diff --git a/build/pgo/server-locations.txt b/build/pgo/server-locations.txt index da4c66e411..bb359d9f70 100644 --- a/build/pgo/server-locations.txt +++ b/build/pgo/server-locations.txt @@ -171,6 +171,8 @@ http://malware.example.com:80 http://tracking.example.com:80 http://not-tracking.example.com:80 http://tracking.example.org:80 +http://itisatracker.org:80 +http://trackertest.org:80 https://malware.example.com:443 https://tracking.example.com:443 diff --git a/dom/base/nsContentUtils.cpp b/dom/base/nsContentUtils.cpp index 5c56d6737b..59f2a7b58d 100644 --- a/dom/base/nsContentUtils.cpp +++ b/dom/base/nsContentUtils.cpp @@ -3623,14 +3623,14 @@ nsContentUtils::IsPlainTextType(const nsACString& aContentType) return aContentType.EqualsLiteral(TEXT_PLAIN) || aContentType.EqualsLiteral(TEXT_CSS) || aContentType.EqualsLiteral(TEXT_CACHE_MANIFEST) || - aContentType.EqualsLiteral(TEXT_VTT) || aContentType.EqualsLiteral(APPLICATION_JAVASCRIPT) || aContentType.EqualsLiteral(APPLICATION_XJAVASCRIPT) || aContentType.EqualsLiteral(TEXT_ECMASCRIPT) || aContentType.EqualsLiteral(APPLICATION_ECMASCRIPT) || aContentType.EqualsLiteral(TEXT_JAVASCRIPT) || aContentType.EqualsLiteral(APPLICATION_JSON) || - aContentType.EqualsLiteral(TEXT_JSON); + aContentType.EqualsLiteral(TEXT_JSON) || + aContentType.EqualsLiteral(TEXT_VTT); } bool diff --git a/dom/bindings/Errors.msg b/dom/bindings/Errors.msg index d986617d41..0fb5345197 100644 --- a/dom/bindings/Errors.msg +++ b/dom/bindings/Errors.msg @@ -54,6 +54,7 @@ MSG_DEF(MSG_NOT_DATE, 1, JSEXN_TYPEERR, "{0} is not a date.") MSG_DEF(MSG_INVALID_ADVANCE_COUNT, 0, JSEXN_TYPEERR, "0 (Zero) is not a valid advance count.") MSG_DEF(MSG_DEFINEPROPERTY_ON_GSP, 0, JSEXN_TYPEERR, "Not allowed to define a property on the named properties object.") MSG_DEF(MSG_INVALID_URL, 1, JSEXN_TYPEERR, "{0} is not a valid URL.") +MSG_DEF(MSG_URL_HAS_CREDENTIALS, 1, JSEXN_TYPEERR, "{0} is an url with embedded credentials.") MSG_DEF(MSG_METADATA_NOT_CONFIGURED, 0, JSEXN_TYPEERR, "Either size or lastModified should be true.") MSG_DEF(MSG_INVALID_READ_SIZE, 0, JSEXN_TYPEERR, "0 (Zero) is not a valid read size.") MSG_DEF(MSG_HEADERS_IMMUTABLE, 0, JSEXN_TYPEERR, "Headers are immutable and cannot be modified.") diff --git a/dom/cache/TypeUtils.cpp b/dom/cache/TypeUtils.cpp index fbf25cf286..4f17a28380 100644 --- a/dom/cache/TypeUtils.cpp +++ b/dom/cache/TypeUtils.cpp @@ -418,25 +418,15 @@ TypeUtils::ProcessURL(nsACString& aUrl, bool* aSchemeValidOut, uint32_t queryPos; int32_t queryLen; - uint32_t refPos; - int32_t refLen; aRv = urlParser->ParsePath(url + pathPos, flatURL.Length() - pathPos, nullptr, nullptr, // ignore filepath &queryPos, &queryLen, - &refPos, &refLen); + nullptr, nullptr); if (NS_WARN_IF(aRv.Failed())) { return; } - // TODO: Remove this once Request/Response properly strip the fragment (bug 1110476) - if (refLen >= 0) { - // ParsePath gives us ref position relative to the start of the path - refPos += pathPos; - - aUrl = Substring(aUrl, 0, refPos - 1); - } - if (!aUrlWithoutQueryOut) { return; } diff --git a/dom/fetch/FetchDriver.cpp b/dom/fetch/FetchDriver.cpp index a96b473a23..16094ec641 100644 --- a/dom/fetch/FetchDriver.cpp +++ b/dom/fetch/FetchDriver.cpp @@ -4,6 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/DebugOnly.h" #include "mozilla/dom/FetchDriver.h" #include "nsIDocument.h" @@ -645,7 +646,8 @@ FetchDriver::BeginAndGetFilteredResponse(InternalResponse* aResponse, nsIURI* aF } else { mRequest->GetURL(reqURL); } - aResponse->SetUrl(reqURL); + DebugOnly rv = aResponse->StripFragmentAndSetUrl(reqURL); + MOZ_ASSERT(NS_SUCCEEDED(rv)); // FIXME(nsm): Handle mixed content check, step 7 of fetch. diff --git a/dom/fetch/InternalRequest.h b/dom/fetch/InternalRequest.h index 85e96c6e30..9cc8aa6932 100644 --- a/dom/fetch/InternalRequest.h +++ b/dom/fetch/InternalRequest.h @@ -388,6 +388,7 @@ private: MapContentPolicyTypeToRequestContext(nsContentPolicyType aContentPolicyType); nsCString mMethod; + // mURL always stores the url with the ref stripped nsCString mURL; nsRefPtr mHeaders; nsCOMPtr mBodyStream; diff --git a/dom/fetch/InternalResponse.cpp b/dom/fetch/InternalResponse.cpp index f1d97f59de..4a598ea02c 100644 --- a/dom/fetch/InternalResponse.cpp +++ b/dom/fetch/InternalResponse.cpp @@ -6,9 +6,11 @@ #include "InternalResponse.h" +#include "mozilla/Assertions.h" #include "mozilla/dom/InternalHeaders.h" #include "mozilla/dom/cache/CacheTypes.h" #include "mozilla/ipc/PBackgroundSharedTypes.h" +#include "nsIURI.h" #include "nsStreamUtils.h" namespace mozilla { @@ -85,6 +87,37 @@ InternalResponse::SetPrincipalInfo(UniquePtr aPrinc mPrincipalInfo = Move(aPrincipalInfo); } +nsresult +InternalResponse::StripFragmentAndSetUrl(const nsACString& aUrl) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr iuri; + nsresult rv; + + rv = NS_NewURI(getter_AddRefs(iuri), aUrl); + if(NS_WARN_IF(NS_FAILED(rv))){ + return rv; + } + + nsCOMPtr iuriClone; + // We use CloneIgnoringRef to strip away the fragment even if the original URI + // is immutable. + rv = iuri->CloneIgnoringRef(getter_AddRefs(iuriClone)); + if(NS_WARN_IF(NS_FAILED(rv))){ + return rv; + } + + nsCString spec; + rv = iuriClone->GetSpec(spec); + if(NS_WARN_IF(NS_FAILED(rv))){ + return rv; + } + + SetUrl(spec); + return NS_OK; +} + already_AddRefed InternalResponse::OpaqueResponse() { diff --git a/dom/fetch/InternalResponse.h b/dom/fetch/InternalResponse.h index 6e109eb243..eb1a352ae3 100644 --- a/dom/fetch/InternalResponse.h +++ b/dom/fetch/InternalResponse.h @@ -82,6 +82,7 @@ public: aURL.Assign(mURL); } + // SetUrl should only be called when the fragment has alredy been stripped void SetUrl(const nsACString& aURL) { @@ -204,6 +205,9 @@ public: void SetPrincipalInfo(UniquePtr aPrincipalInfo); + nsresult + StripFragmentAndSetUrl(const nsACString& aUrl); + private: ~InternalResponse(); diff --git a/dom/fetch/Request.cpp b/dom/fetch/Request.cpp index 1979e72dbb..18572542bb 100644 --- a/dom/fetch/Request.cpp +++ b/dom/fetch/Request.cpp @@ -17,6 +17,7 @@ #include "mozilla/dom/URL.h" #include "mozilla/dom/WorkerPrivate.h" #include "mozilla/dom/workers/bindings/URL.h" +#include "mozilla/unused.h" #include "WorkerPrivate.h" @@ -72,30 +73,44 @@ Request::GetInternalRequest() namespace { void -GetRequestURLFromWindow(const GlobalObject& aGlobal, nsPIDOMWindow* aWindow, - const nsAString& aInput, nsAString& aRequestURL, - ErrorResult& aRv) +GetRequestURLFromDocument(nsIDocument* aDocument, const nsAString& aInput, + nsAString& aRequestURL, ErrorResult& aRv) { - MOZ_ASSERT(aWindow); + MOZ_ASSERT(aDocument); MOZ_ASSERT(NS_IsMainThread()); - nsCOMPtr docURI = aWindow->GetDocumentURI(); + nsCOMPtr baseURI = aDocument->GetBaseURI(); + nsCOMPtr resolvedURI; + aRv = NS_NewURI(getter_AddRefs(resolvedURI), aInput, nullptr, baseURI); + if (NS_WARN_IF(aRv.Failed())) { + aRv.ThrowTypeError(MSG_INVALID_URL, &aInput); + return; + } + + // This fails with URIs with weird protocols, even when they are valid, + // so we ignore the failure + nsAutoCString credentials; + unused << resolvedURI->GetUserPass(credentials); + if (!credentials.IsEmpty()) { + aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput); + return; + } + + nsCOMPtr resolvedURIClone; + // We use CloneIgnoringRef to strip away the fragment even if the original URI + // is immutable. + aRv = resolvedURI->CloneIgnoringRef(getter_AddRefs(resolvedURIClone)); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + nsAutoCString spec; - aRv = docURI->GetSpec(spec); + aRv = resolvedURIClone->GetSpec(spec); if (NS_WARN_IF(aRv.Failed())) { return; } - nsRefPtr url = - dom::URL::Constructor(aGlobal, aInput, NS_ConvertUTF8toUTF16(spec), aRv); - if (NS_WARN_IF(aRv.Failed())) { - return; - } - - url->Stringify(aRequestURL, aRv); - if (NS_WARN_IF(aRv.Failed())) { - return; - } + CopyUTF8toUTF16(spec, aRequestURL); } void @@ -104,23 +119,37 @@ GetRequestURLFromChrome(const nsAString& aInput, nsAString& aRequestURL, { MOZ_ASSERT(NS_IsMainThread()); -#ifdef DEBUG nsCOMPtr uri; aRv = NS_NewURI(getter_AddRefs(uri), aInput, nullptr, nullptr); + if (NS_WARN_IF(aRv.Failed())) { + aRv.ThrowTypeError(MSG_INVALID_URL, &aInput); + return; + } + + // This fails with URIs with weird protocols, even when they are valid, + // so we ignore the failure + nsAutoCString credentials; + unused << uri->GetUserPass(credentials); + if (!credentials.IsEmpty()) { + aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput); + return; + } + + nsCOMPtr uriClone; + // We use CloneIgnoringRef to strip away the fragment even if the original URI + // is immutable. + aRv = uri->CloneIgnoringRef(getter_AddRefs(uriClone)); if (NS_WARN_IF(aRv.Failed())) { return; } nsAutoCString spec; - aRv = uri->GetSpec(spec); + aRv = uriClone->GetSpec(spec); if (NS_WARN_IF(aRv.Failed())) { return; } CopyUTF8toUTF16(spec, aRequestURL); -#else - aRequestURL = aInput; -#endif } void @@ -134,6 +163,29 @@ GetRequestURLFromWorker(const GlobalObject& aGlobal, const nsAString& aInput, NS_ConvertUTF8toUTF16 baseURL(worker->GetLocationInfo().mHref); nsRefPtr url = workers::URL::Constructor(aGlobal, aInput, baseURL, aRv); + if (NS_WARN_IF(aRv.Failed())) { + aRv.ThrowTypeError(MSG_INVALID_URL, &aInput); + return; + } + + nsString username; + url->GetUsername(username, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + nsString password; + url->GetPassword(password, aRv); + if (NS_WARN_IF(aRv.Failed())) { + return; + } + + if (!username.IsEmpty() || !password.IsEmpty()) { + aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput); + return; + } + + url->SetHash(EmptyString(), aRv); if (NS_WARN_IF(aRv.Failed())) { return; } @@ -187,11 +239,11 @@ Request::Constructor(const GlobalObject& aGlobal, nsAutoString requestURL; if (NS_IsMainThread()) { - nsCOMPtr window = do_QueryInterface(global); - if (window) { - GetRequestURLFromWindow(aGlobal, window, input, requestURL, aRv); + nsIDocument* doc = GetEntryDocument(); + if (doc) { + GetRequestURLFromDocument(doc, input, requestURL, aRv); } else { - // If we don't have a window, we must assume that this is a full URL. + // If we don't have a document, we must assume that this is a full URL. GetRequestURLFromChrome(input, requestURL, aRv); } } else { diff --git a/dom/fetch/Response.cpp b/dom/fetch/Response.cpp index fa2fe13a90..c0ad78108e 100644 --- a/dom/fetch/Response.cpp +++ b/dom/fetch/Response.cpp @@ -62,21 +62,24 @@ Response::Redirect(const GlobalObject& aGlobal, const nsAString& aUrl, nsAutoString parsedURL; if (NS_IsMainThread()) { - nsCOMPtr window = do_QueryInterface(aGlobal.GetAsSupports()); - nsCOMPtr docURI = window->GetDocumentURI(); - nsAutoCString spec; - aRv = docURI->GetSpec(spec); + nsCOMPtr baseURI; + nsIDocument* doc = GetEntryDocument(); + if (doc) { + baseURI = doc->GetBaseURI(); + } + nsCOMPtr resolvedURI; + aRv = NS_NewURI(getter_AddRefs(resolvedURI), aUrl, nullptr, baseURI); if (NS_WARN_IF(aRv.Failed())) { return nullptr; } - nsRefPtr url = - dom::URL::Constructor(aGlobal, aUrl, NS_ConvertUTF8toUTF16(spec), aRv); - if (aRv.Failed()) { + nsAutoCString spec; + aRv = resolvedURI->GetSpec(spec); + if (NS_WARN_IF(aRv.Failed())) { return nullptr; } - url->Stringify(parsedURL, aRv); + CopyUTF8toUTF16(spec, parsedURL); } else { workers::WorkerPrivate* worker = workers::GetCurrentThreadWorkerPrivate(); MOZ_ASSERT(worker); diff --git a/dom/tests/mochitest/fetch/test_fetch_cors.js b/dom/tests/mochitest/fetch/test_fetch_cors.js index d67d179652..d04e2d81ff 100644 --- a/dom/tests/mochitest/fetch/test_fetch_cors.js +++ b/dom/tests/mochitest/fetch/test_fetch_cors.js @@ -748,10 +748,13 @@ function testModeCors() { if (test.preflightBody) req.url += "&preflightBody=" + escape(test.preflightBody); - var request = new Request(req.url, { method: req.method, mode: "cors", - headers: req.headers, body: req.body }); - fetches.push((function(test, request) { - return fetch(request).then(function(res) { + fetches.push((function(test) { + return new Promise(function(resolve) { + resolve(new Request(req.url, { method: req.method, mode: "cors", + headers: req.headers, body: req.body })); + }).then(function(request) { + return fetch(request); + }).then(function(res) { ok(test.pass, "Expected test to pass for " + test.toSource()); if (test.status) { is(res.status, test.status, "wrong status in test for " + test.toSource()); @@ -776,21 +779,21 @@ function testModeCors() { } } - return res.text().then(function(v) { - if (test.method !== "HEAD") { - is(v, "hello pass\n", - "wrong responseText in test for " + test.toSource()); - } - else { - is(v, "", - "wrong responseText in HEAD test for " + test.toSource()); - } - }); - }, function(e) { - ok(!test.pass, "Expected test failure for " + test.toSource()); - ok(e instanceof TypeError, "Exception should be TypeError for " + test.toSource()); + return res.text(); + }).then(function(v) { + if (test.method !== "HEAD") { + is(v, "hello pass\n", + "wrong responseText in test for " + test.toSource()); + } + else { + is(v, "", + "wrong responseText in HEAD test for " + test.toSource()); + } + }).catch(function(e) { + ok(!test.pass, "Expected test failure for " + test.toSource()); + ok(e instanceof TypeError, "Exception should be TypeError for " + test.toSource()); }); - })(test, request)); + })(test)); } return Promise.all(fetches); diff --git a/dom/tests/mochitest/fetch/test_request.js b/dom/tests/mochitest/fetch/test_request.js index cb3435c1bd..ec88a752f3 100644 --- a/dom/tests/mochitest/fetch/test_request.js +++ b/dom/tests/mochitest/fetch/test_request.js @@ -216,7 +216,32 @@ function testMethod() { function testUrlFragment() { var req = new Request("./request#withfragment"); - ok(req.url, (new URL("./request", self.location.href)).href, "request.url should be serialized with exclude fragment flag set"); + is(req.url, (new URL("./request", self.location.href)).href, "request.url should be serialized with exclude fragment flag set"); +} + +function testUrlMalformed() { + try { + var req = new Request("http:// example.com"); + ok(false, "Creating a Request with a malformed URL should throw a TypeError"); + } catch(e) { + is(e.name, "TypeError", "Creating a Request with a malformed URL should throw a TypeError"); + } +} + +function testUrlCredentials() { + try { + var req = new Request("http://user@example.com"); + ok(false, "URLs with credentials should be rejected"); + } catch(e) { + is(e.name, "TypeError", "URLs with credentials should be rejected"); + } + + try { + var req = new Request("http://user:password@example.com"); + ok(false, "URLs with credentials should be rejected"); + } catch(e) { + is(e.name, "TypeError", "URLs with credentials should be rejected"); + } } function testBodyUsed() { @@ -471,6 +496,8 @@ function runTest() { testDefaultCtor(); testSimpleUrlParse(); testUrlFragment(); + testUrlCredentials(); + testUrlMalformed(); testMethod(); testBug1109574(); testHeaderGuard(); diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 148e08dd23..1ffb5cadc3 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -3460,7 +3460,7 @@ pref("intl.imm.vertical_writing.always_assume_not_supported", false); // We cannot retrieve active IME name with IMM32 API if a TIP of TSF is active. // This pref can specify active IME name when Japanese TIP is active. // For example: -// Google Japanese Input: "Google ?¥æœ¬èªžå…¥??IMM32 ?¢ã‚¸?¥ãƒ¼?? +// Google Japanese Input: "Google ?�本語入??IMM32 ?�ジ?�ー?? // ATOK 2011: "ATOK 2011" (similarly, e.g., ATOK 2013 is "ATOK 2013") pref("intl.imm.japanese.assume_active_tip_name_as", ""); @@ -4952,11 +4952,12 @@ pref("urlclassifier.malwareTable", "goog-malware-shavar,test-malware-simple"); pref("urlclassifier.phishTable", "goog-phish-shavar,test-phish-simple"); pref("urlclassifier.downloadBlockTable", ""); pref("urlclassifier.downloadAllowTable", ""); -pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,goog-downloadwhite-digest256,mozpub-track-digest256"); +pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,goog-downloadwhite-digest256,mozstd-track-digest256,mozstd-trackwhite-digest256,mozfull-track-digest256"); // The table and update/gethash URLs for Safebrowsing phishing and malware // checks. -pref("urlclassifier.trackingTable", "mozpub-track-digest256"); +pref("urlclassifier.trackingTable", "test-track-simple,mozpub-track-digest256"); +pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozpub-trackwhite-digest256"); pref("browser.trackingprotection.updateURL", "https://tracking.services.mozilla.com/downloads?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2"); pref("browser.trackingprotection.gethashURL", "https://tracking.services.mozilla.com/gethash?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2"); diff --git a/netwerk/base/nsChannelClassifier.cpp b/netwerk/base/nsChannelClassifier.cpp index c645baa706..a422e2778b 100644 --- a/netwerk/base/nsChannelClassifier.cpp +++ b/netwerk/base/nsChannelClassifier.cpp @@ -472,12 +472,83 @@ nsChannelClassifier::SetBlockedTrackingContent(nsIChannel *channel) return NS_OK; } +nsresult +nsChannelClassifier::IsTrackerWhitelisted() +{ + nsresult rv; + nsCOMPtr uriClassifier = + do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString tables; + Preferences::GetCString("urlclassifier.trackingWhitelistTable", &tables); + + if (tables.IsEmpty()) { + LOG(("nsChannelClassifier[%p]:IsTrackerWhitelisted whitelist disabled", + this)); + return NS_ERROR_TRACKING_URI; + } + + nsCOMPtr chan = do_QueryInterface(mChannel, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr topWinURI; + rv = chan->GetTopWindowURI(getter_AddRefs(topWinURI)); + NS_ENSURE_SUCCESS(rv, rv); + if (!topWinURI) { + LOG(("nsChannelClassifier[%p]: No window URI", this)); + return NS_ERROR_TRACKING_URI; + } + + nsCOMPtr securityManager = + do_GetService(NS_SCRIPTSECURITYMANAGER_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr chanPrincipal; + rv = securityManager->GetChannelURIPrincipal(mChannel, + getter_AddRefs(chanPrincipal)); + NS_ENSURE_SUCCESS(rv, rv); + + // Craft a whitelist URL like "toplevel.page/?resource=third.party.domain" + nsAutoCString pageHostname, resourceDomain; + rv = topWinURI->GetHost(pageHostname); + NS_ENSURE_SUCCESS(rv, rv); + rv = chanPrincipal->GetBaseDomain(resourceDomain); + NS_ENSURE_SUCCESS(rv, rv); + nsAutoCString whitelistEntry = NS_LITERAL_CSTRING("http://") + + pageHostname + NS_LITERAL_CSTRING("/?resource=") + resourceDomain; + LOG(("nsChannelClassifier[%p]: Looking for %s in the whitelist", + this, whitelistEntry.get())); + + nsCOMPtr whitelistURI; + rv = NS_NewURI(getter_AddRefs(whitelistURI), whitelistEntry); + NS_ENSURE_SUCCESS(rv, rv); + + // Check whether or not the tracker is in the entity whitelist + nsAutoCString results; + rv = uriClassifier->ClassifyLocalWithTables(whitelistURI, tables, results); + NS_ENSURE_SUCCESS(rv, rv); + if (!results.IsEmpty()) { + return NS_OK; // found it on the whitelist, must not be blocked + } + + LOG(("nsChannelClassifier[%p]: %s is not in the whitelist", + this, whitelistEntry.get())); + return NS_ERROR_TRACKING_URI; +} + NS_IMETHODIMP nsChannelClassifier::OnClassifyComplete(nsresult aErrorCode) { // Should only be called in the parent process. MOZ_ASSERT(XRE_IsParentProcess()); + if (aErrorCode == NS_ERROR_TRACKING_URI && + NS_SUCCEEDED(IsTrackerWhitelisted())) { + LOG(("nsChannelClassifier[%p]:OnClassifyComplete tracker found " + "in whitelist so we won't block it)", this)); + aErrorCode = NS_OK; + } + LOG(("nsChannelClassifier[%p]:OnClassifyComplete %d", this, aErrorCode)); if (mSuspendedChannel) { MarkEntryClassified(aErrorCode); diff --git a/netwerk/base/nsChannelClassifier.h b/netwerk/base/nsChannelClassifier.h index 5a3bfee5c0..24a625829b 100644 --- a/netwerk/base/nsChannelClassifier.h +++ b/netwerk/base/nsChannelClassifier.h @@ -41,6 +41,8 @@ private: // Start is called. Returns NS_OK if and only if we will get a callback // from the classifier service. nsresult StartInternal(); + // Helper function to check a tracking URI against the whitelist + nsresult IsTrackerWhitelisted(); public: // If we are blocking tracking content, update the corresponding flag in diff --git a/netwerk/base/nsIPackagedAppService.idl b/netwerk/base/nsIPackagedAppService.idl index 1c20ebd576..099481c678 100644 --- a/netwerk/base/nsIPackagedAppService.idl +++ b/netwerk/base/nsIPackagedAppService.idl @@ -5,8 +5,7 @@ #include "nsISupports.idl" -interface nsIPrincipal; -interface nsILoadContextInfo; +interface nsIChannel; interface nsICacheEntryOpenCallback; %{C++ @@ -16,16 +15,16 @@ interface nsICacheEntryOpenCallback; /** * nsIPackagedAppService */ -[scriptable, builtinclass, uuid(f35e5229-d08a-46eb-a574-2db4e22aee98)] +[scriptable, builtinclass, uuid(9c96c638-e80c-4dce-abec-c96fdb7a25d8)] interface nsIPackagedAppService : nsISupports { /** - * @param aPrincipal - * the principal associated to the URL of a packaged resource - * URL format: package_url + PACKAGED_APP_TOKEN + resource_path - * example: http://test.com/path/to/package!//resource.html - * @param aFlags - * the load flags used for downloading the package + * @param aChannel + * this param is passed to the packaged app service in order to provide + * info about the requesting channel, which wants to access the contents + * of a packaged app resource. Its URI has the following format: + * http://domain.com/path/to/package.pak!//path/to/subresource.html + * * @param aCallback * an object implementing nsICacheEntryOpenCallback * this is the target of the async result of the operation @@ -34,17 +33,12 @@ interface nsIPackagedAppService : nsISupports * the cached entry, if one exists, or an error code otherwise * aCallback is kept alive using an nsCOMPtr until OnCacheEntryAvailable * is called - * @param aInfo - * an object used to determine the cache jar this resource goes in. - * usually created by calling GetLoadContextInfo(requestingChannel) * * Calling this method will either download the package containing the given * resource URI, store it in the cache and pass the cache entry to aCallback, * or if that resource has already been downloaded it will be served from * the cache. */ - void getResource(in nsIPrincipal aPrincipal, - in uint32_t aFlags, - in nsILoadContextInfo aInfo, + void getResource(in nsIChannel aChannel, in nsICacheEntryOpenCallback aCallback); }; diff --git a/netwerk/base/nsIURIClassifier.idl b/netwerk/base/nsIURIClassifier.idl index f6fd0214cc..a8f6098a78 100644 --- a/netwerk/base/nsIURIClassifier.idl +++ b/netwerk/base/nsIURIClassifier.idl @@ -4,8 +4,9 @@ #include "nsISupports.idl" -interface nsIPrincipal; interface nsIChannel; +interface nsIPrincipal; +interface nsIURI; /** * Callback function for nsIURIClassifier lookups. @@ -30,7 +31,7 @@ interface nsIURIClassifierCallback : nsISupports * The URI classifier service checks a URI against lists of phishing * and malware sites. */ -[scriptable, uuid(03d26681-0ef5-4718-9777-33c9e1ee3e32)] +[scriptable, uuid(596620cc-76e3-4133-9d90-360e59a794cf)] interface nsIURIClassifier : nsISupports { /** @@ -56,10 +57,9 @@ interface nsIURIClassifier : nsISupports in nsIURIClassifierCallback aCallback); /** - * Synchronously classify a Principal locally using its URI. This does not - * make network requests. The result is an error code with which the channel - * should be cancelled, or NS_OK if no result was found. + * Synchronously classify a URI with a comma-separated string + * containing the given tables. This does not make network requests. + * The result is a comma-separated string of tables that match. */ - nsresult classifyLocal(in nsIPrincipal aPrincipal, - in boolean aTrackingProtectionEnabled); + ACString classifyLocalWithTables(in nsIURI aURI, in ACString aTables); }; diff --git a/netwerk/cache2/CacheEntry.cpp b/netwerk/cache2/CacheEntry.cpp index 1b0cf9771e..2588e53b77 100644 --- a/netwerk/cache2/CacheEntry.cpp +++ b/netwerk/cache2/CacheEntry.cpp @@ -248,7 +248,7 @@ nsresult CacheEntry::HashingKey(nsCSubstring const& aStorageID, * Changing it will cause we will not be able to find files on disk. */ - aResult.Append(aStorageID); + aResult.Assign(aStorageID); if (!aEnhanceID.IsEmpty()) { CacheFileUtils::AppendTagWithValue(aResult, '~', aEnhanceID); @@ -1574,7 +1574,7 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync) // Because CacheFile::Set*() are not thread-safe to use (uses WeakReference that // is not thread-safe) we must post to the main thread... nsRefPtr > event = - NS_NewRunnableMethod(this, &CacheEntry::StoreFrecency); + NS_NewRunnableMethodWithArg(this, &CacheEntry::StoreFrecency, mFrecency); NS_DispatchToMainThread(event); } @@ -1598,14 +1598,12 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync) } } -void CacheEntry::StoreFrecency() +void CacheEntry::StoreFrecency(double aFrecency) { - // No need for thread safety over mFrecency, it will be rewriten - // correctly on following invocation if broken by concurrency. MOZ_ASSERT(NS_IsMainThread()); if (NS_SUCCEEDED(mFileStatus)) { - mFile->SetFrecency(FRECENCY2INT(mFrecency)); + mFile->SetFrecency(FRECENCY2INT(aFrecency)); } } diff --git a/netwerk/cache2/CacheEntry.h b/netwerk/cache2/CacheEntry.h index 2d1c8637f7..9bec9f0c1f 100644 --- a/netwerk/cache2/CacheEntry.h +++ b/netwerk/cache2/CacheEntry.h @@ -246,7 +246,7 @@ private: // When executed on the management thread directly, the operation(s) // is (are) executed immediately. void BackgroundOp(uint32_t aOperation, bool aForceAsync = false); - void StoreFrecency(); + void StoreFrecency(double aFrecency); // Called only from DoomAlreadyRemoved() void DoomFile(); diff --git a/netwerk/cache2/CacheFileIOManager.cpp b/netwerk/cache2/CacheFileIOManager.cpp index 7f86b9c371..86a7bb1f61 100644 --- a/netwerk/cache2/CacheFileIOManager.cpp +++ b/netwerk/cache2/CacheFileIOManager.cpp @@ -109,7 +109,6 @@ NS_INTERFACE_MAP_END_THREADSAFE CacheFileHandle::CacheFileHandle(const SHA1Sum::Hash *aHash, bool aPriority) : mHash(aHash) - , mIsDoomed(false) , mPriority(aPriority) , mClosed(false) , mSpecialFile(false) @@ -118,13 +117,17 @@ CacheFileHandle::CacheFileHandle(const SHA1Sum::Hash *aHash, bool aPriority) , mFileSize(-1) , mFD(nullptr) { + // If we initialize mDoomed in the initialization list, that initialization is + // not guaranteeded to be atomic. Whereas this assignment here is guaranteed + // to be atomic. TSan will see this (atomic) assignment and be satisfied + // that cross-thread accesses to mIsDoomed are properly synchronized. + mIsDoomed = false; LOG(("CacheFileHandle::CacheFileHandle() [this=%p, hash=%08x%08x%08x%08x%08x]" , this, LOGSHA1(aHash))); } CacheFileHandle::CacheFileHandle(const nsACString &aKey, bool aPriority) : mHash(nullptr) - , mIsDoomed(false) , mPriority(aPriority) , mClosed(false) , mSpecialFile(true) @@ -134,6 +137,8 @@ CacheFileHandle::CacheFileHandle(const nsACString &aKey, bool aPriority) , mFD(nullptr) , mKey(aKey) { + // See comment above about the initialization of mIsDoomed. + mIsDoomed = false; LOG(("CacheFileHandle::CacheFileHandle() [this=%p, key=%s]", this, PromiseFlatCString(aKey).get())); } @@ -161,13 +166,13 @@ CacheFileHandle::Log() if (mSpecialFile) { LOG(("CacheFileHandle::Log() - special file [this=%p, isDoomed=%d, " "priority=%d, closed=%d, invalid=%d, fileExists=%d, fileSize=%lld, " - "leafName=%s, key=%s]", this, mIsDoomed, mPriority, mClosed, mInvalid, + "leafName=%s, key=%s]", this, int(mIsDoomed), mPriority, mClosed, mInvalid, mFileExists, mFileSize, leafName.get(), mKey.get())); } else { LOG(("CacheFileHandle::Log() - entry file [this=%p, hash=%08x%08x%08x%08x" "%08x, isDoomed=%d, priority=%d, closed=%d, invalid=%d, fileExists=%d," " fileSize=%lld, leafName=%s, key=%s]", this, LOGSHA1(mHash), - mIsDoomed, mPriority, mClosed, mInvalid, mFileExists, mFileSize, + int(mIsDoomed), mPriority, mClosed, mInvalid, mFileExists, mFileSize, leafName.get(), mKey.get())); } } diff --git a/netwerk/cache2/CacheFileIOManager.h b/netwerk/cache2/CacheFileIOManager.h index aaae0a790b..1c378e1f63 100644 --- a/netwerk/cache2/CacheFileIOManager.h +++ b/netwerk/cache2/CacheFileIOManager.h @@ -10,6 +10,7 @@ #include "nsIEventTarget.h" #include "nsITimer.h" #include "nsCOMPtr.h" +#include "mozilla/Atomics.h" #include "mozilla/SHA1.h" #include "mozilla/TimeStamp.h" #include "nsTArray.h" @@ -69,7 +70,7 @@ private: virtual ~CacheFileHandle(); const SHA1Sum::Hash *mHash; - bool mIsDoomed; + mozilla::Atomic mIsDoomed; bool mPriority; bool mClosed; bool mSpecialFile; diff --git a/netwerk/cache2/CacheFileMetadata.cpp b/netwerk/cache2/CacheFileMetadata.cpp index 0c0960c2ba..7467128139 100644 --- a/netwerk/cache2/CacheFileMetadata.cpp +++ b/netwerk/cache2/CacheFileMetadata.cpp @@ -24,6 +24,16 @@ namespace net { #define kMinMetadataRead 1024 // TODO find optimal value from telemetry #define kAlignSize 4096 +// Most of the cache entries fit into one chunk due to current chunk size. Make +// sure to tweak this value if kChunkSize is going to change. +#define kInitialHashArraySize 1 + +// Initial elements buffer size. +#define kInitialBufSize 64 + +// Max size of elements in bytes. +#define kMaxElementsSize 64*1024 + #define kCacheEntryVersion 1 #define NOW_SECONDS() (uint32_t(PR_Now() / PR_USEC_PER_SEC)) @@ -44,6 +54,7 @@ CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString , mIsDirty(false) , mAnonymous(false) , mInBrowser(false) + , mAllocExactSize(false) , mAppId(nsILoadContextInfo::NO_APP_ID) { LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, handle=%p, key=%s]", @@ -74,6 +85,7 @@ CacheFileMetadata::CacheFileMetadata(bool aMemoryOnly, const nsACString &aKey) , mIsDirty(true) , mAnonymous(false) , mInBrowser(false) + , mAllocExactSize(false) , mAppId(nsILoadContextInfo::NO_APP_ID) { LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, key=%s]", @@ -105,6 +117,7 @@ CacheFileMetadata::CacheFileMetadata() , mIsDirty(false) , mAnonymous(false) , mInBrowser(false) + , mAllocExactSize(false) , mAppId(nsILoadContextInfo::NO_APP_ID) { LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p]", this)); @@ -221,6 +234,17 @@ CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener) return NS_OK; } +uint32_t +CacheFileMetadata::CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount) +{ + return sizeof(uint32_t) + // hash of the metadata + aHashCount * sizeof(CacheHash::Hash16_t) + // array of chunk hashes + sizeof(CacheFileMetadataHeader) + // metadata header + mKey.Length() + 1 + // key with trailing null + aElementsSize + // elements + sizeof(uint32_t); // offset +} + nsresult CacheFileMetadata::WriteMetadata(uint32_t aOffset, CacheFileMetadataListener *aListener) @@ -235,10 +259,11 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset, mIsDirty = false; - mWriteBuf = static_cast(moz_xmalloc(sizeof(uint32_t) + - mHashCount * sizeof(CacheHash::Hash16_t) + - sizeof(CacheFileMetadataHeader) + mKey.Length() + 1 + - mElementsSize + sizeof(uint32_t))); + mWriteBuf = static_cast(malloc(CalcMetadataSize(mElementsSize, + mHashCount))); + if (!mWriteBuf) { + return NS_ERROR_OUT_OF_MEMORY; + } char *p = mWriteBuf + sizeof(uint32_t); memcpy(p, mHashArray, mHashCount * sizeof(CacheHash::Hash16_t)); @@ -332,8 +357,11 @@ CacheFileMetadata::SyncReadMetadata(nsIFile *aFile) return NS_ERROR_FAILURE; } + mBuf = static_cast(malloc(fileSize - metaOffset)); + if (!mBuf) { + return NS_ERROR_OUT_OF_MEMORY; + } mBufSize = fileSize - metaOffset; - mBuf = static_cast(moz_xmalloc(mBufSize)); DoMemoryReport(MemoryUsage()); @@ -388,6 +416,8 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue) MarkDirty(); + nsresult rv; + const uint32_t keySize = strlen(aKey) + 1; char *pos = const_cast(GetElement(aKey)); @@ -413,7 +443,10 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue) // Update the value in place newSize -= oldValueSize; - EnsureBuffer(newSize); + rv = EnsureBuffer(newSize); + if (NS_FAILED(rv)) { + return rv; + } // Move the remainder to the right place pos = mBuf + offset; @@ -421,7 +454,10 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue) } else { // allocate new meta data element newSize += keySize; - EnsureBuffer(newSize); + rv = EnsureBuffer(newSize); + if (NS_FAILED(rv)) { + return rv; + } // Add after last element pos = mBuf + mElementsSize; @@ -480,10 +516,11 @@ CacheFileMetadata::SetHash(uint32_t aIndex, CacheHash::Hash16_t aHash) } else if (aIndex == mHashCount) { if ((aIndex + 1) * sizeof(CacheHash::Hash16_t) > mHashArraySize) { // reallocate hash array buffer - if (mHashArraySize == 0) - mHashArraySize = 32 * sizeof(CacheHash::Hash16_t); - else + if (mHashArraySize == 0) { + mHashArraySize = kInitialHashArraySize * sizeof(CacheHash::Hash16_t); + } else { mHashArraySize *= 2; + } mHashArray = static_cast( moz_xrealloc(mHashArray, mHashArraySize)); } @@ -613,7 +650,7 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, MOZ_ASSERT(mListener); - nsresult rv, retval; + nsresult rv; nsCOMPtr listener; if (NS_FAILED(aResult)) { @@ -621,10 +658,9 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, ", creating empty metadata. [this=%p, rv=0x%08x]", this, aResult)); InitEmptyMetadata(); - retval = NS_OK; mListener.swap(listener); - listener->OnMetadataRead(retval); + listener->OnMetadataRead(NS_OK); return NS_OK; } @@ -637,14 +673,28 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, if (realOffset >= size) { LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, creating " - "empty metadata. [this=%p, realOffset=%d, size=%lld]", this, + "empty metadata. [this=%p, realOffset=%u, size=%lld]", this, realOffset, size)); InitEmptyMetadata(); - retval = NS_OK; mListener.swap(listener); - listener->OnMetadataRead(retval); + listener->OnMetadataRead(NS_OK); + return NS_OK; + } + + uint32_t maxHashCount = size / kChunkSize; + uint32_t maxMetadataSize = CalcMetadataSize(kMaxElementsSize, maxHashCount); + if (size - realOffset > maxMetadataSize) { + LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, metadata would " + "be too big, creating empty metadata. [this=%p, realOffset=%u, " + "maxMetadataSize=%u, size=%lld]", this, realOffset, maxMetadataSize, + size)); + + InitEmptyMetadata(); + + mListener.swap(listener); + listener->OnMetadataRead(NS_OK); return NS_OK; } @@ -653,7 +703,20 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, if (realOffset < usedOffset) { uint32_t missing = usedOffset - realOffset; // we need to read more data - mBuf = static_cast(moz_xrealloc(mBuf, mBufSize + missing)); + char *newBuf = static_cast(realloc(mBuf, mBufSize + missing)); + if (!newBuf) { + LOG(("CacheFileMetadata::OnDataRead() - Error allocating %d more bytes " + "for the missing part of the metadata, creating empty metadata. " + "[this=%p]", missing, this)); + + InitEmptyMetadata(); + + mListener.swap(listener); + listener->OnMetadataRead(NS_OK); + return NS_OK; + } + + mBuf = newBuf; memmove(mBuf + missing, mBuf, mBufSize); mBufSize += missing; @@ -669,10 +732,9 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, "rv=0x%08x]", this, rv)); InitEmptyMetadata(); - retval = NS_OK; mListener.swap(listener); - listener->OnMetadataRead(retval); + listener->OnMetadataRead(NS_OK); return NS_OK; } @@ -686,14 +748,19 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf, LOG(("CacheFileMetadata::OnDataRead() - Error parsing metadata, creating " "empty metadata. [this=%p]", this)); InitEmptyMetadata(); - retval = NS_OK; - } - else { - retval = NS_OK; + } else { + // Shrink elements buffer. + mBuf = static_cast(moz_xrealloc(mBuf, mElementsSize)); + mBufSize = mElementsSize; + + // There is usually no or just one call to SetMetadataElement() when the + // metadata is parsed from disk. Avoid allocating power of two sized buffer + // which we do in case of newly created metadata. + mAllocExactSize = true; } mListener.swap(listener); - listener->OnMetadataRead(retval); + listener->OnMetadataRead(NS_OK); return NS_OK; } @@ -844,15 +911,12 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, memcpy(mHashArray, mBuf + hashesOffset, mHashArraySize); } - MarkDirty(); mElementsSize = metaposOffset - elementsOffset; memmove(mBuf, mBuf + elementsOffset, mElementsSize); mOffset = aMetaOffset; - // TODO: shrink memory if buffer is too big - DoMemoryReport(MemoryUsage()); return NS_OK; @@ -886,15 +950,44 @@ CacheFileMetadata::CheckElements(const char *aBuf, uint32_t aSize) return NS_OK; } -void +nsresult CacheFileMetadata::EnsureBuffer(uint32_t aSize) { - if (mBufSize < aSize) { - mBufSize = aSize; - mBuf = static_cast(moz_xrealloc(mBuf, mBufSize)); + if (aSize > kMaxElementsSize) { + return NS_ERROR_FAILURE; } - DoMemoryReport(MemoryUsage()); + if (mBufSize < aSize) { + if (mAllocExactSize) { + // If this is not the only allocation, use power of two for following + // allocations. + mAllocExactSize = false; + } else { + // find smallest power of 2 greater than or equal to aSize + --aSize; + aSize |= aSize >> 1; + aSize |= aSize >> 2; + aSize |= aSize >> 4; + aSize |= aSize >> 8; + aSize |= aSize >> 16; + ++aSize; + } + + if (aSize < kInitialBufSize) { + aSize = kInitialBufSize; + } + + char *newBuf = static_cast(realloc(mBuf, aSize)); + if (!newBuf) { + return NS_ERROR_OUT_OF_MEMORY; + } + mBufSize = aSize; + mBuf = newBuf; + + DoMemoryReport(MemoryUsage()); + } + + return NS_OK; } nsresult diff --git a/netwerk/cache2/CacheFileMetadata.h b/netwerk/cache2/CacheFileMetadata.h index 2053e0bbba..940625c3ae 100644 --- a/netwerk/cache2/CacheFileMetadata.h +++ b/netwerk/cache2/CacheFileMetadata.h @@ -121,6 +121,7 @@ public: nsresult GetKey(nsACString &_retval); nsresult ReadMetadata(CacheFileMetadataListener *aListener); + uint32_t CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount); nsresult WriteMetadata(uint32_t aOffset, CacheFileMetadataListener *aListener); nsresult SyncReadMetadata(nsIFile *aFile); @@ -171,7 +172,7 @@ private: void InitEmptyMetadata(); nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, bool aHaveKey); nsresult CheckElements(const char *aBuf, uint32_t aSize); - void EnsureBuffer(uint32_t aSize); + nsresult EnsureBuffer(uint32_t aSize); nsresult ParseKey(const nsACString &aKey); nsRefPtr mHandle; @@ -186,9 +187,10 @@ private: char *mWriteBuf; CacheFileMetadataHeader mMetaHdr; uint32_t mElementsSize; - bool mIsDirty; - bool mAnonymous; - bool mInBrowser; + bool mIsDirty : 1; + bool mAnonymous : 1; + bool mInBrowser : 1; + bool mAllocExactSize : 1; uint32_t mAppId; nsCOMPtr mListener; }; diff --git a/netwerk/cache2/CacheFileOutputStream.cpp b/netwerk/cache2/CacheFileOutputStream.cpp index b400da7d3f..385ddda47d 100644 --- a/netwerk/cache2/CacheFileOutputStream.cpp +++ b/netwerk/cache2/CacheFileOutputStream.cpp @@ -103,6 +103,18 @@ CacheFileOutputStream::Write(const char * aBuf, uint32_t aCount, return NS_ERROR_FILE_TOO_BIG; } + // We use 64-bit offset when accessing the file, unfortunatelly we use 32-bit + // metadata offset, so we cannot handle data bigger than 4GB. + if (mPos + aCount > PR_UINT32_MAX) { + LOG(("CacheFileOutputStream::Write() - Entry's size exceeds 4GB while it " + "isn't too big according to CacheObserver::EntryIsTooBig(). Failing " + "and dooming the entry. [this=%p]", this)); + + mFile->DoomLocked(nullptr); + CloseWithStatusLocked(NS_ERROR_FILE_TOO_BIG); + return NS_ERROR_FILE_TOO_BIG; + } + *_retval = aCount; while (aCount) { diff --git a/netwerk/cache2/CacheIOThread.h b/netwerk/cache2/CacheIOThread.h index c91a4249c7..077d4d98db 100644 --- a/netwerk/cache2/CacheIOThread.h +++ b/netwerk/cache2/CacheIOThread.h @@ -12,6 +12,7 @@ #include "nsAutoPtr.h" #include "mozilla/Monitor.h" #include "mozilla/DebugOnly.h" +#include "mozilla/Atomics.h" class nsIRunnable; @@ -88,11 +89,11 @@ private: mozilla::Monitor mMonitor; PRThread* mThread; nsCOMPtr mXPCOMThread; - uint32_t mLowestLevelWaiting; + Atomic mLowestLevelWaiting; uint32_t mCurrentlyExecutingLevel; nsTArray > mEventQueue[LAST_LEVEL]; - bool mHasXPCOMEvents; + Atomic mHasXPCOMEvents; bool mRerunCurrentEvent; bool mShutdown; DebugOnly mInsideLoop; diff --git a/netwerk/cache2/CacheIndex.cpp b/netwerk/cache2/CacheIndex.cpp index bb99599f14..1a5c0713b2 100644 --- a/netwerk/cache2/CacheIndex.cpp +++ b/netwerk/cache2/CacheIndex.cpp @@ -37,6 +37,28 @@ namespace mozilla { namespace net { +namespace { + +class FrecencyComparator +{ +public: + bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const { + return a->mFrecency == b->mFrecency; + } + bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const { + // Place entries with frecency 0 at the end of the array. + if (a->mFrecency == 0) { + return false; + } + if (b->mFrecency == 0) { + return true; + } + return a->mFrecency < b->mFrecency; + } +}; + +} // namespace + /** * This helper class is responsible for keeping CacheIndex::mIndexStats and * CacheIndex::mFrecencyArray up to date. @@ -47,7 +69,6 @@ public: CacheIndexEntryAutoManage(const SHA1Sum::Hash *aHash, CacheIndex *aIndex) : mIndex(aIndex) , mOldRecord(nullptr) - , mOldFrecency(0) , mDoNotSearchInIndex(false) , mDoNotSearchInUpdates(false) { @@ -58,7 +79,6 @@ public: mIndex->mIndexStats.BeforeChange(entry); if (entry && entry->IsInitialized() && !entry->IsRemoved()) { mOldRecord = entry->mRec; - mOldFrecency = entry->mRec->mFrecency; } } @@ -79,22 +99,9 @@ public: mIndex->RemoveRecordFromFrecencyArray(mOldRecord); mIndex->RemoveRecordFromIterators(mOldRecord); } else if (entry && mOldRecord) { - bool replaceFrecency = false; - if (entry->mRec != mOldRecord) { // record has a different address, we have to replace it - replaceFrecency = true; mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec); - } else if (entry->mRec->mFrecency == 0 && - entry->mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME) { - // This is a special case when we want to make sure that the entry is - // placed at the end of the lists even when the values didn't change. - replaceFrecency = true; - } else if (entry->mRec->mFrecency != mOldFrecency) { - replaceFrecency = true; - } - - if (replaceFrecency) { mIndex->RemoveRecordFromFrecencyArray(mOldRecord); mIndex->InsertRecordToFrecencyArray(entry->mRec); } @@ -141,7 +148,6 @@ private: const SHA1Sum::Hash *mHash; nsRefPtr mIndex; CacheIndexRecord *mOldRecord; - uint32_t mOldFrecency; bool mDoNotSearchInIndex; bool mDoNotSearchInUpdates; }; @@ -1079,6 +1085,17 @@ CacheIndex::RemoveAll() index->mIndexStats.Clear(); index->mFrecencyArray.Clear(); index->mIndex.Clear(); + + for (uint32_t i = 0; i < index->mIterators.Length(); ) { + nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE); + if (NS_FAILED(rv)) { + // CacheIndexIterator::CloseInternal() removes itself from mIterators + // iff it returns success. + LOG(("CacheIndex::RemoveAll() - Failed to remove iterator %p. " + "[rv=0x%08x]", rv)); + i++; + } + } } if (file) { @@ -1177,6 +1194,7 @@ CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries, SHA1Sum::Hash *aHash, uint32_t i; // find first non-forced valid entry with the lowest frecency + index->mFrecencyArray.Sort(FrecencyComparator()); for (i = 0; i < index->mFrecencyArray.Length(); ++i) { memcpy(&hash, &index->mFrecencyArray[i]->mHash, sizeof(SHA1Sum::Hash)); @@ -1347,6 +1365,7 @@ CacheIndex::GetIterator(nsILoadContextInfo *aInfo, bool aAddNew, iter = new CacheIndexIterator(index, aAddNew); } + index->mFrecencyArray.Sort(FrecencyComparator()); iter->AddRecords(index->mFrecencyArray); index->mIterators.AppendElement(iter); @@ -3105,28 +3124,6 @@ CacheIndex::ReleaseBuffer() mRWBufPos = 0; } -namespace { - -class FrecencyComparator -{ -public: - bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const { - return a->mFrecency == b->mFrecency; - } - bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const { - // Place entries with frecency 0 at the end of the array. - if (a->mFrecency == 0) { - return false; - } - if (b->mFrecency == 0) { - return true; - } - return a->mFrecency < b->mFrecency; - } -}; - -} // namespace - void CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord) { @@ -3134,7 +3131,7 @@ CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord) "%08x%08x]", aRecord, LOGSHA1(aRecord->mHash))); MOZ_ASSERT(!mFrecencyArray.Contains(aRecord)); - mFrecencyArray.InsertElementSorted(aRecord, FrecencyComparator()); + mFrecencyArray.AppendElement(aRecord); } void diff --git a/netwerk/cache2/CacheObserver.cpp b/netwerk/cache2/CacheObserver.cpp index 3fc94b79de..ad3dbfadc3 100644 --- a/netwerk/cache2/CacheObserver.cpp +++ b/netwerk/cache2/CacheObserver.cpp @@ -31,8 +31,8 @@ static int32_t sAutoDeleteCacheVersion = kAutoDeleteCacheVersion; static int32_t const kDefaultHalfLifeExperiment = -1; // Disabled int32_t CacheObserver::sHalfLifeExperiment = kDefaultHalfLifeExperiment; -static uint32_t const kDefaultHalfLifeHours = 6; // 6 hours -uint32_t CacheObserver::sHalfLifeHours = kDefaultHalfLifeHours; +static float const kDefaultHalfLifeHours = 1.0F; // 1 hour +float CacheObserver::sHalfLifeHours = kDefaultHalfLifeHours; static bool const kDefaultUseDiskCache = true; bool CacheObserver::sUseDiskCache = kDefaultUseDiskCache; @@ -63,11 +63,11 @@ bool CacheObserver::sSmartCacheSizeEnabled = kDefaultSmartCacheSizeEnabled; static uint32_t const kDefaultPreloadChunkCount = 4; uint32_t CacheObserver::sPreloadChunkCount = kDefaultPreloadChunkCount; -static uint32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB -uint32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize; +static int32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB +int32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize; -static uint32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB -uint32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize; +static int32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB +int32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize; static uint32_t const kDefaultMaxDiskChunksMemoryUsage = 10 * 1024; // 10MB uint32_t CacheObserver::sMaxDiskChunksMemoryUsage = kDefaultMaxDiskChunksMemoryUsage; @@ -160,9 +160,9 @@ CacheObserver::AttachToPreferences() mozilla::Preferences::AddUintVarCache( &sPreloadChunkCount, "browser.cache.disk.preload_chunk_count", kDefaultPreloadChunkCount); - mozilla::Preferences::AddUintVarCache( + mozilla::Preferences::AddIntVarCache( &sMaxDiskEntrySize, "browser.cache.disk.max_entry_size", kDefaultMaxDiskEntrySize); - mozilla::Preferences::AddUintVarCache( + mozilla::Preferences::AddIntVarCache( &sMaxMemoryEntrySize, "browser.cache.memory.max_entry_size", kDefaultMaxMemoryEntrySize); mozilla::Preferences::AddUintVarCache( @@ -204,22 +204,22 @@ CacheObserver::AttachToPreferences() switch (sHalfLifeExperiment) { case 1: // The experiment is engaged - sHalfLifeHours = 6; + sHalfLifeHours = 0.083F; // ~5 mintues break; case 2: - sHalfLifeHours = 24; + sHalfLifeHours = 0.25F; // 15 mintues break; case 3: - sHalfLifeHours = 7 * 24; + sHalfLifeHours = 1.0F; break; case 4: - sHalfLifeHours = 50 * 24; + sHalfLifeHours = 6.0F; break; case -1: default: // The experiment is off or broken sHalfLifeExperiment = -1; - sHalfLifeHours = std::max(1U, std::min(1440U, mozilla::Preferences::GetUint( + sHalfLifeHours = std::max(0.01F, std::min(1440.0F, mozilla::Preferences::GetFloat( "browser.cache.frecency_half_life_hours", kDefaultHalfLifeHours))); break; } @@ -407,9 +407,12 @@ CacheStorageEvictHelper::ClearStorage(bool const aPrivate, bool const CacheObserver::EntryIsTooBig(int64_t aSize, bool aUsingDisk) { // If custom limit is set, check it. - int64_t preferredLimit = aUsingDisk - ? static_cast(sMaxDiskEntrySize) << 10 - : static_cast(sMaxMemoryEntrySize) << 10; + int64_t preferredLimit = aUsingDisk ? sMaxDiskEntrySize : sMaxMemoryEntrySize; + + // do not convert to bytes when the limit is -1, which means no limit + if (preferredLimit > 0) { + preferredLimit <<= 10; + } if (preferredLimit != -1 && aSize > preferredLimit) return true; diff --git a/netwerk/cache2/CacheObserver.h b/netwerk/cache2/CacheObserver.h index 43dd410b75..8dd20db1bf 100644 --- a/netwerk/cache2/CacheObserver.h +++ b/netwerk/cache2/CacheObserver.h @@ -56,7 +56,7 @@ class CacheObserver : public nsIObserver static uint32_t const CompressionLevel() { return sCompressionLevel; } static uint32_t const HalfLifeSeconds() - { return sHalfLifeHours * 60 * 60; } + { return sHalfLifeHours * 60.0F * 60.0F; } static int32_t const HalfLifeExperiment() { return sHalfLifeExperiment; } static bool const ClearCacheOnShutdown() @@ -82,12 +82,12 @@ private: static uint32_t sDiskFreeSpaceHardLimit; static bool sSmartCacheSizeEnabled; static uint32_t sPreloadChunkCount; - static uint32_t sMaxMemoryEntrySize; - static uint32_t sMaxDiskEntrySize; + static int32_t sMaxMemoryEntrySize; + static int32_t sMaxDiskEntrySize; static uint32_t sMaxDiskChunksMemoryUsage; static uint32_t sMaxDiskPriorityChunksMemoryUsage; static uint32_t sCompressionLevel; - static uint32_t sHalfLifeHours; + static float sHalfLifeHours; static int32_t sHalfLifeExperiment; static bool sSanitizeOnShutdown; static bool sClearCacheOnShutdown; diff --git a/netwerk/cache2/CacheStorageService.cpp b/netwerk/cache2/CacheStorageService.cpp index 12050c452a..ba38ee9cc1 100644 --- a/netwerk/cache2/CacheStorageService.cpp +++ b/netwerk/cache2/CacheStorageService.cpp @@ -892,8 +892,8 @@ CacheStorageService::RegisterEntry(CacheEntry* aEntry) LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry)); MemoryPool& pool = Pool(aEntry->IsUsingDisk()); - pool.mFrecencyArray.InsertElementSorted(aEntry, FrecencyComparator()); - pool.mExpirationArray.InsertElementSorted(aEntry, ExpirationComparator()); + pool.mFrecencyArray.AppendElement(aEntry); + pool.mExpirationArray.AppendElement(aEntry); aEntry->SetRegistered(true); } diff --git a/netwerk/protocol/http/PackagedAppService.cpp b/netwerk/protocol/http/PackagedAppService.cpp index d4d5fc1360..1235b264c5 100644 --- a/netwerk/protocol/http/PackagedAppService.cpp +++ b/netwerk/protocol/http/PackagedAppService.cpp @@ -690,28 +690,55 @@ PackagedAppService::GetPackageURI(nsIURI *aURI, nsIURI **aPackageURI) } NS_IMETHODIMP -PackagedAppService::GetResource(nsIPrincipal *aPrincipal, - uint32_t aLoadFlags, - nsILoadContextInfo *aInfo, +PackagedAppService::GetResource(nsIChannel *aChannel, nsICacheEntryOpenCallback *aCallback) { - // Check arguments are not null - if (!aPrincipal || !aCallback || !aInfo) { + MOZ_RELEASE_ASSERT(NS_IsMainThread(), "mDownloadingPackages hashtable is not thread safe"); + LOG(("[%p] PackagedAppService::GetResource(aChannel: %p, aCallback: %p)\n", + this, aChannel, aCallback)); + + if (!aChannel || !aCallback) { return NS_ERROR_INVALID_ARG; } nsresult rv; + nsIScriptSecurityManager *securityManager = + nsContentUtils::GetSecurityManager(); + if (!securityManager) { + LOG(("[%p] > No securityManager\n", this)); + return NS_ERROR_UNEXPECTED; + } + nsCOMPtr principal; + rv = securityManager->GetChannelURIPrincipal(aChannel, getter_AddRefs(principal)); + if (NS_FAILED(rv) || !principal) { + LOG(("[%p] > Error getting principal rv=%X principal=%p\n", + this, rv, principal.get())); + return NS_FAILED(rv) ? rv : NS_ERROR_NULL_POINTER; + } - nsCOMPtr uri; - rv = aPrincipal->GetURI(getter_AddRefs(uri)); - if (NS_WARN_IF(NS_FAILED(rv))) { + nsCOMPtr loadContextInfo = GetLoadContextInfo(aChannel); + if (!loadContextInfo) { + LOG(("[%p] > Channel has no loadContextInfo\n", this)); + return NS_ERROR_NULL_POINTER; + } + + nsLoadFlags loadFlags = 0; + rv = aChannel->GetLoadFlags(&loadFlags); + if (NS_FAILED(rv)) { + LOG(("[%p] > Error calling GetLoadFlags rv=%X\n", this, rv)); return rv; } - LogURI("PackagedAppService::GetResource", this, uri, aInfo); + nsCOMPtr loadInfo = aChannel->GetLoadInfo(); - MOZ_RELEASE_ASSERT(NS_IsMainThread(), "mDownloadingPackages hashtable is not thread safe"); + nsCOMPtr uri; + rv = principal->GetURI(getter_AddRefs(uri)); + if (NS_WARN_IF(NS_FAILED(rv))) { + LOG(("[%p] > Error calling GetURI rv=%X\n", this, rv)); + return rv; + } + LogURI("PackagedAppService::GetResource", this, uri, loadContextInfo); nsCOMPtr packageURI; rv = GetPackageURI(uri, getter_AddRefs(packageURI)); if (NS_FAILED(rv)) { @@ -719,7 +746,7 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal, } nsAutoCString key; - CacheFileUtils::AppendKeyPrefix(aInfo, key); + CacheFileUtils::AppendKeyPrefix(loadContextInfo, key); { nsAutoCString spec; @@ -740,10 +767,10 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal, } nsCOMPtr channel; - rv = NS_NewChannel( - getter_AddRefs(channel), packageURI, aPrincipal, - nsILoadInfo::SEC_NORMAL, nsIContentPolicy::TYPE_OTHER, nullptr, nullptr, - aLoadFlags); + rv = NS_NewChannelInternal( + getter_AddRefs(channel), packageURI, + loadInfo, + nullptr, nullptr, loadFlags); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; @@ -758,7 +785,7 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal, } downloader = new PackagedAppDownloader(); - rv = downloader->Init(aInfo, key); + rv = downloader->Init(loadContextInfo, key); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } @@ -784,6 +811,10 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal, nsRefPtr listener = new PackagedAppChannelListener(downloader, mimeConverter); + if (loadInfo && loadInfo->GetEnforceSecurity()) { + return channel->AsyncOpen2(listener); + } + return channel->AsyncOpen(listener, nullptr); } diff --git a/netwerk/protocol/http/nsHttpChannel.cpp b/netwerk/protocol/http/nsHttpChannel.cpp index aa5cc4d9fe..d37dab20c3 100644 --- a/netwerk/protocol/http/nsHttpChannel.cpp +++ b/netwerk/protocol/http/nsHttpChannel.cpp @@ -5208,24 +5208,29 @@ nsHttpChannel::BeginConnect() nsRefPtr channelClassifier = new nsChannelClassifier(); if (mLoadFlags & LOAD_CLASSIFY_URI) { nsCOMPtr classifier = do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID); - if (classifier) { - bool tp = false; - channelClassifier->ShouldEnableTrackingProtection(this, &tp); + bool tpEnabled = false; + channelClassifier->ShouldEnableTrackingProtection(this, &tpEnabled); + if (classifier && tpEnabled) { // We skip speculative connections by setting mLocalBlocklist only // when tracking protection is enabled. Though we could do this for // both phishing and malware, it is not necessary for correctness, // since no network events will be received while the // nsChannelClassifier is in progress. See bug 1122691. - if (tp) { - nsCOMPtr principal = GetURIPrincipal(); - nsresult response = NS_OK; - classifier->ClassifyLocal(principal, tp, &response); - if (NS_FAILED(response)) { - LOG(("nsHttpChannel::ClassifyLocal found principal on local " - "blocklist [this=%p]", this)); + nsCOMPtr uri; + rv = GetURI(getter_AddRefs(uri)); + if (NS_SUCCEEDED(rv) && uri) { + nsAutoCString tables; + Preferences::GetCString("urlclassifier.trackingTable", &tables); + nsAutoCString results; + rv = classifier->ClassifyLocalWithTables(uri, tables, results); + if (NS_SUCCEEDED(rv) && !results.IsEmpty()) { + LOG(("nsHttpChannel::ClassifyLocalWithTables found " + "uri on local tracking blocklist [this=%p]", + this)); mLocalBlocklist = true; } else { - LOG(("nsHttpChannel::ClassifyLocal no result found [this=%p]", this)); + LOG(("nsHttpChannel::ClassifyLocalWithTables no result " + "found [this=%p]", this)); } } } @@ -5241,12 +5246,6 @@ nsHttpChannel::BeginConnect() // by the packaged app service into the cache, and the cache entry will // be passed to OnCacheEntryAvailable. - // Pass the original load flags to the packaged app request. - uint32_t loadFlags = mLoadFlags; - - mLoadFlags |= LOAD_ONLY_FROM_CACHE; - mLoadFlags |= LOAD_FROM_CACHE; - mLoadFlags &= ~VALIDATE_ALWAYS; nsCOMPtr pas = do_GetService("@mozilla.org/network/packaged-app-service;1", &rv); if (NS_WARN_IF(NS_FAILED(rv))) { @@ -5254,12 +5253,18 @@ nsHttpChannel::BeginConnect() return rv; } - nsCOMPtr principal = GetURIPrincipal(); - nsCOMPtr loadInfo = GetLoadContextInfo(this); - rv = pas->GetResource(principal, loadFlags, loadInfo, this); + rv = pas->GetResource(this, this); if (NS_FAILED(rv)) { AsyncAbort(rv); } + + // We need to alter the flags so the cache entry returned by the + // packaged app service is always accepted. Revalidation is handled + // by the service. + mLoadFlags |= LOAD_ONLY_FROM_CACHE; + mLoadFlags |= LOAD_FROM_CACHE; + mLoadFlags &= ~VALIDATE_ALWAYS; + return rv; } diff --git a/netwerk/test/unit/test_packaged_app_service.js b/netwerk/test/unit/test_packaged_app_service.js index ecb2f850b4..2e8198e8a5 100644 --- a/netwerk/test/unit/test_packaged_app_service.js +++ b/netwerk/test/unit/test_packaged_app_service.js @@ -34,6 +34,7 @@ Cu.import('resource://gre/modules/LoadContextInfo.jsm'); Cu.import("resource://testing-common/httpd.js"); Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/NetUtil.jsm"); // The number of times this package has been requested // This number might be reset by tests that use it @@ -59,11 +60,24 @@ function packagedAppContentHandler(metadata, response) response.bodyOutputStream.write(body, body.length); } -function getPrincipal(url) { +function getChannelForURL(url) { let uri = createURI(url); - return Components.classes["@mozilla.org/scriptsecuritymanager;1"] - .getService(Ci.nsIScriptSecurityManager) - .getNoAppCodebasePrincipal(uri); + let ssm = Cc["@mozilla.org/scriptsecuritymanager;1"] + .getService(Ci.nsIScriptSecurityManager); + let principal = ssm.createCodebasePrincipal(uri, {}); + let tmpChannel = + NetUtil.newChannel({ + uri: url, + loadingPrincipal: principal, + contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER + }); + + tmpChannel.notificationCallbacks = + new LoadContextCallback(principal.appId, + principal.isInBrowserElement, + false, + false); + return tmpChannel; } // The package content @@ -129,6 +143,15 @@ function run_test() add_test(test_bad_package); add_test(test_bad_package_404); + // Channels created by addons could have no load info. + // In debug mode this triggers an assertion, but we still want to test that + // it works in optimized mode. See bug 1196021 comment 17 + if (Components.classes["@mozilla.org/xpcom/debug;1"] + .getService(Components.interfaces.nsIDebug2) + .isDebugBuild == false) { + add_test(test_channel_no_loadinfo); + } + // run tests run_next_test(); } @@ -179,10 +202,10 @@ var cacheListener = new packagedResourceListener(testData.content[0].data); // These calls should fail, since one of the arguments is invalid or null function test_bad_args() { - Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com"), 0, LoadContextInfo.default, cacheListener); }, "url's with no !// aren't allowed"); - Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com/package!//test"), 0, LoadContextInfo.default, null); }, "should have a callback"); - Assert.throws(() => { paservice.getResource(null, 0, LoadContextInfo.default, cacheListener); }, "should have a URI"); - Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com/package!//test"), null, cacheListener); }, "should have a LoadContextInfo"); + Assert.throws(() => { paservice.getResource(getChannelForURL("http://test.com"), cacheListener); }, "url's with no !// aren't allowed"); + Assert.throws(() => { paservice.getResource(getChannelForURL("http://test.com/package!//test"), null); }, "should have a callback"); + Assert.throws(() => { paservice.getResource(null, cacheListener); }, "should have a channel"); + run_next_test(); } @@ -191,13 +214,15 @@ function test_bad_args() { // This tests that the callback gets called, and the cacheListener gets the proper content. function test_callback_gets_called() { packagePath = "/package"; - paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener); + let url = uri + packagePath + "!//index.html"; + paservice.getResource(getChannelForURL(url), cacheListener); } // Tests that requesting the same resource returns the same content function test_same_content() { packagePath = "/package"; - paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener); + let url = uri + packagePath + "!//index.html"; + paservice.getResource(getChannelForURL(url), cacheListener); } // Check the content handler has been called the expected number of times. @@ -209,8 +234,9 @@ function test_request_number() { // This tests that new content is returned if the package has been updated function test_updated_package() { packagePath = "/package"; - paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, - new packagedResourceListener(testData.content[0].data.replace(/\.\.\./g, 'xxx'))); + let url = uri + packagePath + "!//index.html"; + paservice.getResource(getChannelForURL(url), + new packagedResourceListener(testData.content[0].data.replace(/\.\.\./g, 'xxx'))); } // ---------------------------------------------------------------------------- @@ -233,13 +259,15 @@ var listener404 = { // Tests that an error is returned for a non existing package function test_package_does_not_exist() { packagePath = "/package_non_existent"; - paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, listener404); + let url = uri + packagePath + "!//index.html"; + paservice.getResource(getChannelForURL(url), listener404); } // Tests that an error is returned for a non existing resource in a package function test_file_does_not_exist() { packagePath = "/package"; // This package exists - paservice.getResource(getPrincipal(uri + packagePath + "!//file_non_existent.html"), 0, LoadContextInfo.default, listener404); + let url = uri + packagePath + "!//file_non_existent.html"; + paservice.getResource(getChannelForURL(url), listener404); } // ---------------------------------------------------------------------------- @@ -280,13 +308,24 @@ function packagedAppBadContentHandler(metadata, response) // Checks that the resource with the proper headers inside the bad package is still returned function test_bad_package() { packagePath = "/badPackage"; - paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener); + let url = uri + packagePath + "!//index.html"; + paservice.getResource(getChannelForURL(url), cacheListener); } // Checks that the request for a non-existent resource doesn't hang for a bad package function test_bad_package_404() { packagePath = "/badPackage"; - paservice.getResource(getPrincipal(uri + packagePath + "!//file_non_existent.html"), 0, LoadContextInfo.default, listener404); + let url = uri + packagePath + "!//file_non_existent.html"; + paservice.getResource(getChannelForURL(url), listener404); } // ---------------------------------------------------------------------------- + +// NOTE: This test only runs in NON-DEBUG mode. +function test_channel_no_loadinfo() { + packagePath = "/package"; + let url = uri + packagePath + "!//index.html"; + let channel = getChannelForURL(url); + channel.loadInfo = null; + paservice.getResource(channel, cacheListener); +} diff --git a/netwerk/test/unit/test_packaged_app_service_paths.js b/netwerk/test/unit/test_packaged_app_service_paths.js index 306676ccc6..568d23d85a 100644 --- a/netwerk/test/unit/test_packaged_app_service_paths.js +++ b/netwerk/test/unit/test_packaged_app_service_paths.js @@ -1,6 +1,7 @@ Cu.import('resource://gre/modules/LoadContextInfo.jsm'); Cu.import("resource://testing-common/httpd.js"); Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/NetUtil.jsm"); var gRequestNo = 0; function packagedAppContentHandler(metadata, response) @@ -11,11 +12,24 @@ function packagedAppContentHandler(metadata, response) gRequestNo++; } -function getPrincipal(url) { +function getChannelForURL(url) { let uri = createURI(url); - return Components.classes["@mozilla.org/scriptsecuritymanager;1"] - .getService(Ci.nsIScriptSecurityManager) - .getNoAppCodebasePrincipal(uri); + let ssm = Cc["@mozilla.org/scriptsecuritymanager;1"] + .getService(Ci.nsIScriptSecurityManager); + let principal = ssm.createCodebasePrincipal(uri, {}); + let tmpChannel = + NetUtil.newChannel({ + uri: url, + loadingPrincipal: principal, + contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER + }); + + tmpChannel.notificationCallbacks = + new LoadContextCallback(principal.appId, + principal.isInBrowserElement, + false, + false); + return tmpChannel; } var subresourcePaths = [ @@ -119,8 +133,8 @@ function test_paths() { for (var i in subresourcePaths) { packagePath = "/package/" + i; dump("Iteration " + i + "\n"); - paservice.getResource(getPrincipal(uri + packagePath + "!//" + subresourcePaths[i][1]), 0, - LoadContextInfo.default, + let url = uri + packagePath + "!//" + subresourcePaths[i][1]; + paservice.getResource(getChannelForURL(url), new packagedResourceListener(subresourcePaths[i][1], content)); yield undefined; } diff --git a/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-match.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-match.https.html.ini index 52b6da31c2..2ca31d4ddd 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-match.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-match.https.html.ini @@ -1,3 +1,6 @@ [cache-match.https.html] type: testharness prefs: [dom.serviceWorkers.enabled: true, dom.serviceWorkers.interception.enabled: true, dom.serviceWorkers.exemptFromPerDomainMax:true, dom.caches.enabled:true] + [Cache.match and Cache.matchAll] + expected: FAIL + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-put.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-put.https.html.ini index 4414167e39..a4be22c2b9 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-put.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-put.https.html.ini @@ -1,3 +1,6 @@ [cache-put.https.html] type: testharness prefs: [dom.serviceWorkers.enabled: true, dom.serviceWorkers.interception.enabled: true, dom.serviceWorkers.exemptFromPerDomainMax:true, dom.caches.enabled:true] + [Cache.put with request URLs containing embedded credentials] + expected: FAIL + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/testing/web-platform/meta/service-workers/cache-storage/window/cache-match.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/window/cache-match.https.html.ini index c88be458d2..860f0963c7 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/window/cache-match.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/window/cache-match.https.html.ini @@ -1,3 +1,5 @@ [cache-match.https.html] type: testharness prefs: [dom.caches.enabled:true] + expected: ERROR + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/testing/web-platform/meta/service-workers/cache-storage/window/cache-put.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/window/cache-put.https.html.ini index 236befe797..66a4a3c7ec 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/window/cache-put.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/window/cache-put.https.html.ini @@ -1,3 +1,6 @@ [cache-put.https.html] type: testharness prefs: [dom.caches.enabled:true] + [Cache.put with request URLs containing embedded credentials] + expected: FAIL + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/testing/web-platform/meta/service-workers/cache-storage/worker/cache-match.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/worker/cache-match.https.html.ini index c88be458d2..860f0963c7 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/worker/cache-match.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/worker/cache-match.https.html.ini @@ -1,3 +1,5 @@ [cache-match.https.html] type: testharness prefs: [dom.caches.enabled:true] + expected: ERROR + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/testing/web-platform/meta/service-workers/cache-storage/worker/cache-put.https.html.ini b/testing/web-platform/meta/service-workers/cache-storage/worker/cache-put.https.html.ini index 236befe797..66a4a3c7ec 100644 --- a/testing/web-platform/meta/service-workers/cache-storage/worker/cache-put.https.html.ini +++ b/testing/web-platform/meta/service-workers/cache-storage/worker/cache-put.https.html.ini @@ -1,3 +1,6 @@ [cache-put.https.html] type: testharness prefs: [dom.caches.enabled:true] + [Cache.put with request URLs containing embedded credentials] + expected: FAIL + bug: https://github.com/w3c/web-platform-tests/issues/2098 diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index a4653bb59c..a1bc04b87b 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -1542,11 +1542,17 @@ "description": "HTTP Disk Cache Hit, Reval, Failed-Reval, Miss" }, "HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT": { - "expires_in_version": "never", + "expires_in_version": "40", "kind": "enumerated", "n_values": 4, "description": "HTTP Cache v2 Miss by half-life value (6h, 1d, 7d, 50d)" }, + "HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT_2": { + "expires_in_version": "never", + "kind": "enumerated", + "n_values": 4, + "description": "HTTP Cache v2 Miss by half-life value (5 min, 15 min, 1 hour, 6 hours)" + }, "HTTP_CACHE_ENTRY_RELOAD_TIME": { "expires_in_version": "never", "kind": "exponential", diff --git a/toolkit/components/url-classifier/SafeBrowsing.jsm b/toolkit/components/url-classifier/SafeBrowsing.jsm new file mode 100644 index 0000000000..17a24b62fe --- /dev/null +++ b/toolkit/components/url-classifier/SafeBrowsing.jsm @@ -0,0 +1,274 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +this.EXPORTED_SYMBOLS = ["SafeBrowsing"]; + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/Services.jsm"); + +// Skip all the ones containining "test", because we never need to ask for +// updates for them. +function getLists(prefName) { + let pref = Services.prefs.getCharPref(prefName); + // Splitting an empty string returns [''], we really want an empty array. + if (!pref) { + return []; + } + return pref.split(",") + .filter(function(value) { return value.indexOf("test-") == -1; }) + .map(function(value) { return value.trim(); }); +} + +// These may be a comma-separated lists of tables. +const phishingLists = getLists("urlclassifier.phishTable"); +const malwareLists = getLists("urlclassifier.malwareTable"); +const downloadBlockLists = getLists("urlclassifier.downloadBlockTable"); +const downloadAllowLists = getLists("urlclassifier.downloadAllowTable"); +const trackingProtectionLists = getLists("urlclassifier.trackingTable"); +const trackingProtectionWhitelists = getLists("urlclassifier.trackingWhitelistTable"); + +var debug = false; +function log(...stuff) { + if (!debug) + return; + + var d = new Date(); + let msg = "SafeBrowsing: " + d.toTimeString() + ": " + stuff.join(" "); + Services.console.logStringMessage(msg); + dump(msg + "\n"); +} + +this.SafeBrowsing = { + + init: function() { + if (this.initialized) { + log("Already initialized"); + return; + } + + Services.prefs.addObserver("browser.safebrowsing", this.readPrefs.bind(this), false); + Services.prefs.addObserver("privacy.trackingprotection", this.readPrefs.bind(this), false); + this.readPrefs(); + + // Register our two types of tables, and add custom Mozilla entries + let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + for (let i = 0; i < phishingLists.length; ++i) { + listManager.registerTable(phishingLists[i], this.updateURL, this.gethashURL); + } + for (let i = 0; i < malwareLists.length; ++i) { + listManager.registerTable(malwareLists[i], this.updateURL, this.gethashURL); + } + for (let i = 0; i < downloadBlockLists.length; ++i) { + listManager.registerTable(downloadBlockLists[i], this.updateURL, this.gethashURL); + } + for (let i = 0; i < downloadAllowLists.length; ++i) { + listManager.registerTable(downloadAllowLists[i], this.updateURL, this.gethashURL); + } + for (let i = 0; i < trackingProtectionLists.length; ++i) { + listManager.registerTable(trackingProtectionLists[i], + this.trackingUpdateURL, + this.trackingGethashURL); + } + for (let i = 0; i < trackingProtectionWhitelists.length; ++i) { + listManager.registerTable(trackingProtectionWhitelists[i], + this.trackingUpdateURL, + this.trackingGethashURL); + } + this.addMozEntries(); + + this.controlUpdateChecking(); + this.initialized = true; + + log("init() finished"); + }, + + + initialized: false, + phishingEnabled: false, + malwareEnabled: false, + + updateURL: null, + gethashURL: null, + + reportURL: null, + + getReportURL: function(kind, URI) { + let pref; + switch (kind) { + case "Phish": + pref = "browser.safebrowsing.reportPhishURL"; + break; + case "PhishMistake": + pref = "browser.safebrowsing.reportPhishMistakeURL"; + break; + case "MalwareMistake": + pref = "browser.safebrowsing.reportMalwareMistakeURL"; + break; + + default: + let err = "SafeBrowsing getReportURL() called with unknown kind: " + kind; + Components.utils.reportError(err); + throw err; + } + let reportUrl = Services.urlFormatter.formatURLPref(pref); + + let pageUri = URI.clone(); + + // Remove the query to avoid including potentially sensitive data + if (pageUri instanceof Ci.nsIURL) + pageUri.query = ''; + + reportUrl += encodeURIComponent(pageUri.asciiSpec); + + return reportUrl; + }, + + + readPrefs: function() { + log("reading prefs"); + + debug = Services.prefs.getBoolPref("browser.safebrowsing.debug"); + this.phishingEnabled = Services.prefs.getBoolPref("browser.safebrowsing.enabled"); + this.malwareEnabled = Services.prefs.getBoolPref("browser.safebrowsing.malware.enabled"); + this.trackingEnabled = Services.prefs.getBoolPref("privacy.trackingprotection.enabled") || Services.prefs.getBoolPref("privacy.trackingprotection.pbmode.enabled"); + this.updateProviderURLs(); + + // XXX The listManager backend gets confused if this is called before the + // lists are registered. So only call it here when a pref changes, and not + // when doing initialization. I expect to refactor this later, so pardon the hack. + if (this.initialized) { + this.controlUpdateChecking(); + } + }, + + + updateProviderURLs: function() { + try { + var clientID = Services.prefs.getCharPref("browser.safebrowsing.id"); + } catch(e) { + var clientID = Services.appinfo.name; + } + + log("initializing safe browsing URLs, client id ", clientID); + + // Urls used to update DB + this.updateURL = Services.urlFormatter.formatURLPref("browser.safebrowsing.updateURL"); + this.gethashURL = Services.urlFormatter.formatURLPref("browser.safebrowsing.gethashURL"); + + this.updateURL = this.updateURL.replace("SAFEBROWSING_ID", clientID); + this.gethashURL = this.gethashURL.replace("SAFEBROWSING_ID", clientID); + this.trackingUpdateURL = Services.urlFormatter.formatURLPref( + "browser.trackingprotection.updateURL"); + this.trackingGethashURL = Services.urlFormatter.formatURLPref( + "browser.trackingprotection.gethashURL"); + }, + + controlUpdateChecking: function() { + log("phishingEnabled:", this.phishingEnabled, "malwareEnabled:", + this.malwareEnabled, "trackingEnabled:", this.trackingEnabled); + + let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]. + getService(Ci.nsIUrlListManager); + + for (let i = 0; i < phishingLists.length; ++i) { + if (this.phishingEnabled) { + listManager.enableUpdate(phishingLists[i]); + } else { + listManager.disableUpdate(phishingLists[i]); + } + } + for (let i = 0; i < malwareLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(malwareLists[i]); + } else { + listManager.disableUpdate(malwareLists[i]); + } + } + for (let i = 0; i < downloadBlockLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(downloadBlockLists[i]); + } else { + listManager.disableUpdate(downloadBlockLists[i]); + } + } + for (let i = 0; i < downloadAllowLists.length; ++i) { + if (this.malwareEnabled) { + listManager.enableUpdate(downloadAllowLists[i]); + } else { + listManager.disableUpdate(downloadAllowLists[i]); + } + } + for (let i = 0; i < trackingProtectionLists.length; ++i) { + if (this.trackingEnabled) { + listManager.enableUpdate(trackingProtectionLists[i]); + listManager.enableUpdate(trackingProtectionWhitelists[i]); + } else { + listManager.disableUpdate(trackingProtectionLists[i]); + listManager.disableUpdate(trackingProtectionWhitelists[i]); + } + } + listManager.maybeToggleUpdateChecking(); + }, + + + addMozEntries: function() { + // Add test entries to the DB. + // XXX bug 779008 - this could be done by DB itself? + const phishURL = "itisatrap.org/firefox/its-a-trap.html"; + const malwareURL = "itisatrap.org/firefox/its-an-attack.html"; + const unwantedURL = "itisatrap.org/firefox/unwanted.html"; + const trackerURLs = [ + "trackertest.org/", + "itisatracker.org/", + ]; + const whitelistURL = "itisatrap.org/?resource=itisatracker.org"; + + let update = "n:1000\ni:test-malware-simple\nad:1\n" + + "a:1:32:" + malwareURL.length + "\n" + + malwareURL + "\n"; + update += "n:1000\ni:test-phish-simple\nad:1\n" + + "a:1:32:" + phishURL.length + "\n" + + phishURL + "\n"; + update += "n:1000\ni:test-unwanted-simple\nad:1\n" + + "a:1:32:" + unwantedURL.length + "\n" + + unwantedURL + "\n"; + update += "n:1000\ni:test-track-simple\n" + + "ad:" + trackerURLs.length + "\n"; + trackerURLs.forEach((trackerURL, i) => { + update += "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" + + trackerURL + "\n"; + }); + update += "n:1000\ni:test-trackwhite-simple\nad:1\n" + + "a:1:32:" + whitelistURL.length + "\n" + + whitelistURL; + log("addMozEntries:", update); + + let db = Cc["@mozilla.org/url-classifier/dbservice;1"]. + getService(Ci.nsIUrlClassifierDBService); + + // nsIUrlClassifierUpdateObserver + let dummyListener = { + updateUrlRequested: function() { }, + streamFinished: function() { }, + updateError: function() { }, + updateSuccess: function() { } + }; + + try { + let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple"; + db.beginUpdate(dummyListener, tables, ""); + db.beginStream("", ""); + db.updateStream(update); + db.finishStream(); + db.finishUpdate(); + } catch(ex) { + // beginUpdate will throw harmlessly if there's an existing update in progress, ignore failures. + log("addMozEntries failed!", ex); + } + }, +}; diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp new file mode 100644 index 0000000000..985068b0f9 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp @@ -0,0 +1,1607 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsAutoPtr.h" +#include "nsCOMPtr.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsCRT.h" +#include "nsICryptoHash.h" +#include "nsICryptoHMAC.h" +#include "nsIDirectoryService.h" +#include "nsIKeyModule.h" +#include "nsIObserverService.h" +#include "nsIPermissionManager.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsIProperties.h" +#include "nsToolkitCompsCID.h" +#include "nsIUrlClassifierUtils.h" +#include "nsIXULRuntime.h" +#include "nsUrlClassifierDBService.h" +#include "nsUrlClassifierUtils.h" +#include "nsUrlClassifierProxies.h" +#include "nsURILoader.h" +#include "nsString.h" +#include "nsReadableUtils.h" +#include "nsTArray.h" +#include "nsNetUtil.h" +#include "nsNetCID.h" +#include "nsThreadUtils.h" +#include "nsXPCOMStrings.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "mozilla/Atomics.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Mutex.h" +#include "mozilla/Preferences.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/Telemetry.h" +#include "prlog.h" +#include "prprf.h" +#include "prnetdb.h" +#include "Entries.h" +#include "HashStore.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "mozilla/Attributes.h" +#include "nsIPrincipal.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "nsContentUtils.h" + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +// NSPR_LOG_MODULES=UrlClassifierDbService:5 +#if defined(PR_LOGGING) +PRLogModuleInfo *gUrlClassifierDbServiceLog = nullptr; +#define LOG(args) PR_LOG(gUrlClassifierDbServiceLog, PR_LOG_DEBUG, args) +#define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierDbServiceLog, 4) +#else +#define LOG(args) +#define LOG_ENABLED() (false) +#endif + +// Prefs for implementing nsIURIClassifier to block page loads +#define CHECK_MALWARE_PREF "browser.safebrowsing.malware.enabled" +#define CHECK_MALWARE_DEFAULT false + +#define CHECK_PHISHING_PREF "browser.safebrowsing.enabled" +#define CHECK_PHISHING_DEFAULT false + +#define CHECK_TRACKING_PREF "privacy.trackingprotection.enabled" +#define CHECK_TRACKING_DEFAULT false + +#define GETHASH_NOISE_PREF "urlclassifier.gethashnoise" +#define GETHASH_NOISE_DEFAULT 4 + +// Comma-separated lists +#define MALWARE_TABLE_PREF "urlclassifier.malwareTable" +#define PHISH_TABLE_PREF "urlclassifier.phishTable" +#define TRACKING_TABLE_PREF "urlclassifier.trackingTable" +#define TRACKING_WHITELIST_TABLE_PREF "urlclassifier.trackingWhitelistTable" +#define DOWNLOAD_BLOCK_TABLE_PREF "urlclassifier.downloadBlockTable" +#define DOWNLOAD_ALLOW_TABLE_PREF "urlclassifier.downloadAllowTable" +#define DISALLOW_COMPLETION_TABLE_PREF "urlclassifier.disallow_completions" + +#define CONFIRM_AGE_PREF "urlclassifier.max-complete-age" +#define CONFIRM_AGE_DEFAULT_SEC (45 * 60) + +class nsUrlClassifierDBServiceWorker; + +// Singleton instance. +static nsUrlClassifierDBService* sUrlClassifierDBService; + +nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nullptr; + +// Once we've committed to shutting down, don't do work in the background +// thread. +static bool gShuttingDownThread = false; + +static mozilla::Atomic gFreshnessGuarantee(CONFIRM_AGE_DEFAULT_SEC); + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker, + nsIUrlClassifierDBServiceWorker, + nsIUrlClassifierDBService) + +nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker() + : mInStream(false) + , mGethashNoise(0) + , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock") +{ +} + +nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker() +{ + NS_ASSERTION(!mClassifier, + "Db connection not closed, leaking memory! Call CloseDb " + "to close the connection."); +} + +nsresult +nsUrlClassifierDBServiceWorker::Init(uint32_t aGethashNoise, + nsCOMPtr aCacheDir) +{ + mGethashNoise = aGethashNoise; + mCacheDir = aCacheDir; + + ResetUpdate(); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* callback) +{ + MutexAutoLock lock(mPendingLookupLock); + + PendingLookup* lookup = mPendingLookups.AppendElement(); + if (!lookup) return NS_ERROR_OUT_OF_MEMORY; + + lookup->mStartTime = TimeStamp::Now(); + lookup->mKey = spec; + lookup->mCallback = callback; + lookup->mTables = tables; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::DoLocalLookup(const nsACString& spec, + const nsACString& tables, + LookupResultArray* results) +{ + MOZ_ASSERT(!NS_IsMainThread(), "DoLocalLookup must be on background thread"); + if (!results) { + return NS_ERROR_FAILURE; + } + // Bail if we haven't been initialized on the background thread. + if (!mClassifier) { + return NS_ERROR_NOT_AVAILABLE; + } + + // We ignore failures from Check because we'd rather return the + // results that were found than fail. + mClassifier->Check(spec, tables, gFreshnessGuarantee, *results); + + LOG(("Found %d results.", results->Length())); + return NS_OK; +} + +static nsresult +TablesToResponse(const nsACString& tables) +{ + // We don't check mCheckMalware and friends because BuildTables never + // includes a table that is not enabled. + if (FindInReadable(NS_LITERAL_CSTRING("-malware-"), tables)) { + return NS_ERROR_MALWARE_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-phish-"), tables)) { + return NS_ERROR_PHISHING_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-track-"), tables)) { + return NS_ERROR_TRACKING_URI; + } + return NS_OK; +} + +static nsCString +ProcessLookupResults(LookupResultArray* results) +{ + // Build a stringified list of result tables. + nsTArray tables; + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + MOZ_ASSERT(!result.mNoise, "Lookup results should not have noise added"); + LOG(("Found result from table %s", result.mTableName.get())); + if (tables.IndexOf(result.mTableName) == nsTArray::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + return tableStr; +} + +/** + * Lookup up a key in the database is a two step process: + * + * a) First we look for any Entries in the database that might apply to this + * url. For each URL there are one or two possible domain names to check: + * the two-part domain name (example.com) and the three-part name + * (www.example.com). We check the database for both of these. + * b) If we find any entries, we check the list of fragments for that entry + * against the possible subfragments of the URL as described in the + * "Simplified Regular Expression Lookup" section of the protocol doc. + */ +nsresult +nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* c) +{ + if (gShuttingDownThread) { + c->LookupComplete(nullptr); + return NS_ERROR_NOT_INITIALIZED; + } + +#if defined(PR_LOGGING) + PRIntervalTime clockStart = 0; + if (LOG_ENABLED()) { + clockStart = PR_IntervalNow(); + } +#endif + + nsAutoPtr results(new LookupResultArray()); + if (!results) { + c->LookupComplete(nullptr); + return NS_ERROR_OUT_OF_MEMORY; + } + + nsresult rv = DoLocalLookup(spec, tables, results); + if (NS_FAILED(rv)) { + c->LookupComplete(nullptr); + return rv; + } + + LOG(("Found %d results.", results->Length())); + + +#if defined(PR_LOGGING) + if (LOG_ENABLED()) { + PRIntervalTime clockEnd = PR_IntervalNow(); + LOG(("query took %dms\n", + PR_IntervalToMilliseconds(clockEnd - clockStart))); + } +#endif + + nsAutoPtr completes(new LookupResultArray()); + + for (uint32_t i = 0; i < results->Length(); i++) { + if (!mMissCache.Contains(results->ElementAt(i).hash.prefix)) { + completes->AppendElement(results->ElementAt(i)); + } + } + + for (uint32_t i = 0; i < completes->Length(); i++) { + if (!completes->ElementAt(i).Confirmed()) { + // We're going to be doing a gethash request, add some extra entries. + // Note that we cannot pass the first two by reference, because we + // add to completes, whicah can cause completes to reallocate and move. + AddNoise(completes->ElementAt(i).hash.prefix, + completes->ElementAt(i).mTableName, + mGethashNoise, *completes); + break; + } + } + + // At this point ownership of 'results' is handed to the callback. + c->LookupComplete(completes.forget()); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::HandlePendingLookups() +{ + MutexAutoLock lock(mPendingLookupLock); + while (mPendingLookups.Length() > 0) { + PendingLookup lookup = mPendingLookups[0]; + mPendingLookups.RemoveElementAt(0); + { + MutexAutoUnlock unlock(mPendingLookupLock); + DoLookup(lookup.mKey, lookup.mTables, lookup.mCallback); + } + double lookupTime = (TimeStamp::Now() - lookup.mStartTime).ToMilliseconds(); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LOOKUP_TIME, + static_cast(lookupTime)); + } + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::AddNoise(const Prefix aPrefix, + const nsCString tableName, + uint32_t aCount, + LookupResultArray& results) +{ + if (aCount < 1) { + return NS_OK; + } + + PrefixArray noiseEntries; + nsresult rv = mClassifier->ReadNoiseEntries(aPrefix, tableName, + aCount, &noiseEntries); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < noiseEntries.Length(); i++) { + LookupResult *result = results.AppendElement(); + if (!result) + return NS_ERROR_OUT_OF_MEMORY; + + result->hash.prefix = noiseEntries[i]; + result->mNoise = true; + + result->mTableName.Assign(tableName); + } + + return NS_OK; +} + +// Lookup a key in the db. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::Lookup(nsIPrincipal* aPrincipal, + const nsACString& aTables, + nsIUrlClassifierCallback* c) +{ + return HandlePendingLookups(); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c) +{ + if (gShuttingDownThread) + return NS_ERROR_NOT_INITIALIZED; + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString response; + mClassifier->TableRequest(response); + LOG(("GetTables: %s", response.get())); + c->HandleEvent(response); + + return rv; +} + +void +nsUrlClassifierDBServiceWorker::ResetStream() +{ + LOG(("ResetStream")); + mInStream = false; + mProtocolParser = nullptr; +} + +void +nsUrlClassifierDBServiceWorker::ResetUpdate() +{ + LOG(("ResetUpdate")); + mUpdateWait = 0; + mUpdateStatus = NS_OK; + mUpdateObserver = nullptr; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &tables) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginUpdate [%s]", PromiseFlatCString(tables).get())); + + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(!mUpdateObserver); + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + mUpdateStatus = NS_OK; + mUpdateObserver = observer; + Classifier::SplitTables(tables, mUpdateTables); + + return NS_OK; +} + +// Called from the stream updater. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginStream")); + MOZ_ASSERT(!NS_IsMainThread(), "Streaming must be on the background thread"); + + if (gShuttingDownThread) + return NS_ERROR_NOT_INITIALIZED; + + NS_ENSURE_STATE(mUpdateObserver); + NS_ENSURE_STATE(!mInStream); + + mInStream = true; + + NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser."); + + mProtocolParser = new ProtocolParser(); + if (!mProtocolParser) + return NS_ERROR_OUT_OF_MEMORY; + + mProtocolParser->Init(mCryptoHash); + + if (!table.IsEmpty()) { + mProtocolParser->SetCurrentTable(table); + } + + return NS_OK; +} + +/** + * Updating the database: + * + * The Update() method takes a series of chunks separated with control data, + * as described in + * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec + * + * It will iterate through the control data until it reaches a chunk. By + * the time it reaches a chunk, it should have received + * a) the table to which this chunk applies + * b) the type of chunk (add, delete, expire add, expire delete). + * c) the chunk ID + * d) the length of the chunk. + * + * For add and subtract chunks, it needs to read the chunk data (expires + * don't have any data). Chunk data is a list of URI fragments whose + * encoding depends on the type of table (which is indicated by the end + * of the table name): + * a) tables ending with -exp are a zlib-compressed list of URI fragments + * separated by newlines. + * b) tables ending with -sha128 have the form + * [domain][N][frag0]...[fragN] + * 16 1 16 16 + * If N is 0, the domain is reused as a fragment. + * c) any other tables are assumed to be a plaintext list of URI fragments + * separated by newlines. + * + * Update() can be fed partial data; It will accumulate data until there is + * enough to act on. Finish() should be called when there will be no more + * data. + */ +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk) +{ + if (gShuttingDownThread) + return NS_ERROR_NOT_INITIALIZED; + + NS_ENSURE_STATE(mInStream); + + HandlePendingLookups(); + + // Feed the chunk to the parser. + return mProtocolParser->AppendStream(chunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishStream() +{ + if (gShuttingDownThread) { + LOG(("shutting down")); + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mInStream); + NS_ENSURE_STATE(mUpdateObserver); + + mInStream = false; + + if (NS_SUCCEEDED(mProtocolParser->Status())) { + if (mProtocolParser->UpdateWait()) { + mUpdateWait = mProtocolParser->UpdateWait(); + } + // XXX: Only allow forwards from the initial update? + const nsTArray &forwards = + mProtocolParser->Forwards(); + for (uint32_t i = 0; i < forwards.Length(); i++) { + const ProtocolParser::ForwardedUpdate &forward = forwards[i]; + mUpdateObserver->UpdateUrlRequested(forward.url, forward.table); + } + // Hold on to any TableUpdate objects that were created by the + // parser. + mTableUpdates.AppendElements(mProtocolParser->GetTableUpdates()); + mProtocolParser->ForgetTableUpdates(); + } else { + mUpdateStatus = mProtocolParser->Status(); + } + mUpdateObserver->StreamFinished(mProtocolParser->Status(), 0); + + if (NS_SUCCEEDED(mUpdateStatus)) { + if (mProtocolParser->ResetRequested()) { + mClassifier->Reset(); + } + } + + mProtocolParser = nullptr; + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishUpdate() +{ + if (gShuttingDownThread) + return NS_ERROR_NOT_INITIALIZED; + NS_ENSURE_STATE(mUpdateObserver); + + if (NS_SUCCEEDED(mUpdateStatus)) { + mUpdateStatus = ApplyUpdate(); + } + + mMissCache.Clear(); + + if (NS_SUCCEEDED(mUpdateStatus)) { + LOG(("Notifying success: %d", mUpdateWait)); + mUpdateObserver->UpdateSuccess(mUpdateWait); + } else { + LOG(("Notifying error: %d", mUpdateStatus)); + mUpdateObserver->UpdateError(mUpdateStatus); + /* + * mark the tables as spoiled, we don't want to block hosts + * longer than normal because our update failed + */ + mClassifier->MarkSpoiled(mUpdateTables); + } + mUpdateObserver = nullptr; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::ApplyUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate()")); + return mClassifier->ApplyUpdates(&mTableUpdates); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ResetDatabase() +{ + nsresult rv = OpenDb(); + + if (NS_SUCCEEDED(rv)) { + mClassifier->Reset(); + } + + rv = CloseDb(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CancelUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::CancelUpdate")); + + if (mUpdateObserver) { + LOG(("UpdateObserver exists, cancelling")); + + mUpdateStatus = NS_BINDING_ABORTED; + + mUpdateObserver->UpdateError(mUpdateStatus); + + /* + * mark the tables as spoiled, we don't want to block hosts + * longer than normal because our update failed + */ + mClassifier->MarkSpoiled(mUpdateTables); + + ResetStream(); + ResetUpdate(); + } else { + LOG(("No UpdateObserver, nothing to cancel")); + } + + return NS_OK; +} + +// Allows the main thread to delete the connection which may be in +// a background thread. +// XXX This could be turned into a single shutdown event so the logic +// is simpler in nsUrlClassifierDBService::Shutdown. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CloseDb() +{ + if (mClassifier) { + mClassifier->Close(); + mClassifier = nullptr; + } + + mCryptoHash = nullptr; + LOG(("urlclassifier db closed\n")); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CacheCompletions(CacheResultArray *results) +{ + LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this)); + if (!mClassifier) + return NS_OK; + + // Ownership is transferred in to us + nsAutoPtr resultsPtr(results); + + nsAutoPtr pParse(new ProtocolParser()); + nsTArray updates; + + // Only cache results for tables that we have, don't take + // in tables we might accidentally have hit during a completion. + // This happens due to goog vs googpub lists existing. + nsTArray tables; + nsresult rv = mClassifier->ActiveTables(tables); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + bool activeTable = false; + for (uint32_t table = 0; table < tables.Length(); table++) { + if (tables[table].Equals(resultsPtr->ElementAt(i).table)) { + activeTable = true; + break; + } + } + if (activeTable) { + TableUpdate * tu = pParse->GetTableUpdate(resultsPtr->ElementAt(i).table); + LOG(("CacheCompletion Addchunk %d hash %X", resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.ToUint32())); + rv = tu->NewAddComplete(resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.complete); + if (NS_FAILED(rv)) { + // We can bail without leaking here because ForgetTableUpdates + // hasn't been called yet. + return rv; + } + rv = tu->NewAddChunk(resultsPtr->ElementAt(i).entry.addChunk); + if (NS_FAILED(rv)) { + return rv; + } + tu->SetLocalUpdate(); + updates.AppendElement(tu); + pParse->ForgetTableUpdates(); + } else { + LOG(("Completion received, but table is not active, so not caching.")); + } + } + + mClassifier->ApplyUpdates(&updates); + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CacheMisses(PrefixArray *results) +{ + LOG(("nsUrlClassifierDBServiceWorker::CacheMisses [%p] %d", + this, results->Length())); + + // Ownership is transferred in to us + nsAutoPtr resultsPtr(results); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + mMissCache.AppendElement(resultsPtr->ElementAt(i)); + } + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::OpenDb() +{ + MOZ_ASSERT(!NS_IsMainThread(), "Must initialize DB on background thread"); + // Connection already open, don't do anything. + if (mClassifier) { + return NS_OK; + } + + nsresult rv; + mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr classifier(new Classifier()); + if (!classifier) { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = classifier->Open(*mCacheDir); + NS_ENSURE_SUCCESS(rv, rv); + + mClassifier = classifier; + + return NS_OK; +} + +// ------------------------------------------------------------------------- +// nsUrlClassifierLookupCallback +// +// This class takes the results of a lookup found on the worker thread +// and handles any necessary partial hash expansions before calling +// the client callback. + +class nsUrlClassifierLookupCallback final : public nsIUrlClassifierLookupCallback + , public nsIUrlClassifierHashCompleterCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK + NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK + + nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice, + nsIUrlClassifierCallback *c) + : mDBService(dbservice) + , mResults(nullptr) + , mPendingCompletions(0) + , mCallback(c) + {} + +private: + ~nsUrlClassifierLookupCallback(); + + nsresult HandleResults(); + + nsRefPtr mDBService; + nsAutoPtr mResults; + + // Completed results to send back to the worker for caching. + nsAutoPtr mCacheResults; + + uint32_t mPendingCompletions; + nsCOMPtr mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback, + nsIUrlClassifierLookupCallback, + nsIUrlClassifierHashCompleterCallback) + +nsUrlClassifierLookupCallback::~nsUrlClassifierLookupCallback() +{ + nsCOMPtr thread; + (void)NS_GetMainThread(getter_AddRefs(thread)); + + if (mCallback) { + (void)NS_ProxyRelease(thread, mCallback, false); + } +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::LookupComplete(nsTArray* results) +{ + NS_ASSERTION(mResults == nullptr, + "Should only get one set of results per nsUrlClassifierLookupCallback!"); + + if (!results) { + HandleResults(); + return NS_OK; + } + + mResults = results; + + // Check the results entries that need to be completed. + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + + // We will complete partial matches and matches that are stale. + if (!result.Confirmed()) { + nsCOMPtr completer; + nsCString gethashUrl; + nsresult rv; + nsCOMPtr listManager = do_GetService( + "@mozilla.org/url-classifier/listmanager;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = listManager->GetGethashUrl(result.mTableName, gethashUrl); + NS_ENSURE_SUCCESS(rv, rv); + // gethashUrls may be empty in 2 cases: test tables, and on startup where + // we may have found a prefix in an existing table before the listmanager + // has registered the table. In the second case we should not call + // complete. + if ((!gethashUrl.IsEmpty() || + StringBeginsWith(result.mTableName, NS_LITERAL_CSTRING("test-"))) && + mDBService->GetCompleter(result.mTableName, + getter_AddRefs(completer))) { + nsAutoCString partialHash; + partialHash.Assign(reinterpret_cast(&result.hash.prefix), + PREFIX_SIZE); + + nsresult rv = completer->Complete(partialHash, gethashUrl, this); + if (NS_SUCCEEDED(rv)) { + mPendingCompletions++; + } + } else { + // For tables with no hash completer, a complete hash match is + // good enough, we'll consider it fresh, even if it hasn't been updated + // in 45 minutes. + if (result.Complete()) { + result.mFresh = true; + } else { + NS_WARNING("Partial match in a table without a valid completer, ignoring partial match."); + } + } + } + } + + if (mPendingCompletions == 0) { + // All results were complete, we're ready! + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::CompletionFinished(nsresult status) +{ + LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %08x]", + this, status)); + if (NS_FAILED(status)) { + NS_WARNING("gethash response failed."); + } + + mPendingCompletions--; + if (mPendingCompletions == 0) { + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash, + const nsACString& tableName, + uint32_t chunkId) +{ + LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d]", + this, PromiseFlatCString(tableName).get(), chunkId)); + mozilla::safebrowsing::Completion hash; + hash.Assign(completeHash); + + // Send this completion to the store for caching. + if (!mCacheResults) { + mCacheResults = new CacheResultArray(); + if (!mCacheResults) + return NS_ERROR_OUT_OF_MEMORY; + } + + CacheResult result; + result.entry.addChunk = chunkId; + result.entry.complete = hash; + result.table = tableName; + + // OK if this fails, we just won't cache the item. + mCacheResults->AppendElement(result); + + // Check if this matched any of our results. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Now, see if it verifies a lookup + if (result.CompleteHash() == hash && result.mTableName.Equals(tableName)) { + result.mProtocolConfirmed = true; + } + } + + return NS_OK; +} + +nsresult +nsUrlClassifierLookupCallback::HandleResults() +{ + if (!mResults) { + // No results, this URI is clean. + return mCallback->HandleEvent(NS_LITERAL_CSTRING("")); + } + + nsTArray tables; + // Build a stringified list of result tables. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Leave out results that weren't confirmed, as their existence on + // the list can't be verified. Also leave out randomly-generated + // noise. + if (!result.Confirmed() || result.mNoise) { + LOG(("Skipping result from table %s", result.mTableName.get())); + continue; + } + + LOG(("Confirmed result from table %s", result.mTableName.get())); + + if (tables.IndexOf(result.mTableName) == nsTArray::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + + // Some parts of this gethash request generated no hits at all. + // Prefixes must have been removed from the database since our last update. + // Save the prefixes we checked to prevent repeated requests + // until the next update. + nsAutoPtr cacheMisses(new PrefixArray()); + if (cacheMisses) { + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult &result = mResults->ElementAt(i); + if (!result.Confirmed() && !result.mNoise) { + cacheMisses->AppendElement(result.PrefixHash()); + } + } + // Hands ownership of the miss array back to the worker thread. + mDBService->CacheMisses(cacheMisses.forget()); + } + + if (mCacheResults) { + // This hands ownership of the cache results array back to the worker + // thread. + mDBService->CacheCompletions(mCacheResults.forget()); + } + + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + + return mCallback->HandleEvent(tableStr); +} + + +// ------------------------------------------------------------------------- +// Helper class for nsIURIClassifier implementation, translates table names +// to nsIURIClassifier enums. + +class nsUrlClassifierClassifyCallback final : public nsIUrlClassifierCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERCALLBACK + + nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c, + bool checkMalware, + bool checkPhishing, + bool checkTracking) + : mCallback(c) + {} + +private: + ~nsUrlClassifierClassifyCallback() {} + + nsCOMPtr mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback, + nsIUrlClassifierCallback) + +NS_IMETHODIMP +nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables) +{ + nsresult response = TablesToResponse(tables); + mCallback->OnClassifyComplete(response); + return NS_OK; +} + + +// ------------------------------------------------------------------------- +// Proxy class implementation + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBService, + nsIUrlClassifierDBService, + nsIURIClassifier, + nsIObserver) + +/* static */ nsUrlClassifierDBService* +nsUrlClassifierDBService::GetInstance(nsresult *result) +{ + *result = NS_OK; + if (!sUrlClassifierDBService) { + sUrlClassifierDBService = new nsUrlClassifierDBService(); + if (!sUrlClassifierDBService) { + *result = NS_ERROR_OUT_OF_MEMORY; + return nullptr; + } + + NS_ADDREF(sUrlClassifierDBService); // addref the global + + *result = sUrlClassifierDBService->Init(); + if (NS_FAILED(*result)) { + NS_RELEASE(sUrlClassifierDBService); + return nullptr; + } + } else { + // Already exists, just add a ref + NS_ADDREF(sUrlClassifierDBService); // addref the return result + } + return sUrlClassifierDBService; +} + + +nsUrlClassifierDBService::nsUrlClassifierDBService() + : mCheckMalware(CHECK_MALWARE_DEFAULT) + , mCheckPhishing(CHECK_PHISHING_DEFAULT) + , mCheckTracking(CHECK_TRACKING_DEFAULT) + , mInUpdate(false) +{ +} + +nsUrlClassifierDBService::~nsUrlClassifierDBService() +{ + sUrlClassifierDBService = nullptr; +} + +nsresult +nsUrlClassifierDBService::ReadTablesFromPrefs() +{ + nsCString allTables; + nsCString tables; + Preferences::GetCString(PHISH_TABLE_PREF, &allTables); + + Preferences::GetCString(MALWARE_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_BLOCK_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_ALLOW_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Classifier::SplitTables(allTables, mGethashTables); + + Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, &tables); + Classifier::SplitTables(tables, mDisallowCompletionsTables); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBService::Init() +{ +#if defined(PR_LOGGING) + if (!gUrlClassifierDbServiceLog) + gUrlClassifierDbServiceLog = PR_NewLogModule("UrlClassifierDbService"); +#endif + MOZ_ASSERT(NS_IsMainThread(), "Must initialize DB service on main thread"); + nsCOMPtr appInfo = do_GetService("@mozilla.org/xre/app-info;1"); + if (appInfo) { + bool inSafeMode = false; + appInfo->GetInSafeMode(&inSafeMode); + if (inSafeMode) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + // Retrieve all the preferences. + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + mCheckTracking = Preferences::GetBool(CHECK_TRACKING_PREF, + CHECK_TRACKING_DEFAULT); + uint32_t gethashNoise = Preferences::GetUint(GETHASH_NOISE_PREF, + GETHASH_NOISE_DEFAULT); + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + ReadTablesFromPrefs(); + + // Do we *really* need to be able to change all of these at runtime? + Preferences::AddStrongObserver(this, CHECK_MALWARE_PREF); + Preferences::AddStrongObserver(this, CHECK_PHISHING_PREF); + Preferences::AddStrongObserver(this, CHECK_TRACKING_PREF); + Preferences::AddStrongObserver(this, GETHASH_NOISE_PREF); + Preferences::AddStrongObserver(this, CONFIRM_AGE_PREF); + Preferences::AddStrongObserver(this, PHISH_TABLE_PREF); + Preferences::AddStrongObserver(this, MALWARE_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_WHITELIST_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_BLOCK_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_ALLOW_TABLE_PREF); + Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF); + + // Force PSM loading on main thread + nsresult rv; + nsCOMPtr dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + // Directory providers must also be accessed on the main thread. + nsCOMPtr cacheDir; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + return rv; + } + } + + // Start the background thread. + rv = NS_NewNamedThread("URL Classifier", &gDbBackgroundThread); + if (NS_FAILED(rv)) + return rv; + + mWorker = new nsUrlClassifierDBServiceWorker(); + if (!mWorker) + return NS_ERROR_OUT_OF_MEMORY; + + rv = mWorker->Init(gethashNoise, cacheDir); + if (NS_FAILED(rv)) { + mWorker = nullptr; + return rv; + } + + // Proxy for calling the worker on the background thread + mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker); + rv = mWorkerProxy->OpenDb(); + if (NS_FAILED(rv)) { + return rv; + } + + // Add an observer for shutdown + nsCOMPtr observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + observerService->AddObserver(this, "profile-before-change", false); + observerService->AddObserver(this, "xpcom-shutdown-threads", false); + + return NS_OK; +} + +void +nsUrlClassifierDBService::BuildTables(bool aTrackingProtectionEnabled, + nsCString &tables) +{ + nsAutoCString malware; + // LookupURI takes a comma-separated list already. + Preferences::GetCString(MALWARE_TABLE_PREF, &malware); + if (mCheckMalware && !malware.IsEmpty()) { + tables.Append(malware); + } + nsAutoCString phishing; + Preferences::GetCString(PHISH_TABLE_PREF, &phishing); + if (mCheckPhishing && !phishing.IsEmpty()) { + tables.Append(','); + tables.Append(phishing); + } + if (aTrackingProtectionEnabled) { + nsAutoCString tracking, trackingWhitelist; + Preferences::GetCString(TRACKING_TABLE_PREF, &tracking); + if (!tracking.IsEmpty()) { + tables.Append(','); + tables.Append(tracking); + } + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &trackingWhitelist); + if (!trackingWhitelist.IsEmpty()) { + tables.Append(','); + tables.Append(trackingWhitelist); + } + } + + if (StringBeginsWith(tables, NS_LITERAL_CSTRING(","))) { + tables.Cut(0, 1); + } +} + +// nsChannelClassifier is the only consumer of this interface. +NS_IMETHODIMP +nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal, + bool aTrackingProtectionEnabled, + nsIURIClassifierCallback* c, + bool* result) +{ + NS_ENSURE_ARG(aPrincipal); + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (!(mCheckMalware || mCheckPhishing)) { + *result = false; + return NS_OK; + } + + nsRefPtr callback = + new nsUrlClassifierClassifyCallback(c, mCheckMalware, mCheckPhishing, + mCheckTracking); + if (!callback) return NS_ERROR_OUT_OF_MEMORY; + + nsAutoCString tables; + BuildTables(aTrackingProtectionEnabled, tables); + + nsresult rv = LookupURI(aPrincipal, tables, callback, false, result); + if (rv == NS_ERROR_MALFORMED_URI) { + *result = false; + // The URI had no hostname, don't try to classify it. + return NS_OK; + } + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ClassifyLocalWithTables(nsIURI *aURI, + const nsACString & aTables, + nsACString & aTableResults) +{ + MOZ_ASSERT(NS_IsMainThread(), "ClassifyLocalWithTables must be on main thread"); + + nsCOMPtr uri = NS_GetInnermostURI(aURI); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsresult rv = utilsService->GetKeyForURI(uri, key); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr results(new LookupResultArray()); + if (!results) { + return NS_ERROR_OUT_OF_MEMORY; + } + + // In unittests, we may not have been initalized, so don't crash. + rv = mWorkerProxy->DoLocalLookup(key, aTables, results); + if (NS_SUCCEEDED(rv)) { + aTableResults = ProcessLookupResults(results); + } + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + bool dummy; + return LookupURI(aPrincipal, tables, c, true, &dummy); +} + +nsresult +nsUrlClassifierDBService::LookupURI(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c, + bool forceLookup, + bool *didLookup) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + NS_ENSURE_ARG(aPrincipal); + + if (nsContentUtils::IsSystemPrincipal(aPrincipal)) { + *didLookup = false; + return NS_OK; + } + + nsCOMPtr uri; + nsresult rv = aPrincipal->GetURI(getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + uri = NS_GetInnermostURI(uri); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + rv = utilsService->GetKeyForURI(uri, key); + if (NS_FAILED(rv)) + return rv; + + if (forceLookup) { + *didLookup = true; + } else { + bool clean = false; + + if (!clean) { + nsCOMPtr permissionManager = + services::GetPermissionManager(); + + if (permissionManager) { + uint32_t perm; + rv = permissionManager->TestPermissionFromPrincipal(aPrincipal, + "safe-browsing", &perm); + NS_ENSURE_SUCCESS(rv, rv); + + clean |= (perm == nsIPermissionManager::ALLOW_ACTION); + } + } + + *didLookup = !clean; + if (clean) { + return NS_OK; + } + } + + // Create an nsUrlClassifierLookupCallback object. This object will + // take care of confirming partial hash matches if necessary before + // calling the client's callback. + nsCOMPtr callback = + new nsUrlClassifierLookupCallback(this, c); + if (!callback) + return NS_ERROR_OUT_OF_MEMORY; + + nsCOMPtr proxyCallback = + new UrlClassifierLookupCallbackProxy(callback); + + // Queue this lookup and call the lookup function to flush the queue if + // necessary. + rv = mWorker->QueueLookup(key, tables, proxyCallback); + NS_ENSURE_SUCCESS(rv, rv); + + // This seems to just call HandlePendingLookups. + nsAutoCString dummy; + return mWorkerProxy->Lookup(nullptr, dummy, nullptr); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + // The proxy callback uses the current thread. + nsCOMPtr proxyCallback = + new UrlClassifierCallbackProxy(c); + + return mWorkerProxy->GetTables(proxyCallback); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + if (completer) { + mCompleters.Put(tableName, completer); + } else { + mCompleters.Remove(tableName); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &updateTables) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (mInUpdate) { + LOG(("Already updating, not available")); + return NS_ERROR_NOT_AVAILABLE; + } + + mInUpdate = true; + + // The proxy observer uses the current thread + nsCOMPtr proxyObserver = + new UrlClassifierUpdateObserverProxy(observer); + + return mWorkerProxy->BeginUpdate(proxyObserver, updateTables); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginStream(const nsACString &table) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->BeginStream(table); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->UpdateStream(aUpdateChunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishStream() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->FinishStream(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->FinishUpdate(); +} + + +NS_IMETHODIMP +nsUrlClassifierDBService::CancelUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->CancelUpdate(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ResetDatabase() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ResetDatabase(); +} + +nsresult +nsUrlClassifierDBService::CacheCompletions(CacheResultArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheCompletions(results); +} + +nsresult +nsUrlClassifierDBService::CacheMisses(PrefixArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheMisses(results); +} + +bool +nsUrlClassifierDBService::GetCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter **completer) +{ + // If we have specified a completer, go ahead and query it. This is only + // used by tests. + if (mCompleters.Get(tableName, completer)) { + return true; + } + + // If we don't know about this table at all, or are disallowing completions + // for it, skip completion checks. + if (!mGethashTables.Contains(tableName) || + mDisallowCompletionsTables.Contains(tableName)) { + return false; + } + + MOZ_ASSERT(!StringBeginsWith(tableName, NS_LITERAL_CSTRING("test-")), + "We should never fetch hash completions for test tables"); + + // Otherwise, call gethash to find the hash completions. + return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID, + completer)); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *aData) +{ + if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { + nsresult rv; + nsCOMPtr prefs(do_QueryInterface(aSubject, &rv)); + NS_ENSURE_SUCCESS(rv, rv); + if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) { + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) { + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_TRACKING_PREF).Equals(aData)) { + mCheckTracking = Preferences::GetBool(CHECK_TRACKING_PREF, + CHECK_TRACKING_DEFAULT); + } else if ( + NS_LITERAL_STRING(PHISH_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(MALWARE_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_WHITELIST_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_BLOCK_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_ALLOW_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DISALLOW_COMPLETION_TABLE_PREF).Equals(aData)) { + // Just read everything again. + ReadTablesFromPrefs(); + } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) { + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + } + } else if (!strcmp(aTopic, "profile-before-change") || + !strcmp(aTopic, "xpcom-shutdown-threads")) { + Shutdown(); + } else { + return NS_ERROR_UNEXPECTED; + } + + return NS_OK; +} + +// Join the background thread if it exists. +nsresult +nsUrlClassifierDBService::Shutdown() +{ + LOG(("shutting down db service\n")); + + if (!gDbBackgroundThread) + return NS_OK; + + mCompleters.Clear(); + + nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (prefs) { + prefs->RemoveObserver(CHECK_MALWARE_PREF, this); + prefs->RemoveObserver(CHECK_PHISHING_PREF, this); + prefs->RemoveObserver(CHECK_TRACKING_PREF, this); + prefs->RemoveObserver(PHISH_TABLE_PREF, this); + prefs->RemoveObserver(MALWARE_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_WHITELIST_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_BLOCK_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_ALLOW_TABLE_PREF, this); + prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this); + prefs->RemoveObserver(CONFIRM_AGE_PREF, this); + } + + DebugOnly rv; + // First close the db connection. + if (mWorker) { + rv = mWorkerProxy->CancelUpdate(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event"); + + rv = mWorkerProxy->CloseDb(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event"); + } + + mWorkerProxy = nullptr; + + LOG(("joining background thread")); + + gShuttingDownThread = true; + + nsIThread *backgroundThread = gDbBackgroundThread; + gDbBackgroundThread = nullptr; + backgroundThread->Shutdown(); + NS_RELEASE(backgroundThread); + + return NS_OK; +} + +nsIThread* +nsUrlClassifierDBService::BackgroundThread() +{ + return gDbBackgroundThread; +} + diff --git a/toolkit/components/url-classifier/tests/mochitest/good.js b/toolkit/components/url-classifier/tests/mochitest/good.js new file mode 100644 index 0000000000..015b9fe520 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/good.js @@ -0,0 +1 @@ +scriptItem = "loaded whitelisted javascript!"; diff --git a/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html new file mode 100644 index 0000000000..0d3e376d27 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html @@ -0,0 +1,150 @@ + + + + + Test Tracking Protection in Private Browsing mode + + + + + +

+ +
+
+
+
+
+ + + diff --git a/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html b/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html new file mode 100644 index 0000000000..620416fc74 --- /dev/null +++ b/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html @@ -0,0 +1,15 @@ + + + + + + + + + + + + + +