import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1196021 - Use asyncOpen2 in PackagedAppService r=ckerschb (94301092a0)
- Bug 1196021 - Pass requesting channel to PackagedAppService::GetResource r=ckerschb (bcfa9d9d4f)
- rearrange Bug 1209658 part 2 (e7f9d4e75f)
- Bug 1161625. Use nsIURI, not dom::URL, to create URIs on main thread in fetch code. Also, use the entry settings document's base URI, not the callee document's document URI, as the base. r=nsm (fd824cc408)
- Bug 1110476 - Stripped url fragment from Request::GetUrl() by calling either nsIURI::SetRef() or workers::URL::SetHash() in Request's url getter utility methods. Stripped url fragment from Response::GetUrl() by adding the method InternalRequest::StripFragmentAndSetUrl() which calls nsIURI::SetRef(). Added a test in dom/tests/mochitest/fetch/test_request.js for Request::GetUrl(). Removed manual url stripping from dom/cache/TypeUtils.cpp. r=bkelly (a4f394434a)
- Bug 1195820 - Request constructor should throw TypeError if URL has credentials or parse fails. r=bkelly (347c65872d)
- Bug 1144786: Create ClassifyLocalWithTables (r=gcp,francois) (26c00bc725)
- Bug 1180323 - Only look at TP table before cancelling speculative connections. r=gcp (8c762cd347)
- Bug 1141352 - add a pairwise allowlist to tracking protection. r=gcp (4ec253bff6)
- Bug 1098422 - Change the HTTP cache half-life experiment values. r=jduell (5e39571c88)
-  Bg 1139014 - Optimize memory allocations in CacheFileMetadata, r=honzab (59a8c4d7eb)
- Bug 1177278 - Large OOMs in CacheFileMetadata::WriteMetadata, r=honzab (bfe491ebd1)
- Bug 1171724 - Large OOMs in CacheFileMetadata, r=honzab (58a4d81058)
- Bug 1122070 - TSan: data race netwerk/cache2/CacheIOThread.cpp:97 DispatchInternal, r=honzab (34240071df)
- Bug 1157322 - TSan: data race netwerk/cache2/CacheIOThread.cpp:315 OnDispatchedEvent, r=honzab (66d90f16ae)
- Bug 1156974 - mark CacheFileHandle::mIsDoomed as a release/acquire Atomic variable; r=michal (38eb63304d)
- Bug 1082735 - Don't use InsertElementSorted in HTTP cache, r=michal (06b3ca3f41)
- Bug 1159500 - crash in mozilla::net::CacheIndexIterator::GetNextHash(unsigned char (*)[20]), r=honzab (21a4bf2fdb)
- fix namespace (f642fe8df0)
- Bug 1132172 - Don't access CacheEntry::mFrecency on non-cache threads. r=michal (68e9cf33a6)
- Bug 1139924 - Dooming an unloaded HTTP cache entry by its URL may fail. r=michal (1d259ea5f3)

build/pgo/server-locations.txt

@@ -171,6 +171,8 @@ http://malware.example.com:80
 http://tracking.example.com:80
 http://not-tracking.example.com:80
 http://tracking.example.org:80
+http://itisatracker.org:80
+http://trackertest.org:80
 
 https://malware.example.com:443
 https://tracking.example.com:443

dom/base/nsContentUtils.cpp

@@ -3623,14 +3623,14 @@ nsContentUtils::IsPlainTextType(const nsACString& aContentType)
   return aContentType.EqualsLiteral(TEXT_PLAIN) ||
          aContentType.EqualsLiteral(TEXT_CSS) ||
          aContentType.EqualsLiteral(TEXT_CACHE_MANIFEST) ||
-         aContentType.EqualsLiteral(TEXT_VTT) ||
          aContentType.EqualsLiteral(APPLICATION_JAVASCRIPT) ||
          aContentType.EqualsLiteral(APPLICATION_XJAVASCRIPT) ||
          aContentType.EqualsLiteral(TEXT_ECMASCRIPT) ||
          aContentType.EqualsLiteral(APPLICATION_ECMASCRIPT) ||
          aContentType.EqualsLiteral(TEXT_JAVASCRIPT) ||
          aContentType.EqualsLiteral(APPLICATION_JSON) ||
-         aContentType.EqualsLiteral(TEXT_JSON);
+         aContentType.EqualsLiteral(TEXT_JSON) ||
+         aContentType.EqualsLiteral(TEXT_VTT);
 }
 
 bool

dom/bindings/Errors.msg

@@ -54,6 +54,7 @@ MSG_DEF(MSG_NOT_DATE, 1, JSEXN_TYPEERR, "{0} is not a date.")
 MSG_DEF(MSG_INVALID_ADVANCE_COUNT, 0, JSEXN_TYPEERR, "0 (Zero) is not a valid advance count.")
 MSG_DEF(MSG_DEFINEPROPERTY_ON_GSP, 0, JSEXN_TYPEERR, "Not allowed to define a property on the named properties object.")
 MSG_DEF(MSG_INVALID_URL, 1, JSEXN_TYPEERR, "{0} is not a valid URL.")
+MSG_DEF(MSG_URL_HAS_CREDENTIALS, 1, JSEXN_TYPEERR, "{0} is an url with embedded credentials.")
 MSG_DEF(MSG_METADATA_NOT_CONFIGURED, 0, JSEXN_TYPEERR, "Either size or lastModified should be true.")
 MSG_DEF(MSG_INVALID_READ_SIZE, 0, JSEXN_TYPEERR, "0 (Zero) is not a valid read size.")
 MSG_DEF(MSG_HEADERS_IMMUTABLE, 0, JSEXN_TYPEERR, "Headers are immutable and cannot be modified.")

dom/cache/TypeUtils.cpp

@@ -418,25 +418,15 @@ TypeUtils::ProcessURL(nsACString& aUrl, bool* aSchemeValidOut,
 
   uint32_t queryPos;
   int32_t queryLen;
-  uint32_t refPos;
-  int32_t refLen;
 
   aRv = urlParser->ParsePath(url + pathPos, flatURL.Length() - pathPos,
                              nullptr, nullptr,               // ignore filepath
                              &queryPos, &queryLen,
-                             &refPos, &refLen);
+                             nullptr, nullptr);
   if (NS_WARN_IF(aRv.Failed())) {
     return;
   }
 
-  // TODO: Remove this once Request/Response properly strip the fragment (bug 1110476)
-  if (refLen >= 0) {
-    // ParsePath gives us ref position relative to the start of the path
-    refPos += pathPos;
-
-    aUrl = Substring(aUrl, 0, refPos - 1);
-  }
-
   if (!aUrlWithoutQueryOut) {
     return;
   }

dom/fetch/FetchDriver.cpp

@@ -4,6 +4,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "mozilla/DebugOnly.h"
 #include "mozilla/dom/FetchDriver.h"
 
 #include "nsIDocument.h"
@@ -645,7 +646,8 @@ FetchDriver::BeginAndGetFilteredResponse(InternalResponse* aResponse, nsIURI* aF
   } else {
     mRequest->GetURL(reqURL);
   }
-  aResponse->SetUrl(reqURL);
+  DebugOnly<nsresult> rv = aResponse->StripFragmentAndSetUrl(reqURL);
+  MOZ_ASSERT(NS_SUCCEEDED(rv));
 
   // FIXME(nsm): Handle mixed content check, step 7 of fetch.
 

dom/fetch/InternalRequest.h

@@ -388,6 +388,7 @@ private:
   MapContentPolicyTypeToRequestContext(nsContentPolicyType aContentPolicyType);
 
   nsCString mMethod;
+  // mURL always stores the url with the ref stripped
   nsCString mURL;
   nsRefPtr<InternalHeaders> mHeaders;
   nsCOMPtr<nsIInputStream> mBodyStream;

dom/fetch/InternalResponse.cpp

@@ -6,9 +6,11 @@
 
 #include "InternalResponse.h"
 
+#include "mozilla/Assertions.h"
 #include "mozilla/dom/InternalHeaders.h"
 #include "mozilla/dom/cache/CacheTypes.h"
 #include "mozilla/ipc/PBackgroundSharedTypes.h"
+#include "nsIURI.h"
 #include "nsStreamUtils.h"
 
 namespace mozilla {
@@ -85,6 +87,37 @@ InternalResponse::SetPrincipalInfo(UniquePtr<mozilla::ipc::PrincipalInfo> aPrinc
   mPrincipalInfo = Move(aPrincipalInfo);
 }
 
+nsresult
+InternalResponse::StripFragmentAndSetUrl(const nsACString& aUrl)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  nsCOMPtr<nsIURI> iuri;
+  nsresult rv;
+
+  rv = NS_NewURI(getter_AddRefs(iuri), aUrl);
+  if(NS_WARN_IF(NS_FAILED(rv))){
+    return rv;
+  }
+
+  nsCOMPtr<nsIURI> iuriClone;
+  // We use CloneIgnoringRef to strip away the fragment even if the original URI
+  // is immutable.
+  rv = iuri->CloneIgnoringRef(getter_AddRefs(iuriClone));
+  if(NS_WARN_IF(NS_FAILED(rv))){
+    return rv;
+  }
+
+  nsCString spec;
+  rv = iuriClone->GetSpec(spec);
+  if(NS_WARN_IF(NS_FAILED(rv))){
+    return rv;
+  }
+
+  SetUrl(spec);
+  return NS_OK;
+}
+
 already_AddRefed<InternalResponse>
 InternalResponse::OpaqueResponse()
 {

dom/fetch/InternalResponse.h

@@ -82,6 +82,7 @@ public:
     aURL.Assign(mURL);
   }
 
+  // SetUrl should only be called when the fragment has alredy been stripped
   void
   SetUrl(const nsACString& aURL)
   {
@@ -204,6 +205,9 @@ public:
   void
   SetPrincipalInfo(UniquePtr<mozilla::ipc::PrincipalInfo> aPrincipalInfo);
 
+  nsresult
+  StripFragmentAndSetUrl(const nsACString& aUrl);
+
 private:
   ~InternalResponse();
 

dom/fetch/Request.cpp

@@ -17,6 +17,7 @@
 #include "mozilla/dom/URL.h"
 #include "mozilla/dom/WorkerPrivate.h"
 #include "mozilla/dom/workers/bindings/URL.h"
+#include "mozilla/unused.h"
 
 #include "WorkerPrivate.h"
 
@@ -72,30 +73,44 @@ Request::GetInternalRequest()
 
 namespace {
 void
-GetRequestURLFromWindow(const GlobalObject& aGlobal, nsPIDOMWindow* aWindow,
-                        const nsAString& aInput, nsAString& aRequestURL,
-                        ErrorResult& aRv)
+GetRequestURLFromDocument(nsIDocument* aDocument, const nsAString& aInput,
+                          nsAString& aRequestURL, ErrorResult& aRv)
 {
-  MOZ_ASSERT(aWindow);
+  MOZ_ASSERT(aDocument);
   MOZ_ASSERT(NS_IsMainThread());
 
-  nsCOMPtr<nsIURI> docURI = aWindow->GetDocumentURI();
+  nsCOMPtr<nsIURI> baseURI = aDocument->GetBaseURI();
+  nsCOMPtr<nsIURI> resolvedURI;
+  aRv = NS_NewURI(getter_AddRefs(resolvedURI), aInput, nullptr, baseURI);
+  if (NS_WARN_IF(aRv.Failed())) {
+    aRv.ThrowTypeError(MSG_INVALID_URL, &aInput);
+    return;
+  }
+
+  // This fails with URIs with weird protocols, even when they are valid,
+  // so we ignore the failure
+  nsAutoCString credentials;
+  unused << resolvedURI->GetUserPass(credentials);
+  if (!credentials.IsEmpty()) {
+    aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput);
+    return;
+  }
+
+  nsCOMPtr<nsIURI> resolvedURIClone;
+  // We use CloneIgnoringRef to strip away the fragment even if the original URI
+  // is immutable.
+  aRv = resolvedURI->CloneIgnoringRef(getter_AddRefs(resolvedURIClone));
+  if (NS_WARN_IF(aRv.Failed())) {
+    return;
+  }
+
   nsAutoCString spec;
-  aRv = docURI->GetSpec(spec);
+  aRv = resolvedURIClone->GetSpec(spec);
   if (NS_WARN_IF(aRv.Failed())) {
     return;
   }
 
-  nsRefPtr<dom::URL> url =
-    dom::URL::Constructor(aGlobal, aInput, NS_ConvertUTF8toUTF16(spec), aRv);
-  if (NS_WARN_IF(aRv.Failed())) {
-    return;
-  }
-
-  url->Stringify(aRequestURL, aRv);
-  if (NS_WARN_IF(aRv.Failed())) {
-    return;
-  }
+  CopyUTF8toUTF16(spec, aRequestURL);
 }
 
 void
@@ -104,23 +119,37 @@ GetRequestURLFromChrome(const nsAString& aInput, nsAString& aRequestURL,
 {
   MOZ_ASSERT(NS_IsMainThread());
 
-#ifdef DEBUG
   nsCOMPtr<nsIURI> uri;
   aRv = NS_NewURI(getter_AddRefs(uri), aInput, nullptr, nullptr);
+  if (NS_WARN_IF(aRv.Failed())) {
+    aRv.ThrowTypeError(MSG_INVALID_URL, &aInput);
+    return;
+  }
+
+  // This fails with URIs with weird protocols, even when they are valid,
+  // so we ignore the failure
+  nsAutoCString credentials;
+  unused << uri->GetUserPass(credentials);
+  if (!credentials.IsEmpty()) {
+    aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput);
+    return;
+  }
+
+  nsCOMPtr<nsIURI> uriClone;
+  // We use CloneIgnoringRef to strip away the fragment even if the original URI
+  // is immutable.
+  aRv = uri->CloneIgnoringRef(getter_AddRefs(uriClone));
   if (NS_WARN_IF(aRv.Failed())) {
     return;
   }
 
   nsAutoCString spec;
-  aRv = uri->GetSpec(spec);
+  aRv = uriClone->GetSpec(spec);
   if (NS_WARN_IF(aRv.Failed())) {
     return;
   }
 
   CopyUTF8toUTF16(spec, aRequestURL);
-#else
-  aRequestURL = aInput;
-#endif
 }
 
 void
@@ -134,6 +163,29 @@ GetRequestURLFromWorker(const GlobalObject& aGlobal, const nsAString& aInput,
   NS_ConvertUTF8toUTF16 baseURL(worker->GetLocationInfo().mHref);
   nsRefPtr<workers::URL> url =
     workers::URL::Constructor(aGlobal, aInput, baseURL, aRv);
+  if (NS_WARN_IF(aRv.Failed())) {
+    aRv.ThrowTypeError(MSG_INVALID_URL, &aInput);
+    return;
+  }
+
+  nsString username;
+  url->GetUsername(username, aRv);
+  if (NS_WARN_IF(aRv.Failed())) {
+    return;
+  }
+
+  nsString password;
+  url->GetPassword(password, aRv);
+  if (NS_WARN_IF(aRv.Failed())) {
+    return;
+  }
+
+  if (!username.IsEmpty() || !password.IsEmpty()) {
+    aRv.ThrowTypeError(MSG_URL_HAS_CREDENTIALS, &aInput);
+    return;
+  }
+
+  url->SetHash(EmptyString(), aRv);
   if (NS_WARN_IF(aRv.Failed())) {
     return;
   }
@@ -187,11 +239,11 @@ Request::Constructor(const GlobalObject& aGlobal,
 
     nsAutoString requestURL;
     if (NS_IsMainThread()) {
-      nsCOMPtr<nsPIDOMWindow> window = do_QueryInterface(global);
-      if (window) {
-        GetRequestURLFromWindow(aGlobal, window, input, requestURL, aRv);
+      nsIDocument* doc = GetEntryDocument();
+      if (doc) {
+        GetRequestURLFromDocument(doc, input, requestURL, aRv);
       } else {
-        // If we don't have a window, we must assume that this is a full URL.
+        // If we don't have a document, we must assume that this is a full URL.
         GetRequestURLFromChrome(input, requestURL, aRv);
       }
     } else {

dom/fetch/Response.cpp

@@ -62,21 +62,24 @@ Response::Redirect(const GlobalObject& aGlobal, const nsAString& aUrl,
   nsAutoString parsedURL;
 
   if (NS_IsMainThread()) {
-    nsCOMPtr<nsPIDOMWindow> window = do_QueryInterface(aGlobal.GetAsSupports());
-    nsCOMPtr<nsIURI> docURI = window->GetDocumentURI();
-    nsAutoCString spec;
-    aRv = docURI->GetSpec(spec);
+    nsCOMPtr<nsIURI> baseURI;
+    nsIDocument* doc = GetEntryDocument();
+    if (doc) {
+      baseURI = doc->GetBaseURI();
+    }
+    nsCOMPtr<nsIURI> resolvedURI;
+    aRv = NS_NewURI(getter_AddRefs(resolvedURI), aUrl, nullptr, baseURI);
     if (NS_WARN_IF(aRv.Failed())) {
       return nullptr;
     }
 
-    nsRefPtr<mozilla::dom::URL> url =
-      dom::URL::Constructor(aGlobal, aUrl, NS_ConvertUTF8toUTF16(spec), aRv);
-    if (aRv.Failed()) {
+    nsAutoCString spec;
+    aRv = resolvedURI->GetSpec(spec);
+    if (NS_WARN_IF(aRv.Failed())) {
       return nullptr;
     }
 
-    url->Stringify(parsedURL, aRv);
+    CopyUTF8toUTF16(spec, parsedURL);
   } else {
     workers::WorkerPrivate* worker = workers::GetCurrentThreadWorkerPrivate();
     MOZ_ASSERT(worker);

dom/tests/mochitest/fetch/test_fetch_cors.js

@@ -748,10 +748,13 @@ function testModeCors() {
     if (test.preflightBody)
       req.url += "&preflightBody=" + escape(test.preflightBody);
 
-    var request = new Request(req.url, { method: req.method, mode: "cors",
-                                         headers: req.headers, body: req.body });
-    fetches.push((function(test, request) {
-      return fetch(request).then(function(res) {
+    fetches.push((function(test) {
+      return new Promise(function(resolve) {
+        resolve(new Request(req.url, { method: req.method, mode: "cors",
+                                         headers: req.headers, body: req.body }));
+      }).then(function(request) {
+        return fetch(request);
+      }).then(function(res) {
         ok(test.pass, "Expected test to pass for " + test.toSource());
         if (test.status) {
           is(res.status, test.status, "wrong status in test for " + test.toSource());
@@ -776,21 +779,21 @@ function testModeCors() {
           }
         }
 
-        return res.text().then(function(v) {
-          if (test.method !== "HEAD") {
-            is(v, "<res>hello pass</res>\n",
-               "wrong responseText in test for " + test.toSource());
-          }
-          else {
-            is(v, "",
-               "wrong responseText in HEAD test for " + test.toSource());
-          }
-        });
-      }, function(e) {
-          ok(!test.pass, "Expected test failure for " + test.toSource());
-          ok(e instanceof TypeError, "Exception should be TypeError for " + test.toSource());
+        return res.text();
+      }).then(function(v) {
+        if (test.method !== "HEAD") {
+          is(v, "<res>hello pass</res>\n",
+             "wrong responseText in test for " + test.toSource());
+        }
+        else {
+          is(v, "",
+             "wrong responseText in HEAD test for " + test.toSource());
+        }
+      }).catch(function(e) {
+        ok(!test.pass, "Expected test failure for " + test.toSource());
+        ok(e instanceof TypeError, "Exception should be TypeError for " + test.toSource());
       });
-    })(test, request));
+    })(test));
   }
 
   return Promise.all(fetches);

dom/tests/mochitest/fetch/test_request.js

@@ -216,7 +216,32 @@ function testMethod() {
 
 function testUrlFragment() {
   var req = new Request("./request#withfragment");
-  ok(req.url, (new URL("./request", self.location.href)).href, "request.url should be serialized with exclude fragment flag set");
+  is(req.url, (new URL("./request", self.location.href)).href, "request.url should be serialized with exclude fragment flag set");
+}
+
+function testUrlMalformed() {
+  try {
+    var req = new Request("http:// example.com");
+    ok(false, "Creating a Request with a malformed URL should throw a TypeError");
+  } catch(e) {
+    is(e.name, "TypeError", "Creating a Request with a malformed URL should throw a TypeError");
+  }
+}
+
+function testUrlCredentials() {
+  try {
+    var req = new Request("http://user@example.com");
+    ok(false, "URLs with credentials should be rejected");
+  } catch(e) {
+    is(e.name, "TypeError", "URLs with credentials should be rejected");
+  }
+
+  try {
+    var req = new Request("http://user:password@example.com");
+    ok(false, "URLs with credentials should be rejected");
+  } catch(e) {
+    is(e.name, "TypeError", "URLs with credentials should be rejected");
+  }
 }
 
 function testBodyUsed() {
@@ -471,6 +496,8 @@ function runTest() {
   testDefaultCtor();
   testSimpleUrlParse();
   testUrlFragment();
+  testUrlCredentials();
+  testUrlMalformed();
   testMethod();
   testBug1109574();
   testHeaderGuard();

modules/libpref/init/all.js

@@ -3460,7 +3460,7 @@ pref("intl.imm.vertical_writing.always_assume_not_supported", false);
 // We cannot retrieve active IME name with IMM32 API if a TIP of TSF is active.
 // This pref can specify active IME name when Japanese TIP is active.
 // For example:
-//   Google Japanese Input: "Google ?Ąćś¬čŞžĺ…Ą??IMM32 ?˘ă‚¸?Ąă�Ľ??
+//   Google Japanese Input: "Google ?�本語入??IMM32 ?�ジ?�ー??
 //   ATOK 2011: "ATOK 2011" (similarly, e.g., ATOK 2013 is "ATOK 2013")
 pref("intl.imm.japanese.assume_active_tip_name_as", "");
 
@@ -4952,11 +4952,12 @@ pref("urlclassifier.malwareTable", "goog-malware-shavar,test-malware-simple");
 pref("urlclassifier.phishTable", "goog-phish-shavar,test-phish-simple");
 pref("urlclassifier.downloadBlockTable", "");
 pref("urlclassifier.downloadAllowTable", "");
-pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,goog-downloadwhite-digest256,mozpub-track-digest256");
+pref("urlclassifier.disallow_completions", "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple,goog-downloadwhite-digest256,mozstd-track-digest256,mozstd-trackwhite-digest256,mozfull-track-digest256");
 
 // The table and update/gethash URLs for Safebrowsing phishing and malware
 // checks.
-pref("urlclassifier.trackingTable", "mozpub-track-digest256");
+pref("urlclassifier.trackingTable", "test-track-simple,mozpub-track-digest256");
+pref("urlclassifier.trackingWhitelistTable", "test-trackwhite-simple,mozpub-trackwhite-digest256");
 pref("browser.trackingprotection.updateURL", "https://tracking.services.mozilla.com/downloads?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2");
 pref("browser.trackingprotection.gethashURL", "https://tracking.services.mozilla.com/gethash?client=SAFEBROWSING_ID&appver=%VERSION%&pver=2.2");
 

netwerk/base/nsChannelClassifier.cpp

@@ -472,12 +472,83 @@ nsChannelClassifier::SetBlockedTrackingContent(nsIChannel *channel)
   return NS_OK;
 }
 
+nsresult
+nsChannelClassifier::IsTrackerWhitelisted()
+{
+  nsresult rv;
+  nsCOMPtr<nsIURIClassifier> uriClassifier =
+    do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoCString tables;
+  Preferences::GetCString("urlclassifier.trackingWhitelistTable", &tables);
+
+  if (tables.IsEmpty()) {
+    LOG(("nsChannelClassifier[%p]:IsTrackerWhitelisted whitelist disabled",
+         this));
+    return NS_ERROR_TRACKING_URI;
+  }
+
+  nsCOMPtr<nsIHttpChannelInternal> chan = do_QueryInterface(mChannel, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr<nsIURI> topWinURI;
+  rv = chan->GetTopWindowURI(getter_AddRefs(topWinURI));
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (!topWinURI) {
+    LOG(("nsChannelClassifier[%p]: No window URI", this));
+    return NS_ERROR_TRACKING_URI;
+  }
+
+  nsCOMPtr<nsIScriptSecurityManager> securityManager =
+    do_GetService(NS_SCRIPTSECURITYMANAGER_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsCOMPtr<nsIPrincipal> chanPrincipal;
+  rv = securityManager->GetChannelURIPrincipal(mChannel,
+                                               getter_AddRefs(chanPrincipal));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Craft a whitelist URL like "toplevel.page/?resource=third.party.domain"
+  nsAutoCString pageHostname, resourceDomain;
+  rv = topWinURI->GetHost(pageHostname);
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = chanPrincipal->GetBaseDomain(resourceDomain);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsAutoCString whitelistEntry = NS_LITERAL_CSTRING("http://") +
+    pageHostname + NS_LITERAL_CSTRING("/?resource=") + resourceDomain;
+  LOG(("nsChannelClassifier[%p]: Looking for %s in the whitelist",
+       this, whitelistEntry.get()));
+
+  nsCOMPtr<nsIURI> whitelistURI;
+  rv = NS_NewURI(getter_AddRefs(whitelistURI), whitelistEntry);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Check whether or not the tracker is in the entity whitelist
+  nsAutoCString results;
+  rv = uriClassifier->ClassifyLocalWithTables(whitelistURI, tables, results);
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (!results.IsEmpty()) {
+    return NS_OK; // found it on the whitelist, must not be blocked
+  }
+
+  LOG(("nsChannelClassifier[%p]: %s is not in the whitelist",
+       this, whitelistEntry.get()));
+  return NS_ERROR_TRACKING_URI;
+}
+
 NS_IMETHODIMP
 nsChannelClassifier::OnClassifyComplete(nsresult aErrorCode)
 {
     // Should only be called in the parent process.
     MOZ_ASSERT(XRE_IsParentProcess());
 
+    if (aErrorCode == NS_ERROR_TRACKING_URI &&
+        NS_SUCCEEDED(IsTrackerWhitelisted())) {
+      LOG(("nsChannelClassifier[%p]:OnClassifyComplete tracker found "
+           "in whitelist so we won't block it)", this));
+      aErrorCode = NS_OK;
+    }
+
     LOG(("nsChannelClassifier[%p]:OnClassifyComplete %d", this, aErrorCode));
     if (mSuspendedChannel) {
         MarkEntryClassified(aErrorCode);

netwerk/base/nsChannelClassifier.h

@@ -41,6 +41,8 @@ private:
     // Start is called. Returns NS_OK if and only if we will get a callback
     // from the classifier service.
     nsresult StartInternal();
+    // Helper function to check a tracking URI against the whitelist
+    nsresult IsTrackerWhitelisted();
 
 public:
     // If we are blocking tracking content, update the corresponding flag in

netwerk/base/nsIPackagedAppService.idl

@@ -5,8 +5,7 @@
 
 #include "nsISupports.idl"
 
-interface nsIPrincipal;
-interface nsILoadContextInfo;
+interface nsIChannel;
 interface nsICacheEntryOpenCallback;
 
 %{C++
@@ -16,16 +15,16 @@ interface nsICacheEntryOpenCallback;
 /**
  * nsIPackagedAppService
  */
-[scriptable, builtinclass, uuid(f35e5229-d08a-46eb-a574-2db4e22aee98)]
+[scriptable, builtinclass, uuid(9c96c638-e80c-4dce-abec-c96fdb7a25d8)]
 interface nsIPackagedAppService : nsISupports
 {
   /**
-   * @param aPrincipal
-   *    the principal associated to the URL of a packaged resource
-   *    URL format:  package_url + PACKAGED_APP_TOKEN + resource_path
-   *    example: http://test.com/path/to/package!//resource.html
-   * @param aFlags
-   *    the load flags used for downloading the package
+   * @param aChannel
+   *    this param is passed to the packaged app service in order to provide
+   *    info about the requesting channel, which wants to access the contents
+   *    of a packaged app resource. Its URI has the following format:
+   *          http://domain.com/path/to/package.pak!//path/to/subresource.html
+   *
    * @param aCallback
    *    an object implementing nsICacheEntryOpenCallback
    *    this is the target of the async result of the operation
@@ -34,17 +33,12 @@ interface nsIPackagedAppService : nsISupports
    *    the cached entry, if one exists, or an error code otherwise
    *    aCallback is kept alive using an nsCOMPtr until OnCacheEntryAvailable
    *    is called
-   * @param aInfo
-   *    an object used to determine the cache jar this resource goes in.
-   *    usually created by calling GetLoadContextInfo(requestingChannel)
    *
    * Calling this method will either download the package containing the given
    * resource URI, store it in the cache and pass the cache entry to aCallback,
    * or if that resource has already been downloaded it will be served from
    * the cache.
    */
-  void getResource(in nsIPrincipal aPrincipal,
-                   in uint32_t aFlags,
-                   in nsILoadContextInfo aInfo,
+  void getResource(in nsIChannel aChannel,
                    in nsICacheEntryOpenCallback aCallback);
 };

netwerk/base/nsIURIClassifier.idl

@@ -4,8 +4,9 @@
 
 #include "nsISupports.idl"
 
-interface nsIPrincipal;
 interface nsIChannel;
+interface nsIPrincipal;
+interface nsIURI;
 
 /**
  * Callback function for nsIURIClassifier lookups.
@@ -30,7 +31,7 @@ interface nsIURIClassifierCallback : nsISupports
  * The URI classifier service checks a URI against lists of phishing
  * and malware sites.
  */
-[scriptable, uuid(03d26681-0ef5-4718-9777-33c9e1ee3e32)]
+[scriptable, uuid(596620cc-76e3-4133-9d90-360e59a794cf)]
 interface nsIURIClassifier : nsISupports
 {
   /**
@@ -56,10 +57,9 @@ interface nsIURIClassifier : nsISupports
                    in nsIURIClassifierCallback aCallback);
 
   /**
-   * Synchronously classify a Principal locally using its URI. This does not
-   * make network requests. The result is an error code with which the channel
-   * should be cancelled, or NS_OK if no result was found.
+   * Synchronously classify a URI with a comma-separated string
+   * containing the given tables. This does not make network requests.
+   * The result is a comma-separated string of tables that match.
    */
-  nsresult classifyLocal(in nsIPrincipal aPrincipal,
-                         in boolean aTrackingProtectionEnabled);
+  ACString classifyLocalWithTables(in nsIURI aURI, in ACString aTables);
 };

netwerk/cache2/CacheEntry.cpp

@@ -248,7 +248,7 @@ nsresult CacheEntry::HashingKey(nsCSubstring const& aStorageID,
    * Changing it will cause we will not be able to find files on disk.
    */
 
-  aResult.Append(aStorageID);
+  aResult.Assign(aStorageID);
 
   if (!aEnhanceID.IsEmpty()) {
     CacheFileUtils::AppendTagWithValue(aResult, '~', aEnhanceID);
@@ -1574,7 +1574,7 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync)
       // Because CacheFile::Set*() are not thread-safe to use (uses WeakReference that
       // is not thread-safe) we must post to the main thread...
       nsRefPtr<nsRunnableMethod<CacheEntry> > event =
-        NS_NewRunnableMethod(this, &CacheEntry::StoreFrecency);
+        NS_NewRunnableMethodWithArg<double>(this, &CacheEntry::StoreFrecency, mFrecency);
       NS_DispatchToMainThread(event);
     }
 
@@ -1598,14 +1598,12 @@ void CacheEntry::BackgroundOp(uint32_t aOperations, bool aForceAsync)
   }
 }
 
-void CacheEntry::StoreFrecency()
+void CacheEntry::StoreFrecency(double aFrecency)
 {
-  // No need for thread safety over mFrecency, it will be rewriten
-  // correctly on following invocation if broken by concurrency.
   MOZ_ASSERT(NS_IsMainThread());
 
   if (NS_SUCCEEDED(mFileStatus)) {
-    mFile->SetFrecency(FRECENCY2INT(mFrecency));
+    mFile->SetFrecency(FRECENCY2INT(aFrecency));
   }
 }
 

netwerk/cache2/CacheEntry.h

@@ -246,7 +246,7 @@ private:
   // When executed on the management thread directly, the operation(s)
   // is (are) executed immediately.
   void BackgroundOp(uint32_t aOperation, bool aForceAsync = false);
-  void StoreFrecency();
+  void StoreFrecency(double aFrecency);
 
   // Called only from DoomAlreadyRemoved()
   void DoomFile();

netwerk/cache2/CacheFileIOManager.cpp

@@ -109,7 +109,6 @@ NS_INTERFACE_MAP_END_THREADSAFE
 
 CacheFileHandle::CacheFileHandle(const SHA1Sum::Hash *aHash, bool aPriority)
   : mHash(aHash)
-  , mIsDoomed(false)
   , mPriority(aPriority)
   , mClosed(false)
   , mSpecialFile(false)
@@ -118,13 +117,17 @@ CacheFileHandle::CacheFileHandle(const SHA1Sum::Hash *aHash, bool aPriority)
   , mFileSize(-1)
   , mFD(nullptr)
 {
+  // If we initialize mDoomed in the initialization list, that initialization is
+  // not guaranteeded to be atomic.  Whereas this assignment here is guaranteed
+  // to be atomic.  TSan will see this (atomic) assignment and be satisfied
+  // that cross-thread accesses to mIsDoomed are properly synchronized.
+  mIsDoomed = false;
   LOG(("CacheFileHandle::CacheFileHandle() [this=%p, hash=%08x%08x%08x%08x%08x]"
        , this, LOGSHA1(aHash)));
 }
 
 CacheFileHandle::CacheFileHandle(const nsACString &aKey, bool aPriority)
   : mHash(nullptr)
-  , mIsDoomed(false)
   , mPriority(aPriority)
   , mClosed(false)
   , mSpecialFile(true)
@@ -134,6 +137,8 @@ CacheFileHandle::CacheFileHandle(const nsACString &aKey, bool aPriority)
   , mFD(nullptr)
   , mKey(aKey)
 {
+  // See comment above about the initialization of mIsDoomed.
+  mIsDoomed = false;
   LOG(("CacheFileHandle::CacheFileHandle() [this=%p, key=%s]", this,
        PromiseFlatCString(aKey).get()));
 }
@@ -161,13 +166,13 @@ CacheFileHandle::Log()
   if (mSpecialFile) {
     LOG(("CacheFileHandle::Log() - special file [this=%p, isDoomed=%d, "
          "priority=%d, closed=%d, invalid=%d, fileExists=%d, fileSize=%lld, "
-         "leafName=%s, key=%s]", this, mIsDoomed, mPriority, mClosed, mInvalid,
+         "leafName=%s, key=%s]", this, int(mIsDoomed), mPriority, mClosed, mInvalid,
          mFileExists, mFileSize, leafName.get(), mKey.get()));
   } else {
     LOG(("CacheFileHandle::Log() - entry file [this=%p, hash=%08x%08x%08x%08x"
          "%08x, isDoomed=%d, priority=%d, closed=%d, invalid=%d, fileExists=%d,"
          " fileSize=%lld, leafName=%s, key=%s]", this, LOGSHA1(mHash),
-         mIsDoomed, mPriority, mClosed, mInvalid, mFileExists, mFileSize,
+         int(mIsDoomed), mPriority, mClosed, mInvalid, mFileExists, mFileSize,
          leafName.get(), mKey.get()));
   }
 }

netwerk/cache2/CacheFileIOManager.h

@@ -10,6 +10,7 @@
 #include "nsIEventTarget.h"
 #include "nsITimer.h"
 #include "nsCOMPtr.h"
+#include "mozilla/Atomics.h"
 #include "mozilla/SHA1.h"
 #include "mozilla/TimeStamp.h"
 #include "nsTArray.h"
@@ -69,7 +70,7 @@ private:
   virtual ~CacheFileHandle();
 
   const SHA1Sum::Hash *mHash;
-  bool                 mIsDoomed;
+  mozilla::Atomic<bool,ReleaseAcquire> mIsDoomed;
   bool                 mPriority;
   bool                 mClosed;
   bool                 mSpecialFile;

netwerk/cache2/CacheFileMetadata.cpp

@@ -24,6 +24,16 @@ namespace net {
 #define kMinMetadataRead 1024  // TODO find optimal value from telemetry
 #define kAlignSize       4096
 
+// Most of the cache entries fit into one chunk due to current chunk size. Make
+// sure to tweak this value if kChunkSize is going to change.
+#define kInitialHashArraySize 1
+
+// Initial elements buffer size.
+#define kInitialBufSize 64
+
+// Max size of elements in bytes.
+#define kMaxElementsSize 64*1024
+
 #define kCacheEntryVersion 1
 
 #define NOW_SECONDS() (uint32_t(PR_Now() / PR_USEC_PER_SEC))
@@ -44,6 +54,7 @@ CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString
   , mIsDirty(false)
   , mAnonymous(false)
   , mInBrowser(false)
+  , mAllocExactSize(false)
   , mAppId(nsILoadContextInfo::NO_APP_ID)
 {
   LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, handle=%p, key=%s]",
@@ -74,6 +85,7 @@ CacheFileMetadata::CacheFileMetadata(bool aMemoryOnly, const nsACString &aKey)
   , mIsDirty(true)
   , mAnonymous(false)
   , mInBrowser(false)
+  , mAllocExactSize(false)
   , mAppId(nsILoadContextInfo::NO_APP_ID)
 {
   LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p, key=%s]",
@@ -105,6 +117,7 @@ CacheFileMetadata::CacheFileMetadata()
   , mIsDirty(false)
   , mAnonymous(false)
   , mInBrowser(false)
+  , mAllocExactSize(false)
   , mAppId(nsILoadContextInfo::NO_APP_ID)
 {
   LOG(("CacheFileMetadata::CacheFileMetadata() [this=%p]", this));
@@ -221,6 +234,17 @@ CacheFileMetadata::ReadMetadata(CacheFileMetadataListener *aListener)
   return NS_OK;
 }
 
+uint32_t
+CacheFileMetadata::CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount)
+{
+  return sizeof(uint32_t) +                         // hash of the metadata
+         aHashCount * sizeof(CacheHash::Hash16_t) + // array of chunk hashes
+         sizeof(CacheFileMetadataHeader) +          // metadata header
+         mKey.Length() + 1 +                        // key with trailing null
+         aElementsSize +                            // elements
+         sizeof(uint32_t);                          // offset
+}
+
 nsresult
 CacheFileMetadata::WriteMetadata(uint32_t aOffset,
                                  CacheFileMetadataListener *aListener)
@@ -235,10 +259,11 @@ CacheFileMetadata::WriteMetadata(uint32_t aOffset,
 
   mIsDirty = false;
 
-  mWriteBuf = static_cast<char *>(moz_xmalloc(sizeof(uint32_t) +
-                mHashCount * sizeof(CacheHash::Hash16_t) +
-                sizeof(CacheFileMetadataHeader) + mKey.Length() + 1 +
-                mElementsSize + sizeof(uint32_t)));
+  mWriteBuf = static_cast<char *>(malloc(CalcMetadataSize(mElementsSize,
+                                                          mHashCount)));
+  if (!mWriteBuf) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
 
   char *p = mWriteBuf + sizeof(uint32_t);
   memcpy(p, mHashArray, mHashCount * sizeof(CacheHash::Hash16_t));
@@ -332,8 +357,11 @@ CacheFileMetadata::SyncReadMetadata(nsIFile *aFile)
     return NS_ERROR_FAILURE;
   }
 
+  mBuf = static_cast<char *>(malloc(fileSize - metaOffset));
+  if (!mBuf) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
   mBufSize = fileSize - metaOffset;
-  mBuf = static_cast<char *>(moz_xmalloc(mBufSize));
 
   DoMemoryReport(MemoryUsage());
 
@@ -388,6 +416,8 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue)
 
   MarkDirty();
 
+  nsresult rv;
+
   const uint32_t keySize = strlen(aKey) + 1;
   char *pos = const_cast<char *>(GetElement(aKey));
 
@@ -413,7 +443,10 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue)
 
     // Update the value in place
     newSize -= oldValueSize;
-    EnsureBuffer(newSize);
+    rv = EnsureBuffer(newSize);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
 
     // Move the remainder to the right place
     pos = mBuf + offset;
@@ -421,7 +454,10 @@ CacheFileMetadata::SetElement(const char *aKey, const char *aValue)
   } else {
     // allocate new meta data element
     newSize += keySize;
-    EnsureBuffer(newSize);
+    rv = EnsureBuffer(newSize);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
 
     // Add after last element
     pos = mBuf + mElementsSize;
@@ -480,10 +516,11 @@ CacheFileMetadata::SetHash(uint32_t aIndex, CacheHash::Hash16_t aHash)
   } else if (aIndex == mHashCount) {
     if ((aIndex + 1) * sizeof(CacheHash::Hash16_t) > mHashArraySize) {
       // reallocate hash array buffer
-      if (mHashArraySize == 0)
-        mHashArraySize = 32 * sizeof(CacheHash::Hash16_t);
-      else
+      if (mHashArraySize == 0) {
+        mHashArraySize = kInitialHashArraySize * sizeof(CacheHash::Hash16_t);
+      } else {
         mHashArraySize *= 2;
+      }
       mHashArray = static_cast<CacheHash::Hash16_t *>(
                      moz_xrealloc(mHashArray, mHashArraySize));
     }
@@ -613,7 +650,7 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
 
   MOZ_ASSERT(mListener);
 
-  nsresult rv, retval;
+  nsresult rv;
   nsCOMPtr<CacheFileMetadataListener> listener;
 
   if (NS_FAILED(aResult)) {
@@ -621,10 +658,9 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
          ", creating empty metadata. [this=%p, rv=0x%08x]", this, aResult));
 
     InitEmptyMetadata();
-    retval = NS_OK;
 
     mListener.swap(listener);
-    listener->OnMetadataRead(retval);
+    listener->OnMetadataRead(NS_OK);
     return NS_OK;
   }
 
@@ -637,14 +673,28 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
 
   if (realOffset >= size) {
     LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, creating "
-         "empty metadata. [this=%p, realOffset=%d, size=%lld]", this,
+         "empty metadata. [this=%p, realOffset=%u, size=%lld]", this,
          realOffset, size));
 
     InitEmptyMetadata();
-    retval = NS_OK;
 
     mListener.swap(listener);
-    listener->OnMetadataRead(retval);
+    listener->OnMetadataRead(NS_OK);
+    return NS_OK;
+  }
+
+  uint32_t maxHashCount = size / kChunkSize;
+  uint32_t maxMetadataSize = CalcMetadataSize(kMaxElementsSize, maxHashCount);
+  if (size - realOffset > maxMetadataSize) {
+    LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, metadata would "
+         "be too big, creating empty metadata. [this=%p, realOffset=%u, "
+         "maxMetadataSize=%u, size=%lld]", this, realOffset, maxMetadataSize,
+         size));
+
+    InitEmptyMetadata();
+
+    mListener.swap(listener);
+    listener->OnMetadataRead(NS_OK);
     return NS_OK;
   }
 
@@ -653,7 +703,20 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
   if (realOffset < usedOffset) {
     uint32_t missing = usedOffset - realOffset;
     // we need to read more data
-    mBuf = static_cast<char *>(moz_xrealloc(mBuf, mBufSize + missing));
+    char *newBuf = static_cast<char *>(realloc(mBuf, mBufSize + missing));
+    if (!newBuf) {
+      LOG(("CacheFileMetadata::OnDataRead() - Error allocating %d more bytes "
+           "for the missing part of the metadata, creating empty metadata. "
+           "[this=%p]", missing, this));
+
+      InitEmptyMetadata();
+
+      mListener.swap(listener);
+      listener->OnMetadataRead(NS_OK);
+      return NS_OK;
+    }
+
+    mBuf = newBuf;
     memmove(mBuf + missing, mBuf, mBufSize);
     mBufSize += missing;
 
@@ -669,10 +732,9 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
            "rv=0x%08x]", this, rv));
 
       InitEmptyMetadata();
-      retval = NS_OK;
 
       mListener.swap(listener);
-      listener->OnMetadataRead(retval);
+      listener->OnMetadataRead(NS_OK);
       return NS_OK;
     }
 
@@ -686,14 +748,19 @@ CacheFileMetadata::OnDataRead(CacheFileHandle *aHandle, char *aBuf,
     LOG(("CacheFileMetadata::OnDataRead() - Error parsing metadata, creating "
          "empty metadata. [this=%p]", this));
     InitEmptyMetadata();
-    retval = NS_OK;
-  }
-  else {
-    retval = NS_OK;
+  } else {
+    // Shrink elements buffer.
+    mBuf = static_cast<char *>(moz_xrealloc(mBuf, mElementsSize));
+    mBufSize = mElementsSize;
+
+    // There is usually no or just one call to SetMetadataElement() when the
+    // metadata is parsed from disk. Avoid allocating power of two sized buffer
+    // which we do in case of newly created metadata.
+    mAllocExactSize = true;
   }
 
   mListener.swap(listener);
-  listener->OnMetadataRead(retval);
+  listener->OnMetadataRead(NS_OK);
 
   return NS_OK;
 }
@@ -844,15 +911,12 @@ CacheFileMetadata::ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset,
     memcpy(mHashArray, mBuf + hashesOffset, mHashArraySize);
   }
 
-
   MarkDirty();
 
   mElementsSize = metaposOffset - elementsOffset;
   memmove(mBuf, mBuf + elementsOffset, mElementsSize);
   mOffset = aMetaOffset;
 
-  // TODO: shrink memory if buffer is too big
-
   DoMemoryReport(MemoryUsage());
 
   return NS_OK;
@@ -886,15 +950,44 @@ CacheFileMetadata::CheckElements(const char *aBuf, uint32_t aSize)
   return NS_OK;
 }
 
-void
+nsresult
 CacheFileMetadata::EnsureBuffer(uint32_t aSize)
 {
-  if (mBufSize < aSize) {
-    mBufSize = aSize;
-    mBuf = static_cast<char *>(moz_xrealloc(mBuf, mBufSize));
+  if (aSize > kMaxElementsSize) {
+    return NS_ERROR_FAILURE;
   }
 
-  DoMemoryReport(MemoryUsage());
+  if (mBufSize < aSize) {
+    if (mAllocExactSize) {
+      // If this is not the only allocation, use power of two for following
+      // allocations.
+      mAllocExactSize = false;
+    } else {
+      // find smallest power of 2 greater than or equal to aSize
+      --aSize;
+      aSize |= aSize >> 1;
+      aSize |= aSize >> 2;
+      aSize |= aSize >> 4;
+      aSize |= aSize >> 8;
+      aSize |= aSize >> 16;
+      ++aSize;
+    }
+
+    if (aSize < kInitialBufSize) {
+      aSize = kInitialBufSize;
+    }
+
+    char *newBuf = static_cast<char *>(realloc(mBuf, aSize));
+    if (!newBuf) {
+      return NS_ERROR_OUT_OF_MEMORY;
+    }
+    mBufSize = aSize;
+    mBuf = newBuf;
+
+    DoMemoryReport(MemoryUsage());
+  }
+
+  return NS_OK;
 }
 
 nsresult

netwerk/cache2/CacheFileMetadata.h

@@ -121,6 +121,7 @@ public:
   nsresult GetKey(nsACString &_retval);
 
   nsresult ReadMetadata(CacheFileMetadataListener *aListener);
+  uint32_t CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount);
   nsresult WriteMetadata(uint32_t aOffset,
                          CacheFileMetadataListener *aListener);
   nsresult SyncReadMetadata(nsIFile *aFile);
@@ -171,7 +172,7 @@ private:
   void     InitEmptyMetadata();
   nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, bool aHaveKey);
   nsresult CheckElements(const char *aBuf, uint32_t aSize);
-  void     EnsureBuffer(uint32_t aSize);
+  nsresult EnsureBuffer(uint32_t aSize);
   nsresult ParseKey(const nsACString &aKey);
 
   nsRefPtr<CacheFileHandle>           mHandle;
@@ -186,9 +187,10 @@ private:
   char                               *mWriteBuf;
   CacheFileMetadataHeader             mMetaHdr;
   uint32_t                            mElementsSize;
-  bool                                mIsDirty;
-  bool                                mAnonymous;
-  bool                                mInBrowser;
+  bool                                mIsDirty        : 1;
+  bool                                mAnonymous      : 1;
+  bool                                mInBrowser      : 1;
+  bool                                mAllocExactSize : 1;
   uint32_t                            mAppId;
   nsCOMPtr<CacheFileMetadataListener> mListener;
 };

netwerk/cache2/CacheFileOutputStream.cpp

@@ -103,6 +103,18 @@ CacheFileOutputStream::Write(const char * aBuf, uint32_t aCount,
     return NS_ERROR_FILE_TOO_BIG;
   }
 
+  // We use 64-bit offset when accessing the file, unfortunatelly we use 32-bit
+  // metadata offset, so we cannot handle data bigger than 4GB.
+  if (mPos + aCount > PR_UINT32_MAX) {
+    LOG(("CacheFileOutputStream::Write() - Entry's size exceeds 4GB while it "
+         "isn't too big according to CacheObserver::EntryIsTooBig(). Failing "
+         "and dooming the entry. [this=%p]", this));
+
+    mFile->DoomLocked(nullptr);
+    CloseWithStatusLocked(NS_ERROR_FILE_TOO_BIG);
+    return NS_ERROR_FILE_TOO_BIG;
+  }
+
   *_retval = aCount;
 
   while (aCount) {

netwerk/cache2/CacheIOThread.h

@@ -12,6 +12,7 @@
 #include "nsAutoPtr.h"
 #include "mozilla/Monitor.h"
 #include "mozilla/DebugOnly.h"
+#include "mozilla/Atomics.h"
 
 class nsIRunnable;
 
@@ -88,11 +89,11 @@ private:
   mozilla::Monitor mMonitor;
   PRThread* mThread;
   nsCOMPtr<nsIThread> mXPCOMThread;
-  uint32_t mLowestLevelWaiting;
+  Atomic<uint32_t, Relaxed> mLowestLevelWaiting;
   uint32_t mCurrentlyExecutingLevel;
   nsTArray<nsCOMPtr<nsIRunnable> > mEventQueue[LAST_LEVEL];
 
-  bool mHasXPCOMEvents;
+  Atomic<bool, Relaxed> mHasXPCOMEvents;
   bool mRerunCurrentEvent;
   bool mShutdown;
   DebugOnly<bool> mInsideLoop;

netwerk/cache2/CacheIndex.cpp

@@ -37,6 +37,28 @@
 namespace mozilla {
 namespace net {
 
+namespace {
+
+class FrecencyComparator
+{
+public:
+  bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
+    return a->mFrecency == b->mFrecency;
+  }
+  bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
+    // Place entries with frecency 0 at the end of the array.
+    if (a->mFrecency == 0) {
+      return false;
+    }
+    if (b->mFrecency == 0) {
+      return true;
+    }
+    return a->mFrecency < b->mFrecency;
+  }
+};
+
+} // namespace
+
 /**
  * This helper class is responsible for keeping CacheIndex::mIndexStats and
  * CacheIndex::mFrecencyArray up to date.
@@ -47,7 +69,6 @@ public:
   CacheIndexEntryAutoManage(const SHA1Sum::Hash *aHash, CacheIndex *aIndex)
     : mIndex(aIndex)
     , mOldRecord(nullptr)
-    , mOldFrecency(0)
     , mDoNotSearchInIndex(false)
     , mDoNotSearchInUpdates(false)
   {
@@ -58,7 +79,6 @@ public:
     mIndex->mIndexStats.BeforeChange(entry);
     if (entry && entry->IsInitialized() && !entry->IsRemoved()) {
       mOldRecord = entry->mRec;
-      mOldFrecency = entry->mRec->mFrecency;
     }
   }
 
@@ -79,22 +99,9 @@ public:
       mIndex->RemoveRecordFromFrecencyArray(mOldRecord);
       mIndex->RemoveRecordFromIterators(mOldRecord);
     } else if (entry && mOldRecord) {
-      bool replaceFrecency = false;
-
       if (entry->mRec != mOldRecord) {
         // record has a different address, we have to replace it
-        replaceFrecency = true;
         mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec);
-      } else if (entry->mRec->mFrecency == 0 &&
-                 entry->mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME) {
-        // This is a special case when we want to make sure that the entry is
-        // placed at the end of the lists even when the values didn't change.
-        replaceFrecency = true;
-      } else if (entry->mRec->mFrecency != mOldFrecency) {
-        replaceFrecency = true;
-      }
-
-      if (replaceFrecency) {
         mIndex->RemoveRecordFromFrecencyArray(mOldRecord);
         mIndex->InsertRecordToFrecencyArray(entry->mRec);
       }
@@ -141,7 +148,6 @@ private:
   const SHA1Sum::Hash *mHash;
   nsRefPtr<CacheIndex> mIndex;
   CacheIndexRecord    *mOldRecord;
-  uint32_t             mOldFrecency;
   bool                 mDoNotSearchInIndex;
   bool                 mDoNotSearchInUpdates;
 };
@@ -1079,6 +1085,17 @@ CacheIndex::RemoveAll()
     index->mIndexStats.Clear();
     index->mFrecencyArray.Clear();
     index->mIndex.Clear();
+
+    for (uint32_t i = 0; i < index->mIterators.Length(); ) {
+      nsresult rv = index->mIterators[i]->CloseInternal(NS_ERROR_NOT_AVAILABLE);
+      if (NS_FAILED(rv)) {
+        // CacheIndexIterator::CloseInternal() removes itself from mIterators
+        // iff it returns success.
+        LOG(("CacheIndex::RemoveAll() - Failed to remove iterator %p. "
+             "[rv=0x%08x]", rv));
+        i++;
+      }
+    }
   }
 
   if (file) {
@@ -1177,6 +1194,7 @@ CacheIndex::GetEntryForEviction(bool aIgnoreEmptyEntries, SHA1Sum::Hash *aHash,
   uint32_t i;
 
   // find first non-forced valid entry with the lowest frecency
+  index->mFrecencyArray.Sort(FrecencyComparator());
   for (i = 0; i < index->mFrecencyArray.Length(); ++i) {
     memcpy(&hash, &index->mFrecencyArray[i]->mHash, sizeof(SHA1Sum::Hash));
 
@@ -1347,6 +1365,7 @@ CacheIndex::GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
     iter = new CacheIndexIterator(index, aAddNew);
   }
 
+  index->mFrecencyArray.Sort(FrecencyComparator());
   iter->AddRecords(index->mFrecencyArray);
 
   index->mIterators.AppendElement(iter);
@@ -3105,28 +3124,6 @@ CacheIndex::ReleaseBuffer()
   mRWBufPos = 0;
 }
 
-namespace {
-
-class FrecencyComparator
-{
-public:
-  bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const {
-    return a->mFrecency == b->mFrecency;
-  }
-  bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const {
-    // Place entries with frecency 0 at the end of the array.
-    if (a->mFrecency == 0) {
-      return false;
-    }
-    if (b->mFrecency == 0) {
-      return true;
-    }
-    return a->mFrecency < b->mFrecency;
-  }
-};
-
-} // namespace
-
 void
 CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord)
 {
@@ -3134,7 +3131,7 @@ CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord)
        "%08x%08x]", aRecord, LOGSHA1(aRecord->mHash)));
 
   MOZ_ASSERT(!mFrecencyArray.Contains(aRecord));
-  mFrecencyArray.InsertElementSorted(aRecord, FrecencyComparator());
+  mFrecencyArray.AppendElement(aRecord);
 }
 
 void

netwerk/cache2/CacheObserver.cpp

@@ -31,8 +31,8 @@ static int32_t sAutoDeleteCacheVersion = kAutoDeleteCacheVersion;
 static int32_t const kDefaultHalfLifeExperiment = -1; // Disabled
 int32_t CacheObserver::sHalfLifeExperiment = kDefaultHalfLifeExperiment;
 
-static uint32_t const kDefaultHalfLifeHours = 6; // 6 hours
-uint32_t CacheObserver::sHalfLifeHours = kDefaultHalfLifeHours;
+static float const kDefaultHalfLifeHours = 1.0F; // 1 hour
+float CacheObserver::sHalfLifeHours = kDefaultHalfLifeHours;
 
 static bool const kDefaultUseDiskCache = true;
 bool CacheObserver::sUseDiskCache = kDefaultUseDiskCache;
@@ -63,11 +63,11 @@ bool CacheObserver::sSmartCacheSizeEnabled = kDefaultSmartCacheSizeEnabled;
 static uint32_t const kDefaultPreloadChunkCount = 4;
 uint32_t CacheObserver::sPreloadChunkCount = kDefaultPreloadChunkCount;
 
-static uint32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
-uint32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
+static int32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
+int32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
 
-static uint32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB
-uint32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize;
+static int32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB
+int32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize;
 
 static uint32_t const kDefaultMaxDiskChunksMemoryUsage = 10 * 1024; // 10MB
 uint32_t CacheObserver::sMaxDiskChunksMemoryUsage = kDefaultMaxDiskChunksMemoryUsage;
@@ -160,9 +160,9 @@ CacheObserver::AttachToPreferences()
   mozilla::Preferences::AddUintVarCache(
     &sPreloadChunkCount, "browser.cache.disk.preload_chunk_count", kDefaultPreloadChunkCount);
 
-  mozilla::Preferences::AddUintVarCache(
+  mozilla::Preferences::AddIntVarCache(
     &sMaxDiskEntrySize, "browser.cache.disk.max_entry_size", kDefaultMaxDiskEntrySize);
-  mozilla::Preferences::AddUintVarCache(
+  mozilla::Preferences::AddIntVarCache(
     &sMaxMemoryEntrySize, "browser.cache.memory.max_entry_size", kDefaultMaxMemoryEntrySize);
 
   mozilla::Preferences::AddUintVarCache(
@@ -204,22 +204,22 @@ CacheObserver::AttachToPreferences()
 
   switch (sHalfLifeExperiment) {
   case 1: // The experiment is engaged
-    sHalfLifeHours = 6;
+    sHalfLifeHours = 0.083F; // ~5 mintues
     break;
   case 2:
-    sHalfLifeHours = 24;
+    sHalfLifeHours = 0.25F; // 15 mintues
     break;
   case 3:
-    sHalfLifeHours = 7 * 24;
+    sHalfLifeHours = 1.0F;
     break;
   case 4:
-    sHalfLifeHours = 50 * 24;
+    sHalfLifeHours = 6.0F;
     break;
 
   case -1:
   default: // The experiment is off or broken
     sHalfLifeExperiment = -1;
-    sHalfLifeHours = std::max(1U, std::min(1440U, mozilla::Preferences::GetUint(
+    sHalfLifeHours = std::max(0.01F, std::min(1440.0F, mozilla::Preferences::GetFloat(
       "browser.cache.frecency_half_life_hours", kDefaultHalfLifeHours)));
     break;
   }
@@ -407,9 +407,12 @@ CacheStorageEvictHelper::ClearStorage(bool const aPrivate,
 bool const CacheObserver::EntryIsTooBig(int64_t aSize, bool aUsingDisk)
 {
   // If custom limit is set, check it.
-  int64_t preferredLimit = aUsingDisk
-    ? static_cast<int64_t>(sMaxDiskEntrySize) << 10
-    : static_cast<int64_t>(sMaxMemoryEntrySize) << 10;
+  int64_t preferredLimit = aUsingDisk ? sMaxDiskEntrySize : sMaxMemoryEntrySize;
+
+  // do not convert to bytes when the limit is -1, which means no limit
+  if (preferredLimit > 0) {
+    preferredLimit <<= 10;
+  }
 
   if (preferredLimit != -1 && aSize > preferredLimit)
     return true;

netwerk/cache2/CacheObserver.h

@@ -56,7 +56,7 @@ class CacheObserver : public nsIObserver
   static uint32_t const CompressionLevel()
     { return sCompressionLevel; }
   static uint32_t const HalfLifeSeconds()
-    { return sHalfLifeHours * 60 * 60; }
+    { return sHalfLifeHours * 60.0F * 60.0F; }
   static int32_t const HalfLifeExperiment()
     { return sHalfLifeExperiment; }
   static bool const ClearCacheOnShutdown()
@@ -82,12 +82,12 @@ private:
   static uint32_t sDiskFreeSpaceHardLimit;
   static bool sSmartCacheSizeEnabled;
   static uint32_t sPreloadChunkCount;
-  static uint32_t sMaxMemoryEntrySize;
-  static uint32_t sMaxDiskEntrySize;
+  static int32_t sMaxMemoryEntrySize;
+  static int32_t sMaxDiskEntrySize;
   static uint32_t sMaxDiskChunksMemoryUsage;
   static uint32_t sMaxDiskPriorityChunksMemoryUsage;
   static uint32_t sCompressionLevel;
-  static uint32_t sHalfLifeHours;
+  static float sHalfLifeHours;
   static int32_t sHalfLifeExperiment;
   static bool sSanitizeOnShutdown;
   static bool sClearCacheOnShutdown;

netwerk/cache2/CacheStorageService.cpp

@@ -892,8 +892,8 @@ CacheStorageService::RegisterEntry(CacheEntry* aEntry)
   LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry));
 
   MemoryPool& pool = Pool(aEntry->IsUsingDisk());
-  pool.mFrecencyArray.InsertElementSorted(aEntry, FrecencyComparator());
-  pool.mExpirationArray.InsertElementSorted(aEntry, ExpirationComparator());
+  pool.mFrecencyArray.AppendElement(aEntry);
+  pool.mExpirationArray.AppendElement(aEntry);
 
   aEntry->SetRegistered(true);
 }

netwerk/protocol/http/PackagedAppService.cpp

@@ -690,28 +690,55 @@ PackagedAppService::GetPackageURI(nsIURI *aURI, nsIURI **aPackageURI)
 }
 
 NS_IMETHODIMP
-PackagedAppService::GetResource(nsIPrincipal *aPrincipal,
-                                uint32_t aLoadFlags,
-                                nsILoadContextInfo *aInfo,
+PackagedAppService::GetResource(nsIChannel *aChannel,
                                 nsICacheEntryOpenCallback *aCallback)
 {
-  // Check arguments are not null
-  if (!aPrincipal || !aCallback || !aInfo) {
+  MOZ_RELEASE_ASSERT(NS_IsMainThread(), "mDownloadingPackages hashtable is not thread safe");
+  LOG(("[%p] PackagedAppService::GetResource(aChannel: %p, aCallback: %p)\n",
+       this, aChannel, aCallback));
+
+  if (!aChannel || !aCallback) {
     return NS_ERROR_INVALID_ARG;
   }
 
   nsresult rv;
+  nsIScriptSecurityManager *securityManager =
+    nsContentUtils::GetSecurityManager();
+  if (!securityManager) {
+    LOG(("[%p]    > No securityManager\n", this));
+    return NS_ERROR_UNEXPECTED;
+  }
+  nsCOMPtr<nsIPrincipal> principal;
+  rv = securityManager->GetChannelURIPrincipal(aChannel, getter_AddRefs(principal));
+  if (NS_FAILED(rv) || !principal) {
+    LOG(("[%p]    > Error getting principal rv=%X principal=%p\n",
+         this, rv, principal.get()));
+    return NS_FAILED(rv) ? rv : NS_ERROR_NULL_POINTER;
+  }
 
-  nsCOMPtr<nsIURI> uri;
-  rv = aPrincipal->GetURI(getter_AddRefs(uri));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
+  nsCOMPtr<nsILoadContextInfo> loadContextInfo = GetLoadContextInfo(aChannel);
+  if (!loadContextInfo) {
+    LOG(("[%p]    > Channel has no loadContextInfo\n", this));
+    return NS_ERROR_NULL_POINTER;
+  }
+
+  nsLoadFlags loadFlags = 0;
+  rv = aChannel->GetLoadFlags(&loadFlags);
+  if (NS_FAILED(rv)) {
+    LOG(("[%p]    > Error calling GetLoadFlags rv=%X\n", this, rv));
     return rv;
   }
 
-  LogURI("PackagedAppService::GetResource", this, uri, aInfo);
+  nsCOMPtr<nsILoadInfo> loadInfo = aChannel->GetLoadInfo();
 
-  MOZ_RELEASE_ASSERT(NS_IsMainThread(), "mDownloadingPackages hashtable is not thread safe");
+  nsCOMPtr<nsIURI> uri;
+  rv = principal->GetURI(getter_AddRefs(uri));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    LOG(("[%p]    > Error calling GetURI rv=%X\n", this, rv));
+    return rv;
+  }
 
+  LogURI("PackagedAppService::GetResource", this, uri, loadContextInfo);
   nsCOMPtr<nsIURI> packageURI;
   rv = GetPackageURI(uri, getter_AddRefs(packageURI));
   if (NS_FAILED(rv)) {
@@ -719,7 +746,7 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal,
   }
 
   nsAutoCString key;
-  CacheFileUtils::AppendKeyPrefix(aInfo, key);
+  CacheFileUtils::AppendKeyPrefix(loadContextInfo, key);
 
   {
     nsAutoCString spec;
@@ -740,10 +767,10 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal,
   }
 
   nsCOMPtr<nsIChannel> channel;
-  rv = NS_NewChannel(
-    getter_AddRefs(channel), packageURI, aPrincipal,
-    nsILoadInfo::SEC_NORMAL, nsIContentPolicy::TYPE_OTHER, nullptr, nullptr,
-    aLoadFlags);
+  rv = NS_NewChannelInternal(
+    getter_AddRefs(channel), packageURI,
+    loadInfo,
+    nullptr, nullptr, loadFlags);
 
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -758,7 +785,7 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal,
   }
 
   downloader = new PackagedAppDownloader();
-  rv = downloader->Init(aInfo, key);
+  rv = downloader->Init(loadContextInfo, key);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -784,6 +811,10 @@ PackagedAppService::GetResource(nsIPrincipal *aPrincipal,
   nsRefPtr<PackagedAppChannelListener> listener =
     new PackagedAppChannelListener(downloader, mimeConverter);
 
+  if (loadInfo && loadInfo->GetEnforceSecurity()) {
+    return channel->AsyncOpen2(listener);
+  }
+
   return channel->AsyncOpen(listener, nullptr);
 }
 

netwerk/protocol/http/nsHttpChannel.cpp

@@ -5208,24 +5208,29 @@ nsHttpChannel::BeginConnect()
     nsRefPtr<nsChannelClassifier> channelClassifier = new nsChannelClassifier();
     if (mLoadFlags & LOAD_CLASSIFY_URI) {
         nsCOMPtr<nsIURIClassifier> classifier = do_GetService(NS_URICLASSIFIERSERVICE_CONTRACTID);
-        if (classifier) {
-            bool tp = false;
-            channelClassifier->ShouldEnableTrackingProtection(this, &tp);
+        bool tpEnabled = false;
+        channelClassifier->ShouldEnableTrackingProtection(this, &tpEnabled);
+        if (classifier && tpEnabled) {
             // We skip speculative connections by setting mLocalBlocklist only
             // when tracking protection is enabled. Though we could do this for
             // both phishing and malware, it is not necessary for correctness,
             // since no network events will be received while the
             // nsChannelClassifier is in progress. See bug 1122691.
-            if (tp) {
-                nsCOMPtr<nsIPrincipal> principal = GetURIPrincipal();
-                nsresult response = NS_OK;
-                classifier->ClassifyLocal(principal, tp, &response);
-                if (NS_FAILED(response)) {
-                    LOG(("nsHttpChannel::ClassifyLocal found principal on local "
-                         "blocklist [this=%p]", this));
+            nsCOMPtr<nsIURI> uri;
+            rv = GetURI(getter_AddRefs(uri));
+            if (NS_SUCCEEDED(rv) && uri) {
+                nsAutoCString tables;
+                Preferences::GetCString("urlclassifier.trackingTable", &tables);
+                nsAutoCString results;
+                rv = classifier->ClassifyLocalWithTables(uri, tables, results);
+                if (NS_SUCCEEDED(rv) && !results.IsEmpty()) {
+                    LOG(("nsHttpChannel::ClassifyLocalWithTables found "
+                         "uri on local tracking blocklist [this=%p]",
+                         this));
                     mLocalBlocklist = true;
                 } else {
-                    LOG(("nsHttpChannel::ClassifyLocal no result found [this=%p]", this));
+                    LOG(("nsHttpChannel::ClassifyLocalWithTables no result "
+                         "found [this=%p]", this));
                 }
             }
         }
@@ -5241,12 +5246,6 @@ nsHttpChannel::BeginConnect()
         // by the packaged app service into the cache, and the cache entry will
         // be passed to OnCacheEntryAvailable.
 
-        // Pass the original load flags to the packaged app request.
-        uint32_t loadFlags = mLoadFlags;
-
-        mLoadFlags |= LOAD_ONLY_FROM_CACHE;
-        mLoadFlags |= LOAD_FROM_CACHE;
-        mLoadFlags &= ~VALIDATE_ALWAYS;
         nsCOMPtr<nsIPackagedAppService> pas =
             do_GetService("@mozilla.org/network/packaged-app-service;1", &rv);
         if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -5254,12 +5253,18 @@ nsHttpChannel::BeginConnect()
             return rv;
         }
 
-        nsCOMPtr<nsIPrincipal> principal = GetURIPrincipal();
-        nsCOMPtr<nsILoadContextInfo> loadInfo = GetLoadContextInfo(this);
-        rv = pas->GetResource(principal, loadFlags, loadInfo, this);
+        rv = pas->GetResource(this, this);
         if (NS_FAILED(rv)) {
             AsyncAbort(rv);
         }
+
+        // We need to alter the flags so the cache entry returned by the
+        // packaged app service is always accepted. Revalidation is handled
+        // by the service.
+        mLoadFlags |= LOAD_ONLY_FROM_CACHE;
+        mLoadFlags |= LOAD_FROM_CACHE;
+        mLoadFlags &= ~VALIDATE_ALWAYS;
+
         return rv;
     }
 

netwerk/test/unit/test_packaged_app_service.js

@@ -34,6 +34,7 @@
 Cu.import('resource://gre/modules/LoadContextInfo.jsm');
 Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
 
 // The number of times this package has been requested
 // This number might be reset by tests that use it
@@ -59,11 +60,24 @@ function packagedAppContentHandler(metadata, response)
   response.bodyOutputStream.write(body, body.length);
 }
 
-function getPrincipal(url) {
+function getChannelForURL(url) {
   let uri = createURI(url);
-  return Components.classes["@mozilla.org/scriptsecuritymanager;1"]
-         .getService(Ci.nsIScriptSecurityManager)
-         .getNoAppCodebasePrincipal(uri);
+  let ssm = Cc["@mozilla.org/scriptsecuritymanager;1"]
+              .getService(Ci.nsIScriptSecurityManager);
+  let principal = ssm.createCodebasePrincipal(uri, {});
+  let tmpChannel =
+    NetUtil.newChannel({
+      uri: url,
+      loadingPrincipal: principal,
+      contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER
+    });
+
+  tmpChannel.notificationCallbacks =
+    new LoadContextCallback(principal.appId,
+                            principal.isInBrowserElement,
+                            false,
+                            false);
+  return tmpChannel;
 }
 
 // The package content
@@ -129,6 +143,15 @@ function run_test()
   add_test(test_bad_package);
   add_test(test_bad_package_404);
 
+  // Channels created by addons could have no load info.
+  // In debug mode this triggers an assertion, but we still want to test that
+  // it works in optimized mode. See bug 1196021 comment 17
+  if (Components.classes["@mozilla.org/xpcom/debug;1"]
+                .getService(Components.interfaces.nsIDebug2)
+                .isDebugBuild == false) {
+    add_test(test_channel_no_loadinfo);
+  }
+
   // run tests
   run_next_test();
 }
@@ -179,10 +202,10 @@ var cacheListener = new packagedResourceListener(testData.content[0].data);
 
 // These calls should fail, since one of the arguments is invalid or null
 function test_bad_args() {
-  Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com"), 0, LoadContextInfo.default, cacheListener); }, "url's with no !// aren't allowed");
-  Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com/package!//test"), 0, LoadContextInfo.default, null); }, "should have a callback");
-  Assert.throws(() => { paservice.getResource(null, 0, LoadContextInfo.default, cacheListener); }, "should have a URI");
-  Assert.throws(() => { paservice.getResource(getPrincipal("http://test.com/package!//test"), null, cacheListener); }, "should have a LoadContextInfo");
+  Assert.throws(() => { paservice.getResource(getChannelForURL("http://test.com"), cacheListener); }, "url's with no !// aren't allowed");
+  Assert.throws(() => { paservice.getResource(getChannelForURL("http://test.com/package!//test"), null); }, "should have a callback");
+  Assert.throws(() => { paservice.getResource(null, cacheListener); }, "should have a channel");
+
   run_next_test();
 }
 
@@ -191,13 +214,15 @@ function test_bad_args() {
 // This tests that the callback gets called, and the cacheListener gets the proper content.
 function test_callback_gets_called() {
   packagePath = "/package";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener);
+  let url = uri + packagePath + "!//index.html";
+  paservice.getResource(getChannelForURL(url), cacheListener);
 }
 
 // Tests that requesting the same resource returns the same content
 function test_same_content() {
   packagePath = "/package";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener);
+  let url = uri + packagePath + "!//index.html";
+  paservice.getResource(getChannelForURL(url), cacheListener);
 }
 
 // Check the content handler has been called the expected number of times.
@@ -209,8 +234,9 @@ function test_request_number() {
 // This tests that new content is returned if the package has been updated
 function test_updated_package() {
   packagePath = "/package";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default,
-      new packagedResourceListener(testData.content[0].data.replace(/\.\.\./g, 'xxx')));
+  let url = uri + packagePath + "!//index.html";
+  paservice.getResource(getChannelForURL(url),
+    new packagedResourceListener(testData.content[0].data.replace(/\.\.\./g, 'xxx')));
 }
 
 // ----------------------------------------------------------------------------
@@ -233,13 +259,15 @@ var listener404 = {
 // Tests that an error is returned for a non existing package
 function test_package_does_not_exist() {
   packagePath = "/package_non_existent";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, listener404);
+  let url = uri + packagePath + "!//index.html";
+  paservice.getResource(getChannelForURL(url), listener404);
 }
 
 // Tests that an error is returned for a non existing resource in a package
 function test_file_does_not_exist() {
   packagePath = "/package"; // This package exists
-  paservice.getResource(getPrincipal(uri + packagePath + "!//file_non_existent.html"), 0, LoadContextInfo.default, listener404);
+  let url = uri + packagePath + "!//file_non_existent.html";
+  paservice.getResource(getChannelForURL(url), listener404);
 }
 
 // ----------------------------------------------------------------------------
@@ -280,13 +308,24 @@ function packagedAppBadContentHandler(metadata, response)
 // Checks that the resource with the proper headers inside the bad package is still returned
 function test_bad_package() {
   packagePath = "/badPackage";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//index.html"), 0, LoadContextInfo.default, cacheListener);
+  let url = uri + packagePath + "!//index.html";
+  paservice.getResource(getChannelForURL(url), cacheListener);
 }
 
 // Checks that the request for a non-existent resource doesn't hang for a bad package
 function test_bad_package_404() {
   packagePath = "/badPackage";
-  paservice.getResource(getPrincipal(uri + packagePath + "!//file_non_existent.html"), 0, LoadContextInfo.default, listener404);
+  let url = uri + packagePath + "!//file_non_existent.html";
+  paservice.getResource(getChannelForURL(url), listener404);
 }
 
 // ----------------------------------------------------------------------------
+
+// NOTE: This test only runs in NON-DEBUG mode.
+function test_channel_no_loadinfo() {
+  packagePath = "/package";
+  let url = uri + packagePath + "!//index.html";
+  let channel = getChannelForURL(url);
+  channel.loadInfo = null;
+  paservice.getResource(channel, cacheListener);
+}

netwerk/test/unit/test_packaged_app_service_paths.js

@@ -1,6 +1,7 @@
 Cu.import('resource://gre/modules/LoadContextInfo.jsm');
 Cu.import("resource://testing-common/httpd.js");
 Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
 
 var gRequestNo = 0;
 function packagedAppContentHandler(metadata, response)
@@ -11,11 +12,24 @@ function packagedAppContentHandler(metadata, response)
   gRequestNo++;
 }
 
-function getPrincipal(url) {
+function getChannelForURL(url) {
   let uri = createURI(url);
-  return Components.classes["@mozilla.org/scriptsecuritymanager;1"]
-         .getService(Ci.nsIScriptSecurityManager)
-         .getNoAppCodebasePrincipal(uri);
+  let ssm = Cc["@mozilla.org/scriptsecuritymanager;1"]
+              .getService(Ci.nsIScriptSecurityManager);
+  let principal = ssm.createCodebasePrincipal(uri, {});
+  let tmpChannel =
+    NetUtil.newChannel({
+      uri: url,
+      loadingPrincipal: principal,
+      contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER
+    });
+
+  tmpChannel.notificationCallbacks =
+    new LoadContextCallback(principal.appId,
+                            principal.isInBrowserElement,
+                            false,
+                            false);
+  return tmpChannel;
 }
 
 var subresourcePaths = [
@@ -119,8 +133,8 @@ function test_paths() {
   for (var i in subresourcePaths) {
     packagePath = "/package/" + i;
     dump("Iteration " + i + "\n");
-    paservice.getResource(getPrincipal(uri + packagePath + "!//" + subresourcePaths[i][1]), 0,
-      LoadContextInfo.default,
+    let url = uri + packagePath + "!//" + subresourcePaths[i][1];
+    paservice.getResource(getChannelForURL(url),
       new packagedResourceListener(subresourcePaths[i][1], content));
     yield undefined;
   }

testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-match.https.html.ini

@@ -1,3 +1,6 @@
 [cache-match.https.html]
   type: testharness
   prefs: [dom.serviceWorkers.enabled: true, dom.serviceWorkers.interception.enabled: true, dom.serviceWorkers.exemptFromPerDomainMax:true, dom.caches.enabled:true]
+  [Cache.match and Cache.matchAll]
+    expected: FAIL
+    bug: https://github.com/w3c/web-platform-tests/issues/2098

testing/web-platform/meta/service-workers/cache-storage/serviceworker/cache-put.https.html.ini

@@ -1,3 +1,6 @@
 [cache-put.https.html]
   type: testharness
   prefs: [dom.serviceWorkers.enabled: true, dom.serviceWorkers.interception.enabled: true, dom.serviceWorkers.exemptFromPerDomainMax:true, dom.caches.enabled:true]
+  [Cache.put with request URLs containing embedded credentials]
+    expected: FAIL
+    bug: https://github.com/w3c/web-platform-tests/issues/2098

testing/web-platform/meta/service-workers/cache-storage/window/cache-match.https.html.ini

@@ -1,3 +1,5 @@
 [cache-match.https.html]
   type: testharness
   prefs: [dom.caches.enabled:true]
+  expected: ERROR
+  bug: https://github.com/w3c/web-platform-tests/issues/2098

testing/web-platform/meta/service-workers/cache-storage/window/cache-put.https.html.ini

@@ -1,3 +1,6 @@
 [cache-put.https.html]
   type: testharness
   prefs: [dom.caches.enabled:true]
+  [Cache.put with request URLs containing embedded credentials]
+    expected: FAIL
+    bug: https://github.com/w3c/web-platform-tests/issues/2098

testing/web-platform/meta/service-workers/cache-storage/worker/cache-match.https.html.ini

@@ -1,3 +1,5 @@
 [cache-match.https.html]
   type: testharness
   prefs: [dom.caches.enabled:true]
+  expected: ERROR
+  bug: https://github.com/w3c/web-platform-tests/issues/2098

testing/web-platform/meta/service-workers/cache-storage/worker/cache-put.https.html.ini

@@ -1,3 +1,6 @@
 [cache-put.https.html]
   type: testharness
   prefs: [dom.caches.enabled:true]
+  [Cache.put with request URLs containing embedded credentials]
+    expected: FAIL
+    bug: https://github.com/w3c/web-platform-tests/issues/2098

toolkit/components/telemetry/Histograms.json

@@ -1542,11 +1542,17 @@
     "description": "HTTP Disk Cache Hit, Reval, Failed-Reval, Miss"
   },
   "HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT": {
-    "expires_in_version": "never",
+    "expires_in_version": "40",
     "kind": "enumerated",
     "n_values": 4,
     "description": "HTTP Cache v2 Miss by half-life value (6h, 1d, 7d, 50d)"
   },
+  "HTTP_CACHE_MISS_HALFLIFE_EXPERIMENT_2": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": 4,
+    "description": "HTTP Cache v2 Miss by half-life value (5 min, 15 min, 1 hour, 6 hours)"
+  },
   "HTTP_CACHE_ENTRY_RELOAD_TIME": {
     "expires_in_version": "never",
     "kind": "exponential",

toolkit/components/url-classifier/SafeBrowsing.jsm

@@ -0,0 +1,274 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["SafeBrowsing"];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+
+// Skip all the ones containining "test", because we never need to ask for
+// updates for them.
+function getLists(prefName) {
+  let pref = Services.prefs.getCharPref(prefName);
+  // Splitting an empty string returns [''], we really want an empty array.
+  if (!pref) {
+    return [];
+  }
+  return pref.split(",")
+    .filter(function(value) { return value.indexOf("test-") == -1; })
+    .map(function(value) { return value.trim(); });
+}
+
+// These may be a comma-separated lists of tables.
+const phishingLists = getLists("urlclassifier.phishTable");
+const malwareLists = getLists("urlclassifier.malwareTable");
+const downloadBlockLists = getLists("urlclassifier.downloadBlockTable");
+const downloadAllowLists = getLists("urlclassifier.downloadAllowTable");
+const trackingProtectionLists = getLists("urlclassifier.trackingTable");
+const trackingProtectionWhitelists = getLists("urlclassifier.trackingWhitelistTable");
+
+var debug = false;
+function log(...stuff) {
+  if (!debug)
+    return;
+
+  var d = new Date();
+  let msg = "SafeBrowsing: " + d.toTimeString() + ": " + stuff.join(" ");
+  Services.console.logStringMessage(msg);
+  dump(msg + "\n");
+}
+
+this.SafeBrowsing = {
+
+  init: function() {
+    if (this.initialized) {
+      log("Already initialized");
+      return;
+    }
+
+    Services.prefs.addObserver("browser.safebrowsing", this.readPrefs.bind(this), false);
+    Services.prefs.addObserver("privacy.trackingprotection", this.readPrefs.bind(this), false);
+    this.readPrefs();
+
+    // Register our two types of tables, and add custom Mozilla entries
+    let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"].
+                      getService(Ci.nsIUrlListManager);
+    for (let i = 0; i < phishingLists.length; ++i) {
+      listManager.registerTable(phishingLists[i], this.updateURL, this.gethashURL);
+    }
+    for (let i = 0; i < malwareLists.length; ++i) {
+      listManager.registerTable(malwareLists[i], this.updateURL, this.gethashURL);
+    }
+    for (let i = 0; i < downloadBlockLists.length; ++i) {
+      listManager.registerTable(downloadBlockLists[i], this.updateURL, this.gethashURL);
+    }
+    for (let i = 0; i < downloadAllowLists.length; ++i) {
+      listManager.registerTable(downloadAllowLists[i], this.updateURL, this.gethashURL);
+    }
+    for (let i = 0; i < trackingProtectionLists.length; ++i) {
+      listManager.registerTable(trackingProtectionLists[i],
+                                this.trackingUpdateURL,
+                                this.trackingGethashURL);
+    }
+    for (let i = 0; i < trackingProtectionWhitelists.length; ++i) {
+      listManager.registerTable(trackingProtectionWhitelists[i],
+                                this.trackingUpdateURL,
+                                this.trackingGethashURL);
+    }
+    this.addMozEntries();
+
+    this.controlUpdateChecking();
+    this.initialized = true;
+
+    log("init() finished");
+  },
+
+
+  initialized:     false,
+  phishingEnabled: false,
+  malwareEnabled:  false,
+
+  updateURL:             null,
+  gethashURL:            null,
+
+  reportURL:             null,
+
+  getReportURL: function(kind, URI) {
+    let pref;
+    switch (kind) {
+      case "Phish":
+        pref = "browser.safebrowsing.reportPhishURL";
+        break;
+      case "PhishMistake":
+        pref = "browser.safebrowsing.reportPhishMistakeURL";
+        break;
+      case "MalwareMistake":
+        pref = "browser.safebrowsing.reportMalwareMistakeURL";
+        break;
+
+      default:
+        let err = "SafeBrowsing getReportURL() called with unknown kind: " + kind;
+        Components.utils.reportError(err);
+        throw err;
+    }
+    let reportUrl = Services.urlFormatter.formatURLPref(pref);
+
+    let pageUri = URI.clone();
+
+    // Remove the query to avoid including potentially sensitive data
+    if (pageUri instanceof Ci.nsIURL)
+      pageUri.query = '';
+
+    reportUrl += encodeURIComponent(pageUri.asciiSpec);
+
+    return reportUrl;
+  },
+
+
+  readPrefs: function() {
+    log("reading prefs");
+
+    debug = Services.prefs.getBoolPref("browser.safebrowsing.debug");
+    this.phishingEnabled = Services.prefs.getBoolPref("browser.safebrowsing.enabled");
+    this.malwareEnabled = Services.prefs.getBoolPref("browser.safebrowsing.malware.enabled");
+    this.trackingEnabled = Services.prefs.getBoolPref("privacy.trackingprotection.enabled") || Services.prefs.getBoolPref("privacy.trackingprotection.pbmode.enabled");
+    this.updateProviderURLs();
+
+    // XXX The listManager backend gets confused if this is called before the
+    // lists are registered. So only call it here when a pref changes, and not
+    // when doing initialization. I expect to refactor this later, so pardon the hack.
+    if (this.initialized) {
+      this.controlUpdateChecking();
+    }
+  },
+
+
+  updateProviderURLs: function() {
+    try {
+      var clientID = Services.prefs.getCharPref("browser.safebrowsing.id");
+    } catch(e) {
+      var clientID = Services.appinfo.name;
+    }
+
+    log("initializing safe browsing URLs, client id ", clientID);
+
+    // Urls used to update DB
+    this.updateURL  = Services.urlFormatter.formatURLPref("browser.safebrowsing.updateURL");
+    this.gethashURL = Services.urlFormatter.formatURLPref("browser.safebrowsing.gethashURL");
+
+    this.updateURL  = this.updateURL.replace("SAFEBROWSING_ID", clientID);
+    this.gethashURL = this.gethashURL.replace("SAFEBROWSING_ID", clientID);
+    this.trackingUpdateURL = Services.urlFormatter.formatURLPref(
+      "browser.trackingprotection.updateURL");
+    this.trackingGethashURL = Services.urlFormatter.formatURLPref(
+      "browser.trackingprotection.gethashURL");
+  },
+
+  controlUpdateChecking: function() {
+    log("phishingEnabled:", this.phishingEnabled, "malwareEnabled:",
+        this.malwareEnabled, "trackingEnabled:", this.trackingEnabled);
+
+    let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"].
+                      getService(Ci.nsIUrlListManager);
+
+    for (let i = 0; i < phishingLists.length; ++i) {
+      if (this.phishingEnabled) {
+        listManager.enableUpdate(phishingLists[i]);
+      } else {
+        listManager.disableUpdate(phishingLists[i]);
+      }
+    }
+    for (let i = 0; i < malwareLists.length; ++i) {
+      if (this.malwareEnabled) {
+        listManager.enableUpdate(malwareLists[i]);
+      } else {
+        listManager.disableUpdate(malwareLists[i]);
+      }
+    }
+    for (let i = 0; i < downloadBlockLists.length; ++i) {
+      if (this.malwareEnabled) {
+        listManager.enableUpdate(downloadBlockLists[i]);
+      } else {
+        listManager.disableUpdate(downloadBlockLists[i]);
+      }
+    }
+    for (let i = 0; i < downloadAllowLists.length; ++i) {
+      if (this.malwareEnabled) {
+        listManager.enableUpdate(downloadAllowLists[i]);
+      } else {
+        listManager.disableUpdate(downloadAllowLists[i]);
+      }
+    }
+    for (let i = 0; i < trackingProtectionLists.length; ++i) {
+      if (this.trackingEnabled) {
+        listManager.enableUpdate(trackingProtectionLists[i]);
+        listManager.enableUpdate(trackingProtectionWhitelists[i]);
+      } else {
+        listManager.disableUpdate(trackingProtectionLists[i]);
+        listManager.disableUpdate(trackingProtectionWhitelists[i]);
+      }
+    }
+    listManager.maybeToggleUpdateChecking();
+  },
+
+
+  addMozEntries: function() {
+    // Add test entries to the DB.
+    // XXX bug 779008 - this could be done by DB itself?
+    const phishURL    = "itisatrap.org/firefox/its-a-trap.html";
+    const malwareURL  = "itisatrap.org/firefox/its-an-attack.html";
+    const unwantedURL = "itisatrap.org/firefox/unwanted.html";
+    const trackerURLs = [
+      "trackertest.org/",
+      "itisatracker.org/",
+    ];
+    const whitelistURL = "itisatrap.org/?resource=itisatracker.org";
+
+    let update = "n:1000\ni:test-malware-simple\nad:1\n" +
+                 "a:1:32:" + malwareURL.length + "\n" +
+                 malwareURL + "\n";
+    update += "n:1000\ni:test-phish-simple\nad:1\n" +
+              "a:1:32:" + phishURL.length + "\n" +
+              phishURL  + "\n";
+    update += "n:1000\ni:test-unwanted-simple\nad:1\n" +
+              "a:1:32:" + unwantedURL.length + "\n" +
+              unwantedURL + "\n";
+    update += "n:1000\ni:test-track-simple\n" +
+              "ad:" + trackerURLs.length + "\n";
+    trackerURLs.forEach((trackerURL, i) => {
+      update += "a:" + (i + 1) + ":32:" + trackerURL.length + "\n" +
+                trackerURL + "\n";
+    });
+    update += "n:1000\ni:test-trackwhite-simple\nad:1\n" +
+              "a:1:32:" + whitelistURL.length + "\n" +
+              whitelistURL;
+    log("addMozEntries:", update);
+
+    let db = Cc["@mozilla.org/url-classifier/dbservice;1"].
+             getService(Ci.nsIUrlClassifierDBService);
+
+    // nsIUrlClassifierUpdateObserver
+    let dummyListener = {
+      updateUrlRequested: function() { },
+      streamFinished:     function() { },
+      updateError:        function() { },
+      updateSuccess:      function() { }
+    };
+
+    try {
+      let tables = "test-malware-simple,test-phish-simple,test-unwanted-simple,test-track-simple,test-trackwhite-simple";
+      db.beginUpdate(dummyListener, tables, "");
+      db.beginStream("", "");
+      db.updateStream(update);
+      db.finishStream();
+      db.finishUpdate();
+    } catch(ex) {
+      // beginUpdate will throw harmlessly if there's an existing update in progress, ignore failures.
+      log("addMozEntries failed!", ex);
+    }
+  },
+};

toolkit/components/url-classifier/tests/mochitest/good.js

@@ -0,0 +1 @@
+scriptItem = "loaded whitelisted javascript!";

toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html

@@ -0,0 +1,150 @@
+<!DOCTYPE HTML>
+<!-- Any copyright is dedicated to the Public Domain.
+     http://creativecommons.org/publicdomain/zero/1.0/ -->
+<html>
+<head>
+  <title>Test Tracking Protection in Private Browsing mode</title>
+  <script type="text/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css">
+</head>
+
+<body>
+<p id="display"></p>
+<div id="content" style="display: none">
+</div>
+<pre id="test">
+
+<script class="testbody" type="text/javascript">
+
+var Cc = SpecialPowers.Cc;
+var Ci = SpecialPowers.Ci;
+
+var mainWindow = window.QueryInterface(Ci.nsIInterfaceRequestor)
+                    .getInterface(Ci.nsIWebNavigation)
+                    .QueryInterface(Ci.nsIDocShellTreeItem)
+                    .rootTreeItem
+                    .QueryInterface(Ci.nsIInterfaceRequestor)
+                    .getInterface(Ci.nsIDOMWindow);
+var contentPage1 = "http://www.itisatrap.org/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html";
+var contentPage2 = "http://example.com/tests/toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html";
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://testing-common/UrlClassifierTestUtils.jsm");
+
+function whenDelayedStartupFinished(aWindow, aCallback) {
+  Services.obs.addObserver(function observer(aSubject, aTopic) {
+    if (aWindow == aSubject) {
+      Services.obs.removeObserver(observer, aTopic);
+      setTimeout(aCallback, 0);
+    }
+  }, "browser-delayed-startup-finished", false);
+}
+
+function testOnWindow(contentPage, aCallback) {
+  var win = mainWindow.OpenBrowserWindow();
+  win.addEventListener("load", function onLoad() {
+    win.removeEventListener("load", onLoad, false);
+    whenDelayedStartupFinished(win, function() {
+      win.addEventListener("DOMContentLoaded", function onInnerLoad() {
+        if (win.content.location.href != contentPage) {
+          win.gBrowser.loadURI(contentPage);
+          return;
+        }
+        win.removeEventListener("DOMContentLoaded", onInnerLoad, true);
+
+        win.content.addEventListener('load', function innerLoad2() {
+          win.content.removeEventListener('load', innerLoad2, false);
+          SimpleTest.executeSoon(function() { aCallback(win); });
+        }, false, true);
+      }, true);
+      SimpleTest.executeSoon(function() { win.gBrowser.loadURI(contentPage); });
+    });
+  }, true);
+}
+
+var alwaysbadids = [
+  "badscript",
+];
+
+function checkLoads(aWindow, aWhitelisted) {
+  var win = aWindow.content;
+  is(win.document.getElementById("badscript").dataset.touched, "no", "Should not load tracking javascript");
+  is(win.document.getElementById("goodscript").dataset.touched, aWhitelisted ? "yes" : "no", "Should load whitelisted tracking javascript");
+
+  var badids = alwaysbadids.slice();
+  if (!aWhitelisted) {
+    badids.push("goodscript");
+  }
+  is(win.document.blockedTrackingNodeCount, badids.length, "Should identify all tracking elements");
+
+  var blockedTrackingNodes = win.document.blockedTrackingNodes;
+
+  // Make sure that every node in blockedTrackingNodes exists in the tree
+  // (that may not always be the case but do not expect any nodes to disappear
+  // from the tree here)
+  var allNodeMatch = true;
+  for (var i = 0; i < blockedTrackingNodes.length; i++) {
+    var nodeMatch = false;
+    for (var j = 0; j < badids.length && !nodeMatch; j++) {
+      nodeMatch = nodeMatch ||
+        (blockedTrackingNodes[i] == win.document.getElementById(badids[j]));
+    }
+
+    allNodeMatch = allNodeMatch && nodeMatch;
+  }
+  is(allNodeMatch, true, "All annotated nodes are expected in the tree");
+
+  // Make sure that every node with a badid (see badids) is found in the
+  // blockedTrackingNodes. This tells us if we are neglecting to annotate
+  // some nodes
+  allNodeMatch = true;
+  for (var j = 0; j < badids.length; j++) {
+    var nodeMatch = false;
+    for (var i = 0; i < blockedTrackingNodes.length && !nodeMatch; i++) {
+      nodeMatch = nodeMatch ||
+        (blockedTrackingNodes[i] == win.document.getElementById(badids[j]));
+    }
+
+    allNodeMatch = allNodeMatch && nodeMatch;
+  }
+  is(allNodeMatch, true, "All tracking nodes are expected to be annotated as such");
+}
+
+SpecialPowers.pushPrefEnv(
+  {"set" : [["privacy.trackingprotection.enabled", true],
+            ["channelclassifier.allowlist_example", true]]},
+  test);
+
+function test() {
+  SimpleTest.registerCleanupFunction(UrlClassifierTestUtils.cleanupTestTrackers);
+  UrlClassifierTestUtils.addTestTrackers().then(() => {
+    // Load the test from a URL on the whitelist
+    testOnWindow(contentPage1, function(aWindow) {
+      checkLoads(aWindow, true);
+      aWindow.close();
+
+      // Load the test from a URL that's NOT on the whitelist
+      testOnWindow(contentPage2, function(aWindow) {
+        checkLoads(aWindow, false);
+        aWindow.close();
+
+        // Load the test from a URL on the whitelist but without the whitelist
+        SpecialPowers.setCharPref("urlclassifier.trackingWhitelistTable", "");
+        testOnWindow(contentPage1, function(aWindow) {
+        checkLoads(aWindow, false);
+          aWindow.close();
+          SimpleTest.finish();
+        });
+      });
+    });
+  });
+}
+
+SimpleTest.waitForExplicitFinish();
+
+</script>
+
+</pre>
+<iframe id="testFrame" width="100%" height="100%" onload=""></iframe>
+</body>
+</html>

toolkit/components/url-classifier/tests/mochitest/whitelistFrame.html

@@ -0,0 +1,15 @@
+<!DOCTYPE HTML>
+<!-- Any copyright is dedicated to the Public Domain.
+     http://creativecommons.org/publicdomain/zero/1.0/ -->
+<html>
+<head>
+<title></title>
+</head>
+<body>
+
+<script id="badscript" data-touched="not sure" src="http://trackertest.org/tests/toolkit/components/url-classifier/tests/mochitest/evil.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
+
+<script id="goodscript" data-touched="not sure" src="http://itisatracker.org/tests/toolkit/components/url-classifier/tests/mochitest/good.js" onload="this.dataset.touched = 'yes';" onerror="this.dataset.touched = 'no';"></script>
+
+</body>
+</html>