import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1048131 - Remove MOZ_CAPTIVEDETECT and enable captive portal detector for all products. r=mcmanus (d2b2980c1)
- Bug 1048131 - Implement CaptivePortalService using nsICaptivePortalDetector r=mcmanus (d62c390d4)
- Bug 1048131 - Make some network events trigger a captive portal recheck r=mcmanus (8c19273f0)
- Bug 1166910 - Gecko changes for adding referrer attribute. r=hsivonen (46676ae77)
- Bug 1166910 - Referrer attribute for img tag. r=ckerschb, r=hsivonen, r=bz (f833f3e6c)
- Bug 1174913 - anchor and area referrer attributes. r=ckerschb, r=bz (f99a956be) (backend parts only)
- missing bits of Bug 1210302 - Part 1: Rename nsINetUtil.parseContentType() to parseResponseContentType(); r=mcmanus (eb8fba336)
- Bug 1210302 - Part 2: Rename NS_ParseContentType to NS_ParseResponseContentType; r=mcmanus,sicking (74f4cd98d)
- remaining bits of Bug 1210302 - Part 3: Add a NS_ParseRequestContentType API; r=mcmanus,sicking (56477b434)
- pointer style (ca5142301)
- Bug 1172109 - Name some anonymous structs to help out the analysis, r=terrence (775d17318)
- Bug 1173042 - Remove unused GCChunkSet and chunk hashing functions; r=ehoogeveen (357e0887d)
- Bug 1177778 - Allow HeapPtr destruction while shutting down the GC r=terrence (793654cb2)
- Bug 1181306 - Skip compacting phase when finishing an incremental GC non-incrementally r=terrence (7547c11ad)
- Bug 1162961 - Fix logspam in mochitest-dt by passing the URL along in network progress events;r=past (3b4c9048f)
- Bug 1149235 part 1. Move nsScriptLoadRequest into nsScriptLoader.h. r=sicking (df56a3e9d)
- Bug 1149235 part 2. Switch to using linked lists for nsScriptLoadRequest. r=sicking (7a0536b53)
- Bug 1149235 part 3. Store async requests in the scriptloader in two lists, so we don't have to grovel about looking for loaded ones. r=sicking (db45689c6)
- Bug 1154598. Keep better track of our script requests. r=smaug (a2232a237)
- Bug 1157995 - Tell the cycle collector about Navigator::mMediaDevices. r=smaug (9a9147bdf)
- Bug 1181619. Make sure we've entered a microtask before we call nsJSUtils::EvaluateString, and put those microtasks outside the relevant AutoEntryScripts so we report any possible exceptions before doing the microtask checkpoint. r=bholley (851e371ac)

b2g/app/b2g.js

@@ -870,10 +870,6 @@ pref("consoleservice.buffered", false);
 pref("toolkit.storage.pageSize", 2048);
 #endif
 
-// Enable captive portal detection.
-pref("captivedetect.canonicalURL", "http://detectportal.firefox.com/success.txt");
-pref("captivedetect.canonicalContent", "success\n");
-
 // The url of the manifest we use for ADU pings.
 pref("ping.manifestURL", "https://marketplace.firefox.com/packaged.webapp");
 

b2g/chrome/content/shell.js

@@ -68,11 +68,9 @@ XPCOMUtils.defineLazyGetter(this, "libcutils", function () {
 });
 #endif
 
-#ifdef MOZ_CAPTIVEDETECT
 XPCOMUtils.defineLazyServiceGetter(Services, 'captivePortalDetector',
                                   '@mozilla.org/toolkit/captive-detector;1',
                                   'nsICaptivePortalDetector');
-#endif
 
 function getContentWindow() {
   return shell.contentBrowser.contentWindow;

b2g/confvars.sh

@@ -19,7 +19,6 @@ MOZ_OFFICIAL_BRANDING_DIRECTORY=b2g/branding/official
 
 MOZ_SERVICES_COMMON=1
 MOZ_SERVICES_METRICS=1
-MOZ_CAPTIVEDETECT=1
 
 MOZ_WEBSMS_BACKEND=1
 MOZ_NO_SMART_CARDS=1

b2g/installer/package-manifest.in

@@ -313,9 +313,7 @@
 @RESPATH@/components/services-crypto.xpt
 #endif
 @RESPATH@/components/services-crypto-component.xpt
-#ifdef MOZ_CAPTIVEDETECT
 @RESPATH@/components/captivedetect.xpt
-#endif
 @RESPATH@/components/shellservice.xpt
 @RESPATH@/components/shistory.xpt
 @RESPATH@/components/spellchecker.xpt
@@ -616,10 +614,8 @@
 @RESPATH@/components/HealthReportComponents.manifest
 @RESPATH@/components/HealthReportService.js
 #endif
-#ifdef MOZ_CAPTIVEDETECT
 @RESPATH@/components/CaptivePortalDetectComponents.manifest
 @RESPATH@/components/captivedetect.js
-#endif
 @RESPATH@/components/TelemetryStartup.js
 @RESPATH@/components/TelemetryStartup.manifest
 @RESPATH@/components/XULStore.js

browser/installer/package-manifest.in

@@ -325,9 +325,7 @@
 @RESPATH@/components/saxparser.xpt
 @RESPATH@/browser/components/sessionstore.xpt
 @RESPATH@/components/services-crypto-component.xpt
-#ifdef MOZ_CAPTIVEDETECT
 @RESPATH@/components/captivedetect.xpt
-#endif
 @RESPATH@/browser/components/shellservice.xpt
 #ifdef MOZ_BROWSER_STATUSBAR
 @RESPATH@/browser/components/status4evar.xpt
@@ -535,10 +533,8 @@
 @RESPATH@/components/SyncComponents.manifest
 @RESPATH@/components/Weave.js
 #endif
-#ifdef MOZ_CAPTIVEDETECT
 @RESPATH@/components/CaptivePortalDetectComponents.manifest
 @RESPATH@/components/captivedetect.js
-#endif
 @RESPATH@/components/servicesComponents.manifest
 @RESPATH@/components/cryptoComponents.manifest
 @RESPATH@/components/TelemetryStartup.js

configure.in

@@ -8038,12 +8038,6 @@ if test -n "$MOZ_SERVICES_CLOUDSYNC"; then
   AC_DEFINE(MOZ_SERVICES_CLOUDSYNC)
 fi
 
-dnl Build Captive Portal Detector if required
-AC_SUBST(MOZ_CAPTIVEDETECT)
-if test -n "$MOZ_CAPTIVEDETECT"; then
-  AC_DEFINE(MOZ_CAPTIVEDETECT)
-fi
-
 dnl Build second screen and casting features for external devices if required
 AC_SUBST(MOZ_DEVICES)
 if test -n "$MOZ_DEVICES"; then

docshell/base/nsDocShell.cpp

@@ -13708,6 +13708,22 @@ nsDocShell::OnLinkClickSync(nsIContent* aContent,
   nsCOMPtr<nsIURI> referer = refererDoc->GetDocumentURI();
   uint32_t refererPolicy = refererDoc->GetReferrerPolicy();
 
+  // get referrer attribute from clicked link and parse it
+  // if per element referrer is enabled, the element referrer overrules
+  // the document wide referrer
+  if (IsElementAnchor(aContent)) {
+    MOZ_ASSERT(aContent->IsHTMLElement());
+    if (Preferences::GetBool("network.http.enablePerElementReferrer", false)) {
+      nsAutoString referrerPolicy;
+      if (aContent->GetAttr(kNameSpaceID_None, nsGkAtoms::referrer, referrerPolicy)) {
+        uint32_t refPolEnum = mozilla::net::ReferrerPolicyFromString(referrerPolicy);
+        if (refPolEnum != mozilla::net::RP_Unset) {
+          refererPolicy = refPolEnum;
+        }
+      }
+    }
+  }
+
   // referer could be null here in some odd cases, but that's ok,
   // we'll just load the link w/o sending a referer in those cases.
 

dom/apps/AppsUtils.jsm

@@ -508,7 +508,7 @@ this.AppsUtils = {
     let hadCharset = { };
     let charset = { };
     let netutil = Cc["@mozilla.org/network/util;1"].getService(Ci.nsINetUtil);
-    let contentType = netutil.parseContentType(aContentType, charset, hadCharset);
+    let contentType = netutil.parseResponseContentType(aContentType, charset, hadCharset);
     if (aInstallOrigin != aWebappOrigin &&
         !(contentType == "application/x-web-app-manifest+json" ||
           contentType == "application/manifest+json")) {

dom/base/Navigator.cpp

@@ -198,6 +198,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(Navigator)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mAudioChannelManager)
 #endif
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mCameraManager)
+  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaDevices)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMessagesManager)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mDeviceStorageStores)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTimeManager)
@@ -303,6 +304,7 @@ Navigator::Invalidate()
 #endif
 
   mCameraManager = nullptr;
+  mMediaDevices = nullptr;
 
   if (mMessagesManager) {
     mMessagesManager = nullptr;

dom/base/crashtests/1154598.xhtml

@@ -0,0 +1,9 @@
+<html xmlns="http://www.w3.org/1999/xhtml">
+<body><body>
+<script>
+s = document.createElement('script');
+s.src="";
+document.body.appendChild(s);
+</script>
+</body>
+</html>

dom/base/crashtests/1157995.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <script>
+    // This should not leak.
+    var a = navigator;
+    navigator.mediaDevices._ = null;
+  </script>
+</body>

dom/base/crashtests/1181619.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<body>
+  <script>
+    var obs = new MutationObserver(function() {
+      // Just need something here to assert exception is not pending.  Any
+      // binding method will do.
+      console.log("hello");
+    });
+    obs.observe(document.body, { childList: true });
+  </script>
+  <script>
+    noSuchMethodYo();
+  </script>
+</body>

dom/base/crashtests/crashtests.list

@@ -201,3 +201,6 @@ HTTP(..) load xhr_abortinprogress.html
 load xhr_empty_datauri.html
 load xhr_html_nullresponse.html
 load structured_clone_container_throws.html
+load 1154598.xhtml
+load 1157995.html
+load 1181619.html

dom/base/nsGlobalWindow.cpp

@@ -12153,6 +12153,7 @@ nsGlobalWindow::RunTimeoutHandler(nsTimeout* aTimeout,
 
     // New script entry point required, due to the "Create a script" sub-step of
     // http://www.whatwg.org/specs/web-apps/current-work/#timer-initialisation-steps
+    nsAutoMicroTask mt;
     AutoEntryScript entryScript(this, reason, true, aScx->GetNativeContext());
     entryScript.TakeOwnershipOfErrorReporting();
     JS::CompileOptions options(entryScript.cx());

dom/base/nsImageLoadingContent.cpp

@@ -48,6 +48,7 @@
 #include "mozilla/EventStates.h"
 #include "mozilla/dom/Element.h"
 #include "mozilla/dom/ScriptSettings.h"
+#include "mozilla/Preferences.h"
 
 #ifdef LoadImage
 // Undefine LoadImage to prevent naming conflict with Windows.
@@ -940,14 +941,26 @@ nsImageLoadingContent::LoadImage(nsIURI* aNewURI,
     loadFlags |= imgILoader::LOAD_CORS_USE_CREDENTIALS;
   }
 
+  // get document wide referrer policy
+  mozilla::net::ReferrerPolicy referrerPolicy = aDocument->GetReferrerPolicy();
+  bool referrerAttributeEnabled = Preferences::GetBool("network.http.enablePerElementReferrer", false);
+  // if referrer attributes are enabled in preferences, load img referrer attribute
+  nsresult rv;
+  if (referrerAttributeEnabled) {
+    mozilla::net::ReferrerPolicy imgReferrerPolicy = GetImageReferrerPolicy();
+    // if the image does not provide a referrer attribute, ignore this
+    if (imgReferrerPolicy != mozilla::net::RP_Unset) {
+      referrerPolicy = imgReferrerPolicy;
+    }
+  }
+
   // Not blocked. Do the load.
   nsRefPtr<imgRequestProxy>& req = PrepareNextRequest(aImageLoadType);
   nsIContent* content = AsContent();
-  nsresult rv;
   rv = nsContentUtils::LoadImage(aNewURI, aDocument,
                                  aDocument->NodePrincipal(),
                                  aDocument->GetDocumentURI(),
-                                 aDocument->GetReferrerPolicy(),
+                                 referrerPolicy,
                                  this, loadFlags,
                                  content->LocalName(),
                                  getter_AddRefs(req),
@@ -1586,3 +1599,11 @@ nsImageLoadingContent::ImageObserver::~ImageObserver()
   MOZ_COUNT_DTOR(ImageObserver);
   NS_CONTENT_DELETE_LIST_MEMBER(ImageObserver, this, mNext);
 }
+
+// Only HTMLInputElement.h overrides this for <img> tags
+// all other subclasses use this one, i.e. ignore referrer attributes
+mozilla::net::ReferrerPolicy
+nsImageLoadingContent::GetImageReferrerPolicy()
+{
+  return mozilla::net::RP_Unset;
+};

dom/base/nsImageLoadingContent.h

@@ -25,6 +25,7 @@
 #include "nsAutoPtr.h"
 #include "nsIContentPolicy.h"
 #include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/net/ReferrerPolicy.h"
 
 class nsIURI;
 class nsIDocument;
@@ -203,6 +204,8 @@ protected:
    */
   virtual mozilla::CORSMode GetCORSMode();
 
+  virtual mozilla::net::ReferrerPolicy GetImageReferrerPolicy();
+
   // Subclasses are *required* to call BindToTree/UnbindFromTree.
   void BindToTree(nsIDocument* aDocument, nsIContent* aParent,
                   nsIContent* aBindingParent, bool aCompileEventHandlers);

dom/base/nsJSUtils.cpp

@@ -174,6 +174,8 @@ nsJSUtils::EvaluateString(JSContext* aCx,
   MOZ_ASSERT(js::GetGlobalForObjectCrossCompartment(aEvaluationGlobal) ==
              aEvaluationGlobal);
   MOZ_ASSERT_IF(aOffThreadToken, aCompileOptions.noScriptRval);
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(nsContentUtils::IsInMicroTask());
 
   // Unfortunately, the JS engine actually compiles scripts with a return value
   // in a different, less efficient way.  Furthermore, it can't JIT them in many
@@ -183,7 +185,6 @@ nsJSUtils::EvaluateString(JSContext* aCx,
   // aCompileOptions.noScriptRval set to true.
   aRetValue.setUndefined();
 
-  nsAutoMicroTask mt;
   nsresult rv = NS_OK;
 
   nsIScriptSecurityManager* ssm = nsContentUtils::GetSecurityManager();

dom/base/nsObjectLoadingContent.cpp

@@ -1850,7 +1850,7 @@ nsObjectLoadingContent::UpdateObjectParameters(bool aJavaURI)
       // end up trying to dispatch to a nsFrameLoader, which will complain that
       // it couldn't find a way to handle application/octet-stream
       nsAutoCString parsedMime, dummy;
-      NS_ParseContentType(newMime, parsedMime, dummy);
+      NS_ParseResponseContentType(newMime, parsedMime, dummy);
       if (!parsedMime.IsEmpty()) {
         mChannel->SetContentType(parsedMime);
       }

dom/base/nsScriptLoader.cpp

@@ -51,7 +51,6 @@
 #include "ImportManager.h"
 #include "mozilla/dom/EncodingUtils.h"
 
-#include "mozilla/CORSMode.h"
 #include "mozilla/Attributes.h"
 #include "mozilla/unused.h"
 
@@ -60,70 +59,41 @@ static PRLogModuleInfo* gCspPRLog;
 using namespace mozilla;
 using namespace mozilla::dom;
 
-//////////////////////////////////////////////////////////////
-// Per-request data structure
-//////////////////////////////////////////////////////////////
-
-class nsScriptLoadRequest final : public nsISupports {
-  ~nsScriptLoadRequest()
-  {
-    js_free(mScriptTextBuf);
-  }
-
-public:
-  nsScriptLoadRequest(nsIScriptElement* aElement,
-                      uint32_t aVersion,
-                      CORSMode aCORSMode)
-    : mElement(aElement),
-      mLoading(true),
-      mIsInline(true),
-      mHasSourceMapURL(false),
-      mScriptTextBuf(nullptr),
-      mScriptTextLength(0),
-      mJSVersion(aVersion),
-      mLineNo(1),
-      mCORSMode(aCORSMode),
-      mReferrerPolicy(mozilla::net::RP_Default)
-  {
-  }
-
-  NS_DECL_THREADSAFE_ISUPPORTS
-
-  void FireScriptAvailable(nsresult aResult)
-  {
-    mElement->ScriptAvailable(aResult, mElement, mIsInline, mURI, mLineNo);
-  }
-  void FireScriptEvaluated(nsresult aResult)
-  {
-    mElement->ScriptEvaluated(aResult, mElement, mIsInline);
-  }
-
-  bool IsPreload()
-  {
-    return mElement == nullptr;
-  }
-
-  nsCOMPtr<nsIScriptElement> mElement;
-  bool mLoading;          // Are we still waiting for a load to complete?
-  bool mIsInline;         // Is the script inline or loaded?
-  bool mHasSourceMapURL;  // Does the HTTP header have a source map url?
-  nsString mSourceMapURL; // Holds source map url for loaded scripts
-  char16_t* mScriptTextBuf; // Holds script text for non-inline scripts. Don't
-  size_t mScriptTextLength; // use nsString so we can give ownership to jsapi.
-  uint32_t mJSVersion;
-  nsCOMPtr<nsIURI> mURI;
-  nsCOMPtr<nsIPrincipal> mOriginPrincipal;
-  nsAutoCString mURL;   // Keep the URI's filename alive during off thread parsing.
-  int32_t mLineNo;
-  const CORSMode mCORSMode;
-  mozilla::net::ReferrerPolicy mReferrerPolicy;
-};
-
 // The nsScriptLoadRequest is passed as the context to necko, and thus
 // it needs to be threadsafe. Necko won't do anything with this
 // context, but it will AddRef and Release it on other threads.
 NS_IMPL_ISUPPORTS0(nsScriptLoadRequest)
 
+nsScriptLoadRequestList::~nsScriptLoadRequestList()
+{
+  Clear();
+}
+
+void
+nsScriptLoadRequestList::Clear()
+{
+  while (!isEmpty()) {
+    nsRefPtr<nsScriptLoadRequest> first = StealFirst();
+    first->Cancel();
+    // And just let it go out of scope and die.
+  }
+}
+
+#ifdef DEBUG
+bool
+nsScriptLoadRequestList::Contains(nsScriptLoadRequest* aElem)
+{
+  for (nsScriptLoadRequest* req = getFirst();
+       req; req = req->getNext()) {
+    if (req == aElem) {
+      return true;
+    }
+  }
+
+  return false;
+}
+#endif // DEBUG
+
 //////////////////////////////////////////////////////////////
 //
 //////////////////////////////////////////////////////////////
@@ -149,20 +119,30 @@ nsScriptLoader::~nsScriptLoader()
     mParserBlockingRequest->FireScriptAvailable(NS_ERROR_ABORT);
   }
 
-  for (uint32_t i = 0; i < mXSLTRequests.Length(); i++) {
-    mXSLTRequests[i]->FireScriptAvailable(NS_ERROR_ABORT);
+  for (nsScriptLoadRequest* req = mXSLTRequests.getFirst(); req;
+       req = req->getNext()) {
+    req->FireScriptAvailable(NS_ERROR_ABORT);
   }
 
-  for (uint32_t i = 0; i < mDeferRequests.Length(); i++) {
-    mDeferRequests[i]->FireScriptAvailable(NS_ERROR_ABORT);
+  for (nsScriptLoadRequest* req = mDeferRequests.getFirst(); req;
+       req = req->getNext()) {
+    req->FireScriptAvailable(NS_ERROR_ABORT);
   }
 
-  for (uint32_t i = 0; i < mAsyncRequests.Length(); i++) {
-    mAsyncRequests[i]->FireScriptAvailable(NS_ERROR_ABORT);
+  for (nsScriptLoadRequest* req = mLoadingAsyncRequests.getFirst(); req;
+       req = req->getNext()) {
+    req->FireScriptAvailable(NS_ERROR_ABORT);
   }
 
-  for (uint32_t i = 0; i < mNonAsyncExternalScriptInsertedRequests.Length(); i++) {
-    mNonAsyncExternalScriptInsertedRequests[i]->FireScriptAvailable(NS_ERROR_ABORT);
+  for (nsScriptLoadRequest* req = mLoadedAsyncRequests.getFirst(); req;
+       req = req->getNext()) {
+    req->FireScriptAvailable(NS_ERROR_ABORT);
+  }
+
+  for(nsScriptLoadRequest* req = mNonAsyncExternalScriptInsertedRequests.getFirst();
+      req;
+      req = req->getNext()) {
+    req->FireScriptAvailable(NS_ERROR_ABORT);
   }
 
   // Unblock the kids, in case any of them moved to a different document
@@ -667,11 +647,14 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
     request->mJSVersion = version;
 
     if (aElement->GetScriptAsync()) {
-      mAsyncRequests.AppendElement(request);
+      request->mIsAsync = true;
       if (!request->mLoading) {
+        mLoadedAsyncRequests.AppendElement(request);
         // The script is available already. Run it ASAP when the event
         // loop gets a chance to spin.
         ProcessPendingRequestsAsync();
+      } else {
+        mLoadingAsyncRequests.AppendElement(request);
       }
       return false;
     }
@@ -679,6 +662,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
       // Violate the HTML5 spec in order to make LABjs and the "order" plug-in
       // for RequireJS work with their Gecko-sniffed code path. See
       // http://lists.w3.org/Archives/Public/public-html/2010Oct/0088.html
+      request->mIsNonAsyncScriptInserted = true;
       mNonAsyncExternalScriptInsertedRequests.AppendElement(request);
       if (!request->mLoading) {
         // The script is available already. Run it ASAP when the event
@@ -707,6 +691,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
       // Need to maintain order for XSLT-inserted scripts
       NS_ASSERTION(!mParserBlockingRequest,
           "Parser-blocking scripts and XSLT scripts in the same doc!");
+      request->mIsXSLT = true;
       mXSLTRequests.AppendElement(request);
       if (!request->mLoading) {
         // The script is available already. Run it ASAP when the event
@@ -727,7 +712,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
       // Web page.
       NS_ASSERTION(!mParserBlockingRequest,
           "There can be only one parser-blocking script at a time");
-      NS_ASSERTION(mXSLTRequests.IsEmpty(),
+      NS_ASSERTION(mXSLTRequests.isEmpty(),
           "Parser-blocking scripts and XSLT scripts in the same doc!");
       mParserBlockingRequest = request;
       ProcessPendingRequestsAsync();
@@ -737,7 +722,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
     // The script will be run when it loads or the style sheet loads.
     NS_ASSERTION(!mParserBlockingRequest,
         "There can be only one parser-blocking script at a time");
-    NS_ASSERTION(mXSLTRequests.IsEmpty(),
+    NS_ASSERTION(mXSLTRequests.isEmpty(),
         "Parser-blocking scripts and XSLT scripts in the same doc!");
     mParserBlockingRequest = request;
     return true;
@@ -763,7 +748,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
   request->mLineNo = aElement->GetScriptLineNumber();
 
   if (aElement->GetParserCreated() == FROM_PARSER_XSLT &&
-      (!ReadyToExecuteScripts() || !mXSLTRequests.IsEmpty())) {
+      (!ReadyToExecuteScripts() || !mXSLTRequests.isEmpty())) {
     // Need to maintain order for XSLT-inserted scripts
     NS_ASSERTION(!mParserBlockingRequest,
         "Parser-blocking scripts and XSLT scripts in the same doc!");
@@ -782,7 +767,7 @@ nsScriptLoader::ProcessScriptElement(nsIScriptElement *aElement)
     NS_ASSERTION(!mParserBlockingRequest,
         "There can be only one parser-blocking script at a time");
     mParserBlockingRequest = request;
-    NS_ASSERTION(mXSLTRequests.IsEmpty(),
+    NS_ASSERTION(mXSLTRequests.isEmpty(),
         "Parser-blocking scripts and XSLT scripts in the same doc!");
     return true;
   }
@@ -1148,6 +1133,7 @@ nsScriptLoader::EvaluateScript(nsScriptLoadRequest* aRequest,
 
   // New script entry point required, due to the "Create a script" sub-step of
   // http://www.whatwg.org/specs/web-apps/current-work/#execute-the-script-block
+  nsAutoMicroTask mt;
   AutoEntryScript entryScript(globalObject, "<script> element", true,
                               context->GetNativeContext());
   entryScript.TakeOwnershipOfErrorReporting();
@@ -1207,39 +1193,30 @@ nsScriptLoader::ProcessPendingRequests()
   }
 
   while (ReadyToExecuteScripts() && 
-         !mXSLTRequests.IsEmpty() && 
-         !mXSLTRequests[0]->mLoading) {
-    request.swap(mXSLTRequests[0]);
-    mXSLTRequests.RemoveElementAt(0);
+         !mXSLTRequests.isEmpty() &&
+         !mXSLTRequests.getFirst()->mLoading) {
+    request = mXSLTRequests.StealFirst();
     ProcessRequest(request);
   }
 
-  uint32_t i = 0;
-  while (mEnabled && i < mAsyncRequests.Length()) {
-    if (!mAsyncRequests[i]->mLoading) {
-      request.swap(mAsyncRequests[i]);
-      mAsyncRequests.RemoveElementAt(i);
-      ProcessRequest(request);
-      continue;
-    }
-    ++i;
+  while (mEnabled && !mLoadedAsyncRequests.isEmpty()) {
+    request = mLoadedAsyncRequests.StealFirst();
+    ProcessRequest(request);
   }
 
-  while (mEnabled && !mNonAsyncExternalScriptInsertedRequests.IsEmpty() &&
-         !mNonAsyncExternalScriptInsertedRequests[0]->mLoading) {
+  while (mEnabled && !mNonAsyncExternalScriptInsertedRequests.isEmpty() &&
+         !mNonAsyncExternalScriptInsertedRequests.getFirst()->mLoading) {
     // Violate the HTML5 spec and execute these in the insertion order in
     // order to make LABjs and the "order" plug-in for RequireJS work with
     // their Gecko-sniffed code path. See
     // http://lists.w3.org/Archives/Public/public-html/2010Oct/0088.html
-    request.swap(mNonAsyncExternalScriptInsertedRequests[0]);
-    mNonAsyncExternalScriptInsertedRequests.RemoveElementAt(0);
+    request = mNonAsyncExternalScriptInsertedRequests.StealFirst();
     ProcessRequest(request);
   }
 
-  if (mDocumentParsingDone && mXSLTRequests.IsEmpty()) {
-    while (!mDeferRequests.IsEmpty() && !mDeferRequests[0]->mLoading) {
-      request.swap(mDeferRequests[0]);
-      mDeferRequests.RemoveElementAt(0);
+  if (mDocumentParsingDone && mXSLTRequests.isEmpty()) {
+    while (!mDeferRequests.isEmpty() && !mDeferRequests.getFirst()->mLoading) {
+      request = mDeferRequests.StealFirst();
       ProcessRequest(request);
     }
   }
@@ -1251,9 +1228,10 @@ nsScriptLoader::ProcessPendingRequests()
   }
 
   if (mDocumentParsingDone && mDocument &&
-      !mParserBlockingRequest && mAsyncRequests.IsEmpty() &&
-      mNonAsyncExternalScriptInsertedRequests.IsEmpty() &&
-      mXSLTRequests.IsEmpty() && mDeferRequests.IsEmpty()) {
+      !mParserBlockingRequest && mLoadingAsyncRequests.isEmpty() &&
+      mLoadedAsyncRequests.isEmpty() &&
+      mNonAsyncExternalScriptInsertedRequests.isEmpty() &&
+      mXSLTRequests.isEmpty() && mDeferRequests.isEmpty()) {
     if (MaybeRemovedDeferRequests()) {
       return ProcessPendingRequests();
     }
@@ -1459,11 +1437,27 @@ nsScriptLoader::OnStreamComplete(nsIStreamLoader* aLoader,
       mDocument->AddBlockedTrackingNode(cont);
     }
 
-    if (mDeferRequests.RemoveElement(request) ||
-        mAsyncRequests.RemoveElement(request) ||
-        mNonAsyncExternalScriptInsertedRequests.RemoveElement(request) ||
-        mXSLTRequests.RemoveElement(request)) {
-      FireScriptAvailable(rv, request);
+    if (request->mIsDefer) {
+      if (request->isInList()) {
+        nsRefPtr<nsScriptLoadRequest> req = mDeferRequests.Steal(request);
+        FireScriptAvailable(rv, req);
+      }
+    } else if (request->mIsAsync) {
+      if (request->isInList()) {
+        nsRefPtr<nsScriptLoadRequest> req = mLoadingAsyncRequests.Steal(request);
+        FireScriptAvailable(rv, req);
+      }
+    } else if (request->mIsNonAsyncScriptInserted) {
+      if (request->isInList()) {
+        nsRefPtr<nsScriptLoadRequest> req =
+          mNonAsyncExternalScriptInsertedRequests.Steal(request);
+        FireScriptAvailable(rv, req);
+      }
+    } else if (request->mIsXSLT) {
+      if (request->isInList()) {
+        nsRefPtr<nsScriptLoadRequest> req = mXSLTRequests.Steal(request);
+        FireScriptAvailable(rv, req);
+      }
     } else if (mParserBlockingRequest == request) {
       mParserBlockingRequest = nullptr;
       UnblockParser(request);
@@ -1507,6 +1501,10 @@ nsScriptLoader::PrepareLoadedRequest(nsScriptLoadRequest* aRequest,
     return aStatus;
   }
 
+  if (aRequest->IsCanceled()) {
+    return NS_BINDING_ABORTED;
+  }
+
   // If we don't have a document, then we need to abort further
   // evaluation.
   if (!mDocument) {
@@ -1570,7 +1568,7 @@ nsScriptLoader::PrepareLoadedRequest(nsScriptLoadRequest* aRequest,
   // so if you see this assertion it is likely something else that is
   // wrong, especially if you see it more than once.
   NS_ASSERTION(mDeferRequests.Contains(aRequest) ||
-               mAsyncRequests.Contains(aRequest) ||
+               mLoadingAsyncRequests.Contains(aRequest) ||
                mNonAsyncExternalScriptInsertedRequests.Contains(aRequest) ||
                mXSLTRequests.Contains(aRequest)  ||
                mPreloads.Contains(aRequest, PreloadRequestComparator()) ||
@@ -1580,6 +1578,17 @@ nsScriptLoader::PrepareLoadedRequest(nsScriptLoadRequest* aRequest,
   // Mark this as loaded
   aRequest->mLoading = false;
 
+  // And if it's async, move it to the loaded list.  aRequest->mIsAsync really
+  // _should_ be in a list, but the consequences if it's not are bad enough we
+  // want to avoid trying to move it if it's not.
+  if (aRequest->mIsAsync) {
+    MOZ_ASSERT(aRequest->isInList());
+    if (aRequest->isInList()) {
+      nsRefPtr<nsScriptLoadRequest> req = mLoadingAsyncRequests.Steal(aRequest);
+      mLoadedAsyncRequests.AppendElement(req);
+    }
+  }
+
   return NS_OK;
 }
 
@@ -1594,10 +1603,14 @@ nsScriptLoader::ParsingComplete(bool aTerminated)
   mDeferEnabled = false;
   if (aTerminated) {
     mDeferRequests.Clear();
-    mAsyncRequests.Clear();
+    mLoadingAsyncRequests.Clear();
+    mLoadedAsyncRequests.Clear();
     mNonAsyncExternalScriptInsertedRequests.Clear();
     mXSLTRequests.Clear();
-    mParserBlockingRequest = nullptr;
+    if (mParserBlockingRequest) {
+      mParserBlockingRequest->Cancel();
+      mParserBlockingRequest = nullptr;
+    }
   }
 
   // Have to call this even if aTerminated so we'll correctly unblock
@@ -1638,9 +1651,10 @@ nsScriptLoader::PreloadURI(nsIURI *aURI, const nsAString &aCharset,
 void
 nsScriptLoader::AddDeferRequest(nsScriptLoadRequest* aRequest)
 {
+  aRequest->mIsDefer = true;
   mDeferRequests.AppendElement(aRequest);
-  if (mDeferEnabled && mDeferRequests.Length() == 1 && mDocument &&
-      !mBlockingDOMContentLoaded) {
+  if (mDeferEnabled && aRequest == mDeferRequests.getFirst() &&
+      mDocument && !mBlockingDOMContentLoaded) {
     MOZ_ASSERT(mDocument->GetReadyStateEnum() == nsIDocument::READYSTATE_LOADING);
     mBlockingDOMContentLoaded = true;
     mDocument->BlockDOMContentLoaded();
@@ -1650,7 +1664,7 @@ nsScriptLoader::AddDeferRequest(nsScriptLoadRequest* aRequest)
 bool
 nsScriptLoader::MaybeRemovedDeferRequests()
 {
-  if (mDeferRequests.Length() == 0 && mDocument &&
+  if (mDeferRequests.isEmpty() && mDocument &&
       mBlockingDOMContentLoaded) {
     mBlockingDOMContentLoaded = false;
     mDocument->UnblockDOMContentLoaded();

dom/base/nsScriptLoader.h

@@ -18,9 +18,11 @@
 #include "nsAutoPtr.h"
 #include "nsIDocument.h"
 #include "nsIStreamLoader.h"
+#include "mozilla/CORSMode.h"
+#include "mozilla/LinkedList.h"
 #include "mozilla/net/ReferrerPolicy.h"
 
-class nsScriptLoadRequest;
+class nsScriptLoadRequestList;
 class nsIURI;
 
 namespace JS {
@@ -33,6 +35,134 @@ class AutoJSAPI;
 } // namespace dom
 } // namespace mozilla
 
+//////////////////////////////////////////////////////////////
+// Per-request data structure
+//////////////////////////////////////////////////////////////
+
+class nsScriptLoadRequest final : public nsISupports,
+                                  private mozilla::LinkedListElement<nsScriptLoadRequest>
+{
+  ~nsScriptLoadRequest()
+  {
+    js_free(mScriptTextBuf);
+  }
+
+  typedef LinkedListElement<nsScriptLoadRequest> super;
+
+  // Allow LinkedListElement<nsScriptLoadRequest> to cast us to itself as needed.
+  friend class mozilla::LinkedListElement<nsScriptLoadRequest>;
+  friend class nsScriptLoadRequestList;
+
+public:
+  nsScriptLoadRequest(nsIScriptElement* aElement,
+                      uint32_t aVersion,
+                      mozilla::CORSMode aCORSMode)
+    : mElement(aElement),
+      mLoading(true),
+      mIsInline(true),
+      mHasSourceMapURL(false),
+      mIsDefer(false),
+      mIsAsync(false),
+      mIsNonAsyncScriptInserted(false),
+      mIsXSLT(false),
+      mIsCanceled(false),
+      mScriptTextBuf(nullptr),
+      mScriptTextLength(0),
+      mJSVersion(aVersion),
+      mLineNo(1),
+      mCORSMode(aCORSMode),
+      mReferrerPolicy(mozilla::net::RP_Default)
+  {
+  }
+
+  NS_DECL_THREADSAFE_ISUPPORTS
+
+  void FireScriptAvailable(nsresult aResult)
+  {
+    mElement->ScriptAvailable(aResult, mElement, mIsInline, mURI, mLineNo);
+  }
+  void FireScriptEvaluated(nsresult aResult)
+  {
+    mElement->ScriptEvaluated(aResult, mElement, mIsInline);
+  }
+
+  bool IsPreload()
+  {
+    return mElement == nullptr;
+  }
+
+  void Cancel()
+  {
+    mIsCanceled = true;
+  }
+
+  bool IsCanceled() const
+  {
+    return mIsCanceled;
+  }
+
+  using super::getNext;
+  using super::isInList;
+
+  nsCOMPtr<nsIScriptElement> mElement;
+  bool mLoading;          // Are we still waiting for a load to complete?
+  bool mIsInline;         // Is the script inline or loaded?
+  bool mHasSourceMapURL;  // Does the HTTP header have a source map url?
+  bool mIsDefer;          // True if we live in mDeferRequests.
+  bool mIsAsync;          // True if we live in mLoadingAsyncRequests or mLoadedAsyncRequests.
+  bool mIsNonAsyncScriptInserted; // True if we live in mNonAsyncExternalScriptInsertedRequests
+  bool mIsXSLT;           // True if we live in mXSLTRequests.
+  bool mIsCanceled;       // True if we have been explicitly canceled.
+  nsString mSourceMapURL; // Holds source map url for loaded scripts
+  char16_t* mScriptTextBuf; // Holds script text for non-inline scripts. Don't
+  size_t mScriptTextLength; // use nsString so we can give ownership to jsapi.
+  uint32_t mJSVersion;
+  nsCOMPtr<nsIURI> mURI;
+  nsCOMPtr<nsIPrincipal> mOriginPrincipal;
+  nsAutoCString mURL;   // Keep the URI's filename alive during off thread parsing.
+  int32_t mLineNo;
+  const mozilla::CORSMode mCORSMode;
+  mozilla::net::ReferrerPolicy mReferrerPolicy;
+};
+
+class nsScriptLoadRequestList : private mozilla::LinkedList<nsScriptLoadRequest>
+{
+  typedef mozilla::LinkedList<nsScriptLoadRequest> super;
+
+public:
+  ~nsScriptLoadRequestList();
+
+  void Clear();
+
+#ifdef DEBUG
+  bool Contains(nsScriptLoadRequest* aElem);
+#endif // DEBUG
+
+  using super::getFirst;
+  using super::isEmpty;
+
+  void AppendElement(nsScriptLoadRequest* aElem)
+  {
+    MOZ_ASSERT(!aElem->isInList());
+    NS_ADDREF(aElem);
+    insertBack(aElem);
+  }
+
+  MOZ_WARN_UNUSED_RESULT
+  already_AddRefed<nsScriptLoadRequest> Steal(nsScriptLoadRequest* aElem)
+  {
+    aElem->removeFrom(*this);
+    return dont_AddRef(aElem);
+  }
+
+  MOZ_WARN_UNUSED_RESULT
+  already_AddRefed<nsScriptLoadRequest> StealFirst()
+  {
+    MOZ_ASSERT(!isEmpty());
+    return Steal(getFirst());
+  }
+};
+
 //////////////////////////////////////////////////////////////
 // Script loader implementation
 //////////////////////////////////////////////////////////////
@@ -337,10 +467,13 @@ private:
 
   nsIDocument* mDocument;                   // [WEAK]
   nsCOMArray<nsIScriptLoaderObserver> mObservers;
-  nsTArray<nsRefPtr<nsScriptLoadRequest> > mNonAsyncExternalScriptInsertedRequests;
-  nsTArray<nsRefPtr<nsScriptLoadRequest> > mAsyncRequests;
-  nsTArray<nsRefPtr<nsScriptLoadRequest> > mDeferRequests;
-  nsTArray<nsRefPtr<nsScriptLoadRequest> > mXSLTRequests;
+  nsScriptLoadRequestList mNonAsyncExternalScriptInsertedRequests;
+  // mLoadingAsyncRequests holds async requests while they're loading; when they
+  // have been loaded they are moved to mLoadedAsyncRequests.
+  nsScriptLoadRequestList mLoadingAsyncRequests;
+  nsScriptLoadRequestList mLoadedAsyncRequests;
+  nsScriptLoadRequestList mDeferRequests;
+  nsScriptLoadRequestList mXSLTRequests;
   nsRefPtr<nsScriptLoadRequest> mParserBlockingRequest;
 
   // In mRequests, the additional information here is stored by the element.

dom/html/HTMLAnchorElement.h

@@ -123,6 +123,14 @@ public:
   {
     SetHTMLAttr(nsGkAtoms::rel, aValue, rv);
   }
+  void SetReferrer(const nsAString& aValue, mozilla::ErrorResult& rv)
+  {
+    SetHTMLAttr(nsGkAtoms::referrer, aValue, rv);
+  }
+  void GetReferrer(nsAString& aReferrer)
+  {
+    GetHTMLAttr(nsGkAtoms::referrer, aReferrer);
+  }
   nsDOMTokenList* RelList();
   void GetHreflang(DOMString& aValue)
   {

dom/html/HTMLAreaElement.h

@@ -128,6 +128,15 @@ public:
   } 
   nsDOMTokenList* RelList();
 
+  void SetReferrer(const nsAString& aValue, mozilla::ErrorResult& rv)
+  {
+    SetHTMLAttr(nsGkAtoms::referrer, aValue, rv);
+  }
+  void GetReferrer(nsAString& aReferrer)
+  {
+    GetHTMLAttr(nsGkAtoms::referrer, aReferrer);
+  }
+
   // The Link::GetOrigin is OK for us
 
   using Link::GetProtocol;

dom/html/HTMLImageElement.h

@@ -189,6 +189,20 @@ public:
   {
     SetHTMLAttr(nsGkAtoms::border, aBorder, aError);
   }
+  void SetReferrer(const nsAString& aReferrer, ErrorResult& aError)
+  {
+    SetHTMLAttr(nsGkAtoms::referrer, aReferrer, aError);
+  }
+  void GetReferrer(nsAString& aReferrer)
+  {
+    GetEnumAttr(nsGkAtoms::referrer, nullptr, aReferrer);
+  }
+
+  mozilla::net::ReferrerPolicy
+  GetImageReferrerPolicy()
+  {
+    return GetReferrerPolicy();
+  }
 
   int32_t X();
   int32_t Y();

dom/html/nsGenericHTMLElement.cpp

@@ -108,6 +108,8 @@
 #include "imgIContainer.h"
 #include "nsComputedDOMStyle.h"
 
+#include "mozilla/net/ReferrerPolicy.h"
+
 using namespace mozilla;
 using namespace mozilla::dom;
 
@@ -1004,6 +1006,10 @@ nsGenericHTMLElement::ParseAttribute(int32_t aNamespaceID,
       return aResult.ParseIntValue(aValue);
     }
 
+    if (aAttribute == nsGkAtoms::referrer) {
+      return ParseReferrerAttribute(aValue, aResult);
+    }
+
     if (aAttribute == nsGkAtoms::name) {
       // Store name as an atom.  name="" means that the element has no name,
       // not that it has an emptystring as the name.
@@ -1271,6 +1277,19 @@ nsGenericHTMLElement::ParseImageAttribute(nsIAtom* aAttribute,
   return false;
 }
 
+bool
+nsGenericHTMLElement::ParseReferrerAttribute(const nsAString& aString,
+                                             nsAttrValue& aResult)
+{
+  static const nsAttrValue::EnumTable kReferrerTable[] = {
+    { "no-referrer", net::RP_No_Referrer },
+    { "origin", net::RP_Origin },
+    { "unsafe-url", net::RP_Unsafe_URL },
+    { 0 }
+  };
+  return aResult.ParseEnumValue(aString, kReferrerTable, false);
+}
+
 bool
 nsGenericHTMLElement::ParseFrameborderValue(const nsAString& aString,
                                             nsAttrValue& aResult)

dom/html/nsGenericHTMLElement.h

@@ -243,6 +243,17 @@ public:
   void GetInnerText(mozilla::dom::DOMString& aValue, mozilla::ErrorResult& aError);
   void SetInnerText(const nsAString& aValue);
 
+  mozilla::net::ReferrerPolicy
+  GetReferrerPolicy()
+  {
+    nsAutoString aPolicyString;
+    GetEnumAttr(nsGkAtoms::referrer, nullptr, aPolicyString);
+    if (aPolicyString.IsEmpty()) {
+      return mozilla::net::RP_Unset;
+    }
+    return mozilla::net::ReferrerPolicyFromString(aPolicyString);
+  }
+
   /**
    * Determine whether an attribute is an event (onclick, etc.)
    * @param aName the attribute
@@ -721,6 +732,10 @@ public:
   static bool ParseImageAttribute(nsIAtom* aAttribute,
                                     const nsAString& aString,
                                     nsAttrValue& aResult);
+
+  static bool ParseReferrerAttribute(const nsAString& aString,
+                                     nsAttrValue& aResult);
+
   /**
    * Convert a frameborder string to value (yes/no/1/0)
    *

dom/jsurl/nsJSProtocolHandler.cpp

@@ -250,6 +250,7 @@ nsresult nsJSThunk::EvaluateScript(nsIChannel *aChannel,
 
     // New script entry point required, due to the "Create a script" step of
     // http://www.whatwg.org/specs/web-apps/current-work/#javascript-protocol
+    nsAutoMicroTask mt;
     AutoEntryScript entryScript(innerGlobal, "javascript: URI", true,
                                 scriptContext->GetNativeContext());
     // We want to make sure we report any exceptions that happen before we

dom/manifest/ImageObjectProcessor.jsm

@@ -92,7 +92,7 @@ ImageObjectProcessor.prototype.process = function(
     };
     let value = extractor.extractValue(spec);
     if (value) {
-      value = netutil.parseContentType(value, charset, hadCharset);
+      value = netutil.parseRequestContentType(value, charset, hadCharset);
     }
     return value || undefined;
   }

dom/plugins/base/nsNPAPIPlugin.cpp

@@ -1504,6 +1504,7 @@ _evaluate(NPP npp, NPObject* npobj, NPString *script, NPVariant *result)
     return false;
   }
 
+  nsAutoMicroTask mt;
   dom::AutoEntryScript aes(win, "NPAPI NPN_evaluate");
   aes.TakeOwnershipOfErrorReporting();
   JSContext* cx = aes.cx();

dom/webidl/HTMLAnchorElement.webidl

@@ -21,6 +21,8 @@ interface HTMLAnchorElement : HTMLElement {
            attribute DOMString ping;
            [SetterThrows]
            attribute DOMString rel;
+           [SetterThrows, Pref="network.http.enablePerElementReferrer"]
+           attribute DOMString referrer;
   readonly attribute DOMTokenList relList;
            [SetterThrows]
            attribute DOMString hreflang;

dom/webidl/HTMLAreaElement.webidl

@@ -28,6 +28,8 @@ interface HTMLAreaElement : HTMLElement {
            attribute DOMString ping;
            [SetterThrows]
            attribute DOMString rel;
+           [SetterThrows, Pref="network.http.enablePerElementReferrer"]
+           attribute DOMString referrer;
   readonly attribute DOMTokenList relList;
 
   // not implemented.

dom/webidl/HTMLImageElement.webidl

@@ -30,6 +30,8 @@ interface HTMLImageElement : HTMLElement {
            [SetterThrows]
            attribute DOMString useMap;
            [SetterThrows]
+           attribute DOMString referrer;
+           [SetterThrows]
            attribute boolean isMap;
            [SetterThrows]
            attribute unsigned long width;

image/imgLoader.cpp

@@ -666,6 +666,9 @@ ValidateSecurityInfo(imgRequest* request, bool forcePrincipalCheck,
                      nsISupports* aCX, ReferrerPolicy referrerPolicy)
 {
   // If the entry's Referrer Policy doesn't match, we can't use this request.
+  // XXX: this will return false if an image has different referrer attributes,
+  // i.e. we currently don't use the cached image but reload the image with
+  // the new referrer policy bug 1174921
   if (referrerPolicy != request->GetReferrerPolicy()) {
     return false;
   }

js/src/gc/GCRuntime.h

@@ -1086,6 +1086,9 @@ class GCRuntime
     /* The invocation kind of the current GC, taken from the first slice. */
     JSGCInvocationKind invocationKind;
 
+    /* The initial GC reason, taken from the first slice. */
+    JS::gcreason::Reason initialReason;
+
     /*
      * If this is 0, all cross-compartment proxies must be registered in the
      * wrapper map. This checking must be disabled temporarily while creating

js/src/jit-test/tests/gc/bug-1177778.js

@@ -0,0 +1,16 @@
+if (!("setGCCallback" in this))
+    quit();
+
+setGCCallback({
+    action: "majorGC",
+    phases: "both"
+});
+var g = newGlobal();
+var dbg = new Debugger;
+var gw = dbg.addDebuggee(g);
+g.eval("function h() { debugger; }");
+dbg.onDebuggerStatement = function(hframe) {
+    var env = hframe.older.environment;
+};
+g.eval("h();");
+

js/src/jsgc.cpp

@@ -1337,6 +1337,7 @@ GCRuntime::finish()
 
     /* Delete all remaining zones. */
     if (rt->gcInitialized) {
+        AutoSetThreadIsSweeping threadIsSweeping;
         for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
             for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
                 js_delete(comp.get());
@@ -2115,12 +2116,18 @@ ArenaList::relocateArenas(ArenaHeader *toRelocate, ArenaHeader *relocated, Slice
 // heap memory.
 static const double MIN_ZONE_RECLAIM_PERCENT = 2.0;
 
+static bool isOOMReason(JS::gcreason::Reason reason)
+{
+    return reason == JS::gcreason::LAST_DITCH ||
+           reason == JS::gcreason::MEM_PRESSURE;
+}
+
 static bool ShouldRelocateZone(size_t arenaCount, size_t relocCount, JS::gcreason::Reason reason)
 {
     if (relocCount == 0)
         return false;
 
-    if (reason == JS::gcreason::MEM_PRESSURE || reason == JS::gcreason::LAST_DITCH)
+    if (isOOMReason(reason))
         return true;
 
     return (relocCount * 100.0) / arenaCount >= MIN_ZONE_RECLAIM_PERCENT;
@@ -4053,6 +4060,25 @@ GCRuntime::markAllGrayReferences(gcstats::Phase phase)
 
 #ifdef DEBUG
 
+struct GCChunkHasher {
+    typedef gc::Chunk* Lookup;
+
+    /*
+     * Strip zeros for better distribution after multiplying by the golden
+     * ratio.
+     */
+    static HashNumber hash(gc::Chunk* chunk) {
+        MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
+        return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
+    }
+
+    static bool match(gc::Chunk* k, gc::Chunk* l) {
+        MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
+        MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
+        return k == l;
+    }
+};
+
 class js::gc::MarkingValidator
 {
   public:
@@ -5487,10 +5513,7 @@ GCRuntime::finishCollection(JS::gcreason::Reason reason)
     // before returning to ensure that we free as much as possible. If this is
     // a zeal-triggered GC, we want to ensure that the mutator can continue
     // allocating on the same pages to reduce fragmentation.
-    if (reason == JS::gcreason::LAST_DITCH ||
-        reason == JS::gcreason::MEM_PRESSURE ||
-        reason == JS::gcreason::DEBUG_GC)
-    {
+    if (isOOMReason(reason) || reason == JS::gcreason::DEBUG_GC) {
         gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
         rt->gc.waitBackgroundSweepOrAllocEnd();
     }
@@ -5774,6 +5797,7 @@ GCRuntime::incrementalCollectSlice(SliceBudget& budget, JS::gcreason::Reason rea
 
     switch (incrementalState) {
       case NO_INCREMENTAL:
+        initialReason = reason;
         cleanUpEverything = ShouldCleanUpEverything(reason, invocationKind);
         isCompacting = shouldCompact();
         lastMarkSlice = false;
@@ -6226,6 +6250,19 @@ void
 GCRuntime::finishGC(JS::gcreason::Reason reason)
 {
     MOZ_ASSERT(isIncrementalGCInProgress());
+
+    // If we're not collecting because we're out of memory then skip the
+    // compacting phase if we need to finish an ongoing incremental GC
+    // non-incrementally to avoid janking the browser.
+    if (!isOOMReason(initialReason)) {
+        if (incrementalState == COMPACT) {
+            abortGC();
+            return;
+        }
+
+        isCompacting = false;
+    }
+
     collect(true, SliceBudget(), reason);
 }
 
@@ -7359,14 +7396,14 @@ DummyGetter(JSContext* cx, unsigned argc, Value* vp)
 
 } /* namespace MemInfo */
 
-JSObject *
-NewMemoryInfoObject(JSContext *cx)
+JSObject*
+NewMemoryInfoObject(JSContext* cx)
 {
     RootedObject obj(cx, JS_NewObject(cx, nullptr));
 
     using namespace MemInfo;
-    struct {
-        const char *name;
+    struct NamedGetter {
+        const char* name;
         JSNative getter;
     } getters[] = {
         { "gcBytes", GCBytesGetter },
@@ -7379,13 +7416,13 @@ NewMemoryInfoObject(JSContext *cx)
         { "minorGCCount", MinorGCCountGetter }
     };
 
-    for (size_t i = 0; i < mozilla::ArrayLength(getters); i++) {
- #ifdef JS_MORE_DETERMINISTIC
+    for (auto pair : getters) {
+#ifdef JS_MORE_DETERMINISTIC
         JSNative getter = DummyGetter;
 #else
-        JSNative getter = getters[i].getter;
+        JSNative getter = pair.getter;
 #endif
-        if (!JS_DefineProperty(cx, obj, getters[i].name, UndefinedHandleValue,
+        if (!JS_DefineProperty(cx, obj, pair.name, UndefinedHandleValue,
                                JSPROP_ENUMERATE | JSPROP_SHARED,
                                getter, nullptr))
         {
@@ -7400,8 +7437,8 @@ NewMemoryInfoObject(JSContext *cx)
     if (!JS_DefineProperty(cx, obj, "zone", zoneObj, JSPROP_ENUMERATE))
         return nullptr;
 
-    struct {
-        const char *name;
+    struct NamedZoneGetter {
+        const char* name;
         JSNative getter;
     } zoneGetters[] = {
         { "gcBytes", ZoneGCBytesGetter },
@@ -7414,13 +7451,13 @@ NewMemoryInfoObject(JSContext *cx)
         { "gcNumber", ZoneGCNumberGetter }
     };
 
-    for (size_t i = 0; i < mozilla::ArrayLength(zoneGetters); i++) {
+    for (auto pair : zoneGetters) {
  #ifdef JS_MORE_DETERMINISTIC
         JSNative getter = DummyGetter;
 #else
-        JSNative getter = zoneGetters[i].getter;
+        JSNative getter = pair.getter;
 #endif
-        if (!JS_DefineProperty(cx, zoneObj, zoneGetters[i].name, UndefinedHandleValue,
+        if (!JS_DefineProperty(cx, zoneObj, pair.name, UndefinedHandleValue,
                                JSPROP_ENUMERATE | JSPROP_SHARED,
                                getter, nullptr))
         {

js/src/jsgc.h

@@ -1032,27 +1032,6 @@ class GCParallelTask
     virtual void runFromHelperThread();
 };
 
-struct GCChunkHasher {
-    typedef gc::Chunk* Lookup;
-
-    /*
-     * Strip zeros for better distribution after multiplying by the golden
-     * ratio.
-     */
-    static HashNumber hash(gc::Chunk* chunk) {
-        MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
-        return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
-    }
-
-    static bool match(gc::Chunk* k, gc::Chunk* l) {
-        MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
-        MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
-        return k == l;
-    }
-};
-
-typedef HashSet<js::gc::Chunk*, GCChunkHasher, SystemAllocPolicy> GCChunkSet;
-
 typedef void (*IterateChunkCallback)(JSRuntime* rt, void* data, gc::Chunk* chunk);
 typedef void (*IterateZoneCallback)(JSRuntime* rt, void* data, JS::Zone* zone);
 typedef void (*IterateArenaCallback)(JSRuntime* rt, void* data, gc::Arena* arena,

modules/libjar/nsJARChannel.cpp

@@ -765,7 +765,7 @@ nsJARChannel::SetContentType(const nsACString &aContentType)
     // doing our guessing.  So we don't care when this is being called.
 
     // mContentCharset is unchanged if not parsed
-    NS_ParseContentType(aContentType, mContentType, mContentCharset);
+    NS_ParseResponseContentType(aContentType, mContentType, mContentCharset);
     return NS_OK;
 }
 
@@ -1134,7 +1134,7 @@ nsJARChannel::OnDownloadComplete(MemoryDownloader* aDownloader,
                                            header);
             nsAutoCString contentType;
             nsAutoCString charset;
-            NS_ParseContentType(header, contentType, charset);
+            NS_ParseResponseContentType(header, contentType, charset);
             nsAutoCString channelContentType;
             channel->GetContentType(channelContentType);
             mIsUnsafe = !(contentType.Equals(channelContentType) &&

modules/libpref/init/all.js

@@ -1572,6 +1572,9 @@ pref("network.http.referer.XOriginPolicy", 0);
 // By default this is enabled for compatibility (see bug 141641)
 pref("network.http.sendSecureXSiteReferrer", true);
 
+// Controls whether referrer attributes in <a>, <img>, <area>, and <iframe> are honoured
+pref("network.http.enablePerElementReferrer", false);
+
 // Maximum number of consecutive redirects before aborting.
 pref("network.http.redirection-limit", 20);
 
@@ -4800,11 +4803,19 @@ pref("ui.touch_activation.duration_ms", 10);
 // actions when the fifo is written to.  Disable this in general.
 pref("memory_info_dumper.watch_fifo.enabled", false);
 
-#ifdef MOZ_CAPTIVEDETECT
+// If minInterval is 0, the check will only happen
+// when the service has a strong suspicion we are in a captive portal
+pref("network.captive-portal-service.minInterval", 60000); // 60 seconds
+pref("network.captive-portal-service.maxInterval", 1500000); // 25 minutes
+// Every 10 checks, the delay is increased by a factor of 5
+pref("network.captive-portal-service.backoffFactor", "5.0");
+pref("network.captive-portal-service.enabled", false);
+
+pref("captivedetect.canonicalURL", "http://detectportal.firefox.com/success.txt");
+pref("captivedetect.canonicalContent", "success\n");
 pref("captivedetect.maxWaitingTime", 5000);
 pref("captivedetect.pollingTime", 3000);
 pref("captivedetect.maxRetryCount", 5);
-#endif
 
 #ifdef RELEASE_BUILD
 pref("dom.forms.inputmode", false);

netwerk/base/CaptivePortalService.cpp

@@ -0,0 +1,311 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "CaptivePortalService.h"
+#include "mozilla/Services.h"
+#include "mozilla/Preferences.h"
+#include "nsIObserverService.h"
+#include "nsXULAppAPI.h"
+
+#define kInterfaceName "captive-portal-inteface"
+
+static const char kOpenCaptivePortalLoginEvent[] = "captive-portal-login";
+static const char kAbortCaptivePortalLoginEvent[] = "captive-portal-login-abort";
+static const char kCaptivePortalLoginSuccessEvent[] = "captive-portal-login-success";
+
+static const uint32_t kDefaultInterval = 60*1000; // check every 60 seconds
+
+static PRLogModuleInfo *gCaptivePortalLog = nullptr;
+#undef LOG
+#define LOG(args) MOZ_LOG(gCaptivePortalLog, mozilla::LogLevel::Debug, args)
+
+namespace mozilla {
+namespace net {
+
+NS_IMPL_ISUPPORTS(CaptivePortalService, nsICaptivePortalService, nsIObserver,
+                  nsISupportsWeakReference, nsITimerCallback,
+                  nsICaptivePortalCallback)
+
+CaptivePortalService::CaptivePortalService()
+  : mState(UNKNOWN)
+  , mStarted(false)
+  , mInitialized(false)
+  , mRequestInProgress(false)
+  , mEverBeenCaptive(false)
+  , mDelay(kDefaultInterval)
+  , mSlackCount(0)
+  , mMinInterval(kDefaultInterval)
+  , mMaxInterval(25*kDefaultInterval)
+  , mBackoffFactor(5.0)
+{
+  mLastChecked = TimeStamp::Now();
+}
+
+CaptivePortalService::~CaptivePortalService()
+{
+}
+
+nsresult
+CaptivePortalService::PerformCheck()
+{
+  LOG(("CaptivePortalService::PerformCheck mRequestInProgress:%d mInitialized:%d mStarted:%d\n",
+        mRequestInProgress, mInitialized, mStarted));
+  // Don't issue another request if last one didn't complete
+  if (mRequestInProgress || !mInitialized || !mStarted) {
+    return NS_OK;
+  }
+
+  nsresult rv;
+  if (!mCaptivePortalDetector) {
+    mCaptivePortalDetector =
+      do_GetService("@mozilla.org/toolkit/captive-detector;1", &rv);
+    if (NS_FAILED(rv)) {
+        LOG(("Unable to get a captive portal detector\n"));
+        return rv;
+    }
+  }
+
+  LOG(("CaptivePortalService::PerformCheck - Calling CheckCaptivePortal\n"));
+  mRequestInProgress = true;
+  mCaptivePortalDetector->CheckCaptivePortal(
+    NS_LITERAL_STRING(kInterfaceName).get(), this);
+  return NS_OK;
+}
+
+nsresult
+CaptivePortalService::RearmTimer()
+{
+  // Start a timer to recheck
+  if (mTimer) {
+    mTimer->Cancel();
+  }
+
+  if (!mTimer) {
+    mTimer = do_CreateInstance(NS_TIMER_CONTRACTID);
+  }
+
+  if (mTimer && mDelay > 0) {
+    LOG(("CaptivePortalService - Reloading timer with delay %u\n", mDelay));
+    return mTimer->InitWithCallback(this, mDelay, nsITimer::TYPE_ONE_SHOT);
+  }
+
+  return NS_OK;
+}
+
+nsresult
+CaptivePortalService::Initialize()
+{
+  if (mInitialized || XRE_GetProcessType() != GeckoProcessType_Default) {
+    return NS_OK;
+  }
+  mInitialized = true;
+
+  if (!gCaptivePortalLog) {
+    gCaptivePortalLog = PR_NewLogModule("CaptivePortalService");
+  }
+
+  nsCOMPtr<nsIObserverService> observerService =
+    mozilla::services::GetObserverService();
+  if (observerService) {
+    observerService->AddObserver(this, kOpenCaptivePortalLoginEvent, true);
+    observerService->AddObserver(this, kAbortCaptivePortalLoginEvent, true);
+    observerService->AddObserver(this, kCaptivePortalLoginSuccessEvent, true);
+  }
+
+  LOG(("Initialized CaptivePortalService\n"));
+  return NS_OK;
+}
+
+nsresult
+CaptivePortalService::Start()
+{
+  if (!mInitialized) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  if (mStarted) {
+    return NS_OK;
+  }
+
+  mStarted = true;
+  mEverBeenCaptive = false;
+
+  // Get the delay prefs
+  Preferences::GetUint("network.captive-portal-service.minInterval", &mMinInterval);
+  Preferences::GetUint("network.captive-portal-service.maxInterval", &mMaxInterval);
+  Preferences::GetFloat("network.captive-portal-service.backoffFactor", &mBackoffFactor);
+
+  LOG(("CaptivePortalService::Start min:%u max:%u backoff:%.2f\n",
+       mMinInterval, mMaxInterval, mBackoffFactor));
+
+  mSlackCount = 0;
+  mDelay = mMinInterval;
+
+  // When Start is called, perform a check immediately
+  PerformCheck();
+  RearmTimer();
+  return NS_OK;
+}
+
+nsresult
+CaptivePortalService::Stop()
+{
+  LOG(("CaptivePortalService::Stop\n"));
+
+  if (!mStarted) {
+    return NS_OK;
+  }
+
+  if (mTimer) {
+    mTimer->Cancel();
+  }
+  mTimer = nullptr;
+  mRequestInProgress = false;
+  mStarted = false;
+  if (mCaptivePortalDetector) {
+    mCaptivePortalDetector->Abort(NS_LITERAL_STRING(kInterfaceName).get());
+  }
+  mCaptivePortalDetector = nullptr;
+  return NS_OK;
+}
+
+//-----------------------------------------------------------------------------
+// CaptivePortalService::nsICaptivePortalService
+//-----------------------------------------------------------------------------
+
+NS_IMETHODIMP
+CaptivePortalService::GetState(int32_t *aState)
+{
+  *aState = UNKNOWN;
+  if (!mInitialized) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  *aState = mState;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+CaptivePortalService::RecheckCaptivePortal()
+{
+  LOG(("CaptivePortalService::RecheckCaptivePortal\n"));
+
+  // This is called for user activity. We need to reset the slack count,
+  // so the checks continue to be quite frequent.
+  mSlackCount = 0;
+  mDelay = mMinInterval;
+
+  PerformCheck();
+  RearmTimer();
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+CaptivePortalService::GetLastChecked(uint64_t *aLastChecked)
+{
+  double duration = (TimeStamp::Now() - mLastChecked).ToMilliseconds();
+  *aLastChecked = static_cast<uint64_t>(duration);
+  return NS_OK;
+}
+
+//-----------------------------------------------------------------------------
+// CaptivePortalService::nsITimer
+// This callback gets called every mDelay miliseconds
+// It issues a checkCaptivePortal operation if one isn't already in progress
+//-----------------------------------------------------------------------------
+NS_IMETHODIMP
+CaptivePortalService::Notify(nsITimer *aTimer)
+{
+  LOG(("CaptivePortalService::Notify\n"));
+  MOZ_ASSERT(aTimer == mTimer);
+
+  PerformCheck();
+
+  // This is needed because we don't want to always make requests very often.
+  // Every 10 checks, we the delay is increased mBackoffFactor times
+  // to a maximum delay of mMaxInterval
+  mSlackCount++;
+  if (mSlackCount % 10 == 0) {
+    mDelay = mDelay * mBackoffFactor;
+  }
+  if (mDelay > mMaxInterval) {
+    mDelay = mMaxInterval;
+  }
+
+  // Note - if mDelay is 0, the timer will not be rearmed.
+  RearmTimer();
+
+  return NS_OK;
+}
+
+//-----------------------------------------------------------------------------
+// CaptivePortalService::nsIObserver
+//-----------------------------------------------------------------------------
+NS_IMETHODIMP
+CaptivePortalService::Observe(nsISupports *aSubject,
+                              const char * aTopic,
+                              const char16_t * aData)
+{
+  LOG(("CaptivePortalService::Observe() topic=%s\n", aTopic));
+  if (!strcmp(aTopic, kOpenCaptivePortalLoginEvent)) {
+    // A redirect or altered content has been detected.
+    // The user needs to log in. We are in a captive portal.
+    mState = LOCKED_PORTAL;
+    mLastChecked = TimeStamp::Now();
+    mEverBeenCaptive = true;
+  } else if (!strcmp(aTopic, kCaptivePortalLoginSuccessEvent)) {
+    // The user has successfully logged in. We have connectivity.
+    mState = UNLOCKED_PORTAL;
+    mLastChecked = TimeStamp::Now();
+    mRequestInProgress = false;
+    mSlackCount = 0;
+    mDelay = mMinInterval;
+    RearmTimer();
+  } else if (!strcmp(aTopic, kAbortCaptivePortalLoginEvent)) {
+    // The login has been aborted
+    mRequestInProgress = false;
+    mState = UNKNOWN;
+    mLastChecked = TimeStamp::Now();
+    mSlackCount = 0;
+  }
+  return NS_OK;
+}
+
+//-----------------------------------------------------------------------------
+// CaptivePortalService::nsICaptivePortalCallback
+//-----------------------------------------------------------------------------
+NS_IMETHODIMP
+CaptivePortalService::Prepare()
+{
+  LOG(("CaptivePortalService::Prepare\n"));
+  // XXX: Finish preparation shouldn't be called until dns and routing is available.
+  if (mCaptivePortalDetector) {
+    mCaptivePortalDetector->FinishPreparation(NS_LITERAL_STRING(kInterfaceName).get());
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+CaptivePortalService::Complete(bool success)
+{
+  LOG(("CaptivePortalService::Complete(success=%d) mState=%d\n", success, mState));
+  mLastChecked = TimeStamp::Now();
+  if ((mState == UNKNOWN || mState == NOT_CAPTIVE) && success) {
+    mState = NOT_CAPTIVE;
+    // If this check succeeded and we have never been in a captive portal
+    // since the service was started, there is no need to keep polling
+    if (!mEverBeenCaptive) {
+      mDelay = 0;
+      if (mTimer) {
+        mTimer->Cancel();
+      }
+    }
+  }
+
+  mRequestInProgress = false;
+  return NS_OK;
+}
+
+} // namespace net
+} // namespace mozilla

netwerk/base/CaptivePortalService.h

@@ -0,0 +1,66 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef CaptivePortalService_h_
+#define CaptivePortalService_h_
+
+#include "nsICaptivePortalService.h"
+#include "nsICaptivePortalDetector.h"
+#include "nsIObserver.h"
+#include "nsWeakReference.h"
+#include "nsITimer.h"
+#include "nsCOMArray.h"
+#include "mozilla/TimeStamp.h"
+
+namespace mozilla {
+namespace net {
+
+class CaptivePortalService
+  : public nsICaptivePortalService
+  , public nsIObserver
+  , public nsSupportsWeakReference
+  , public nsITimerCallback
+  , public nsICaptivePortalCallback
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSICAPTIVEPORTALSERVICE
+  NS_DECL_NSIOBSERVER
+  NS_DECL_NSITIMERCALLBACK
+  NS_DECL_NSICAPTIVEPORTALCALLBACK
+
+  CaptivePortalService();
+  nsresult Initialize();
+  nsresult Start();
+  nsresult Stop();
+private:
+  virtual ~CaptivePortalService();
+  nsresult PerformCheck();
+  nsresult RearmTimer();
+
+  nsCOMPtr<nsICaptivePortalDetector>    mCaptivePortalDetector;
+  int32_t                               mState;
+
+  nsCOMPtr<nsITimer>                    mTimer;
+  bool                                  mStarted;
+  bool                                  mInitialized;
+  bool                                  mRequestInProgress;
+  bool                                  mEverBeenCaptive;
+
+  uint32_t                              mDelay;
+  int32_t                               mSlackCount;
+
+  uint32_t                              mMinInterval;
+  uint32_t                              mMaxInterval;
+  float                                 mBackoffFactor;
+
+  // This holds a timestamp when the last time when the captive portal check
+  // has changed state.
+  mozilla::TimeStamp                    mLastChecked;
+};
+
+} // namespace net
+} // namespace mozilla
+
+#endif // CaptivePortalService_h_

netwerk/base/ReferrerPolicy.h

@@ -25,31 +25,53 @@ enum ReferrerPolicy {
   RP_Origin_When_Crossorigin     = nsIHttpChannel::REFERRER_POLICY_ORIGIN_WHEN_XORIGIN,
 
   /* spec tokens: always unsafe-url */
-  RP_Unsafe_URL                  = nsIHttpChannel::REFERRER_POLICY_UNSAFE_URL
+  RP_Unsafe_URL                  = nsIHttpChannel::REFERRER_POLICY_UNSAFE_URL,
+
+  /* referrer policy is not set */
+  RP_Unset                       = nsIHttpChannel::REFERRER_POLICY_NO_REFERRER_WHEN_DOWNGRADE
 };
 
+/* spec tokens: never no-referrer */
+const char kRPS_Never[]                       = "never";
+const char kRPS_No_Referrer[]                 = "no-referrer";
+
+/* spec tokens: origin */
+const char kRPS_Origin[]                      = "origin";
+
+/* spec tokens: default no-referrer-when-downgrade */
+const char kRPS_Default[]                     = "default";
+const char kRPS_No_Referrer_When_Downgrade[]  = "no-referrer-when-downgrade";
+
+/* spec tokens: origin-when-cross-origin */
+const char kRPS_Origin_When_Cross_Origin[]    = "origin-when-cross-origin";
+const char kRPS_Origin_When_Crossorigin[]     = "origin-when-crossorigin";
+
+/* spec tokens: always unsafe-url */
+const char kRPS_Always[]                      = "always";
+const char kRPS_Unsafe_URL[]                  = "unsafe-url";
+
 inline ReferrerPolicy
 ReferrerPolicyFromString(const nsAString& content)
 {
   // This is implemented step by step as described in the Referrer Policy
   // specification, section 6.4 "Determine token's Policy".
-  if (content.LowerCaseEqualsLiteral("never") ||
-      content.LowerCaseEqualsLiteral("no-referrer")) {
+  if (content.LowerCaseEqualsLiteral(kRPS_Never) ||
+      content.LowerCaseEqualsLiteral(kRPS_No_Referrer)) {
     return RP_No_Referrer;
   }
-  if (content.LowerCaseEqualsLiteral("origin")) {
+  if (content.LowerCaseEqualsLiteral(kRPS_Origin)) {
     return RP_Origin;
   }
-  if (content.LowerCaseEqualsLiteral("default") ||
-      content.LowerCaseEqualsLiteral("no-referrer-when-downgrade")) {
+  if (content.LowerCaseEqualsLiteral(kRPS_Default) ||
+      content.LowerCaseEqualsLiteral(kRPS_No_Referrer_When_Downgrade)) {
     return RP_No_Referrer_When_Downgrade;
   }
-  if (content.LowerCaseEqualsLiteral("origin-when-cross-origin") ||
-      content.LowerCaseEqualsLiteral("origin-when-crossorigin")) {
+  if (content.LowerCaseEqualsLiteral(kRPS_Origin_When_Cross_Origin) ||
+      content.LowerCaseEqualsLiteral(kRPS_Origin_When_Crossorigin)) {
     return RP_Origin_When_Crossorigin;
   }
-  if (content.LowerCaseEqualsLiteral("always") ||
-      content.LowerCaseEqualsLiteral("unsafe-url")) {
+  if (content.LowerCaseEqualsLiteral(kRPS_Always) ||
+      content.LowerCaseEqualsLiteral(kRPS_Unsafe_URL)) {
     return RP_Unsafe_URL;
   }
   // Spec says if none of the previous match, use No_Referrer.
@@ -60,15 +82,42 @@ ReferrerPolicyFromString(const nsAString& content)
 inline bool
 IsValidReferrerPolicy(const nsAString& content)
 {
-  return content.LowerCaseEqualsLiteral("never")
-      || content.LowerCaseEqualsLiteral("no-referrer")
-      || content.LowerCaseEqualsLiteral("origin")
-      || content.LowerCaseEqualsLiteral("default")
-      || content.LowerCaseEqualsLiteral("no-referrer-when-downgrade")
-      || content.LowerCaseEqualsLiteral("origin-when-cross-origin")
-      || content.LowerCaseEqualsLiteral("origin-when-crossorigin")
-      || content.LowerCaseEqualsLiteral("always")
-      || content.LowerCaseEqualsLiteral("unsafe-url");
+  return content.LowerCaseEqualsLiteral(kRPS_Never)
+      || content.LowerCaseEqualsLiteral(kRPS_No_Referrer)
+      || content.LowerCaseEqualsLiteral(kRPS_Origin)
+      || content.LowerCaseEqualsLiteral(kRPS_Default)
+      || content.LowerCaseEqualsLiteral(kRPS_No_Referrer_When_Downgrade)
+      || content.LowerCaseEqualsLiteral(kRPS_Origin_When_Cross_Origin)
+      || content.LowerCaseEqualsLiteral(kRPS_Origin_When_Crossorigin)
+      || content.LowerCaseEqualsLiteral(kRPS_Always)
+      || content.LowerCaseEqualsLiteral(kRPS_Unsafe_URL);
+}
+
+inline bool
+IsValidAttributeReferrerPolicy(const nsAString& aContent)
+{
+  // Spec allows only these three policies at the moment
+  // See bug 1178337
+  return aContent.LowerCaseEqualsLiteral(kRPS_No_Referrer)
+      || aContent.LowerCaseEqualsLiteral(kRPS_Origin)
+      || aContent.LowerCaseEqualsLiteral(kRPS_Unsafe_URL);
+}
+
+inline ReferrerPolicy
+AttributeReferrerPolicyFromString(const nsAString& aContent)
+{
+  // if the referrer attribute string is empty, return RP_Unset
+  if (aContent.IsEmpty()) {
+    return RP_Unset;
+  }
+  // if the referrer attribute string is not empty and contains a valid
+  // referrer policy, return the according enum value
+  if (IsValidAttributeReferrerPolicy(aContent)) {
+    return ReferrerPolicyFromString(aContent);
+  }
+  // in any other case the referrer attribute contains an invalid
+  // policy value, we thus return RP_No_Referrer
+  return RP_No_Referrer;
 }
 
 } // namespace net

netwerk/base/moz.build

@@ -27,6 +27,7 @@ XPIDL_SOURCES += [
     'nsICacheInfoChannel.idl',
     'nsICachingChannel.idl',
     'nsICancelable.idl',
+    'nsICaptivePortalService.idl',
     'nsIChannel.idl',
     'nsIChannelEventSink.idl',
     'nsIChildChannel.idl',
@@ -165,6 +166,7 @@ EXPORTS.mozilla += [
 ]
 
 EXPORTS.mozilla.net += [
+    'CaptivePortalService.h',
     'ChannelDiverterChild.h',
     'ChannelDiverterParent.h',
     'Dashboard.h',
@@ -182,6 +184,7 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk':
 UNIFIED_SOURCES += [
     'ArrayBufferInputStream.cpp',
     'BackgroundFileSaver.cpp',
+    'CaptivePortalService.cpp',
     'ChannelDiverterChild.cpp',
     'ChannelDiverterParent.cpp',
     'Dashboard.cpp',

netwerk/base/nsICaptivePortalService.idl

@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+[scriptable, uuid(b5fd5629-d04c-4138-9529-9311f291ecd4)]
+interface nsICaptivePortalServiceCallback : nsISupports
+{
+  /**
+   * Invoke callbacks after captive portal detection finished.
+   */
+  void complete(in bool success, in nsresult error);
+};
+
+/**
+ * Service used for captive portal detection.
+ */
+[scriptable, uuid(bdbe0555-fc3d-4f7b-9205-c309ceb2d641)]
+interface nsICaptivePortalService : nsISupports
+{
+  const long UNKNOWN          = 0;
+  const long NOT_CAPTIVE      = 1;
+  const long UNLOCKED_PORTAL  = 2;
+  const long LOCKED_PORTAL    = 3;
+
+  /**
+   * Called from XPCOM to trigger a captive portal recheck.
+   * A network request will only be performed if no other checks are currently
+   * ongoing.
+   */
+  void recheckCaptivePortal();
+
+  /**
+   * Returns the state of the captive portal.
+   */
+  readonly attribute long state;
+
+  /**
+   * Returns the time difference between NOW and the last time a request was
+   * completed in milliseconds.
+   */
+  readonly attribute unsigned long long lastChecked;
+};

netwerk/base/nsINetUtil.idl

@@ -11,7 +11,7 @@ interface nsIPrefBranch;
 /**
  * nsINetUtil provides various network-related utility methods.
  */
-[scriptable, uuid(ca68c485-9db3-4c12-82a6-4fab7948e2d5)]
+[scriptable, uuid(fe2625ec-b884-4df1-b39c-9e830e47aa94)]
 interface nsINetUtil : nsISupports
 {
   /**
@@ -31,7 +31,7 @@ interface nsINetUtil : nsISupports
                                       out AUTF8String aCharset,
                                       out boolean aHadCharset);
 
-    /**
+  /**
    * Parse a Content-Type header value in relaxed mode.  This is a more
    * permissive parser that ignores things that go against RFC 7231 section
    * 3.1.1.1.  This is typically useful for parsing Content-Type header values
@@ -215,4 +215,16 @@ interface nsINetUtil : nsISupports
                                         out AUTF8String aCharset,
                                         out long aCharsetStart,
                                         out long aCharsetEnd);
+
+/**
+   * Parse an attribute referrer policy string (no-referrer, origin, unsafe-url)
+   * and return the according integer code (defined in nsIHttpChannel.idl)
+   *
+   * @param aPolicyString
+   *        the policy string given as attribute
+   * @return aPolicyEnum
+   *         referrer policy code from nsIHttpChannel.idl, (see parser in
+   *         ReferrerPolicy.h for details)
+   */
+  unsigned long parseAttributePolicyString(in AString aPolicyString);
 };

netwerk/base/nsIOService.cpp

@@ -42,6 +42,9 @@
 #include "nsThreadUtils.h"
 #include "mozilla/LoadInfo.h"
 #include "mozilla/net/NeckoCommon.h"
+#include "mozilla/net/DNS.h"
+#include "CaptivePortalService.h"
+#include "ReferrerPolicy.h"
 
 #ifdef MOZ_WIDGET_GONK
 #include "nsINetworkManager.h"
@@ -53,6 +56,7 @@
 
 using namespace mozilla;
 using mozilla::net::IsNeckoChild;
+using mozilla::net::CaptivePortalService;
 
 #define PORT_PREF_PREFIX           "network.security.ports."
 #define PORT_PREF(x)               PORT_PREF_PREFIX x
@@ -66,6 +70,7 @@ using mozilla::net::IsNeckoChild;
 #define NECKO_BUFFER_CACHE_COUNT_PREF "network.buffer.cache.count"
 #define NECKO_BUFFER_CACHE_SIZE_PREF  "network.buffer.cache.size"
 #define NETWORK_NOTIFY_CHANGED_PREF   "network.notify.changed"
+#define NETWORK_CAPTIVE_PORTAL_PREF   "network.captive-portal-service.enabled"
 
 #define MAX_RECURSION_COUNT 50
 
@@ -192,7 +197,9 @@ nsIOService::Init()
     }
     else
         NS_WARNING("failed to get error service");
-    
+
+    InitializeCaptivePortalService();
+
     // setup our bad port list stuff
     for(int i=0; gBadPortList[i]; i++)
         mRestrictedPortList.AppendElement(gBadPortList[i]);
@@ -207,6 +214,7 @@ nsIOService::Init()
         prefBranch->AddObserver(NECKO_BUFFER_CACHE_COUNT_PREF, this, true);
         prefBranch->AddObserver(NECKO_BUFFER_CACHE_SIZE_PREF, this, true);
         prefBranch->AddObserver(NETWORK_NOTIFY_CHANGED_PREF, this, true);
+        prefBranch->AddObserver(NETWORK_CAPTIVE_PORTAL_PREF, this, true);
         PrefsChanged(prefBranch);
     }
     
@@ -241,6 +249,22 @@ nsIOService::~nsIOService()
     gIOService = nullptr;
 }
 
+nsresult
+nsIOService::InitializeCaptivePortalService()
+{
+    if (XRE_GetProcessType() != GeckoProcessType_Default) {
+        // We only initalize a captive portal service in the main process
+        return NS_OK;
+    }
+
+    mCaptivePortalService = do_GetService(NS_CAPTIVEPORTAL_CID);
+    if (mCaptivePortalService) {
+        return static_cast<CaptivePortalService*>(mCaptivePortalService.get())->Initialize();
+    }
+
+    return NS_OK;
+}
+
 nsresult
 nsIOService::InitializeSocketTransportService()
 {
@@ -322,11 +346,53 @@ NS_IMPL_ISUPPORTS(nsIOService,
 
 ////////////////////////////////////////////////////////////////////////////////
 
+nsresult
+nsIOService::RecheckCaptivePortalIfLocalRedirect(nsIChannel* newChan)
+{
+    nsresult rv;
+
+    if (!mCaptivePortalService) {
+        return NS_OK;
+    }
+
+    nsCOMPtr<nsIURI> uri;
+    rv = newChan->GetURI(getter_AddRefs(uri));
+    if (NS_FAILED(rv)) {
+        return rv;
+    }
+
+    nsCString host;
+    rv = uri->GetHost(host);
+    if (NS_FAILED(rv)) {
+        return rv;
+    }
+
+    PRNetAddr prAddr;
+    if (PR_StringToNetAddr(host.BeginReading(), &prAddr) != PR_SUCCESS) {
+        // The redirect wasn't to an IP literal, so there's probably no need
+        // to trigger the captive portal detection right now. It can wait.
+        return NS_OK;
+    }
+
+    mozilla::net::NetAddr netAddr;
+    PRNetAddrToNetAddr(&prAddr, &netAddr);
+    if (IsIPAddrLocal(&netAddr)) {
+        // Redirects to local IP addresses are probably captive portals
+        mCaptivePortalService->RecheckCaptivePortal();
+    }
+
+    return NS_OK;
+}
+
 nsresult
 nsIOService::AsyncOnChannelRedirect(nsIChannel* oldChan, nsIChannel* newChan,
                                     uint32_t flags,
                                     nsAsyncRedirectVerifyHelper *helper)
 {
+    // If a redirect to a local network address occurs, then chances are we
+    // are in a captive portal, so we trigger a recheck.
+    RecheckCaptivePortalIfLocalRedirect(newChan);
+
     nsCOMPtr<nsIChannelEventSink> sink =
         do_GetService(NS_GLOBAL_CHANNELEVENTSINK_CONTRACTID);
     if (sink) {
@@ -1127,6 +1193,28 @@ nsIOService::PrefsChanged(nsIPrefBranch *prefs, const char *pref)
             mNetworkNotifyChanged = allow;
         }
     }
+
+    if (!pref || strcmp(pref, NETWORK_CAPTIVE_PORTAL_PREF) == 0) {
+        static int disabledForTest = -1;
+        if (disabledForTest == -1) {
+            char *s = getenv("MOZ_DISABLE_NONLOCAL_CONNECTIONS");
+            if (s) {
+                disabledForTest = (strncmp(s, "0", 1) == 0) ? 0 : 1;
+            } else {
+                disabledForTest = 0;
+            }
+        }
+
+        bool captivePortalEnabled;
+        nsresult rv = prefs->GetBoolPref(NETWORK_CAPTIVE_PORTAL_PREF, &captivePortalEnabled);
+        if (NS_SUCCEEDED(rv) && mCaptivePortalService) {
+            if (captivePortalEnabled && !disabledForTest) {
+                static_cast<CaptivePortalService*>(mCaptivePortalService.get())->Start();
+            } else {
+                static_cast<CaptivePortalService*>(mCaptivePortalService.get())->Stop();
+            }
+        }
+    }
 }
 
 void
@@ -1274,6 +1362,10 @@ nsIOService::Observe(nsISupports *subject,
 
         SetOffline(true);
 
+        if (mCaptivePortalService) {
+            static_cast<CaptivePortalService*>(mCaptivePortalService.get())->Stop();
+        }
+
         // Break circular reference.
         mProxyService = nullptr;
     } else if (!strcmp(topic, NS_NETWORK_LINK_TOPIC)) {
@@ -1291,6 +1383,10 @@ nsIOService::Observe(nsISupports *subject,
                                 NS_NETWORK_LINK_TOPIC,
                                 MOZ_UTF16(NS_NETWORK_LINK_DATA_CHANGED));
         }
+
+        if (mCaptivePortalService) {
+            mCaptivePortalService->RecheckCaptivePortal();
+        }
     } else if (!strcmp(topic, kNetworkActiveChanged)) {
 #ifdef MOZ_WIDGET_GONK
         if (IsNeckoChild()) {
@@ -1497,6 +1593,10 @@ nsIOService::OnNetworkLinkEvent(const char *data)
     } else if (!strcmp(data, NS_NETWORK_LINK_DATA_DOWN)) {
         isUp = false;
     } else if (!strcmp(data, NS_NETWORK_LINK_DATA_UP)) {
+        if (mCaptivePortalService) {
+            // Interface is up. Triggering a captive portal recheck.
+            mCaptivePortalService->RecheckCaptivePortal();
+        }
         isUp = true;
     } else if (!strcmp(data, NS_NETWORK_LINK_DATA_UNKNOWN)) {
         nsresult rv = mNetworkLinkService->GetIsLinkUp(&isUp);
@@ -1563,6 +1663,16 @@ nsIOService::ExtractCharsetFromContentType(const nsACString &aTypeHeader,
     return NS_OK;
 }
 
+// parse policyString to policy enum value (see ReferrerPolicy.h)
+NS_IMETHODIMP
+nsIOService::ParseAttributePolicyString(const nsAString& policyString,
+                                                uint32_t *outPolicyEnum)
+{
+  NS_ENSURE_ARG(outPolicyEnum);
+  *outPolicyEnum = (uint32_t)mozilla::net::AttributeReferrerPolicyFromString(policyString);
+  return NS_OK;
+}
+
 // nsISpeculativeConnect
 class IOServiceProxyCallback final : public nsIProtocolProxyCallback
 {

netwerk/base/nsIOService.h

@@ -19,6 +19,7 @@
 #include "nsISpeculativeConnect.h"
 #include "nsDataHashtable.h"
 #include "mozilla/Attributes.h"
+#include "nsICaptivePortalService.h"
 
 #define NS_N(x) (sizeof(x)/sizeof(*x))
 
@@ -100,6 +101,9 @@ private:
     nsresult CacheProtocolHandler(const char *scheme,
                                               nsIProtocolHandler* hdlr);
 
+    nsresult InitializeCaptivePortalService();
+    nsresult RecheckCaptivePortalIfLocalRedirect(nsIChannel* newChan);
+
     // Prefs wrangling
     void PrefsChanged(nsIPrefBranch *prefs, const char *pref = nullptr);
     void GetPrefBranch(nsIPrefBranch **);
@@ -146,6 +150,7 @@ private:
     nsCOMPtr<nsPISocketTransportService> mSocketTransportService;
     nsCOMPtr<nsPIDNSService>             mDNSService;
     nsCOMPtr<nsIProtocolProxyService2>   mProxyService;
+    nsCOMPtr<nsICaptivePortalService>    mCaptivePortalService;
     nsCOMPtr<nsINetworkLinkService>      mNetworkLinkService;
     bool                                 mNetworkLinkServiceInitialized;
 

netwerk/base/nsNetUtil.cpp

@@ -945,9 +945,9 @@ NS_ParseRequestContentType(const nsACString &rawContentType,
 }
 
 nsresult
-NS_ParseContentType(const nsACString &rawContentType,
-                    nsCString        &contentType,
-                    nsCString        &contentCharset)
+NS_ParseResponseContentType(const nsACString &rawContentType,
+                            nsCString        &contentType,
+                            nsCString        &contentCharset)
 {
     // contentCharset is left untouched if not present in rawContentType
     nsresult rv;

netwerk/base/nsNetUtil.h

@@ -493,9 +493,9 @@ nsresult NS_ParseRequestContentType(const nsACString &rawContentType,
                                     nsCString        &contentType,
                                     nsCString        &contentCharset);
 
-nsresult NS_ParseContentType(const nsACString &rawContentType,
-                             nsCString        &contentType,
-                             nsCString        &contentCharset);
+nsresult NS_ParseResponseContentType(const nsACString &rawContentType,
+                                     nsCString        &contentType,
+                                     nsCString        &contentCharset);
 
 nsresult NS_ExtractCharsetFromContentType(const nsACString &rawContentType,
                                           nsCString        &contentCharset,

netwerk/build/nsNetCID.h

@@ -469,6 +469,17 @@
     { 0xae, 0xcf, 0x05, 0xf8, 0xfa, 0xf0, 0x0c, 0x9b } \
 }
 
+// captive portal service implementing nsICaptivePortalService
+#define NS_CAPTIVEPORTAL_CONTRACTID \
+    "@mozilla.org/network/captive-portal-service;1"
+#define NS_CAPTIVEPORTAL_CID \
+{ /* bdbe0555-fc3d-4f7b-9205-c309ceb2d641 */ \
+    0xbdbe0555, \
+    0xfc3d, \
+    0x4f7b, \
+  { 0x92, 0x05, 0xc3, 0x09, 0xce, 0xb2, 0xd6, 0x41 } \
+}
+
 /******************************************************************************
  * netwerk/cache/ classes
  */

netwerk/build/nsNetModule.cpp

@@ -131,6 +131,16 @@ NS_GENERIC_FACTORY_CONSTRUCTOR(RedirectChannelRegistrar)
 typedef mozilla::net::CacheStorageService CacheStorageService;
 NS_GENERIC_FACTORY_CONSTRUCTOR(CacheStorageService)
 
+
+///////////////////////////////////////////////////////////////////////////////
+
+#include "mozilla/net/CaptivePortalService.h"
+namespace mozilla {
+namespace net {
+  NS_GENERIC_FACTORY_CONSTRUCTOR(CaptivePortalService)
+}
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 extern nsresult
@@ -807,6 +817,7 @@ NS_DEFINE_NAMED_CID(NS_SERIALIZATION_HELPER_CID);
 NS_DEFINE_NAMED_CID(NS_REDIRECTCHANNELREGISTRAR_CID);
 NS_DEFINE_NAMED_CID(NS_CACHE_STORAGE_SERVICE_CID);
 NS_DEFINE_NAMED_CID(NS_NETWORKPREDICTOR_CID);
+NS_DEFINE_NAMED_CID(NS_CAPTIVEPORTAL_CID);
 
 static const mozilla::Module::CIDEntry kNeckoCIDs[] = {
     { &kNS_IOSERVICE_CID, false, nullptr, nsIOServiceConstructor },
@@ -953,6 +964,7 @@ static const mozilla::Module::CIDEntry kNeckoCIDs[] = {
     { &kNS_REDIRECTCHANNELREGISTRAR_CID, false, nullptr, RedirectChannelRegistrarConstructor },
     { &kNS_CACHE_STORAGE_SERVICE_CID, false, nullptr, CacheStorageServiceConstructor },
     { &kNS_NETWORKPREDICTOR_CID, false, nullptr, mozilla::net::Predictor::Create },
+    { &kNS_CAPTIVEPORTAL_CID, false, nullptr, mozilla::net::CaptivePortalServiceConstructor },
     { nullptr }
 };
 
@@ -1104,6 +1116,7 @@ static const mozilla::Module::ContractIDEntry kNeckoContracts[] = {
     { NS_CACHE_STORAGE_SERVICE_CONTRACTID, &kNS_CACHE_STORAGE_SERVICE_CID },
     { NS_CACHE_STORAGE_SERVICE_CONTRACTID2, &kNS_CACHE_STORAGE_SERVICE_CID },
     { NS_NETWORKPREDICTOR_CONTRACTID, &kNS_NETWORKPREDICTOR_CID },
+    { NS_CAPTIVEPORTAL_CONTRACTID, &kNS_CAPTIVEPORTAL_CID },
     { nullptr }
 };
 

netwerk/protocol/http/nsHttpChannel.cpp

@@ -15,6 +15,7 @@
 #include "nsICacheStorageService.h"
 #include "nsICacheStorage.h"
 #include "nsICacheEntry.h"
+#include "nsICaptivePortalService.h"
 #include "nsICryptoHash.h"
 #include "nsINetworkInterceptController.h"
 #include "nsIStringBundle.h"

netwerk/test/unit/test_parse_content_type.js

@@ -25,95 +25,95 @@ function run_test() {
   var netutil = Components.classes["@mozilla.org/network/util;1"]
                           .getService(Components.interfaces.nsINetUtil);
 
-  type = netutil.parseContentType("text/html", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/html", charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType("TEXT/HTML", charset, hadCharset);
+  type = netutil.parseResponseContentType("TEXT/HTML", charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType("text/html, text/html", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/html, text/html", charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType("text/html, text/plain",
+  type = netutil.parseResponseContentType("text/html, text/plain",
 				  charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/html, ', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/html, ', charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType('text/html, */*', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/html, */*', charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType('text/html, foo', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/html, foo', charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType("text/html; charset=ISO-8859-1",
+  type = netutil.parseResponseContentType("text/html; charset=ISO-8859-1",
 				  charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType('text/html; charset="ISO-8859-1"',
+  type = netutil.parseResponseContentType('text/html; charset="ISO-8859-1"',
 				  charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType("text/html; charset='ISO-8859-1'",
+  type = netutil.parseResponseContentType("text/html; charset='ISO-8859-1'",
 				  charset, hadCharset);
   check("text/html", "'ISO-8859-1'", true);
 
-  type = netutil.parseContentType("text/html; charset=\"ISO-8859-1\", text/html",
+  type = netutil.parseResponseContentType("text/html; charset=\"ISO-8859-1\", text/html",
 				  charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType("text/html; charset=\"ISO-8859-1\", text/html; charset=UTF8",
+  type = netutil.parseResponseContentType("text/html; charset=\"ISO-8859-1\", text/html; charset=UTF8",
 				  charset, hadCharset);
   check("text/html", "UTF8", true);
 
-  type = netutil.parseContentType("text/html; charset=ISO-8859-1, TEXT/HTML", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/html; charset=ISO-8859-1, TEXT/HTML", charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType("text/html; charset=ISO-8859-1, TEXT/plain", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/html; charset=ISO-8859-1, TEXT/plain", charset, hadCharset);
   check("text/plain", "", true);
 
-  type = netutil.parseContentType("text/plain, TEXT/HTML; charset=ISO-8859-1, text/html, TEXT/HTML", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/plain, TEXT/HTML; charset=ISO-8859-1, text/html, TEXT/HTML", charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType('text/plain, TEXT/HTML; param="charset=UTF8"; charset="ISO-8859-1"; param2="charset=UTF16", text/html, TEXT/HTML', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain, TEXT/HTML; param="charset=UTF8"; charset="ISO-8859-1"; param2="charset=UTF16", text/html, TEXT/HTML', charset, hadCharset);
   check("text/html", "ISO-8859-1", true);
 
-  type = netutil.parseContentType('text/plain, TEXT/HTML; param=charset=UTF8; charset="ISO-8859-1"; param2=charset=UTF16, text/html, TEXT/HTML', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain, TEXT/HTML; param=charset=UTF8; charset="ISO-8859-1"; param2=charset=UTF16, text/html, TEXT/HTML', charset, hadCharset);
   check("text/html", "ISO-8859-1", true);  
 
-  type = netutil.parseContentType("text/plain; param= , text/html", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/plain; param= , text/html", charset, hadCharset);
   check("text/html", "", false);
 
-  type = netutil.parseContentType('text/plain; param=", text/html"', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain; param=", text/html"', charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/plain; param=", \\" , text/html"', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain; param=", \\" , text/html"', charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/plain; param=", \\" , text/html , "', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain; param=", \\" , text/html , "', charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/plain param=", \\" , text/html , "', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain param=", \\" , text/html , "', charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/plain charset=UTF8', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain charset=UTF8', charset, hadCharset);
   check("text/plain", "", false);
 
-  type = netutil.parseContentType('text/plain, TEXT/HTML; param="charset=UTF8"; ; param2="charset=UTF16", text/html, TEXT/HTML', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain, TEXT/HTML; param="charset=UTF8"; ; param2="charset=UTF16", text/html, TEXT/HTML', charset, hadCharset);
   check("text/html", "", false);
 
   // Bug 562915 - correctness: "\x" is "x"
-  type = netutil.parseContentType('text/plain; charset="UTF\\-8"', charset, hadCharset);
+  type = netutil.parseResponseContentType('text/plain; charset="UTF\\-8"', charset, hadCharset);
   check("text/plain", "UTF-8", true);
 
   // Bug 700589
 
   // check that single quote doesn't confuse parsing of subsequent parameters
-  type = netutil.parseContentType("text/plain; x='; charset=\"UTF-8\"", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/plain; x='; charset=\"UTF-8\"", charset, hadCharset);
   check("text/plain", "UTF-8", true);
 
   // check that single quotes do not get removed from extracted charset
-  type = netutil.parseContentType("text/plain; charset='UTF-8'", charset, hadCharset);
+  type = netutil.parseResponseContentType("text/plain; charset='UTF-8'", charset, hadCharset);
   check("text/plain", "'UTF-8'", true);
 }

parser/html/javasrc/AttributeName.java

@@ -989,6 +989,7 @@ public final class AttributeName
     public static final AttributeName READONLY = new AttributeName(ALL_NO_NS, SAME_LOCAL("readonly"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG | CASE_FOLDED | BOOLEAN);
     public static final AttributeName SELECTED = new AttributeName(ALL_NO_NS, SAME_LOCAL("selected"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG | CASE_FOLDED | BOOLEAN);
     public static final AttributeName ROWLINES = new AttributeName(ALL_NO_NS, SAME_LOCAL("rowlines"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG);
+    public static final AttributeName REFERRER = new AttributeName(ALL_NO_NS, SAME_LOCAL("referrer"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG);
     public static final AttributeName SEAMLESS = new AttributeName(ALL_NO_NS, SAME_LOCAL("seamless"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG);
     public static final AttributeName ROWALIGN = new AttributeName(ALL_NO_NS, SAME_LOCAL("rowalign"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG);
     public static final AttributeName STRETCHY = new AttributeName(ALL_NO_NS, SAME_LOCAL("stretchy"), ALL_NO_PREFIX, NCNAME_HTML | NCNAME_FOREIGN | NCNAME_LANG);
@@ -1571,6 +1572,7 @@ public final class AttributeName
     READONLY,
     SELECTED,
     ROWLINES,
+    REFERRER,
     SEAMLESS,
     ROWALIGN,
     STRETCHY,
@@ -2154,6 +2156,7 @@ public final class AttributeName
     291557706,
     291665349,
     291804100,
+    291862420,
     292138018,
     292166446,
     292418738,

parser/html/nsHtml5AtomList.h

@@ -348,6 +348,7 @@ HTML5_ATOM(tabindex, "tabindex")
 HTML5_ATOM(readonly, "readonly")
 HTML5_ATOM(selected, "selected")
 HTML5_ATOM(rowlines, "rowlines")
+HTML5_ATOM(referrer, "referrer")
 HTML5_ATOM(seamless, "seamless")
 HTML5_ATOM(rowalign, "rowalign")
 HTML5_ATOM(stretchy, "stretchy")

parser/html/nsHtml5AttributeName.h

@@ -364,6 +364,7 @@ class nsHtml5AttributeName
     static nsHtml5AttributeName* ATTR_READONLY;
     static nsHtml5AttributeName* ATTR_SELECTED;
     static nsHtml5AttributeName* ATTR_ROWLINES;
+    static nsHtml5AttributeName* ATTR_REFERRER;
     static nsHtml5AttributeName* ATTR_SEAMLESS;
     static nsHtml5AttributeName* ATTR_ROWALIGN;
     static nsHtml5AttributeName* ATTR_STRETCHY;

parser/html/nsHtml5SpeculativeLoad.cpp

@@ -28,10 +28,10 @@ nsHtml5SpeculativeLoad::Perform(nsHtml5TreeOpExecutor* aExecutor)
       aExecutor->SetSpeculationBase(mUrl);
       break;
     case eSpeculativeLoadMetaReferrer:
-      aExecutor->SetSpeculationReferrerPolicy(mMetaReferrerPolicy);
+      aExecutor->SetSpeculationReferrerPolicy(mReferrerPolicy);
       break;
     case eSpeculativeLoadImage:
-      aExecutor->PreloadImage(mUrl, mCrossOrigin, mSrcset, mSizes);
+      aExecutor->PreloadImage(mUrl, mCrossOrigin, mSrcset, mSizes, mReferrerPolicy);
       break;
     case eSpeculativeLoadOpenPicture:
       aExecutor->PreloadOpenPicture();

parser/html/nsHtml5SpeculativeLoad.h

@@ -45,12 +45,13 @@ class nsHtml5SpeculativeLoad {
       NS_PRECONDITION(mOpCode == eSpeculativeLoadUninitialized,
                       "Trying to reinitialize a speculative load!");
       mOpCode = eSpeculativeLoadMetaReferrer;
-      mMetaReferrerPolicy.Assign(
+      mReferrerPolicy.Assign(
         nsContentUtils::TrimWhitespace<nsContentUtils::IsHTMLWhitespace>(aReferrerPolicy));
     }
 
     inline void InitImage(const nsAString& aUrl,
                           const nsAString& aCrossOrigin,
+                          const nsAString& aReferrerPolicy,
                           const nsAString& aSrcset,
                           const nsAString& aSizes)
     {
@@ -59,6 +60,8 @@ class nsHtml5SpeculativeLoad {
       mOpCode = eSpeculativeLoadImage;
       mUrl.Assign(aUrl);
       mCrossOrigin.Assign(aCrossOrigin);
+      mReferrerPolicy.Assign(
+        nsContentUtils::TrimWhitespace<nsContentUtils::IsHTMLWhitespace>(aReferrerPolicy));
       mSrcset.Assign(aSrcset);
       mSizes.Assign(aSizes);
     }
@@ -179,7 +182,7 @@ class nsHtml5SpeculativeLoad {
   private:
     eHtml5SpeculativeLoad mOpCode;
     nsString mUrl;
-    nsString mMetaReferrerPolicy;
+    nsString mReferrerPolicy;
     /**
      * If mOpCode is eSpeculativeLoadStyle or eSpeculativeLoadScript[FromHead]
      * then this is the value of the "charset" attribute. For

parser/html/nsHtml5TreeBuilderCppSupplement.h

@@ -125,11 +125,14 @@ nsHtml5TreeBuilder::createElement(int32_t aNamespace, nsIAtom* aName,
             aAttributes->getValue(nsHtml5AttributeName::ATTR_SRCSET);
           nsString* crossOrigin =
             aAttributes->getValue(nsHtml5AttributeName::ATTR_CROSSORIGIN);
+          nsString* referrerPolicy =
+            aAttributes->getValue(nsHtml5AttributeName::ATTR_REFERRER);
           nsString* sizes =
             aAttributes->getValue(nsHtml5AttributeName::ATTR_SIZES);
           mSpeculativeLoadQueue.AppendElement()->
             InitImage(url ? *url : NullString(),
                       crossOrigin ? *crossOrigin : NullString(),
+                      referrerPolicy ? *referrerPolicy : NullString(),
                       srcset ? *srcset : NullString(),
                       sizes ? *sizes : NullString());
         } else if (nsHtml5Atoms::source == aName) {
@@ -202,6 +205,7 @@ nsHtml5TreeBuilder::createElement(int32_t aNamespace, nsIAtom* aName,
           nsString* url = aAttributes->getValue(nsHtml5AttributeName::ATTR_POSTER);
           if (url) {
             mSpeculativeLoadQueue.AppendElement()->InitImage(*url, NullString(),
+                                                             NullString(),
                                                              NullString(),
                                                              NullString());
           }
@@ -238,6 +242,7 @@ nsHtml5TreeBuilder::createElement(int32_t aNamespace, nsIAtom* aName,
           nsString* url = aAttributes->getValue(nsHtml5AttributeName::ATTR_XLINK_HREF);
           if (url) {
             mSpeculativeLoadQueue.AppendElement()->InitImage(*url, NullString(),
+                                                             NullString(),
                                                              NullString(),
                                                              NullString());
           }

parser/html/nsHtml5TreeOpExecutor.cpp

@@ -947,13 +947,25 @@ void
 nsHtml5TreeOpExecutor::PreloadImage(const nsAString& aURL,
                                     const nsAString& aCrossOrigin,
                                     const nsAString& aSrcset,
-                                    const nsAString& aSizes)
+                                    const nsAString& aSizes,
+                                    const nsAString& aImageReferrerPolicy)
 {
   nsCOMPtr<nsIURI> baseURI = BaseURIForPreload();
   nsCOMPtr<nsIURI> uri = mDocument->ResolvePreloadImage(baseURI, aURL, aSrcset,
                                                         aSizes);
   if (uri && ShouldPreloadURI(uri)) {
-    mDocument->MaybePreLoadImage(uri, aCrossOrigin, mSpeculationReferrerPolicy);
+    // use document wide referrer policy
+    mozilla::net::ReferrerPolicy referrerPolicy = mSpeculationReferrerPolicy;
+    // if enabled in preferences, use the referrer attribute from the image, if provided
+    bool referrerAttributeEnabled = Preferences::GetBool("network.http.enablePerElementReferrer", false);
+    if (referrerAttributeEnabled) {
+      mozilla::net::ReferrerPolicy imageReferrerPolicy = mozilla::net::ReferrerPolicyFromString(aImageReferrerPolicy);
+      if (imageReferrerPolicy != mozilla::net::RP_Unset) {
+        referrerPolicy = imageReferrerPolicy;
+      }
+    }
+
+    mDocument->MaybePreLoadImage(uri, aCrossOrigin, referrerPolicy);
   }
 }
 

parser/html/nsHtml5TreeOpExecutor.h

@@ -256,7 +256,8 @@ class nsHtml5TreeOpExecutor final : public nsHtml5DocumentBuilder,
     void PreloadImage(const nsAString& aURL,
                       const nsAString& aCrossOrigin,
                       const nsAString& aSrcset,
-                      const nsAString& aSizes);
+                      const nsAString& aSizes,
+                      const nsAString& aImageReferrerPolicy);
 
     void PreloadOpenPicture();
 

toolkit/components/moz.build

@@ -79,8 +79,7 @@ if CONFIG['MOZ_XUL']:
 if CONFIG['MOZ_TOOLKIT_SEARCH']:
     DIRS += ['search']
 
-if CONFIG['MOZ_CAPTIVEDETECT']:
-    DIRS += ['captivedetect']
+DIRS += ['captivedetect']
 
 if CONFIG['MOZ_WIDGET_TOOLKIT'] != "gonk" and CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
     DIRS += ['terminator']

toolkit/devtools/netmonitor/netmonitor-controller.js

@@ -520,35 +520,35 @@ NetworkEventsHandler.prototype = {
    *        The network request information.
    */
   _onNetworkEventUpdate: function(type, { packet, networkInfo }) {
-    let actor = networkInfo.actor;
+    let { actor, request: { url } } = networkInfo;
 
     switch (packet.updateType) {
       case "requestHeaders":
         this.webConsoleClient.getRequestHeaders(actor, this._onRequestHeaders);
-        window.emit(EVENTS.UPDATING_REQUEST_HEADERS, actor);
+        window.emit(EVENTS.UPDATING_REQUEST_HEADERS, [actor, url]);
         break;
       case "requestCookies":
         this.webConsoleClient.getRequestCookies(actor, this._onRequestCookies);
-        window.emit(EVENTS.UPDATING_REQUEST_COOKIES, actor);
+        window.emit(EVENTS.UPDATING_REQUEST_COOKIES, [actor, url]);
         break;
       case "requestPostData":
         this.webConsoleClient.getRequestPostData(actor, this._onRequestPostData);
-        window.emit(EVENTS.UPDATING_REQUEST_POST_DATA, actor);
+        window.emit(EVENTS.UPDATING_REQUEST_POST_DATA, [actor, url]);
         break;
       case "securityInfo":
         NetMonitorView.RequestsMenu.updateRequest(actor, {
           securityState: networkInfo.securityInfo,
         });
         this.webConsoleClient.getSecurityInfo(actor, this._onSecurityInfo);
-        window.emit(EVENTS.UPDATING_SECURITY_INFO, actor);
+        window.emit(EVENTS.UPDATING_SECURITY_INFO, [actor, url]);
         break;
       case "responseHeaders":
         this.webConsoleClient.getResponseHeaders(actor, this._onResponseHeaders);
-        window.emit(EVENTS.UPDATING_RESPONSE_HEADERS, actor);
+        window.emit(EVENTS.UPDATING_RESPONSE_HEADERS, [actor, url]);
         break;
       case "responseCookies":
         this.webConsoleClient.getResponseCookies(actor, this._onResponseCookies);
-        window.emit(EVENTS.UPDATING_RESPONSE_COOKIES, actor);
+        window.emit(EVENTS.UPDATING_RESPONSE_COOKIES, [actor, url]);
         break;
       case "responseStart":
         NetMonitorView.RequestsMenu.updateRequest(actor, {
@@ -559,7 +559,7 @@ NetworkEventsHandler.prototype = {
           statusText: networkInfo.response.statusText,
           headersSize: networkInfo.response.headersSize
         });
-        window.emit(EVENTS.STARTED_RECEIVING_RESPONSE, actor);
+        window.emit(EVENTS.STARTED_RECEIVING_RESPONSE, [actor, url]);
         break;
       case "responseContent":
         NetMonitorView.RequestsMenu.updateRequest(actor, {
@@ -568,14 +568,14 @@ NetworkEventsHandler.prototype = {
           mimeType: networkInfo.response.content.mimeType
         });
         this.webConsoleClient.getResponseContent(actor, this._onResponseContent);
-        window.emit(EVENTS.UPDATING_RESPONSE_CONTENT, actor);
+        window.emit(EVENTS.UPDATING_RESPONSE_CONTENT, [actor, url]);
         break;
       case "eventTimings":
         NetMonitorView.RequestsMenu.updateRequest(actor, {
           totalTime: networkInfo.totalTime
         });
         this.webConsoleClient.getEventTimings(actor, this._onEventTimings);
-        window.emit(EVENTS.UPDATING_EVENT_TIMINGS, actor);
+        window.emit(EVENTS.UPDATING_EVENT_TIMINGS, [actor, url]);
         break;
     }
   },

toolkit/devtools/netmonitor/test/head.js

@@ -195,7 +195,6 @@ function waitForNetworkEvents(aMonitor, aGetRequests, aPostRequests = 0) {
 
   let panel = aMonitor.panelWin;
   let events = panel.EVENTS;
-  let menu = panel.NetMonitorView.RequestsMenu;
 
   let progress = {};
   let genericEvents = 0;
@@ -240,13 +239,12 @@ function waitForNetworkEvents(aMonitor, aGetRequests, aPostRequests = 0) {
     maybeResolve(event, actor);
   }
 
-  function maybeResolve(event, actor) {
+  function maybeResolve(event, [actor, url]) {
     info("> Network events progress: " +
       genericEvents + "/" + ((aGetRequests + aPostRequests) * 13) + ", " +
       postEvents + "/" + (aPostRequests * 2) + ", " +
       "got " + event + " for " + actor);
 
-    let url = menu.getItemByValue(actor).attachment.url;
     updateProgressForURL(url, event);
     info("> Current state: " + JSON.stringify(progress, null, 2));
 

toolkit/modules/Services.jsm

@@ -90,6 +90,7 @@ let initTable = [
   ["focus", "@mozilla.org/focus-manager;1", "nsIFocusManager"],
   ["uriFixup", "@mozilla.org/docshell/urifixup;1", "nsIURIFixup"],
   ["blocklist", "@mozilla.org/extensions/blocklist;1", "nsIBlocklistService"],
+  ["netUtils", "@mozilla.org/network/util;1", "nsINetUtil"],
 #ifdef XP_WIN
 #ifdef MOZ_METRO
   ["metro", "@mozilla.org/windows-metroutils;1", "nsIWinMetroUtils"],