import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1110446 P2 Cleanup stale caches/bodies if last session didn't shutdown cleanly. r=ehsan (7925cf5fa)
- Bug 1110446 P3 Add a test that forces a Cache object to be orphaned and reclaimed. r=ehsan (c61409240)
- Bug 1110446 P4 Add a test that orphanes Cache API body files. r=ehsan (97e0a6f73)
- Bug 1157670 - Fixing an incorrect assertion in QuotaManager.cpp leads to an assertion failure; r=bent (0a19eac66)
- Bug 1165119 Remove corrupt morgue directories polluting nightly profiles. r=janv (d148170d8)
- Bug 1165119 - Follow-up to address review feedback accidentally left out of last push. r=me (f7ef96873)
- Fixup to make bug 1165119 ride the trains properly. r=trivial,DONTBUILD (87d186da4)
- Bug 1162624 - Add support for restoring corrupted or missing metadata files; r=bent (57e4341e6)
- Bug 1174113 - QuotaManager: Origin initialization fails on moz-safe-about+++home; r=bent (b7673128c)
- Bug 1142694 - QuotaManager default/temporary initialization fails on some profiles; r=bent (29a286137)
- Bug 1166871 - Always force a repaint before handling a wheel event so that we don't untransform it into some other scrollframe. r=botond (28e56646d)
- Don't vertically scroll APZCs that have less than one pixel of vertical scroll range. (bug 1154134, r=kats) (1bac9c054)
- Bug 1166871 - Add a test. r=botond (45d398bb6)
- Bug 1164557 - Do not start an overscroll animation if one is already running. r=kats (287a27910)
- Bug 1163832 - Add an API to flush pending APZ repaint requests and dispatch a notification upon completion. r=botond (8b3f9e06f)
- Bug 858680 - Part 1: Perform incremental_vacuum on open databases while idle, r=janv. (715f77ad6)
- Bug 858680 - Part 2: Add idle notifications to QuotaClient, r=janv. (9f245b1bb)
- Bug 1135166 - Initialize Telemetry histogram id cache early to avoid races. r=froydnj,vladan (f0bd8278c)
- Bug 1162176, Part 1. r=mak. (f92ba4061)
- Bug 1162176, Part 2. r=janv. (f313e1cf3)
- Bug 1155634 - Move ConnectionPool creation closer to where we actually use it and at a point guaranteed to be after QuotaManager has been started. r=khuey relanding CLOSED TREE (ce489e8f4)
- Bug 1155652 - Fix two incorrect assertions r=janv (2417d91ed)
- Bug 1156063 - Intermittent application crashed [@ mozilla::dom::indexedDB::::ConnectionPool::Start] in various tests. r=janv (b1126ac71)
- Bug 1157029 - More changes to bulletproof shutdown of failed connections, r=janv. (93a425abb)
- Bug 858680 - Part 4: Perform maintenance on databases while idle, r=janv. (017d536fe)
- Bug 1130775 - Convert synchronized ops and storage registration into unified directory locks; r=bent (300f635f7)
- Bug 1130775 followup: Add missing 'override' keyword to SendResults() methods in QuotaManager.cpp. rs=ehsan (397338f5b)
- Bug 1170021 - Part 1: Merge QuotaManager with QuotaObject; r=bent (168264350)
- Bug 1170021 - Part 2: Move DirectoryLock out of QuotaManager class; r=bent (278964f88)
- pointer style (99453953c)
- Bug 1171931 - Refactor duplicated code using XRE_IsParent/ContentProcess. r=froydnj (6d1ddbff1)

dom/cache/Manager.cpp

@@ -30,15 +30,12 @@
 #include "nsThreadUtils.h"
 #include "nsTObserverArray.h"
 
-namespace {
 
-using mozilla::unused;
-using mozilla::dom::cache::Action;
-using mozilla::dom::cache::BodyCreateDir;
-using mozilla::dom::cache::BodyDeleteFiles;
-using mozilla::dom::cache::QuotaInfo;
-using mozilla::dom::cache::SyncDBAction;
-using mozilla::dom::cache::db::CreateSchema;
+namespace mozilla {
+namespace dom {
+namespace cache {
+
+namespace {
 
 // An Action that is executed when a Context is first created.  It ensures that
 // the directory and database are setup properly.  This lets other actions
@@ -54,23 +51,56 @@ public:
   RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir,
                         mozIStorageConnection* aConn) override
   {
-    // TODO: init maintainance marker (bug 1110446)
-    // TODO: perform maintainance if necessary (bug 1110446)
-    // TODO: find orphaned caches in database (bug 1110446)
-    // TODO: have Context create/delete marker files in constructor/destructor
-    //       and only do expensive maintenance if that marker is present (bug 1110446)
-
     nsresult rv = BodyCreateDir(aDBDir);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-    mozStorageTransaction trans(aConn, false,
-                                mozIStorageConnection::TRANSACTION_IMMEDIATE);
+    {
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
-    rv = CreateSchema(aConn);
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      rv = db::CreateSchema(aConn);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-    rv = trans.Commit();
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      rv = trans.Commit();
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
+
+    // If the Context marker file exists, then the last session was
+    // not cleanly shutdown.  In these cases sqlite will ensure that
+    // the database is valid, but we might still orphan data.  Both
+    // Cache objects and body files can be referenced by DOM objects
+    // after they are "removed" from their parent.  So we need to
+    // look and see if any of these late access objects have been
+    // orphaned.
+    //
+    // Note, this must be done after any schema version updates to
+    // ensure our DBSchema methods work correctly.
+    if (MarkerFileExists(aQuotaInfo)) {
+      NS_WARNING("Cache not shutdown cleanly! Cleaning up stale data...");
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+      // Clean up orphaned Cache objects
+      nsAutoTArray<CacheId, 8> orphanedCacheIdList;
+      nsresult rv = db::FindOrphanedCacheIds(aConn, orphanedCacheIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      for (uint32_t i = 0; i < orphanedCacheIdList.Length(); ++i) {
+        nsAutoTArray<nsID, 16> deletedBodyIdList;
+        rv = db::DeleteCacheId(aConn, orphanedCacheIdList[i], deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+        rv = BodyDeleteFiles(aDBDir, deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      }
+
+      // Clean up orphaned body objects
+      nsAutoTArray<nsID, 64> knownBodyIdList;
+      rv = db::GetKnownBodyIds(aConn, knownBodyIdList);
+
+      rv = BodyDeleteOrphanedFiles(aDBDir, knownBodyIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
 
     return rv;
   }
@@ -127,14 +157,6 @@ private:
   nsTArray<nsID> mDeletedBodyIdList;
 };
 
-} // namespace
-
-namespace mozilla {
-namespace dom {
-namespace cache {
-
-namespace {
-
 bool IsHeadRequest(CacheRequest aRequest, CacheQueryParams aParams)
 {
   return !aParams.ignoreMethod() && aRequest.method().LowerCaseEqualsLiteral("head");
@@ -229,6 +251,26 @@ public:
     MaybeDestroyInstance();
   }
 
+  static void
+  StartAbortOnMainThread(const nsACString& aOrigin)
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+
+    // Lock for sBackgroundThread.
+    StaticMutexAutoLock lock(sMutex);
+
+    if (!sBackgroundThread) {
+      return;
+    }
+
+    // Guaranteed to succeed because we should get abort only before the
+    // background thread is destroyed.
+    nsCOMPtr<nsIRunnable> runnable = new AbortRunnable(aOrigin);
+    nsresult rv = sBackgroundThread->Dispatch(runnable,
+                                              nsIThread::DISPATCH_NORMAL);
+    MOZ_ALWAYS_TRUE(NS_SUCCEEDED(rv));
+  }
+
   static void
   StartShutdownAllOnMainThread()
   {
@@ -338,6 +380,36 @@ private:
     sFactory = nullptr;
   }
 
+  static void
+  AbortOnBackgroundThread(const nsACString& aOrigin)
+  {
+    mozilla::ipc::AssertIsOnBackgroundThread();
+
+    // The factory was destroyed between when abort started on main thread and
+    // when we could start abort on the worker thread.  Just declare abort
+    // complete.
+    if (!sFactory) {
+#ifdef DEBUG
+      StaticMutexAutoLock lock(sMutex);
+      MOZ_ASSERT(!sBackgroundThread);
+#endif
+      return;
+    }
+
+    MOZ_ASSERT(!sFactory->mManagerList.IsEmpty());
+
+    {
+      ManagerList::ForwardIterator iter(sFactory->mManagerList);
+      while (iter.HasMore()) {
+        nsRefPtr<Manager> manager = iter.GetNext();
+        if (aOrigin.IsVoid() ||
+            manager->mManagerId->ExtendedOrigin() == aOrigin) {
+          manager->Abort();
+        }
+      }
+    }
+  }
+
   static void
   ShutdownAllOnBackgroundThread()
   {
@@ -374,6 +446,26 @@ private:
     MaybeDestroyInstance();
   }
 
+  class AbortRunnable final : public nsRunnable
+  {
+  public:
+    explicit AbortRunnable(const nsACString& aOrigin)
+      : mOrigin(aOrigin)
+    { }
+
+    NS_IMETHOD
+    Run() override
+    {
+      mozilla::ipc::AssertIsOnBackgroundThread();
+      AbortOnBackgroundThread(mOrigin);
+      return NS_OK;
+    }
+  private:
+    ~AbortRunnable() { }
+
+    const nsCString mOrigin;
+  };
+
   class ShutdownAllRunnable final : public nsRunnable
   {
   public:
@@ -1463,6 +1555,15 @@ Manager::ShutdownAllOnMainThread()
   }
 }
 
+// static
+void
+Manager::AbortOnMainThread(const nsACString& aOrigin)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  Factory::StartAbortOnMainThread(aOrigin);
+}
+
 void
 Manager::RemoveListener(Listener* aListener)
 {
@@ -1838,6 +1939,22 @@ Manager::Shutdown()
   }
 }
 
+void
+Manager::Abort()
+{
+  NS_ASSERT_OWNINGTHREAD(Manager);
+  MOZ_ASSERT(mContext);
+
+  // Note that we are closing to prevent any new requests from coming in and
+  // creating a new Context.  We must ensure all Contexts and IO operations are
+  // complete before origin clear proceeds.
+  NoteClosing();
+
+  // Cancel and only note that we are done after the context is cleaned up.
+  nsRefPtr<Context> context = mContext;
+  context->CancelAll();
+}
+
 Manager::ListenerId
 Manager::SaveListener(Listener* aListener)
 {