diff --git a/image/imgLoader.cpp b/image/imgLoader.cpp
index 71644d382..c199ed154 100644
--- a/image/imgLoader.cpp
+++ b/image/imgLoader.cpp
@@ -1003,6 +1003,12 @@ imgCacheQueue::GetNumElements() const
   return mQueue.size();
 }
 
+bool
+imgCacheQueue::Contains(imgCacheEntry* aEntry) const
+{
+  return mQueue.Contains(aEntry);
+}
+
 imgCacheQueue::iterator
 imgCacheQueue::begin()
 {
@@ -1583,7 +1589,9 @@ imgLoader::CheckCacheLimits(imgCacheTable& cache, imgCacheQueue& queue)
     }
 
     if (entry) {
-      RemoveFromCache(entry);
+      // We just popped this entry from the queue, so pass AlreadyRemoved
+      // to avoid searching the queue again in RemoveFromCache.
+      RemoveFromCache(entry, QueueState::AlreadyRemoved);
     }
   }
 }
@@ -1884,7 +1892,7 @@ imgLoader::RemoveFromCache(const ImageCacheKey& aKey)
 }
 
 bool
-imgLoader::RemoveFromCache(imgCacheEntry* entry)
+imgLoader::RemoveFromCache(imgCacheEntry* entry, QueueState aQueueState)
 {
   LOG_STATIC_FUNC(gImgLog, "imgLoader::RemoveFromCache entry");
 
@@ -1898,6 +1906,8 @@ imgLoader::RemoveFromCache(imgCacheEntry* entry)
                                "imgLoader::RemoveFromCache", "entry's uri",
                                key.Spec());
 
+    cache.Remove(key);
+
     if (queue.IsDirty()) {
       queue.Refresh();
     }
@@ -1908,11 +1918,16 @@ imgLoader::RemoveFromCache(imgCacheEntry* entry)
       if (mCacheTracker) {
         mCacheTracker->RemoveObject(entry);
       }
-      queue.Remove(entry);
+      // Only search the queue to remove the entry if its possible it might
+      // be in the queue.  If we know its not in the queue this would be
+      // wasted work.
+      MOZ_ASSERT_IF(aQueueState == QueueState::AlreadyRemoved,
+                    !queue.Contains(entry));
+      if (aQueueState == QueueState::MaybeExists) {
+        queue.Remove(entry);
+      }
     }
 
-    cache.Remove(key);
-
     entry->SetEvicted(true);
     request->SetIsInCache(false);
     AddToUncachedImages(request);
diff --git a/image/imgLoader.h b/image/imgLoader.h
index 84272c483..2d5fd2e5e 100644
--- a/image/imgLoader.h
+++ b/image/imgLoader.h
@@ -315,7 +315,16 @@ public:
   nsresult InitCache();
 
   bool RemoveFromCache(const ImageCacheKey& aKey);
-  bool RemoveFromCache(imgCacheEntry* entry);
+
+  // Enumeration describing if a given entry is in the cache queue or not.
+  // There are some cases we know the entry is definitely not in the queue.
+  enum class QueueState {
+    MaybeExists,
+    AlreadyRemoved
+  };
+
+  bool RemoveFromCache(imgCacheEntry* entry,
+                       QueueState aQueueState = QueueState::MaybeExists);
 
   bool PutIntoCache(const ImageCacheKey& aKey, imgCacheEntry* aEntry);