Merge remote-tracking branch 'origin/media-works' into master

b2g/installer/package-manifest.in

@@ -146,6 +146,7 @@
 #ifndef MOZ_FOLD_LIBS
 @BINPATH@/@DLL_PREFIX@mozsqlite3@DLL_SUFFIX@
 #endif
+@BINPATH@/@DLL_PREFIX@lgpllibs@DLL_SUFFIX@
 @RESPATH@/blocklist.xml
 @RESPATH@/ua-update.json
 #ifdef XP_UNIX

browser/installer/package-manifest.in

@@ -162,6 +162,7 @@
 @BINPATH@/@DLL_PREFIX@mozsqlite3@DLL_SUFFIX@
 #endif
 #endif
+@BINPATH@/@DLL_PREFIX@lgpllibs@DLL_SUFFIX@
 @RESPATH@/browser/blocklist.xml
 #ifdef XP_UNIX
 #ifndef XP_MACOSX

config/external/lgpllibs/lgpllibs.def

@@ -0,0 +1,10 @@
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+LIBRARY lgpllibs.dll
+
+EXPORTS
+        av_rdft_init
+        av_rdft_calc
+        av_rdft_end

config/external/lgpllibs/moz.build

@@ -0,0 +1,14 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# The lgpllibs library stores symbols from third-party LGPL licensed libraries,
+# such as libav and libsoundtouch. It fulfills the requirement of dynamically
+# linking these symbols into gecko.
+#
+# Any library added here should also be reflected in the about:license page.
+
+SharedLibrary('lgpllibs')
+SHARED_LIBRARY_NAME = 'lgpllibs'

config/external/moz.build

@@ -7,6 +7,7 @@
 external_dirs = []
 
 DIRS += [
+    'lgpllibs',
     'sqlite',
 ]
 if not CONFIG['MOZ_NATIVE_JPEG']:

config/system-headers

@@ -1262,6 +1262,10 @@ X11/Xos.h
 X11/Xutil.h
 zmouse.h
 soundtouch/SoundTouch.h
+soundtouch/SoundTouchFactory.h
+#if MOZ_LIBAV_FFT==1
+libavcodec/avfft.h
+#endif
 #if MOZ_NATIVE_PNG==1
 png.h
 #endif

docshell/base/nsDocShell.cpp

@@ -781,6 +781,7 @@ nsDocShell::nsDocShell()
   , mUseRemoteTabs(false)
   , mDeviceSizeIsPageSize(false)
   , mWindowDraggingAllowed(false)
+  , mInFrameSwap(false)
   , mCanExecuteScripts(false)
   , mFiredUnloadEvent(false)
   , mEODForCurrentDocument(false)
@@ -14375,3 +14376,16 @@ nsDocShell::GetPaymentRequestId(nsAString& aPaymentRequestId)
   aPaymentRequestId = GetInheritedPaymentRequestId();
   return NS_OK;
 }
+
+bool
+nsDocShell::InFrameSwap()
+{
+  nsRefPtr<nsDocShell> shell = this;
+  do {
+    if (shell->mInFrameSwap) {
+      return true;
+    }
+    shell = shell->GetParentDocshell();
+  } while (shell);
+  return false;
+}

docshell/base/nsDocShell.h

@@ -267,6 +267,12 @@ public:
 
   mozilla::OriginAttributes GetOriginAttributes();
 
+  void SetInFrameSwap(bool aInSwap)
+  {
+    mInFrameSwap = aInSwap;
+  }
+  bool InFrameSwap();
+
 private:
   // An observed docshell wrapper is created when recording markers is enabled.
   mozilla::UniquePtr<mozilla::ObservedDocShell> mObserved;
@@ -921,6 +927,7 @@ protected:
   bool mUseRemoteTabs;
   bool mDeviceSizeIsPageSize;
   bool mWindowDraggingAllowed;
+  bool mInFrameSwap;
 
   // Because scriptability depends on the mAllowJavascript values of our
   // ancestors, we cache the effective scriptability and recompute it when

dom/apps/PermissionsTable.jsm

@@ -298,6 +298,12 @@ this.PermissionsTable =  { geolocation: {
                              privileged: ALLOW_ACTION,
                              certified: ALLOW_ACTION
                            },
+                           "audio-channel-system": {
+                             app: DENY_ACTION,
+                             trusted: DENY_ACTION,
+                             privileged: ALLOW_ACTION,
+                             certified: ALLOW_ACTION
+                           },
                            "audio-channel-telephony": {
                              app: DENY_ACTION,
                              privileged: DENY_ACTION,

dom/audiochannel/AudioChannelAgent.cpp

@@ -72,6 +72,7 @@ AudioChannelAgent::InitInternal(nsIDOMWindow* aWindow, int32_t aChannelType,
              int(AUDIO_AGENT_CHANNEL_ALARM) == int(AudioChannel::Alarm) &&
              int(AUDIO_AGENT_CHANNEL_TELEPHONY) == int(AudioChannel::Telephony) &&
              int(AUDIO_AGENT_CHANNEL_RINGER) == int(AudioChannel::Ringer) &&
+             int(AUDIO_AGENT_CHANNEL_SYSTEM) == int(AudioChannel::System) &&
              int(AUDIO_AGENT_CHANNEL_PUBLICNOTIFICATION) == int(AudioChannel::Publicnotification),
              "Enum of channel on nsIAudioChannelAgent.idl should be the same with AudioChannelBinding.h");
 

dom/audiochannel/AudioChannelService.cpp

@@ -27,9 +27,7 @@
 
 #ifdef MOZ_WIDGET_GONK
 #include "nsJSUtils.h"
-#include "nsIAudioManager.h"
 #include "SpeakerManagerService.h"
-#define NS_AUDIOMANAGER_CONTRACTID "@mozilla.org/telephony/audiomanager;1"
 #endif
 
 #include "mozilla/Preferences.h"
@@ -205,8 +203,7 @@ NS_IMPL_ADDREF(AudioChannelService)
 NS_IMPL_RELEASE(AudioChannelService)
 
 AudioChannelService::AudioChannelService()
-  : mDisabled(false)
-  , mDefChannelChildID(CONTENT_PROCESS_ID_UNKNOWN)
+  : mDefChannelChildID(CONTENT_PROCESS_ID_UNKNOWN)
   , mTelephonyChannel(false)
   , mContentOrNormalChannel(false)
   , mAnyChannel(false)
@@ -236,10 +233,6 @@ void
 AudioChannelService::RegisterAudioChannelAgent(AudioChannelAgent* aAgent,
                                                AudioChannel aChannel)
 {
-  if (mDisabled) {
-    return;
-  }
-
   uint64_t windowID = aAgent->WindowID();
   AudioChannelWindow* winData = GetWindowData(windowID);
   if (!winData) {
@@ -270,10 +263,6 @@ AudioChannelService::RegisterAudioChannelAgent(AudioChannelAgent* aAgent,
 void
 AudioChannelService::UnregisterAudioChannelAgent(AudioChannelAgent* aAgent)
 {
-  if (mDisabled) {
-    return;
-  }
-
   AudioChannelWindow* winData = GetWindowData(aAgent->WindowID());
   if (!winData) {
     return;
@@ -463,49 +452,9 @@ AudioChannelService::Observe(nsISupports* aSubject, const char* aTopic,
                              const char16_t* aData)
 {
   if (!strcmp(aTopic, "xpcom-shutdown")) {
-    mDisabled = true;
     mWindows.Clear();
-  }
-
-#ifdef MOZ_WIDGET_GONK
-  // To process the volume control on each audio channel according to
-  // change of settings
-  else if (!strcmp(aTopic, "mozsettings-changed")) {
-    RootedDictionary<SettingChangeNotification> setting(nsContentUtils::RootingCxForThread());
-    if (!WrappedJSToDictionary(aSubject, setting)) {
-      return NS_OK;
-    }
-    if (!StringBeginsWith(setting.mKey, NS_LITERAL_STRING("audio.volume."))) {
-      return NS_OK;
-    }
-    if (!setting.mValue.isNumber()) {
-      return NS_OK;
-    }
-
-    nsCOMPtr<nsIAudioManager> audioManager = do_GetService(NS_AUDIOMANAGER_CONTRACTID);
-    NS_ENSURE_TRUE(audioManager, NS_OK);
-
-    int32_t index = setting.mValue.toNumber();
-    if (setting.mKey.EqualsLiteral("audio.volume.content")) {
-      audioManager->SetAudioChannelVolume((int32_t)AudioChannel::Content, index);
-    } else if (setting.mKey.EqualsLiteral("audio.volume.notification")) {
-      audioManager->SetAudioChannelVolume((int32_t)AudioChannel::Notification, index);
-    } else if (setting.mKey.EqualsLiteral("audio.volume.alarm")) {
-      audioManager->SetAudioChannelVolume((int32_t)AudioChannel::Alarm, index);
-    } else if (setting.mKey.EqualsLiteral("audio.volume.telephony")) {
-      audioManager->SetAudioChannelVolume((int32_t)AudioChannel::Telephony, index);
-    } else if (!setting.mKey.EqualsLiteral("audio.volume.bt_sco")) {
-      // bt_sco is not a valid audio channel so we manipulate it in
-      // AudioManager.cpp. And the others should not be used.
-      // We didn't use MOZ_CRASH or MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE here
-      // because any web content who has permission of mozSettings can set any
-      // names then it can be easy to crash the B2G.
-      NS_WARNING("unexpected audio channel for volume control");
-    }
-  }
-#endif
-
-  else if (!strcmp(aTopic, "outer-window-destroyed")) {
+    Shutdown();
+  } else if (!strcmp(aTopic, "outer-window-destroyed")) {
     nsCOMPtr<nsISupportsPRUint64> wrapper = do_QueryInterface(aSubject);
     NS_ENSURE_TRUE(wrapper, NS_ERROR_FAILURE);
 
@@ -544,9 +493,7 @@ AudioChannelService::Observe(nsISupports* aSubject, const char* aTopic,
       mSpeakerManager[i]->SetAudioChannelActive(active);
     }
 #endif
-  }
-
-  else if (!strcmp(aTopic, "ipc:content-shutdown")) {
+  } else if (!strcmp(aTopic, "ipc:content-shutdown")) {
     nsCOMPtr<nsIPropertyBag2> props = do_QueryInterface(aSubject);
     if (!props) {
       NS_WARNING("ipc:content-shutdown message without property bag as subject");

dom/audiochannel/AudioChannelService.h

@@ -43,11 +43,6 @@ public:
    */
   static already_AddRefed<AudioChannelService> GetOrCreate();
 
-  /**
-   * Shutdown the singleton.
-   */
-  static void Shutdown();
-
   static bool IsAudioChannelMutedByDefault();
 
   /**
@@ -144,6 +139,11 @@ private:
   AudioChannelService();
   ~AudioChannelService();
 
+  /**
+   * Shutdown the singleton.
+   */
+  static void Shutdown();
+
   void MaybeSendStatusUpdate();
 
   bool ContentOrNormalChannelIsActive();
@@ -212,8 +212,6 @@ private:
   nsTArray<SpeakerManagerService*>  mSpeakerManager;
 #endif
 
-  bool mDisabled;
-
   nsCOMPtr<nsIRunnable> mRunnable;
 
   uint64_t mDefChannelChildID;

dom/audiochannel/nsIAudioChannelAgent.idl

@@ -34,7 +34,7 @@ interface nsIAudioChannelAgentCallback : nsISupports
  *   1. Changes to the playable status of this channel.
  */
 
-[uuid(e28e1569-2a44-4f71-9cd0-216874b05d57)]
+[uuid(ee39a34b-a5c7-4b30-b1ac-cd64ceedef67)]
 interface nsIAudioChannelAgent : nsISupports
 {
   const long AUDIO_AGENT_CHANNEL_NORMAL             = 0;
@@ -44,6 +44,7 @@ interface nsIAudioChannelAgent : nsISupports
   const long AUDIO_AGENT_CHANNEL_TELEPHONY          = 4;
   const long AUDIO_AGENT_CHANNEL_RINGER             = 5;
   const long AUDIO_AGENT_CHANNEL_PUBLICNOTIFICATION = 6;
+  const long AUDIO_AGENT_CHANNEL_SYSTEM             = 7;
 
   const long AUDIO_AGENT_CHANNEL_ERROR              = 1000;
 

dom/base/nsFrameLoader.cpp

@@ -987,6 +987,60 @@ nsFrameLoader::SwapWithOtherRemoteLoader(nsFrameLoader* aOther,
   return NS_OK;
 }
 
+class MOZ_STACK_CLASS AutoResetInFrameSwap final
+{
+public:
+  AutoResetInFrameSwap(nsFrameLoader* aThisFrameLoader,
+                       nsFrameLoader* aOtherFrameLoader,
+                       nsDocShell* aThisDocShell,
+                       nsDocShell* aOtherDocShell,
+                       EventTarget* aThisEventTarget,
+                       EventTarget* aOtherEventTarget
+                       MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
+    : mThisFrameLoader(aThisFrameLoader)
+    , mOtherFrameLoader(aOtherFrameLoader)
+    , mThisDocShell(aThisDocShell)
+    , mOtherDocShell(aOtherDocShell)
+    , mThisEventTarget(aThisEventTarget)
+    , mOtherEventTarget(aOtherEventTarget)
+  {
+    MOZ_GUARD_OBJECT_NOTIFIER_INIT;
+
+    mThisFrameLoader->mInSwap = true;
+    mOtherFrameLoader->mInSwap = true;
+    mThisDocShell->SetInFrameSwap(true);
+    mOtherDocShell->SetInFrameSwap(true);
+
+    // Fire pageshow events on still-loading pages, and then fire pagehide
+    // events.  Note that we do NOT fire these in the normal way, but just fire
+    // them on the chrome event handlers.
+    nsContentUtils::FirePageShowEvent(mThisDocShell, mThisEventTarget, false);
+    nsContentUtils::FirePageShowEvent(mOtherDocShell, mOtherEventTarget, false);
+    nsContentUtils::FirePageHideEvent(mThisDocShell, mThisEventTarget);
+    nsContentUtils::FirePageHideEvent(mOtherDocShell, mOtherEventTarget);
+  }
+
+  ~AutoResetInFrameSwap()
+  {
+    nsContentUtils::FirePageShowEvent(mThisDocShell, mThisEventTarget, true);
+    nsContentUtils::FirePageShowEvent(mOtherDocShell, mOtherEventTarget, true);
+
+    mThisFrameLoader->mInSwap = false;
+    mOtherFrameLoader->mInSwap = false;
+    mThisDocShell->SetInFrameSwap(false);
+    mOtherDocShell->SetInFrameSwap(false);
+  }
+
+private:
+  nsRefPtr<nsFrameLoader> mThisFrameLoader;
+  nsRefPtr<nsFrameLoader> mOtherFrameLoader;
+  nsRefPtr<nsDocShell> mThisDocShell;
+  nsRefPtr<nsDocShell> mOtherDocShell;
+  nsCOMPtr<EventTarget> mThisEventTarget;
+  nsCOMPtr<EventTarget> mOtherEventTarget;
+  MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
+};
+
 nsresult
 nsFrameLoader::SwapWithOtherLoader(nsFrameLoader* aOther,
                                    nsRefPtr<nsFrameLoader>& aFirstToSwap,
@@ -1023,8 +1077,8 @@ nsFrameLoader::SwapWithOtherLoader(nsFrameLoader* aOther,
     return NS_ERROR_DOM_SECURITY_ERR;
   }
 
-  nsCOMPtr<nsIDocShell> ourDocshell = GetExistingDocShell();
-  nsCOMPtr<nsIDocShell> otherDocshell = aOther->GetExistingDocShell();
+  nsRefPtr<nsDocShell> ourDocshell = static_cast<nsDocShell*>(GetExistingDocShell());
+  nsRefPtr<nsDocShell> otherDocshell = static_cast<nsDocShell*>(aOther->GetExistingDocShell());
   if (!ourDocshell || !otherDocshell) {
     // How odd
     return NS_ERROR_NOT_IMPLEMENTED;
@@ -1155,39 +1209,23 @@ nsFrameLoader::SwapWithOtherLoader(nsFrameLoader* aOther,
   if (mInSwap || aOther->mInSwap) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
-  mInSwap = aOther->mInSwap = true;
+  AutoResetInFrameSwap autoFrameSwap(this, aOther, ourDocshell, otherDocshell,
+                                     ourEventTarget, otherEventTarget);
 
-  // Fire pageshow events on still-loading pages, and then fire pagehide
-  // events.  Note that we do NOT fire these in the normal way, but just fire
-  // them on the chrome event handlers.
-  nsContentUtils::FirePageShowEvent(ourDocshell, ourEventTarget, false);
-  nsContentUtils::FirePageShowEvent(otherDocshell, otherEventTarget, false);
-  nsContentUtils::FirePageHideEvent(ourDocshell, ourEventTarget);
-  nsContentUtils::FirePageHideEvent(otherDocshell, otherEventTarget);
-  
   nsIFrame* ourFrame = ourContent->GetPrimaryFrame();
   nsIFrame* otherFrame = otherContent->GetPrimaryFrame();
   if (!ourFrame || !otherFrame) {
-    mInSwap = aOther->mInSwap = false;
-    nsContentUtils::FirePageShowEvent(ourDocshell, ourEventTarget, true);
-    nsContentUtils::FirePageShowEvent(otherDocshell, otherEventTarget, true);
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   nsSubDocumentFrame* ourFrameFrame = do_QueryFrame(ourFrame);
   if (!ourFrameFrame) {
-    mInSwap = aOther->mInSwap = false;
-    nsContentUtils::FirePageShowEvent(ourDocshell, ourEventTarget, true);
-    nsContentUtils::FirePageShowEvent(otherDocshell, otherEventTarget, true);
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   // OK.  First begin to swap the docshells in the two nsIFrames
   rv = ourFrameFrame->BeginSwapDocShells(otherFrame);
   if (NS_FAILED(rv)) {
-    mInSwap = aOther->mInSwap = false;
-    nsContentUtils::FirePageShowEvent(ourDocshell, ourEventTarget, true);
-    nsContentUtils::FirePageShowEvent(otherDocshell, otherEventTarget, true);
     return rv;
   }
 
@@ -1290,10 +1328,6 @@ nsFrameLoader::SwapWithOtherLoader(nsFrameLoader* aOther,
   ourParentDocument->FlushPendingNotifications(Flush_Layout);
   otherParentDocument->FlushPendingNotifications(Flush_Layout);
 
-  nsContentUtils::FirePageShowEvent(ourDocshell, ourEventTarget, true);
-  nsContentUtils::FirePageShowEvent(otherDocshell, otherEventTarget, true);
-
-  mInSwap = aOther->mInSwap = false;
   return NS_OK;
 }
 

dom/base/nsFrameLoader.h

@@ -31,6 +31,7 @@ class nsSubDocumentFrame;
 class nsView;
 class nsIInProcessContentFrameMessageManager;
 class AutoResetInShow;
+class AutoResetInFrameSwap;
 class nsITabParent;
 class nsIDocShellTreeItem;
 class nsIDocShellTreeOwner;
@@ -61,6 +62,7 @@ class nsFrameLoader final : public nsIFrameLoader,
                             public mozilla::dom::ipc::MessageManagerCallback
 {
   friend class AutoResetInShow;
+  friend class AutoResetInFrameSwap;
   typedef mozilla::dom::PBrowserParent PBrowserParent;
   typedef mozilla::dom::TabParent TabParent;
   typedef mozilla::layout::RenderFrameParent RenderFrameParent;

dom/base/nsGlobalWindow.cpp

@@ -6365,7 +6365,6 @@ nsGlobalWindow::SetFullScreenInternal(bool aFullScreen, bool aFullscreenMode,
   } else if (mWakeLock && !mFullScreen) {
     ErrorResult rv;
     mWakeLock->Unlock(rv);
-    NS_WARN_IF_FALSE(!rv.Failed(), "Failed to unlock the wakelock.");
     mWakeLock = nullptr;
   }
 

dom/base/test/mochitest.ini

@@ -245,6 +245,7 @@ support-files =
   file_webaudioLoop.html
   file_webaudioLoop2.html
   file_pluginAudio.html
+  noaudio.webm
   referrer_helper.js
   referrer_testserver.sjs
   script_postmessages_fileList.js
@@ -265,6 +266,8 @@ support-files =
 skip-if = buildapp == 'mulet'
 [test_audioNotificationStopOnNavigation.html]
 skip-if = buildapp == 'mulet'
+[test_audioNotificationWithEarlyPlay.html]
+skip-if = buildapp == 'mulet'
 [test_bug1091883.html]
 [test_bug116083.html]
 [test_bug793311.html]
@@ -306,6 +309,10 @@ skip-if = buildapp == 'mulet' || buildapp == 'b2g' || toolkit == 'android' || e1
 [test_named_frames.html]
 [test_navigator_resolve_identity.html]
 [test_navigator_language.html]
+[test_noAudioNotification.html]
+[test_noAudioNotificationOnMutedElement.html]
+[test_noAudioNotificationOnMutedOrVolume0Element.html]
+[test_noAudioNotificationOnVolume0Element.html]
 [test_openDialogChromeOnly.html]
 [test_open_null_features.html]
 skip-if = (buildapp == 'b2g' && toolkit != 'gonk') # Fails on b2g-desktop, tracked in bug 1011874

dom/base/test/test_audioNotificationWithEarlyPlay.html

@@ -0,0 +1,73 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for audio controller in windows</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<pre id="test">
+</pre>
+
+<script type="application/javascript">
+
+SimpleTest.waitForExplicitFinish();
+
+var expectedNotification = null;
+
+var observer = {
+  observe: function(subject, topic, data) {
+    is(topic, "audio-playback", "audio-playback received");
+    is(data, expectedNotification, "This is the right notification");
+    runTest();
+  }
+};
+
+var observerService = SpecialPowers.Cc["@mozilla.org/observer-service;1"]
+                                   .getService(SpecialPowers.Ci.nsIObserverService);
+
+var audio = new Audio();
+audio.loop = true;
+audio.preload = "metadata";
+
+var tests = [
+  function() {
+    observerService.addObserver(observer, "audio-playback", false);
+    ok(true, "Observer set");
+    runTest();
+  },
+
+  function() {
+    expectedNotification = 'active';
+    audio.src = "audio.ogg";
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.pause();
+  },
+
+  function() {
+    observerService.removeObserver(observer, "audio-playback");
+    ok(true, "Observer removed");
+    runTest();
+  }
+];
+
+function runTest() {
+  if (!tests.length) {
+    SimpleTest.finish();
+    return;
+  }
+
+  var test = tests.shift();
+  test();
+}
+
+runTest();
+
+</script>
+</body>
+</html>
+

dom/base/test/test_noAudioNotification.html

@@ -0,0 +1,81 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for video controller in windows</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<pre id="test">
+</pre>
+
+<script type="application/javascript">
+
+SimpleTest.waitForExplicitFinish();
+
+var observer = {
+  observe: function(subject, topic, data) {
+    ok(false, "should not receive media-playback notification!");
+  }
+};
+
+var observerService = SpecialPowers.Cc["@mozilla.org/observer-service;1"]
+                                   .getService(SpecialPowers.Ci.nsIObserverService);
+
+var video = document.createElement("video");
+video.loop = true;
+video.src = "noaudio.webm";
+
+video.onplay = video.onpause  = function() {
+  // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+  SimpleTest.executeSoon(function() {
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        runTest();
+      });
+    });
+  });
+};
+
+var tests = [
+  function() {
+    SpecialPowers.pushPrefEnv({"set": [["media.useAudioChannelService", true]]}, runTest);
+  },
+
+  function() {
+    observerService.addObserver(observer, "media-playback", false);
+    ok(true, "Observer set");
+    runTest();
+  },
+
+  function() {
+    video.play();
+  },
+
+  function() {
+    video.pause();
+  },
+
+  function() {
+    observerService.removeObserver(observer, "media-playback");
+    ok(true, "Observer removed");
+    runTest();
+  }
+];
+
+function runTest() {
+  if (!tests.length) {
+    SimpleTest.finish();
+    return;
+  }
+
+  var test = tests.shift();
+  test();
+}
+
+runTest();
+
+</script>
+</body>
+</html>
+

dom/base/test/test_noAudioNotificationOnMutedElement.html

@@ -0,0 +1,133 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for audio controller in windows</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<pre id="test">
+</pre>
+
+<script type="application/javascript">
+
+SimpleTest.waitForExplicitFinish();
+
+var expectedNotification = null;
+
+var observer = {
+  observe: function(subject, topic, data) {
+    if (expectedNotification !== null) {
+      is(topic, "media-playback", "media-playback received");
+      is(data, expectedNotification, "This is the right notification");
+      runTest();
+    } else {
+      ok(false, "should not receive media-playback notification!");
+    }
+  }
+};
+
+var observerService = SpecialPowers.Cc["@mozilla.org/observer-service;1"]
+                                   .getService(SpecialPowers.Ci.nsIObserverService);
+
+var audio = new Audio();
+audio.loop = true;
+
+var tests = [
+  function() {
+    SpecialPowers.pushPrefEnv({"set": [["media.useAudioChannelService", true]]}, runTest);
+  },
+
+  function() {
+    observerService.addObserver(observer, "media-playback", false);
+    ok(true, "Observer set");
+
+    audio.src = "audio.ogg";
+    // Wait for the audio metadata to become available so that we have an audio track.
+    audio.onloadedmetadata = runTest;
+  },
+
+  // Verify that muting and unmuting dispatches the events as expected.
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.muted = true;
+  },
+
+  function() {
+    expectedNotification = 'active';
+    audio.muted = false;
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.pause();
+  },
+
+  // Verify that no events are dispatched when muted.
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.muted = true;
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.onpause  = function() {
+      // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          SimpleTest.executeSoon(function() {
+            audio.onpause = null;
+            runTest();
+          });
+        });
+      });
+    };
+    audio.pause();
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.muted = false;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    observerService.removeObserver(observer, "media-playback");
+    ok(true, "Observer removed");
+    runTest();
+  }
+];
+
+function runTest() {
+  if (!tests.length) {
+    SimpleTest.finish();
+    return;
+  }
+
+  var test = tests.shift();
+  test();
+}
+
+runTest();
+
+</script>
+</body>
+</html>
+

dom/base/test/test_noAudioNotificationOnMutedOrVolume0Element.html

@@ -0,0 +1,166 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for audio controller in windows</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<pre id="test">
+</pre>
+
+<script type="application/javascript">
+
+SimpleTest.waitForExplicitFinish();
+
+var expectedNotification = null;
+
+var observer = {
+  observe: function(subject, topic, data) {
+    if (expectedNotification !== null) {
+      is(topic, "media-playback", "media-playback received");
+      is(data, expectedNotification, "This is the right notification");
+      runTest();
+    } else {
+      ok(false, "should not receive media-playback notification!");
+    }
+  }
+};
+
+var observerService = SpecialPowers.Cc["@mozilla.org/observer-service;1"]
+                                   .getService(SpecialPowers.Ci.nsIObserverService);
+
+var audio = new Audio();
+audio.loop = true;
+
+var tests = [
+  function() {
+    SpecialPowers.pushPrefEnv({"set": [["media.useAudioChannelService", true]]}, runTest);
+  },
+
+  function() {
+    observerService.addObserver(observer, "media-playback", false);
+    ok(true, "Observer set");
+
+    audio.src = "audio.ogg";
+    // Wait for the audio metadata to become available so that we have an audio track.
+    audio.onloadedmetadata = runTest;
+  },
+
+  // Verify that unmuting when the volume is 0 doesn't dispatch the events.
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.muted = true;
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.volume = 0;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.muted = false;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    expectedNotification = 'active';
+    audio.volume = 0.5;
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.pause();
+  },
+
+  // Verify that raising the volume when muted doesn't dispatch the events.
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.muted = true;
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.volume = 0;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.volume = 0.5;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    expectedNotification = 'active';
+    audio.muted = false;
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.pause();
+  },
+
+  function() {
+    observerService.removeObserver(observer, "media-playback");
+    ok(true, "Observer removed");
+    runTest();
+  }
+];
+
+function runTest() {
+  if (!tests.length) {
+    SimpleTest.finish();
+    return;
+  }
+
+  var test = tests.shift();
+  test();
+}
+
+runTest();
+
+</script>
+</body>
+</html>
+

dom/base/test/test_noAudioNotificationOnVolume0Element.html

@@ -0,0 +1,133 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Test for audio controller in windows</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+</head>
+<body>
+<pre id="test">
+</pre>
+
+<script type="application/javascript">
+
+SimpleTest.waitForExplicitFinish();
+
+var expectedNotification = null;
+
+var observer = {
+  observe: function(subject, topic, data) {
+    if (expectedNotification !== null) {
+      is(topic, "media-playback", "media-playback received");
+      is(data, expectedNotification, "This is the right notification");
+      runTest();
+    } else {
+      ok(false, "should not receive media-playback notification!");
+    }
+  }
+};
+
+var observerService = SpecialPowers.Cc["@mozilla.org/observer-service;1"]
+                                   .getService(SpecialPowers.Ci.nsIObserverService);
+
+var audio = new Audio();
+audio.loop = true;
+
+var tests = [
+  function() {
+    SpecialPowers.pushPrefEnv({"set": [["media.useAudioChannelService", true]]}, runTest);
+  },
+
+  function() {
+    observerService.addObserver(observer, "media-playback", false);
+    ok(true, "Observer set");
+
+    audio.src = "audio.ogg";
+    // Wait for the audio metadata to become available so that we have an audio track.
+    audio.onloadedmetadata = runTest;
+  },
+
+  // Verify that muting and unmuting dispatches the events as expected.
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.volume = 0;
+  },
+
+  function() {
+    expectedNotification = 'active';
+    audio.volume = 1;
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.pause();
+  },
+
+  // Verify that no events are dispatched when volume is set to 0..
+  function() {
+    expectedNotification = 'active';
+    audio.play();
+  },
+
+  function() {
+    expectedNotification = 'inactive';
+    audio.volume = 0;
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.onpause  = function() {
+      // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          SimpleTest.executeSoon(function() {
+            audio.onpause = null;
+            runTest();
+          });
+        });
+      });
+    };
+    audio.pause();
+  },
+
+  function() {
+    expectedNotification = null;
+    audio.volume = 1;
+    // Yield to the event loop a few times to make sure that media-playback is not dispatched.
+    SimpleTest.executeSoon(function() {
+      SimpleTest.executeSoon(function() {
+        SimpleTest.executeSoon(function() {
+          runTest();
+        });
+      });
+    });
+  },
+
+  function() {
+    observerService.removeObserver(observer, "media-playback");
+    ok(true, "Observer removed");
+    runTest();
+  }
+];
+
+function runTest() {
+  if (!tests.length) {
+    SimpleTest.finish();
+    return;
+  }
+
+  var test = tests.shift();
+  test();
+}
+
+runTest();
+
+</script>
+</body>
+</html>
+

dom/camera/CameraPreviewMediaStream.cpp

@@ -36,7 +36,9 @@ CameraPreviewMediaStream::CameraPreviewMediaStream(DOMMediaStream* aWrapper)
   , mRateLimit(false)
   , mTrackCreated(false)
 {
-  SetGraphImpl(MediaStreamGraph::GetInstance());
+  SetGraphImpl(
+      MediaStreamGraph::GetInstance(
+        MediaStreamGraph::SYSTEM_THREAD_DRIVER, AudioChannel::Normal));
   mFakeMediaStreamGraph = new FakeMediaStreamGraph();
   mIsConsumed = false;
 }

dom/events/NotifyPaintEvent.cpp

@@ -26,7 +26,7 @@ NotifyPaintEvent::NotifyPaintEvent(EventTarget* aOwner,
     mEvent->mMessage = aEventMessage;
   }
   if (aInvalidateRequests) {
-    mInvalidateRequests.MoveElementsFrom(aInvalidateRequests->mRequests);
+    mInvalidateRequests.AppendElements(Move(aInvalidateRequests->mRequests));
   }
 }
 

dom/html/HTMLMediaElement.cpp

@@ -87,6 +87,7 @@
 #include "ImageContainer.h"
 #include "nsRange.h"
 #include <algorithm>
+#include <cmath>
 
 static PRLogModuleInfo* gMediaElementLog;
 static PRLogModuleInfo* gMediaElementEventsLog;
@@ -100,12 +101,44 @@ static PRLogModuleInfo* gMediaElementEventsLog;
 
 #include "nsIPermissionManager.h"
 #include "nsContentTypeParser.h"
+#include "nsDocShell.h"
 
 #include "mozilla/EventStateManager.h"
 
 using namespace mozilla::layers;
 using mozilla::net::nsMediaFragmentURIParser;
 
+class MOZ_STACK_CLASS AutoNotifyAudioChannelAgent
+{
+  nsRefPtr<mozilla::dom::HTMLMediaElement> mElement;
+  bool mShouldNotify;
+  MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER;
+public:
+  AutoNotifyAudioChannelAgent(mozilla::dom::HTMLMediaElement* aElement,
+                              bool aNotify
+                              MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
+    : mElement(aElement)
+    , mShouldNotify(aNotify)
+  {
+    MOZ_GUARD_OBJECT_NOTIFIER_INIT;
+    if (mShouldNotify) {
+      // The audio channel agent may not exist now.
+      if (mElement->MaybeCreateAudioChannelAgent()) {
+        mElement->NotifyAudioChannelAgent(false);
+      }
+    }
+  }
+  ~AutoNotifyAudioChannelAgent()
+  {
+    if (mShouldNotify) {
+      // The audio channel agent is destroyed at this point.
+      if (mElement->MaybeCreateAudioChannelAgent()) {
+        mElement->NotifyAudioChannelAgent(true);
+      }
+    }
+  }
+};
+
 namespace mozilla {
 namespace dom {
 
@@ -430,7 +463,6 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream)
-  NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlaybackStream)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer)
   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc)
@@ -576,8 +608,8 @@ NS_IMETHODIMP HTMLMediaElement::GetError(nsIDOMMediaError * *aError)
 bool
 HTMLMediaElement::Ended()
 {
-  if (mSrcStream) {
-    return GetSrcMediaStream()->IsFinished();
+  if (MediaStream* stream = GetSrcMediaStream()) {
+    return stream->IsFinished();
   }
 
   if (mDecoder) {
@@ -1337,11 +1369,11 @@ NS_IMETHODIMP HTMLMediaElement::GetSeeking(bool* aSeeking)
 double
 HTMLMediaElement::CurrentTime() const
 {
-  if (mSrcStream) {
-    MediaStream* stream = GetSrcMediaStream();
-    if (stream) {
-      return stream->StreamTimeToSeconds(stream->GetCurrentTime());
+  if (MediaStream* stream = GetSrcMediaStream()) {
+    if (mSrcStreamPausedCurrentTime >= 0) {
+      return mSrcStreamPausedCurrentTime;
     }
+    return stream->StreamTimeToSeconds(stream->GetCurrentTime());
   }
 
   if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) {
@@ -1641,14 +1673,9 @@ HTMLMediaElement::Pause(ErrorResult& aRv)
   mAutoplaying = false;
   // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference
   AddRemoveSelfReference();
+  UpdateSrcMediaStreamPlaying();
 
   if (!oldPaused) {
-    if (mSrcStream) {
-      MediaStream* stream = GetSrcMediaStream();
-      if (stream) {
-        stream->ChangeExplicitBlockerCount(1);
-      }
-    }
     FireTimeUpdate(false);
     DispatchAsyncEvent(NS_LITERAL_STRING("pause"));
   }
@@ -1789,9 +1816,13 @@ void HTMLMediaElement::SetVolumeInternal()
 
   if (mDecoder) {
     mDecoder->SetVolume(effectiveVolume);
-  } else if (mSrcStream) {
-    GetSrcMediaStream()->SetAudioOutputVolume(this, effectiveVolume);
+  } else if (MediaStream* stream = GetSrcMediaStream()) {
+    if (mSrcStreamIsPlaying) {
+      stream->SetAudioOutputVolume(this, effectiveVolume);
+    }
   }
+
+  UpdateAudioChannelPlayingState();
 }
 
 NS_IMETHODIMP HTMLMediaElement::SetMuted(bool aMuted)
@@ -1818,6 +1849,14 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded,
   if (!window) {
     return nullptr;
   }
+
+  if (!aGraph) {
+    MediaStreamGraph::GraphDriverType graphDriverType =
+      HasAudio() ? MediaStreamGraph::AUDIO_THREAD_DRIVER
+                 : MediaStreamGraph::SYSTEM_THREAD_DRIVER;
+    aGraph = MediaStreamGraph::GetInstance(graphDriverType, mAudioChannel);
+  }
+
   OutputMediaStream* out = mOutputStreams.AppendElement();
   out->mStream = DOMMediaStream::CreateTrackUnionStream(window, aGraph);
   nsRefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
@@ -1987,6 +2026,7 @@ HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI)
 HTMLMediaElement::HTMLMediaElement(already_AddRefed<mozilla::dom::NodeInfo>& aNodeInfo)
   : nsGenericHTMLElement(aNodeInfo),
     mWatchManager(this, AbstractThread::MainThread()),
+    mSrcStreamPausedCurrentTime(-1),
     mCurrentLoadID(0),
     mNetworkState(nsIDOMHTMLMediaElement::NETWORK_EMPTY),
     mReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING, "HTMLMediaElement::mReadyState"),
@@ -2029,6 +2069,7 @@ HTMLMediaElement::HTMLMediaElement(already_AddRefed<mozilla::dom::NodeInfo>& aNo
     mHasSelfReference(false),
     mShuttingDown(false),
     mSuspendedForPreloadNone(false),
+    mSrcStreamIsPlaying(false),
     mMediaSecurityVerified(false),
     mCORSMode(CORS_NONE),
     mIsEncrypted(false),
@@ -2225,9 +2266,6 @@ HTMLMediaElement::PlayInternal(bool aCallerIsChrome)
   // TODO: If the playback has ended, then the user agent must set
   // seek to the effective start.
   if (mPaused) {
-    if (mSrcStream) {
-      GetSrcMediaStream()->ChangeExplicitBlockerCount(-1);
-    }
     DispatchAsyncEvent(NS_LITERAL_STRING("play"));
     switch (mReadyState) {
     case nsIDOMHTMLMediaElement::HAVE_NOTHING:
@@ -2251,6 +2289,7 @@ HTMLMediaElement::PlayInternal(bool aCallerIsChrome)
   // and our preload status.
   AddRemoveSelfReference();
   UpdatePreloadAction();
+  UpdateSrcMediaStreamPlaying();
 
   return NS_OK;
 }
@@ -2343,7 +2382,6 @@ HTMLMediaElement::WakeLockRelease()
   if (mWakeLock) {
     ErrorResult rv;
     mWakeLock->Unlock(rv);
-    NS_WARN_IF_FALSE(!rv.Failed(), "Failed to unlock the wakelock.");
     mWakeLock = nullptr;
   }
 }
@@ -2847,6 +2885,7 @@ public:
     mElement(aElement),
     mHaveCurrentData(false),
     mBlocked(false),
+    mFinished(false),
     mMutex(aName),
     mPendingNotifyOutput(false)
   {}
@@ -2855,19 +2894,29 @@ public:
   // Main thread
   void DoNotifyFinished()
   {
+    mFinished = true;
     if (mElement) {
       nsRefPtr<HTMLMediaElement> deathGrip = mElement;
-      mElement->PlaybackEnded();
+
+      // Update NextFrameStatus() to move to NEXT_FRAME_UNAVAILABLE and
+      // HAVE_CURRENT_DATA.
+      mElement = nullptr;
+      // NotifyWatchers before calling PlaybackEnded since PlaybackEnded
+      // can remove watchers.
+      NotifyWatchers();
+
+      deathGrip->PlaybackEnded();
     }
   }
 
   MediaDecoderOwner::NextFrameStatus NextFrameStatus()
   {
-    if (!mElement || !mHaveCurrentData) {
+    if (!mElement || !mHaveCurrentData || mFinished) {
       return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
     }
-    return mBlocked ? MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING
-                    : MediaDecoderOwner::NEXT_FRAME_AVAILABLE;
+    return mBlocked
+        ? MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING
+        : MediaDecoderOwner::NEXT_FRAME_AVAILABLE;
   }
 
   void DoNotifyBlocked()
@@ -2947,6 +2996,7 @@ private:
   HTMLMediaElement* mElement;
   bool mHaveCurrentData;
   bool mBlocked;
+  bool mFinished;
 
   // mMutex protects the fields below; they can be accessed on any thread
   Mutex mMutex;
@@ -3028,6 +3078,82 @@ private:
   WeakPtr<HTMLMediaElement> mElement;
 };
 
+void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags)
+{
+  if (!mSrcStream) {
+    return;
+  }
+  // We might be in cycle collection with mSrcStream->GetStream() already
+  // returning null due to unlinking.
+
+  MediaStream* stream = mSrcStream->GetStream();
+  bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused &&
+      !mPausedForInactiveDocumentOrChannel && stream;
+  if (shouldPlay == mSrcStreamIsPlaying) {
+    return;
+  }
+  mSrcStreamIsPlaying = shouldPlay;
+
+  if (shouldPlay) {
+    mSrcStreamPausedCurrentTime = -1;
+
+    mMediaStreamListener = new StreamListener(this,
+        "HTMLMediaElement::mMediaStreamListener");
+    mMediaStreamSizeListener = new StreamSizeListener(this);
+    stream->AddListener(mMediaStreamListener);
+    stream->AddListener(mMediaStreamSizeListener);
+
+    mWatchManager.Watch(*mMediaStreamListener,
+        &HTMLMediaElement::UpdateReadyStateInternal);
+
+  nsRefPtr<MediaStream> stream = mSrcStream->GetStream();
+  if (stream) {
+    stream->SetAudioChannelType(mAudioChannel);
+  }
+
+    stream->AddAudioOutput(this);
+    SetVolumeInternal();
+
+  #ifdef MOZ_WIDGET_GONK
+    bool bUseOverlayImage = mSrcStream->AsDOMHwMediaStream() != nullptr;
+  #else
+    bool bUseOverlayImage = false;
+  #endif
+    VideoFrameContainer* container;
+    if (bUseOverlayImage) {
+      container = GetOverlayImageVideoFrameContainer();
+    } else {
+      container = GetVideoFrameContainer();
+    }
+    if (container) {
+      stream->AddVideoOutput(container);
+    }
+  } else {
+    if (stream) {
+      mSrcStreamPausedCurrentTime = CurrentTime();
+
+      stream->RemoveListener(mMediaStreamListener);
+      stream->RemoveListener(mMediaStreamSizeListener);
+
+      stream->RemoveAudioOutput(this);
+      VideoFrameContainer* container = GetVideoFrameContainer();
+      if (container) {
+        stream->RemoveVideoOutput(container);
+      }
+    }
+    // If stream is null, then DOMMediaStream::Destroy must have been
+    // called and that will remove all listeners/outputs.
+
+    mWatchManager.Unwatch(*mMediaStreamListener,
+        &HTMLMediaElement::UpdateReadyStateInternal);
+
+    mMediaStreamListener->Forget();
+    mMediaStreamListener = nullptr;
+    mMediaStreamSizeListener->Forget();
+    mMediaStreamSizeListener = nullptr;
+  }
+}
+
 void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
 {
   NS_ASSERTION(!mSrcStream && !mMediaStreamListener && !mMediaStreamSizeListener,
@@ -3040,116 +3166,38 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream)
     return;
   }
 
-  // XXX Remove this if with CameraPreviewMediaStream per bug 1124630.
-  if (!mSrcStream->GetStream()->AsCameraPreviewStream()) {
-    // Now that we have access to |mSrcStream| we can pipe it to our shadow
-    // version |mPlaybackStream|. If two media elements are playing the
-    // same realtime DOMMediaStream, this allows them to pause playback
-    // independently of each other.
-    mPlaybackStream = DOMMediaStream::CreateTrackUnionStream(window);
-    mPlaybackStreamInputPort = mPlaybackStream->GetStream()->AsProcessedStream()->
-      AllocateInputPort(mSrcStream->GetStream(), MediaInputPort::FLAG_BLOCK_OUTPUT);
-
-    nsRefPtr<nsIPrincipal> principal = GetCurrentPrincipal();
-    mPlaybackStream->CombineWithPrincipal(principal);
-
-    // Let |mSrcStream| decide when the stream has finished.
-    GetSrcMediaStream()->AsProcessedStream()->SetAutofinish(true);
-  }
-
   nsRefPtr<MediaStream> stream = mSrcStream->GetStream();
   if (stream) {
     stream->SetAudioChannelType(mAudioChannel);
   }
 
-  // XXX if we ever support capturing the output of a media element which is
-  // playing a stream, we'll need to add a CombineWithPrincipal call here.
-  mMediaStreamListener = new StreamListener(this, "HTMLMediaElement::mMediaStreamListener");
-  mMediaStreamSizeListener = new StreamSizeListener(this);
-  mWatchManager.Watch(*mMediaStreamListener, &HTMLMediaElement::UpdateReadyStateInternal);
-
-  GetSrcMediaStream()->AddListener(mMediaStreamListener);
-  // Listen for an initial image size on mSrcStream so we can get results even
-  // if we block the mPlaybackStream.
-  stream->AddListener(mMediaStreamSizeListener);
-  if (mPaused) {
-    GetSrcMediaStream()->ChangeExplicitBlockerCount(1);
-  }
-  if (mPausedForInactiveDocumentOrChannel) {
-    GetSrcMediaStream()->ChangeExplicitBlockerCount(1);
-  }
-
-  ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE);
-
-  ChangeDelayLoadStatus(false);
-  GetSrcMediaStream()->AddAudioOutput(this);
-  SetVolumeInternal();
-
-  bool bUseOverlayImage = mSrcStream->AsDOMHwMediaStream() != nullptr;
-  VideoFrameContainer* container;
-
-  if (bUseOverlayImage) {
-    container = GetOverlayImageVideoFrameContainer();
-  }
-  else {
-    container = GetVideoFrameContainer();
-  }
-
-  if (container) {
-    GetSrcMediaStream()->AddVideoOutput(container);
-  }
-
-  CheckAutoplayDataReady();
+  UpdateSrcMediaStreamPlaying();
 
   // Note: we must call DisconnectTrackListListeners(...)  before dropping
-  // mSrcStream
+  // mSrcStream.
+  // If we pause this media element, track changes in the underlying stream
+  // will continue to fire events at this element and alter its track list.
+  // That's simpler than delaying the events, but probably confusing...
   mSrcStream->ConstructMediaTracks(AudioTracks(), VideoTracks());
 
   mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this));
 
+  ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE);
+  ChangeDelayLoadStatus(false);
+  CheckAutoplayDataReady();
+
   // FirstFrameLoaded() will be called when the stream has current data.
 }
 
 void HTMLMediaElement::EndSrcMediaStreamPlayback()
 {
-  MediaStream* stream = GetSrcMediaStream();
-  if (stream) {
-    stream->RemoveListener(mMediaStreamListener);
-  }
-  if (mSrcStream->GetStream()) {
-    mSrcStream->GetStream()->RemoveListener(mMediaStreamSizeListener);
-  }
+  MOZ_ASSERT(mSrcStream);
+
+  UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM);
+
   mSrcStream->DisconnectTrackListListeners(AudioTracks(), VideoTracks());
 
-  if (mPlaybackStreamInputPort) {
-    mPlaybackStreamInputPort->Destroy();
-  }
-
-  // Kill its reference to this element
-  mWatchManager.Unwatch(*mMediaStreamListener, &HTMLMediaElement::UpdateReadyStateInternal);
-  mMediaStreamListener->Forget();
-  mMediaStreamListener = nullptr;
-  mMediaStreamSizeListener->Forget();
-  mMediaStreamSizeListener = nullptr;
-  if (stream) {
-    stream->RemoveAudioOutput(this);
-  }
-  VideoFrameContainer* container = GetVideoFrameContainer();
-  if (container) {
-    if (stream) {
-      stream->RemoveVideoOutput(container);
-    }
-    container->ClearCurrentFrame();
-  }
-  if (mPaused && stream) {
-    stream->ChangeExplicitBlockerCount(-1);
-  }
-  if (mPausedForInactiveDocumentOrChannel && stream) {
-    stream->ChangeExplicitBlockerCount(-1);
-  }
   mSrcStream = nullptr;
-  mPlaybackStreamInputPort = nullptr;
-  mPlaybackStream = nullptr;
 }
 
 void HTMLMediaElement::ProcessMediaFragmentURI()
@@ -3171,6 +3219,14 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo,
 {
   MOZ_ASSERT(NS_IsMainThread());
 
+  // If the element is gaining or losing an audio track, we need to notify
+  // the audio channel agent so that the correct audio-playback events will
+  // get dispatched.
+  bool audioTrackChanging = mMediaInfo.HasAudio() != aInfo->HasAudio();
+  AutoNotifyAudioChannelAgent autoNotify(this,
+                                         audioTrackChanging &&
+                                         mPlayingThroughTheAudioChannel);
+
   mMediaInfo = *aInfo;
   mIsEncrypted = aInfo->IsEncrypted();
   mTags = aTags.forget();
@@ -3741,6 +3797,7 @@ void HTMLMediaElement::CheckAutoplayDataReady()
   mPaused = false;
   // We changed mPaused which can affect AddRemoveSelfReference
   AddRemoveSelfReference();
+  UpdateSrcMediaStreamPlaying();
 
   if (mDecoder) {
     SetPlayedOrSeeked(true);
@@ -3750,7 +3807,6 @@ void HTMLMediaElement::CheckAutoplayDataReady()
     mDecoder->Play();
   } else if (mSrcStream) {
     SetPlayedOrSeeked(true);
-    GetSrcMediaStream()->ChangeExplicitBlockerCount(-1);
   }
   DispatchAsyncEvent(NS_LITERAL_STRING("play"));
 
@@ -3941,6 +3997,7 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE
 
   if (aPauseElement != mPausedForInactiveDocumentOrChannel) {
     mPausedForInactiveDocumentOrChannel = aPauseElement;
+    UpdateSrcMediaStreamPlaying();
     if (aPauseElement) {
       if (mMediaSource) {
         ReportMSETelemetry();
@@ -3949,8 +4006,6 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE
       if (mDecoder) {
         mDecoder->Pause();
         mDecoder->Suspend();
-      } else if (mSrcStream) {
-        GetSrcMediaStream()->ChangeExplicitBlockerCount(1);
       }
       mEventDeliveryPaused = aSuspendEvents;
     } else {
@@ -3959,8 +4014,6 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE
         if (!mPaused && !mDecoder->IsEndedOrShutdown()) {
           mDecoder->Play();
         }
-      } else if (mSrcStream) {
-        GetSrcMediaStream()->ChangeExplicitBlockerCount(-1);
       }
       if (mEventDeliveryPaused) {
         mEventDeliveryPaused = false;
@@ -3987,7 +4040,14 @@ void HTMLMediaElement::NotifyOwnerDocumentActivityChanged()
   if (pauseElement && mAudioChannelAgent) {
     // If the element is being paused since we are navigating away from the
     // document, notify the audio channel agent.
-    NotifyAudioChannelAgent(false);
+    // Be careful to ignore this event during a docshell frame swap.
+    auto docShell = static_cast<nsDocShell*>(OwnerDoc()->GetDocShell());
+    if (!docShell) {
+      return;
+    }
+    if (!docShell->InFrameSwap()) {
+      NotifyAudioChannelAgent(false);
+    }
   }
 }
 
@@ -4450,7 +4510,25 @@ nsresult HTMLMediaElement::UpdateChannelMuteState(float aVolume, bool aMuted)
   return NS_OK;
 }
 
-void HTMLMediaElement::UpdateAudioChannelPlayingState()
+bool
+HTMLMediaElement::MaybeCreateAudioChannelAgent()
+{
+  if (!mAudioChannelAgent) {
+    nsresult rv;
+    mAudioChannelAgent = do_CreateInstance("@mozilla.org/audiochannelagent;1", &rv);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return false;
+    }
+    MOZ_ASSERT(mAudioChannelAgent);
+    mAudioChannelAgent->InitWithWeakCallback(OwnerDoc()->GetInnerWindow(),
+                                             static_cast<int32_t>(mAudioChannel),
+                                             this);
+  }
+  return true;
+}
+
+void
+HTMLMediaElement::UpdateAudioChannelPlayingState()
 {
   if (!UseAudioChannelService()) {
     return;
@@ -4458,6 +4536,8 @@ void HTMLMediaElement::UpdateAudioChannelPlayingState()
 
   bool playingThroughTheAudioChannel =
      (!mPaused &&
+      !Muted() &&
+      std::fabs(Volume()) > 1e-7 &&
       (HasAttr(kNameSpaceID_None, nsGkAtoms::loop) ||
        (mReadyState >= nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA &&
         !IsPlaybackEnded()) ||
@@ -4470,24 +4550,20 @@ void HTMLMediaElement::UpdateAudioChannelPlayingState()
        return;
     }
 
-    if (!mAudioChannelAgent) {
-      nsresult rv;
-      mAudioChannelAgent = do_CreateInstance("@mozilla.org/audiochannelagent;1", &rv);
-      if (!mAudioChannelAgent) {
-        return;
-      }
-      mAudioChannelAgent->InitWithWeakCallback(OwnerDoc()->GetInnerWindow(),
-                                               static_cast<int32_t>(mAudioChannel),
-                                               this);
+    if (MaybeCreateAudioChannelAgent()) {
+      NotifyAudioChannelAgent(mPlayingThroughTheAudioChannel);
     }
-
-    NotifyAudioChannelAgent(mPlayingThroughTheAudioChannel);
   }
 }
 
 void
 HTMLMediaElement::NotifyAudioChannelAgent(bool aPlaying)
 {
+  // Don't do anything if this element doesn't have any audio tracks.
+  if (!HasAudio()) {
+    return;
+  }
+
   // Immediately check if this should go to the MSG instead of the normal
   // media playback route.
   WindowAudioCaptureChanged();
@@ -4536,13 +4612,15 @@ NS_IMETHODIMP HTMLMediaElement::WindowAudioCaptureChanged()
       nsCOMPtr<nsPIDOMWindow> window =
         do_QueryInterface(OwnerDoc()->GetParentObject());
       uint64_t id = window->WindowID();
-      MediaStreamGraph* msg = MediaStreamGraph::GetInstance();
+      MediaStreamGraph* msg =
+        MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER,
+                                      AudioChannel::Normal);
 
-      if (!mPlaybackStream) {
+      if (mSrcStream) {
+        mCaptureStreamPort = msg->ConnectToCaptureStream(id, mSrcStream->GetStream());
+      } else {
         nsRefPtr<DOMMediaStream> stream = CaptureStreamInternal(false, msg);
         mCaptureStreamPort = msg->ConnectToCaptureStream(id, stream->GetStream());
-      } else {
-        mCaptureStreamPort = msg->ConnectToCaptureStream(id, mPlaybackStream->GetStream());
       }
     } else {
       mAudioCapturedByWindow = false;

dom/html/HTMLMediaElement.h

@@ -33,10 +33,6 @@
 // Define to output information on decoding and painting framerate
 /* #define DEBUG_FRAME_RATE 1 */
 
-class nsIChannel;
-class nsIHttpChannel;
-class nsILoadGroup;
-
 typedef uint16_t nsMediaNetworkState;
 typedef uint16_t nsMediaReadyState;
 
@@ -54,9 +50,13 @@ class MediaTrack;
 } // namespace dom
 } // namespace mozilla
 
+class AutoNotifyAudioChannelAgent;
+class nsIChannel;
+class nsIHttpChannel;
+class nsILoadGroup;
+class nsIRunnable;
 class nsITimer;
 class nsRange;
-class nsIRunnable;
 
 namespace mozilla {
 namespace dom {
@@ -77,6 +77,8 @@ class HTMLMediaElement : public nsGenericHTMLElement,
                          public nsIAudioChannelAgentCallback,
                          public SupportsWeakPtr<HTMLMediaElement>
 {
+  friend AutoNotifyAudioChannelAgent;
+
 public:
   typedef mozilla::TimeStamp TimeStamp;
   typedef mozilla::layers::ImageContainer ImageContainer;
@@ -346,14 +348,14 @@ public:
    */
   virtual void FireTimeUpdate(bool aPeriodic) final override;
 
+  /**
+   * This will return null if mSrcStream is null, or if mSrcStream is not
+   * null but its GetStream() returns null --- which can happen during
+   * cycle collection unlinking!
+   */
   MediaStream* GetSrcMediaStream() const
   {
-    NS_ASSERTION(mSrcStream, "Don't call this when not playing a stream");
-    if (!mPlaybackStream) {
-      // XXX Remove this check with CameraPreviewMediaStream per bug 1124630.
-      return mSrcStream->GetStream();
-    }
-    return mPlaybackStream->GetStream();
+    return mSrcStream ? mSrcStream->GetStream() : nullptr;
   }
 
   // WebIDL
@@ -727,6 +729,11 @@ protected:
    * Stop playback on mSrcStream.
    */
   void EndSrcMediaStreamPlayback();
+  /**
+   * Ensure we're playing mSrcStream if and only if we're not paused.
+   */
+  enum { REMOVING_SRC_STREAM = 0x1 };
+  void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0);
 
   /**
    * Returns an nsDOMMediaStream containing the played contents of this
@@ -1029,6 +1036,10 @@ protected:
   // Notifies the audio channel agent when the element starts or stops playing.
   void NotifyAudioChannelAgent(bool aPlaying);
 
+  // Creates the audio channel agent if needed.  Returns true if the audio
+  // channel agent is ready to be used.
+  bool MaybeCreateAudioChannelAgent();
+
   class nsAsyncEventRunner;
   using nsGenericHTMLElement::DispatchEvent;
   // For nsAsyncEventRunner.
@@ -1054,8 +1065,9 @@ protected:
   // At most one of mDecoder and mSrcStream can be non-null.
   nsRefPtr<DOMMediaStream> mSrcStream;
 
-  // Holds a reference to a MediaInputPort connecting mSrcStream to mPlaybackStream.
-  nsRefPtr<MediaInputPort> mPlaybackStreamInputPort;
+  // If non-negative, the time we should return for currentTime while playing
+  // mSrcStream.
+  double mSrcStreamPausedCurrentTime;
 
   // Holds a reference to the stream connecting this stream to the capture sink.
   nsRefPtr<MediaInputPort> mCaptureStreamPort;
@@ -1338,6 +1350,9 @@ protected:
   // stored in mPreloadURI.
   bool mSuspendedForPreloadNone;
 
+  // True if we've connected mSrcStream to the media element output.
+  bool mSrcStreamIsPlaying;
+
   // True if a same-origin check has been done for the media element and resource.
   bool mMediaSecurityVerified;
 

dom/html/HTMLVideoElement.cpp

@@ -262,7 +262,6 @@ HTMLVideoElement::UpdateScreenWakeLock()
   if (mScreenWakeLock && (mPaused || hidden)) {
     ErrorResult rv;
     mScreenWakeLock->Unlock(rv);
-    NS_WARN_IF_FALSE(!rv.Failed(), "Failed to unlock the wakelock.");
     mScreenWakeLock = nullptr;
     return;
   }

dom/inputport/InputPort.cpp

@@ -4,6 +4,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "MediaStreamGraph.h"
 #include "DOMMediaStream.h"
 #include "InputPortData.h"
 #include "InputPortListeners.h"
@@ -60,7 +61,10 @@ InputPort::Init(nsIInputPortData* aData, nsIInputPortListener* aListener, ErrorR
   mInputPortListener = static_cast<InputPortListener*>(aListener);
   mInputPortListener->RegisterInputPort(this);
 
-  mStream = DOMMediaStream::CreateSourceStream(GetOwner());
+  MediaStreamGraph* graph =
+    MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER,
+                                  AudioChannel::Normal);
+  mStream = DOMMediaStream::CreateSourceStream(GetOwner(), graph);
 }
 
 void

dom/ipc/TabChild.cpp

@@ -2463,11 +2463,18 @@ TabChild::RecvSwappedWithOtherRemoteLoader()
     return true;
   }
 
+  nsRefPtr<nsDocShell> docShell = static_cast<nsDocShell*>(ourDocShell.get());
+
   nsCOMPtr<EventTarget> ourEventTarget = ourWindow->GetParentTarget();
 
+  docShell->SetInFrameSwap(true);
+
   nsContentUtils::FirePageShowEvent(ourDocShell, ourEventTarget, false);
   nsContentUtils::FirePageHideEvent(ourDocShell, ourEventTarget);
   nsContentUtils::FirePageShowEvent(ourDocShell, ourEventTarget, true);
+
+  docShell->SetInFrameSwap(false);
+
   return true;
 }
 

dom/media/ADTSDemuxer.cpp

@@ -315,7 +315,7 @@ ADTSDemuxer::Init()
     ADTSLOG("Init() failure: waiting for data");
 
     return InitPromise::CreateAndReject(
-      DemuxerFailureReason::WAITING_FOR_DATA, __func__);
+      DemuxerFailureReason::DEMUXER_ERROR, __func__);
   }
 
   ADTSLOG("Init() successful");
@@ -429,16 +429,6 @@ ADTSTrackDemuxer::GetInfo() const
   return mInfo->Clone();
 }
 
-already_AddRefed<MediaDataDemuxer>
-ADTSDemuxer::Clone() const {
-  nsRefPtr<ADTSDemuxer> demuxer = new ADTSDemuxer(mSource);
-  if (!demuxer->InitInternal()) {
-    NS_WARNING("Couldn't recreate ADTSTrackDemuxer");
-    return nullptr;
-  }
-  return demuxer.forget();
-}
-
 nsRefPtr<ADTSTrackDemuxer::SeekPromise>
 ADTSTrackDemuxer::Seek(media::TimeUnit aTime)
 {
@@ -450,13 +440,6 @@ ADTSTrackDemuxer::Seek(media::TimeUnit aTime)
   return SeekPromise::CreateAndResolve(seekTime, __func__);
 }
 
-int64_t
-ADTSTrackDemuxer::GetEvictionOffset(media::TimeUnit aTime)
-{
-  // Unused.
-  return 0;
-}
-
 media::TimeUnit
 ADTSTrackDemuxer::FastSeek(const media::TimeUnit& aTime)
 {

dom/media/ADTSDemuxer.h

@@ -28,7 +28,6 @@ public:
   explicit ADTSDemuxer(MediaResource* aSource);
   nsRefPtr<InitPromise>  Init() override;
   bool HasTrackType(TrackInfo::TrackType aType) const override;
-  already_AddRefed<MediaDataDemuxer> Clone() const override;
   uint32_t GetNumberTracks(TrackInfo::TrackType aType) const override;
   already_AddRefed<MediaTrackDemuxer> GetTrackDemuxer(
     TrackInfo::TrackType aType, uint32_t aTrackNumber) override;
@@ -69,8 +68,6 @@ public:
   int64_t GetResourceOffset() const override;
   media::TimeIntervals GetBuffered() override;
 
-  int64_t GetEvictionOffset(media::TimeUnit aTime) override;
-
 private:
   // Destructor.
   ~ADTSTrackDemuxer();

dom/media/AbstractMediaDecoder.h

@@ -49,9 +49,6 @@ public:
   // state.
   virtual ReentrantMonitor& GetReentrantMonitor() = 0;
 
-  // Returns true if the decoder is shut down.
-  virtual bool IsShutdown() const = 0;
-
   // A special version of the above for the ogg decoder that is allowed to be
   // called cross-thread.
   virtual bool IsOggDecoderShutdown() { return false; }
@@ -90,6 +87,13 @@ public:
   // Set the media as being seekable or not.
   virtual void SetMediaSeekable(bool aMediaSeekable) = 0;
 
+  void DispatchSetMediaSeekable(bool aMediaSeekable)
+  {
+    nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
+      this, &AbstractMediaDecoder::SetMediaSeekable, aMediaSeekable);
+    NS_DispatchToMainThread(r);
+  }
+
   virtual VideoFrameContainer* GetVideoFrameContainer() = 0;
   virtual mozilla::layers::ImageContainer* GetImageContainer() = 0;
 
@@ -100,11 +104,8 @@ public:
   virtual bool IsMediaSeekable() = 0;
 
   virtual void MetadataLoaded(nsAutoPtr<MediaInfo> aInfo, nsAutoPtr<MetadataTags> aTags, MediaDecoderEventVisibility aEventVisibility) = 0;
-  virtual void QueueMetadata(int64_t aTime, nsAutoPtr<MediaInfo> aInfo, nsAutoPtr<MetadataTags> aTags) = 0;
   virtual void FirstFrameLoaded(nsAutoPtr<MediaInfo> aInfo, MediaDecoderEventVisibility aEventVisibility) = 0;
 
-  virtual void RemoveMediaTracks() = 0;
-
   // May be called by the reader to notify this decoder that the metadata from
   // the media file has been read. Call on the decode thread only.
   virtual void OnReadMetadataCompleted() = 0;
@@ -213,45 +214,6 @@ public:
   }
 };
 
-class MetadataUpdatedEventRunner : public nsRunnable, private MetadataContainer
-{
-public:
-  MetadataUpdatedEventRunner(AbstractMediaDecoder* aDecoder,
-                             nsAutoPtr<MediaInfo> aInfo,
-                             nsAutoPtr<MetadataTags> aTags,
-                             MediaDecoderEventVisibility aEventVisibility = MediaDecoderEventVisibility::Observable)
-    : MetadataContainer(aDecoder, aInfo, aTags, aEventVisibility)
-  {}
-
-  NS_IMETHOD Run() override
-  {
-    nsAutoPtr<MediaInfo> info(new MediaInfo());
-    *info = *mInfo;
-    mDecoder->MetadataLoaded(info, mTags, mEventVisibility);
-    mDecoder->FirstFrameLoaded(mInfo, mEventVisibility);
-    return NS_OK;
-  }
-};
-
-class RemoveMediaTracksEventRunner : public nsRunnable
-{
-public:
-  explicit RemoveMediaTracksEventRunner(AbstractMediaDecoder* aDecoder)
-    : mDecoder(aDecoder)
-  {}
-
-  NS_IMETHOD Run() override
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-
-    mDecoder->RemoveMediaTracks();
-    return NS_OK;
-  }
-
-private:
-  nsRefPtr<AbstractMediaDecoder> mDecoder;
-};
-
 } // namespace mozilla
 
 #endif

dom/media/AudioChannelFormat.cpp

@@ -15,79 +15,4 @@ GetAudioChannelsSuperset(uint32_t aChannels1, uint32_t aChannels2)
   return std::max(aChannels1, aChannels2);
 }
 
-/**
- * UpMixMatrix represents a conversion matrix by exploiting the fact that
- * each output channel comes from at most one input channel.
- */
-struct UpMixMatrix {
-  uint8_t mInputDestination[CUSTOM_CHANNEL_LAYOUTS];
-};
-
-static const UpMixMatrix
-gUpMixMatrices[CUSTOM_CHANNEL_LAYOUTS*(CUSTOM_CHANNEL_LAYOUTS - 1)/2] =
-{
-  // Upmixes from mono
-  { { 0, 0 } },
-  { { 0, IGNORE, IGNORE } },
-  { { 0, 0, IGNORE, IGNORE } },
-  { { 0, IGNORE, IGNORE, IGNORE, IGNORE } },
-  { { IGNORE, IGNORE, 0, IGNORE, IGNORE, IGNORE } },
-  // Upmixes from stereo
-  { { 0, 1, IGNORE } },
-  { { 0, 1, IGNORE, IGNORE } },
-  { { 0, 1, IGNORE, IGNORE, IGNORE } },
-  { { 0, 1, IGNORE, IGNORE, IGNORE, IGNORE } },
-  // Upmixes from 3-channel
-  { { 0, 1, 2, IGNORE } },
-  { { 0, 1, 2, IGNORE, IGNORE } },
-  { { 0, 1, 2, IGNORE, IGNORE, IGNORE } },
-  // Upmixes from quad
-  { { 0, 1, 2, 3, IGNORE } },
-  { { 0, 1, IGNORE, IGNORE, 2, 3 } },
-  // Upmixes from 5-channel
-  { { 0, 1, 2, 3, 4, IGNORE } }
-};
-
-void
-AudioChannelsUpMix(nsTArray<const void*>* aChannelArray,
-                   uint32_t aOutputChannelCount,
-                   const void* aZeroChannel)
-{
-  uint32_t inputChannelCount = aChannelArray->Length();
-  uint32_t outputChannelCount =
-    GetAudioChannelsSuperset(aOutputChannelCount, inputChannelCount);
-  NS_ASSERTION(outputChannelCount > inputChannelCount,
-               "No up-mix needed");
-  MOZ_ASSERT(inputChannelCount > 0, "Bad number of channels");
-  MOZ_ASSERT(outputChannelCount > 0, "Bad number of channels");
-
-  aChannelArray->SetLength(outputChannelCount);
-
-  if (inputChannelCount < CUSTOM_CHANNEL_LAYOUTS &&
-      outputChannelCount <= CUSTOM_CHANNEL_LAYOUTS) {
-    const UpMixMatrix& m = gUpMixMatrices[
-      gMixingMatrixIndexByChannels[inputChannelCount - 1] +
-      outputChannelCount - inputChannelCount - 1];
-
-    const void* outputChannels[CUSTOM_CHANNEL_LAYOUTS];
-
-    for (uint32_t i = 0; i < outputChannelCount; ++i) {
-      uint8_t channelIndex = m.mInputDestination[i];
-      if (channelIndex == IGNORE) {
-        outputChannels[i] = aZeroChannel;
-      } else {
-        outputChannels[i] = aChannelArray->ElementAt(channelIndex);
-      }
-    }
-    for (uint32_t i = 0; i < outputChannelCount; ++i) {
-      aChannelArray->ElementAt(i) = outputChannels[i];
-    }
-    return;
-  }
-
-  for (uint32_t i = inputChannelCount; i < outputChannelCount; ++i) {
-    aChannelArray->ElementAt(i) = aZeroChannel;
-  }
-}
-
 } // namespace mozilla

dom/media/AudioChannelFormat.h

@@ -58,24 +58,6 @@ const int gMixingMatrixIndexByChannels[CUSTOM_CHANNEL_LAYOUTS - 1] =
 uint32_t
 GetAudioChannelsSuperset(uint32_t aChannels1, uint32_t aChannels2);
 
-/**
- * Given an array of input channel data, and an output channel count,
- * replaces the array with an array of upmixed channels.
- * This shuffles the array and may set some channel buffers to aZeroChannel.
- * Don't call this with input count >= output count.
- * This may return *more* channels than requested. In that case, downmixing
- * is required to to get to aOutputChannelCount. (This is how we handle
- * odd cases like 3 -> 4 upmixing.)
- * If aChannelArray.Length() was the input to one of a series of
- * GetAudioChannelsSuperset calls resulting in aOutputChannelCount,
- * no downmixing will be required.
- */
-void
-AudioChannelsUpMix(nsTArray<const void*>* aChannelArray,
-                   uint32_t aOutputChannelCount,
-                   const void* aZeroChannel);
-
-
 /**
  * DownMixMatrix represents a conversion matrix efficiently by exploiting the
  * fact that each input channel contributes to at most one output channel,
@@ -124,19 +106,19 @@ gDownMixMatrices[CUSTOM_CHANNEL_LAYOUTS*(CUSTOM_CHANNEL_LAYOUTS - 1)/2] =
  * input count <= output count.
  */
 template<typename T>
-void AudioChannelsDownMix(const nsTArray<const void*>& aChannelArray,
-                     T** aOutputChannels,
-                     uint32_t aOutputChannelCount,
-                     uint32_t aDuration)
+void AudioChannelsDownMix(const nsTArray<const T*>& aChannelArray,
+                          T** aOutputChannels,
+                          uint32_t aOutputChannelCount,
+                          uint32_t aDuration)
 {
   uint32_t inputChannelCount = aChannelArray.Length();
-  const void* const* inputChannels = aChannelArray.Elements();
+  const T* const* inputChannels = aChannelArray.Elements();
   NS_ASSERTION(inputChannelCount > aOutputChannelCount, "Nothing to do");
 
   if (inputChannelCount > 6) {
     // Just drop the unknown channels.
     for (uint32_t o = 0; o < aOutputChannelCount; ++o) {
-      memcpy(aOutputChannels[o], inputChannels[o], aDuration*sizeof(T));
+      PodCopy(aOutputChannels[o], inputChannels[o], aDuration);
     }
     return;
   }
@@ -153,8 +135,7 @@ void AudioChannelsDownMix(const nsTArray<const void*>& aChannelArray,
   for (uint32_t s = 0; s < aDuration; ++s) {
     // Reserve an extra junk channel at the end for the cases where we
     // want an input channel to contribute to nothing
-    T outputChannels[CUSTOM_CHANNEL_LAYOUTS + 1];
-    memset(outputChannels, 0, sizeof(T)*(CUSTOM_CHANNEL_LAYOUTS));
+    T outputChannels[CUSTOM_CHANNEL_LAYOUTS + 1] = {0};
     for (uint32_t c = 0; c < inputChannelCount; ++c) {
       outputChannels[m.mInputDestination[c]] +=
         m.mInputCoefficient[c]*(static_cast<const T*>(inputChannels[c]))[s];
@@ -171,6 +152,94 @@ void AudioChannelsDownMix(const nsTArray<const void*>& aChannelArray,
   }
 }
 
+/**
+ * UpMixMatrix represents a conversion matrix by exploiting the fact that
+ * each output channel comes from at most one input channel.
+ */
+struct UpMixMatrix {
+  uint8_t mInputDestination[CUSTOM_CHANNEL_LAYOUTS];
+};
+
+static const UpMixMatrix
+gUpMixMatrices[CUSTOM_CHANNEL_LAYOUTS*(CUSTOM_CHANNEL_LAYOUTS - 1)/2] =
+{
+  // Upmixes from mono
+  { { 0, 0 } },
+  { { 0, IGNORE, IGNORE } },
+  { { 0, 0, IGNORE, IGNORE } },
+  { { 0, IGNORE, IGNORE, IGNORE, IGNORE } },
+  { { IGNORE, IGNORE, 0, IGNORE, IGNORE, IGNORE } },
+  // Upmixes from stereo
+  { { 0, 1, IGNORE } },
+  { { 0, 1, IGNORE, IGNORE } },
+  { { 0, 1, IGNORE, IGNORE, IGNORE } },
+  { { 0, 1, IGNORE, IGNORE, IGNORE, IGNORE } },
+  // Upmixes from 3-channel
+  { { 0, 1, 2, IGNORE } },
+  { { 0, 1, 2, IGNORE, IGNORE } },
+  { { 0, 1, 2, IGNORE, IGNORE, IGNORE } },
+  // Upmixes from quad
+  { { 0, 1, 2, 3, IGNORE } },
+  { { 0, 1, IGNORE, IGNORE, 2, 3 } },
+  // Upmixes from 5-channel
+  { { 0, 1, 2, 3, 4, IGNORE } }
+};
+
+
+/**
+ * Given an array of input channel data, and an output channel count,
+ * replaces the array with an array of upmixed channels.
+ * This shuffles the array and may set some channel buffers to aZeroChannel.
+ * Don't call this with input count >= output count.
+ * This may return *more* channels than requested. In that case, downmixing
+ * is required to to get to aOutputChannelCount. (This is how we handle
+ * odd cases like 3 -> 4 upmixing.)
+ * If aChannelArray.Length() was the input to one of a series of
+ * GetAudioChannelsSuperset calls resulting in aOutputChannelCount,
+ * no downmixing will be required.
+ */
+template<typename T>
+void
+AudioChannelsUpMix(nsTArray<const T*>* aChannelArray,
+                   uint32_t aOutputChannelCount,
+                   const T* aZeroChannel)
+{
+  uint32_t inputChannelCount = aChannelArray->Length();
+  uint32_t outputChannelCount =
+    GetAudioChannelsSuperset(aOutputChannelCount, inputChannelCount);
+  NS_ASSERTION(outputChannelCount > inputChannelCount,
+               "No up-mix needed");
+  MOZ_ASSERT(inputChannelCount > 0, "Bad number of channels");
+  MOZ_ASSERT(outputChannelCount > 0, "Bad number of channels");
+
+  aChannelArray->SetLength(outputChannelCount);
+
+  if (inputChannelCount < CUSTOM_CHANNEL_LAYOUTS &&
+      outputChannelCount <= CUSTOM_CHANNEL_LAYOUTS) {
+    const UpMixMatrix& m = gUpMixMatrices[
+      gMixingMatrixIndexByChannels[inputChannelCount - 1] +
+      outputChannelCount - inputChannelCount - 1];
+
+    const T* outputChannels[CUSTOM_CHANNEL_LAYOUTS];
+
+    for (uint32_t i = 0; i < outputChannelCount; ++i) {
+      uint8_t channelIndex = m.mInputDestination[i];
+      if (channelIndex == IGNORE) {
+        outputChannels[i] = aZeroChannel;
+      } else {
+        outputChannels[i] = aChannelArray->ElementAt(channelIndex);
+      }
+    }
+    for (uint32_t i = 0; i < outputChannelCount; ++i) {
+      aChannelArray->ElementAt(i) = outputChannels[i];
+    }
+    return;
+  }
+
+  for (uint32_t i = inputChannelCount; i < outputChannelCount; ++i) {
+    aChannelArray->ElementAt(i) = aZeroChannel;
+  }
+}
 
 } // namespace mozilla
 

dom/media/AudioPacketizer.h

@@ -0,0 +1,196 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AudioPacketizer_h_
+#define AudioPacketizer_h_
+
+#include <mozilla/PodOperations.h>
+#include <mozilla/Assertions.h>
+#include <nsAutoPtr.h>
+#include <AudioSampleFormat.h>
+
+// Enable this to warn when `Output` has been called but not enough data was
+// buffered.
+// #define LOG_PACKETIZER_UNDERRUN
+
+namespace mozilla {
+/**
+ * This class takes arbitrary input data, and returns packets of a specific
+ * size. In the process, it can convert audio samples from 16bit integers to
+ * float (or vice-versa).
+ *
+ * Input and output, as well as length units in the public interface are
+ * interleaved frames.
+ *
+ * Allocations of output buffer can be performed by this class.  Buffers can
+ * simply be delete-d.  This is because packets are intended to be sent off to
+ * non-gecko code using normal pointers/length pairs
+ *
+ * Alternatively, consumers can pass in a buffer in which the output is copied.
+ * The buffer needs to be large enough to store a packet worth of audio.
+ *
+ * The implementation uses a circular buffer using absolute virtual indices.
+ */
+template <typename InputType, typename OutputType>
+class AudioPacketizer
+{
+public:
+  AudioPacketizer(uint32_t aPacketSize, uint32_t aChannels)
+    : mPacketSize(aPacketSize)
+    , mChannels(aChannels)
+    , mReadIndex(0)
+    , mWriteIndex(0)
+    // Start off with a single packet
+    , mStorage(new InputType[aPacketSize * aChannels])
+    , mLength(aPacketSize * aChannels)
+  {
+     MOZ_ASSERT(aPacketSize > 0 && aChannels > 0,
+       "The packet size and the number of channel should be strictly positive");
+  }
+
+  void Input(const InputType* aFrames, uint32_t aFrameCount)
+  {
+    uint32_t inputSamples = aFrameCount * mChannels;
+    // Need to grow the storage. This should rarely happen, if at all, once the
+    // array has the right size.
+    if (inputSamples > EmptySlots()) {
+      // Calls to Input and Output are roughtly interleaved
+      // (Input,Output,Input,Output, etc.), or balanced
+      // (Input,Input,Input,Output,Output,Output), so we update the buffer to
+      // the exact right size in order to not waste space.
+      uint32_t newLength = AvailableSamples() + inputSamples;
+      uint32_t toCopy = AvailableSamples();
+      nsAutoPtr<InputType> oldStorage = mStorage;
+      mStorage = new InputType[newLength];
+      // Copy the old data at the beginning of the new storage.
+      if (WriteIndex() >= ReadIndex()) {
+        PodCopy(mStorage.get(),
+                oldStorage.get() + ReadIndex(),
+                AvailableSamples());
+      } else {
+        uint32_t firstPartLength = mLength - ReadIndex();
+        uint32_t secondPartLength = AvailableSamples() - firstPartLength;
+        PodCopy(mStorage.get(),
+                oldStorage.get() + ReadIndex(),
+                firstPartLength);
+        PodCopy(mStorage.get() + firstPartLength,
+                oldStorage.get(),
+                secondPartLength);
+      }
+      mWriteIndex = toCopy;
+      mReadIndex = 0;
+      mLength = newLength;
+    }
+
+    if (WriteIndex() + inputSamples <= mLength) {
+      PodCopy(mStorage.get() + WriteIndex(), aFrames, aFrameCount * mChannels);
+    } else {
+      uint32_t firstPartLength = mLength - WriteIndex();
+      uint32_t secondPartLength = inputSamples - firstPartLength;
+      PodCopy(mStorage.get() + WriteIndex(), aFrames, firstPartLength);
+      PodCopy(mStorage.get(), aFrames + firstPartLength, secondPartLength);
+    }
+
+    mWriteIndex += inputSamples;
+  }
+
+  OutputType* Output()
+  {
+    uint32_t samplesNeeded = mPacketSize * mChannels;
+    OutputType* out = new OutputType[samplesNeeded];
+
+    Output(out);
+
+    return out;
+  }
+
+  void Output(OutputType* aOutputBuffer)
+  {
+    uint32_t samplesNeeded = mPacketSize * mChannels;
+
+    // Under-run. Pad the end of the buffer with silence.
+    if (AvailableSamples() < samplesNeeded) {
+#ifdef LOG_PACKETIZER_UNDERRUN
+      char buf[256];
+      snprintf(buf, 256,
+               "AudioPacketizer %p underrun: available: %u, needed: %u\n",
+               this, AvailableSamples(), samplesNeeded);
+      NS_WARNING(buf);
+#endif
+      uint32_t zeros = samplesNeeded - AvailableSamples();
+      PodZero(aOutputBuffer + AvailableSamples(), zeros);
+      samplesNeeded -= zeros;
+    }
+    if (ReadIndex() + samplesNeeded <= mLength) {
+      ConvertAudioSamples<InputType,OutputType>(mStorage.get() + ReadIndex(),
+                                                aOutputBuffer,
+                                                samplesNeeded);
+    } else {
+      uint32_t firstPartLength = mLength - ReadIndex();
+      uint32_t secondPartLength = samplesNeeded - firstPartLength;
+      ConvertAudioSamples<InputType, OutputType>(mStorage.get() + ReadIndex(),
+                                                 aOutputBuffer,
+                                                 firstPartLength);
+      ConvertAudioSamples<InputType, OutputType>(mStorage.get(),
+                                                 aOutputBuffer + firstPartLength,
+                                                 secondPartLength);
+    }
+    mReadIndex += samplesNeeded;
+  }
+
+  uint32_t PacketsAvailable() const {
+    return AvailableSamples() / mChannels / mPacketSize;
+  }
+
+  bool Empty() const {
+   return mWriteIndex == mReadIndex;
+  }
+
+  bool Full() const {
+    return mWriteIndex - mReadIndex == mLength;
+  }
+
+  uint32_t PacketSize() const {
+    return mPacketSize;
+  }
+
+  uint32_t Channels() const {
+    return mChannels;
+  }
+
+private:
+  uint32_t ReadIndex() const {
+    return mReadIndex % mLength;
+  }
+
+  uint32_t WriteIndex() const {
+    return mWriteIndex % mLength;
+  }
+
+  uint32_t AvailableSamples() const {
+    return mWriteIndex - mReadIndex;
+  }
+
+  uint32_t EmptySlots() const {
+    return mLength - AvailableSamples();
+  }
+
+  // Size of one packet of audio, in frames
+  uint32_t mPacketSize;
+  // Number of channels of the stream flowing through this packetizer
+  uint32_t mChannels;
+  // Two virtual index into the buffer: the read position and the write
+  // position.
+  uint64_t mReadIndex;
+  uint64_t mWriteIndex;
+  // Storage for the samples
+  nsAutoPtr<InputType> mStorage;
+  // Length of the buffer, in samples
+  uint32_t mLength;
+};
+
+} // mozilla
+
+#endif // AudioPacketizer_h_

dom/media/AudioSampleFormat.h

@@ -96,6 +96,51 @@ FloatToAudioSample<int16_t>(float aValue)
   return int16_t(clamped);
 }
 
+template <typename T> T IntegerToAudioSample(int16_t aValue);
+
+template <> inline float
+IntegerToAudioSample<float>(int16_t aValue)
+{
+  return aValue / 32768.0f;
+}
+template <> inline int16_t
+IntegerToAudioSample<int16_t>(int16_t aValue)
+{
+  return aValue;
+}
+
+template<typename SrcT, typename DstT>
+inline void
+ConvertAudioSample(SrcT aIn, DstT& aOut);
+
+template<>
+inline void
+ConvertAudioSample(int16_t aIn, int16_t & aOut)
+{
+  aOut = aIn;
+}
+
+template<>
+inline void
+ConvertAudioSample(int16_t aIn, float& aOut)
+{
+  aOut = AudioSampleToFloat(aIn);
+}
+
+template<>
+inline void
+ConvertAudioSample(float aIn, float& aOut)
+{
+  aOut = aIn;
+}
+
+template<>
+inline void
+ConvertAudioSample(float aIn, int16_t& aOut)
+{
+  aOut = FloatToAudioSample<int16_t>(aIn);
+}
+
 // Sample buffer conversion
 
 template <typename From, typename To> inline void

dom/media/AudioSegment.cpp

@@ -5,7 +5,6 @@
 
 #include "AudioSegment.h"
 
-#include "AudioStream.h"
 #include "AudioMixer.h"
 #include "AudioChannelFormat.h"
 #include "Latency.h"
@@ -13,49 +12,18 @@
 
 namespace mozilla {
 
-template <class SrcT, class DestT>
-static void
-InterleaveAndConvertBuffer(const SrcT** aSourceChannels,
-                           int32_t aLength, float aVolume,
-                           int32_t aChannels,
-                           DestT* aOutput)
+const uint8_t SilentChannel::gZeroChannel[MAX_AUDIO_SAMPLE_SIZE*SilentChannel::AUDIO_PROCESSING_FRAMES] = {0};
+
+template<>
+const float* SilentChannel::ZeroChannel<float>()
 {
-  DestT* output = aOutput;
-  for (int32_t i = 0; i < aLength; ++i) {
-    for (int32_t channel = 0; channel < aChannels; ++channel) {
-      float v = AudioSampleToFloat(aSourceChannels[channel][i])*aVolume;
-      *output = FloatToAudioSample<DestT>(v);
-      ++output;
-    }
-  }
+  return reinterpret_cast<const float*>(SilentChannel::gZeroChannel);
 }
 
-void
-InterleaveAndConvertBuffer(const void** aSourceChannels,
-                           AudioSampleFormat aSourceFormat,
-                           int32_t aLength, float aVolume,
-                           int32_t aChannels,
-                           AudioDataValue* aOutput)
+template<>
+const int16_t* SilentChannel::ZeroChannel<int16_t>()
 {
-  switch (aSourceFormat) {
-  case AUDIO_FORMAT_FLOAT32:
-    InterleaveAndConvertBuffer(reinterpret_cast<const float**>(aSourceChannels),
-                               aLength,
-                               aVolume,
-                               aChannels,
-                               aOutput);
-    break;
-  case AUDIO_FORMAT_S16:
-    InterleaveAndConvertBuffer(reinterpret_cast<const int16_t**>(aSourceChannels),
-                               aLength,
-                               aVolume,
-                               aChannels,
-                               aOutput);
-    break;
-   case AUDIO_FORMAT_SILENCE:
-    // nothing to do here.
-    break;
-  }
+  return reinterpret_cast<const int16_t*>(SilentChannel::gZeroChannel);
 }
 
 void
@@ -66,54 +34,6 @@ AudioSegment::ApplyVolume(float aVolume)
   }
 }
 
-static const int AUDIO_PROCESSING_FRAMES = 640; /* > 10ms of 48KHz audio */
-static const uint8_t gZeroChannel[MAX_AUDIO_SAMPLE_SIZE*AUDIO_PROCESSING_FRAMES] = {0};
-
-void
-DownmixAndInterleave(const nsTArray<const void*>& aChannelData,
-                     AudioSampleFormat aSourceFormat, int32_t aDuration,
-                     float aVolume, uint32_t aOutputChannels,
-                     AudioDataValue* aOutput)
-{
-  nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channelData;
-  nsAutoTArray<float,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> downmixConversionBuffer;
-  nsAutoTArray<float,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> downmixOutputBuffer;
-
-  channelData.SetLength(aChannelData.Length());
-  if (aSourceFormat != AUDIO_FORMAT_FLOAT32) {
-    NS_ASSERTION(aSourceFormat == AUDIO_FORMAT_S16, "unknown format");
-    downmixConversionBuffer.SetLength(aDuration*aChannelData.Length());
-    for (uint32_t i = 0; i < aChannelData.Length(); ++i) {
-      float* conversionBuf = downmixConversionBuffer.Elements() + (i*aDuration);
-      const int16_t* sourceBuf = static_cast<const int16_t*>(aChannelData[i]);
-      for (uint32_t j = 0; j < (uint32_t)aDuration; ++j) {
-        conversionBuf[j] = AudioSampleToFloat(sourceBuf[j]);
-      }
-      channelData[i] = conversionBuf;
-    }
-  } else {
-    for (uint32_t i = 0; i < aChannelData.Length(); ++i) {
-      channelData[i] = aChannelData[i];
-    }
-  }
-
-  downmixOutputBuffer.SetLength(aDuration*aOutputChannels);
-  nsAutoTArray<float*,GUESS_AUDIO_CHANNELS> outputChannelBuffers;
-  nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> outputChannelData;
-  outputChannelBuffers.SetLength(aOutputChannels);
-  outputChannelData.SetLength(aOutputChannels);
-  for (uint32_t i = 0; i < (uint32_t)aOutputChannels; ++i) {
-    outputChannelData[i] = outputChannelBuffers[i] =
-        downmixOutputBuffer.Elements() + aDuration*i;
-  }
-  if (channelData.Length() > aOutputChannels) {
-    AudioChannelsDownMix(channelData, outputChannelBuffers.Elements(),
-                         aOutputChannels, aDuration);
-  }
-  InterleaveAndConvertBuffer(outputChannelData.Elements(), AUDIO_FORMAT_FLOAT32,
-                             aDuration, aVolume, aOutputChannels, aOutput);
-}
-
 void AudioSegment::ResampleChunks(SpeexResamplerState* aResampler, uint32_t aInRate, uint32_t aOutRate)
 {
   if (mChunks.IsEmpty()) {
@@ -165,9 +85,9 @@ void
 AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels,
                   uint32_t aSampleRate)
 {
-  nsAutoTArray<AudioDataValue, AUDIO_PROCESSING_FRAMES* GUESS_AUDIO_CHANNELS>
+  nsAutoTArray<AudioDataValue, SilentChannel::AUDIO_PROCESSING_FRAMES* GUESS_AUDIO_CHANNELS>
   buf;
-  nsAutoTArray<const void*, GUESS_AUDIO_CHANNELS> channelData;
+  nsAutoTArray<const AudioDataValue*, GUESS_AUDIO_CHANNELS> channelData;
   uint32_t offsetSamples = 0;
   uint32_t duration = GetDuration();
 
@@ -197,11 +117,11 @@ AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels,
       // desired input and output channels.
       channelData.SetLength(c.mChannelData.Length());
       for (uint32_t i = 0; i < channelData.Length(); ++i) {
-        channelData[i] = c.mChannelData[i];
+        channelData[i] = static_cast<const AudioDataValue*>(c.mChannelData[i]);
       }
       if (channelData.Length() < aOutputChannels) {
         // Up-mix.
-        AudioChannelsUpMix(&channelData, aOutputChannels, gZeroChannel);
+        AudioChannelsUpMix(&channelData, aOutputChannels, SilentChannel::ZeroChannel<AudioDataValue>());
         for (uint32_t channel = 0; channel < aOutputChannels; channel++) {
           AudioDataValue* ptr =
             PointerForOffsetInChannel(buf.Elements(), outBufferLength,
@@ -246,9 +166,8 @@ AudioSegment::Mix(AudioMixer& aMixer, uint32_t aOutputChannels,
 void
 AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels, uint32_t aSampleRate)
 {
-  nsAutoTArray<AudioDataValue,AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> buf;
-  nsAutoTArray<const void*,GUESS_AUDIO_CHANNELS> channelData;
-  // Offset in the buffer that will end up sent to the AudioStream, in samples.
+  nsAutoTArray<AudioDataValue,SilentChannel::AUDIO_PROCESSING_FRAMES*GUESS_AUDIO_CHANNELS> buf;
+  // Offset in the buffer that will be written to the mixer, in samples.
   uint32_t offset = 0;
 
   if (GetDuration() <= 0) {
@@ -262,39 +181,23 @@ AudioSegment::WriteTo(uint64_t aID, AudioMixer& aMixer, uint32_t aOutputChannels
 
   for (ChunkIterator ci(*this); !ci.IsEnded(); ci.Next()) {
     AudioChunk& c = *ci;
-    uint32_t frames = c.mDuration;
 
-    // If we have written data in the past, or we have real (non-silent) data
-    // to write, we can proceed. Otherwise, it means we just started the
-    // AudioStream, and we don't have real data to write to it (just silence).
-    // To avoid overbuffering in the AudioStream, we simply drop the silence,
-    // here. The stream will underrun and output silence anyways.
-    if (c.mBuffer && c.mBufferFormat != AUDIO_FORMAT_SILENCE) {
-      channelData.SetLength(c.mChannelData.Length());
-      for (uint32_t i = 0; i < channelData.Length(); ++i) {
-        channelData[i] = c.mChannelData[i];
-      }
-      if (channelData.Length() < aOutputChannels) {
-        // Up-mix. Note that this might actually make channelData have more
-        // than aOutputChannels temporarily.
-        AudioChannelsUpMix(&channelData, aOutputChannels, gZeroChannel);
-      }
-      if (channelData.Length() > aOutputChannels) {
-        // Down-mix.
-        DownmixAndInterleave(channelData, c.mBufferFormat, frames,
-                             c.mVolume, aOutputChannels, buf.Elements() + offset);
-      } else {
-        InterleaveAndConvertBuffer(channelData.Elements(), c.mBufferFormat,
-                                   frames, c.mVolume,
-                                   aOutputChannels,
-                                   buf.Elements() + offset);
-      }
-    } else {
-      // Assumes that a bit pattern of zeroes == 0.0f
-      memset(buf.Elements() + offset, 0, aOutputChannels * frames * sizeof(AudioDataValue));
+    switch (c.mBufferFormat) {
+      case AUDIO_FORMAT_S16:
+        WriteChunk<int16_t>(c, aOutputChannels, buf.Elements() + offset);
+        break;
+      case AUDIO_FORMAT_FLOAT32:
+        WriteChunk<float>(c, aOutputChannels, buf.Elements() + offset);
+        break;
+      case AUDIO_FORMAT_SILENCE:
+        // The mixer is expecting interleaved data, so this is ok.
+        PodZero(buf.Elements() + offset, c.mDuration * aOutputChannels);
+        break;
+      default:
+        MOZ_ASSERT(false, "Not handled");
     }
 
-    offset += frames * aOutputChannels;
+    offset += c.mDuration * aOutputChannels;
 
 #if !defined(MOZILLA_XPCOMRT_API)
     if (!c.mTimeStamp.IsNull()) {

dom/media/AudioSegment.h

@@ -8,11 +8,13 @@
 
 #include "MediaSegment.h"
 #include "AudioSampleFormat.h"
+#include "AudioChannelFormat.h"
 #include "SharedBuffer.h"
 #include "WebAudioUtils.h"
 #ifdef MOZILLA_INTERNAL_API
 #include "mozilla/TimeStamp.h"
 #endif
+#include <float.h>
 
 namespace mozilla {
 
@@ -55,21 +57,82 @@ const int GUESS_AUDIO_CHANNELS = 2;
 const uint32_t WEBAUDIO_BLOCK_SIZE_BITS = 7;
 const uint32_t WEBAUDIO_BLOCK_SIZE = 1 << WEBAUDIO_BLOCK_SIZE_BITS;
 
-void InterleaveAndConvertBuffer(const void** aSourceChannels,
-                                AudioSampleFormat aSourceFormat,
-                                int32_t aLength, float aVolume,
-                                int32_t aChannels,
-                                AudioDataValue* aOutput);
+template <typename SrcT, typename DestT>
+static void
+InterleaveAndConvertBuffer(const SrcT* const* aSourceChannels,
+                           uint32_t aLength, float aVolume,
+                           uint32_t aChannels,
+                           DestT* aOutput)
+{
+  DestT* output = aOutput;
+  for (size_t i = 0; i < aLength; ++i) {
+    for (size_t channel = 0; channel < aChannels; ++channel) {
+      float v = AudioSampleToFloat(aSourceChannels[channel][i])*aVolume;
+      *output = FloatToAudioSample<DestT>(v);
+      ++output;
+    }
+  }
+}
+
+template <typename SrcT, typename DestT>
+static void
+DeinterleaveAndConvertBuffer(const SrcT* aSourceBuffer,
+                             uint32_t aFrames, uint32_t aChannels,
+                             DestT** aOutput)
+{
+  for (size_t i = 0; i < aChannels; i++) {
+    size_t interleavedIndex = i;
+    for (size_t j = 0; j < aFrames; j++) {
+      ConvertAudioSample(aSourceBuffer[interleavedIndex],
+                         aOutput[i][j]);
+      interleavedIndex += aChannels;
+    }
+  }
+}
+
+class SilentChannel
+{
+public:
+  static const int AUDIO_PROCESSING_FRAMES = 640; /* > 10ms of 48KHz audio */
+  static const uint8_t gZeroChannel[MAX_AUDIO_SAMPLE_SIZE*AUDIO_PROCESSING_FRAMES];
+  // We take advantage of the fact that zero in float and zero in int have the
+  // same all-zeros bit layout.
+  template<typename T>
+  static const T* ZeroChannel();
+};
+
 
 /**
  * Given an array of input channels (aChannelData), downmix to aOutputChannels,
  * interleave the channel data. A total of aOutputChannels*aDuration
  * interleaved samples will be copied to a channel buffer in aOutput.
  */
-void DownmixAndInterleave(const nsTArray<const void*>& aChannelData,
-                          AudioSampleFormat aSourceFormat, int32_t aDuration,
-                          float aVolume, uint32_t aOutputChannels,
-                          AudioDataValue* aOutput);
+template <typename SrcT, typename DestT>
+void
+DownmixAndInterleave(const nsTArray<const SrcT*>& aChannelData,
+                     int32_t aDuration, float aVolume, uint32_t aOutputChannels,
+                     DestT* aOutput)
+{
+
+  if (aChannelData.Length() == aOutputChannels) {
+    InterleaveAndConvertBuffer(aChannelData.Elements(),
+                               aDuration, aVolume, aOutputChannels, aOutput);
+  } else {
+    nsAutoTArray<SrcT*,GUESS_AUDIO_CHANNELS> outputChannelData;
+    nsAutoTArray<SrcT, SilentChannel::AUDIO_PROCESSING_FRAMES * GUESS_AUDIO_CHANNELS> outputBuffers;
+    outputChannelData.SetLength(aOutputChannels);
+    outputBuffers.SetLength(aDuration * aOutputChannels);
+    for (uint32_t i = 0; i < aOutputChannels; i++) {
+      outputChannelData[i] = outputBuffers.Elements() + aDuration * i;
+    }
+    AudioChannelsDownMix(aChannelData,
+                         outputChannelData.Elements(),
+                         aOutputChannels,
+                         aDuration);
+    InterleaveAndConvertBuffer(outputChannelData.Elements(),
+                               aDuration, aVolume, aOutputChannels, aOutput);
+  }
+}
 
 /**
  * An AudioChunk represents a multi-channel buffer of audio samples.
@@ -128,8 +191,44 @@ struct AudioChunk {
     mVolume = 1.0f;
     mBufferFormat = AUDIO_FORMAT_SILENCE;
   }
+
+  bool IsSilentOrSubnormal() const
+  {
+    if (!mBuffer) {
+      return true;
+    }
+
+    for (uint32_t i = 0, length = mChannelData.Length(); i < length; ++i) {
+      const float* channel = static_cast<const float*>(mChannelData[i]);
+      for (StreamTime frame = 0; frame < mDuration; ++frame) {
+        if (fabs(channel[frame]) >= FLT_MIN) {
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
   int ChannelCount() const { return mChannelData.Length(); }
 
+  float* ChannelFloatsForWrite(size_t aChannel)
+  {
+    MOZ_ASSERT(mBufferFormat == AUDIO_FORMAT_FLOAT32);
+    MOZ_ASSERT(!mBuffer->IsShared());
+    return static_cast<float*>(const_cast<void*>(mChannelData[aChannel]));
+  }
+
+  void ReleaseBufferIfShared()
+  {
+    if (mBuffer && mBuffer->IsShared()) {
+      // Remove pointers into the buffer, but keep the array allocation for
+      // chunk re-use.
+      mChannelData.ClearAndRetainStorage();
+      mBuffer = nullptr;
+    }
+  }
+
   bool IsMuted() const { return mVolume == 0.0f; }
 
   size_t SizeOfExcludingThisIfUnshared(MallocSizeOf aMallocSizeOf) const
@@ -153,6 +252,13 @@ struct AudioChunk {
     return amount;
   }
 
+  template<typename T>
+  const nsTArray<const T*>& ChannelData()
+  {
+    MOZ_ASSERT(AudioSampleTypeToFormat<T>::Format == mBufferFormat);
+    return *reinterpret_cast<nsTArray<const T*>*>(&mChannelData);
+  }
+
   StreamTime mDuration; // in frames within the buffer
   nsRefPtr<ThreadSharedObject> mBuffer; // the buffer object whose lifetime is managed; null means data is all zeroes
   nsTArray<const void*> mChannelData; // one pointer per channel; empty if and only if mBuffer is null
@@ -319,6 +425,34 @@ public:
   }
 };
 
+template<typename SrcT>
+void WriteChunk(AudioChunk& aChunk,
+                uint32_t aOutputChannels,
+                AudioDataValue* aOutputBuffer)
+{
+  nsAutoTArray<const SrcT*,GUESS_AUDIO_CHANNELS> channelData;
+
+  channelData = aChunk.ChannelData<SrcT>();
+
+  if (channelData.Length() < aOutputChannels) {
+    // Up-mix. Note that this might actually make channelData have more
+    // than aOutputChannels temporarily.
+    AudioChannelsUpMix(&channelData, aOutputChannels, SilentChannel::ZeroChannel<SrcT>());
+  }
+  if (channelData.Length() > aOutputChannels) {
+    // Down-mix.
+    DownmixAndInterleave(channelData, aChunk.mDuration,
+        aChunk.mVolume, aOutputChannels, aOutputBuffer);
+  } else {
+    InterleaveAndConvertBuffer(channelData.Elements(),
+        aChunk.mDuration, aChunk.mVolume,
+        aOutputChannels,
+        aOutputBuffer);
+  }
+}
+
+
+
 } // namespace mozilla
 
 #endif /* MOZILLA_AUDIOSEGMENT_H_ */

dom/media/AudioStream.cpp

@@ -14,7 +14,6 @@
 #include "mozilla/Mutex.h"
 #include "mozilla/Snprintf.h"
 #include <algorithm>
-#include "soundtouch/SoundTouch.h"
 #include "Latency.h"
 #include "CubebUtils.h"
 #include "nsPrintfCString.h"
@@ -129,6 +128,7 @@ AudioStream::AudioStream()
   , mOutChannels(0)
   , mWritten(0)
   , mAudioClock(this)
+  , mTimeStretcher(nullptr)
   , mLatencyRequest(HighLatency)
   , mReadPoint(0)
   , mDumpFile(nullptr)
@@ -151,6 +151,9 @@ AudioStream::~AudioStream()
   if (mDumpFile) {
     fclose(mDumpFile);
   }
+  if (mTimeStretcher) {
+    soundtouch::destroySoundTouchObj(mTimeStretcher);
+  }
 }
 
 size_t
@@ -173,7 +176,7 @@ nsresult AudioStream::EnsureTimeStretcherInitializedUnlocked()
 {
   mMonitor.AssertCurrentThreadOwns();
   if (!mTimeStretcher) {
-    mTimeStretcher = new soundtouch::SoundTouch();
+    mTimeStretcher = soundtouch::createSoundTouchObj();
     mTimeStretcher->setSampleRate(mInRate);
     mTimeStretcher->setChannels(mOutChannels);
     mTimeStretcher->setPitch(1.0);

dom/media/AudioStream.h

@@ -15,10 +15,7 @@
 #include "mozilla/RefPtr.h"
 #include "mozilla/UniquePtr.h"
 #include "CubebUtils.h"
-
-namespace soundtouch {
-class SoundTouch;
-}
+#include "soundtouch/SoundTouchFactory.h"
 
 namespace mozilla {
 
@@ -332,7 +329,7 @@ private:
   // Number of frames written to the buffers.
   int64_t mWritten;
   AudioClock mAudioClock;
-  nsAutoPtr<soundtouch::SoundTouch> mTimeStretcher;
+  soundtouch::SoundTouch* mTimeStretcher;
   nsRefPtr<AsyncLatencyLogger> mLatencyLog;
 
   // copy of Latency logger's starting time for offset calculations

dom/media/BufferMediaResource.h

@@ -59,45 +59,15 @@ private:
   // The mode is initially MODE_PLAYBACK.
   virtual void SetReadMode(MediaCacheStream::ReadMode aMode) override {}
   virtual void SetPlaybackRate(uint32_t aBytesPerSecond) override {}
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) override
-  {
-    *aBytes = std::min(mLength - mOffset, aCount);
-    memcpy(aBuffer, mBuffer + mOffset, *aBytes);
-    mOffset += *aBytes;
-    MOZ_ASSERT(mOffset <= mLength);
-    return NS_OK;
-  }
   virtual nsresult ReadAt(int64_t aOffset, char* aBuffer,
                           uint32_t aCount, uint32_t* aBytes) override
   {
-    nsresult rv = Seek(nsISeekableStream::NS_SEEK_SET, aOffset);
-    if (NS_FAILED(rv)) return rv;
-    return Read(aBuffer, aCount, aBytes);
-  }
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) override
-  {
-    MOZ_ASSERT(aOffset <= UINT32_MAX);
-    switch (aWhence) {
-    case nsISeekableStream::NS_SEEK_SET:
-      if (aOffset < 0 || aOffset > mLength) {
-        return NS_ERROR_FAILURE;
-      }
-      mOffset = static_cast<uint32_t> (aOffset);
-      break;
-    case nsISeekableStream::NS_SEEK_CUR:
-      if (aOffset >= mLength - mOffset) {
-        return NS_ERROR_FAILURE;
-      }
-      mOffset += static_cast<uint32_t> (aOffset);
-      break;
-    case nsISeekableStream::NS_SEEK_END:
-      if (aOffset < 0 || aOffset > mLength) {
-        return NS_ERROR_FAILURE;
-      }
-      mOffset = mLength - aOffset;
-      break;
+    if (aOffset < 0 || aOffset > mLength) {
+      return NS_ERROR_FAILURE;
     }
-
+    *aBytes = std::min(mLength - static_cast<uint32_t>(aOffset), aCount);
+    memcpy(aBuffer, mBuffer + aOffset, *aBytes);
+    mOffset = aOffset + *aBytes;
     return NS_OK;
   }
   virtual int64_t Tell() override { return mOffset; }

dom/media/CanvasCaptureMediaStream.cpp

@@ -235,7 +235,10 @@ CanvasCaptureMediaStream::CreateSourceStream(nsIDOMWindow* aWindow,
                                              HTMLCanvasElement* aCanvas)
 {
   nsRefPtr<CanvasCaptureMediaStream> stream = new CanvasCaptureMediaStream(aCanvas);
-  stream->InitSourceStream(aWindow);
+  MediaStreamGraph* graph =
+    MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER,
+                                  AudioChannel::Normal);
+  stream->InitSourceStream(aWindow, graph);
   return stream.forget();
 }
 

dom/media/CubebUtils.cpp

@@ -181,6 +181,8 @@ cubeb_stream_type ConvertChannelToCubebType(dom::AudioChannel aChannel)
       return CUBEB_STREAM_TYPE_VOICE_CALL;
     case dom::AudioChannel::Ringer:
       return CUBEB_STREAM_TYPE_RING;
+    case dom::AudioChannel::System:
+      return CUBEB_STREAM_TYPE_SYSTEM;
     case dom::AudioChannel::Publicnotification:
       return CUBEB_STREAM_TYPE_SYSTEM_ENFORCED;
     default:

dom/media/DOMMediaStream.cpp

@@ -10,6 +10,7 @@
 #include "mozilla/dom/MediaStreamBinding.h"
 #include "mozilla/dom/LocalMediaStreamBinding.h"
 #include "mozilla/dom/AudioNode.h"
+#include "AudioChannelAgent.h"
 #include "mozilla/dom/AudioTrack.h"
 #include "mozilla/dom/AudioTrackList.h"
 #include "mozilla/dom/VideoTrack.h"
@@ -283,9 +284,6 @@ DOMMediaStream::InitSourceStream(nsIDOMWindow* aWindow,
                                  MediaStreamGraph* aGraph)
 {
   mWindow = aWindow;
-  if (!aGraph) {
-    aGraph = MediaStreamGraph::GetInstance();
-  }
   InitStreamCommon(aGraph->CreateSourceStream(this));
 }
 
@@ -295,9 +293,6 @@ DOMMediaStream::InitTrackUnionStream(nsIDOMWindow* aWindow,
 {
   mWindow = aWindow;
 
-  if (!aGraph) {
-    aGraph = MediaStreamGraph::GetInstance();
-  }
   InitStreamCommon(aGraph->CreateTrackUnionStream(this));
 }
 
@@ -307,9 +302,6 @@ DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow,
 {
   mWindow = aWindow;
 
-  if (!aGraph) {
-    aGraph = MediaStreamGraph::GetInstance();
-  }
   InitStreamCommon(aGraph->CreateAudioCaptureStream(this));
 }
 
@@ -722,7 +714,11 @@ already_AddRefed<DOMHwMediaStream>
 DOMHwMediaStream::CreateHwStream(nsIDOMWindow* aWindow)
 {
   nsRefPtr<DOMHwMediaStream> stream = new DOMHwMediaStream();
-  stream->InitSourceStream(aWindow);
+
+  MediaStreamGraph* graph =
+    MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER,
+                                  AudioChannel::Normal);
+  stream->InitSourceStream(aWindow, graph);
   stream->Init(stream->GetStream());
 
   return stream.forget();

dom/media/DOMMediaStream.h

@@ -190,20 +190,20 @@ public:
    * Create an nsDOMMediaStream whose underlying stream is a SourceMediaStream.
    */
   static already_AddRefed<DOMMediaStream> CreateSourceStream(nsIDOMWindow* aWindow,
-                                                             MediaStreamGraph* aGraph = nullptr);
+                                                             MediaStreamGraph* aGraph);
 
   /**
    * Create an nsDOMMediaStream whose underlying stream is a TrackUnionStream.
    */
   static already_AddRefed<DOMMediaStream> CreateTrackUnionStream(nsIDOMWindow* aWindow,
-                                                                 MediaStreamGraph* aGraph = nullptr);
+                                                                 MediaStreamGraph* aGraph);
 
   /**
    * Create an nsDOMMediaStream whose underlying stream is an
    * AudioCaptureStream
    */
   static already_AddRefed<DOMMediaStream> CreateAudioCaptureStream(
-    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph);
 
   void SetLogicalStreamStartTime(StreamTime aTime)
   {
@@ -266,11 +266,11 @@ protected:
 
   void Destroy();
   void InitSourceStream(nsIDOMWindow* aWindow,
-                        MediaStreamGraph* aGraph = nullptr);
+                        MediaStreamGraph* aGraph);
   void InitTrackUnionStream(nsIDOMWindow* aWindow,
-                            MediaStreamGraph* aGraph = nullptr);
+                            MediaStreamGraph* aGraph);
   void InitAudioCaptureStream(nsIDOMWindow* aWindow,
-                              MediaStreamGraph* aGraph = nullptr);
+                              MediaStreamGraph* aGraph);
   void InitStreamCommon(MediaStream* aStream);
   already_AddRefed<AudioTrack> CreateAudioTrack(AudioStreamTrack* aStreamTrack);
   already_AddRefed<VideoTrack> CreateVideoTrack(VideoStreamTrack* aStreamTrack);
@@ -352,20 +352,20 @@ public:
    */
   static already_AddRefed<DOMLocalMediaStream>
   CreateSourceStream(nsIDOMWindow* aWindow,
-                     MediaStreamGraph* aGraph = nullptr);
+                     MediaStreamGraph* aGraph);
 
   /**
    * Create an nsDOMLocalMediaStream whose underlying stream is a TrackUnionStream.
    */
   static already_AddRefed<DOMLocalMediaStream>
   CreateTrackUnionStream(nsIDOMWindow* aWindow,
-                         MediaStreamGraph* aGraph = nullptr);
+                         MediaStreamGraph* aGraph);
 
   /**
    * Create an nsDOMLocalMediaStream whose underlying stream is an
    * AudioCaptureStream. */
   static already_AddRefed<DOMLocalMediaStream> CreateAudioCaptureStream(
-    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr);
+    nsIDOMWindow* aWindow, MediaStreamGraph* aGraph);
 
 protected:
   virtual ~DOMLocalMediaStream();
@@ -389,7 +389,7 @@ public:
   static already_AddRefed<DOMAudioNodeMediaStream>
   CreateTrackUnionStream(nsIDOMWindow* aWindow,
                          AudioNode* aNode,
-                         MediaStreamGraph* aGraph = nullptr);
+                         MediaStreamGraph* aGraph);
 
 protected:
   ~DOMAudioNodeMediaStream();

dom/media/DecodedStream.cpp

@@ -1,667 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#include "DecodedStream.h"
-#include "MediaStreamGraph.h"
-#include "AudioSegment.h"
-#include "VideoSegment.h"
-#include "MediaQueue.h"
-#include "MediaData.h"
-#include "SharedBuffer.h"
-#include "VideoUtils.h"
-
-namespace mozilla {
-
-class DecodedStreamGraphListener : public MediaStreamListener {
-  typedef MediaStreamListener::MediaStreamGraphEvent MediaStreamGraphEvent;
-public:
-  explicit DecodedStreamGraphListener(MediaStream* aStream)
-    : mMutex("DecodedStreamGraphListener::mMutex")
-    , mStream(aStream)
-    , mLastOutputTime(aStream->StreamTimeToMicroseconds(aStream->GetCurrentTime()))
-    , mStreamFinishedOnMainThread(false) {}
-
-  void NotifyOutput(MediaStreamGraph* aGraph, GraphTime aCurrentTime) override
-  {
-    MutexAutoLock lock(mMutex);
-    if (mStream) {
-      mLastOutputTime = mStream->StreamTimeToMicroseconds(mStream->GraphTimeToStreamTime(aCurrentTime));
-    }
-  }
-
-  void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent event) override
-  {
-    if (event == EVENT_FINISHED) {
-      nsCOMPtr<nsIRunnable> event =
-        NS_NewRunnableMethod(this, &DecodedStreamGraphListener::DoNotifyFinished);
-      aGraph->DispatchToMainThreadAfterStreamStateUpdate(event.forget());
-    }
-  }
-
-  void DoNotifyFinished()
-  {
-    MutexAutoLock lock(mMutex);
-    mStreamFinishedOnMainThread = true;
-  }
-
-  int64_t GetLastOutputTime()
-  {
-    MutexAutoLock lock(mMutex);
-    return mLastOutputTime;
-  }
-
-  void Forget()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    MutexAutoLock lock(mMutex);
-    mStream = nullptr;
-  }
-
-  bool IsFinishedOnMainThread()
-  {
-    MutexAutoLock lock(mMutex);
-    return mStreamFinishedOnMainThread;
-  }
-
-private:
-  Mutex mMutex;
-  // Members below are protected by mMutex.
-  nsRefPtr<MediaStream> mStream;
-  int64_t mLastOutputTime; // microseconds
-  bool mStreamFinishedOnMainThread;
-};
-
-static void
-UpdateStreamBlocking(MediaStream* aStream, bool aBlocking)
-{
-  int32_t delta = aBlocking ? 1 : -1;
-  if (NS_IsMainThread()) {
-    aStream->ChangeExplicitBlockerCount(delta);
-  } else {
-    nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<int32_t>(
-      aStream, &MediaStream::ChangeExplicitBlockerCount, delta);
-    AbstractThread::MainThread()->Dispatch(r.forget());
-  }
-}
-
-DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream, bool aPlaying)
-  : mAudioFramesWritten(0)
-  , mNextVideoTime(-1)
-  , mNextAudioTime(-1)
-  , mStreamInitialized(false)
-  , mHaveSentFinish(false)
-  , mHaveSentFinishAudio(false)
-  , mHaveSentFinishVideo(false)
-  , mStream(aStream)
-  , mPlaying(aPlaying)
-  , mEOSVideoCompensation(false)
-{
-  mListener = new DecodedStreamGraphListener(mStream);
-  mStream->AddListener(mListener);
-
-  // Block the stream if we are not playing.
-  if (!aPlaying) {
-    UpdateStreamBlocking(mStream, true);
-  }
-}
-
-DecodedStreamData::~DecodedStreamData()
-{
-  mListener->Forget();
-  mStream->Destroy();
-}
-
-bool
-DecodedStreamData::IsFinished() const
-{
-  return mListener->IsFinishedOnMainThread();
-}
-
-int64_t
-DecodedStreamData::GetPosition() const
-{
-  return mListener->GetLastOutputTime();
-}
-
-void
-DecodedStreamData::SetPlaying(bool aPlaying)
-{
-  if (mPlaying != aPlaying) {
-    mPlaying = aPlaying;
-    UpdateStreamBlocking(mStream, !mPlaying);
-  }
-}
-
-class OutputStreamListener : public MediaStreamListener {
-  typedef MediaStreamListener::MediaStreamGraphEvent MediaStreamGraphEvent;
-public:
-  OutputStreamListener(DecodedStream* aDecodedStream, MediaStream* aStream)
-    : mDecodedStream(aDecodedStream), mStream(aStream) {}
-
-  void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent event) override
-  {
-    if (event == EVENT_FINISHED) {
-      nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethod(
-        this, &OutputStreamListener::DoNotifyFinished);
-      aGraph->DispatchToMainThreadAfterStreamStateUpdate(r.forget());
-    }
-  }
-
-  void Forget()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    mDecodedStream = nullptr;
-  }
-
-private:
-  void DoNotifyFinished()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    if (mDecodedStream) {
-      // Remove the finished stream so it won't block the decoded stream.
-      mDecodedStream->Remove(mStream);
-    }
-  }
-
-  // Main thread only
-  DecodedStream* mDecodedStream;
-  nsRefPtr<MediaStream> mStream;
-};
-
-OutputStreamData::~OutputStreamData()
-{
-  mListener->Forget();
-}
-
-void
-OutputStreamData::Init(DecodedStream* aDecodedStream, ProcessedMediaStream* aStream)
-{
-  mStream = aStream;
-  mListener = new OutputStreamListener(aDecodedStream, aStream);
-  aStream->AddListener(mListener);
-}
-
-DecodedStream::DecodedStream(MediaQueue<AudioData>& aAudioQueue,
-                             MediaQueue<VideoData>& aVideoQueue)
-  : mMonitor("DecodedStream::mMonitor")
-  , mPlaying(false)
-  , mAudioQueue(aAudioQueue)
-  , mVideoQueue(aVideoQueue)
-{
-  //
-}
-
-void
-DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo)
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  if (mStartTime.isNothing()) {
-    mStartTime.emplace(aStartTime);
-    mInfo = aInfo;
-  }
-}
-
-void DecodedStream::StopPlayback()
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  mStartTime.reset();
-}
-
-void
-DecodedStream::DestroyData()
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-
-  // Avoid the redundant blocking to output stream.
-  if (!mData) {
-    return;
-  }
-
-  // All streams are having their SourceMediaStream disconnected, so they
-  // need to be explicitly blocked again.
-  auto& outputStreams = OutputStreams();
-  for (int32_t i = outputStreams.Length() - 1; i >= 0; --i) {
-    OutputStreamData& os = outputStreams[i];
-    // Explicitly remove all existing ports.
-    // This is not strictly necessary but it's good form.
-    MOZ_ASSERT(os.mPort, "Double-delete of the ports!");
-    os.mPort->Destroy();
-    os.mPort = nullptr;
-    // During cycle collection, nsDOMMediaStream can be destroyed and send
-    // its Destroy message before this decoder is destroyed. So we have to
-    // be careful not to send any messages after the Destroy().
-    if (os.mStream->IsDestroyed()) {
-      // Probably the DOM MediaStream was GCed. Clean up.
-      outputStreams.RemoveElementAt(i);
-    } else {
-      os.mStream->ChangeExplicitBlockerCount(1);
-    }
-  }
-
-  mData = nullptr;
-}
-
-void
-DecodedStream::RecreateData()
-{
-  nsRefPtr<DecodedStream> self = this;
-  nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () -> void {
-    self->RecreateData(nullptr);
-  });
-  AbstractThread::MainThread()->Dispatch(r.forget());
-}
-
-void
-DecodedStream::RecreateData(MediaStreamGraph* aGraph)
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  MOZ_ASSERT((aGraph && !mData && OutputStreams().IsEmpty()) || // first time
-             (!aGraph && mData)); // 2nd time and later
-
-  auto source = aGraph->CreateSourceStream(nullptr);
-  DestroyData();
-  mData.reset(new DecodedStreamData(source, mPlaying));
-
-  // Note that the delay between removing ports in DestroyDecodedStream
-  // and adding new ones won't cause a glitch since all graph operations
-  // between main-thread stable states take effect atomically.
-  auto& outputStreams = OutputStreams();
-  for (int32_t i = outputStreams.Length() - 1; i >= 0; --i) {
-    OutputStreamData& os = outputStreams[i];
-    MOZ_ASSERT(!os.mStream->IsDestroyed(), "Should've been removed in DestroyData()");
-    Connect(&os);
-  }
-}
-
-nsTArray<OutputStreamData>&
-DecodedStream::OutputStreams()
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  GetReentrantMonitor().AssertCurrentThreadIn();
-  return mOutputStreams;
-}
-
-bool
-DecodedStream::HasConsumers() const
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  return mOutputStreams.IsEmpty();
-}
-
-ReentrantMonitor&
-DecodedStream::GetReentrantMonitor() const
-{
-  return mMonitor;
-}
-
-void
-DecodedStream::Connect(OutputStreamData* aStream)
-{
-	MOZ_ASSERT(NS_IsMainThread());
-  GetReentrantMonitor().AssertCurrentThreadIn();
-  NS_ASSERTION(!aStream->mPort, "Already connected?");
-
-  // The output stream must stay in sync with the decoded stream, so if
-  // either stream is blocked, we block the other.
-  aStream->mPort = aStream->mStream->AllocateInputPort(mData->mStream,
-      MediaInputPort::FLAG_BLOCK_INPUT | MediaInputPort::FLAG_BLOCK_OUTPUT);
-  // Unblock the output stream now. While it's connected to DecodedStream,
-  // DecodedStream is responsible for controlling blocking.
-  aStream->mStream->ChangeExplicitBlockerCount(-1);
-}
-
-void
-DecodedStream::Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded)
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-
-  if (!mData) {
-    RecreateData(aStream->Graph());
-  }
-
-  OutputStreamData* os = OutputStreams().AppendElement();
-  os->Init(this, aStream);
-  Connect(os);
-  if (aFinishWhenEnded) {
-    // Ensure that aStream finishes the moment mDecodedStream does.
-    aStream->SetAutofinish(true);
-  }
-}
-
-void
-DecodedStream::Remove(MediaStream* aStream)
-{
-  MOZ_ASSERT(NS_IsMainThread());
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-
-  auto& streams = OutputStreams();
-  for (int32_t i = streams.Length() - 1; i >= 0; --i) {
-    auto& os = streams[i];
-    MediaStream* p = os.mStream.get();
-    if (p == aStream) {
-      if (os.mPort) {
-        os.mPort->Destroy();
-        os.mPort = nullptr;
-      }
-      streams.RemoveElementAt(i);
-      break;
-    }
-  }
-}
-
-void
-DecodedStream::SetPlaying(bool aPlaying)
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  mPlaying = aPlaying;
-  if (mData) {
-    mData->SetPlaying(aPlaying);
-  }
-}
-
-void
-DecodedStream::InitTracks()
-{
-  GetReentrantMonitor().AssertCurrentThreadIn();
-
-  if (mData->mStreamInitialized) {
-    return;
-  }
-
-  SourceMediaStream* sourceStream = mData->mStream;
-
-  if (mInfo.HasAudio()) {
-    TrackID audioTrackId = mInfo.mAudio.mTrackId;
-    AudioSegment* audio = new AudioSegment();
-    sourceStream->AddAudioTrack(audioTrackId, mInfo.mAudio.mRate, 0, audio,
-                                SourceMediaStream::ADDTRACK_QUEUED);
-    mData->mNextAudioTime = mStartTime.ref();
-  }
-
-  if (mInfo.HasVideo()) {
-    TrackID videoTrackId = mInfo.mVideo.mTrackId;
-    VideoSegment* video = new VideoSegment();
-    sourceStream->AddTrack(videoTrackId, 0, video,
-                           SourceMediaStream::ADDTRACK_QUEUED);
-    mData->mNextVideoTime = mStartTime.ref();
-  }
-
-  sourceStream->FinishAddTracks();
-  mData->mStreamInitialized = true;
-}
-
-static void
-SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime,
-                AudioData* aAudio, AudioSegment* aOutput,
-                uint32_t aRate, double aVolume)
-{
-  // This logic has to mimic AudioSink closely to make sure we write
-  // the exact same silences
-  CheckedInt64 audioWrittenOffset = aStream->mAudioFramesWritten +
-                                    UsecsToFrames(aStartTime, aRate);
-  CheckedInt64 frameOffset = UsecsToFrames(aAudio->mTime, aRate);
-
-  if (!audioWrittenOffset.isValid() ||
-      !frameOffset.isValid() ||
-      // ignore packet that we've already processed
-      frameOffset.value() + aAudio->mFrames <= audioWrittenOffset.value()) {
-    return;
-  }
-
-  if (audioWrittenOffset.value() < frameOffset.value()) {
-    int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value();
-    // Write silence to catch up
-    AudioSegment silence;
-    silence.InsertNullDataAtStart(silentFrames);
-    aStream->mAudioFramesWritten += silentFrames;
-    audioWrittenOffset += silentFrames;
-    aOutput->AppendFrom(&silence);
-  }
-
-  MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value());
-
-  int64_t offset = audioWrittenOffset.value() - frameOffset.value();
-  size_t framesToWrite = aAudio->mFrames - offset;
-
-  aAudio->EnsureAudioBuffer();
-  nsRefPtr<SharedBuffer> buffer = aAudio->mAudioBuffer;
-  AudioDataValue* bufferData = static_cast<AudioDataValue*>(buffer->Data());
-  nsAutoTArray<const AudioDataValue*, 2> channels;
-  for (uint32_t i = 0; i < aAudio->mChannels; ++i) {
-    channels.AppendElement(bufferData + i * aAudio->mFrames + offset);
-  }
-  aOutput->AppendFrames(buffer.forget(), channels, framesToWrite);
-  aStream->mAudioFramesWritten += framesToWrite;
-  aOutput->ApplyVolume(aVolume);
-
-  aStream->mNextAudioTime = aAudio->GetEndTime();
-}
-
-void
-DecodedStream::SendAudio(double aVolume, bool aIsSameOrigin)
-{
-  GetReentrantMonitor().AssertCurrentThreadIn();
-
-  if (!mInfo.HasAudio()) {
-    return;
-  }
-
-  AudioSegment output;
-  uint32_t rate = mInfo.mAudio.mRate;
-  nsAutoTArray<nsRefPtr<AudioData>,10> audio;
-  TrackID audioTrackId = mInfo.mAudio.mTrackId;
-  SourceMediaStream* sourceStream = mData->mStream;
-
-  // It's OK to hold references to the AudioData because AudioData
-  // is ref-counted.
-  mAudioQueue.GetElementsAfter(mData->mNextAudioTime, &audio);
-  for (uint32_t i = 0; i < audio.Length(); ++i) {
-    SendStreamAudio(mData.get(), mStartTime.ref(), audio[i], &output, rate, aVolume);
-  }
-
-  if (!aIsSameOrigin) {
-    output.ReplaceWithDisabled();
-  }
-
-  // |mNextAudioTime| is updated as we process each audio sample in
-  // SendStreamAudio(). This is consistent with how |mNextVideoTime|
-  // is updated for video samples.
-  if (output.GetDuration() > 0) {
-    sourceStream->AppendToTrack(audioTrackId, &output);
-  }
-
-  if (mAudioQueue.IsFinished() && !mData->mHaveSentFinishAudio) {
-    sourceStream->EndTrack(audioTrackId);
-    mData->mHaveSentFinishAudio = true;
-  }
-}
-
-static void
-WriteVideoToMediaStream(MediaStream* aStream,
-                        layers::Image* aImage,
-                        int64_t aEndMicroseconds,
-                        int64_t aStartMicroseconds,
-                        const mozilla::gfx::IntSize& aIntrinsicSize,
-                        VideoSegment* aOutput)
-{
-  nsRefPtr<layers::Image> image = aImage;
-  StreamTime duration =
-      aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) -
-      aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds);
-  aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize);
-}
-
-static bool
-ZeroDurationAtLastChunk(VideoSegment& aInput)
-{
-  // Get the last video frame's start time in VideoSegment aInput.
-  // If the start time is equal to the duration of aInput, means the last video
-  // frame's duration is zero.
-  StreamTime lastVideoStratTime;
-  aInput.GetLastFrame(&lastVideoStratTime);
-  return lastVideoStratTime == aInput.GetDuration();
-}
-
-void
-DecodedStream::SendVideo(bool aIsSameOrigin)
-{
-  GetReentrantMonitor().AssertCurrentThreadIn();
-
-  if (!mInfo.HasVideo()) {
-    return;
-  }
-
-  VideoSegment output;
-  TrackID videoTrackId = mInfo.mVideo.mTrackId;
-  nsAutoTArray<nsRefPtr<VideoData>, 10> video;
-  SourceMediaStream* sourceStream = mData->mStream;
-
-  // It's OK to hold references to the VideoData because VideoData
-  // is ref-counted.
-  mVideoQueue.GetElementsAfter(mData->mNextVideoTime, &video);
-
-  for (uint32_t i = 0; i < video.Length(); ++i) {
-    VideoData* v = video[i];
-
-    if (mData->mNextVideoTime < v->mTime) {
-      // Write last video frame to catch up. mLastVideoImage can be null here
-      // which is fine, it just means there's no video.
-
-      // TODO: |mLastVideoImage| should come from the last image rendered
-      // by the state machine. This will avoid the black frame when capture
-      // happens in the middle of playback (especially in th middle of a
-      // video frame). E.g. if we have a video frame that is 30 sec long
-      // and capture happens at 15 sec, we'll have to append a black frame
-      // that is 15 sec long.
-      WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, v->mTime,
-          mData->mNextVideoTime, mData->mLastVideoImageDisplaySize, &output);
-      mData->mNextVideoTime = v->mTime;
-    }
-
-    if (mData->mNextVideoTime < v->GetEndTime()) {
-      WriteVideoToMediaStream(sourceStream, v->mImage,
-          v->GetEndTime(), mData->mNextVideoTime, v->mDisplay, &output);
-      mData->mNextVideoTime = v->GetEndTime();
-      mData->mLastVideoImage = v->mImage;
-      mData->mLastVideoImageDisplaySize = v->mDisplay;
-    }
-  }
-
-  // Check the output is not empty.
-  if (output.GetLastFrame()) {
-    mData->mEOSVideoCompensation = ZeroDurationAtLastChunk(output);
-  }
-
-  if (!aIsSameOrigin) {
-    output.ReplaceWithDisabled();
-  }
-
-  if (output.GetDuration() > 0) {
-    sourceStream->AppendToTrack(videoTrackId, &output);
-  }
-
-  if (mVideoQueue.IsFinished() && !mData->mHaveSentFinishVideo) {
-    if (mData->mEOSVideoCompensation) {
-      VideoSegment endSegment;
-      // Calculate the deviation clock time from DecodedStream.
-      int64_t deviation_usec = sourceStream->StreamTimeToMicroseconds(1);
-      WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage,
-          mData->mNextVideoTime + deviation_usec, mData->mNextVideoTime,
-          mData->mLastVideoImageDisplaySize, &endSegment);
-      mData->mNextVideoTime += deviation_usec;
-      MOZ_ASSERT(endSegment.GetDuration() > 0);
-      if (!aIsSameOrigin) {
-        endSegment.ReplaceWithDisabled();
-      }
-      sourceStream->AppendToTrack(videoTrackId, &endSegment);
-    }
-    sourceStream->EndTrack(videoTrackId);
-    mData->mHaveSentFinishVideo = true;
-  }
-}
-
-void
-DecodedStream::AdvanceTracks()
-{
-  GetReentrantMonitor().AssertCurrentThreadIn();
-
-  StreamTime endPosition = 0;
-
-  if (mInfo.HasAudio()) {
-    StreamTime audioEnd = mData->mStream->TicksToTimeRoundDown(
-        mInfo.mAudio.mRate, mData->mAudioFramesWritten);
-    endPosition = std::max(endPosition, audioEnd);
-  }
-
-  if (mInfo.HasVideo()) {
-    StreamTime videoEnd = mData->mStream->MicrosecondsToStreamTimeRoundDown(
-        mData->mNextVideoTime - mStartTime.ref());
-    endPosition = std::max(endPosition, videoEnd);
-  }
-
-  if (!mData->mHaveSentFinish) {
-    mData->mStream->AdvanceKnownTracksTime(endPosition);
-  }
-}
-
-bool
-DecodedStream::SendData(double aVolume, bool aIsSameOrigin)
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  MOZ_ASSERT(mStartTime.isSome(), "Must be called after StartPlayback()");
-
-  InitTracks();
-  SendAudio(aVolume, aIsSameOrigin);
-  SendVideo(aIsSameOrigin);
-  AdvanceTracks();
-
-  bool finished = (!mInfo.HasAudio() || mAudioQueue.IsFinished()) &&
-                  (!mInfo.HasVideo() || mVideoQueue.IsFinished());
-
-  if (finished && !mData->mHaveSentFinish) {
-    mData->mHaveSentFinish = true;
-    mData->mStream->Finish();
-  }
-
-  return finished;
-}
-
-int64_t
-DecodedStream::AudioEndTime() const
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  if (mStartTime.isSome() && mInfo.HasAudio()) {
-    CheckedInt64 t = mStartTime.ref() +
-      FramesToUsecs(mData->mAudioFramesWritten, mInfo.mAudio.mRate);
-    if (t.isValid()) {
-      return t.value();
-    }
-  }
-  return -1;
-}
-
-int64_t
-DecodedStream::GetPosition() const
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  // This is only called after MDSM starts playback. So mStartTime is
-  // guaranteed to be something.
-  MOZ_ASSERT(mStartTime.isSome());
-  return mStartTime.ref() + mData->GetPosition();
-}
-
-bool
-DecodedStream::IsFinished() const
-{
-  ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
-  return mData->IsFinished();
-}
-
-} // namespace mozilla

dom/media/DecodedStream.h

@@ -1,158 +0,0 @@
-/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim: set ts=8 sts=2 et sw=2 tw=80: */
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-#ifndef DecodedStream_h_
-#define DecodedStream_h_
-
-#include "mozilla/nsRefPtr.h"
-#include "nsTArray.h"
-#include "MediaInfo.h"
-
-#include "mozilla/UniquePtr.h"
-#include "mozilla/gfx/Point.h"
-#include "mozilla/CheckedInt.h"
-#include "mozilla/ReentrantMonitor.h"
-#include "mozilla/Maybe.h"
-
-namespace mozilla {
-
-class AudioData;
-class VideoData;
-class AudioSegment;
-class MediaStream;
-class MediaInputPort;
-class SourceMediaStream;
-class ProcessedMediaStream;
-class DecodedStream;
-class DecodedStreamGraphListener;
-class OutputStreamListener;
-class ReentrantMonitor;
-class MediaStreamGraph;
-
-template <class T> class MediaQueue;
-
-namespace layers {
-class Image;
-} // namespace layers
-
-/*
- * All MediaStream-related data is protected by the decoder's monitor.
- * We have at most one DecodedStreamDaata per MediaDecoder. Its stream
- * is used as the input for each ProcessedMediaStream created by calls to
- * captureStream(UntilEnded). Seeking creates a new source stream, as does
- * replaying after the input as ended. In the latter case, the new source is
- * not connected to streams created by captureStreamUntilEnded.
- */
-class DecodedStreamData {
-public:
-  DecodedStreamData(SourceMediaStream* aStream, bool aPlaying);
-  ~DecodedStreamData();
-  bool IsFinished() const;
-  int64_t GetPosition() const;
-  void SetPlaying(bool aPlaying);
-
-  /* The following group of fields are protected by the decoder's monitor
-   * and can be read or written on any thread.
-   */
-  // Count of audio frames written to the stream
-  int64_t mAudioFramesWritten;
-  // mNextVideoTime is the end timestamp for the last packet sent to the stream.
-  // Therefore video packets starting at or after this time need to be copied
-  // to the output stream.
-  int64_t mNextVideoTime; // microseconds
-  int64_t mNextAudioTime; // microseconds
-  // The last video image sent to the stream. Useful if we need to replicate
-  // the image.
-  nsRefPtr<layers::Image> mLastVideoImage;
-  gfx::IntSize mLastVideoImageDisplaySize;
-  // This is set to true when the stream is initialized (audio and
-  // video tracks added).
-  bool mStreamInitialized;
-  bool mHaveSentFinish;
-  bool mHaveSentFinishAudio;
-  bool mHaveSentFinishVideo;
-
-  // The decoder is responsible for calling Destroy() on this stream.
-  const nsRefPtr<SourceMediaStream> mStream;
-  nsRefPtr<DecodedStreamGraphListener> mListener;
-  bool mPlaying;
-  // True if we need to send a compensation video frame to ensure the
-  // StreamTime going forward.
-  bool mEOSVideoCompensation;
-};
-
-class OutputStreamData {
-public:
-  ~OutputStreamData();
-  void Init(DecodedStream* aDecodedStream, ProcessedMediaStream* aStream);
-  nsRefPtr<ProcessedMediaStream> mStream;
-  // mPort connects DecodedStreamData::mStream to our mStream.
-  nsRefPtr<MediaInputPort> mPort;
-  nsRefPtr<OutputStreamListener> mListener;
-};
-
-class DecodedStream {
-  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStream);
-public:
-  DecodedStream(MediaQueue<AudioData>& aAudioQueue,
-                MediaQueue<VideoData>& aVideoQueue);
-
-  // Mimic MDSM::StartAudioThread.
-  // Must be called before any calls to SendData().
-  void StartPlayback(int64_t aStartTime, const MediaInfo& aInfo);
-  // Mimic MDSM::StopAudioThread.
-  void StopPlayback();
-
-  void DestroyData();
-  void RecreateData();
-  void Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded);
-  void Remove(MediaStream* aStream);
-  void SetPlaying(bool aPlaying);
-  int64_t AudioEndTime() const;
-  int64_t GetPosition() const;
-  bool IsFinished() const;
-  bool HasConsumers() const;
-
-  // Return true if stream is finished.
-  bool SendData(double aVolume, bool aIsSameOrigin);
-
-protected:
-  virtual ~DecodedStream() {}
-
-private:
-  ReentrantMonitor& GetReentrantMonitor() const;
-  void RecreateData(MediaStreamGraph* aGraph);
-  void Connect(OutputStreamData* aStream);
-  nsTArray<OutputStreamData>& OutputStreams();
-  void InitTracks();
-  void AdvanceTracks();
-  void SendAudio(double aVolume, bool aIsSameOrigin);
-  void SendVideo(bool aIsSameOrigin);
-
-  UniquePtr<DecodedStreamData> mData;
-  // Data about MediaStreams that are being fed by the decoder.
-  nsTArray<OutputStreamData> mOutputStreams;
-
-  // TODO: This is a temp solution to get rid of decoder monitor on the main
-  // thread in MDSM::AddOutputStream and MDSM::RecreateDecodedStream as
-  // required by bug 1146482. DecodedStream needs to release monitor before
-  // calling back into MDSM functions in order to prevent deadlocks.
-  //
-  // Please move all capture-stream related code from MDSM into DecodedStream
-  // and apply "dispatch + mirroring" to get rid of this monitor in the future.
-  mutable ReentrantMonitor mMonitor;
-
-  bool mPlaying;
-  Maybe<int64_t> mStartTime;
-  MediaInfo mInfo;
-
-  MediaQueue<AudioData>& mAudioQueue;
-  MediaQueue<VideoData>& mVideoQueue;
-};
-
-} // namespace mozilla
-
-#endif // DecodedStream_h_

dom/media/GraphDriver.cpp

@@ -45,8 +45,6 @@ struct AutoProfilerUnregisterThread
 GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl)
   : mIterationStart(0),
     mIterationEnd(0),
-    mStateComputedTime(0),
-    mNextStateComputedTime(0),
     mGraphImpl(aGraphImpl),
     mWaitState(WAITSTATE_RUNNING),
     mCurrentTimeStamp(TimeStamp::Now()),
@@ -56,9 +54,7 @@ GraphDriver::GraphDriver(MediaStreamGraphImpl* aGraphImpl)
 
 void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver,
                                GraphTime aLastSwitchNextIterationStart,
-                               GraphTime aLastSwitchNextIterationEnd,
-                               GraphTime aLastSwitchStateComputedTime,
-                               GraphTime aLastSwitchNextStateComputedTime)
+                               GraphTime aLastSwitchNextIterationEnd)
 {
   // We set mIterationEnd here, because the first thing a driver do when it
   // does an iteration is to update graph times, so we are in fact setting
@@ -66,8 +62,6 @@ void GraphDriver::SetGraphTime(GraphDriver* aPreviousDriver,
   // iteration.
   mIterationStart = aLastSwitchNextIterationStart;
   mIterationEnd = aLastSwitchNextIterationEnd;
-  mStateComputedTime = aLastSwitchStateComputedTime;
-  mNextStateComputedTime = aLastSwitchNextStateComputedTime;
 
   STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", aPreviousDriver, aPreviousDriver->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
   MOZ_ASSERT(!mPreviousDriver);
@@ -102,17 +96,10 @@ void GraphDriver::EnsureImmediateWakeUpLocked()
   mGraphImpl->GetMonitor().Notify();
 }
 
-void GraphDriver::UpdateStateComputedTime(GraphTime aStateComputedTime)
+GraphTime
+GraphDriver::StateComputedTime() const
 {
-  MOZ_ASSERT(aStateComputedTime > mIterationEnd);
-  // The next state computed time can be the same as the previous, here: it
-  // means the driver would be have been blocking indefinitly, but the graph has
-  // been woken up right after having been to sleep.
-  if (aStateComputedTime < mStateComputedTime) {
-    printf("State time can't go backward %ld < %ld.\n", static_cast<long>(aStateComputedTime), static_cast<long>(mStateComputedTime));
-  }
-
-  mStateComputedTime = aStateComputedTime;
+  return mGraphImpl->mStateComputedTime;
 }
 
 void GraphDriver::EnsureNextIteration()
@@ -241,8 +228,7 @@ ThreadedDriver::Revive()
   // loop again.
   MonitorAutoLock mon(mGraphImpl->GetMonitor());
   if (mNextDriver) {
-    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                               mStateComputedTime, mNextStateComputedTime);
+    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
     mGraphImpl->SetCurrentDriver(mNextDriver);
     mNextDriver->Start();
   } else {
@@ -280,29 +266,46 @@ ThreadedDriver::RunThread()
 
   bool stillProcessing = true;
   while (stillProcessing) {
-    GraphTime prevCurrentTime, nextCurrentTime;
-    GetIntervalForIteration(prevCurrentTime, nextCurrentTime);
+    mIterationStart = IterationEnd();
+    mIterationEnd += GetIntervalForIteration();
 
-    mStateComputedTime = mNextStateComputedTime;
-    mNextStateComputedTime =
+    GraphTime stateComputedTime = StateComputedTime();
+    if (stateComputedTime < mIterationEnd) {
+      STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
+      mIterationEnd = stateComputedTime;
+    }
+
+    if (mIterationStart >= mIterationEnd) {
+      NS_ASSERTION(mIterationStart == mIterationEnd ,
+                   "Time can't go backwards!");
+      // This could happen due to low clock resolution, maybe?
+      STREAM_LOG(LogLevel::Debug, ("Time did not advance"));
+    }
+
+    GraphTime nextStateComputedTime =
       mGraphImpl->RoundUpToNextAudioBlock(
-        nextCurrentTime + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS));
+        mIterationEnd + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS));
+    if (nextStateComputedTime < stateComputedTime) {
+      // A previous driver may have been processing further ahead of
+      // iterationEnd.
+      STREAM_LOG(LogLevel::Warning,
+                 ("Prevent state from going backwards. interval[%ld; %ld] state[%ld; %ld]",
+                  (long)mIterationStart, (long)mIterationEnd,
+                  (long)stateComputedTime, (long)nextStateComputedTime));
+      nextStateComputedTime = stateComputedTime;
+    }
     STREAM_LOG(LogLevel::Debug,
                ("interval[%ld; %ld] state[%ld; %ld]",
                (long)mIterationStart, (long)mIterationEnd,
-               (long)mStateComputedTime, (long)mNextStateComputedTime));
+               (long)stateComputedTime, (long)nextStateComputedTime));
 
     mGraphImpl->mFlushSourcesNow = mGraphImpl->mFlushSourcesOnNextIteration;
     mGraphImpl->mFlushSourcesOnNextIteration = false;
-    stillProcessing = mGraphImpl->OneIteration(prevCurrentTime,
-                                               nextCurrentTime,
-                                               StateComputedTime(),
-                                               mNextStateComputedTime);
+    stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
 
     if (mNextDriver && stillProcessing) {
       STREAM_LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver"));
-      mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                                 mStateComputedTime, mNextStateComputedTime);
+      mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
       mGraphImpl->SetCurrentDriver(mNextDriver);
       mNextDriver->Start();
       return;
@@ -310,36 +313,21 @@ ThreadedDriver::RunThread()
   }
 }
 
-void
-SystemClockDriver::GetIntervalForIteration(GraphTime& aFrom, GraphTime& aTo)
+MediaTime
+SystemClockDriver::GetIntervalForIteration()
 {
   TimeStamp now = TimeStamp::Now();
-  aFrom = mIterationStart = IterationEnd();
-  aTo = mIterationEnd = mGraphImpl->SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds()) + IterationEnd();
-
+  MediaTime interval =
+    mGraphImpl->SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
   mCurrentTimeStamp = now;
 
-  MOZ_LOG(gMediaStreamGraphLog, LogLevel::Verbose, ("Updating current time to %f (real %f, mStateComputedTime %f)",
-         mGraphImpl->MediaTimeToSeconds(aTo),
-         (now - mInitialTimeStamp).ToSeconds(),
-         mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
+  MOZ_LOG(gMediaStreamGraphLog, LogLevel::Verbose,
+          ("Updating current time to %f (real %f, StateComputedTime() %f)",
+           mGraphImpl->MediaTimeToSeconds(IterationEnd() + interval),
+           (now - mInitialTimeStamp).ToSeconds(),
+           mGraphImpl->MediaTimeToSeconds(StateComputedTime())));
 
-  if (mStateComputedTime < aTo) {
-    STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
-    aTo = mIterationEnd = mStateComputedTime;
-  }
-
-  if (aFrom >= aTo) {
-    NS_ASSERTION(aFrom == aTo , "Time can't go backwards!");
-    // This could happen due to low clock resolution, maybe?
-    STREAM_LOG(LogLevel::Debug, ("Time did not advance"));
-  }
-}
-
-GraphTime
-SystemClockDriver::GetCurrentTime()
-{
-  return IterationEnd();
+  return interval;
 }
 
 TimeStamp
@@ -432,31 +420,12 @@ OfflineClockDriver::~OfflineClockDriver()
   }
 }
 
-void
-OfflineClockDriver::GetIntervalForIteration(GraphTime& aFrom, GraphTime& aTo)
+MediaTime
+OfflineClockDriver::GetIntervalForIteration()
 {
-  aFrom = mIterationStart = IterationEnd();
-  aTo = mIterationEnd = IterationEnd() + mGraphImpl->MillisecondsToMediaTime(mSlice);
-
-  if (mStateComputedTime < aTo) {
-    STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
-    aTo = mIterationEnd = mStateComputedTime;
-  }
-
-  if (aFrom >= aTo) {
-    NS_ASSERTION(aFrom == aTo , "Time can't go backwards!");
-    // This could happen due to low clock resolution, maybe?
-    STREAM_LOG(LogLevel::Debug, ("Time did not advance"));
-  }
+  return mGraphImpl->MillisecondsToMediaTime(mSlice);
 }
 
-GraphTime
-OfflineClockDriver::GetCurrentTime()
-{
-  return mIterationEnd;
-}
-
-
 void
 OfflineClockDriver::WaitForNextIteration()
 {
@@ -599,8 +568,7 @@ AudioCallbackDriver::Init()
     NS_WARNING("Could not create a cubeb stream for MediaStreamGraph, falling back to a SystemClockDriver");
     // Fall back to a driver using a normal thread.
     mNextDriver = new SystemClockDriver(GraphImpl());
-    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                               mStateComputedTime, mNextStateComputedTime);
+    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
     mGraphImpl->SetCurrentDriver(mNextDriver);
     DebugOnly<bool> found = mGraphImpl->RemoveMixerCallback(this);
     NS_WARN_IF_FALSE(!found, "Mixer callback not added when switching?");
@@ -693,8 +661,7 @@ AudioCallbackDriver::Revive()
   // If we were switching, switch now. Otherwise, start the audio thread again.
   MonitorAutoLock mon(mGraphImpl->GetMonitor());
   if (mNextDriver) {
-    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                              mStateComputedTime, mNextStateComputedTime);
+    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
     mGraphImpl->SetCurrentDriver(mNextDriver);
     mNextDriver->Start();
   } else {
@@ -705,24 +672,6 @@ AudioCallbackDriver::Revive()
   }
 }
 
-void
-AudioCallbackDriver::GetIntervalForIteration(GraphTime& aFrom,
-                                             GraphTime& aTo)
-{
-}
-
-GraphTime
-AudioCallbackDriver::GetCurrentTime()
-{
-  uint64_t position = 0;
-
-  if (cubeb_stream_get_position(mAudioStream, &position) != CUBEB_OK) {
-    NS_WARNING("Could not get current time from cubeb.");
-  }
-
-  return mSampleRate * position;
-}
-
 void AudioCallbackDriver::WaitForNextIteration()
 {
 }
@@ -830,7 +779,8 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
   AutoInCallback aic(this);
 #endif
 
-  if (mStateComputedTime == 0) {
+  GraphTime stateComputedTime = StateComputedTime();
+  if (stateComputedTime == 0) {
     MonitorAutoLock mon(mGraphImpl->GetMonitor());
     // Because this function is called during cubeb_stream_init (to prefill the
     // audio buffers), it can be that we don't have a message here (because this
@@ -862,21 +812,19 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
   // we don't need to run an iteration and if we do so we may overflow.
   if (mBuffer.Available()) {
 
-    mStateComputedTime = mNextStateComputedTime;
-
     // State computed time is decided by the audio callback's buffer length. We
     // compute the iteration start and end from there, trying to keep the amount
     // of buffering in the graph constant.
-    mNextStateComputedTime =
-      mGraphImpl->RoundUpToNextAudioBlock(mStateComputedTime + mBuffer.Available());
+    GraphTime nextStateComputedTime =
+      mGraphImpl->RoundUpToNextAudioBlock(stateComputedTime + mBuffer.Available());
 
     mIterationStart = mIterationEnd;
     // inGraph is the number of audio frames there is between the state time and
     // the current time, i.e. the maximum theoretical length of the interval we
     // could use as [mIterationStart; mIterationEnd].
-    GraphTime inGraph = mStateComputedTime - mIterationStart;
+    GraphTime inGraph = stateComputedTime - mIterationStart;
     // We want the interval [mIterationStart; mIterationEnd] to be before the
-    // interval [mStateComputedTime; mNextStateComputedTime]. We also want
+    // interval [stateComputedTime; nextStateComputedTime]. We also want
     // the distance between these intervals to be roughly equivalent each time, to
     // ensure there is no clock drift between current time and state time. Since
     // we can't act on the state time because we have to fill the audio buffer, we
@@ -885,21 +833,18 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
 
     STREAM_LOG(LogLevel::Debug, ("interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) (duration ticks: %ld)\n",
                               (long)mIterationStart, (long)mIterationEnd,
-                              (long)mStateComputedTime, (long)mNextStateComputedTime,
+                              (long)stateComputedTime, (long)nextStateComputedTime,
                               (long)aFrames, (uint32_t)durationMS,
-                              (long)(mNextStateComputedTime - mStateComputedTime)));
+                              (long)(nextStateComputedTime - stateComputedTime)));
 
     mCurrentTimeStamp = TimeStamp::Now();
 
-    if (mStateComputedTime < mIterationEnd) {
+    if (stateComputedTime < mIterationEnd) {
       STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
-      mIterationEnd = mStateComputedTime;
+      mIterationEnd = stateComputedTime;
     }
 
-    stillProcessing = mGraphImpl->OneIteration(mIterationStart,
-                                               mIterationEnd,
-                                               mStateComputedTime,
-                                               mNextStateComputedTime);
+    stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
   } else {
     NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
     stillProcessing = true;
@@ -917,8 +862,7 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames)
       }
     }
     STREAM_LOG(LogLevel::Debug, ("Switching to system driver."));
-    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                               mStateComputedTime, mNextStateComputedTime);
+    mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
     mGraphImpl->SetCurrentDriver(mNextDriver);
     mNextDriver->Start();
     // Returning less than aFrames starts the draining and eventually stops the
@@ -1025,8 +969,7 @@ AudioCallbackDriver::DeviceChangedCallback() {
   mCallbackReceivedWhileSwitching = 0;
   mGraphImpl->mFlushSourcesOnNextIteration = true;
   mNextDriver = new SystemClockDriver(GraphImpl());
-  mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd,
-                            mStateComputedTime, mNextStateComputedTime);
+  mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd);
   mGraphImpl->SetCurrentDriver(mNextDriver);
   mNextDriver->Start();
 #endif

dom/media/GraphDriver.h

@@ -50,15 +50,6 @@ static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
 static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
     SCHEDULE_SAFETY_MARGIN_MS;
 
-/**
- * Try have this much video buffered. Video frames are set
- * near the end of the iteration of the control loop. The maximum delay
- * to the setting of the next video frame is 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
- * SCHEDULE_SAFETY_MARGIN_MS. This is not optimal yet.
- */
-static const int VIDEO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
-    SCHEDULE_SAFETY_MARGIN_MS;
-
 class MediaStreamGraphImpl;
 
 class AudioCallbackDriver;
@@ -78,13 +69,6 @@ public:
   explicit GraphDriver(MediaStreamGraphImpl* aGraphImpl);
 
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GraphDriver);
-  /* When the graph wakes up to do an iteration, this returns the range of time
-   * that will be processed. */
-  virtual void GetIntervalForIteration(GraphTime& aFrom,
-                                       GraphTime& aTo) = 0;
-  /* Returns the current time for this graph. This is the end of the current
-   * iteration. */
-  virtual GraphTime GetCurrentTime() = 0;
   /* For real-time graphs, this waits until it's time to process more data. For
    * offline graphs, this is a no-op. */
   virtual void WaitForNextIteration() = 0;
@@ -143,10 +127,6 @@ public:
     return mIterationEnd;
   }
 
-  GraphTime StateComputedTime() {
-    return mStateComputedTime;
-  }
-
   virtual void GetAudioBuffer(float** aBuffer, long& aFrames) {
     MOZ_CRASH("This is not an Audio GraphDriver!");
   }
@@ -171,16 +151,7 @@ public:
    */
   void SetGraphTime(GraphDriver* aPreviousDriver,
                     GraphTime aLastSwitchNextIterationStart,
-                    GraphTime aLastSwitchNextIterationEnd,
-                    GraphTime aLastSwitchNextStateComputedTime,
-                    GraphTime aLastSwitchStateComputedTime);
-
-  /**
-   * Whenever the graph has computed the time until it has all state
-   * (mStateComputedState), it calls this to indicate the new time until which
-   * we have computed state.
-   */
-  void UpdateStateComputedTime(GraphTime aStateComputedTime);
+                    GraphTime aLastSwitchNextIterationEnd);
 
   /**
    * Call this to indicate that another iteration of the control loop is
@@ -207,13 +178,12 @@ public:
   virtual bool OnThread() = 0;
 
 protected:
+  GraphTime StateComputedTime() const;
+
   // Time of the start of this graph iteration.
   GraphTime mIterationStart;
   // Time of the end of this graph iteration.
   GraphTime mIterationEnd;
-  // Time, in the future, for which blocking has been computed.
-  GraphTime mStateComputedTime;
-  GraphTime mNextStateComputedTime;
   // The MediaStreamGraphImpl that owns this driver. This has a lifetime longer
   // than the driver, and will never be null.
   MediaStreamGraphImpl* mGraphImpl;
@@ -271,6 +241,11 @@ public:
 
   virtual bool OnThread() override { return !mThread || NS_GetCurrentThread() == mThread; }
 
+  /* When the graph wakes up to do an iteration, implementations return the
+   * range of time that will be processed.  This is called only once per
+   * iteration; it may determine the interval from state in a previous
+   * call. */
+  virtual MediaTime GetIntervalForIteration() = 0;
 protected:
   nsCOMPtr<nsIThread> mThread;
 };
@@ -284,9 +259,7 @@ class SystemClockDriver : public ThreadedDriver
 public:
   explicit SystemClockDriver(MediaStreamGraphImpl* aGraphImpl);
   virtual ~SystemClockDriver();
-  virtual void GetIntervalForIteration(GraphTime& aFrom,
-                                       GraphTime& aTo) override;
-  virtual GraphTime GetCurrentTime() override;
+  virtual MediaTime GetIntervalForIteration() override;
   virtual void WaitForNextIteration() override;
   virtual void WakeUp() override;
 
@@ -305,9 +278,7 @@ class OfflineClockDriver : public ThreadedDriver
 public:
   OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice);
   virtual ~OfflineClockDriver();
-  virtual void GetIntervalForIteration(GraphTime& aFrom,
-                                       GraphTime& aTo) override;
-  virtual GraphTime GetCurrentTime() override;
+  virtual MediaTime GetIntervalForIteration() override;
   virtual void WaitForNextIteration() override;
   virtual void WakeUp() override;
   virtual TimeStamp GetCurrentTimeStamp() override;
@@ -368,9 +339,6 @@ public:
   virtual void Stop() override;
   virtual void Resume() override;
   virtual void Revive() override;
-  virtual void GetIntervalForIteration(GraphTime& aFrom,
-                                       GraphTime& aTo) override;
-  virtual GraphTime GetCurrentTime() override;
   virtual void WaitForNextIteration() override;
   virtual void WakeUp() override;
 

dom/media/Intervals.h

@@ -269,7 +269,7 @@ public:
 
   IntervalSet(SelfType&& aOther)
   {
-    mIntervals.MoveElementsFrom(Move(aOther.mIntervals));
+    mIntervals.AppendElements(Move(aOther.mIntervals));
   }
 
   explicit IntervalSet(const ElemType& aOther)
@@ -379,7 +379,7 @@ public:
       normalized.AppendElement(Move(mIntervals[i]));
     }
     mIntervals.Clear();
-    mIntervals.MoveElementsFrom(Move(normalized));
+    mIntervals.AppendElements(Move(normalized));
 
     return *this;
   }
@@ -481,9 +481,8 @@ public:
         j++;
       }
     }
-
     mIntervals.Clear();
-    mIntervals.MoveElementsFrom(Move(intersection));
+    mIntervals.AppendElements(Move(intersection));
     return *this;
   }
 
@@ -683,7 +682,7 @@ private:
       normalized.AppendElement(Move(current));
 
       mIntervals.Clear();
-      mIntervals.MoveElementsFrom(Move(normalized));
+      mIntervals.AppendElements(Move(normalized));
     }
   }
 

dom/media/MP3Demuxer.cpp

@@ -13,6 +13,18 @@
 #include "mozilla/Endian.h"
 #include "VideoUtils.h"
 #include "TimeUnits.h"
+#include "prenv.h"
+
+#ifdef PR_LOGGING
+PRLogModuleInfo* gMP3DemuxerLog;
+#define MP3DEMUXER_LOG(msg, ...) \
+  MOZ_LOG(gMP3DemuxerLog, LogLevel::Debug, ("MP3Demuxer " msg, ##__VA_ARGS__))
+#define MP3DEMUXER_LOGV(msg, ...) \
+  MOZ_LOG(gMP3DemuxerLog, LogLevel::Verbose, ("MP3Demuxer " msg, ##__VA_ARGS__))
+#else
+#define MP3DEMUXER_LOG(msg, ...)
+#define MP3DEMUXER_LOGV(msg, ...)
+#endif
 
 using mozilla::media::TimeUnit;
 using mozilla::media::TimeIntervals;
@@ -38,23 +50,16 @@ MP3Demuxer::InitInternal() {
 nsRefPtr<MP3Demuxer::InitPromise>
 MP3Demuxer::Init() {
   if (!InitInternal()) {
+    MP3DEMUXER_LOG("MP3Demuxer::Init() failure: waiting for data");
+
     return InitPromise::CreateAndReject(
-      DemuxerFailureReason::WAITING_FOR_DATA, __func__);
+      DemuxerFailureReason::DEMUXER_ERROR, __func__);
   }
 
+  MP3DEMUXER_LOG("MP3Demuxer::Init() successful");
   return InitPromise::CreateAndResolve(NS_OK, __func__);
 }
 
-already_AddRefed<MediaDataDemuxer>
-MP3Demuxer::Clone() const {
-  nsRefPtr<MP3Demuxer> demuxer = new MP3Demuxer(mSource);
-  if (!demuxer->InitInternal()) {
-    NS_WARNING("Couldn't recreate MP3Demuxer");
-    return nullptr;
-  }
-  return demuxer.forget();
-}
-
 bool
 MP3Demuxer::HasTrackType(TrackInfo::TrackType aType) const {
   return aType == TrackInfo::kAudioTrack;
@@ -79,15 +84,17 @@ MP3Demuxer::IsSeekable() const {
 }
 
 void
-MP3Demuxer::NotifyDataArrived(uint32_t aLength, int64_t aOffset) {
+MP3Demuxer::NotifyDataArrived() {
   // TODO: bug 1169485.
   NS_WARNING("Unimplemented function NotifyDataArrived");
+  MP3DEMUXER_LOGV("NotifyDataArrived()");
 }
 
 void
 MP3Demuxer::NotifyDataRemoved() {
   // TODO: bug 1169485.
   NS_WARNING("Unimplemented function NotifyDataRemoved");
+  MP3DEMUXER_LOGV("NotifyDataRemoved()");
 }
 
 
@@ -105,6 +112,12 @@ MP3TrackDemuxer::MP3TrackDemuxer(MediaResource* aSource)
   , mChannels(0)
 {
   Reset();
+
+#ifdef PR_LOGGING
+  if (!gMP3DemuxerLog) {
+    gMP3DemuxerLog = PR_NewLogModule("MP3Demuxer");
+  }
+#endif
 }
 
 bool
@@ -113,6 +126,10 @@ MP3TrackDemuxer::Init() {
   FastSeek(TimeUnit());
   // Read the first frame to fetch sample rate and other meta data.
   nsRefPtr<MediaRawData> frame(GetNextFrame(FindFirstFrame()));
+
+  MP3DEMUXER_LOG("Init StreamLength()=%" PRId64 " first-frame-found=%d",
+                 StreamLength(), !!frame);
+
   if (!frame) {
     return false;
   }
@@ -130,6 +147,10 @@ MP3TrackDemuxer::Init() {
   mInfo->mMimeType = "audio/mpeg";
   mInfo->mDuration = Duration().ToMicroseconds();
 
+  MP3DEMUXER_LOG("Init mInfo={mRate=%d mChannels=%d mBitDepth=%d mDuration=%" PRId64 "}",
+                 mInfo->mRate, mInfo->mChannels, mInfo->mBitDepth,
+                 mInfo->mDuration);
+
   return mSamplesPerSecond && mChannels;
 }
 
@@ -205,6 +226,11 @@ MP3TrackDemuxer::FastSeek(const TimeUnit& aTime) {
 
 TimeUnit
 MP3TrackDemuxer::ScanUntil(const TimeUnit& aTime) {
+  MP3DEMUXER_LOG("ScanUntil(%" PRId64 ") avgFrameLen=%f mNumParsedFrames=%" PRIu64
+                 " mFrameIndex=%" PRId64 " mOffset=%" PRIu64,
+                 aTime, AverageFrameLength(), mNumParsedFrames, mFrameIndex,
+                 mOffset);
+
   if (!aTime.ToMicroseconds()) {
     return FastSeek(aTime);
   }
@@ -220,13 +246,30 @@ MP3TrackDemuxer::ScanUntil(const TimeUnit& aTime) {
   MediaByteRange nextRange = FindNextFrame();
   while (SkipNextFrame(nextRange) && Duration(mFrameIndex + 1) < aTime) {
     nextRange = FindNextFrame();
+    MP3DEMUXER_LOGV("ScanUntil* avgFrameLen=%f mNumParsedFrames=%" PRIu64
+                " mFrameIndex=%" PRId64 " mOffset=%" PRIu64 " Duration=%" PRId64,
+                aTime, AverageFrameLength(), mNumParsedFrames, mFrameIndex,
+                mOffset, Duration(mFrameIndex + 1));
   }
 
+  MP3DEMUXER_LOG("ScanUntil End avgFrameLen=%f mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64 " mOffset=%" PRIu64,
+              aTime, AverageFrameLength(), mNumParsedFrames, mFrameIndex,
+              mOffset);
+
   return SeekPosition();
 }
 
 nsRefPtr<MP3TrackDemuxer::SamplesPromise>
 MP3TrackDemuxer::GetSamples(int32_t aNumSamples) {
+  MP3DEMUXER_LOGV("GetSamples(%d) Begin mOffset=%" PRIu64 " mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64
+              " mTotalFrameLen=%" PRIu64 " mSamplesPerFrame=%d mSamplesPerSecond=%d "
+              "mChannels=%d",
+              aNumSamples,
+              mOffset, mNumParsedFrames, mFrameIndex, mTotalFrameLen, mSamplesPerFrame,
+              mSamplesPerSecond, mChannels);
+
   if (!aNumSamples) {
     return SamplesPromise::CreateAndReject(
         DemuxerFailureReason::DEMUXER_ERROR, __func__);
@@ -243,6 +286,15 @@ MP3TrackDemuxer::GetSamples(int32_t aNumSamples) {
     frames->mSamples.AppendElement(frame);
   }
 
+  MP3DEMUXER_LOGV("GetSamples() End mSamples.Size()=%d aNumSamples=%d mOffset=%" PRIu64
+              " mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64
+              " mTotalFrameLen=%" PRIu64 " mSamplesPerFrame=%d mSamplesPerSecond=%d "
+              "mChannels=%d",
+              frames->mSamples.Length(), aNumSamples,
+              mOffset, mNumParsedFrames, mFrameIndex, mTotalFrameLen, mSamplesPerFrame,
+              mSamplesPerSecond, mChannels);
+
   if (frames->mSamples.IsEmpty()) {
     return SamplesPromise::CreateAndReject(
         DemuxerFailureReason::END_OF_STREAM, __func__);
@@ -252,6 +304,8 @@ MP3TrackDemuxer::GetSamples(int32_t aNumSamples) {
 
 void
 MP3TrackDemuxer::Reset() {
+  MP3DEMUXER_LOG("Reset()");
+
   FastSeek(TimeUnit());
   mParser.Reset();
 }
@@ -270,19 +324,18 @@ MP3TrackDemuxer::GetResourceOffset() const {
 
 TimeIntervals
 MP3TrackDemuxer::GetBuffered() {
-  // TODO: bug 1169485.
-  NS_WARNING("Unimplemented function GetBuffered");
-  return TimeIntervals();
-}
+  TimeUnit duration = Duration();
 
-int64_t
-MP3TrackDemuxer::GetEvictionOffset(TimeUnit aTime) {
-  return 0;
+  if (duration <= TimeUnit()) {
+    return TimeIntervals();
+  }
+  AutoPinned<MediaResource> stream(mSource.GetResource());
+  return GetEstimatedBufferedTimeRanges(stream, duration.ToMicroseconds());
 }
 
 int64_t
 MP3TrackDemuxer::StreamLength() const {
-  return mSource->GetLength();
+  return mSource.GetLength();
 }
 
 TimeUnit
@@ -372,6 +425,13 @@ MP3TrackDemuxer::FindNextFrame() {
   static const int BUFFER_SIZE = 64;
   static const int MAX_SKIPPED_BYTES = 1024 * BUFFER_SIZE;
 
+  MP3DEMUXER_LOGV("FindNext() Begin mOffset=%" PRIu64 " mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64
+              " mTotalFrameLen=%" PRIu64 " mSamplesPerFrame=%d mSamplesPerSecond=%d "
+              "mChannels=%d",
+              mOffset, mNumParsedFrames, mFrameIndex, mTotalFrameLen, mSamplesPerFrame,
+              mSamplesPerSecond, mChannels);
+
   uint8_t buffer[BUFFER_SIZE];
   int32_t read = 0;
   bool foundFrame = false;
@@ -428,11 +488,19 @@ MP3TrackDemuxer::SkipNextFrame(const MediaByteRange& aRange) {
 
   UpdateState(aRange);
 
+  MP3DEMUXER_LOGV("SkipNext() End mOffset=%" PRIu64 " mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64
+              " mTotalFrameLen=%" PRIu64 " mSamplesPerFrame=%d mSamplesPerSecond=%d "
+              "mChannels=%d",
+              mOffset, mNumParsedFrames, mFrameIndex, mTotalFrameLen, mSamplesPerFrame,
+              mSamplesPerSecond, mChannels);
+
   return true;
 }
 
 already_AddRefed<MediaRawData>
 MP3TrackDemuxer::GetNextFrame(const MediaByteRange& aRange) {
+  MP3DEMUXER_LOG("GetNext() Begin({mStart=%" PRId64 " Length()=%" PRId64 "})");
   if (!aRange.Length()) {
     return nullptr;
   }
@@ -442,6 +510,7 @@ MP3TrackDemuxer::GetNextFrame(const MediaByteRange& aRange) {
 
   nsAutoPtr<MediaRawDataWriter> frameWriter(frame->CreateWriter());
   if (!frameWriter->SetSize(aRange.Length())) {
+    MP3DEMUXER_LOG("GetNext() Exit failed to allocated media buffer");
     return nullptr;
   }
 
@@ -470,6 +539,13 @@ MP3TrackDemuxer::GetNextFrame(const MediaByteRange& aRange) {
     mFirstFrameOffset = frame->mOffset;
   }
 
+  MP3DEMUXER_LOGV("GetNext() End mOffset=%" PRIu64 " mNumParsedFrames=%" PRIu64
+              " mFrameIndex=%" PRId64
+              " mTotalFrameLen=%" PRIu64 " mSamplesPerFrame=%d mSamplesPerSecond=%d "
+              "mChannels=%d",
+              mOffset, mNumParsedFrames, mFrameIndex, mTotalFrameLen, mSamplesPerFrame,
+              mSamplesPerSecond, mChannels);
+
   return frame.forget();
 }
 
@@ -577,8 +653,9 @@ MP3TrackDemuxer::Read(uint8_t* aBuffer, int64_t aOffset, int32_t aSize) {
   aSize = ClampReadSize(aOffset, aSize);
 
   uint32_t read = 0;
-  const nsresult rv = mSource->ReadAt(aOffset, reinterpret_cast<char*>(aBuffer),
-                                      static_cast<uint32_t>(aSize), &read);
+  MP3DEMUXER_LOGV("MP3TrackDemuxer::Read        -> ReadAt(%d)", aSize);
+  const nsresult rv = mSource.ReadAt(aOffset, reinterpret_cast<char*>(aBuffer),
+                                     static_cast<uint32_t>(aSize), &read);
   NS_ENSURE_SUCCESS(rv, 0);
   return static_cast<int32_t>(read);
 }

dom/media/MP3Demuxer.h

@@ -21,14 +21,18 @@ public:
   // MediaDataDemuxer interface.
   explicit MP3Demuxer(MediaResource* aSource);
   nsRefPtr<InitPromise> Init() override;
-  already_AddRefed<MediaDataDemuxer> Clone() const override;
   bool HasTrackType(TrackInfo::TrackType aType) const override;
   uint32_t GetNumberTracks(TrackInfo::TrackType aType) const override;
   already_AddRefed<MediaTrackDemuxer> GetTrackDemuxer(
       TrackInfo::TrackType aType, uint32_t aTrackNumber) override;
   bool IsSeekable() const override;
-  void NotifyDataArrived(uint32_t aLength, int64_t aOffset) override;
+  void NotifyDataArrived() override;
   void NotifyDataRemoved() override;
+  // Do not shift the calculated buffered range by the start time of the first
+  // decoded frame. The mac MP3 decoder will buffer some samples and the first
+  // frame returned has typically a start time that is non-zero, causing our
+  // buffered range to have a negative start time.
+  bool ShouldComputeStartTime() const override { return false; }
 
 private:
   // Synchronous initialization.
@@ -388,7 +392,6 @@ public:
     media::TimeUnit aTimeThreshold) override;
   int64_t GetResourceOffset() const override;
   media::TimeIntervals GetBuffered() override;
-  int64_t GetEvictionOffset(media::TimeUnit aTime) override;
 
 private:
   // Destructor.
@@ -442,7 +445,7 @@ private:
   double AverageFrameLength() const;
 
   // The (hopefully) MPEG resource.
-  nsRefPtr<MediaResource> mSource;
+  MediaResourceIndex mSource;
 
   // MPEG frame parser used to detect frames and extract side info.
   FrameParser mParser;

dom/media/MediaData.cpp

@@ -115,7 +115,7 @@ VideoData::VideoData(int64_t aOffset,
                      int64_t aTimecode,
                      IntSize aDisplay,
                      layers::ImageContainer::FrameID aFrameID)
-  : MediaData(VIDEO_DATA, aOffset, aTime, aDuration)
+  : MediaData(VIDEO_DATA, aOffset, aTime, aDuration, 1)
   , mDisplay(aDisplay)
   , mFrameID(aFrameID)
   , mSentToCompositor(false)
@@ -486,20 +486,20 @@ VideoData::Create(const VideoInfo& aInfo,
 #define RAW_DATA_ALIGNMENT 31U
 
 MediaRawData::MediaRawData()
-  : MediaData(RAW_DATA)
-  , mCrypto(mCryptoInternal)
+  : MediaData(RAW_DATA, 0)
   , mData(nullptr)
   , mSize(0)
+  , mCrypto(mCryptoInternal)
   , mBuffer(nullptr)
   , mCapacity(0)
 {
 }
 
 MediaRawData::MediaRawData(const uint8_t* aData, size_t aSize)
-  : MediaData(RAW_DATA)
-  , mCrypto(mCryptoInternal)
+  : MediaData(RAW_DATA, 0)
   , mData(nullptr)
   , mSize(0)
+  , mCrypto(mCryptoInternal)
   , mBuffer(nullptr)
   , mCapacity(0)
 {

dom/media/MediaData.h

@@ -40,12 +40,14 @@ public:
   MediaData(Type aType,
             int64_t aOffset,
             int64_t aTimestamp,
-            int64_t aDuration)
+            int64_t aDuration,
+            uint32_t aFrames)
     : mType(aType)
     , mOffset(aOffset)
     , mTime(aTimestamp)
     , mTimecode(aTimestamp)
     , mDuration(aDuration)
+    , mFrames(aFrames)
     , mKeyframe(false)
     , mDiscontinuity(false)
   {}
@@ -66,6 +68,9 @@ public:
   // Duration of sample, in microseconds.
   int64_t mDuration;
 
+  // Amount of frames for contained data.
+  const uint32_t mFrames;
+
   bool mKeyframe;
 
   // True if this is the first sample after a gap or discontinuity in
@@ -79,13 +84,29 @@ public:
     mTime = mTime - aStartTime;
     return mTime >= 0;
   }
+
+  template <typename ReturnType>
+  const ReturnType* As() const
+  {
+    MOZ_ASSERT(this->mType == ReturnType::sType);
+    return static_cast<const ReturnType*>(this);
+  }
+
+  template <typename ReturnType>
+  ReturnType* As()
+  {
+    MOZ_ASSERT(this->mType == ReturnType::sType);
+    return static_cast<ReturnType*>(this);
+  }
+
 protected:
-  explicit MediaData(Type aType)
+  MediaData(Type aType, uint32_t aFrames)
     : mType(aType)
     , mOffset(0)
     , mTime(0)
     , mTimecode(0)
     , mDuration(0)
+    , mFrames(aFrames)
     , mKeyframe(false)
     , mDiscontinuity(false)
   {}
@@ -105,8 +126,7 @@ public:
             AudioDataValue* aData,
             uint32_t aChannels,
             uint32_t aRate)
-    : MediaData(sType, aOffset, aTime, aDuration)
-    , mFrames(aFrames)
+    : MediaData(sType, aOffset, aTime, aDuration, aFrames)
     , mChannels(aChannels)
     , mRate(aRate)
     , mAudioData(aData) {}
@@ -128,7 +148,6 @@ public:
   // If mAudioBuffer is null, creates it from mAudioData.
   void EnsureAudioBuffer();
 
-  const uint32_t mFrames;
   const uint32_t mChannels;
   const uint32_t mRate;
   // At least one of mAudioBuffer/mAudioData must be non-null.
@@ -289,7 +308,7 @@ public:
             bool aKeyframe,
             int64_t aTimecode,
             IntSize aDisplay,
-            int32_t aFrameID);
+            uint32_t aFrameID);
 
 protected:
   ~VideoData();
@@ -413,6 +432,8 @@ private:
   // MediaByteBuffer is a ref counted infallible TArray.
 class MediaByteBuffer : public nsTArray<uint8_t> {
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaByteBuffer);
+  MediaByteBuffer() = default;
+  explicit MediaByteBuffer(size_t aCapacity) : nsTArray<uint8_t>(aCapacity) {}
 
 private:
   ~MediaByteBuffer() {}

dom/media/MediaDataDemuxer.h

@@ -46,24 +46,12 @@ public:
   // Initializes the demuxer. Other methods cannot be called unless
   // initialization has completed and succeeded.
   // Typically a demuxer will wait to parse the metadata before resolving the
-  // promise. The promise will be rejected with WAITING_FOR_DATA should
-  // insufficient data be available at the time. Init() would have to be called
-  // again to retry once more data has been received.
+  // promise. The promise must not be resolved until sufficient data is
+  // supplied. For example, an incomplete metadata would cause the promise to be
+  // rejected should no more data be coming, while the demuxer would wait
+  // otherwise.
   virtual nsRefPtr<InitPromise> Init() = 0;
 
-  // MediaFormatReader ensures that calls to the MediaDataDemuxer are thread-safe.
-  // This is done by having multiple demuxers, created with Clone(), one per
-  // running thread.
-  // However, should the MediaDataDemuxer object guaranteed to be thread-safe
-  // such cloning is unnecessary and only one demuxer will be used across
-  // all threads.
-  virtual bool IsThreadSafe() { return false; }
-
-  // Clone the demuxer and return a new initialized demuxer.
-  // This can only be called once Init() has succeeded.
-  // The new demuxer can be immediately use to retrieve the track demuxers.
-  virtual already_AddRefed<MediaDataDemuxer> Clone() const = 0;
-
   // Returns true if a aType track type is available.
   virtual bool HasTrackType(TrackInfo::TrackType aType) const = 0;
 
@@ -89,15 +77,17 @@ public:
     return nullptr;
   }
 
-  // Notifies the demuxer that the underlying resource has received more data.
+  // Notifies the demuxer that the underlying resource has received more data
+  // since the demuxer was initialized.
   // The demuxer can use this mechanism to inform all track demuxers that new
-  // data is available.
-  virtual void NotifyDataArrived(uint32_t aLength, int64_t aOffset) { }
+  // data is available and to refresh its buffered range.
+  virtual void NotifyDataArrived() { }
 
-  // Notifies the demuxer that the underlying resource has had data removed.
+  // Notifies the demuxer that the underlying resource has had data removed
+  // since the demuxer was initialized.
   // The demuxer can use this mechanism to inform all track demuxers to update
-  // its TimeIntervals.
-  // This will be called should the demuxer be used with MediaSource.
+  // its buffered range.
+  // This will be called should the demuxer be used with MediaSourceResource.
   virtual void NotifyDataRemoved() { }
 
   // Indicate to MediaFormatReader if it should compute the start time
@@ -207,8 +197,6 @@ public:
 
   virtual media::TimeIntervals GetBuffered() = 0;
 
-  virtual int64_t GetEvictionOffset(media::TimeUnit aTime) = 0;
-
   // If the MediaTrackDemuxer and MediaDataDemuxer hold cross references.
   // BreakCycles must be overridden.
   virtual void BreakCycles()

dom/media/MediaDecoder.h

@@ -184,6 +184,7 @@ destroying the MediaDecoder object.
 #if !defined(MediaDecoder_h_)
 #define MediaDecoder_h_
 
+#include "mozilla/Atomics.h"
 #include "mozilla/MozPromise.h"
 #include "mozilla/ReentrantMonitor.h"
 #include "mozilla/StateMirroring.h"
@@ -191,17 +192,20 @@ destroying the MediaDecoder object.
 
 #include "mozilla/dom/AudioChannelBinding.h"
 
-#include "nsISupports.h"
+#include "necko-config.h"
+#include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
 #include "nsIObserver.h"
-#include "nsAutoPtr.h"
+#include "nsISupports.h"
 #include "nsITimer.h"
-#include "MediaResource.h"
-#include "MediaDecoderOwner.h"
-#include "MediaStreamGraph.h"
+
 #include "AbstractMediaDecoder.h"
-#include "DecodedStream.h"
-#include "necko-config.h"
+#include "MediaDecoderOwner.h"
+#include "MediaEventSource.h"
+#include "MediaMetadataManager.h"
+#include "MediaResource.h"
+#include "MediaStatistics.h"
+#include "MediaStreamGraph.h"
 #include "TimeUnits.h"
 
 class nsIStreamListener;
@@ -515,16 +519,13 @@ public:
 
   bool OnDecodeTaskQueue() const override;
 
-  MediaDecoderStateMachine* GetStateMachine() { return mDecoderStateMachine; }
+  MediaDecoderStateMachine* GetStateMachine() const;
   void SetStateMachine(MediaDecoderStateMachine* aStateMachine);
 
   // Returns the monitor for other threads to synchronise access to
   // state.
   ReentrantMonitor& GetReentrantMonitor() override;
 
-  // Returns true if the decoder is shut down
-  bool IsShutdown() const final override;
-
   // Constructs the time ranges representing what segments of the media
   // are buffered and playable.
   virtual media::TimeIntervals GetBuffered();
@@ -534,6 +535,39 @@ public:
   size_t SizeOfVideoQueue();
   size_t SizeOfAudioQueue();
 
+  // Helper struct for accumulating resource sizes that need to be measured
+  // asynchronously. Once all references are dropped the callback will be
+  // invoked.
+  struct ResourceSizes
+  {
+    typedef MozPromise<size_t, size_t, true> SizeOfPromise;
+    NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ResourceSizes)
+    explicit ResourceSizes(MallocSizeOf aMallocSizeOf)
+      : mMallocSizeOf(aMallocSizeOf)
+      , mByteSize(0)
+      , mCallback()
+    {
+    }
+
+    mozilla::MallocSizeOf mMallocSizeOf;
+    mozilla::Atomic<size_t> mByteSize;
+
+    nsRefPtr<SizeOfPromise> Promise()
+    {
+      return mCallback.Ensure(__func__);
+    }
+
+private:
+    ~ResourceSizes()
+    {
+      mCallback.ResolveIfExists(mByteSize, __func__);
+    }
+
+    MozPromiseHolder<SizeOfPromise> mCallback;
+  };
+
+  virtual void AddSizeOfResources(ResourceSizes* aSizes);
+
   VideoFrameContainer* GetVideoFrameContainer() final override
   {
     return mVideoFrameContainer;
@@ -561,17 +595,15 @@ public:
   // Records activity stopping on the channel.
   void DispatchPlaybackStopped() {
     nsRefPtr<MediaDecoder> self = this;
-    nsCOMPtr<nsIRunnable> r =
-      NS_NewRunnableFunction([self] () { self->mPlaybackStatistics->Stop(); });
+    nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () {
+      self->mPlaybackStatistics->Stop();
+      self->ComputePlaybackRate();
+    });
     AbstractThread::MainThread()->Dispatch(r.forget());
   }
 
   // The actual playback rate computation. The monitor must be held.
-  virtual double ComputePlaybackRate(bool* aReliable);
-
-  // Return true when the media is same-origin with the element. The monitor
-  // must be held.
-  bool IsSameOriginMedia();
+  void ComputePlaybackRate();
 
   // Returns true if we can play the entire media through without stopping
   // to buffer, given the current download and playback rates.
@@ -580,13 +612,6 @@ public:
   void SetAudioChannel(dom::AudioChannel aChannel) { mAudioChannel = aChannel; }
   dom::AudioChannel GetAudioChannel() { return mAudioChannel; }
 
-  // Send a new set of metadata to the state machine, to be dispatched to the
-  // main thread to be presented when the |currentTime| of the media is greater
-  // or equal to aPublishTime.
-  void QueueMetadata(int64_t aPublishTime,
-                     nsAutoPtr<MediaInfo> aInfo,
-                     nsAutoPtr<MetadataTags> aTags) override;
-
   /******
    * The following methods must only be called on the main
    * thread.
@@ -619,14 +644,7 @@ public:
 
   // Removes all audio tracks and video tracks that are previously added into
   // the track list. Call on the main thread only.
-  virtual void RemoveMediaTracks() override;
-
-  // Returns true if the this decoder is expecting any more data to arrive
-  // sometime in the not-too-distant future, either from the network or from
-  // an appendBuffer call on a MediaSource element.
-  //
-  // Acquires the monitor. Call from any thread.
-  virtual bool IsExpectingMoreData();
+  void RemoveMediaTracks();
 
   // Called when the video has completed playing.
   // Call on the main thread only.
@@ -663,13 +681,6 @@ public:
   // position.
   int64_t GetDownloadPosition();
 
-  // Updates the approximate byte offset which playback has reached. This is
-  // used to calculate the readyState transitions.
-  void UpdatePlaybackOffset(int64_t aOffset);
-
-  // Provide access to the state machine object
-  MediaDecoderStateMachine* GetStateMachine() const;
-
   // Drop reference to state machine.  Only called during shutdown dance.
   virtual void BreakCycles();
 
@@ -716,41 +727,11 @@ public:
   static bool IsAppleMP3Enabled();
 #endif
 
-  // Schedules the state machine to run one cycle on the shared state
-  // machine thread. Main thread only.
-  nsresult ScheduleStateMachine();
-
-  struct Statistics {
-    // Estimate of the current playback rate (bytes/second).
-    double mPlaybackRate;
-    // Estimate of the current download rate (bytes/second). This
-    // ignores time that the channel was paused by Gecko.
-    double mDownloadRate;
-    // Total length of media stream in bytes; -1 if not known
-    int64_t mTotalBytes;
-    // Current position of the download, in bytes. This is the offset of
-    // the first uncached byte after the decoder position.
-    int64_t mDownloadPosition;
-    // Current position of decoding, in bytes (how much of the stream
-    // has been consumed)
-    int64_t mDecoderPosition;
-    // Current position of playback, in bytes
-    int64_t mPlaybackPosition;
-    // If false, then mDownloadRate cannot be considered a reliable
-    // estimate (probably because the download has only been running
-    // a short time).
-    bool mDownloadRateReliable;
-    // If false, then mPlaybackRate cannot be considered a reliable
-    // estimate (probably because playback has only been running
-    // a short time).
-    bool mPlaybackRateReliable;
-  };
-
   // Return statistics. This is used for progress events and other things.
   // This can be called from any thread. It's only a snapshot of the
   // current state, since other threads might be changing the state
   // at any time.
-  virtual Statistics GetStatistics();
+  MediaStatistics GetStatistics();
 
   // Frame decoding/painting related performance counters.
   // Threadsafe.
@@ -909,17 +890,6 @@ protected:
   // Whether the decoder implementation supports dormant mode.
   bool mDormantSupported;
 
-  // Current decoding position in the stream. This is where the decoder
-  // is up to consuming the stream. This is not adjusted during decoder
-  // seek operations, but it's updated at the end when we start playing
-  // back again.
-  int64_t mDecoderPosition;
-  // Current playback position in the stream. This is (approximately)
-  // where we're up to playing back the stream. This is not adjusted
-  // during decoder seek operations, but it's updated at the end when we
-  // start playing back again.
-  int64_t mPlaybackPosition;
-
   // The logical playback position of the media resource in units of
   // seconds. This corresponds to the "official position" in HTML5. Note that
   // we need to store this as a double, rather than an int64_t (like
@@ -936,13 +906,6 @@ protected:
   // Official duration of the media resource as observed by script.
   double mDuration;
 
-  // True if the media is seekable (i.e. supports random access).
-  bool mMediaSeekable;
-
-  // True if the media is same-origin with the element. Data can only be
-  // passed to MediaStreams when this is true.
-  bool mSameOriginMedia;
-
   /******
    * The following member variables can be accessed from any thread.
    ******/
@@ -971,8 +934,12 @@ private:
   ReentrantMonitor mReentrantMonitor;
 
 protected:
+
   virtual void CallSeek(const SeekTarget& aTarget);
 
+  // Returns true if heuristic dormant is supported.
+  bool IsHeuristicDormantSupported() const;
+
   MozPromiseRequestHolder<SeekPromise> mSeekRequest;
 
   // True when seeking or otherwise moving the play position around in
@@ -992,6 +959,8 @@ protected:
 
   const char* PlayStateStr();
 
+  void OnMetadataUpdate(TimedMetadata&& aMetadata);
+
   // This should only ever be accessed from the main thread.
   // It is set in Init and cleared in Shutdown when the element goes away.
   // The decoder does not add a reference the element.
@@ -1062,6 +1031,9 @@ protected:
   // Timer to schedule updating dormant state.
   nsCOMPtr<nsITimer> mDormantTimer;
 
+  // A listener to receive metadata updates from MDSM.
+  MediaEventListener mTimedMetadataListener;
+
 protected:
   // Whether the state machine is shut down.
   Mirror<bool> mStateMachineIsShutdown;
@@ -1078,6 +1050,12 @@ protected:
   // Duration of the media resource according to the state machine.
   Mirror<media::NullableTimeUnit> mStateMachineDuration;
 
+  // Current playback position in the stream. This is (approximately)
+  // where we're up to playing back the stream. This is not adjusted
+  // during decoder seek operations, but it's updated at the end when we
+  // start playing back again.
+  Mirror<int64_t> mPlaybackPosition;
+
   // Volume of playback.  0.0 = muted. 1.0 = full volume.
   Canonical<double> mVolume;
 
@@ -1101,20 +1079,35 @@ protected:
   // This can only be changed on the main thread while holding the decoder
   // monitor. Thus, it can be safely read while holding the decoder monitor
   // OR on the main thread.
-  // Any change to the state on the main thread must call NotifyAll on the
-  // monitor so the decode thread can wake up.
   Canonical<PlayState> mPlayState;
 
   // This can only be changed on the main thread while holding the decoder
   // monitor. Thus, it can be safely read while holding the decoder monitor
   // OR on the main thread.
-  // Any change to the state must call NotifyAll on the monitor.
-  // This can only be PLAY_STATE_PAUSED or PLAY_STATE_PLAYING.
   Canonical<PlayState> mNextState;
 
   // True if the decoder is seeking.
   Canonical<bool> mLogicallySeeking;
 
+  // True if the media is same-origin with the element. Data can only be
+  // passed to MediaStreams when this is true.
+  Canonical<bool> mSameOriginMedia;
+
+  // Estimate of the current playback rate (bytes/second).
+  Canonical<double> mPlaybackBytesPerSecond;
+
+  // True if mPlaybackBytesPerSecond is a reliable estimate.
+  Canonical<bool> mPlaybackRateReliable;
+
+  // Current decoding position in the stream. This is where the decoder
+  // is up to consuming the stream. This is not adjusted during decoder
+  // seek operations, but it's updated at the end when we start playing
+  // back again.
+  Canonical<int64_t> mDecoderPosition;
+
+  // True if the media is seekable (i.e. supports random access).
+  Canonical<bool> mMediaSeekable;
+
 public:
   AbstractCanonical<media::NullableTimeUnit>* CanonicalDurationOrNull() override;
   AbstractCanonical<double>* CanonicalVolume() {
@@ -1141,6 +1134,21 @@ public:
   AbstractCanonical<bool>* CanonicalLogicallySeeking() {
     return &mLogicallySeeking;
   }
+  AbstractCanonical<bool>* CanonicalSameOriginMedia() {
+    return &mSameOriginMedia;
+  }
+  AbstractCanonical<double>* CanonicalPlaybackBytesPerSecond() {
+    return &mPlaybackBytesPerSecond;
+  }
+  AbstractCanonical<bool>* CanonicalPlaybackRateReliable() {
+    return &mPlaybackRateReliable;
+  }
+  AbstractCanonical<int64_t>* CanonicalDecoderPosition() {
+    return &mDecoderPosition;
+  }
+  AbstractCanonical<bool>* CanonicalMediaSeekable() {
+    return &mMediaSeekable;
+  }
 };
 
 } // namespace mozilla

dom/media/MediaDecoderReader.cpp

@@ -192,6 +192,8 @@ MediaDecoderReader::ThrottledNotifyDataArrived(const Interval<int64_t>& aInterva
 
   if (mThrottledInterval.isNothing()) {
     mThrottledInterval.emplace(aInterval);
+  } else if (mThrottledInterval.ref().Contains(aInterval)) {
+    return;
   } else if (!mThrottledInterval.ref().Contiguous(aInterval)) {
     DoThrottledNotify();
     mThrottledInterval.emplace(aInterval);
@@ -295,8 +297,7 @@ public:
 
     // Make sure ResetDecode hasn't been called in the mean time.
     if (!mReader->mBaseVideoPromise.IsEmpty()) {
-      mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold,
-                                /* aForceDecodeAhead = */ false);
+      mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold);
     }
 
     return NS_OK;
@@ -333,8 +334,7 @@ private:
 
 nsRefPtr<MediaDecoderReader::VideoDataPromise>
 MediaDecoderReader::RequestVideoData(bool aSkipToNextKeyframe,
-                                     int64_t aTimeThreshold,
-                                     bool aForceDecodeAhead)
+                                     int64_t aTimeThreshold)
 {
   nsRefPtr<VideoDataPromise> p = mBaseVideoPromise.Ensure(__func__);
   bool skip = aSkipToNextKeyframe;

dom/media/MediaDecoderReader.h

@@ -11,6 +11,7 @@
 #include "AbstractMediaDecoder.h"
 #include "MediaInfo.h"
 #include "MediaData.h"
+#include "MediaMetadataManager.h"
 #include "MediaQueue.h"
 #include "MediaTimer.h"
 #include "AudioCompactor.h"
@@ -148,7 +149,7 @@ public:
   // If aSkipToKeyframe is true, the decode should skip ahead to the
   // the next keyframe at or after aTimeThreshold microseconds.
   virtual nsRefPtr<VideoDataPromise>
-  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead);
+  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold);
 
   friend class ReRequestVideoWithSkipTask;
   friend class ReRequestAudioTask;
@@ -279,7 +280,6 @@ public:
 
   // Notify the reader that data from the resource was evicted (MediaSource only)
   virtual void NotifyDataRemoved() {}
-  virtual int64_t GetEvictionOffset(double aTime) { return -1; }
 
   virtual MediaQueue<AudioData>& AudioQueue() { return mAudioQueue; }
   virtual MediaQueue<VideoData>& VideoQueue() { return mVideoQueue; }
@@ -321,12 +321,16 @@ public:
   // the newer async model.
   virtual bool IsAsync() const { return false; }
 
-  virtual void DisableHardwareAcceleration() {}
-
   // Returns true if this decoder reader uses hardware accelerated video
   // decoding.
   virtual bool VideoIsHardwareAccelerated() const { return false; }
 
+  virtual void DisableHardwareAcceleration() {}
+
+  TimedMetadataEventSource& TimedMetadataEvent() {
+    return mTimedMetadataEvent;
+  }
+
 protected:
   virtual ~MediaDecoderReader();
 
@@ -418,6 +422,9 @@ protected:
   bool mHitAudioDecodeError;
   bool mShutdown;
 
+  // Used to send TimedMetadata to the listener.
+  TimedMetadataEventProducer mTimedMetadataEvent;
+
 private:
   // Promises used only for the base-class (sync->async adapter) implementation
   // of Request{Audio,Video}Data.

dom/media/MediaDecoderStateMachine.h

@@ -91,15 +91,20 @@ hardware (via AudioStream).
 #include "MediaDecoder.h"
 #include "MediaDecoderReader.h"
 #include "MediaDecoderOwner.h"
+#include "MediaEventSource.h"
 #include "MediaMetadataManager.h"
+#include "MediaStatistics.h"
 #include "MediaTimer.h"
-#include "DecodedStream.h"
 #include "ImageContainer.h"
 
 namespace mozilla {
 
+namespace media {
+class MediaSink;
+}
+
 class AudioSegment;
-class AudioSink;
+class DecodedStream;
 class TaskQueue;
 
 extern PRLogModuleInfo* gMediaDecoderLog;
@@ -111,9 +116,8 @@ extern PRLogModuleInfo* gMediaSampleLog;
   state machine thread, and controls the audio "push" thread.
 
   All internal state is synchronised via the decoder monitor. State changes
-  are either propagated by NotifyAll on the monitor (typically when state
-  changes need to be propagated to non-state machine threads) or by scheduling
-  the state machine to run another cycle on the shared state machine thread.
+  are propagated by scheduling the state machine to run another cycle on the
+  shared state machine thread.
 
   See MediaDecoder.h for more details.
 */
@@ -138,7 +142,6 @@ public:
     DECODER_STATE_DECODING_METADATA,
     DECODER_STATE_WAIT_FOR_RESOURCES,
     DECODER_STATE_WAIT_FOR_CDM,
-    DECODER_STATE_DECODING_FIRSTFRAME,
     DECODER_STATE_DORMANT,
     DECODER_STATE_DECODING,
     DECODER_STATE_SEEKING,
@@ -153,7 +156,11 @@ public:
   void RemoveOutputStream(MediaStream* aStream);
 
   // Set/Unset dormant state.
-  void SetDormant(bool aDormant);
+  void DispatchSetDormant(bool aDormant);
+
+  TimedMetadataEventSource& TimedMetadataEvent() {
+    return mMetadataManager.TimedMetadataEvent();
+  }
 
 private:
   // Initialization that needs to happen on the task queue. This is the first
@@ -161,18 +168,18 @@ private:
   // constructor immediately after the task queue is created.
   void InitializationTask();
 
-  void DispatchAudioCaptured();
-  void DispatchAudioUncaptured();
+  void SetDormant(bool aDormant);
+
+  void SetAudioCaptured(bool aCaptured);
+
+  void NotifyWaitingForResourcesStatusChanged();
+
+  nsRefPtr<MediaDecoder::SeekPromise> Seek(SeekTarget aTarget);
 
   void Shutdown();
-public:
 
-  void DispatchShutdown()
-  {
-    nsCOMPtr<nsIRunnable> runnable =
-      NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown);
-    OwnerThread()->Dispatch(runnable.forget());
-  }
+public:
+  void DispatchShutdown();
 
   void FinishShutdown();
 
@@ -185,8 +192,7 @@ public:
   bool OnTaskQueue() const;
 
   // Seeks to the decoder to aTarget asynchronously.
-  // Must be called on the state machine thread.
-  nsRefPtr<MediaDecoder::SeekPromise> Seek(SeekTarget aTarget);
+  nsRefPtr<MediaDecoder::SeekPromise> InvokeSeek(SeekTarget aTarget);
 
   // Clear the flag indicating that a playback position change event
   // is currently queued. This is called from the main thread and must
@@ -205,8 +211,12 @@ private:
   // immediately stop playback and buffer downloaded data. Called on
   // the state machine thread.
   void StartBuffering();
-public:
 
+  bool CanPlayThrough();
+
+  MediaStatistics GetStatistics();
+
+public:
   void DispatchStartBuffering()
   {
     nsCOMPtr<nsIRunnable> runnable =
@@ -317,15 +327,12 @@ public:
     if (mReader) {
       mReader->BreakCycles();
     }
-    mDecodedStream->DestroyData();
     mResource = nullptr;
     mDecoder = nullptr;
   }
 
-  // Copy queued audio/video data in the reader to any output MediaStreams that
-  // need it.
-  void SendStreamData();
-  void FinishStreamData();
+  // Discard audio/video data that are already played by MSG.
+  void DiscardStreamData();
   bool HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs);
   bool HaveEnoughDecodedVideo();
 
@@ -333,17 +340,13 @@ public:
   // shutting down. The decoder monitor must be held while calling this.
   bool IsShutdown();
 
-  void QueueMetadata(int64_t aPublishTime,
-                     nsAutoPtr<MediaInfo> aInfo,
-                     nsAutoPtr<MetadataTags> aTags);
-
   // Returns true if we're currently playing. The decoder monitor must
   // be held.
   bool IsPlaying() const;
 
   // Called when the reader may have acquired the hardware resources required
   // to begin decoding.
-  void NotifyWaitingForResourcesStatusChanged();
+  void DispatchWaitingForResourcesStatusChanged();
 
   // Notifies the state machine that should minimize the number of samples
   // decoded we preroll, until playback starts. The first time playback starts
@@ -400,17 +403,15 @@ protected:
   void PushFront(AudioData* aSample);
   void PushFront(VideoData* aSample);
 
-  void OnAudioPopped(const AudioData* aSample);
-  void OnVideoPopped(const VideoData* aSample);
+  void OnAudioPopped(const nsRefPtr<MediaData>& aSample);
+  void OnVideoPopped(const nsRefPtr<MediaData>& aSample);
 
   void VolumeChanged();
   void LogicalPlaybackRateChanged();
   void PreservesPitchChanged();
 
-  MediaQueue<AudioData>& AudioQueue() { return mAudioQueue; }
-  MediaQueue<VideoData>& VideoQueue() { return mVideoQueue; }
-
-  nsresult FinishDecodeFirstFrame();
+  MediaQueue<MediaData>& AudioQueue() { return mAudioQueue; }
+  MediaQueue<MediaData>& VideoQueue() { return mVideoQueue; }
 
   // True if our buffers of decoded audio are not full, and we should
   // decode more.
@@ -459,22 +460,6 @@ protected:
   // parties.
   void UpdateNextFrameStatus();
 
-  // Called when AudioSink reaches the end. |mPlayStartTime| and
-  // |mPlayDuration| are updated to provide a good base for calculating video
-  // stream time.
-  void ResyncAudioClock();
-
-  // Returns the audio clock, if we have audio, or -1 if we don't.
-  // Called on the state machine thread.
-  int64_t GetAudioClock() const;
-
-  int64_t GetStreamClock() const;
-
-  // Get the video stream position, taking the |playbackRate| change into
-  // account. This is a position in the media, not the duration of the playback
-  // so far. Returns the position for the given time aTimeStamp.
-  int64_t GetVideoStreamPosition(TimeStamp aTimeStamp) const;
-
   // Return the current time, either the audio clock if available (if the media
   // has audio, and the playback is possible), or a clock for the video.
   // Called on the state machine thread.
@@ -494,7 +479,7 @@ protected:
   void UpdatePlaybackPositionInternal(int64_t aTime);
 
   // Decode monitor must be held.
-  void CheckTurningOffHardwareDecoder(VideoData* aData);
+  bool CheckFrameValidity(VideoData* aData);
 
   // Sets VideoQueue images into the VideoFrameContainer. Called on the shared
   // state machine thread. Decode monitor must be held. The first aMaxFrames
@@ -513,13 +498,17 @@ protected:
   // state machine thread.
   void UpdateRenderedVideoFrames();
 
-  // Stops the audio thread. The decoder monitor must be held with exactly
-  // one lock count. Called on the state machine thread.
-  void StopAudioThread();
+  media::MediaSink* CreateAudioSink();
 
-  // Starts the audio thread. The decoder monitor must be held with exactly
-  // one lock count. Called on the state machine thread.
-  void StartAudioThread();
+  // Stops the media sink and shut it down.
+  // The decoder monitor must be held with exactly one lock count.
+  // Called on the state machine thread.
+  void StopMediaSink();
+
+  // Create and start the media sink.
+  // The decoder monitor must be held with exactly one lock count.
+  // Called on the state machine thread.
+  void StartMediaSink();
 
   // Notification method invoked when mPlayState changes.
   void PlayStateChanged();
@@ -527,6 +516,9 @@ protected:
   // Notification method invoked when mLogicallySeeking changes.
   void LogicallySeekingChanged();
 
+  // Notification method invoked when mSameOriginMedia changes.
+  void SameOriginMediaChanged();
+
   // Sets internal state which causes playback of media to pause.
   // The decoder monitor must be held.
   void StopPlayback();
@@ -558,11 +550,6 @@ protected:
 
   void EnqueueFirstFrameLoadedEvent();
 
-  // Dispatches a task to the decode task queue to begin decoding content.
-  // This is threadsafe and can be called on any thread.
-  // The decoder monitor must be held.
-  nsresult EnqueueDecodeFirstFrameTask();
-
   // Clears any previous seeking state and initiates a new see on the decoder.
   // The decoder monitor must be held.
   void InitiateSeek();
@@ -620,16 +607,14 @@ protected:
   void OnMetadataRead(MetadataHolder* aMetadata);
   void OnMetadataNotRead(ReadMetadataFailureReason aReason);
 
-  // Initiate first content decoding. Called on the state machine thread.
-  // The decoder monitor must be held with exactly one lock count.
-  nsresult DecodeFirstFrame();
-
-  // Wraps the call to DecodeFirstFrame(), signals a DecodeError() on failure.
-  void CallDecodeFirstFrame();
-
-  // Checks whether we're finished decoding first audio and/or video packets,
-  // and switches to DECODING state if so.
-  void MaybeFinishDecodeFirstFrame();
+  // Checks whether we're finished decoding first audio and/or video packets.
+  // If so will trigger firing loadeddata event.
+  // If there are any queued seek, will change state to DECODER_STATE_SEEKING
+  // and return true.
+  bool MaybeFinishDecodeFirstFrame();
+  // Return true if we are currently decoding the first frames.
+  bool IsDecodingFirstFrame();
+  void FinishDecodeFirstFrame();
 
   // Seeks to mSeekTarget. Called on the decode thread. The decoder monitor
   // must be held with exactly one lock count.
@@ -670,12 +655,12 @@ protected:
   void SetPlayStartTime(const TimeStamp& aTimeStamp);
 
 private:
-  // Resolved by the AudioSink to signal that all outstanding work is complete
+  // Resolved by the MediaSink to signal that all outstanding work is complete
   // and the sink is shutting down.
-  void OnAudioSinkComplete();
+  void OnMediaSinkComplete();
 
-  // Rejected by the AudioSink to signal errors.
-  void OnAudioSinkError();
+  // Rejected by the MediaSink to signal errors.
+  void OnMediaSinkError();
 
   // Return true if the video decoder's decode speed can not catch up the
   // play time.
@@ -889,15 +874,12 @@ private:
 
   // Queue of audio frames. This queue is threadsafe, and is accessed from
   // the audio, decoder, state machine, and main threads.
-  MediaQueue<AudioData> mAudioQueue;
-
+  MediaQueue<MediaData> mAudioQueue;
   // Queue of video frames. This queue is threadsafe, and is accessed from
   // the decoder, state machine, and main threads.
-  MediaQueue<VideoData> mVideoQueue;
+  MediaQueue<MediaData> mVideoQueue;
 
   // The decoder monitor must be obtained before modifying this state.
-  // NotifyAll on the monitor must be called when the state is changed so
-  // that interested threads can wake up and alter behaviour if appropriate
   // Accessed on state machine, audio, main, and AV thread.
   Watchable<State> mState;
 
@@ -998,15 +980,15 @@ private:
   // Media Fragment end time in microseconds. Access controlled by decoder monitor.
   int64_t mFragmentEndTime;
 
-  // The audio sink resource.  Used on state machine and audio threads.
-  RefPtr<AudioSink> mAudioSink;
+  // The media sink resource.  Used on the state machine thread.
+  nsRefPtr<media::MediaSink> mMediaSink;
 
   // The reader, don't call its methods with the decoder monitor held.
   // This is created in the state machine's constructor.
   nsRefPtr<MediaDecoderReader> mReader;
 
-  // The end time of the last audio frame that's been pushed onto the audio sink
-  // or DecodedStream in microseconds. This will approximately be the end time
+  // The end time of the last audio frame that's been pushed onto the media sink
+  // in microseconds. This will approximately be the end time
   // of the audio stream, unless another frame is pushed to the hardware.
   int64_t AudioEndTime() const;
 
@@ -1264,11 +1246,9 @@ private:
 
   mozilla::RollingMean<uint32_t, uint32_t> mCorruptFrames;
 
-  bool mDisabledHardwareAcceleration;
-
-  // mDecodingFrozenAtStateDecoding: turn on/off at
-  //                                 SetDormant/Seek,Play.
-  bool mDecodingFrozenAtStateDecoding;
+  // True if we need to call FinishDecodeFirstFrame() upon frame decoding
+  // successeeding.
+  bool mDecodingFirstFrame;
 
   // True if we are back from DECODER_STATE_DORMANT state and
   // LoadedMetadataEvent was already sent.
@@ -1287,12 +1267,15 @@ private:
   // Only written on the main thread while holding the monitor. Therefore it
   // can be read on any thread while holding the monitor, or on the main thread
   // without holding the monitor.
-  nsRefPtr<DecodedStream> mDecodedStream;
+  nsRefPtr<DecodedStream> mStreamSink;
 
   // Media data resource from the decoder.
   nsRefPtr<MediaResource> mResource;
 
-  MozPromiseRequestHolder<GenericPromise> mAudioSinkPromise;
+  MozPromiseRequestHolder<GenericPromise> mMediaSinkPromise;
+
+  MediaEventListener mAudioQueueListener;
+  MediaEventListener mVideoQueueListener;
 
 private:
   // The buffered range. Mirrored from the decoder thread.
@@ -1320,6 +1303,22 @@ private:
   // Pitch preservation for the playback rate.
   Mirror<bool> mPreservesPitch;
 
+  // True if the media is same-origin with the element. Data can only be
+  // passed to MediaStreams when this is true.
+  Mirror<bool> mSameOriginMedia;
+
+  // Estimate of the current playback rate (bytes/second).
+  Mirror<double> mPlaybackBytesPerSecond;
+
+  // True if mPlaybackBytesPerSecond is a reliable estimate.
+  Mirror<bool> mPlaybackRateReliable;
+
+  // Current decoding position in the stream.
+  Mirror<int64_t> mDecoderPosition;
+
+  // True if the media is seekable (i.e. supports random access).
+  Mirror<bool> mMediaSeekable;
+
   // Duration of the media. This is guaranteed to be non-null after we finish
   // decoding the first frame.
   Canonical<media::NullableTimeUnit> mDuration;
@@ -1336,6 +1335,9 @@ private:
   // playback position.
   Canonical<int64_t> mCurrentPosition;
 
+  // Current playback position in the stream in bytes.
+  Canonical<int64_t> mPlaybackOffset;
+
 public:
   AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() {
     return mReader->CanonicalBuffered();
@@ -1352,6 +1354,9 @@ public:
   AbstractCanonical<int64_t>* CanonicalCurrentPosition() {
     return &mCurrentPosition;
   }
+  AbstractCanonical<int64_t>* CanonicalPlaybackOffset() {
+    return &mPlaybackOffset;
+  }
 };
 
 } // namespace mozilla

dom/media/MediaEventSource.h

@@ -46,6 +46,15 @@ private:
   Atomic<bool> mRevoked;
 };
 
+enum class ListenerMode : int8_t {
+  // Allow at most one listener. Move will be used when possible
+  // to pass the event data to save copy.
+  Exclusive,
+  // This is the default. Event data will always be copied when passed
+  // to the listeners.
+  NonExclusive
+};
+
 namespace detail {
 
 /**
@@ -66,7 +75,7 @@ struct EventTypeTraits<void> {
  * Test if a method function or lambda accepts one or more arguments.
  */
 template <typename T>
-class TakeArgs {
+class TakeArgsHelper {
   template <typename C> static FalseType test(void(C::*)(), int);
   template <typename C> static FalseType test(void(C::*)() const, int);
   template <typename C> static FalseType test(void(C::*)() volatile, int);
@@ -77,6 +86,9 @@ public:
   typedef decltype(test(DeclVal<T>(), 0)) Type;
 };
 
+template <typename T>
+struct TakeArgs : public TakeArgsHelper<T>::Type {};
+
 template <typename T> struct EventTarget;
 
 template <>
@@ -108,9 +120,196 @@ private:
   T* const mPtr;
 };
 
+/**
+ * A helper class to pass event data to the listeners. Optimized to save
+ * copy when Move is possible or |Function| takes no arguments.
+ */
+template<typename Target, typename Function>
+class ListenerHelper {
+  // Define our custom runnable to minimize copy of the event data.
+  // NS_NewRunnableFunction will result in 2 copies of the event data.
+  // One is captured by the lambda and the other is the copy of the lambda.
+  template <typename T>
+  class R : public nsRunnable {
+    typedef typename RemoveCV<typename RemoveReference<T>::Type>::Type ArgType;
+  public:
+    template <typename U>
+    R(RevocableToken* aToken, const Function& aFunction, U&& aEvent)
+      : mToken(aToken), mFunction(aFunction), mEvent(Forward<U>(aEvent)) {}
+
+    NS_IMETHOD Run() override {
+      // Don't call the listener if it is disconnected.
+      if (!mToken->IsRevoked()) {
+        // Enable move whenever possible since mEvent won't be used anymore.
+        mFunction(Move(mEvent));
+      }
+      return NS_OK;
+    }
+
+  private:
+    nsRefPtr<RevocableToken> mToken;
+    Function mFunction;
+    ArgType mEvent;
+  };
+
+public:
+  ListenerHelper(RevocableToken* aToken, Target* aTarget, const Function& aFunc)
+    : mToken(aToken), mTarget(aTarget), mFunction(aFunc) {}
+
+  // |F| takes one argument.
+  template <typename F, typename T>
+  typename EnableIf<TakeArgs<F>::value, void>::Type
+  Dispatch(const F& aFunc, T&& aEvent) {
+    nsCOMPtr<nsIRunnable> r = new R<T>(mToken, aFunc, Forward<T>(aEvent));
+    EventTarget<Target>::Dispatch(mTarget.get(), r.forget());
+  }
+
+  // |F| takes no arguments. Don't bother passing aEvent.
+  template <typename F, typename T>
+  typename EnableIf<!TakeArgs<F>::value, void>::Type
+  Dispatch(const F& aFunc, T&&) {
+    const nsRefPtr<RevocableToken>& token = mToken;
+    nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([=] () {
+      // Don't call the listener if it is disconnected.
+      if (!token->IsRevoked()) {
+        aFunc();
+      }
+    });
+    EventTarget<Target>::Dispatch(mTarget.get(), r.forget());
+  }
+
+  template <typename T>
+  void Dispatch(T&& aEvent) {
+    Dispatch(mFunction, Forward<T>(aEvent));
+  }
+
+private:
+  nsRefPtr<RevocableToken> mToken;
+  const nsRefPtr<Target> mTarget;
+  Function mFunction;
+};
+
+/**
+ * Define whether an event data should be copied or moved to the listeners.
+ *
+ * @Copy Data will always be copied. Each listener gets a copy.
+ * @Move Data will always be moved.
+ * @Both Data will be moved when possible or copied when necessary.
+ */
+enum class EventPassMode : int8_t {
+  Copy,
+  Move,
+  Both
+};
+
+class ListenerBase {
+public:
+  ListenerBase() : mToken(new RevocableToken()) {}
+  ~ListenerBase() {
+    MOZ_ASSERT(Token()->IsRevoked(), "Must disconnect the listener.");
+  }
+  RevocableToken* Token() const {
+    return mToken;
+  }
+private:
+  const nsRefPtr<RevocableToken> mToken;
+};
+
+/**
+ * Stored by MediaEventSource to send notifications to the listener.
+ * Since virtual methods can not be templated, this class is specialized
+ * to provide different Dispatch() overloads depending on EventPassMode.
+ */
+template <typename ArgType, EventPassMode Mode = EventPassMode::Copy>
+class Listener : public ListenerBase {
+public:
+  virtual ~Listener() {}
+  virtual void Dispatch(const ArgType& aEvent) = 0;
+};
+
+template <typename ArgType>
+class Listener<ArgType, EventPassMode::Both> : public ListenerBase {
+public:
+  virtual ~Listener() {}
+  virtual void Dispatch(const ArgType& aEvent) = 0;
+  virtual void Dispatch(ArgType&& aEvent) = 0;
+};
+
+template <typename ArgType>
+class Listener<ArgType, EventPassMode::Move> : public ListenerBase {
+public:
+  virtual ~Listener() {}
+  virtual void Dispatch(ArgType&& aEvent) = 0;
+};
+
+/**
+ * Store the registered target thread and function so it knows where and to
+ * whom to send the event data.
+ */
+template <typename Target, typename Function, typename ArgType, EventPassMode>
+class ListenerImpl : public Listener<ArgType, EventPassMode::Copy> {
+public:
+  ListenerImpl(Target* aTarget, const Function& aFunction)
+    : mHelper(ListenerBase::Token(), aTarget, aFunction) {}
+  void Dispatch(const ArgType& aEvent) override {
+    mHelper.Dispatch(aEvent);
+  }
+private:
+  ListenerHelper<Target, Function> mHelper;
+};
+
+template <typename Target, typename Function, typename ArgType>
+class ListenerImpl<Target, Function, ArgType, EventPassMode::Both>
+  : public Listener<ArgType, EventPassMode::Both> {
+public:
+  ListenerImpl(Target* aTarget, const Function& aFunction)
+    : mHelper(ListenerBase::Token(), aTarget, aFunction) {}
+  void Dispatch(const ArgType& aEvent) override {
+    mHelper.Dispatch(aEvent);
+  }
+  void Dispatch(ArgType&& aEvent) override {
+    mHelper.Dispatch(Move(aEvent));
+  }
+private:
+  ListenerHelper<Target, Function> mHelper;
+};
+
+template <typename Target, typename Function, typename ArgType>
+class ListenerImpl<Target, Function, ArgType, EventPassMode::Move>
+  : public Listener<ArgType, EventPassMode::Move> {
+public:
+  ListenerImpl(Target* aTarget, const Function& aFunction)
+    : mHelper(ListenerBase::Token(), aTarget, aFunction) {}
+  void Dispatch(ArgType&& aEvent) override {
+    mHelper.Dispatch(Move(aEvent));
+  }
+private:
+  ListenerHelper<Target, Function> mHelper;
+};
+
+/**
+ * Select EventPassMode based on ListenerMode and if the type is copyable.
+ *
+ * @Copy Selected when ListenerMode is NonExclusive because each listener
+ * must get a copy.
+ *
+ * @Move Selected when ListenerMode is Exclusive and the type is move-only.
+ *
+ * @Both Selected when ListenerMode is Exclusive and the type is copyable.
+ * The data will be moved when possible and copied when necessary.
+ */
+template <typename ArgType, ListenerMode Mode>
+struct PassModePicker {
+  // TODO: pick EventPassMode::Both when we can detect if a type is
+  // copy-constructible to allow copy-only types in Exclusive mode.
+  static const EventPassMode Value =
+    Mode == ListenerMode::NonExclusive ?
+    EventPassMode::Copy : EventPassMode::Move;
+};
+
 } // namespace detail
 
-template <typename T> class MediaEventSource;
+template <typename T, ListenerMode> class MediaEventSource;
 
 /**
  * Not thread-safe since this is not meant to be shared and therefore only
@@ -119,7 +318,7 @@ template <typename T> class MediaEventSource;
  * listener from an event source.
  */
 class MediaEventListener {
-  template <typename T>
+  template <typename T, ListenerMode>
   friend class MediaEventSource;
 
 public:
@@ -153,98 +352,25 @@ private:
 /**
  * A generic and thread-safe class to implement the observer pattern.
  */
-template <typename EventType>
+template <typename EventType, ListenerMode Mode = ListenerMode::NonExclusive>
 class MediaEventSource {
   static_assert(!IsReference<EventType>::value, "Ref-type not supported!");
   typedef typename detail::EventTypeTraits<EventType>::ArgType ArgType;
+  static const detail::EventPassMode PassMode
+    = detail::PassModePicker<ArgType, Mode>::Value;
+  typedef detail::Listener<ArgType, PassMode> Listener;
 
-  /**
-   * Stored by MediaEventSource to send notifications to the listener.
-   */
-  class Listener {
-  public:
-    Listener() : mToken(new RevocableToken()) {}
+  template<typename Target, typename Func>
+  using ListenerImpl = detail::ListenerImpl<Target, Func, ArgType, PassMode>;
 
-    virtual ~Listener() {
-      MOZ_ASSERT(Token()->IsRevoked(), "Must disconnect the listener.");
-    }
-
-    virtual void Dispatch(const ArgType& aEvent) = 0;
-
-    RevocableToken* Token() const {
-      return mToken;
-    }
-
-  private:
-    const nsRefPtr<RevocableToken> mToken;
-  };
-
-  /**
-   * Store the registered target thread and function so it knows where and to
-   * whom to send the event data.
-   */
-  template<typename Target, typename Function>
-  class ListenerImpl : public Listener {
-  public:
-    explicit ListenerImpl(Target* aTarget, const Function& aFunction)
-      : mTarget(aTarget), mFunction(aFunction) {}
-
-    // |Function| takes one argument.
-    void Dispatch(const ArgType& aEvent, TrueType) {
-      // Define our custom runnable to minimize copy of the event data.
-      // NS_NewRunnableFunction will result in 2 copies of the event data.
-      // One is captured by the lambda and the other is the copy of the lambda.
-      class R : public nsRunnable {
-      public:
-        R(RevocableToken* aToken,
-          const Function& aFunction, const ArgType& aEvent)
-          : mToken(aToken), mFunction(aFunction), mEvent(aEvent) {}
-
-        NS_IMETHOD Run() override {
-          // Don't call the listener if it is disconnected.
-          if (!mToken->IsRevoked()) {
-            // Enable move whenever possible since mEvent won't be used anymore.
-            mFunction(Move(mEvent));
-          }
-          return NS_OK;
-        }
-
-      private:
-        nsRefPtr<RevocableToken> mToken;
-        Function mFunction;
-        ArgType mEvent;
-      };
-
-      nsCOMPtr<nsIRunnable> r = new R(this->Token(), mFunction, aEvent);
-      detail::EventTarget<Target>::Dispatch(mTarget.get(), r.forget());
-    }
-
-    // |Function| takes no arguments. Don't bother passing aEvent.
-    void Dispatch(const ArgType& aEvent, FalseType) {
-      nsRefPtr<RevocableToken> token = this->Token();
-      const Function& function = mFunction;
-      nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([=] () {
-        // Don't call the listener if it is disconnected.
-        if (!token->IsRevoked()) {
-          function();
-        }
-      });
-      detail::EventTarget<Target>::Dispatch(mTarget.get(), r.forget());
-    }
-
-    void Dispatch(const ArgType& aEvent) override {
-      Dispatch(aEvent, typename detail::TakeArgs<Function>::Type());
-    }
-
-  private:
-    const nsRefPtr<Target> mTarget;
-    Function mFunction;
-  };
+  template <typename Method>
+  using TakeArgs = detail::TakeArgs<Method>;
 
   template<typename Target, typename Function>
   MediaEventListener
   ConnectInternal(Target* aTarget, const Function& aFunction) {
     MutexAutoLock lock(mMutex);
+    MOZ_ASSERT(Mode == ListenerMode::NonExclusive || mListeners.IsEmpty());
     auto l = mListeners.AppendElement();
     l->reset(new ListenerImpl<Target, Function>(aTarget, aFunction));
     return MediaEventListener((*l)->Token());
@@ -252,8 +378,8 @@ class MediaEventSource {
 
   // |Method| takes one argument.
   template <typename Target, typename This, typename Method>
-  MediaEventListener
-  ConnectInternal(Target* aTarget, This* aThis, Method aMethod, TrueType) {
+  typename EnableIf<TakeArgs<Method>::value, MediaEventListener>::Type
+  ConnectInternal(Target* aTarget, This* aThis, Method aMethod) {
     detail::RawPtr<This> thiz(aThis);
     auto f = [=] (ArgType&& aEvent) {
       (thiz.get()->*aMethod)(Move(aEvent));
@@ -263,8 +389,8 @@ class MediaEventSource {
 
   // |Method| takes no arguments. Don't bother passing the event data.
   template <typename Target, typename This, typename Method>
-  MediaEventListener
-  ConnectInternal(Target* aTarget, This* aThis, Method aMethod, FalseType) {
+  typename EnableIf<!TakeArgs<Method>::value, MediaEventListener>::Type
+  ConnectInternal(Target* aTarget, This* aThis, Method aMethod) {
     detail::RawPtr<This> thiz(aThis);
     auto f = [=] () {
       (thiz.get()->*aMethod)();
@@ -307,21 +433,20 @@ public:
   template <typename This, typename Method>
   MediaEventListener
   Connect(AbstractThread* aTarget, This* aThis, Method aMethod) {
-    return ConnectInternal(aTarget, aThis, aMethod,
-      typename detail::TakeArgs<Method>::Type());
+    return ConnectInternal(aTarget, aThis, aMethod);
   }
 
   template <typename This, typename Method>
   MediaEventListener
   Connect(nsIEventTarget* aTarget, This* aThis, Method aMethod) {
-    return ConnectInternal(aTarget, aThis, aMethod,
-      typename detail::TakeArgs<Method>::Type());
+    return ConnectInternal(aTarget, aThis, aMethod);
   }
 
 protected:
   MediaEventSource() : mMutex("MediaEventSource::mMutex") {}
 
-  void NotifyInternal(const ArgType& aEvent) {
+  template <typename T>
+  void NotifyInternal(T&& aEvent) {
     MutexAutoLock lock(mMutex);
     for (int32_t i = mListeners.Length() - 1; i >= 0; --i) {
       auto&& l = mListeners[i];
@@ -331,7 +456,7 @@ protected:
         mListeners.RemoveElementAt(i);
         continue;
       }
-      l->Dispatch(aEvent);
+      l->Dispatch(Forward<T>(aEvent));
     }
   }
 
@@ -345,11 +470,12 @@ private:
  * and event publisher. Mostly used as a member variable to publish events
  * to the listeners.
  */
-template <typename EventType>
-class MediaEventProducer : public MediaEventSource<EventType> {
+template <typename EventType, ListenerMode Mode = ListenerMode::NonExclusive>
+class MediaEventProducer : public MediaEventSource<EventType, Mode> {
 public:
-  void Notify(const EventType& aEvent) {
-    this->NotifyInternal(aEvent);
+  template <typename T>
+  void Notify(T&& aEvent) {
+    this->NotifyInternal(Forward<T>(aEvent));
   }
 };
 

dom/media/MediaFormatReader.cpp

@@ -76,6 +76,7 @@ MediaFormatReader::MediaFormatReader(AbstractMediaDecoder* aDecoder,
   , mSeekable(false)
   , mIsEncrypted(false)
   , mTrackDemuxersMayBlock(false)
+  , mHardwareAccelerationDisabled(false)
 {
   MOZ_ASSERT(aDemuxer);
   MOZ_COUNT_CTOR(MediaFormatReader);
@@ -139,17 +140,6 @@ MediaFormatReader::Shutdown()
 
   mDemuxer = nullptr;
 
-  // shutdown main thread demuxer and track demuxers.
-  if (mAudioTrackDemuxer) {
-    mAudioTrackDemuxer->BreakCycles();
-    mAudioTrackDemuxer = nullptr;
-  }
-  if (mVideoTrackDemuxer) {
-    mVideoTrackDemuxer->BreakCycles();
-    mVideoTrackDemuxer = nullptr;
-  }
-  mMainThreadDemuxer = nullptr;
-
   mPlatform = nullptr;
 
   return MediaDecoderReader::Shutdown();
@@ -293,32 +283,10 @@ MediaFormatReader::OnDemuxerInitDone(nsresult)
 
   mSeekable = mDemuxer->IsSeekable();
 
-  // Create demuxer object for main thread.
-  if (mDemuxer->IsThreadSafe()) {
-    mMainThreadDemuxer = mDemuxer;
-  } else {
-    mMainThreadDemuxer = mDemuxer->Clone();
-  }
-  if (!mMainThreadDemuxer) {
-    mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
-    NS_WARNING("Unable to clone current MediaDataDemuxer");
-    return;
-  }
-
   if (!videoActive && !audioActive) {
     mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
     return;
   }
-  if (videoActive) {
-    mVideoTrackDemuxer =
-      mMainThreadDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
-    MOZ_ASSERT(mVideoTrackDemuxer);
-  }
-  if (audioActive) {
-    mAudioTrackDemuxer =
-      mMainThreadDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
-    MOZ_ASSERT(mAudioTrackDemuxer);
-  }
 
   if (IsWaitingOnCDMResource()) {
     // Decoder can't be allocated before CDM resource is ready, so resolving the
@@ -344,11 +312,7 @@ void
 MediaFormatReader::OnDemuxerInitFailed(DemuxerFailureReason aFailure)
 {
   mDemuxerInitRequest.Complete();
-  if (aFailure == DemuxerFailureReason::WAITING_FOR_DATA) {
-    mMetadataPromise.Reject(ReadMetadataFailureReason::WAITING_FOR_RESOURCES, __func__);
-  } else {
-    mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
-  }
+  mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__);
 }
 
 bool
@@ -387,8 +351,11 @@ MediaFormatReader::EnsureDecodersCreated()
                    false);
 
     mVideo.mDecoderInitialized = false;
+    // If we've disabled hardware acceleration for this reader, then we can't use
+    // the shared decoder.
     if (mSharedDecoderManager &&
-        mPlatform->SupportsSharedDecoders(mInfo.mVideo)) {
+        mPlatform->SupportsSharedDecoders(mInfo.mVideo) &&
+        !mHardwareAccelerationDisabled) {
       mVideo.mDecoder =
         mSharedDecoderManager->CreateVideoDecoder(mPlatform,
                                                   mVideo.mInfo ?
@@ -399,13 +366,16 @@ MediaFormatReader::EnsureDecodersCreated()
                                                   mVideo.mTaskQueue,
                                                   mVideo.mCallback);
     } else {
+      // Decoders use the layers backend to decide if they can use hardware decoding,
+      // so specify LAYERS_NONE if we want to forcibly disable it.
       mVideo.mDecoder =
         mPlatform->CreateDecoder(mVideo.mInfo ?
                                    *mVideo.mInfo->GetAsVideoInfo() :
                                    mInfo.mVideo,
                                  mVideo.mTaskQueue,
                                  mVideo.mCallback,
-                                 mLayersBackendType,
+                                 mHardwareAccelerationDisabled ? LayersBackend::LAYERS_NONE :
+                                                                 mLayersBackendType,
                                  mDecoder->GetImageContainer());
     }
     NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, false);
@@ -510,8 +480,8 @@ void
 MediaFormatReader::DisableHardwareAcceleration()
 {
   MOZ_ASSERT(OnTaskQueue());
-  if (HasVideo()) {
-    mPlatform->DisableHardwareAcceleration();
+  if (HasVideo() && !mHardwareAccelerationDisabled) {
+    mHardwareAccelerationDisabled = true;
     Flush(TrackInfo::kVideoTrack);
     mVideo.mDecoder->Shutdown();
     mVideo.mDecoder = nullptr;
@@ -538,8 +508,7 @@ MediaFormatReader::ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThr
 
 nsRefPtr<MediaDecoderReader::VideoDataPromise>
 MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe,
-                                    int64_t aTimeThreshold,
-                                    bool aForceDecodeAhead)
+                                     int64_t aTimeThreshold)
 {
   MOZ_ASSERT(OnTaskQueue());
   MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking");
@@ -565,7 +534,6 @@ MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe,
     return VideoDataPromise::CreateAndReject(CANCELED, __func__);
   }
 
-  mVideo.mForceDecodeAhead = aForceDecodeAhead;
   media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)};
   if (ShouldSkip(aSkipToNextKeyframe, timeThreshold)) {
     Flush(TrackInfo::kVideoTrack);
@@ -764,12 +732,11 @@ MediaFormatReader::NeedInput(DecoderData& aDecoder)
   return
     !aDecoder.mDraining &&
     !aDecoder.mError &&
-    (aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) &&
+    aDecoder.HasPromise() &&
     !aDecoder.mDemuxRequest.Exists() &&
     aDecoder.mOutput.IsEmpty() &&
     (aDecoder.mInputExhausted || !aDecoder.mQueuedSamples.IsEmpty() ||
      aDecoder.mTimeThreshold.isSome() ||
-     aDecoder.mForceDecodeAhead ||
      aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead);
 }
 
@@ -915,7 +882,7 @@ MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack,
       decoder.mDecoder->Shutdown();
       decoder.mDecoder = nullptr;
       if (sample->mKeyframe) {
-        decoder.mQueuedSamples.MoveElementsFrom(samples);
+        decoder.mQueuedSamples.AppendElements(Move(samples));
         ScheduleUpdate(aTrack);
       } else {
         MOZ_ASSERT(decoder.mTimeThreshold.isNothing());
@@ -1438,22 +1405,6 @@ MediaFormatReader::OnAudioSeekCompleted(media::TimeUnit aTime)
   mSeekPromise.Resolve(aTime.ToMicroseconds(), __func__);
 }
 
-int64_t
-MediaFormatReader::GetEvictionOffset(double aTime)
-{
-  int64_t audioOffset;
-  int64_t videoOffset;
-  if (NS_IsMainThread()) {
-    audioOffset = HasAudio() ? mAudioTrackDemuxer->GetEvictionOffset(media::TimeUnit::FromSeconds(aTime)) : INT64_MAX;
-    videoOffset = HasVideo() ? mVideoTrackDemuxer->GetEvictionOffset(media::TimeUnit::FromSeconds(aTime)) : INT64_MAX;
-  } else {
-    MOZ_ASSERT(OnTaskQueue());
-    audioOffset = HasAudio() ? mAudio.mTrackDemuxer->GetEvictionOffset(media::TimeUnit::FromSeconds(aTime)) : INT64_MAX;
-    videoOffset = HasVideo() ? mVideo.mTrackDemuxer->GetEvictionOffset(media::TimeUnit::FromSeconds(aTime)) : INT64_MAX;
-  }
-  return std::min(audioOffset, videoOffset);
-}
-
 media::TimeIntervals
 MediaFormatReader::GetBuffered()
 {
@@ -1538,15 +1489,18 @@ MediaFormatReader::NotifyDemuxer(uint32_t aLength, int64_t aOffset)
   MOZ_ASSERT(OnTaskQueue());
 
   LOGV("aLength=%u, aOffset=%lld", aLength, aOffset);
-  if (mShutdown) {
+  if (mShutdown || !mDemuxer) {
     return;
   }
 
   if (aLength || aOffset) {
-    mDemuxer->NotifyDataArrived(aLength, aOffset);
+    mDemuxer->NotifyDataArrived();
   } else {
     mDemuxer->NotifyDataRemoved();
   }
+  if (!mInitDone) {
+    return;
+  }
   if (HasVideo()) {
     mVideo.mReceivedNewData = true;
     ScheduleUpdate(TrackType::kVideoTrack);
@@ -1563,19 +1517,6 @@ MediaFormatReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset)
   MOZ_ASSERT(OnTaskQueue());
   MOZ_ASSERT(aLength);
 
-  if (!mInitDone || mShutdown) {
-    return;
-  }
-
-  MOZ_ASSERT(mMainThreadDemuxer);
-
-  // Queue a task to notify our main thread demuxer.
-  nsCOMPtr<nsIRunnable> task =
-    NS_NewRunnableMethodWithArgs<uint32_t, int64_t>(
-      mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataArrived,
-      aLength, aOffset);
-  AbstractThread::MainThread()->Dispatch(task.forget());
-
   NotifyDemuxer(aLength, aOffset);
 }
 
@@ -1584,18 +1525,6 @@ MediaFormatReader::NotifyDataRemoved()
 {
   MOZ_ASSERT(OnTaskQueue());
 
-  if (!mInitDone || mShutdown) {
-    return;
-  }
-
-  MOZ_ASSERT(mMainThreadDemuxer);
-
-  // Queue a task to notify our main thread demuxer.
-  nsCOMPtr<nsIRunnable> task =
-    NS_NewRunnableMethod(
-      mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataRemoved);
-  AbstractThread::MainThread()->Dispatch(task.forget());
-
   NotifyDemuxer(0, 0);
 }
 

dom/media/MediaFormatReader.h

@@ -35,7 +35,7 @@ public:
   size_t SizeOfAudioQueueInFrames() override;
 
   nsRefPtr<VideoDataPromise>
-  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override;
+  RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override;
 
   nsRefPtr<AudioDataPromise> RequestAudioData() override;
 
@@ -61,7 +61,6 @@ public:
     return mSeekable;
   }
 
-  int64_t GetEvictionOffset(double aTime) override;
 protected:
   void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override;
 public:
@@ -176,6 +175,9 @@ private:
     void ReleaseMediaResources() override {
       mReader->ReleaseMediaResources();
     }
+    bool OnReaderTaskQueue() override {
+      return mReader->OnTaskQueue();
+    }
   private:
     MediaFormatReader* mReader;
     TrackType mType;
@@ -188,7 +190,6 @@ private:
       : mOwner(aOwner)
       , mType(aType)
       , mDecodeAhead(aDecodeAhead)
-      , mForceDecodeAhead(false)
       , mUpdateScheduled(false)
       , mDemuxEOS(false)
       , mWaitingForData(false)
@@ -222,7 +223,6 @@ private:
 
     // Only accessed from reader's task queue.
     uint32_t mDecodeAhead;
-    bool mForceDecodeAhead;
     bool mUpdateScheduled;
     bool mDemuxEOS;
     bool mWaitingForData;
@@ -277,7 +277,6 @@ private:
     void ResetState()
     {
       MOZ_ASSERT(mOwner->OnTaskQueue());
-      mForceDecodeAhead = false;
       mDemuxEOS = false;
       mWaitingForData = false;
       mReceivedNewData = false;
@@ -395,6 +394,8 @@ private:
   // Set to true if any of our track buffers may be blocking.
   bool mTrackDemuxersMayBlock;
 
+  bool mHardwareAccelerationDisabled;
+
   // Seeking objects.
   bool IsSeeking() const { return mPendingSeekTime.isSome(); }
   void AttemptSeek();
@@ -421,13 +422,6 @@ private:
 
   nsRefPtr<SharedDecoderManager> mSharedDecoderManager;
 
-  // Main thread objects
-  // Those are only used to calculate our buffered range on the main thread.
-  // The cached buffered range is calculated one when required.
-  nsRefPtr<MediaDataDemuxer> mMainThreadDemuxer;
-  nsRefPtr<MediaTrackDemuxer> mAudioTrackDemuxer;
-  nsRefPtr<MediaTrackDemuxer> mVideoTrackDemuxer;
-
 #if defined(READER_DORMANT_HEURISTIC)
   const bool mDormantEnabled;
 #endif

dom/media/MediaManager.cpp

@@ -126,8 +126,6 @@ using dom::Sequence;
 using dom::OwningBooleanOrMediaTrackConstraints;
 using dom::OwningStringOrStringSequence;
 using dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters;
-using dom::SupportedAudioConstraints;
-using dom::SupportedVideoConstraints;
 using media::Pledge;
 using media::NewRunnableFrom;
 using media::NewTaskFrom;
@@ -486,12 +484,13 @@ public:
   CreateTrackUnionStream(nsIDOMWindow* aWindow,
                          GetUserMediaCallbackMediaStreamListener* aListener,
                          MediaEngineSource* aAudioSource,
-                         MediaEngineSource* aVideoSource)
+                         MediaEngineSource* aVideoSource,
+                         MediaStreamGraph* aMSG)
   {
     nsRefPtr<nsDOMUserMediaStream> stream = new nsDOMUserMediaStream(aListener,
                                                                      aAudioSource,
                                                                      aVideoSource);
-    stream->InitTrackUnionStream(aWindow);
+    stream->InitTrackUnionStream(aWindow, aMSG);
     return stream.forget();
   }
 
@@ -790,7 +789,13 @@ public:
     }
 #endif
 
-    MediaStreamGraph* msg = MediaStreamGraph::GetInstance();
+    MediaStreamGraph::GraphDriverType graphDriverType =
+      mAudioSource ? MediaStreamGraph::AUDIO_THREAD_DRIVER
+                   : MediaStreamGraph::SYSTEM_THREAD_DRIVER;
+    MediaStreamGraph* msg =
+      MediaStreamGraph::GetInstance(graphDriverType,
+                                    dom::AudioChannel::Normal);
+
     nsRefPtr<SourceMediaStream> stream = msg->CreateSourceStream(nullptr);
 
     nsRefPtr<DOMLocalMediaStream> domStream;
@@ -800,7 +805,7 @@ public:
     // them down instead.
     if (mAudioSource &&
         mAudioSource->GetMediaSource() == dom::MediaSourceEnum::AudioCapture) {
-      domStream = DOMLocalMediaStream::CreateAudioCaptureStream(window);
+      domStream = DOMLocalMediaStream::CreateAudioCaptureStream(window, msg);
       // It should be possible to pipe the capture stream to anything. CORS is
       // not a problem here, we got explicit user content.
       domStream->SetPrincipal(window->GetExtantDoc()->NodePrincipal());
@@ -812,7 +817,8 @@ public:
       // avoid us blocking
       nsRefPtr<nsDOMUserMediaStream> trackunion =
         nsDOMUserMediaStream::CreateTrackUnionStream(window, mListener,
-                                                     mAudioSource, mVideoSource);
+                                                     mAudioSource, mVideoSource,
+                                                     msg);
       trackunion->GetStream()->AsProcessedStream()->SetAutofinish(true);
       nsRefPtr<MediaInputPort> port = trackunion->GetStream()->AsProcessedStream()->
         AllocateInputPort(stream, MediaInputPort::FLAG_BLOCK_OUTPUT);
@@ -998,7 +1004,7 @@ ApplyConstraints(const MediaTrackConstraints &aConstraints,
         }
       }
       if (!aSources.Length()) {
-        aSources.MoveElementsFrom(rejects);
+        aSources.AppendElements(Move(rejects));
         aggregateConstraints.RemoveElementAt(aggregateConstraints.Length() - 1);
       }
     }
@@ -1376,6 +1382,8 @@ MediaManager::IsInMediaThread()
 // NOTE: never Dispatch(....,NS_DISPATCH_SYNC) to the MediaManager
 // thread from the MainThread, as we NS_DISPATCH_SYNC to MainThread
 // from MediaManager thread.
+
+// Guaranteed never to return nullptr.
 /* static */  MediaManager*
 MediaManager::Get() {
   if (!sSingleton) {
@@ -1403,6 +1411,7 @@ MediaManager::Get() {
     nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
     if (obs) {
       obs->AddObserver(sSingleton, "xpcom-will-shutdown", false);
+      obs->AddObserver(sSingleton, "last-pb-context-exited", false);
       obs->AddObserver(sSingleton, "getUserMedia:privileged:allow", false);
       obs->AddObserver(sSingleton, "getUserMedia:response:allow", false);
       obs->AddObserver(sSingleton, "getUserMedia:response:deny", false);
@@ -2032,7 +2041,14 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow,
   nsCOMPtr<nsIDOMGetUserMediaErrorCallback> onFailure(aOnFailure);
   uint64_t windowId = aWindow->WindowID();
 
-  AddWindowID(windowId);
+  StreamListeners* listeners = AddWindowID(windowId);
+
+  // Create a disabled listener to act as a placeholder
+  nsRefPtr<GetUserMediaCallbackMediaStreamListener> listener =
+    new GetUserMediaCallbackMediaStreamListener(mMediaThread, windowId);
+
+  // No need for locking because we always do this in the main thread.
+  listeners->AppendElement(listener);
 
   bool fake = Preferences::GetBool("media.navigator.streams.fake");
 
@@ -2040,11 +2056,15 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow,
                                                      dom::MediaSourceEnum::Camera,
                                                      dom::MediaSourceEnum::Microphone,
                                                      fake);
-  p->Then([onSuccess](SourceSet*& aDevices) mutable {
+  p->Then([onSuccess, windowId, listener](SourceSet*& aDevices) mutable {
     ScopedDeletePtr<SourceSet> devices(aDevices); // grab result
+    nsRefPtr<MediaManager> mgr = MediaManager_GetInstance();
+    mgr->RemoveFromWindowList(windowId, listener);
     nsCOMPtr<nsIWritableVariant> array = MediaManager_ToJSArray(*devices);
     onSuccess->OnSuccess(array);
-  }, [onFailure](MediaStreamError& reason) mutable {
+  }, [onFailure, windowId, listener](MediaStreamError& reason) mutable {
+    nsRefPtr<MediaManager> mgr = MediaManager_GetInstance();
+    mgr->RemoveFromWindowList(windowId, listener);
     onFailure->OnError(&reason);
   });
   return NS_OK;
@@ -2274,6 +2294,7 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic,
     sInShutdown = true;
 
     obs->RemoveObserver(this, "xpcom-will-shutdown");
+    obs->RemoveObserver(this, "last-pb-context-exited");
     obs->RemoveObserver(this, "getUserMedia:privileged:allow");
     obs->RemoveObserver(this, "getUserMedia:response:allow");
     obs->RemoveObserver(this, "getUserMedia:response:deny");
@@ -2356,6 +2377,10 @@ MediaManager::Observe(nsISupports* aSubject, const char* aTopic,
     })));
     return NS_OK;
 
+  } else if (!strcmp(aTopic, "last-pb-context-exited")) {
+    // Clear memory of private-browsing-specific deviceIds. Fire and forget.
+    media::SanitizeOriginKeys(0, true);
+    return NS_OK;
   } else if (!strcmp(aTopic, "getUserMedia:privileged:allow") ||
              !strcmp(aTopic, "getUserMedia:response:allow")) {
     nsString key(aData);
@@ -2605,7 +2630,7 @@ MediaManager::SanitizeDeviceIds(int64_t aSinceWhen)
   NS_ASSERTION(NS_IsMainThread(), "Only call on main thread");
   LOG(("%s: sinceWhen = %llu", __FUNCTION__, aSinceWhen));
 
-  media::SanitizeOriginKeys(aSinceWhen); // we fire and forget
+  media::SanitizeOriginKeys(aSinceWhen, false); // we fire and forget
   return NS_OK;
 }
 
@@ -2798,7 +2823,10 @@ GetUserMediaCallbackMediaStreamListener::StopSharing()
     nsCOMPtr<nsPIDOMWindow> window = nsGlobalWindow::GetInnerWindowWithId(mWindowID);
     MOZ_ASSERT(window);
     window->SetAudioCapture(false);
-    MediaStreamGraph::GetInstance()->UnregisterCaptureStreamForWindow(mWindowID);
+    MediaStreamGraph* graph =
+      MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER,
+                                    dom::AudioChannel::Normal);
+    graph->UnregisterCaptureStreamForWindow(mWindowID);
     mStream->Destroy();
   }
 }

dom/media/MediaMetadataManager.h

@@ -6,63 +6,101 @@
 
 #if !defined(MediaMetadataManager_h__)
 #define MediaMetadataManager_h__
-#include "VideoUtils.h"
+
+#include "mozilla/AbstractThread.h"
 #include "mozilla/LinkedList.h"
-#include "AbstractMediaDecoder.h"
+
 #include "nsAutoPtr.h"
+#include "AbstractMediaDecoder.h"
+#include "MediaEventSource.h"
+#include "TimeUnits.h"
+#include "VideoUtils.h"
 
 namespace mozilla {
 
-  // A struct that contains the metadata of a media, and the time at which those
-  // metadata should start to be reported.
-  class TimedMetadata : public LinkedListElement<TimedMetadata> {
-    public:
-      // The time, in microseconds, at which those metadata should be available.
-      int64_t mPublishTime;
-      // The metadata. The ownership is transfered to the element when dispatching to
-      // the main threads.
-      nsAutoPtr<MetadataTags> mTags;
-      // The media info, including the info of audio tracks and video tracks.
-      // The ownership is transfered to MediaDecoder when dispatching to the
-      // main thread.
-      nsAutoPtr<MediaInfo> mInfo;
-  };
+class TimedMetadata;
+typedef MediaEventProducer<TimedMetadata, ListenerMode::Exclusive>
+        TimedMetadataEventProducer;
+typedef MediaEventSource<TimedMetadata, ListenerMode::Exclusive>
+        TimedMetadataEventSource;
 
-  // This class encapsulate the logic to give the metadata from the reader to
-  // the content, at the right time.
-  class MediaMetadataManager
-  {
-    public:
-      ~MediaMetadataManager() {
-        TimedMetadata* element;
-        while((element = mMetadataQueue.popFirst()) != nullptr) {
-          delete element;
-        }
-      }
-      void QueueMetadata(TimedMetadata* aMetadata) {
-        mMetadataQueue.insertBack(aMetadata);
-      }
+// A struct that contains the metadata of a media, and the time at which those
+// metadata should start to be reported.
+class TimedMetadata : public LinkedListElement<TimedMetadata> {
+public:
+  TimedMetadata(const media::TimeUnit& aPublishTime,
+                nsAutoPtr<MetadataTags>&& aTags,
+                nsAutoPtr<MediaInfo>&& aInfo)
+    : mPublishTime(aPublishTime)
+    , mTags(Move(aTags))
+    , mInfo(Move(aInfo)) {}
 
-      void DispatchMetadataIfNeeded(AbstractMediaDecoder* aDecoder, double aCurrentTime) {
-        TimedMetadata* metadata = mMetadataQueue.getFirst();
-        while (metadata && aCurrentTime >= static_cast<double>(metadata->mPublishTime) / USECS_PER_S) {
-          // Remove all media tracks from the list first.
-          nsCOMPtr<nsIRunnable> removeTracksEvent =
-            new RemoveMediaTracksEventRunner(aDecoder);
-          NS_DispatchToMainThread(removeTracksEvent);
+  // Define our move constructor because we don't want to move the members of
+  // LinkedListElement to change the list.
+  TimedMetadata(TimedMetadata&& aOther)
+    : mPublishTime(aOther.mPublishTime)
+    , mTags(Move(aOther.mTags))
+    , mInfo(Move(aOther.mInfo)) {}
+
+  // The time, in microseconds, at which those metadata should be available.
+  media::TimeUnit mPublishTime;
+  // The metadata. The ownership is transfered to the element when dispatching to
+  // the main threads.
+  nsAutoPtr<MetadataTags> mTags;
+  // The media info, including the info of audio tracks and video tracks.
+  // The ownership is transfered to MediaDecoder when dispatching to the
+  // main thread.
+  nsAutoPtr<MediaInfo> mInfo;
+};
+
+// This class encapsulate the logic to give the metadata from the reader to
+// the content, at the right time.
+class MediaMetadataManager {
+public:
+  ~MediaMetadataManager() {
+    TimedMetadata* element;
+    while((element = mMetadataQueue.popFirst()) != nullptr) {
+      delete element;
+    }
+  }
+
+  // Connect to an event source to receive TimedMetadata events.
+  void Connect(TimedMetadataEventSource& aEvent, AbstractThread* aThread) {
+    mListener = aEvent.Connect(
+      aThread, this, &MediaMetadataManager::OnMetadataQueued);
+  }
+
+  // Stop receiving TimedMetadata events.
+  void Disconnect() {
+    mListener.Disconnect();
+  }
+
+  // Return an event source through which we will send TimedMetadata events
+  // when playback position reaches the publish time.
+  TimedMetadataEventSource& TimedMetadataEvent() {
+    return mTimedMetadataEvent;
+  }
+
+  void DispatchMetadataIfNeeded(const media::TimeUnit& aCurrentTime) {
+    TimedMetadata* metadata = mMetadataQueue.getFirst();
+    while (metadata && aCurrentTime >= metadata->mPublishTime) {
+      // Our listener will figure out what to do with TimedMetadata.
+      mTimedMetadataEvent.Notify(Move(*metadata));
+      delete mMetadataQueue.popFirst();
+      metadata = mMetadataQueue.getFirst();
+    }
+  }
+
+protected:
+  void OnMetadataQueued(TimedMetadata&& aMetadata) {
+    mMetadataQueue.insertBack(new TimedMetadata(Move(aMetadata)));
+  }
+
+  LinkedList<TimedMetadata> mMetadataQueue;
+  MediaEventListener mListener;
+  TimedMetadataEventProducer mTimedMetadataEvent;
+};
 
-          nsCOMPtr<nsIRunnable> metadataUpdatedEvent =
-            new MetadataUpdatedEventRunner(aDecoder,
-                                           metadata->mInfo,
-                                           metadata->mTags);
-          NS_DispatchToMainThread(metadataUpdatedEvent);
-          delete mMetadataQueue.popFirst();
-          metadata = mMetadataQueue.getFirst();
-        }
-      }
-    protected:
-      LinkedList<TimedMetadata> mMetadataQueue;
-  };
 } // namespace mozilla
 
 #endif

dom/media/MediaQueue.h

@@ -8,10 +8,9 @@
 
 #include "mozilla/ReentrantMonitor.h"
 #include "mozilla/TaskQueue.h"
-#include "mozilla/UniquePtr.h"
 
 #include "nsDeque.h"
-#include "nsTArray.h"
+#include "MediaEventSource.h"
 
 namespace mozilla {
 
@@ -26,36 +25,6 @@ class MediaQueueDeallocator : public nsDequeFunctor {
 
 template <class T>
 class MediaQueue : private nsDeque {
-  struct Listener {
-    virtual ~Listener() {}
-    virtual void Dispatch(T* aItem) = 0;
-  };
-
-  template<typename Function>
-  class PopListener : public Listener {
-  public:
-    explicit PopListener(const Function& aFunction, TaskQueue* aTarget)
-      : mFunction(aFunction), mTarget(aTarget) {}
-
-    void Dispatch(T* aItem) override {
-      nsRefPtr<T> item = aItem;
-      Function function = mFunction;
-      nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([=] () {
-        function(item);
-      });
-      mTarget->Dispatch(r.forget());
-    }
-  private:
-    Function mFunction;
-    nsRefPtr<TaskQueue> mTarget;
-  };
-
-  void NotifyPopListeners(T* aItem) {
-    for (auto&& l : mPopListeners) {
-      l->Dispatch(aItem);
-    }
-  }
-
 public:
   MediaQueue()
     : nsDeque(new MediaQueueDeallocator<T>()),
@@ -77,6 +46,7 @@ public:
     MOZ_ASSERT(aItem);
     NS_ADDREF(aItem);
     nsDeque::Push(aItem);
+    mPushEvent.Notify();
   }
 
   inline void PushFront(T* aItem) {
@@ -84,13 +54,14 @@ public:
     MOZ_ASSERT(aItem);
     NS_ADDREF(aItem);
     nsDeque::PushFront(aItem);
+    mPushEvent.Notify();
   }
 
   inline already_AddRefed<T> PopFront() {
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
     nsRefPtr<T> rv = dont_AddRef(static_cast<T*>(nsDeque::PopFront()));
     if (rv) {
-      NotifyPopListeners(rv);
+      mPopEvent.Notify(rv);
     }
     return rv.forget();
   }
@@ -135,6 +106,7 @@ public:
   void Finish() {
     ReentrantMonitorAutoEnter mon(mReentrantMonitor);
     mEndOfStream = true;
+    mFinishEvent.Notify();
   }
 
   // Returns the approximate number of microseconds of items in the queue.
@@ -190,21 +162,23 @@ public:
     return frames;
   }
 
-  void ClearListeners() {
-    ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-    mPopListeners.Clear();
+  MediaEventSource<nsRefPtr<T>>& PopEvent() {
+    return mPopEvent;
   }
 
-  template<typename Function>
-  void AddPopListener(const Function& aFunction, TaskQueue* aTarget) {
-    ReentrantMonitorAutoEnter mon(mReentrantMonitor);
-    mPopListeners.AppendElement()->reset(
-      new PopListener<Function>(aFunction, aTarget));
+  MediaEventSource<void>& PushEvent() {
+    return mPushEvent;
+  }
+
+  MediaEventSource<void>& FinishEvent() {
+    return mFinishEvent;
   }
 
 private:
   mutable ReentrantMonitor mReentrantMonitor;
-  nsTArray<UniquePtr<Listener>> mPopListeners;
+  MediaEventProducer<nsRefPtr<T>> mPopEvent;
+  MediaEventProducer<void> mPushEvent;
+  MediaEventProducer<void> mFinishEvent;
   // True when we've decoded the last frame of data in the
   // bitstream for which we're queueing frame data.
   bool mEndOfStream;

dom/media/MediaRecorder.cpp

@@ -770,12 +770,13 @@ MediaRecorder::MediaRecorder(AudioNode& aSrcAudioNode,
   if (aSrcAudioNode.NumberOfOutputs() > 0) {
     AudioContext* ctx = aSrcAudioNode.Context();
     AudioNodeEngine* engine = new AudioNodeEngine(nullptr);
-    mPipeStream = ctx->Graph()->CreateAudioNodeStream(engine,
-                                                      MediaStreamGraph::EXTERNAL_STREAM,
-                                                      ctx->SampleRate());
-    AudioNodeStream* ns = aSrcAudioNode.Stream();
+    AudioNodeStream::Flags flags =
+      AudioNodeStream::EXTERNAL_OUTPUT |
+      AudioNodeStream::NEED_MAIN_THREAD_FINISHED;
+    mPipeStream = AudioNodeStream::Create(ctx->Graph(), engine, flags);
+    AudioNodeStream* ns = aSrcAudioNode.GetStream();
     if (ns) {
-      mInputPort = mPipeStream->AllocateInputPort(aSrcAudioNode.Stream(),
+      mInputPort = mPipeStream->AllocateInputPort(aSrcAudioNode.GetStream(),
                                                   MediaInputPort::FLAG_BLOCK_INPUT,
                                                   0,
                                                   aSrcOutput);
@@ -1151,7 +1152,7 @@ MediaRecorder::GetSourceMediaStream()
     return mDOMStream->GetStream();
   }
   MOZ_ASSERT(mAudioNode != nullptr);
-  return mPipeStream != nullptr ? mPipeStream.get() : mAudioNode->Stream();
+  return mPipeStream ? mPipeStream.get() : mAudioNode->GetStream();
 }
 
 nsIPrincipal*

dom/media/MediaResource.cpp

@@ -718,20 +718,6 @@ nsresult ChannelMediaResource::ReadFromCache(char* aBuffer,
   return mCacheStream.ReadFromCache(aBuffer, aOffset, aCount);
 }
 
-nsresult ChannelMediaResource::Read(char* aBuffer,
-                                    uint32_t aCount,
-                                    uint32_t* aBytes)
-{
-  NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
-
-  int64_t offset = mCacheStream.Tell();
-  nsresult rv = mCacheStream.Read(aBuffer, aCount, aBytes);
-  if (NS_SUCCEEDED(rv)) {
-    DispatchBytesConsumed(*aBytes, offset);
-  }
-  return rv;
-}
-
 nsresult ChannelMediaResource::ReadAt(int64_t aOffset,
                                       char* aBuffer,
                                       uint32_t aCount,
@@ -747,37 +733,30 @@ nsresult ChannelMediaResource::ReadAt(int64_t aOffset,
 }
 
 already_AddRefed<MediaByteBuffer>
-ChannelMediaResource::SilentReadAt(int64_t aOffset, uint32_t aCount)
+ChannelMediaResource::MediaReadAt(int64_t aOffset, uint32_t aCount)
 {
   NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
 
   nsRefPtr<MediaByteBuffer> bytes = new MediaByteBuffer();
-  bool ok = bytes->SetCapacity(aCount, fallible);
+  bool ok = bytes->SetLength(aCount, fallible);
   NS_ENSURE_TRUE(ok, nullptr);
-  int64_t pos = mCacheStream.Tell();
   char* curr = reinterpret_cast<char*>(bytes->Elements());
+  const char* start = curr;
   while (aCount > 0) {
     uint32_t bytesRead;
     nsresult rv = mCacheStream.ReadAt(aOffset, curr, aCount, &bytesRead);
     NS_ENSURE_SUCCESS(rv, nullptr);
-    NS_ENSURE_TRUE(bytesRead > 0, nullptr);
+    if (!bytesRead) {
+      break;
+    }
     aOffset += bytesRead;
     aCount -= bytesRead;
     curr += bytesRead;
   }
-  mCacheStream.Seek(nsISeekableStream::NS_SEEK_SET, pos);
+  bytes->SetLength(curr - start);
   return bytes.forget();
 }
 
-nsresult ChannelMediaResource::Seek(int32_t aWhence, int64_t aOffset)
-{
-  NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
-
-  CMLOG("Seek requested for aOffset [%lld] for decoder [%p]",
-        aOffset, mDecoder);
-  return mCacheStream.Seek(aWhence, aOffset);
-}
-
 int64_t ChannelMediaResource::Tell()
 {
   NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
@@ -1246,11 +1225,9 @@ public:
   // Other thread
   virtual void     SetReadMode(MediaCacheStream::ReadMode aMode) override {}
   virtual void     SetPlaybackRate(uint32_t aBytesPerSecond) override {}
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) override;
   virtual nsresult ReadAt(int64_t aOffset, char* aBuffer,
                           uint32_t aCount, uint32_t* aBytes) override;
-  virtual already_AddRefed<MediaByteBuffer> SilentReadAt(int64_t aOffset, uint32_t aCount) override;
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) override;
+  virtual already_AddRefed<MediaByteBuffer> MediaReadAt(int64_t aOffset, uint32_t aCount) override;
   virtual int64_t  Tell() override;
 
   // Any thread
@@ -1313,6 +1290,8 @@ private:
   // Ensures mSize is initialized, if it can be.
   // mLock must be held when this is called, and mInput must be non-null.
   void EnsureSizeInitialized();
+  already_AddRefed<MediaByteBuffer> UnsafeMediaReadAt(
+                        int64_t aOffset, uint32_t aCount);
 
   // The file size, or -1 if not known. Immutable after Open().
   // Can be used from any thread.
@@ -1534,21 +1513,6 @@ nsresult FileMediaResource::ReadFromCache(char* aBuffer, int64_t aOffset, uint32
   return seekres;
 }
 
-nsresult FileMediaResource::Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes)
-{
-  nsresult rv;
-  int64_t offset = 0;
-  {
-    MutexAutoLock lock(mLock);
-    mSeekable->Tell(&offset);
-    rv = UnsafeRead(aBuffer, aCount, aBytes);
-  }
-  if (NS_SUCCEEDED(rv)) {
-    DispatchBytesConsumed(*aBytes, offset);
-  }
-  return rv;
-}
-
 nsresult FileMediaResource::UnsafeRead(char* aBuffer, uint32_t aCount, uint32_t* aBytes)
 {
   EnsureSizeInitialized();
@@ -1574,41 +1538,38 @@ nsresult FileMediaResource::ReadAt(int64_t aOffset, char* aBuffer,
 }
 
 already_AddRefed<MediaByteBuffer>
-FileMediaResource::SilentReadAt(int64_t aOffset, uint32_t aCount)
+FileMediaResource::MediaReadAt(int64_t aOffset, uint32_t aCount)
 {
   NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
 
   MutexAutoLock lock(mLock);
+  return UnsafeMediaReadAt(aOffset, aCount);
+}
+
+already_AddRefed<MediaByteBuffer>
+FileMediaResource::UnsafeMediaReadAt(int64_t aOffset, uint32_t aCount)
+{
   nsRefPtr<MediaByteBuffer> bytes = new MediaByteBuffer();
-  bool ok = bytes->SetCapacity(aCount, fallible);
+  bool ok = bytes->SetLength(aCount, fallible);
   NS_ENSURE_TRUE(ok, nullptr);
-  int64_t pos = 0;
-  NS_ENSURE_TRUE(mSeekable, nullptr);
-  nsresult rv = mSeekable->Tell(&pos);
-  NS_ENSURE_SUCCESS(rv, nullptr);
-  rv = UnsafeSeek(nsISeekableStream::NS_SEEK_SET, aOffset);
+  nsresult rv = UnsafeSeek(nsISeekableStream::NS_SEEK_SET, aOffset);
   NS_ENSURE_SUCCESS(rv, nullptr);
   char* curr = reinterpret_cast<char*>(bytes->Elements());
+  const char* start = curr;
   while (aCount > 0) {
     uint32_t bytesRead;
     rv = UnsafeRead(curr, aCount, &bytesRead);
     NS_ENSURE_SUCCESS(rv, nullptr);
-    NS_ENSURE_TRUE(bytesRead > 0, nullptr);
+    if (!bytesRead) {
+      break;
+    }
     aCount -= bytesRead;
     curr += bytesRead;
   }
-  UnsafeSeek(nsISeekableStream::NS_SEEK_SET, pos);
+  bytes->SetLength(curr - start);
   return bytes.forget();
 }
 
-nsresult FileMediaResource::Seek(int32_t aWhence, int64_t aOffset)
-{
-  NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");
-
-  MutexAutoLock lock(mLock);
-  return UnsafeSeek(aWhence, aOffset);
-}
-
 nsresult FileMediaResource::UnsafeSeek(int32_t aWhence, int64_t aOffset)
 {
   NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");

dom/media/MediaResource.h

@@ -25,7 +25,11 @@
 // For HTTP seeking, if number of bytes needing to be
 // seeked forward is less than this value then a read is
 // done rather than a byte range request.
-static const int64_t SEEK_VS_READ_THRESHOLD = 32*1024;
+//
+// If we assume a 100Mbit connection, and assume reissuing an HTTP seek causes
+// a delay of 200ms, then in that 200ms we could have simply read ahead 2MB. So
+// setting SEEK_VS_READ_THRESHOLD to 1MB sounds reasonable.
+static const int64_t SEEK_VS_READ_THRESHOLD = 1 * 1024 * 1024;
 
 static const uint32_t HTTP_REQUESTED_RANGE_NOT_SATISFIABLE_CODE = 416;
 
@@ -277,75 +281,42 @@ public:
   // the media plays continuously. The cache can't guess this itself
   // because it doesn't know when the decoder was paused, buffering, etc.
   virtual void SetPlaybackRate(uint32_t aBytesPerSecond) = 0;
-  // Read up to aCount bytes from the stream. The buffer must have
-  // enough room for at least aCount bytes. Stores the number of
-  // actual bytes read in aBytes (0 on end of file).
-  // May read less than aCount bytes if the number of
-  // available bytes is less than aCount. Always check *aBytes after
-  // read, and call again if necessary.
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) = 0;
   // Read up to aCount bytes from the stream. The read starts at
   // aOffset in the stream, seeking to that location initially if
   // it is not the current stream offset. The remaining arguments,
   // results and requirements are the same as per the Read method.
   virtual nsresult ReadAt(int64_t aOffset, char* aBuffer,
                           uint32_t aCount, uint32_t* aBytes) = 0;
-
-  // ReadAt without side-effects. Given that our MediaResource infrastructure
-  // is very side-effecty, this accomplishes its job by checking the initial
-  // position and seeking back to it. If the seek were to fail, a side-effect
-  // might be observable.
-  //
-  // This method returns null if anything fails, including the failure to read
-  // aCount bytes. Otherwise, it returns an owned buffer.
-  virtual already_AddRefed<MediaByteBuffer> SilentReadAt(int64_t aOffset, uint32_t aCount)
+  // This method returns nullptr if anything fails.
+  // Otherwise, it returns an owned buffer.
+  // MediaReadAt may return fewer bytes than requested if end of stream is
+  // encountered. There is no need to call it again to get more data.
+  virtual already_AddRefed<MediaByteBuffer> MediaReadAt(int64_t aOffset, uint32_t aCount)
   {
     nsRefPtr<MediaByteBuffer> bytes = new MediaByteBuffer();
-    bool ok = bytes->SetCapacity(aCount, fallible);
+    bool ok = bytes->SetLength(aCount, fallible);
     NS_ENSURE_TRUE(ok, nullptr);
-    int64_t pos = Tell();
     char* curr = reinterpret_cast<char*>(bytes->Elements());
+    const char* start = curr;
     while (aCount > 0) {
       uint32_t bytesRead;
       nsresult rv = ReadAt(aOffset, curr, aCount, &bytesRead);
       NS_ENSURE_SUCCESS(rv, nullptr);
-      NS_ENSURE_TRUE(bytesRead > 0, nullptr);
+      if (!bytesRead) {
+        break;
+      }
       aOffset += bytesRead;
       aCount -= bytesRead;
       curr += bytesRead;
     }
-    Seek(nsISeekableStream::NS_SEEK_SET, pos);
+    bytes->SetLength(curr - start);
     return bytes.forget();
   }
 
-  // Seek to the given bytes offset in the stream. aWhence can be
-  // one of:
-  //   NS_SEEK_SET
-  //   NS_SEEK_CUR
-  //   NS_SEEK_END
-  //
-  // In the Http strategy case the cancel will cause the http
-  // channel's listener to close the pipe, forcing an i/o error on any
-  // blocked read. This will allow the decode thread to complete the
-  // event.
-  //
-  // In the case of a seek in progress, the byte range request creates
-  // a new listener. This is done on the main thread via seek
-  // synchronously dispatching an event. This avoids the issue of us
-  // closing the listener but an outstanding byte range request
-  // creating a new one. They run on the same thread so no explicit
-  // synchronisation is required. The byte range request checks for
-  // the cancel flag and does not create a new channel or listener if
-  // we are cancelling.
-  //
-  // The default strategy does not do any seeking - the only issue is
-  // a blocked read which it handles by causing the listener to close
-  // the pipe, as per the http case.
-  //
-  // The file strategy doesn't block for any great length of time so
-  // is fine for a no-op cancel.
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) = 0;
   // Report the current offset in bytes from the start of the stream.
+  // This is used to approximate where we currently are in the playback of a
+  // media.
+  // A call to ReadAt will update this position.
   virtual int64_t Tell() = 0;
   // Moves any existing channel loads into or out of background. Background
   // loads don't block the load event. This also determines whether or not any
@@ -380,6 +351,15 @@ public:
   // Returns true if all the data from aOffset to the end of the stream
   // is in cache. If the end of the stream is not known, we return false.
   virtual bool IsDataCachedToEndOfResource(int64_t aOffset) = 0;
+  // Returns true if we are expecting any more data to arrive
+  // sometime in the not-too-distant future, either from the network or from
+  // an appendBuffer call on a MediaSource element.
+  virtual bool IsExpectingMoreData()
+  {
+    // MediaDecoder::mDecoderPosition is roughly the same as Tell() which
+    // returns a position updated by latest Read() or ReadAt().
+    return !IsDataCachedToEndOfResource(Tell()) && !IsSuspended();
+  }
   // Returns true if this stream is suspended by the cache because the
   // cache is full. If true then the decoder should try to start consuming
   // data, otherwise we may not be able to make progress.
@@ -459,6 +439,8 @@ public:
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
 
+  const nsCString& GetContentURL() const { return EmptyCString(); }
+
 protected:
   virtual ~MediaResource() {};
 
@@ -491,6 +473,12 @@ public:
     return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
   }
 
+  // Returns the url of the resource. Safe to call from any thread?
+  const nsCString& GetContentURL() const
+  {
+    return mContentURL;
+  }
+
 protected:
   BaseMediaResource(MediaDecoder* aDecoder,
                     nsIChannel* aChannel,
@@ -504,6 +492,7 @@ protected:
   {
     MOZ_COUNT_CTOR(BaseMediaResource);
     NS_ASSERTION(!mContentType.IsEmpty(), "Must know content type");
+    mURI->GetSpec(mContentURL);
   }
   virtual ~BaseMediaResource()
   {
@@ -542,6 +531,9 @@ protected:
   // is safe.
   const nsAutoCString mContentType;
 
+  // Copy of the url of the channel resource.
+  nsAutoCString mContentURL;
+
   // True if SetLoadInBackground() has been called with
   // aLoadInBackground = true, i.e. when the document load event is not
   // blocked by this resource, and all channel loads will be in the
@@ -667,12 +659,10 @@ public:
   // Other thread
   virtual void     SetReadMode(MediaCacheStream::ReadMode aMode) override;
   virtual void     SetPlaybackRate(uint32_t aBytesPerSecond) override;
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) override;
   virtual nsresult ReadAt(int64_t offset, char* aBuffer,
                           uint32_t aCount, uint32_t* aBytes) override;
-  virtual already_AddRefed<MediaByteBuffer> SilentReadAt(int64_t aOffset, uint32_t aCount) override;
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) override;
-  virtual int64_t  Tell() override;
+  virtual already_AddRefed<MediaByteBuffer> MediaReadAt(int64_t aOffset, uint32_t aCount) override;
+  virtual int64_t Tell() override;
 
   // Any thread
   virtual void    Pin() override;
@@ -900,10 +890,10 @@ public:
   // Otherwise, it returns an owned buffer.
   // MediaReadAt may return fewer bytes than requested if end of stream is
   // encountered. There is no need to call it again to get more data.
-//  already_AddRefed<MediaByteBuffer> MediaReadAt(int64_t aOffset, uint32_t aCount) const
-//  {
-//    return mResource->MediaReadAt(aOffset, aCount);
-//  }
+  already_AddRefed<MediaByteBuffer> MediaReadAt(int64_t aOffset, uint32_t aCount) const
+  {
+    return mResource->MediaReadAt(aOffset, aCount);
+  }
   // Get the length of the stream in bytes. Returns -1 if not known.
   // This can change over time; after a seek operation, a misbehaving
   // server may give us a resource of a different length to what it had

dom/media/MediaSegment.h

@@ -284,6 +284,26 @@ public:
     uint32_t mIndex;
   };
 
+  Chunk* FindChunkContaining(StreamTime aOffset, StreamTime* aStart = nullptr)
+  {
+    if (aOffset < 0) {
+      return nullptr;
+    }
+    StreamTime offset = 0;
+    for (uint32_t i = 0; i < mChunks.Length(); ++i) {
+      Chunk& c = mChunks[i];
+      StreamTime nextOffset = offset + c.GetDuration();
+      if (aOffset < nextOffset) {
+        if (aStart) {
+          *aStart = offset;
+        }
+        return &c;
+      }
+      offset = nextOffset;
+    }
+    return nullptr;
+  }
+
   void RemoveLeading(StreamTime aDuration)
   {
     RemoveLeading(aDuration, 0);
@@ -325,7 +345,7 @@ protected:
       mChunks[mChunks.Length() - 1].mDuration += aSource->mChunks[0].mDuration;
       aSource->mChunks.RemoveElementAt(0);
     }
-    mChunks.MoveElementsFrom(aSource->mChunks);
+    mChunks.AppendElements(Move(aSource->mChunks));
   }
 
   void AppendSliceInternal(const MediaSegmentBase<C, Chunk>& aSource,
@@ -356,26 +376,6 @@ protected:
     return c;
   }
 
-  Chunk* FindChunkContaining(StreamTime aOffset, StreamTime* aStart = nullptr)
-  {
-    if (aOffset < 0) {
-      return nullptr;
-    }
-    StreamTime offset = 0;
-    for (uint32_t i = 0; i < mChunks.Length(); ++i) {
-      Chunk& c = mChunks[i];
-      StreamTime nextOffset = offset + c.GetDuration();
-      if (aOffset < nextOffset) {
-        if (aStart) {
-          *aStart = offset;
-        }
-        return &c;
-      }
-      offset = nextOffset;
-    }
-    return nullptr;
-  }
-
   Chunk* GetLastChunk()
   {
     if (mChunks.IsEmpty()) {

dom/media/MediaStatistics.h

@@ -0,0 +1,82 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MediaStatistics_h_
+#define MediaStatistics_h_
+
+namespace mozilla {
+
+struct MediaStatistics {
+  // Estimate of the current playback rate (bytes/second).
+  double mPlaybackRate;
+  // Estimate of the current download rate (bytes/second). This
+  // ignores time that the channel was paused by Gecko.
+  double mDownloadRate;
+  // Total length of media stream in bytes; -1 if not known
+  int64_t mTotalBytes;
+  // Current position of the download, in bytes. This is the offset of
+  // the first uncached byte after the decoder position.
+  int64_t mDownloadPosition;
+  // Current position of decoding, in bytes (how much of the stream
+  // has been consumed)
+  int64_t mDecoderPosition;
+  // Current position of playback, in bytes
+  int64_t mPlaybackPosition;
+  // If false, then mDownloadRate cannot be considered a reliable
+  // estimate (probably because the download has only been running
+  // a short time).
+  bool mDownloadRateReliable;
+  // If false, then mPlaybackRate cannot be considered a reliable
+  // estimate (probably because playback has only been running
+  // a short time).
+  bool mPlaybackRateReliable;
+
+  bool CanPlayThrough()
+  {
+    // Number of estimated seconds worth of data we need to have buffered
+    // ahead of the current playback position before we allow the media decoder
+    // to report that it can play through the entire media without the decode
+    // catching up with the download. Having this margin make the
+    // CanPlayThrough() calculation more stable in the case of
+    // fluctuating bitrates.
+    static const int64_t CAN_PLAY_THROUGH_MARGIN = 1;
+
+    if ((mTotalBytes < 0 && mDownloadRateReliable) ||
+        (mTotalBytes >= 0 && mTotalBytes == mDownloadPosition)) {
+      return true;
+    }
+
+    if (!mDownloadRateReliable || !mPlaybackRateReliable) {
+      return false;
+    }
+
+    int64_t bytesToDownload = mTotalBytes - mDownloadPosition;
+    int64_t bytesToPlayback = mTotalBytes - mPlaybackPosition;
+    double timeToDownload = bytesToDownload / mDownloadRate;
+    double timeToPlay = bytesToPlayback / mPlaybackRate;
+
+    if (timeToDownload > timeToPlay) {
+      // Estimated time to download is greater than the estimated time to play.
+      // We probably can't play through without having to stop to buffer.
+      return false;
+    }
+
+    // Estimated time to download is less than the estimated time to play.
+    // We can probably play through without having to buffer, but ensure that
+    // we've got a reasonable amount of data buffered after the current
+    // playback position, so that if the bitrate of the media fluctuates, or if
+    // our download rate or decode rate estimation is otherwise inaccurate,
+    // we don't suddenly discover that we need to buffer. This is particularly
+    // required near the start of the media, when not much data is downloaded.
+    int64_t readAheadMargin =
+      static_cast<int64_t>(mPlaybackRate * CAN_PLAY_THROUGH_MARGIN);
+    return mDownloadPosition > mPlaybackPosition + readAheadMargin;
+  }
+};
+
+} // namespace mozilla
+
+#endif // MediaStatistics_h_

dom/media/MediaStreamGraph.h

@@ -313,7 +313,8 @@ class CameraPreviewMediaStream;
  * for those objects in arbitrary order and the MediaStreamGraph has to be able
  * to handle this.
  */
-class MediaStream : public mozilla::LinkedListElement<MediaStream> {
+class MediaStream : public mozilla::LinkedListElement<MediaStream>
+{
 public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
 
@@ -432,8 +433,6 @@ public:
   virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
   virtual CameraPreviewMediaStream* AsCameraPreviewStream() { return nullptr; }
 
-  // media graph thread only
-  void Init();
   // These Impl methods perform the core functionality of the control methods
   // above, on the media graph thread.
   /**
@@ -462,6 +461,9 @@ public:
   }
   void RemoveVideoOutputImpl(VideoFrameContainer* aContainer)
   {
+    // Ensure that any frames currently queued for playback by the compositor
+    // are removed.
+    aContainer->ClearFutureFrames();
     mVideoOutputs.RemoveElement(aContainer);
   }
   void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta)
@@ -556,10 +558,6 @@ public:
   GraphTime StreamTimeToGraphTime(StreamTime aTime);
   bool IsFinishedOnGraphThread() { return mFinished; }
   void FinishOnGraphThread();
-  /**
-   * Identify which graph update index we are currently processing.
-   */
-  int64_t GetProcessingGraphUpdateIndex();
 
   bool HasCurrentData() { return mHasCurrentData; }
 
@@ -586,11 +584,9 @@ public:
   dom::AudioChannel AudioChannelType() const { return mAudioChannelType; }
 
 protected:
-  virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
+  void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime)
   {
     mBufferStartTime += aBlockedTime;
-    mGraphUpdateIndices.InsertTimeAtStart(aBlockedTime);
-    mGraphUpdateIndices.AdvanceCurrentTime(aCurrentTime);
     mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime);
 
     mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime);
@@ -637,15 +633,14 @@ protected:
   };
   nsTArray<AudioOutput> mAudioOutputs;
   nsTArray<nsRefPtr<VideoFrameContainer> > mVideoOutputs;
-  // We record the last played video frame to avoid redundant setting
-  // of the current video frame.
+  // We record the last played video frame to avoid playing the frame again
+  // with a different frame id.
   VideoFrame mLastPlayedVideoFrame;
   // The number of times this stream has been explicitly blocked by the control
   // API, minus the number of times it has been explicitly unblocked.
   TimeVarying<GraphTime,uint32_t,0> mExplicitBlockerCount;
   nsTArray<nsRefPtr<MediaStreamListener> > mListeners;
   nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
-  nsRefPtr<nsRunnable> mNotificationMainThreadRunnable;
   nsTArray<TrackID> mDisabledTrackIDs;
 
   // Precomputed blocking status (over GraphTime).
@@ -655,15 +650,14 @@ protected:
   // as necessary to account for that time instead) --- this avoids us having to
   // record the entire history of the stream's blocking-ness in mBlocked.
   TimeVarying<GraphTime,bool,5> mBlocked;
-  // Maps graph time to the graph update that affected this stream at that time
-  TimeVarying<GraphTime,int64_t,0> mGraphUpdateIndices;
 
   // MediaInputPorts to which this is connected
   nsTArray<MediaInputPort*> mConsumers;
 
   // Where audio output is going. There is one AudioOutputStream per
   // audio track.
-  struct AudioOutputStream {
+  struct AudioOutputStream
+  {
     // When we started audio playback for this track.
     // Add mStream->GetPosition() to find the current audio playback position.
     GraphTime mAudioPlaybackStartTime;
@@ -733,7 +727,8 @@ protected:
  * Audio and video can be written on any thread, but you probably want to
  * always write from the same thread to avoid unexpected interleavings.
  */
-class SourceMediaStream : public MediaStream {
+class SourceMediaStream : public MediaStream
+{
 public:
   explicit SourceMediaStream(DOMMediaStream* aWrapper) :
     MediaStream(aWrapper),
@@ -812,11 +807,6 @@ public:
    * or the stream was already finished.
    */
   bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
-  /**
-   * Returns true if the buffer currently has enough data.
-   * Returns false if there isn't enough data or if no such track exists.
-   */
-  bool HaveEnoughBuffered(TrackID aID);
   /**
    * Get the stream time of the end of the data that has been appended so far.
    * Can be called from any thread but won't be useful if it can race with
@@ -824,14 +814,6 @@ public:
    * that also calls AppendToTrack.
    */
   StreamTime GetEndOfAppendedData(TrackID aID);
-  /**
-   * Ensures that aSignalRunnable will be dispatched to aSignalThread
-   * when we don't have enough buffered data in the track (which could be
-   * immediately). Will dispatch the runnable immediately if the track
-   * does not exist. No op if a runnable is already present for this track.
-   */
-  void DispatchWhenNotEnoughBuffered(TrackID aID,
-      TaskQueue* aSignalQueue, nsIRunnable* aSignalRunnable);
   /**
    * Indicate that a track has ended. Do not do any more API calls
    * affecting this track.
@@ -918,20 +900,16 @@ protected:
     // Resampler if the rate of the input track does not match the
     // MediaStreamGraph's.
     nsAutoRef<SpeexResamplerState> mResampler;
-#ifdef DEBUG
     int mResamplerChannelCount;
-#endif
     StreamTime mStart;
     // End-time of data already flushed to the track (excluding mData)
     StreamTime mEndOfFlushedData;
     // Each time the track updates are flushed to the media graph thread,
     // the segment buffer is emptied.
     nsAutoPtr<MediaSegment> mData;
-    nsTArray<ThreadAndRunnable> mDispatchWhenNotEnough;
     // Each time the track updates are flushed to the media graph thread,
     // this is cleared.
     uint32_t mCommands;
-    bool mHaveEnough;
   };
 
   bool NeedsMixing();
@@ -995,7 +973,8 @@ protected:
  * the Destroy message is processed on the graph manager thread we disconnect
  * the port and drop the graph's reference, destroying the object.
  */
-class MediaInputPort final {
+class MediaInputPort final
+{
 private:
   // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
   MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest,
@@ -1112,7 +1091,8 @@ private:
  * its output. The details of how the output is produced are handled by
  * subclasses overriding the ProcessInput method.
  */
-class ProcessedMediaStream : public MediaStream {
+class ProcessedMediaStream : public MediaStream
+{
 public:
   explicit ProcessedMediaStream(DOMMediaStream* aWrapper)
     : MediaStream(aWrapper), mAutofinish(false)
@@ -1216,21 +1196,33 @@ protected:
 };
 
 /**
- * Initially, at least, we will have a singleton MediaStreamGraph per
- * process.  Each OfflineAudioContext object creates its own MediaStreamGraph
- * object too.
+ * There can be multiple MediaStreamGraph per process: one per AudioChannel.
+ * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph
+ * object too..
  */
-class MediaStreamGraph {
+class MediaStreamGraph
+{
 public:
+
   // We ensure that the graph current time advances in multiples of
   // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
   // never blocks and has a track with the ideal audio rate will produce audio
   // in multiples of the block size.
   //
 
+  // Initializing an graph that outputs audio can be quite long on some
+  // platforms. Code that want to output audio at some point can express the
+  // fact that they will need an audio stream at some point by passing
+  // AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that
+  // the graph starts with the right driver.
+  enum GraphDriverType {
+    AUDIO_THREAD_DRIVER,
+    SYSTEM_THREAD_DRIVER,
+    OFFLINE_THREAD_DRIVER
+  };
   // Main thread only
-  static MediaStreamGraph* GetInstance(bool aStartWithAudioDriver = false,
-                                       dom::AudioChannel aChannel = dom::AudioChannel::Normal);
+  static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
+                                       dom::AudioChannel aChannel);
   static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
@@ -1260,23 +1252,11 @@ public:
    * Create a stream that will mix all its audio input.
    */
   ProcessedMediaStream* CreateAudioCaptureStream(DOMMediaStream* aWrapper);
-  // Internal AudioNodeStreams can only pass their output to another
-  // AudioNode, whereas external AudioNodeStreams can pass their output
-  // to an nsAudioStream for playback.
-  enum AudioNodeStreamKind { SOURCE_STREAM, INTERNAL_STREAM, EXTERNAL_STREAM };
-  /**
-   * Create a stream that will process audio for an AudioNode.
-   * Takes ownership of aEngine.  aSampleRate is the sampling rate used
-   * for the stream.  If 0 is passed, the sampling rate of the engine's
-   * node will get used.
-   */
-  AudioNodeStream* CreateAudioNodeStream(AudioNodeEngine* aEngine,
-                                         AudioNodeStreamKind aKind,
-                                         TrackRate aSampleRate = 0);
 
-  AudioNodeExternalInputStream*
-  CreateAudioNodeExternalInputStream(AudioNodeEngine* aEngine,
-                                     TrackRate aSampleRate = 0);
+  /**
+   * Add a new stream to the graph.  Main thread.
+   */
+  void AddStream(MediaStream* aStream);
 
   /* From the main thread, ask the MSG to send back an event when the graph
    * thread is running, and audio is being processed. */
@@ -1324,8 +1304,7 @@ public:
 
 protected:
   explicit MediaStreamGraph(TrackRate aSampleRate)
-    : mNextGraphUpdateIndex(1)
-    , mSampleRate(aSampleRate)
+    : mSampleRate(aSampleRate)
   {
     MOZ_COUNT_CTOR(MediaStreamGraph);
   }
@@ -1337,11 +1316,6 @@ protected:
   // Media graph thread only
   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
 
-  // Main thread only
-  // The number of updates we have sent to the media graph thread + 1.
-  // We start this at 1 just to ensure that 0 is usable as a special value.
-  int64_t mNextGraphUpdateIndex;
-
   /**
    * Sample rate at which this graph runs. For real time graphs, this is
    * the rate of the audio mixer. For offline graphs, this is the rate specified

dom/media/MediaStreamGraphImpl.h

@@ -31,7 +31,6 @@ class AudioOutputObserver;
  * main thread.
  */
 struct StreamUpdate {
-  int64_t mGraphUpdateIndex;
   nsRefPtr<MediaStream> mStream;
   StreamTime mNextMainThreadCurrentTime;
   bool mNextMainThreadFinished;
@@ -41,7 +40,8 @@ struct StreamUpdate {
  * This represents a message passed from the main thread to the graph thread.
  * A ControlMessage always has a weak reference a particular affected stream.
  */
-class ControlMessage {
+class ControlMessage
+{
 public:
   explicit ControlMessage(MediaStream* aStream) : mStream(aStream)
   {
@@ -53,8 +53,8 @@ public:
     MOZ_COUNT_DTOR(ControlMessage);
   }
   // Do the action of this message on the MediaStreamGraph thread. Any actions
-  // affecting graph processing should take effect at mStateComputedTime.
-  // All stream data for times < mStateComputedTime has already been
+  // affecting graph processing should take effect at mProcessedTime.
+  // All stream data for times < mProcessedTime has already been
   // computed.
   virtual void Run() = 0;
   // When we're shutting down the application, most messages are ignored but
@@ -70,9 +70,9 @@ protected:
   MediaStream* mStream;
 };
 
-class MessageBlock {
+class MessageBlock
+{
 public:
-  int64_t mGraphUpdateIndex;
   nsTArray<nsAutoPtr<ControlMessage> > mMessages;
 };
 
@@ -85,22 +85,23 @@ public:
  * OfflineAudioContext object.
  */
 class MediaStreamGraphImpl : public MediaStreamGraph,
-                             public nsIMemoryReporter {
+                             public nsIMemoryReporter
+{
 public:
   NS_DECL_THREADSAFE_ISUPPORTS
   NS_DECL_NSIMEMORYREPORTER
 
   /**
-   * Set aRealtime to true in order to create a MediaStreamGraph which provides
-   * support for real-time audio and video.  Set it to false in order to create
-   * a non-realtime instance which just churns through its inputs and produces
-   * output.  Those objects currently only support audio, and are used to
-   * implement OfflineAudioContext.  They do not support MediaStream inputs.
+   * Use aGraphDriverRequested with SYSTEM_THREAD_DRIVER or AUDIO_THREAD_DRIVER
+   * to create a MediaStreamGraph which provides support for real-time audio
+   * and/or video.  Set it to false in order to create a non-realtime instance
+   * which just churns through its inputs and produces output.  Those objects
+   * currently only support audio, and are used to implement
+   * OfflineAudioContext.  They do not support MediaStream inputs.
    */
-  explicit MediaStreamGraphImpl(bool aRealtime,
+  explicit MediaStreamGraphImpl(GraphDriverType aGraphDriverRequested,
                                 TrackRate aSampleRate,
-                                bool aStartWithAudioDriver = false,
-                                dom::AudioChannel aChannel = dom::AudioChannel::Normal);
+                                dom::AudioChannel aChannel);
 
   /**
    * Unregisters memory reporting and deletes this instance. This should be
@@ -150,7 +151,8 @@ public:
   void Init();
   // The following methods run on the graph thread (or possibly the main thread if
   // mLifecycleState > LIFECYCLE_RUNNING)
-  void AssertOnGraphThreadOrNotRunning() {
+  void AssertOnGraphThreadOrNotRunning() const
+  {
     // either we're on the right thread (and calling CurrentDriver() is safe),
     // or we're going to assert anyways, so don't cross-check CurrentDriver
 #ifdef DEBUG
@@ -170,23 +172,25 @@ public:
    */
   void DoIteration();
 
-  bool OneIteration(GraphTime aFrom, GraphTime aTo,
-                    GraphTime aStateFrom, GraphTime aStateEnd);
+  bool OneIteration(GraphTime aStateEnd);
 
-  bool Running() {
+  bool Running() const
+  {
     mMonitor.AssertCurrentThreadOwns();
     return mLifecycleState == LIFECYCLE_RUNNING;
   }
 
   // Get the message queue, from the current GraphDriver thread.
-  nsTArray<MessageBlock>& MessageQueue() {
+  nsTArray<MessageBlock>& MessageQueue()
+  {
     mMonitor.AssertCurrentThreadOwns();
     return mFrontMessageQueue;
   }
 
   /* This is the end of the current iteration, that is, the current time of the
    * graph. */
-  GraphTime IterationEnd();
+  GraphTime IterationEnd() const;
+
   /**
    * Ensure there is an event posted to the main thread to run RunInStableState.
    * mMonitor must be held.
@@ -221,7 +225,8 @@ public:
    */
   void UpdateGraph(GraphTime aEndBlockingDecisions);
 
-  void SwapMessageQueues() {
+  void SwapMessageQueues()
+  {
     mMonitor.AssertCurrentThreadOwns();
     mFrontMessageQueue.SwapElements(mBackMessageQueue);
   }
@@ -304,6 +309,7 @@ public:
    * for all streams.
    */
   void RecomputeBlocking(GraphTime aEndBlockingDecisions);
+
   // The following methods are used to help RecomputeBlocking.
   /**
    * If aStream isn't already in aStreams, add it and recursively call
@@ -362,9 +368,11 @@ public:
    * when we don't know whether it's blocked or not, we assume it's not blocked.
    */
   StreamTime GraphTimeToStreamTimeOptimistic(MediaStream* aStream, GraphTime aTime);
-  enum {
+  enum
+  {
     INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01
   };
+
   /**
    * Given a stream time aTime, convert it to a graph time taking into
    * account the time during which aStream is scheduled to be blocked.
@@ -376,10 +384,6 @@ public:
    */
   GraphTime StreamTimeToGraphTime(MediaStream* aStream, StreamTime aTime,
                                   uint32_t aFlags = 0);
-  /**
-   * Get the current audio position of the stream's audio output.
-   */
-  GraphTime GetAudioPosition(MediaStream* aStream);
   /**
    * Call NotifyHaveCurrentData on aStream's listeners.
    */
@@ -411,25 +415,20 @@ public:
   /**
    * Returns true when there are no active streams.
    */
-  bool IsEmpty()
+  bool IsEmpty() const
   {
     return mStreams.IsEmpty() && mSuspendedStreams.IsEmpty() && mPortCount == 0;
   }
 
-  // For use by control messages, on graph thread only.
-  /**
-   * Identify which graph update index we are currently processing.
-   */
-  int64_t GetProcessingGraphUpdateIndex() { return mProcessingGraphUpdateIndex; }
   /**
    * Add aStream to the graph and initializes its graph-specific state.
    */
-  void AddStream(MediaStream* aStream);
+  void AddStreamGraphThread(MediaStream* aStream);
   /**
    * Remove aStream from the graph. Ensures that pending messages about the
    * stream back to the main thread are flushed.
    */
-  void RemoveStream(MediaStream* aStream);
+  void RemoveStreamGraphThread(MediaStream* aStream);
   /**
    * Remove aPort from the graph and release it.
    */
@@ -443,20 +442,23 @@ public:
   }
 
   // Always stereo for now.
-  uint32_t AudioChannelCount() { return 2; }
+  uint32_t AudioChannelCount() const { return 2; }
 
-  double MediaTimeToSeconds(GraphTime aTime)
+  double MediaTimeToSeconds(GraphTime aTime) const
   {
-    NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
+    NS_ASSERTION(aTime > -STREAM_TIME_MAX && aTime <= STREAM_TIME_MAX,
+                 "Bad time");
     return static_cast<double>(aTime)/GraphRate();
   }
-  GraphTime SecondsToMediaTime(double aS)
+
+  GraphTime SecondsToMediaTime(double aS) const
   {
     NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
                  "Bad seconds");
     return GraphRate() * aS;
   }
-  GraphTime MillisecondsToMediaTime(int32_t aMS)
+
+  GraphTime MillisecondsToMediaTime(int32_t aMS) const
   {
     return RateConvertTicksRoundDown(GraphRate(), 1000, aMS);
   }
@@ -471,7 +473,8 @@ public:
   /**
    * Not safe to call off the MediaStreamGraph thread unless monitor is held!
    */
-  GraphDriver* CurrentDriver() {
+  GraphDriver* CurrentDriver() const
+  {
     AssertOnGraphThreadOrNotRunning();
     return mDriver;
   }
@@ -489,16 +492,19 @@ public:
    * We can also switch from Revive() (on MainThread), in which case the
    * monitor is held
    */
-  void SetCurrentDriver(GraphDriver* aDriver) {
+  void SetCurrentDriver(GraphDriver* aDriver)
+  {
     AssertOnGraphThreadOrNotRunning();
     mDriver = aDriver;
   }
 
-  Monitor& GetMonitor() {
+  Monitor& GetMonitor()
+  {
     return mMonitor;
   }
 
-  void EnsureNextIteration() {
+  void EnsureNextIteration()
+  {
     mNeedAnotherIteration = true; // atomic
     if (mGraphDriverAsleep) { // atomic
       MonitorAutoLock mon(mMonitor);
@@ -506,7 +512,8 @@ public:
     }
   }
 
-  void EnsureNextIterationLocked() {
+  void EnsureNextIterationLocked()
+  {
     mNeedAnotherIteration = true; // atomic
     if (mGraphDriverAsleep) { // atomic
       CurrentDriver()->WakeUp(); // Might not be the same driver; might have woken already
@@ -553,14 +560,21 @@ public:
    * cycles.
    */
   uint32_t mFirstCycleBreaker;
+  /**
+   * Blocking decisions have been computed up to this time.
+   * Between each iteration, this is the same as mProcessedTime.
+   */
+  GraphTime mStateComputedTime = 0;
+  /**
+   * All stream contents have been computed up to this time.
+   * The next batch of updates from the main thread will be processed
+   * at this time.  This is behind mStateComputedTime during processing.
+   */
+  GraphTime mProcessedTime = 0;
   /**
    * Date of the last time we updated the main thread with the graph state.
    */
   TimeStamp mLastMainThreadUpdate;
-  /**
-   * Which update batch we are currently processing.
-   */
-  int64_t mProcessingGraphUpdateIndex;
   /**
    * Number of active MediaInputPorts
    */
@@ -599,7 +613,8 @@ public:
   nsTArray<MessageBlock> mBackMessageQueue;
 
   /* True if there will messages to process if we swap the message queues. */
-  bool MessagesQueued() {
+  bool MessagesQueued() const
+  {
     mMonitor.AssertCurrentThreadOwns();
     return !mBackMessageQueue.IsEmpty();
   }
@@ -626,7 +641,8 @@ public:
    * This should be kept in sync with the LifecycleState_str array in
    * MediaStreamGraph.cpp
    */
-  enum LifecycleState {
+  enum LifecycleState
+  {
     // The graph thread hasn't started yet.
     LIFECYCLE_THREAD_NOT_STARTED,
     // RunThread() is running normally.
@@ -719,6 +735,10 @@ public:
 private:
   virtual ~MediaStreamGraphImpl();
 
+  void StreamNotifyOutput(MediaStream* aStream);
+
+  void StreamReadyToFinish(MediaStream* aStream);
+
   MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
 
   /**

dom/media/RtspMediaResource.h

@@ -133,15 +133,6 @@ public:
   // dummy
   virtual void     SetPlaybackRate(uint32_t aBytesPerSecond) override {}
   // dummy
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes)
-  override {
-    return NS_OK;
-  }
-  // dummy
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) override {
-    return NS_OK;
-  }
-  // dummy
   virtual int64_t  Tell() override { return 0; }
 
   // Any thread

dom/media/VideoFrameContainer.cpp

@@ -93,7 +93,24 @@ void VideoFrameContainer::ClearCurrentFrame()
   mImageContainer->GetCurrentImages(&kungFuDeathGrip);
 
   mImageContainer->ClearAllImages();
-  mImageSizeChanged = false;
+}
+
+void VideoFrameContainer::ClearFutureFrames()
+{
+  MutexAutoLock lock(mMutex);
+
+  // See comment in SetCurrentFrame for the reasoning behind
+  // using a kungFuDeathGrip here.
+  nsTArray<ImageContainer::OwningImage> kungFuDeathGrip;
+  mImageContainer->GetCurrentImages(&kungFuDeathGrip);
+
+  if (!kungFuDeathGrip.IsEmpty()) {
+    nsTArray<ImageContainer::NonOwningImage> currentFrame;
+    const ImageContainer::OwningImage& img = kungFuDeathGrip[0];
+    currentFrame.AppendElement(ImageContainer::NonOwningImage(img.mImage,
+        img.mTimeStamp, img.mFrameID, img.mProducerID));
+    mImageContainer->SetCurrentImages(currentFrame);
+  }
 }
 
 ImageContainer* VideoFrameContainer::GetImageContainer() {

dom/media/VideoFrameContainer.h

@@ -52,10 +52,22 @@ public:
   }
 
   void ClearCurrentFrame();
+  // Make the current frame the only frame in the container, i.e. discard
+  // all future frames.
+  void ClearFutureFrames();
   // Time in seconds by which the last painted video frame was late by.
   // E.g. if the last painted frame should have been painted at time t,
   // but was actually painted at t+n, this returns n in seconds. Threadsafe.
   double GetFrameDelay();
+
+  // Returns a new frame ID for SetCurrentFrames().  The client must either
+  // call this on only one thread or provide barriers.  Do not use together
+  // with SetCurrentFrame().
+  ImageContainer::FrameID NewFrameID()
+  {
+    return ++mFrameID;
+  }
+
   // Call on main thread
   enum {
     INVALIDATE_DEFAULT,
@@ -83,8 +95,8 @@ protected:
   // specifies that the Image should be stretched to have the correct aspect
   // ratio.
   gfxIntSize mIntrinsicSize;
-  // For SetCurrentFrame callers we maintain our own mFrameID which is auto-
-  // incremented at every SetCurrentFrame.
+  // We maintain our own mFrameID which is auto-incremented at every
+  // SetCurrentFrame() or NewFrameID() call.
   ImageContainer::FrameID mFrameID;
   // True when the intrinsic size has been changed by SetCurrentFrame() since
   // the last call to Invalidate().

dom/media/VideoSegment.h

@@ -117,14 +117,6 @@ public:
                    StreamTime aDuration,
                    const IntSize& aIntrinsicSize,
                    bool aForceBlack = false);
-  const VideoFrame* GetFrameAt(StreamTime aOffset, StreamTime* aStart = nullptr)
-  {
-    VideoChunk* c = FindChunkContaining(aOffset, aStart);
-    if (!c) {
-      return nullptr;
-    }
-    return &c->mFrame;
-  }
   const VideoFrame* GetLastFrame(StreamTime* aStart = nullptr)
   {
     VideoChunk* c = GetLastChunk();

dom/media/apple/AppleMP3Reader.cpp

@@ -44,6 +44,7 @@ AppleMP3Reader::AppleMP3Reader(AbstractMediaDecoder *aDecoder)
   , mAudioFileStream(nullptr)
   , mAudioConverter(nullptr)
   , mMP3FrameParser(mDecoder->GetResource()->GetLength())
+  , mResource(mDecoder->GetResource())
 {
   MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread");
 }
@@ -90,27 +91,15 @@ static void _AudioSampleCallback(void *aThis,
 nsresult
 AppleMP3Reader::Read(uint32_t *aNumBytes, char *aData)
 {
-  MediaResource *resource = mDecoder->GetResource();
+  nsresult rv = mResource.Read(aData, *aNumBytes, aNumBytes);
 
-  // Loop until we have all the data asked for, or we've reached EOS
-  uint32_t totalBytes = 0;
-  uint32_t numBytes;
-  do {
-    uint32_t bytesWanted = *aNumBytes - totalBytes;
-    nsresult rv = resource->Read(aData + totalBytes, bytesWanted, &numBytes);
-    totalBytes += numBytes;
+  if (NS_FAILED(rv)) {
+    *aNumBytes = 0;
+    return NS_ERROR_FAILURE;
+  }
 
-    if (NS_FAILED(rv)) {
-      *aNumBytes = 0;
-      return NS_ERROR_FAILURE;
-    }
-  } while(totalBytes < *aNumBytes && numBytes);
-
-  *aNumBytes = totalBytes;
-
-  // We will have read some data in the last iteration iff we filled the buffer.
   // XXX Maybe return a better value than NS_ERROR_FAILURE?
-  return numBytes ? NS_OK : NS_ERROR_FAILURE;
+  return *aNumBytes ? NS_OK : NS_ERROR_FAILURE;
 }
 
 nsresult
@@ -257,7 +246,7 @@ AppleMP3Reader::AudioSampleCallback(UInt32 aNumBytes,
     LOGD("pushed audio at time %lfs; duration %lfs\n",
          (double)time / USECS_PER_S, (double)duration / USECS_PER_S);
 
-    AudioData *audio = new AudioData(mDecoder->GetResource()->Tell(),
+    AudioData *audio = new AudioData(mResource.Tell(),
                                      time, duration, numFrames,
                                      reinterpret_cast<AudioDataValue *>(decoded.forget()),
                                      mAudioChannels, mAudioSampleRate);
@@ -516,7 +505,7 @@ AppleMP3Reader::Seek(int64_t aTime, int64_t aEndTime)
        byteOffset,
        (flags & kAudioFileStreamSeekFlag_OffsetIsEstimated) ? "YES" : "NO");
 
-  mDecoder->GetResource()->Seek(nsISeekableStream::NS_SEEK_SET, byteOffset);
+  mResource.Seek(nsISeekableStream::NS_SEEK_SET, byteOffset);
 
   ResetDecode();
 
@@ -531,7 +520,8 @@ AppleMP3Reader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset)
     return;
   }
 
-  nsRefPtr<MediaByteBuffer> bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength);
+  nsRefPtr<MediaByteBuffer> bytes =
+    mDecoder->GetResource()->MediaReadAt(aOffset, aLength);
   NS_ENSURE_TRUE_VOID(bytes);
   mMP3FrameParser.Parse(bytes->Elements(), aLength, aOffset);
   if (!mMP3FrameParser.IsMP3()) {

dom/media/apple/AppleMP3Reader.h

@@ -6,6 +6,7 @@
 #define __AppleMP3Reader_h__
 
 #include "MediaDecoderReader.h"
+#include "MediaResource.h"
 #include "MP3FrameParser.h"
 #include "VideoUtils.h"
 
@@ -79,6 +80,8 @@ private:
   AudioConverterRef mAudioConverter;
 
   MP3FrameParser mMP3FrameParser;
+
+  MediaResourceIndex mResource;
 };
 
 } // namespace mozilla

dom/media/compiledtest/TestAudioPacketizer.cpp

@@ -0,0 +1,174 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <stdint.h>
+#include <assert.h>
+#define _USE_MATH_DEFINES
+#include <math.h>
+#include "../AudioPacketizer.h"
+
+using namespace mozilla;
+
+template<typename T>
+class AutoBuffer
+{
+public:
+  explicit AutoBuffer(size_t aLength)
+  {
+    mStorage = new T[aLength];
+  }
+  ~AutoBuffer() {
+    delete [] mStorage;
+  }
+  T* Get() {
+    return mStorage;
+  }
+private:
+  T* mStorage;
+};
+
+int16_t Sequence(int16_t* aBuffer, uint32_t aSize, uint32_t aStart = 0)
+{
+  uint32_t i;
+  for (i = 0; i < aSize; i++) {
+    aBuffer[i] = aStart + i;
+  }
+  return aStart + i;
+}
+
+void IsSequence(int16_t* aBuffer, uint32_t aSize, uint32_t aStart = 0)
+{
+  for (uint32_t i = 0; i < aSize; i++) {
+    if (aBuffer[i] != static_cast<int64_t>(aStart + i)) {
+      fprintf(stderr, "Buffer is not a sequence at offset %u\n", i);
+      assert(false);
+    }
+  }
+  assert("Buffer is a sequence.");
+}
+
+void Zero(int16_t* aBuffer, uint32_t aSize)
+{
+  for (uint32_t i = 0; i < aSize; i++) {
+    if (aBuffer[i] != 0) {
+      fprintf(stderr, "Buffer is not null at offset %u\n", i);
+      assert(false);
+    }
+  }
+}
+
+double sine(uint32_t aPhase) {
+ return sin(aPhase * 2 * M_PI * 440 / 44100);
+}
+
+int main() {
+  for (int16_t channels = 1; channels < 2; channels++) {
+    // Test that the packetizer returns zero on underrun
+    {
+      AudioPacketizer<int16_t, int16_t> ap(441, channels);
+      for (int16_t i = 0; i < 10; i++) {
+        int16_t* out = ap.Output();
+        Zero(out, 441);
+        delete out;
+      }
+    }
+    // Simple test, with input/output buffer size aligned on the packet size,
+    // alternating Input and Output calls.
+    {
+      AudioPacketizer<int16_t, int16_t> ap(441, channels);
+      int16_t seqEnd = 0;
+      for (int16_t i = 0; i < 10; i++) {
+        AutoBuffer<int16_t> b(441 * channels);
+        int16_t prevEnd = seqEnd;
+        seqEnd = Sequence(b.Get(), channels * 441, prevEnd);
+        ap.Input(b.Get(), 441);
+        int16_t* out = ap.Output();
+        IsSequence(out, 441 * channels, prevEnd);
+        delete out;
+      }
+    }
+    // Simple test, with input/output buffer size aligned on the packet size,
+    // alternating two Input and Output calls.
+    {
+      AudioPacketizer<int16_t, int16_t> ap(441, channels);
+      int16_t seqEnd = 0;
+      for (int16_t i = 0; i < 10; i++) {
+        AutoBuffer<int16_t> b(441 * channels);
+        AutoBuffer<int16_t> b1(441 * channels);
+        int16_t prevEnd0 = seqEnd;
+        seqEnd = Sequence(b.Get(), 441 * channels, prevEnd0);
+        int16_t prevEnd1 = seqEnd;
+        seqEnd = Sequence(b1.Get(), 441 * channels, seqEnd);
+        ap.Input(b.Get(), 441);
+        ap.Input(b1.Get(), 441);
+        int16_t* out = ap.Output();
+        int16_t* out2 = ap.Output();
+        IsSequence(out, 441 * channels, prevEnd0);
+        IsSequence(out2, 441 * channels, prevEnd1);
+        delete out;
+        delete out2;
+      }
+    }
+    // Input/output buffer size not aligned on the packet size,
+    // alternating two Input and Output calls.
+    {
+      AudioPacketizer<int16_t, int16_t> ap(441, channels);
+      int16_t prevEnd = 0;
+      int16_t prevSeq = 0;
+      for (int16_t i = 0; i < 10; i++) {
+        AutoBuffer<int16_t> b(480 * channels);
+        AutoBuffer<int16_t> b1(480 * channels);
+        prevSeq = Sequence(b.Get(), 480 * channels, prevSeq);
+        prevSeq = Sequence(b1.Get(), 480 * channels, prevSeq);
+        ap.Input(b.Get(), 480);
+        ap.Input(b1.Get(), 480);
+        int16_t* out = ap.Output();
+        int16_t* out2 = ap.Output();
+        IsSequence(out, 441 * channels, prevEnd);
+        prevEnd += 441 * channels;
+        IsSequence(out2, 441 * channels, prevEnd);
+        prevEnd += 441 * channels;
+        delete out;
+        delete out2;
+      }
+      printf("Available: %d\n", ap.PacketsAvailable());
+    }
+
+    // "Real-life" test case: streaming a sine wave through a packetizer, and
+    // checking that we have the right output.
+    // 128 is, for example, the size of a Web Audio API block, and 441 is the
+    // size of a webrtc.org packet when the sample rate is 44100 (10ms)
+    {
+      AudioPacketizer<int16_t, int16_t> ap(441, channels);
+      AutoBuffer<int16_t> b(128 * channels);
+      uint32_t phase = 0;
+      uint32_t outPhase = 0;
+      for (int16_t i = 0; i < 1000; i++) {
+        for (int32_t j = 0; j < 128; j++) {
+          for (int32_t c = 0; c < channels; c++) {
+            // int16_t sinewave at 440Hz/44100Hz sample rate
+            b.Get()[j * channels + c] = (2 << 14) * sine(phase);
+          }
+          phase++;
+        }
+        ap.Input(b.Get(), 128);
+        while (ap.PacketsAvailable()) {
+          int16_t* packet = ap.Output();
+          for (uint32_t k = 0; k < ap.PacketSize(); k++) {
+            for (int32_t c = 0; c < channels; c++) {
+              assert(packet[k * channels + c] ==
+                     static_cast<int16_t>(((2 << 14) * sine(outPhase))));
+            }
+            outPhase++;
+          }
+          delete [] packet;
+        }
+      }
+    }
+  }
+
+  printf("OK\n");
+  return 0;
+}

dom/media/compiledtest/TestAudioSegment.cpp

@@ -0,0 +1,268 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AudioSegment.h"
+#include <assert.h>
+#include <iostream>
+
+using namespace mozilla;
+
+namespace mozilla {
+uint32_t
+GetAudioChannelsSuperset(uint32_t aChannels1, uint32_t aChannels2)
+{
+  return std::max(aChannels1, aChannels2);
+}
+}
+
+/* Helper function to give us the maximum and minimum value that don't clip,
+ * for a given sample format (integer or floating-point). */
+template<typename T>
+T GetLowValue();
+
+template<typename T>
+T GetHighValue();
+
+template<typename T>
+T GetSilentValue();
+
+template<>
+float GetLowValue<float>() {
+  return -1.0;
+}
+
+template<>
+int16_t GetLowValue<short>() {
+  return -INT16_MAX;
+}
+
+template<>
+float GetHighValue<float>() {
+  return 1.0;
+}
+
+template<>
+int16_t GetHighValue<short>() {
+  return INT16_MAX;
+}
+
+template<>
+float GetSilentValue() {
+  return 0.0;
+}
+
+template<>
+int16_t GetSilentValue() {
+  return 0;
+}
+
+
+// Get an array of planar audio buffers that has the inverse of the index of the
+// channel (1-indexed) as samples.
+template<typename T>
+const T* const* GetPlanarChannelArray(size_t aChannels, size_t aSize)
+{
+  T** channels = new T*[aChannels];
+  for (size_t c = 0; c < aChannels; c++) {
+    channels[c] = new T[aSize];
+    for (size_t i = 0; i < aSize; i++) {
+      channels[c][i] = FloatToAudioSample<T>(1. / (c + 1));
+    }
+  }
+  return channels;
+}
+
+template<typename T>
+void DeletePlanarChannelsArray(const T* const* aArrays, size_t aChannels)
+{
+  for (size_t channel = 0; channel < aChannels; channel++) {
+    delete [] aArrays[channel];
+  }
+  delete [] aArrays;
+}
+
+template<typename T>
+T** GetPlanarArray(size_t aChannels, size_t aSize)
+{
+  T** channels = new T*[aChannels];
+  for (size_t c = 0; c < aChannels; c++) {
+    channels[c] = new T[aSize];
+    for (size_t i = 0; i < aSize; i++) {
+      channels[c][i] = 0.0f;
+    }
+  }
+  return channels;
+}
+
+template<typename T>
+void DeletePlanarArray(T** aArrays, size_t aChannels)
+{
+  for (size_t channel = 0; channel < aChannels; channel++) {
+    delete [] aArrays[channel];
+  }
+  delete [] aArrays;
+}
+
+// Get an array of audio samples that have the inverse of the index of the
+// channel (1-indexed) as samples.
+template<typename T>
+const T* GetInterleavedChannelArray(size_t aChannels, size_t aSize)
+{
+  size_t sampleCount = aChannels * aSize;
+  T* samples = new T[sampleCount];
+  for (size_t i = 0; i < sampleCount; i++) {
+    uint32_t channel = (i % aChannels) + 1;
+    samples[i] = FloatToAudioSample<T>(1. / channel);
+  }
+  return samples;
+}
+
+template<typename T>
+void DeleteInterleavedChannelArray(const T* aArray)
+{
+  delete [] aArray;
+}
+
+bool FuzzyEqual(float aLhs, float aRhs) {
+  return std::abs(aLhs - aRhs) < 0.01;
+}
+
+template<typename SrcT, typename DstT>
+void TestInterleaveAndConvert()
+{
+  size_t arraySize = 1024;
+  size_t maxChannels = 8; // 7.1
+  for (uint32_t channels = 1; channels < maxChannels; channels++) {
+    const SrcT* const* src = GetPlanarChannelArray<SrcT>(channels, arraySize);
+    DstT* dst = new DstT[channels * arraySize];
+
+    InterleaveAndConvertBuffer(src, arraySize, 1.0, channels, dst);
+
+    uint32_t channelIndex = 0;
+    for (size_t i = 0; i < arraySize * channels; i++) {
+      assert(FuzzyEqual(dst[i],
+                        FloatToAudioSample<DstT>(1. / (channelIndex + 1))));
+      channelIndex++;
+      channelIndex %= channels;
+    }
+
+    DeletePlanarChannelsArray(src, channels);
+    delete [] dst;
+  }
+}
+
+template<typename SrcT, typename DstT>
+void TestDeinterleaveAndConvert()
+{
+  size_t arraySize = 1024;
+  size_t maxChannels = 8; // 7.1
+  for (uint32_t channels = 1; channels < maxChannels; channels++) {
+    const SrcT* src = GetInterleavedChannelArray<SrcT>(channels, arraySize);
+    DstT** dst = GetPlanarArray<DstT>(channels, arraySize);
+
+    DeinterleaveAndConvertBuffer(src, arraySize, channels, dst);
+
+    for (size_t channel = 0; channel < channels; channel++) {
+      for (size_t i = 0; i < arraySize; i++) {
+        assert(FuzzyEqual(dst[channel][i],
+                          FloatToAudioSample<DstT>(1. / (channel + 1))));
+      }
+    }
+
+    DeleteInterleavedChannelArray(src);
+    DeletePlanarArray(dst, channels);
+  }
+}
+
+uint8_t gSilence[4096] = {0};
+
+template<typename T>
+T* SilentChannel()
+{
+  return reinterpret_cast<T*>(gSilence);
+}
+
+template<typename T>
+void TestUpmixStereo()
+{
+  size_t arraySize = 1024;
+  nsTArray<T*> channels;
+  nsTArray<const T*> channelsptr;
+
+  channels.SetLength(1);
+  channelsptr.SetLength(1);
+
+  channels[0] = new T[arraySize];
+
+  for (size_t i = 0; i < arraySize; i++) {
+    channels[0][i] = GetHighValue<T>();
+  }
+  channelsptr[0] = channels[0];
+
+  AudioChannelsUpMix(&channelsptr, 2, ::SilentChannel<T>());
+
+  for (size_t channel = 0; channel < 2; channel++) {
+    for (size_t i = 0; i < arraySize; i++) {
+      if (channelsptr[channel][i] != GetHighValue<T>()) {
+        assert(false);
+      }
+    }
+  }
+  assert(true);
+  delete channels[0];
+}
+
+template<typename T>
+void TestDownmixStereo()
+{
+  const size_t arraySize = 1024;
+  nsTArray<const T*> inputptr;
+  nsTArray<T*> input;
+  T** output;
+
+  output = new T*[1];
+  output[0] = new T[arraySize];
+
+  input.SetLength(2);
+  inputptr.SetLength(2);
+
+  for (size_t channel = 0; channel < input.Length(); channel++) {
+    input[channel] = new T[arraySize];
+    for (size_t i = 0; i < arraySize; i++) {
+      input[channel][i] = channel == 0 ? GetLowValue<T>() : GetHighValue<T>();
+    }
+    inputptr[channel] = input[channel];
+  }
+
+  AudioChannelsDownMix(inputptr, output, 1, arraySize);
+
+  for (size_t i = 0; i < arraySize; i++) {
+    if (output[0][i] != GetSilentValue<T>()) {
+      assert(false);
+    }
+  }
+  assert(true);
+
+  delete output[0];
+  delete output;
+}
+
+int main(int argc, char* argv[]) {
+  TestInterleaveAndConvert<float, float>();
+  TestInterleaveAndConvert<float, int16_t>();
+  TestInterleaveAndConvert<int16_t, float>();
+  TestInterleaveAndConvert<int16_t, int16_t>();
+  TestDeinterleaveAndConvert<float, float>();
+  TestDeinterleaveAndConvert<float, int16_t>();
+  TestDeinterleaveAndConvert<int16_t, float>();
+  TestDeinterleaveAndConvert<int16_t, int16_t>();
+  TestUpmixStereo<float>();
+  TestUpmixStereo<int16_t>();
+  TestDownmixStereo<float>();
+  TestDownmixStereo<int16_t>();
+
+  return 0;
+}
+

dom/media/compiledtest/moz.build

@@ -6,11 +6,15 @@
 
 GeckoCppUnitTests([
     'TestAudioBuffers',
-    'TestAudioMixer'
+    'TestAudioMixer',
+    'TestAudioPacketizer',
+    'TestAudioSegment'
 ])
 
-FAIL_ON_WARNINGS = True
-
 LOCAL_INCLUDES += [
     '..',
 ]
+
+USE_LIBS += [
+    'lgpllibs',
+]

dom/media/directshow/DirectShowReader.cpp

@@ -408,7 +408,7 @@ DirectShowReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset)
     return;
   }
 
-  nsRefPtr<MediaByteBuffer> bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength);
+  nsRefPtr<MediaByteBuffer> bytes = mDecoder->GetResource()->MediaReadAt(aOffset, aLength);
   NS_ENSURE_TRUE_VOID(bytes);
   mMP3FrameParser.Parse(bytes->Elements(), aLength, aOffset);
   if (!mMP3FrameParser.IsMP3()) {

dom/media/directshow/SourceFilter.cpp

@@ -76,7 +76,7 @@ public:
   {}
 
   int64_t GetLength() {
-    int64_t len = mResource->GetLength();
+    int64_t len = mResource.GetLength();
     if (len == -1) {
       return len;
     }
@@ -85,19 +85,20 @@ public:
   nsresult ReadAt(int64_t aOffset, char* aBuffer,
                   uint32_t aCount, uint32_t* aBytes)
   {
-    return mResource->ReadAt(aOffset + mDataOffset,
-                             aBuffer,
-                             aCount,
-                             aBytes);
+    return mResource.ReadAt(aOffset + mDataOffset,
+                            aBuffer,
+                            aCount,
+                            aBytes);
   }
   int64_t GetCachedDataEnd() {
-    int64_t tell = mResource->Tell();
-    int64_t dataEnd = mResource->GetCachedDataEnd(tell) - mDataOffset;
+    int64_t tell = mResource.GetResource()->Tell();
+    int64_t dataEnd =
+      mResource.GetResource()->GetCachedDataEnd(tell) - mDataOffset;
     return dataEnd;
   }
 private:
   // MediaResource from which we read data.
-  RefPtr<MediaResource> mResource;
+  MediaResourceIndex mResource;
   int64_t mDataOffset;
 };
 
@@ -559,23 +560,13 @@ OutputPin::SyncRead(LONGLONG aPosition,
     }
   }
 
-  // Read in a loop to ensure we fill the buffer, when possible.
-  LONG totalBytesRead = 0;
-  while (totalBytesRead < aLength) {
-    BYTE* readBuffer = aBuffer + totalBytesRead;
-    uint32_t bytesRead = 0;
-    LONG length = aLength - totalBytesRead;
-    nsresult rv = mResource.ReadAt(aPosition + totalBytesRead,
-                                   reinterpret_cast<char*>(readBuffer),
-                                   length,
-                                   &bytesRead);
-    if (NS_FAILED(rv)) {
-      return E_FAIL;
-    }
-    totalBytesRead += bytesRead;
-    if (bytesRead == 0) {
-      break;
-    }
+  uint32_t totalBytesRead = 0;
+  nsresult rv = mResource.ReadAt(aPosition,
+                                 reinterpret_cast<char*>(aBuffer),
+                                 aLength,
+                                 &totalBytesRead);
+  if (NS_FAILED(rv)) {
+    return E_FAIL;
   }
   if (totalBytesRead > 0) {
     CriticalSectionAutoEnter lock(*mLock);

dom/media/encoder/OmxTrackEncoder.cpp

@@ -134,11 +134,8 @@ OmxVideoTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
     layers::Image* img = (!mLastFrame.GetImage() || mLastFrame.GetForceBlack())
                          ? nullptr : mLastFrame.GetImage();
     rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs,
-                          OMXCodecWrapper::BUFFER_EOS);
+                          OMXCodecWrapper::BUFFER_EOS, &mEosSetInEncoder);
     NS_ENSURE_SUCCESS(rv, rv);
-
-    // Keep sending EOS signal until OMXVideoEncoder gets it.
-    mEosSetInEncoder = true;
   }
 
   // Dequeue an encoded frame from the output buffers of OMXCodecWrapper.
@@ -219,6 +216,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
   PROFILER_LABEL("OmxAACAudioTrackEncoder", "GetEncodedTrack",
     js::ProfileEntry::Category::OTHER);
   AudioSegment segment;
+  bool EOS;
   // Move all the samples from mRawSegment to segment. We only hold
   // the monitor in this block.
   {
@@ -234,14 +232,15 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
     }
 
     segment.AppendFrom(&mRawSegment);
+    EOS = mEndOfStream;
   }
 
   nsresult rv;
   if (segment.GetDuration() == 0) {
     // Notify EOS at least once, even if segment is empty.
-    if (mEndOfStream && !mEosSetInEncoder) {
-      mEosSetInEncoder = true;
-      rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS);
+    if (EOS && !mEosSetInEncoder) {
+      rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS,
+                            &mEosSetInEncoder);
       NS_ENSURE_SUCCESS(rv, rv);
     }
     // Nothing to encode but encoder could still have encoded data for earlier
@@ -252,8 +251,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
   // OMX encoder has limited input buffers only so we have to feed input and get
   // output more than once if there are too many samples pending in segment.
   while (segment.GetDuration() > 0) {
-    rv = mEncoder->Encode(segment,
-                          mEndOfStream ? OMXCodecWrapper::BUFFER_EOS : 0);
+    rv = mEncoder->Encode(segment, 0);
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = AppendEncodedFrames(aData);

dom/media/encoder/TrackEncoder.cpp

@@ -123,9 +123,6 @@ AudioTrackEncoder::AppendAudioSegment(const AudioSegment& aSegment)
   return NS_OK;
 }
 
-static const int AUDIO_PROCESSING_FRAMES = 640; /* > 10ms of 48KHz audio */
-static const uint8_t gZeroChannel[MAX_AUDIO_SAMPLE_SIZE*AUDIO_PROCESSING_FRAMES] = {0};
-
 /*static*/
 void
 AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
@@ -133,19 +130,29 @@ AudioTrackEncoder::InterleaveTrackData(AudioChunk& aChunk,
                                        uint32_t aOutputChannels,
                                        AudioDataValue* aOutput)
 {
-  if (aChunk.mChannelData.Length() < aOutputChannels) {
-    // Up-mix. This might make the mChannelData have more than aChannels.
-    AudioChannelsUpMix(&aChunk.mChannelData, aOutputChannels, gZeroChannel);
-  }
-
-  if (aChunk.mChannelData.Length() > aOutputChannels) {
-    DownmixAndInterleave(aChunk.mChannelData, aChunk.mBufferFormat, aDuration,
-                         aChunk.mVolume, aOutputChannels, aOutput);
-  } else {
-    InterleaveAndConvertBuffer(aChunk.mChannelData.Elements(),
-                               aChunk.mBufferFormat, aDuration, aChunk.mVolume,
-                               aOutputChannels, aOutput);
-  }
+  switch(aChunk.mBufferFormat) {
+    case AUDIO_FORMAT_S16: {
+      nsAutoTArray<const int16_t*, 2> array;
+      array.SetLength(aOutputChannels);
+      for (uint32_t i = 0; i < array.Length(); i++) {
+        array[i] = static_cast<const int16_t*>(aChunk.mChannelData[i]);
+      }
+      InterleaveTrackData(array, aDuration, aOutputChannels, aOutput, aChunk.mVolume);
+      break;
+    }
+    case AUDIO_FORMAT_FLOAT32: {
+      nsAutoTArray<const float*, 2> array;
+      array.SetLength(aOutputChannels);
+      for (uint32_t i = 0; i < array.Length(); i++) {
+        array[i] = static_cast<const float*>(aChunk.mChannelData[i]);
+      }
+      InterleaveTrackData(array, aDuration, aOutputChannels, aOutput, aChunk.mVolume);
+      break;
+   }
+   case AUDIO_FORMAT_SILENCE: {
+      MOZ_ASSERT(false, "To implement.");
+    }
+  };
 }
 
 /*static*/

dom/media/encoder/TrackEncoder.h

@@ -151,6 +151,28 @@ public:
                                         uint32_t aTrackEvents,
                                         const MediaSegment& aQueuedMedia) override;
 
+  template<typename T>
+  static
+  void InterleaveTrackData(nsTArray<const T*>& aInput,
+                           int32_t aDuration,
+                           uint32_t aOutputChannels,
+                           AudioDataValue* aOutput,
+                           float aVolume)
+  {
+    if (aInput.Length() < aOutputChannels) {
+      // Up-mix. This might make the mChannelData have more than aChannels.
+      AudioChannelsUpMix(&aInput, aOutputChannels, SilentChannel::ZeroChannel<T>());
+    }
+
+    if (aInput.Length() > aOutputChannels) {
+      DownmixAndInterleave(aInput, aDuration,
+                           aVolume, aOutputChannels, aOutput);
+    } else {
+      InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
+                                 aOutputChannels, aOutput);
+    }
+  }
+
   /**
    * Interleaves the track data and stores the result into aOutput. Might need
    * to up-mix or down-mix the channel data if the channels number of this chunk

dom/media/encoder/VP8TrackEncoder.cpp

@@ -448,6 +448,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
 {
   PROFILER_LABEL("VP8TrackEncoder", "GetEncodedTrack",
     js::ProfileEntry::Category::OTHER);
+  bool EOS;
   {
     // Move all the samples from mRawSegment to mSourceSegment. We only hold
     // the monitor in this block.
@@ -463,6 +464,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
       return NS_ERROR_FAILURE;
     }
     mSourceSegment.AppendFrom(&mRawSegment);
+    EOS = mEndOfStream;
   }
 
   VideoSegment::ChunkIterator iter(mSourceSegment);
@@ -536,7 +538,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
   VP8LOG("RemoveLeading %lld\n",totalProcessedDuration);
 
   // End of stream, pull the rest frames in encoder.
-  if (mEndOfStream) {
+  if (EOS) {
     VP8LOG("mEndOfStream is true\n");
     mEncodingComplete = true;
     if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,

dom/media/fmp4/MP4Demuxer.cpp

@@ -46,8 +46,6 @@ public:
 
   virtual media::TimeIntervals GetBuffered() override;
 
-  virtual int64_t GetEvictionOffset(media::TimeUnit aTime) override;
-
   virtual void BreakCycles() override;
 
 private:
@@ -84,7 +82,7 @@ MP4Demuxer::Init()
 
   // Check that we have enough data to read the metadata.
   if (!mp4_demuxer::MP4Metadata::HasCompleteMetadata(stream)) {
-    return InitPromise::CreateAndReject(DemuxerFailureReason::WAITING_FOR_DATA, __func__);
+    return InitPromise::CreateAndReject(DemuxerFailureReason::DEMUXER_ERROR, __func__);
   }
 
   mInitData = mp4_demuxer::MP4Metadata::Metadata(stream);
@@ -106,22 +104,6 @@ MP4Demuxer::Init()
   return InitPromise::CreateAndResolve(NS_OK, __func__);
 }
 
-already_AddRefed<MediaDataDemuxer>
-MP4Demuxer::Clone() const
-{
-  nsRefPtr<MP4Demuxer> demuxer = new MP4Demuxer(mResource);
-  demuxer->mInitData = mInitData;
-  nsRefPtr<mp4_demuxer::BufferStream> bufferstream =
-    new mp4_demuxer::BufferStream(mInitData);
-  demuxer->mMetadata = MakeUnique<mp4_demuxer::MP4Metadata>(bufferstream);
-  if (!mMetadata->GetNumberTracks(mozilla::TrackInfo::kAudioTrack) &&
-      !mMetadata->GetNumberTracks(mozilla::TrackInfo::kVideoTrack)) {
-    NS_WARNING("Couldn't recreate MP4Demuxer");
-    return nullptr;
-  }
-  return demuxer.forget();
-}
-
 bool
 MP4Demuxer::HasTrackType(TrackInfo::TrackType aType) const
 {
@@ -161,7 +143,7 @@ MP4Demuxer::IsSeekable() const
 }
 
 void
-MP4Demuxer::NotifyDataArrived(uint32_t aLength, int64_t aOffset)
+MP4Demuxer::NotifyDataArrived()
 {
   for (uint32_t i = 0; i < mDemuxers.Length(); i++) {
     mDemuxers[i]->NotifyDataArrived();
@@ -369,15 +351,6 @@ MP4TrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold)
   }
 }
 
-int64_t
-MP4TrackDemuxer::GetEvictionOffset(media::TimeUnit aTime)
-{
-  EnsureUpToDateIndex();
-  MonitorAutoLock mon(mMonitor);
-  uint64_t offset = mIndex->GetEvictionOffset(aTime.ToMicroseconds());
-  return int64_t(offset == std::numeric_limits<uint64_t>::max() ? 0 : offset);
-}
-
 media::TimeIntervals
 MP4TrackDemuxer::GetBuffered()
 {

dom/media/fmp4/MP4Demuxer.h

@@ -29,8 +29,6 @@ public:
 
   virtual nsRefPtr<InitPromise> Init() override;
 
-  virtual already_AddRefed<MediaDataDemuxer> Clone() const override;
-
   virtual bool HasTrackType(TrackInfo::TrackType aType) const override;
 
   virtual uint32_t GetNumberTracks(TrackInfo::TrackType aType) const override;
@@ -42,7 +40,7 @@ public:
 
   virtual UniquePtr<EncryptionInfo> GetCrypto() override;
 
-  virtual void NotifyDataArrived(uint32_t aLength, int64_t aOffset) override;
+  virtual void NotifyDataArrived() override;
 
   virtual void NotifyDataRemoved() override;
 

dom/media/fmp4/MP4Stream.cpp

@@ -32,22 +32,17 @@ MP4Stream::BlockingReadIntoCache(int64_t aOffset, size_t aCount, Monitor* aToUnl
     return false;
   }
 
-  uint32_t sum = 0;
   uint32_t bytesRead = 0;
-  do {
-    uint64_t offset = aOffset + sum;
-    char* buffer = block.Buffer() + sum;
-    uint32_t toRead = aCount - sum;
+  {
     MonitorAutoUnlock unlock(*aToUnlock);
-    nsresult rv = mResource->ReadAt(offset, buffer, toRead, &bytesRead);
+    nsresult rv = mResource.ReadAt(aOffset, block.Buffer(), aCount, &bytesRead);
     if (NS_FAILED(rv)) {
       return false;
     }
-    sum += bytesRead;
-  } while (sum < aCount && bytesRead > 0);
+  }
 
-  MOZ_ASSERT(block.mCount >= sum);
-  block.mCount = sum;
+  MOZ_ASSERT(block.mCount >= bytesRead);
+  block.mCount = bytesRead;
 
   mCache.AppendElement(block);
   return true;
@@ -85,8 +80,9 @@ MP4Stream::CachedReadAt(int64_t aOffset, void* aBuffer, size_t aCount,
     }
   }
 
-  nsresult rv = mResource->ReadFromCache(reinterpret_cast<char*>(aBuffer),
-                                         aOffset, aCount);
+  nsresult rv =
+    mResource.GetResource()->ReadFromCache(reinterpret_cast<char*>(aBuffer),
+                                           aOffset, aCount);
   if (NS_FAILED(rv)) {
     *aBytesRead = 0;
     return false;
@@ -98,9 +94,9 @@ MP4Stream::CachedReadAt(int64_t aOffset, void* aBuffer, size_t aCount,
 bool
 MP4Stream::Length(int64_t* aSize)
 {
-  if (mResource->GetLength() < 0)
+  if (mResource.GetLength() < 0)
     return false;
-  *aSize = mResource->GetLength();
+  *aSize = mResource.GetLength();
   return true;
 }
 

dom/media/fmp4/MP4Stream.h

@@ -49,13 +49,13 @@ public:
 
   void Pin()
   {
-    mResource->Pin();
+    mResource.GetResource()->Pin();
     ++mPinCount;
   }
 
   void Unpin()
   {
-    mResource->Unpin();
+    mResource.GetResource()->Unpin();
     MOZ_ASSERT(mPinCount);
     --mPinCount;
     if (mPinCount == 0) {
@@ -64,7 +64,7 @@ public:
   }
 
 private:
-  nsRefPtr<MediaResource> mResource;
+  MediaResourceIndex mResource;
   Maybe<ReadRecord> mFailedRead;
   uint32_t mPinCount;
 

dom/media/gtest/MockMediaResource.h

@@ -32,17 +32,8 @@ public:
   }
   virtual void SetReadMode(MediaCacheStream::ReadMode aMode) override {}
   virtual void SetPlaybackRate(uint32_t aBytesPerSecond) override {}
-  virtual nsresult Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes)
-    override
-  {
-    return NS_OK;
-  }
   virtual nsresult ReadAt(int64_t aOffset, char* aBuffer, uint32_t aCount,
                           uint32_t* aBytes) override;
-  virtual nsresult Seek(int32_t aWhence, int64_t aOffset) override
-  {
-    return NS_OK;
-  }
   virtual int64_t Tell() override { return 0; }
   virtual void Pin() override {}
   virtual void Unpin() override {}

dom/media/gtest/TestMediaEventSource.cpp

@@ -6,6 +6,7 @@
 #include "gtest/gtest.h"
 
 #include "mozilla/TaskQueue.h"
+#include "mozilla/UniquePtr.h"
 #include "MediaEventSource.h"
 #include "VideoUtils.h"
 
@@ -294,3 +295,29 @@ TEST(MediaEventSource, CopyEvent2)
   listener1.Disconnect();
   listener2.Disconnect();
 }
+
+/*
+ * Test move-only types.
+ */
+TEST(MediaEventSource, MoveOnly)
+{
+  nsRefPtr<TaskQueue> queue = new TaskQueue(
+    GetMediaThreadPool(MediaThreadType::PLAYBACK));
+
+  MediaEventProducer<UniquePtr<int>, ListenerMode::Exclusive> source;
+
+  auto func = [] (UniquePtr<int>&& aEvent) {
+    EXPECT_EQ(*aEvent, 20);
+  };
+  MediaEventListener listener = source.Connect(queue, func);
+
+  // It is OK to pass an rvalue which is move-only.
+  source.Notify(UniquePtr<int>(new int(20)));
+  // It is an error to pass an lvalue which is move-only.
+  // UniquePtr<int> event(new int(30));
+  // source.Notify(event);
+
+  queue->BeginShutdown();
+  queue->AwaitShutdownAndIdle();
+  listener.Disconnect();
+}

dom/media/mediasink/AudioSink.h

@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(AudioSink_h__)
+#define AudioSink_h__
+
+#include "mozilla/MozPromise.h"
+#include "mozilla/nsRefPtr.h"
+#include "nsISupportsImpl.h"
+
+namespace mozilla {
+
+class MediaData;
+template <class T> class MediaQueue;
+
+namespace media {
+
+/*
+ * Define basic APIs for derived class instance to operate or obtain
+ * information from it.
+ */
+class AudioSink {
+public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioSink)
+  AudioSink(MediaQueue<MediaData>& aAudioQueue)
+    : mAudioQueue(aAudioQueue)
+  {}
+
+  // Return a promise which will be resolved when AudioSink finishes playing,
+  // or rejected if any error.
+  virtual nsRefPtr<GenericPromise> Init() = 0;
+
+  virtual int64_t GetEndTime() const = 0;
+  virtual int64_t GetPosition() = 0;
+
+  // Check whether we've pushed more frames to the audio
+  // hardware than it has played.
+  virtual bool HasUnplayedFrames() = 0;
+
+  // Shut down the AudioSink's resources.
+  virtual void Shutdown() = 0;
+
+  // Change audio playback setting.
+  virtual void SetVolume(double aVolume) = 0;
+  virtual void SetPlaybackRate(double aPlaybackRate) = 0;
+  virtual void SetPreservesPitch(bool aPreservesPitch) = 0;
+
+  // Change audio playback status pause/resume.
+  virtual void SetPlaying(bool aPlaying) = 0;
+
+protected:
+  virtual ~AudioSink() {}
+
+  virtual MediaQueue<MediaData>& AudioQueue() const {
+    return mAudioQueue;
+  }
+
+  // To queue audio data (no matter it's plain or encoded or encrypted, depends
+  // on the subclass)
+  MediaQueue<MediaData>& mAudioQueue;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif

dom/media/mediasink/AudioSinkWrapper.cpp

@@ -0,0 +1,239 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AudioSink.h"
+#include "AudioSinkWrapper.h"
+
+namespace mozilla {
+namespace media {
+
+AudioSinkWrapper::~AudioSinkWrapper()
+{
+}
+
+void
+AudioSinkWrapper::Shutdown()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped.");
+  mCreator = nullptr;
+}
+
+const MediaSink::PlaybackParams&
+AudioSinkWrapper::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+  return mParams;
+}
+
+void
+AudioSinkWrapper::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+  if (mAudioSink) {
+    mAudioSink->SetVolume(aParams.volume);
+    mAudioSink->SetPlaybackRate(aParams.playbackRate);
+    mAudioSink->SetPreservesPitch(aParams.preservesPitch);
+  }
+  mParams = aParams;
+}
+
+nsRefPtr<GenericPromise>
+AudioSinkWrapper::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType == TrackInfo::kAudioTrack) {
+    return mEndPromise;
+  }
+  return nullptr;
+}
+
+int64_t
+AudioSinkWrapper::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+  if (aType == TrackInfo::kAudioTrack && mAudioSink) {
+    return mAudioSink->GetEndTime();
+  }
+  return -1;
+}
+
+int64_t
+AudioSinkWrapper::GetVideoPosition(TimeStamp aNow) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mPlayStartTime.IsNull());
+  // Time elapsed since we started playing.
+  int64_t delta = (aNow - mPlayStartTime).ToMicroseconds();
+  // Take playback rate into account.
+  return mPlayDuration + delta * mParams.playbackRate;
+}
+
+int64_t
+AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
+
+  int64_t pos = -1;
+  TimeStamp t = TimeStamp::Now();
+
+  if (!mAudioEnded) {
+    // Rely on the audio sink to report playback position when it is not ended.
+    pos = mAudioSink->GetPosition();
+  } else if (!mPlayStartTime.IsNull()) {
+    // Calculate playback position using system clock if we are still playing.
+    pos = GetVideoPosition(t);
+  } else {
+    // Return how long we've played if we are not playing.
+    pos = mPlayDuration;
+  }
+
+  if (aTimeStamp) {
+    *aTimeStamp = t;
+  }
+
+  return pos;
+}
+
+bool
+AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const
+{
+  AssertOwnerThread();
+  return mAudioSink ? mAudioSink->HasUnplayedFrames() : false;
+}
+
+void
+AudioSinkWrapper::SetVolume(double aVolume)
+{
+  AssertOwnerThread();
+  mParams.volume = aVolume;
+  if (mAudioSink) {
+    mAudioSink->SetVolume(aVolume);
+  }
+}
+
+void
+AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+  mParams.playbackRate = aPlaybackRate;
+  if (!mAudioEnded) {
+    // Pass the playback rate to the audio sink. The underlying AudioStream
+    // will handle playback rate changes and report correct audio position.
+    mAudioSink->SetPlaybackRate(aPlaybackRate);
+  } else if (!mPlayStartTime.IsNull()) {
+    // Adjust playback duration and start time when we are still playing.
+    TimeStamp now = TimeStamp::Now();
+    mPlayDuration = GetVideoPosition(now);
+    mPlayStartTime = now;
+  }
+  // Do nothing when not playing. Changes in playback rate will be taken into
+  // account by GetVideoPosition().
+}
+
+void
+AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch)
+{
+  AssertOwnerThread();
+  mParams.preservesPitch = aPreservesPitch;
+  if (mAudioSink) {
+    mAudioSink->SetPreservesPitch(aPreservesPitch);
+  }
+}
+
+void
+AudioSinkWrapper::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+
+  // Resume/pause matters only when playback started.
+  if (!mIsStarted) {
+    return;
+  }
+
+  if (mAudioSink) {
+    mAudioSink->SetPlaying(aPlaying);
+  }
+
+  if (aPlaying) {
+    MOZ_ASSERT(mPlayStartTime.IsNull());
+    mPlayStartTime = TimeStamp::Now();
+  } else {
+    // Remember how long we've played.
+    mPlayDuration = GetPosition();
+    // mPlayStartTime must be updated later since GetPosition()
+    // depends on the value of mPlayStartTime.
+    mPlayStartTime = TimeStamp();
+  }
+}
+
+void
+AudioSinkWrapper::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mIsStarted, "playback already started.");
+
+  mIsStarted = true;
+  mPlayDuration = aStartTime;
+  mPlayStartTime = TimeStamp::Now();
+
+  // no audio is equivalent to audio ended before video starts.
+  mAudioEnded = !aInfo.HasAudio();
+
+  if (aInfo.HasAudio()) {
+    mAudioSink = mCreator->Create();
+    mEndPromise = mAudioSink->Init();
+    SetPlaybackParams(mParams);
+
+    mAudioSinkPromise.Begin(mEndPromise->Then(
+      mOwnerThread.get(), __func__, this,
+      &AudioSinkWrapper::OnAudioEnded,
+      &AudioSinkWrapper::OnAudioEnded));
+  }
+}
+
+void
+AudioSinkWrapper::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mIsStarted, "playback not started.");
+
+  mIsStarted = false;
+  mAudioEnded = true;
+
+  if (mAudioSink) {
+    mAudioSinkPromise.DisconnectIfExists();
+    mAudioSink->Shutdown();
+    mAudioSink = nullptr;
+    mEndPromise = nullptr;
+  }
+}
+
+bool
+AudioSinkWrapper::IsStarted() const
+{
+  AssertOwnerThread();
+  return mIsStarted;
+}
+
+void
+AudioSinkWrapper::OnAudioEnded()
+{
+  AssertOwnerThread();
+  mAudioSinkPromise.Complete();
+  mPlayDuration = GetPosition();
+  if (!mPlayStartTime.IsNull()) {
+    mPlayStartTime = TimeStamp::Now();
+  }
+  mAudioEnded = true;
+}
+
+} // namespace media
+} // namespace mozilla
+

dom/media/mediasink/AudioSinkWrapper.h

@@ -0,0 +1,107 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AudioSinkWrapper_h_
+#define AudioSinkWrapper_h_
+
+#include "mozilla/AbstractThread.h"
+#include "mozilla/dom/AudioChannelBinding.h"
+#include "mozilla/nsRefPtr.h"
+#include "mozilla/TimeStamp.h"
+#include "mozilla/UniquePtr.h"
+
+#include "MediaSink.h"
+
+namespace mozilla {
+
+class MediaData;
+template <class T> class MediaQueue;
+
+namespace media {
+
+class AudioSink;
+
+/**
+ * A wrapper around AudioSink to provide the interface of MediaSink.
+ */
+class AudioSinkWrapper : public MediaSink {
+  // An AudioSink factory.
+  class Creator {
+  public:
+    virtual ~Creator() {}
+    virtual AudioSink* Create() = 0;
+  };
+
+  // Wrap around a function object which creates AudioSinks.
+  template <typename Function>
+  class CreatorImpl : public Creator {
+  public:
+    explicit CreatorImpl(const Function& aFunc) : mFunction(aFunc) {}
+    AudioSink* Create() override { return mFunction(); }
+  private:
+    Function mFunction;
+  };
+
+public:
+  template <typename Function>
+  AudioSinkWrapper(AbstractThread* aOwnerThread, const Function& aFunc)
+    : mOwnerThread(aOwnerThread)
+    , mCreator(new CreatorImpl<Function>(aFunc))
+    , mIsStarted(false)
+    // Give an insane value to facilitate debug if used before playback starts.
+    , mPlayDuration(INT64_MAX)
+    , mAudioEnded(true)
+  {}
+
+  const PlaybackParams& GetPlaybackParams() const override;
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  nsRefPtr<GenericPromise> OnEnded(TrackType aType) override;
+  int64_t GetEndTime(TrackType aType) const override;
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+  bool HasUnplayedFrames(TrackType aType) const override;
+
+  void SetVolume(double aVolume) override;
+  void SetPlaybackRate(double aPlaybackRate) override;
+  void SetPreservesPitch(bool aPreservesPitch) override;
+  void SetPlaying(bool aPlaying) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+  void Stop() override;
+  bool IsStarted() const override;
+
+  void Shutdown() override;
+
+private:
+  virtual ~AudioSinkWrapper();
+
+  void AssertOwnerThread() const {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  int64_t GetVideoPosition(TimeStamp aNow) const;
+
+  void OnAudioEnded();
+
+  const nsRefPtr<AbstractThread> mOwnerThread;
+  UniquePtr<Creator> mCreator;
+  nsRefPtr<AudioSink> mAudioSink;
+  nsRefPtr<GenericPromise> mEndPromise;
+
+  bool mIsStarted;
+  PlaybackParams mParams;
+
+  TimeStamp mPlayStartTime;
+  int64_t mPlayDuration;
+
+  bool mAudioEnded;
+  MozPromiseRequestHolder<GenericPromise> mAudioSinkPromise;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif //AudioSinkWrapper_h_