import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1192694: remove mDecodingFrozenAtStateDecoding so decoder can leave dormant normally. r=jwwang (459e69fc5) - bug 962719 use unsigned ints for FrameID and ProducerID for defined overflow behavior r=roc (d5961e6a7) - Bug 1191192 - Add DecodedStream::SetSameOrigin(). r=roc. (fa76795e9) - Bug 1179451 - Part 4: Don't pass nsRefPtr&& to functions that want raw pointers. r=froydnj (2a7893a33) - Bug 1191696. Part 1 - Handle the promise returned by DecodedStream::StartPlayback in MDSM. r=roc. (56d7fc3ed) - Bug 1191696. Part 2 - implementation in DecodedStream. r=roc. (125f834f8) - Bug 1188268 - Make AudioSink a base class, create DecodedAudioDataSink to act as original AudioSink, and move sink-related files to dom/media/mediasink. r=jwwang, r=cpearce (be9858883) - Bug 1188268 - Correct the logic against |MaybeStartPlayback()| due to rebase mistake. r=jwwang (2d8118ab6) - apparently missing bit of Bug 1163467. Part 1 (baef792d7) - Bug 1195187. Part 1 - Move output stream connection/disconnection code to OutputStreamData. r=roc. (2f57da7c6) - Bug 1195187. Part 2 - add a new class OutputStreamManager for managing output streams. r=roc. (c30031436) - and some local fixes
2026-05-26 14:30:27 +00:00 · 2021-10-01 13:32:06 +08:00
parent 036a7138f8
commit 0e2ad8b8aa
27 changed files with 747 additions and 542 deletions
@@ -14,7 +14,7 @@

 #include "MediaDecoderStateMachine.h"
 #include "MediaTimer.h"
-#include "AudioSink.h"
+#include "mediasink/DecodedAudioDataSink.h"
 #include "nsTArray.h"
 #include "MediaDecoder.h"
 #include "MediaDecoderReader.h"
@@ -39,6 +39,7 @@
 #include "nsPrintfCString.h"
 #include "DOMMediaStream.h"
 #include "DecodedStream.h"
+#include "mozilla/Logging.h"

 #include <algorithm>

@@ -216,7 +217,6 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
  mCorruptFrames(30),
  mDecodingFirstFrame(true),
  mDisabledHardwareAcceleration(false),
-  mDecodingFrozenAtStateDecoding(false),
  mSentLoadedMetadataEvent(false),
  mSentFirstFrameLoadedEvent(false),
  mSentPlaybackEndedEvent(false),
@@ -239,6 +239,8 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
                       "MediaDecoderStateMachine::mLogicalPlaybackRate (Mirror)"),
  mPreservesPitch(mTaskQueue, true,
                  "MediaDecoderStateMachine::mPreservesPitch (Mirror)"),
+  mSameOriginMedia(mTaskQueue, false,
+                   "MediaDecoderStateMachine::mSameOriginMedia (Mirror)"),
  mDuration(mTaskQueue, NullableTimeUnit(),
            "MediaDecoderStateMachine::mDuration (Canonical"),
  mIsShutdown(mTaskQueue, false,
@@ -278,17 +280,10 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
  timeBeginPeriod(1);
 #endif

-  nsRefPtr<MediaDecoderStateMachine> self = this;
-
-  AudioQueue().AddPopListener(
-    [self] (const MediaData* aSample) {
-      self->OnAudioPopped(aSample->As<AudioData>());
-     }, mTaskQueue);
-
-  VideoQueue().AddPopListener(
-    [self] (const MediaData* aSample) {
-      self->OnVideoPopped(aSample->As<VideoData>());
-    }, mTaskQueue);
+  mAudioQueueListener = AudioQueue().PopEvent().Connect(
+    mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped);
+  mVideoQueueListener = VideoQueue().PopEvent().Connect(
+    mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped);
 }

 MediaDecoderStateMachine::~MediaDecoderStateMachine()
@@ -318,6 +313,7 @@ MediaDecoderStateMachine::InitializationTask()
  mVolume.Connect(mDecoder->CanonicalVolume());
  mLogicalPlaybackRate.Connect(mDecoder->CanonicalPlaybackRate());
  mPreservesPitch.Connect(mDecoder->CanonicalPreservesPitch());
+  mSameOriginMedia.Connect(mDecoder->CanonicalSameOriginMedia());

  // Initialize watchers.
  mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated);
@@ -331,6 +327,7 @@ MediaDecoderStateMachine::InitializationTask()
  mWatchManager.Watch(mObservedDuration, &MediaDecoderStateMachine::RecomputeDuration);
  mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged);
  mWatchManager.Watch(mLogicallySeeking, &MediaDecoderStateMachine::LogicallySeekingChanged);
+  mWatchManager.Watch(mSameOriginMedia, &MediaDecoderStateMachine::SameOriginMediaChanged);
 }

 bool MediaDecoderStateMachine::HasFutureAudio()
@@ -374,7 +371,7 @@ void MediaDecoderStateMachine::SendStreamData()
  AssertCurrentThreadInMonitor();
  MOZ_ASSERT(!mAudioSink, "Should've been stopped in RunStateMachine()");

-  bool finished = mDecodedStream->SendData(mVolume, mDecoder->IsSameOriginMedia());
+  mDecodedStream->SendData();

  const auto clockTime = GetClock();
  while (true) {
@@ -390,12 +387,6 @@ void MediaDecoderStateMachine::SendStreamData()
    }
    break;
  }
-
-  // To be consistent with AudioSink, |mAudioCompleted| is not set
-  // until all samples are drained.
-  if (finished && AudioQueue().GetSize() == 0) {
-    mAudioCompleted = true;
-  }
 }

 bool MediaDecoderStateMachine::HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs)
@@ -659,10 +650,6 @@ MediaDecoderStateMachine::Push(VideoData* aSample)
  VideoQueue().Push(aSample);
  UpdateNextFrameStatus();
  DispatchDecodeTasksIfNeeded();
-
-  if (mAudioSink) {
-    mAudioSink->NotifyData();
-  }
 }

 void
@@ -677,7 +664,7 @@ MediaDecoderStateMachine::PushFront(VideoData* aSample)
 }

 void
-MediaDecoderStateMachine::OnAudioPopped(const AudioData* aSample)
+MediaDecoderStateMachine::OnAudioPopped(const nsRefPtr<MediaData>& aSample)
 {
  MOZ_ASSERT(OnTaskQueue());
  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
@@ -687,16 +674,13 @@ MediaDecoderStateMachine::OnAudioPopped(const AudioData* aSample)
 }

 void
-MediaDecoderStateMachine::OnVideoPopped(const VideoData* aSample)
+MediaDecoderStateMachine::OnVideoPopped(const nsRefPtr<MediaData>& aSample)
 {
  MOZ_ASSERT(OnTaskQueue());
  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
  mDecoder->UpdatePlaybackOffset(aSample->mOffset);
  UpdateNextFrameStatus();
  DispatchVideoDecodeTaskIfNeeded();
-  // Notify the decode thread that the video queue's buffers may have
-  // free'd up space for more frames.
-  mDecoder->GetReentrantMonitor().NotifyAll();
 }

 void
@@ -777,11 +761,6 @@ MediaDecoderStateMachine::OnNotDecoded(MediaData::Type aType,
        return;
      }
      CheckIfDecodeComplete();
-      mDecoder->GetReentrantMonitor().NotifyAll();
-      // Tell AudioSink to wake up for audio queue is finished.
-      if (mAudioSink) {
-        mAudioSink->NotifyData();
-      }
      // Schedule the state machine to notify track ended as soon as possible.
      if (mAudioCaptured) {
        ScheduleStateMachine();
@@ -1034,7 +1013,7 @@ nsresult MediaDecoderStateMachine::Init(MediaDecoderStateMachine* aCloneDonor)

  nsresult rv = mReader->Init(cloneReader);
  NS_ENSURE_SUCCESS(rv, rv);
-
+  ScheduleStateMachineCrossThread();
  return NS_OK;
 }

@@ -1088,15 +1067,9 @@ void MediaDecoderStateMachine::MaybeStartPlayback()
  SetPlayStartTime(TimeStamp::Now());
  MOZ_ASSERT(IsPlaying());

-  StartAudioThread();
+  StartAudioSink();
+  StartDecodedStream();

-  // Tell DecodedStream to start playback with specified start time and media
-  // info. This is consistent with how we create AudioSink in StartAudioThread().
-  if (mAudioCaptured) {
-    mDecodedStream->StartPlayback(GetMediaTime(), mInfo);
-  }
-
-  mDecoder->GetReentrantMonitor().NotifyAll();
  DispatchDecodeTasksIfNeeded();
 }

@@ -1172,6 +1145,7 @@ void MediaDecoderStateMachine::VolumeChanged()
  if (mAudioSink) {
    mAudioSink->SetVolume(mVolume);
  }
+  mDecodedStream->SetVolume(mVolume);
 }

 void MediaDecoderStateMachine::RecomputeDuration()
@@ -1259,13 +1233,10 @@ void MediaDecoderStateMachine::SetDormant(bool aDormant)
    // it here as well.
    nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethod(mReader, &MediaDecoderReader::ReleaseMediaResources);
    DecodeTaskQueue()->Dispatch(r.forget());
-    mDecoder->GetReentrantMonitor().NotifyAll();
  } else if ((aDormant != true) && (mState == DECODER_STATE_DORMANT)) {
-    mDecodingFrozenAtStateDecoding = true;
    ScheduleStateMachine();
    mDecodingFirstFrame = true;
    SetState(DECODER_STATE_DECODING_NONE);
-    mDecoder->GetReentrantMonitor().NotifyAll();
  }
 }

@@ -1386,11 +1357,6 @@ void MediaDecoderStateMachine::PlayStateChanged()
    DispatchDecodeTasksIfNeeded();
  }

-  if (mDecodingFrozenAtStateDecoding) {
-    mDecodingFrozenAtStateDecoding = false;
-    DispatchDecodeTasksIfNeeded();
-  }
-
  // Some state transitions still happen synchronously on the main thread. So
  // if the main thread invokes Play() and then Seek(), the seek will initiate
  // synchronously on the main thread, and the asynchronous PlayInternal task
@@ -1422,6 +1388,13 @@ void MediaDecoderStateMachine::LogicallySeekingChanged()
  ScheduleStateMachine();
 }

+void MediaDecoderStateMachine::SameOriginMediaChanged()
+{
+  MOZ_ASSERT(OnTaskQueue());
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  mDecodedStream->SetSameOrigin(mSameOriginMedia);
+}
+
 void MediaDecoderStateMachine::BufferedRangeUpdated()
 {
  MOZ_ASSERT(OnTaskQueue());
@@ -1446,8 +1419,6 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
  MOZ_ASSERT(OnTaskQueue());
  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());

-  mDecodingFrozenAtStateDecoding = false;
-
  if (IsShutdown()) {
    return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
  }
@@ -1480,7 +1451,7 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
  return mPendingSeek.mPromise.Ensure(__func__);
 }

-void MediaDecoderStateMachine::StopAudioThread()
+void MediaDecoderStateMachine::StopAudioSink()
 {
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
@@ -1505,11 +1476,6 @@ MediaDecoderStateMachine::DispatchDecodeTasksIfNeeded()
    return;
  }

-  if (mState == DECODER_STATE_DECODING && mDecodingFrozenAtStateDecoding) {
-    DECODER_LOG("DispatchDecodeTasksIfNeeded return due to "
-                "mFreezeDecodingAtStateDecoding");
-    return;
-  }
  // NeedToDecodeAudio() can go from false to true while we hold the
  // monitor, but it can't go from true to false. This can happen because
  // NeedToDecodeAudio() takes into account the amount of decoded audio
@@ -1778,7 +1744,7 @@ MediaDecoderStateMachine::RequestVideoData()
 }

 void
-MediaDecoderStateMachine::StartAudioThread()
+MediaDecoderStateMachine::StartAudioSink()
 {
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
@@ -1789,9 +1755,9 @@ MediaDecoderStateMachine::StartAudioThread()

  if (HasAudio() && !mAudioSink) {
    mAudioCompleted = false;
-    mAudioSink = new AudioSink(mAudioQueue,
-                               GetMediaTime(), mInfo.mAudio,
-                               mDecoder->GetAudioChannel());
+    mAudioSink = new DecodedAudioDataSink(mAudioQueue,
+                                          GetMediaTime(), mInfo.mAudio,
+                                          mDecoder->GetAudioChannel());

    mAudioSinkPromise.Begin(
      mAudioSink->Init()->Then(
@@ -1805,6 +1771,32 @@ MediaDecoderStateMachine::StartAudioThread()
  }
 }

+void
+MediaDecoderStateMachine::StopDecodedStream()
+{
+  MOZ_ASSERT(OnTaskQueue());
+  AssertCurrentThreadInMonitor();
+  mDecodedStream->StopPlayback();
+  mDecodedStreamPromise.DisconnectIfExists();
+}
+
+void
+MediaDecoderStateMachine::StartDecodedStream()
+{
+  MOZ_ASSERT(OnTaskQueue());
+  AssertCurrentThreadInMonitor();
+
+  // Tell DecodedStream to start playback with specified start time and media
+  // info. This is consistent with how we create AudioSink in StartAudioThread().
+  if (mAudioCaptured && !mDecodedStreamPromise.Exists()) {
+    mDecodedStreamPromise.Begin(
+      mDecodedStream->StartPlayback(GetMediaTime(), mInfo)->Then(
+        OwnerThread(), __func__, this,
+        &MediaDecoderStateMachine::OnDecodedStreamFinish,
+        &MediaDecoderStateMachine::OnDecodedStreamError));
+  }
+}
+
 int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
 {
  MOZ_ASSERT(OnTaskQueue());
@@ -1906,10 +1898,6 @@ MediaDecoderStateMachine::DecodeError()
  ScheduleStateMachine();
  DECODER_WARN("Decode error, changed state to ERROR");

-  // Is anybody actually waiting on this monitor, or is it just
-  // a leftover from when we used to do sync dispatch for the below?
-  mDecoder->GetReentrantMonitor().NotifyAll();
-
  // MediaDecoder::DecodeError notifies the owner, and then shuts down the state
  // machine.
  nsCOMPtr<nsIRunnable> event =
@@ -2203,8 +2191,10 @@ MediaDecoderStateMachine::FinishShutdown()
  // The reader's listeners hold references to the state machine,
  // creating a cycle which keeps the state machine and its shared
  // thread pools alive. So break it here.
-  AudioQueue().ClearListeners();
-  VideoQueue().ClearListeners();
+
+  // Prevent dangling pointers by disconnecting the listeners.
+  mAudioQueueListener.Disconnect();
+  mVideoQueueListener.Disconnect();

  // Disconnect canonicals and mirrors before shutting down our task queue.
  mBuffered.DisconnectIfConnected();
@@ -2216,6 +2206,7 @@ MediaDecoderStateMachine::FinishShutdown()
  mVolume.DisconnectIfConnected();
  mLogicalPlaybackRate.DisconnectIfConnected();
  mPreservesPitch.DisconnectIfConnected();
+  mSameOriginMedia.DisconnectIfConnected();
  mDuration.DisconnectAll();
  mIsShutdown.DisconnectAll();
  mNextFrameStatus.DisconnectAll();
@@ -2350,8 +2341,6 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
      DECODER_LOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds());
      StartDecoding();

-      // Notify to allow blocked decoder thread to continue
-      mDecoder->GetReentrantMonitor().NotifyAll();
      NS_ASSERTION(IsStateMachineScheduled(), "Must have timer scheduled");
      return NS_OK;
    }
@@ -2407,12 +2396,12 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
        AbstractThread::MainThread()->Dispatch(event.forget());

        mSentPlaybackEndedEvent = true;
-      }

-      // Stop audio sink after call to AudioEndTime() above, otherwise it will
-      // return an incorrect value due to a null mAudioSink.
-      StopAudioThread();
-      mDecodedStream->StopPlayback();
+        // Stop audio sink after call to AudioEndTime() above, otherwise it will
+        // return an incorrect value due to a null mAudioSink.
+        StopAudioSink();
+        StopDecodedStream();
+      }

      return NS_OK;
    }
@@ -2440,8 +2429,8 @@ MediaDecoderStateMachine::Reset()
  // Stop the audio thread. Otherwise, AudioSink might be accessing AudioQueue
  // outside of the decoder monitor while we are clearing the queue and causes
  // crash for no samples to be popped.
-  StopAudioThread();
-  mDecodedStream->StopPlayback();
+  StopAudioSink();
+  StopDecodedStream();

  mVideoFrameEndTime = -1;
  mDecodedVideoEndTime = -1;
@@ -2487,6 +2476,7 @@ void MediaDecoderStateMachine::CheckTurningOffHardwareDecoder(VideoData* aData)
          NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration);
        DecodeTaskQueue()->Dispatch(task.forget());
      mDisabledHardwareAcceleration = true;
+      gfxCriticalNote << "Too many dropped/corrupted frames, disabling DXVA";
    }
  } else {
    mCorruptFrames.insert(0);
@@ -2541,7 +2531,7 @@ void MediaDecoderStateMachine::RenderVideoFrames(int32_t aMaxFrames,
    img->mFrameID = frame->mFrameID;
    img->mProducerID = mProducerID;

-    VERBOSE_LOG("playing video frame %lld (id=%d) (queued=%i, state-machine=%i, decoder-queued=%i)",
+    VERBOSE_LOG("playing video frame %lld (id=%x) (queued=%i, state-machine=%i, decoder-queued=%i)",
                frame->mTime, frame->mFrameID,
                VideoQueue().GetSize() + mReader->SizeOfVideoQueueInFrames(),
                VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames());
@@ -3057,7 +3047,7 @@ MediaDecoderStateMachine::AudioEndTime() const
    return mDecodedStream->AudioEndTime();
  }
  // Don't call this after mAudioSink becomes null since we can't distinguish
-  // "before StartAudioThread" and "after StopAudioThread" where mAudioSink
+  // "before StartAudioSink" and "after StopAudioSink" where mAudioSink
  // is null in both cases.
  MOZ_ASSERT(!mAudioCompleted);
  return -1;
@@ -3072,9 +3062,6 @@ void MediaDecoderStateMachine::OnAudioSinkComplete()
  mAudioSinkPromise.Complete();
  ResyncAudioClock();
  mAudioCompleted = true;
-
-  // Kick the decode thread; it may be sleeping waiting for this to finish.
-  mDecoder->GetReentrantMonitor().NotifyAll();
 }

 void MediaDecoderStateMachine::OnAudioSinkError()
@@ -3097,9 +3084,35 @@ void MediaDecoderStateMachine::OnAudioSinkError()
  DecodeError();
 }

+void
+MediaDecoderStateMachine::OnDecodedStreamFinish()
+{
+  MOZ_ASSERT(OnTaskQueue());
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
+
+  mDecodedStreamPromise.Complete();
+  if (mInfo.HasAudio()) {
+    mAudioCompleted = true;
+  }
+  // To notify PlaybackEnded as soon as possible.
+  ScheduleStateMachine();
+}
+
+void
+MediaDecoderStateMachine::OnDecodedStreamError()
+{
+  MOZ_ASSERT(OnTaskQueue());
+  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
+  MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
+
+  mDecodedStreamPromise.Complete();
+  DecodeError();
+}

 uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
 {
+  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
  return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated())
    ? std::max<uint32_t>(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE)
@@ -3115,12 +3128,12 @@ void MediaDecoderStateMachine::DispatchAudioCaptured()
    ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
    if (!self->mAudioCaptured) {
      // Stop the audio sink if it's running.
-      self->StopAudioThread();
+      self->StopAudioSink();
      self->mAudioCaptured = true;
      // Start DecodedStream if we are already playing. Otherwise it will be
      // handled in MaybeStartPlayback().
      if (self->IsPlaying()) {
-        self->mDecodedStream->StartPlayback(self->GetMediaTime(), self->mInfo);
+        self->StartDecodedStream();
      }
      self->ScheduleStateMachine();
    }
@@ -3136,10 +3149,11 @@ void MediaDecoderStateMachine::DispatchAudioUncaptured()
    MOZ_ASSERT(self->OnTaskQueue());
    ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
    if (self->mAudioCaptured) {
-      // Start again the audio sink
+      self->StopDecodedStream();
+      // Start again the audio sink.
      self->mAudioCaptured = false;
      if (self->IsPlaying()) {
-        self->StartAudioThread();
+        self->StartAudioSink();
      }
      self->ScheduleStateMachine();
    }