import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1195185. Part 1 - rename Connect/Remove to AddOutput/RemoveOutput. r=roc. (549b92cb4) - Bug 1195185. Part 2 - Check if all output streams are from the same graph because we don't support connecting streams from different graphs. r=roc. (6b66501ed) - Bug 1195185. Part 3 - align the life cycle of mData with {Start,Stop}Playback. r=roc. (90f9a6fce) - Bug 1195185. Part 4 - remove dead code. r=roc. (1bcfb5837) - Bug 1195632. Part 1 - Let DecodedStream have a worker thread and asset some funtions on the worker thread. r=roc. (5987408e8) - Bug 1195632. Part 2 - Have DecodedStream listen to push events of the media queues and call SendData() on its own without the help of MDSM. r=roc. (d082c7c8a) - Bug 1198568 - Fix build error in non-unified build for DecodedAudioDataSink.cpp. r=kinetik. (dc1289a89) - Bug 1199104. Part 1 - create MediaSink. r=kinetik. (29657636b) - Bug 1199104. Part 2 - create AudioSinkWrapper. r=kinetik. (2a970c2a6) - Bug 1199104. Part 3 - use AudioSinkWrapper in MDSM. r=kinetik. (d31f8986d) - Bug 1195158. Part 1 - Have MediaMetadataManager listen to an event source to receive TimedMetadata events. OggReader will send TimedMetadata events through an event source. This will break OggReader's dependency on AbstractMediaDecoder::QueueMetadata which then can be removed for it is against our goal to run all MediaDecoder's methods on the main thread. r=cpearce. (4f2df907d) - Bug 1195158. Part 2 - Have OggReader send TimedMetadata events through a event source instead of direct calls to AbstractMediaDecoder::QueueMetadata. r=cpearce. (cce94d58a) - Bug 1195158. Part 3 - connect listeners. r=cpearce. (60bcc5f8b) - Bug 1195158. Part 4 - remove unused code. r=cpearce. (072432eef) - Bug 1195158. Part 5 - 1. Fix insufficient includes and sort out include order. 2. Only disconnect |mTimedMetadataListener| when the state machine is created. r=cpearce. (d552e56a8) - Bug 1204407: P1. Remove no longer used mainthread object. r=cpearce (30f6b1c86) - Bug 1185792: [webm] P1. Don't clear mNeedReIndex if GetCachedRanges is emtpy. r=jya (3169897d5) - Bug 1194884: [webm] P1. Use MediaResourceIndex. r=j^ (6dbe08ddf) - Bug 1194884: [webm] P2. Retrieve all VPX decoded frames from decoder. r=j^ (060727eca)
2026-05-26 14:18:48 +00:00 · 2021-10-05 09:55:30 +08:00
parent 6734bafa19
commit c80d3c1b1f
25 changed files with 726 additions and 388 deletions
@@ -15,6 +15,7 @@
 #include "MediaDecoderStateMachine.h"
 #include "MediaTimer.h"
 #include "mediasink/DecodedAudioDataSink.h"
+#include "mediasink/AudioSinkWrapper.h"
 #include "nsTArray.h"
 #include "MediaDecoder.h"
 #include "MediaDecoderReader.h"
@@ -220,7 +221,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
  mSentLoadedMetadataEvent(false),
  mSentFirstFrameLoadedEvent(false),
  mSentPlaybackEndedEvent(false),
-  mDecodedStream(new DecodedStream(mAudioQueue, mVideoQueue)),
+  mDecodedStream(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)),
  mResource(aDecoder->GetResource()),
  mBuffered(mTaskQueue, TimeIntervals(),
            "MediaDecoderStateMachine::mBuffered (Mirror)"),
@@ -284,6 +285,17 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
    mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped);
  mVideoQueueListener = VideoQueue().PopEvent().Connect(
    mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped);
+
+  mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
+
+  nsRefPtr<MediaDecoderStateMachine> self = this;
+  auto audioSinkCreator = [self] () {
+    MOZ_ASSERT(self->OnTaskQueue());
+    return new DecodedAudioDataSink(
+      self->mAudioQueue, self->GetMediaTime(),
+      self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
+  };
+  mAudioSink = new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
 }

 MediaDecoderStateMachine::~MediaDecoderStateMachine()
@@ -359,19 +371,17 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration()
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
  int64_t audioDecoded = AudioQueue().Duration();
-  if (AudioEndTime() != -1) {
+  if (mAudioSink->IsStarted()) {
    audioDecoded += AudioEndTime() - GetMediaTime();
  }
  return audioDecoded;
 }

-void MediaDecoderStateMachine::SendStreamData()
+void MediaDecoderStateMachine::DiscardStreamData()
 {
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
-  MOZ_ASSERT(!mAudioSink, "Should've been stopped in RunStateMachine()");
-
-  mDecodedStream->SendData();
+  MOZ_ASSERT(!mAudioSink->IsStarted(), "Should've been stopped in RunStateMachine()");

  const auto clockTime = GetClock();
  while (true) {
@@ -381,7 +391,9 @@ void MediaDecoderStateMachine::SendStreamData()
    // keep decoding audio samples till the end and consume a lot of memory.
    // Therefore we only discard those behind the stream clock to throttle
    // the decoding speed.
-    if (a && a->mTime <= clockTime) {
+    // Note we don't discard a sample when |a->mTime == clockTime| because that
+    // will discard the 1st sample when clockTime is still 0.
+    if (a && a->mTime < clockTime) {
      nsRefPtr<MediaData> releaseMe = AudioQueue().PopFront();
      continue;
    }
@@ -562,10 +574,6 @@ MediaDecoderStateMachine::OnAudioDecoded(AudioData* aAudioSample)
      if (mIsAudioPrerolling && DonePrerollingAudio()) {
        StopPrerollingAudio();
      }
-      // Schedule the state machine to send stream data as soon as possible.
-      if (mAudioCaptured) {
-        ScheduleStateMachine();
-      }
      return;
    }

@@ -761,10 +769,6 @@ MediaDecoderStateMachine::OnNotDecoded(MediaData::Type aType,
        return;
      }
      CheckIfDecodeComplete();
-      // Schedule the state machine to notify track ended as soon as possible.
-      if (mAudioCaptured) {
-        ScheduleStateMachine();
-      }
      return;
    }
    case DECODER_STATE_SEEKING: {
@@ -851,7 +855,7 @@ MediaDecoderStateMachine::OnVideoDecoded(VideoData* aVideoSample)
      // frame pushed in the queue, schedule the state machine as soon as
      // possible to render the video frame or delay the state machine thread
      // accurately.
-      if (mAudioCaptured || VideoQueue().GetSize() == 1) {
+      if (VideoQueue().GetSize() == 1) {
        ScheduleStateMachine();
      }

@@ -1091,7 +1095,7 @@ void MediaDecoderStateMachine::UpdatePlaybackPosition(int64_t aTime)
  UpdatePlaybackPositionInternal(aTime);

  bool fragmentEnded = mFragmentEndTime >= 0 && GetMediaTime() >= mFragmentEndTime;
-  mMetadataManager.DispatchMetadataIfNeeded(mDecoder, TimeUnit::FromMicroseconds(aTime));
+  mMetadataManager.DispatchMetadataIfNeeded(TimeUnit::FromMicroseconds(aTime));

  if (fragmentEnded) {
    StopPlayback();
@@ -1142,9 +1146,7 @@ void MediaDecoderStateMachine::VolumeChanged()
 {
  MOZ_ASSERT(OnTaskQueue());
  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
-  if (mAudioSink) {
-    mAudioSink->SetVolume(mVolume);
-  }
+  mAudioSink->SetVolume(mVolume);
  mDecodedStream->SetVolume(mVolume);
 }

@@ -1262,6 +1264,8 @@ void MediaDecoderStateMachine::Shutdown()

  Reset();

+  mAudioSink->Shutdown();
+
  // Shut down our start time rendezvous.
  if (mStartTimeRendezvous) {
    mStartTimeRendezvous->Destroy();
@@ -1456,12 +1460,11 @@ void MediaDecoderStateMachine::StopAudioSink()
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();

-  if (mAudioSink) {
-    DECODER_LOG("Shutdown audio thread");
-    mAudioSink->Shutdown();
-    mAudioSink = nullptr;
+  if (mAudioSink->IsStarted()) {
+    DECODER_LOG("Stop AudioSink");
+    mAudioSink->Stop();
+    mAudioSinkPromise.DisconnectIfExists();
  }
-  mAudioSinkPromise.DisconnectIfExists();
 }

 void
@@ -1542,10 +1545,6 @@ MediaDecoderStateMachine::InitiateSeek()
               "Can only seek in range [0,duration]");
  mCurrentSeek.mTarget.mTime = seekTime;

-  if (mAudioCaptured) {
-    mDecodedStream->RecreateData();
-  }
-
  mDropAudioUntilNextDiscontinuity = HasAudio();
  mDropVideoUntilNextDiscontinuity = HasVideo();
  mCurrentTimeBeforeSeek = GetMediaTime();
@@ -1749,25 +1748,19 @@ MediaDecoderStateMachine::StartAudioSink()
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
  if (mAudioCaptured) {
-    MOZ_ASSERT(!mAudioSink);
+    MOZ_ASSERT(!mAudioSink->IsStarted());
    return;
  }

-  if (HasAudio() && !mAudioSink) {
+  if (HasAudio() && !mAudioSink->IsStarted()) {
    mAudioCompleted = false;
-    mAudioSink = new DecodedAudioDataSink(mAudioQueue,
-                                          GetMediaTime(), mInfo.mAudio,
-                                          mDecoder->GetAudioChannel());
+    mAudioSink->Start(GetMediaTime(), mInfo);

    mAudioSinkPromise.Begin(
-      mAudioSink->Init()->Then(
+      mAudioSink->OnEnded(TrackInfo::kAudioTrack)->Then(
        OwnerThread(), __func__, this,
        &MediaDecoderStateMachine::OnAudioSinkComplete,
        &MediaDecoderStateMachine::OnAudioSinkError));
-
-    mAudioSink->SetVolume(mVolume);
-    mAudioSink->SetPlaybackRate(mPlaybackRate);
-    mAudioSink->SetPreservesPitch(mPreservesPitch);
  }
 }

@@ -1805,7 +1798,7 @@ int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
  // The amount of audio we have decoded is the amount of audio data we've
  // already decoded and pushed to the hardware, plus the amount of audio
  // data waiting to be pushed to the hardware.
-  int64_t pushed = (AudioEndTime() != -1) ? (AudioEndTime() - GetMediaTime()) : 0;
+  int64_t pushed = mAudioSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0;

  // Currently for real time streams, AudioQueue().Duration() produce
  // wrong values (Bug 1114434), so we use frame counts to calculate duration.
@@ -1834,7 +1827,7 @@ bool MediaDecoderStateMachine::OutOfDecodedAudio()
    MOZ_ASSERT(OnTaskQueue());
    return IsAudioDecoding() && !AudioQueue().IsFinished() &&
           AudioQueue().GetSize() == 0 &&
-           (!mAudioSink || !mAudioSink->HasUnplayedFrames());
+           !mAudioSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
 }

 bool MediaDecoderStateMachine::HasLowUndecodedData()
@@ -2195,6 +2188,7 @@ MediaDecoderStateMachine::FinishShutdown()
  // Prevent dangling pointers by disconnecting the listeners.
  mAudioQueueListener.Disconnect();
  mVideoQueueListener.Disconnect();
+  mMetadataManager.Disconnect();

  // Disconnect canonicals and mirrors before shutting down our task queue.
  mBuffered.DisconnectIfConnected();
@@ -2397,8 +2391,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()

        mSentPlaybackEndedEvent = true;

-        // Stop audio sink after call to AudioEndTime() above, otherwise it will
-        // return an incorrect value due to a null mAudioSink.
+        // MediaSink::GetEndTime() must be called before stopping playback.
        StopAudioSink();
        StopDecodedStream();
      }
@@ -2561,8 +2554,8 @@ MediaDecoderStateMachine::GetAudioClock() const
  AssertCurrentThreadInMonitor();
  MOZ_ASSERT(HasAudio() && !mAudioCompleted && IsPlaying());
  // Since this function is called while we are playing and AudioSink is
-  // created once playback starts, mAudioSink is guaranteed to be non-null.
-  MOZ_ASSERT(mAudioSink);
+  // started once playback starts, IsStarted() is guaranteed to be true.
+  MOZ_ASSERT(mAudioSink->IsStarted());
  return mAudioSink->GetPosition();
 }

@@ -2631,7 +2624,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames()
  }

  if (mAudioCaptured) {
-    SendStreamData();
+    DiscardStreamData();
  }

  TimeStamp nowTime;
@@ -2905,9 +2898,7 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp)
  AssertCurrentThreadInMonitor();
  mPlayStartTime = aTimeStamp;

-  if (mAudioSink) {
-    mAudioSink->SetPlaying(!mPlayStartTime.IsNull());
-  }
+  mAudioSink->SetPlaying(!mPlayStartTime.IsNull());
  // Have DecodedStream remember the playing state so it doesn't need to
  // ask MDSM about IsPlaying(). Note we have to do this even before capture
  // happens since capture could happen in the middle of playback.
@@ -3000,9 +2991,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged()
  }

  mPlaybackRate = mLogicalPlaybackRate;
-  if (mAudioSink) {
-    mAudioSink->SetPlaybackRate(mPlaybackRate);
-  }
+  mAudioSink->SetPlaybackRate(mPlaybackRate);

  ScheduleStateMachine();
 }
@@ -3011,10 +3000,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged()
 {
  MOZ_ASSERT(OnTaskQueue());
  ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
-
-  if (mAudioSink) {
-    mAudioSink->SetPreservesPitch(mPreservesPitch);
-  }
+  mAudioSink->SetPreservesPitch(mPreservesPitch);
 }

 bool MediaDecoderStateMachine::IsShutdown()
@@ -3023,33 +3009,17 @@ bool MediaDecoderStateMachine::IsShutdown()
  return mIsShutdown;
 }

-void MediaDecoderStateMachine::QueueMetadata(const TimeUnit& aPublishTime,
-                                             nsAutoPtr<MediaInfo> aInfo,
-                                             nsAutoPtr<MetadataTags> aTags)
-{
-  MOZ_ASSERT(OnDecodeTaskQueue());
-  AssertCurrentThreadInMonitor();
-  TimedMetadata* metadata = new TimedMetadata;
-  metadata->mPublishTime = aPublishTime;
-  metadata->mInfo = aInfo.forget();
-  metadata->mTags = aTags.forget();
-  mMetadataManager.QueueMetadata(metadata);
-}
-
 int64_t
 MediaDecoderStateMachine::AudioEndTime() const
 {
  MOZ_ASSERT(OnTaskQueue());
  AssertCurrentThreadInMonitor();
-  if (mAudioSink) {
-    return mAudioSink->GetEndTime();
+  if (mAudioSink->IsStarted()) {
+    return mAudioSink->GetEndTime(TrackInfo::kAudioTrack);
  } else if (mAudioCaptured) {
    return mDecodedStream->AudioEndTime();
  }
-  // Don't call this after mAudioSink becomes null since we can't distinguish
-  // "before StartAudioSink" and "after StopAudioSink" where mAudioSink
-  // is null in both cases.
-  MOZ_ASSERT(!mAudioCompleted);
+  MOZ_ASSERT(!HasAudio());
  return -1;
 }

@@ -3166,7 +3136,7 @@ void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream,
 {
  MOZ_ASSERT(NS_IsMainThread());
  DECODER_LOG("AddOutputStream aStream=%p!", aStream);
-  mDecodedStream->Connect(aStream, aFinishWhenEnded);
+  mDecodedStream->AddOutput(aStream, aFinishWhenEnded);
  DispatchAudioCaptured();
 }

@@ -3174,7 +3144,7 @@ void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream)
 {
  MOZ_ASSERT(NS_IsMainThread());
  DECODER_LOG("RemoveOutputStream=%p!", aStream);
-  mDecodedStream->Remove(aStream);
+  mDecodedStream->RemoveOutput(aStream);
  if (!mDecodedStream->HasConsumers()) {
    DispatchAudioUncaptured();
  }