import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1197669 - Part1 - 1.Enable the testcase on B2G. 2. fix the mimetype check in testcase. r=jwwang (9bbae092f)
- Bug 1197669 - Part2 - 1. Ensure the EOS is sent to MediaCodec. 2. mEndOfStream should be protected by monitor. r=sotaro (62b8e66b7)
- bug 1193922 skip SetIsOnlyNodeForContext after shutdown r=padenot (df991d0af)
- Minor (1d695f7c4)
- Bug 1197051 - Don't try to decode ahead while seeking - r=jya (7d8d877ad)
- Bug 1197075: P1. Revert "Bug 1197051 - Don't try to decode ahead while seeking - r=jya". r=edwin (e837b12e3)
- Bug 1143575. Push all available frames to the compositor. r=cpearce (109e54ae8)
- partial of  Bug 1197075: P2. Revert "Bug 1171257 - Add force decode ahead to MediaFormatReader r=jya,bholley". r=edwin (5f4859e47)
- Minor (a841ca407)
- Bug 1199155. Part 2 - fix includes and forward declarations. r=roc. (ca523595a)
- Bug 1199155. Part 1 - move dom/media/DecodedStream.* to dom/media/mediasink/ and implement the interface of MediaSink. r=roc. (90b376a24)
- Bug 1199155. Part 3 - rename mDecodedStream to mStreamSink. r=roc. (1aae54328)
- Bug 1203374. Part 1 - extract the code of computing canplaythrough so it is reusable. r=jya. (acc5fc8c6)
- Bug 1203374. Part 2 - duplicate the implementation of MediaDecoder::CanPlayThrough so MDSM can call its own CanPlayThrough() on its own thread. r=jya. (a042541a8)
- Bug 1182928 - Disable dormant mode for EME videos in Firefox Beta and Release. r=sotaro (ea5964ae9)
- Bug 1197022 - [EME] Disable dormant mode for EME videos on all channels. r=kentuckyfriedtakahe (742d00672)
- Bug 1179110 - Fix ComputePlaybackRate. r=jww (a43272af5)
- Bug 1178622 - Fix enum-to-string mismatch in MediaDecoder.cpp. r=jya. (b9da7ebf3)
- Bug 1203418. Part 1 - cache the results of ComputePlaybackRate() so they can be mirrored by MDSM. r=cpearce. (95f7ac068)
- Bug 1203418. Part 2 - duplicate the implementation of MediaDecoder::GetStatistics so MDSM can call it on its own thread. r=cpearce. (2755b85a7)
- Bug 1203418. Part 3 - ensure MDSM::mPlaybackOffset and MediaDecoder::mPlaybackPosition are mono-increasing to avoid "jitter" in calculating playback statistics. r=cpearce. (54dcb6219)
- Bug 1104616 - Proxy video capture access to the main process. r=jesup,glandium,mrbkap (cfdd08459)
- Bug 1200614 - Protect the capture engines array from concurrent access (during shutdown). r=jesup (c4dbd6e10)
- Bug 1200614 - Check whether engines are still alive when webrtc ops run. r=jesup (d42d3474f)
- Bug 1194640 - add NSPR logging of camera capabilities, r=jesup (c7365b916)
- Bug 1204413 - Make MediaDecoder::IsTransportSeekable run on the main thread. r=kinetik. (098868b0d)
- Bug 1198202 - Increase hardware video decoding fallback threshold. r=ajones (1059f9b0f)
- Bug 1201197 - add dedicated listener to enumerateDevices. r=jesup (a51477957)
- Bug 1199562. Part 1 - rename mAudioSink to mMediaSink as well as related member names. r=roc. (c2db1b4d4)
- Bug 1199562. Part 2 - replace usage of mStreamSink with mMediaSink in most cases. r=roc. (7aa0754c4)
- Bug 1199562. Part 3 - remove unused code. r=roc. (343c5be2f)
- Bug 1203877 - Remove MediaDecoder::UpdatePlaybackOffset. r=kinetik. (4476dd1f8)
- Bug 1204430. Part 2 - mirror MediaDecoder::mMediaSeekable. r=kinetik. (51ca1197d)
- Bug 1206576 - Dispatch some MDSM functions to hide its internal thread model. r=jya. (74b9a169c)
- Bug 1206574 - Remove AbstractMediaDecoder::IsShutdown(). r=cpearce. (4ed17a27f)
- Bug 1207017. Part 1 - fix coding styles. r=kinetik. (cc10a28c3)
- Bug 1207017. Part 2 - remove duplicated GetStateMachine(). r=kinetik. (19b663098)
- Bug 1200477 - Allow building with WebRTC disabled. r=glandium,mrbkap (56055c3c7)
This commit is contained in:
2021-10-14 09:25:28 +08:00
parent edaff2b0dc
commit 0c01ffebb4
55 changed files with 3877 additions and 1340 deletions
+216 -207
View File
@@ -9,40 +9,42 @@
#include "mmsystem.h"
#endif
#include "mozilla/DebugOnly.h"
#include <algorithm>
#include <stdint.h>
#include "MediaDecoderStateMachine.h"
#include "MediaTimer.h"
#include "gfx2DGlue.h"
#include "mediasink/DecodedAudioDataSink.h"
#include "mediasink/AudioSinkWrapper.h"
#include "nsTArray.h"
#include "MediaDecoder.h"
#include "MediaDecoderReader.h"
#include "mozilla/MathAlgorithms.h"
#include "mediasink/DecodedStream.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/Logging.h"
#include "mozilla/mozalloc.h"
#include "VideoUtils.h"
#include "TimeUnits.h"
#include "nsDeque.h"
#include "AudioSegment.h"
#include "VideoSegment.h"
#include "ImageContainer.h"
#include "nsComponentManagerUtils.h"
#include "nsITimer.h"
#include "nsContentUtils.h"
#include "MediaShutdownManager.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Preferences.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/TaskQueue.h"
#include "nsIEventTarget.h"
#include "prenv.h"
#include "mozilla/Preferences.h"
#include "gfx2DGlue.h"
#include "nsPrintfCString.h"
#include "DOMMediaStream.h"
#include "DecodedStream.h"
#include "mozilla/Logging.h"
#include <algorithm>
#include "nsComponentManagerUtils.h"
#include "nsContentUtils.h"
#include "nsIEventTarget.h"
#include "nsITimer.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
#include "nsDeque.h"
#include "prenv.h"
#include "AudioSegment.h"
#include "DOMMediaStream.h"
#include "ImageContainer.h"
#include "MediaDecoder.h"
#include "MediaDecoderReader.h"
#include "MediaDecoderStateMachine.h"
#include "MediaShutdownManager.h"
#include "MediaTimer.h"
#include "TimeUnits.h"
#include "VideoSegment.h"
#include "VideoUtils.h"
namespace mozilla {
@@ -172,10 +174,11 @@ static int64_t DurationToUsecs(TimeDuration aDuration) {
static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
static const uint32_t SCARCE_VIDEO_QUEUE_SIZE = 1;
static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueHWAccelSize = MIN_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueSendToCompositorSize = VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
MediaDecoderReader* aReader,
@@ -215,12 +218,12 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mDropVideoUntilNextDiscontinuity(false),
mDecodeToSeekTarget(false),
mCurrentTimeBeforeSeek(0),
mCorruptFrames(30),
mCorruptFrames(60),
mDecodingFirstFrame(true),
mSentLoadedMetadataEvent(false),
mSentFirstFrameLoadedEvent(false),
mSentPlaybackEndedEvent(false),
mDecodedStream(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)),
mStreamSink(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)),
mResource(aDecoder->GetResource()),
mBuffered(mTaskQueue, TimeIntervals(),
"MediaDecoderStateMachine::mBuffered (Mirror)"),
@@ -241,6 +244,14 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
"MediaDecoderStateMachine::mPreservesPitch (Mirror)"),
mSameOriginMedia(mTaskQueue, false,
"MediaDecoderStateMachine::mSameOriginMedia (Mirror)"),
mPlaybackBytesPerSecond(mTaskQueue, 0.0,
"MediaDecoderStateMachine::mPlaybackBytesPerSecond (Mirror)"),
mPlaybackRateReliable(mTaskQueue, true,
"MediaDecoderStateMachine::mPlaybackRateReliable (Mirror)"),
mDecoderPosition(mTaskQueue, 0,
"MediaDecoderStateMachine::mDecoderPosition (Mirror)"),
mMediaSeekable(mTaskQueue, true,
"MediaDecoderStateMachine::mMediaSeekable (Mirror)"),
mDuration(mTaskQueue, NullableTimeUnit(),
"MediaDecoderStateMachine::mDuration (Canonical"),
mIsShutdown(mTaskQueue, false,
@@ -248,7 +259,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mNextFrameStatus(mTaskQueue, MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED,
"MediaDecoderStateMachine::mNextFrameStatus (Canonical)"),
mCurrentPosition(mTaskQueue, 0,
"MediaDecoderStateMachine::mCurrentPosition (Canonical)")
"MediaDecoderStateMachine::mCurrentPosition (Canonical)"),
mPlaybackOffset(mTaskQueue, 0,
"MediaDecoderStateMachine::mPlaybackOffset (Canonical)")
{
MOZ_COUNT_CTOR(MediaDecoderStateMachine);
NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
@@ -266,6 +279,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
Preferences::AddUintVarCache(&sVideoQueueHWAccelSize,
"media.video-queue.hw-accel-size",
MIN_VIDEO_QUEUE_SIZE);
Preferences::AddUintVarCache(&sVideoQueueSendToCompositorSize,
"media.video-queue.send-to-compositor-size",
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
}
mBufferingWait = IsRealTime() ? 0 : 15;
@@ -287,14 +303,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
nsRefPtr<MediaDecoderStateMachine> self = this;
auto audioSinkCreator = [self] () {
MOZ_ASSERT(self->OnTaskQueue());
return new DecodedAudioDataSink(
self->mAudioQueue, self->GetMediaTime(),
self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
};
mAudioSink = new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
mMediaSink = CreateAudioSink();
}
MediaDecoderStateMachine::~MediaDecoderStateMachine()
@@ -325,6 +334,10 @@ MediaDecoderStateMachine::InitializationTask()
mLogicalPlaybackRate.Connect(mDecoder->CanonicalPlaybackRate());
mPreservesPitch.Connect(mDecoder->CanonicalPreservesPitch());
mSameOriginMedia.Connect(mDecoder->CanonicalSameOriginMedia());
mPlaybackBytesPerSecond.Connect(mDecoder->CanonicalPlaybackBytesPerSecond());
mPlaybackRateReliable.Connect(mDecoder->CanonicalPlaybackRateReliable());
mDecoderPosition.Connect(mDecoder->CanonicalDecoderPosition());
mMediaSeekable.Connect(mDecoder->CanonicalMediaSeekable());
// Initialize watchers.
mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated);
@@ -344,6 +357,19 @@ MediaDecoderStateMachine::InitializationTask()
SameOriginMediaChanged();
}
media::MediaSink*
MediaDecoderStateMachine::CreateAudioSink()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
auto audioSinkCreator = [self] () {
MOZ_ASSERT(self->OnTaskQueue());
return new DecodedAudioDataSink(
self->mAudioQueue, self->GetMediaTime(),
self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
};
return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
}
bool MediaDecoderStateMachine::HasFutureAudio()
{
MOZ_ASSERT(OnTaskQueue());
@@ -373,7 +399,7 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration()
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
int64_t audioDecoded = AudioQueue().Duration();
if (mAudioSink->IsStarted()) {
if (mMediaSink->IsStarted()) {
audioDecoded += AudioEndTime() - GetMediaTime();
}
return audioDecoded;
@@ -383,7 +409,6 @@ void MediaDecoderStateMachine::DiscardStreamData()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
MOZ_ASSERT(!mAudioSink->IsStarted(), "Should've been stopped in RunStateMachine()");
const auto clockTime = GetClock();
while (true) {
@@ -678,7 +703,7 @@ MediaDecoderStateMachine::OnAudioPopped(const nsRefPtr<MediaData>& aSample)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->UpdatePlaybackOffset(std::max<int64_t>(0, aSample->mOffset));
mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
UpdateNextFrameStatus();
DispatchAudioDecodeTaskIfNeeded();
}
@@ -688,7 +713,7 @@ MediaDecoderStateMachine::OnVideoPopped(const nsRefPtr<MediaData>& aSample)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->UpdatePlaybackOffset(aSample->mOffset);
mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
UpdateNextFrameStatus();
DispatchVideoDecodeTaskIfNeeded();
}
@@ -1073,8 +1098,7 @@ void MediaDecoderStateMachine::MaybeStartPlayback()
SetPlayStartTime(TimeStamp::Now());
MOZ_ASSERT(IsPlaying());
StartAudioSink();
StartDecodedStream();
StartMediaSink();
DispatchDecodeTasksIfNeeded();
}
@@ -1148,8 +1172,7 @@ void MediaDecoderStateMachine::VolumeChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mAudioSink->SetVolume(mVolume);
mDecodedStream->SetVolume(mVolume);
mMediaSink->SetVolume(mVolume);
}
void MediaDecoderStateMachine::RecomputeDuration()
@@ -1184,7 +1207,16 @@ void MediaDecoderStateMachine::RecomputeDuration()
mDuration = Some(duration);
}
void MediaDecoderStateMachine::SetDormant(bool aDormant)
void
MediaDecoderStateMachine::DispatchSetDormant(bool aDormant)
{
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetDormant, aDormant);
OwnerThread()->Dispatch(r.forget());
}
void
MediaDecoderStateMachine::SetDormant(bool aDormant)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
@@ -1266,7 +1298,7 @@ void MediaDecoderStateMachine::Shutdown()
Reset();
mAudioSink->Shutdown();
mMediaSink->Shutdown();
// Shut down our start time rendezvous.
if (mStartTimeRendezvous) {
@@ -1329,7 +1361,16 @@ void MediaDecoderStateMachine::StartDecoding()
ScheduleStateMachine();
}
void MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged()
void
MediaDecoderStateMachine::DispatchWaitingForResourcesStatusChanged()
{
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethod(
this, &MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged);
OwnerThread()->Dispatch(r.forget());
}
void
MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
@@ -1398,7 +1439,7 @@ void MediaDecoderStateMachine::SameOriginMediaChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecodedStream->SetSameOrigin(mSameOriginMedia);
mStreamSink->SetSameOrigin(mSameOriginMedia);
}
void MediaDecoderStateMachine::BufferedRangeUpdated()
@@ -1431,7 +1472,7 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
// We need to be able to seek both at a transport level and at a media level
// to seek.
if (!mDecoder->IsMediaSeekable()) {
if (!mMediaSeekable) {
DECODER_WARN("Seek() function should not be called on a non-seekable state machine");
return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
}
@@ -1457,15 +1498,22 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
return mPendingSeek.mPromise.Ensure(__func__);
}
void MediaDecoderStateMachine::StopAudioSink()
nsRefPtr<MediaDecoder::SeekPromise>
MediaDecoderStateMachine::InvokeSeek(SeekTarget aTarget)
{
return InvokeAsync(OwnerThread(), this, __func__,
&MediaDecoderStateMachine::Seek, aTarget);
}
void MediaDecoderStateMachine::StopMediaSink()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioSink->IsStarted()) {
DECODER_LOG("Stop AudioSink");
mAudioSink->Stop();
mAudioSinkPromise.DisconnectIfExists();
if (mMediaSink->IsStarted()) {
DECODER_LOG("Stop MediaSink");
mMediaSink->Stop();
mMediaSinkPromise.DisconnectIfExists();
}
}
@@ -1714,8 +1762,6 @@ MediaDecoderStateMachine::RequestVideoData()
bool skipToNextKeyFrame = mSentFirstFrameLoadedEvent &&
NeedToSkipToNextKeyframe();
int64_t currentTime = mState == DECODER_STATE_SEEKING ? 0 : GetMediaTime();
bool forceDecodeAhead = mSentFirstFrameLoadedEvent &&
static_cast<uint32_t>(VideoQueue().GetSize()) <= SCARCE_VIDEO_QUEUE_SIZE;
SAMPLE_LOG("Queueing video task - queued=%i, decoder-queued=%o, skip=%i, time=%lld",
VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), skipToNextKeyFrame,
@@ -1725,7 +1771,7 @@ MediaDecoderStateMachine::RequestVideoData()
mVideoDataRequest.Begin(
InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__,
&MediaDecoderReader::RequestVideoData,
skipToNextKeyFrame, currentTime, forceDecodeAhead)
skipToNextKeyFrame, currentTime)
->Then(OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnVideoDecoded,
&MediaDecoderStateMachine::OnVideoNotDecoded));
@@ -1733,7 +1779,7 @@ MediaDecoderStateMachine::RequestVideoData()
mVideoDataRequest.Begin(
InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__,
&MediaDecoderReader::RequestVideoData,
skipToNextKeyFrame, currentTime, forceDecodeAhead)
skipToNextKeyFrame, currentTime)
->Then(OwnerThread(), __func__, mStartTimeRendezvous.get(),
&StartTimeRendezvous::ProcessFirstSample<VideoDataPromise>,
&StartTimeRendezvous::FirstSampleRejected<VideoData>)
@@ -1745,55 +1791,25 @@ MediaDecoderStateMachine::RequestVideoData()
}
void
MediaDecoderStateMachine::StartAudioSink()
MediaDecoderStateMachine::StartMediaSink()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioCaptured) {
MOZ_ASSERT(!mAudioSink->IsStarted());
return;
}
if (!mAudioSink->IsStarted()) {
if (!mMediaSink->IsStarted()) {
mAudioCompleted = false;
mAudioSink->Start(GetMediaTime(), mInfo);
mMediaSink->Start(GetMediaTime(), mInfo);
auto promise = mAudioSink->OnEnded(TrackInfo::kAudioTrack);
auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
if (promise) {
mAudioSinkPromise.Begin(promise->Then(
mMediaSinkPromise.Begin(promise->Then(
OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnAudioSinkComplete,
&MediaDecoderStateMachine::OnAudioSinkError));
&MediaDecoderStateMachine::OnMediaSinkComplete,
&MediaDecoderStateMachine::OnMediaSinkError));
}
}
}
void
MediaDecoderStateMachine::StopDecodedStream()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
mDecodedStream->StopPlayback();
mDecodedStreamPromise.DisconnectIfExists();
}
void
MediaDecoderStateMachine::StartDecodedStream()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
// Tell DecodedStream to start playback with specified start time and media
// info. This is consistent with how we create AudioSink in StartAudioThread().
if (mAudioCaptured && !mDecodedStreamPromise.Exists()) {
mDecodedStreamPromise.Begin(
mDecodedStream->StartPlayback(GetMediaTime(), mInfo)->Then(
OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnDecodedStreamFinish,
&MediaDecoderStateMachine::OnDecodedStreamError));
}
}
int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
{
MOZ_ASSERT(OnTaskQueue());
@@ -1802,7 +1818,7 @@ int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
// The amount of audio we have decoded is the amount of audio data we've
// already decoded and pushed to the hardware, plus the amount of audio
// data waiting to be pushed to the hardware.
int64_t pushed = mAudioSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0;
int64_t pushed = mMediaSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0;
// Currently for real time streams, AudioQueue().Duration() produce
// wrong values (Bug 1114434), so we use frame counts to calculate duration.
@@ -1831,7 +1847,7 @@ bool MediaDecoderStateMachine::OutOfDecodedAudio()
MOZ_ASSERT(OnTaskQueue());
return IsAudioDecoding() && !AudioQueue().IsFinished() &&
AudioQueue().GetSize() == 0 &&
!mAudioSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
!mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
}
bool MediaDecoderStateMachine::HasLowUndecodedData()
@@ -2047,7 +2063,7 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame()
DECODER_LOG("Media duration %lld, "
"transportSeekable=%d, mediaSeekable=%d",
Duration().ToMicroseconds(), mDecoder->IsTransportSeekable(), mDecoder->IsMediaSeekable());
Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable.Ref());
if (HasAudio() && !HasVideo() && !mSentFirstFrameLoadedEvent) {
// We're playing audio only. We don't need to worry about slow video
@@ -2179,6 +2195,15 @@ private:
nsRefPtr<MediaDecoderStateMachine> mStateMachine;
};
void
MediaDecoderStateMachine::DispatchShutdown()
{
mStreamSink->BeginShutdown();
nsCOMPtr<nsIRunnable> runnable =
NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown);
OwnerThread()->Dispatch(runnable.forget());
}
void
MediaDecoderStateMachine::FinishShutdown()
{
@@ -2205,10 +2230,16 @@ MediaDecoderStateMachine::FinishShutdown()
mLogicalPlaybackRate.DisconnectIfConnected();
mPreservesPitch.DisconnectIfConnected();
mSameOriginMedia.DisconnectIfConnected();
mPlaybackBytesPerSecond.DisconnectIfConnected();
mPlaybackRateReliable.DisconnectIfConnected();
mDecoderPosition.DisconnectIfConnected();
mMediaSeekable.DisconnectIfConnected();
mDuration.DisconnectAll();
mIsShutdown.DisconnectAll();
mNextFrameStatus.DisconnectAll();
mCurrentPosition.DisconnectAll();
mPlaybackOffset.DisconnectAll();
// Shut down the watch manager before shutting down our task queue.
mWatchManager.Shutdown();
@@ -2310,7 +2341,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
if (mReader->UseBufferingHeuristics()) {
TimeDuration elapsed = now - mBufferingStart;
bool isLiveStream = resource->IsLiveStream();
if ((isLiveStream || !mDecoder->CanPlayThrough()) &&
if ((isLiveStream || !CanPlayThrough()) &&
elapsed < TimeDuration::FromSeconds(mBufferingWait * mPlaybackRate) &&
(mQuickBuffering ? HasLowDecodedData(mQuickBufferingLowDataThresholdUsecs)
: HasLowUndecodedData(mBufferingWait * USECS_PER_S)) &&
@@ -2359,7 +2390,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
// end of the media, and so that we update the readyState.
if (VideoQueue().GetSize() > 1 ||
(HasAudio() && !mAudioCompleted) ||
(mAudioCaptured && !mDecodedStream->IsFinished()))
(mAudioCaptured && !mStreamSink->IsFinished()))
{
// Start playback if necessary to play the remaining media.
MaybeStartPlayback();
@@ -2396,8 +2427,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
mSentPlaybackEndedEvent = true;
// MediaSink::GetEndTime() must be called before stopping playback.
StopAudioSink();
StopDecodedStream();
StopMediaSink();
}
return NS_OK;
@@ -2423,11 +2453,10 @@ MediaDecoderStateMachine::Reset()
mState == DECODER_STATE_DORMANT ||
mState == DECODER_STATE_DECODING_NONE);
// Stop the audio thread. Otherwise, AudioSink might be accessing AudioQueue
// Stop the audio thread. Otherwise, MediaSink might be accessing AudioQueue
// outside of the decoder monitor while we are clearing the queue and causes
// crash for no samples to be popped.
StopAudioSink();
StopDecodedStream();
StopMediaSink();
mVideoFrameEndTime = -1;
mDecodedVideoEndTime = -1;
@@ -2447,6 +2476,8 @@ MediaDecoderStateMachine::Reset()
mVideoWaitRequest.DisconnectIfExists();
mSeekRequest.DisconnectIfExists();
mPlaybackOffset = 0;
nsCOMPtr<nsIRunnable> resetTask =
NS_NewRunnableMethod(mReader, &MediaDecoderReader::ResetDecode);
DecodeTaskQueue()->Dispatch(resetTask.forget());
@@ -2472,8 +2503,8 @@ bool MediaDecoderStateMachine::CheckFrameValidity(VideoData* aData)
// only supports integer types.
mCorruptFrames.insert(10);
if (mReader->VideoIsHardwareAccelerated() &&
frameStats.GetPresentedFrames() > 30 &&
mCorruptFrames.mean() >= 1 /* 10% */) {
frameStats.GetPresentedFrames() > 60 &&
mCorruptFrames.mean() >= 2 /* 20% */) {
nsCOMPtr<nsIRunnable> task =
NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration);
DecodeTaskQueue()->Dispatch(task.forget());
@@ -2552,13 +2583,6 @@ void MediaDecoderStateMachine::RenderVideoFrames(int32_t aMaxFrames,
container->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
}
int64_t MediaDecoderStateMachine::GetStreamClock() const
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
return mDecodedStream->GetPosition();
}
int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
{
MOZ_ASSERT(OnTaskQueue());
@@ -2573,11 +2597,7 @@ int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
if (!IsPlaying()) {
clock_time = mPlayDuration;
} else {
if (mAudioCaptured) {
clock_time = GetStreamClock();
} else {
clock_time = mAudioSink->GetPosition(&t);
}
clock_time = mMediaSink->GetPosition(&t);
NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards.");
}
if (aTimeStamp) {
@@ -2633,7 +2653,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames()
}
}
RenderVideoFrames(1, clockTime, nowTime);
RenderVideoFrames(sVideoQueueSendToCompositorSize, clockTime, nowTime);
// Check to see if we don't have enough data to play up to the next frame.
// If we don't, switch to buffering mode.
@@ -2828,6 +2848,28 @@ bool MediaDecoderStateMachine::JustExitedQuickBuffering()
(TimeStamp::Now() - mDecodeStartTime) < TimeDuration::FromMicroseconds(QUICK_BUFFER_THRESHOLD_USECS);
}
bool
MediaDecoderStateMachine::CanPlayThrough()
{
MOZ_ASSERT(OnTaskQueue());
return IsRealTime() || GetStatistics().CanPlayThrough();
}
MediaStatistics
MediaDecoderStateMachine::GetStatistics()
{
MOZ_ASSERT(OnTaskQueue());
MediaStatistics result;
result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable);
result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition);
result.mTotalBytes = mResource->GetLength();
result.mPlaybackRate = mPlaybackBytesPerSecond;
result.mPlaybackRateReliable = mPlaybackRateReliable;
result.mDecoderPosition = mDecoderPosition;
result.mPlaybackPosition = mPlaybackOffset;
return result;
}
void MediaDecoderStateMachine::StartBuffering()
{
MOZ_ASSERT(OnTaskQueue());
@@ -2858,7 +2900,7 @@ void MediaDecoderStateMachine::StartBuffering()
SetState(DECODER_STATE_BUFFERING);
DECODER_LOG("Changed state from DECODING to BUFFERING, decoded for %.3lfs",
decodeDuration.ToSeconds());
MediaDecoder::Statistics stats = mDecoder->GetStatistics();
MediaStatistics stats = GetStatistics();
DECODER_LOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s",
stats.mPlaybackRate/1024, stats.mPlaybackRateReliable ? "" : " (unreliable)",
stats.mDownloadRate/1024, stats.mDownloadRateReliable ? "" : " (unreliable)");
@@ -2869,12 +2911,7 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp)
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
mPlayStartTime = aTimeStamp;
mAudioSink->SetPlaying(!mPlayStartTime.IsNull());
// Have DecodedStream remember the playing state so it doesn't need to
// ask MDSM about IsPlaying(). Note we have to do this even before capture
// happens since capture could happen in the middle of playback.
mDecodedStream->SetPlaying(!mPlayStartTime.IsNull());
mMediaSink->SetPlaying(!mPlayStartTime.IsNull());
}
void MediaDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder()
@@ -2952,7 +2989,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged()
}
mPlaybackRate = mLogicalPlaybackRate;
mAudioSink->SetPlaybackRate(mPlaybackRate);
mMediaSink->SetPlaybackRate(mPlaybackRate);
ScheduleStateMachine();
}
@@ -2961,7 +2998,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mAudioSink->SetPreservesPitch(mPreservesPitch);
mMediaSink->SetPreservesPitch(mPreservesPitch);
}
bool MediaDecoderStateMachine::IsShutdown()
@@ -2975,33 +3012,33 @@ MediaDecoderStateMachine::AudioEndTime() const
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioSink->IsStarted()) {
return mAudioSink->GetEndTime(TrackInfo::kAudioTrack);
} else if (mAudioCaptured) {
return mDecodedStream->AudioEndTime();
if (mMediaSink->IsStarted()) {
return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
}
MOZ_ASSERT(!HasAudio());
return -1;
}
void MediaDecoderStateMachine::OnAudioSinkComplete()
void MediaDecoderStateMachine::OnMediaSinkComplete()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
mAudioSinkPromise.Complete();
mAudioCompleted = true;
mMediaSinkPromise.Complete();
// Set true only when we have audio.
mAudioCompleted = mInfo.HasAudio();
// To notify PlaybackEnded as soon as possible.
ScheduleStateMachine();
}
void MediaDecoderStateMachine::OnAudioSinkError()
void MediaDecoderStateMachine::OnMediaSinkError()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
mAudioSinkPromise.Complete();
mAudioCompleted = true;
mMediaSinkPromise.Complete();
// Set true only when we have audio.
mAudioCompleted = mInfo.HasAudio();
// Make the best effort to continue playback when there is video.
if (HasVideo()) {
@@ -3014,31 +3051,41 @@ void MediaDecoderStateMachine::OnAudioSinkError()
}
void
MediaDecoderStateMachine::OnDecodedStreamFinish()
MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
mDecodedStreamPromise.Complete();
if (mInfo.HasAudio()) {
mAudioCompleted = true;
if (aCaptured == mAudioCaptured) {
return;
}
// To notify PlaybackEnded as soon as possible.
// Backup current playback parameters.
MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams();
// Stop and shut down the existing sink.
StopMediaSink();
mMediaSink->Shutdown();
// Create a new sink according to whether audio is captured.
// TODO: We can't really create a new DecodedStream until OutputStreamManager
// is extracted. It is tricky that the implementation of DecodedStream
// happens to allow reuse after shutdown without creating a new one.
mMediaSink = aCaptured ? mStreamSink : CreateAudioSink();
// Restore playback parameters.
mMediaSink->SetPlaybackParams(params);
// Start the sink if we are already playing. Otherwise it will be
// handled in MaybeStartPlayback().
if (IsPlaying()) {
StartMediaSink();
}
mAudioCaptured = aCaptured;
ScheduleStateMachine();
}
void
MediaDecoderStateMachine::OnDecodedStreamError()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
mDecodedStreamPromise.Complete();
DecodeError();
}
uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
{
MOZ_ASSERT(OnTaskQueue());
@@ -3048,64 +3095,26 @@ uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
: std::max<uint32_t>(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE);
}
void MediaDecoderStateMachine::DispatchAudioCaptured()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () -> void
{
MOZ_ASSERT(self->OnTaskQueue());
ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
if (!self->mAudioCaptured) {
// Stop the audio sink if it's running.
self->StopAudioSink();
self->mAudioCaptured = true;
// Start DecodedStream if we are already playing. Otherwise it will be
// handled in MaybeStartPlayback().
if (self->IsPlaying()) {
self->StartDecodedStream();
}
self->ScheduleStateMachine();
}
});
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::DispatchAudioUncaptured()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () -> void
{
MOZ_ASSERT(self->OnTaskQueue());
ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
if (self->mAudioCaptured) {
self->StopDecodedStream();
// Start again the audio sink.
self->mAudioCaptured = false;
if (self->IsPlaying()) {
self->StartAudioSink();
}
self->ScheduleStateMachine();
}
});
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream,
bool aFinishWhenEnded)
{
MOZ_ASSERT(NS_IsMainThread());
DECODER_LOG("AddOutputStream aStream=%p!", aStream);
mDecodedStream->AddOutput(aStream, aFinishWhenEnded);
DispatchAudioCaptured();
mStreamSink->AddOutput(aStream, aFinishWhenEnded);
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetAudioCaptured, true);
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream)
{
MOZ_ASSERT(NS_IsMainThread());
DECODER_LOG("RemoveOutputStream=%p!", aStream);
mDecodedStream->RemoveOutput(aStream);
if (!mDecodedStream->HasConsumers()) {
DispatchAudioUncaptured();
mStreamSink->RemoveOutput(aStream);
if (!mStreamSink->HasConsumers()) {
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetAudioCaptured, false);
OwnerThread()->Dispatch(r.forget());
}
}