From 6431b48dbcdb47c281cb2ce6da9f09c64784d925 Mon Sep 17 00:00:00 2001 From: roytam1 Date: Wed, 12 May 2021 10:04:12 +0800 Subject: [PATCH] import changes from `dev' branch of rmottola/Arctic-Fox: - Bug 1175768 - Remove bogus check. r=jya (6d1b387f4) - Bug 1175768 - Make the logic in MDSM::NotifyDataArrived apply to mObservedDuration. r=jya (90dcae693) - Bug 1175768 - Dispatch UpdateEstimatedMediaDuration. r=jya (f003b4646) - Bug 1175768 - Dispatch NotifyDataArrived and remove the aBuffer argument. r=jya (e6eaf8eb1) - Bug 1175768 - Misc changes to Intervals/TimeUnit. r=jya (786da5ba2) - Bug 1174220 - Part 2: Use MediaByteBuffer instead of MediaLargeByteBuffer. r=jya (59fe3c482) - Bug 1175768 - Use mirroring for buffered ranges. r=jya (5100aba5e) - Bug 1175768 - Throttle NotifyDataArrived. r=jya (4bb13775b) - Bug 1178680 - Remove MediaDecoder::Observe as we have MediaShutdownManager. r=cpearce. (ba9b728ea) - Bug 1178437 - Do the dormant-enabled tracking on the main thread. r=jww,r=jya (b0287df66) --- dom/media/AbstractMediaDecoder.h | 17 +- dom/media/Intervals.h | 10 + dom/media/MP3FrameParser.cpp | 4 +- dom/media/MP3FrameParser.h | 2 +- dom/media/MediaData.h | 1 - dom/media/MediaDecoder.cpp | 31 +- dom/media/MediaDecoder.h | 12 +- dom/media/MediaDecoderReader.cpp | 72 ++++- dom/media/MediaDecoderReader.h | 81 ++++- dom/media/MediaDecoderStateMachine.cpp | 52 ++-- dom/media/MediaDecoderStateMachine.h | 19 +- dom/media/MediaFormatReader.cpp | 130 +++----- dom/media/MediaFormatReader.h | 22 +- dom/media/MediaResource.cpp | 3 +- dom/media/TimeUnits.h | 2 +- dom/media/apple/AppleMP3Reader.cpp | 15 +- dom/media/apple/AppleMP3Reader.h | 7 +- dom/media/directshow/DirectShowReader.cpp | 13 +- dom/media/directshow/DirectShowReader.h | 7 +- dom/media/fmp4/MP4Decoder.cpp | 13 + dom/media/fmp4/MP4Decoder.h | 1 + dom/media/fmp4/MP4Demuxer.cpp | 2 +- dom/media/fmp4/MP4Demuxer.h | 2 +- dom/media/fmp4/MP4Reader.cpp | 22 +- dom/media/fmp4/MP4Reader.h | 16 +- dom/media/gtest/TestMP4Reader.cpp | 9 +- dom/media/mediasource/ContainerParser.cpp | 26 +- dom/media/mediasource/ContainerParser.h | 12 +- dom/media/mediasource/MediaSourceDecoder.cpp | 72 +++-- dom/media/mediasource/MediaSourceDecoder.h | 15 +- dom/media/mediasource/MediaSourceReader.cpp | 79 +++-- dom/media/mediasource/MediaSourceReader.h | 4 +- dom/media/mediasource/ResourceQueue.cpp | 6 +- dom/media/mediasource/ResourceQueue.h | 6 +- dom/media/mediasource/SourceBuffer.cpp | 16 +- dom/media/mediasource/SourceBuffer.h | 11 +- .../SourceBufferContentManager.cpp | 18 ++ .../mediasource/SourceBufferContentManager.h | 2 +- dom/media/mediasource/SourceBufferDecoder.cpp | 15 +- dom/media/mediasource/SourceBufferDecoder.h | 3 +- .../mediasource/SourceBufferResource.cpp | 2 +- dom/media/mediasource/SourceBufferResource.h | 4 +- dom/media/mediasource/TrackBuffer.cpp | 294 ++++++++++++------ dom/media/mediasource/TrackBuffer.h | 38 ++- dom/media/mediasource/TrackBuffersManager.cpp | 12 +- dom/media/mediasource/TrackBuffersManager.h | 12 +- .../test/test_TruncatedDuration.html | 1 - dom/media/ogg/OggReader.cpp | 1 + dom/media/omx/MediaCodecReader.cpp | 15 +- dom/media/omx/MediaCodecReader.h | 7 +- dom/media/omx/MediaOmxCommonDecoder.cpp | 1 + dom/media/omx/MediaOmxReader.cpp | 51 ++- dom/media/omx/MediaOmxReader.h | 7 +- dom/media/raw/RawReader.cpp | 1 + dom/media/wave/WaveReader.cpp | 1 + dom/media/webaudio/BufferDecoder.cpp | 12 - dom/media/webaudio/BufferDecoder.h | 4 +- dom/media/webm/WebMBufferedParser.cpp | 5 +- dom/media/webm/WebMBufferedParser.h | 2 +- dom/media/webm/WebMReader.cpp | 9 +- dom/media/webm/WebMReader.h | 5 +- media/libstagefright/binding/BufferStream.cpp | 4 +- media/libstagefright/binding/MP4Metadata.cpp | 2 +- media/libstagefright/binding/MoofParser.cpp | 4 +- .../include/mp4_demuxer/BufferStream.h | 6 +- .../binding/include/mp4_demuxer/ByteReader.h | 4 - .../binding/include/mp4_demuxer/MP4Metadata.h | 2 +- .../binding/include/mp4_demuxer/MoofParser.h | 2 +- 68 files changed, 779 insertions(+), 579 deletions(-) diff --git a/dom/media/AbstractMediaDecoder.h b/dom/media/AbstractMediaDecoder.h index b5997468b3..95011726eb 100644 --- a/dom/media/AbstractMediaDecoder.h +++ b/dom/media/AbstractMediaDecoder.h @@ -71,10 +71,16 @@ public: virtual AbstractCanonical* CanonicalDurationOrNull() { return nullptr; }; - // Sets the duration of the media in microseconds. The MediaDecoder - // fires a durationchange event to its owner (e.g., an HTML audio - // tag). - virtual void UpdateEstimatedMediaDuration(int64_t aDuration) = 0; +protected: + virtual void UpdateEstimatedMediaDuration(int64_t aDuration) {}; +public: + void DispatchUpdateEstimatedMediaDuration(int64_t aDuration) + { + nsCOMPtr r = + NS_NewRunnableMethodWithArg(this, &AbstractMediaDecoder::UpdateEstimatedMediaDuration, + aDuration); + NS_DispatchToMainThread(r); + } // Set the media as being seekable or not. virtual void SetMediaSeekable(bool aMediaSeekable) = 0; @@ -108,7 +114,8 @@ public: // Called by the reader's MediaResource as data arrives over the network. // Must be called on the main thread. - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) = 0; + virtual void NotifyDataArrived(uint32_t aLength, int64_t aOffset, + bool aThrottleUpdates) = 0; // Set by Reader if the current audio track can be offloaded virtual void SetPlatformCanOffloadAudio(bool aCanOffloadAudio) {} diff --git a/dom/media/Intervals.h b/dom/media/Intervals.h index fd2f088dcc..5fb6dd2bc4 100644 --- a/dom/media/Intervals.h +++ b/dom/media/Intervals.h @@ -286,6 +286,16 @@ public: } } + bool operator== (const SelfType& aOther) const + { + return mIntervals == aOther.mIntervals; + } + + bool operator!= (const SelfType& aOther) const + { + return mIntervals != aOther.mIntervals; + } + SelfType& operator= (const SelfType& aOther) { mIntervals = aOther.mIntervals; diff --git a/dom/media/MP3FrameParser.cpp b/dom/media/MP3FrameParser.cpp index 0ce4fae0d8..6448c6d344 100644 --- a/dom/media/MP3FrameParser.cpp +++ b/dom/media/MP3FrameParser.cpp @@ -466,7 +466,7 @@ nsresult MP3FrameParser::ParseBuffer(const uint8_t* aBuffer, return NS_OK; } -void MP3FrameParser::Parse(const char* aBuffer, uint32_t aLength, uint64_t aOffset) +void MP3FrameParser::Parse(const uint8_t* aBuffer, uint32_t aLength, uint64_t aOffset) { MutexAutoLock mon(mLock); @@ -475,7 +475,7 @@ void MP3FrameParser::Parse(const char* aBuffer, uint32_t aLength, uint64_t aOffs return; } - const uint8_t* buffer = reinterpret_cast(aBuffer); + const uint8_t* buffer = aBuffer; int32_t length = aLength; uint64_t offset = aOffset; diff --git a/dom/media/MP3FrameParser.h b/dom/media/MP3FrameParser.h index 5c36f16270..154bff21e8 100644 --- a/dom/media/MP3FrameParser.h +++ b/dom/media/MP3FrameParser.h @@ -112,7 +112,7 @@ public: return mIsMP3 != NOT_MP3; } - void Parse(const char* aBuffer, uint32_t aLength, uint64_t aStreamOffset); + void Parse(const uint8_t* aBuffer, uint32_t aLength, uint64_t aStreamOffset); // Returns the duration, in microseconds. If the entire stream has not // been parsed yet, this is an estimate based on the bitrate of the diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h index 92e098fb5b..a853b3ed70 100644 --- a/dom/media/MediaData.h +++ b/dom/media/MediaData.h @@ -22,7 +22,6 @@ class Image; class ImageContainer; } // namespace layers -class MediaLargeByteBuffer; class MediaByteBuffer; class SharedTrackInfo; diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 23dd510066..f480bd9712 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -140,7 +140,7 @@ MediaDecoder::InitStatics() NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter) -NS_IMPL_ISUPPORTS(MediaDecoder, nsIObserver) +NS_IMPL_ISUPPORTS0(MediaDecoder) void MediaDecoder::NotifyOwnerActivityChanged() { @@ -166,7 +166,7 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) mPlayState == PLAY_STATE_SHUTDOWN || !mOwner->GetVideoFrameContainer() || (mOwner->GetMediaElement() && mOwner->GetMediaElement()->IsBeingDestroyed()) || - !mDecoderStateMachine->IsDormantNeeded()) + !mDormantSupported) { return; } @@ -333,9 +333,11 @@ bool MediaDecoder::IsInfinite() MediaDecoder::MediaDecoder() : mWatchManager(this, AbstractThread::MainThread()), + mBuffered(AbstractThread::MainThread(), TimeIntervals(), "MediaDecoder::mBuffered (Mirror)"), mNextFrameStatus(AbstractThread::MainThread(), MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED, "MediaDecoder::mNextFrameStatus (Mirror)"), + mDormantSupported(false), mDecoderPosition(0), mPlaybackPosition(0), mLogicalPosition(0.0), @@ -809,18 +811,6 @@ void MediaDecoder::PlaybackEnded() } } -NS_IMETHODIMP MediaDecoder::Observe(nsISupports *aSubjet, - const char *aTopic, - const char16_t *someData) -{ - MOZ_ASSERT(NS_IsMainThread()); - if (strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) { - Shutdown(); - } - - return NS_OK; -} - MediaDecoder::Statistics MediaDecoder::GetStatistics() { @@ -1104,6 +1094,8 @@ void MediaDecoder::DurationChanged() void MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) { + MOZ_ASSERT(NS_IsMainThread()); + if (mPlayState <= PLAY_STATE_LOADING) { return; } @@ -1260,10 +1252,12 @@ MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine) if (mDecoderStateMachine) { mStateMachineDuration.Connect(mDecoderStateMachine->CanonicalDuration()); + mBuffered.Connect(mDecoderStateMachine->CanonicalBuffered()); mNextFrameStatus.Connect(mDecoderStateMachine->CanonicalNextFrameStatus()); mCurrentPosition.Connect(mDecoderStateMachine->CanonicalCurrentPosition()); } else { mStateMachineDuration.DisconnectIfConnected(); + mBuffered.DisconnectIfConnected(); mNextFrameStatus.DisconnectIfConnected(); mCurrentPosition.DisconnectIfConnected(); } @@ -1295,8 +1289,7 @@ void MediaDecoder::Invalidate() // Constructs the time ranges representing what segments of the media // are buffered and playable. media::TimeIntervals MediaDecoder::GetBuffered() { - NS_ENSURE_TRUE(mDecoderStateMachine && !mShuttingDown, media::TimeIntervals::Invalid()); - return mDecoderStateMachine->GetBuffered(); + return mBuffered.Ref(); } size_t MediaDecoder::SizeOfVideoQueue() { @@ -1313,9 +1306,11 @@ size_t MediaDecoder::SizeOfAudioQueue() { return 0; } -void MediaDecoder::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) { +void MediaDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) { + MOZ_ASSERT(NS_IsMainThread()); + if (mDecoderStateMachine) { - mDecoderStateMachine->NotifyDataArrived(aBuffer, aLength, aOffset); + mDecoderStateMachine->DispatchNotifyDataArrived(aLength, aOffset, aThrottleUpdates); } // ReadyState computation depends on MediaDecoder::CanPlayThrough, which diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 76beba38eb..ebe3d47e41 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -274,7 +274,6 @@ public: typedef MediaPromise SeekPromise; NS_DECL_THREADSAFE_ISUPPORTS - NS_DECL_NSIOBSERVER // Enumeration for the valid play states (see mPlayState) enum PlayState { @@ -431,7 +430,8 @@ public: // Called as data arrives on the stream and is read into the cache. Called // on the main thread only. - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) override; + virtual void NotifyDataArrived(uint32_t aLength, int64_t aOffset, + bool aThrottleUpdates) override; // Called by MediaResource when the principal of the resource has // changed. Called on main thread only. @@ -451,6 +451,7 @@ public: // Call on the main thread only. virtual bool IsEndedOrShutdown() const; +protected: // Updates the media duration. This is called while the media is being // played, calls before the media has reached loaded metadata are ignored. // The duration is assumed to be an estimate, and so a degree of @@ -460,6 +461,7 @@ public: // changed, this causes a durationchanged event to fire to the media // element. void UpdateEstimatedMediaDuration(int64_t aDuration) override; +public: // Set a flag indicating whether seeking is supported virtual void SetMediaSeekable(bool aMediaSeekable) override; @@ -876,6 +878,9 @@ protected: // State-watching manager. WatchManager mWatchManager; + // Buffered range, mirrored from the reader. + Mirror mBuffered; + // NextFrameStatus, mirrored from the state machine. Mirror mNextFrameStatus; @@ -883,6 +888,9 @@ protected: * The following members should be accessed with the decoder lock held. ******/ + // Whether the decoder implementation supports dormant mode. + bool mDormantSupported; + // Current decoding position in the stream. This is where the decoder // is up to consuming the stream. This is not adjusted during decoder // seek operations, but it's updated at the end when we start playing diff --git a/dom/media/MediaDecoderReader.cpp b/dom/media/MediaDecoderReader.cpp index 76adfc1531..ab63823d8b 100644 --- a/dom/media/MediaDecoderReader.cpp +++ b/dom/media/MediaDecoderReader.cpp @@ -69,7 +69,12 @@ MediaDecoderReader::MediaDecoderReader(AbstractMediaDecoder* aDecoder, , mTaskQueue(aBorrowedTaskQueue ? aBorrowedTaskQueue : new MediaTaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK), /* aSupportsTailDispatch = */ true)) + , mWatchManager(this, mTaskQueue) + , mTimer(new MediaTimer()) + , mBuffered(mTaskQueue, TimeIntervals(), "MediaDecoderReader::mBuffered (Canonical)") , mDuration(mTaskQueue, NullableTimeUnit(), "MediaDecoderReader::mDuration (Mirror)") + , mThrottleDuration(TimeDuration::FromMilliseconds(500)) + , mLastThrottledNotify(TimeStamp::Now() - mThrottleDuration) , mIgnoreAudioOutputFormat(false) , mStartTime(-1) , mHitAudioDecodeError(false) @@ -92,6 +97,9 @@ MediaDecoderReader::InitializationTask() if (mDecoder->CanonicalDurationOrNull()) { mDuration.Connect(mDecoder->CanonicalDurationOrNull()); } + + // Initialize watchers. + mWatchManager.Watch(mDuration, &MediaDecoderReader::UpdateBuffered); } MediaDecoderReader::~MediaDecoderReader() @@ -161,16 +169,65 @@ VideoData* MediaDecoderReader::DecodeToFirstVideoData() } void -MediaDecoderReader::SetStartTime(int64_t aStartTime) +MediaDecoderReader::UpdateBuffered() { - mDecoder->GetReentrantMonitor().AssertCurrentThreadIn(); - MOZ_ASSERT(mStartTime == -1); - mStartTime = aStartTime; + MOZ_ASSERT(OnTaskQueue()); + NS_ENSURE_TRUE_VOID(!mShutdown); + mBuffered = GetBuffered(); +} + +void +MediaDecoderReader::ThrottledNotifyDataArrived(const Interval& aInterval) +{ + MOZ_ASSERT(OnTaskQueue()); + NS_ENSURE_TRUE_VOID(!mShutdown); + + if (mThrottledInterval.isNothing()) { + mThrottledInterval.emplace(aInterval); + } else if (!mThrottledInterval.ref().Contiguous(aInterval)) { + DoThrottledNotify(); + mThrottledInterval.emplace(aInterval); + } else { + mThrottledInterval = Some(mThrottledInterval.ref().Span(aInterval)); + } + + // If it's been long enough since our last update, do it. + if (TimeStamp::Now() - mLastThrottledNotify > mThrottleDuration) { + DoThrottledNotify(); + } else if (!mThrottledNotify.Exists()) { + // Otherwise, schedule an update if one isn't scheduled already. + nsRefPtr self = this; + mThrottledNotify.Begin( + mTimer->WaitUntil(mLastThrottledNotify + mThrottleDuration, __func__) + ->Then(TaskQueue(), __func__, + [self] () -> void { + self->mThrottledNotify.Complete(); + NS_ENSURE_TRUE_VOID(!self->mShutdown); + self->DoThrottledNotify(); + }, + [self] () -> void { + self->mThrottledNotify.Complete(); + NS_WARNING("throttle callback rejected"); + }) + ); + } +} + +void +MediaDecoderReader::DoThrottledNotify() +{ + MOZ_ASSERT(OnTaskQueue()); + mLastThrottledNotify = TimeStamp::Now(); + mThrottledNotify.DisconnectIfExists(); + Interval interval = mThrottledInterval.ref(); + mThrottledInterval.reset(); + NotifyDataArrived(interval); } media::TimeIntervals MediaDecoderReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); AutoPinned stream(mDecoder->GetResource()); @@ -353,8 +410,14 @@ MediaDecoderReader::Shutdown() mBaseAudioPromise.RejectIfExists(END_OF_STREAM, __func__); mBaseVideoPromise.RejectIfExists(END_OF_STREAM, __func__); + mThrottledNotify.DisconnectIfExists(); + ReleaseMediaResources(); mDuration.DisconnectIfConnected(); + mBuffered.DisconnectAll(); + + // Shut down the watch manager before shutting down our task queue. + mWatchManager.Shutdown(); nsRefPtr p; @@ -370,6 +433,7 @@ MediaDecoderReader::Shutdown() p = ShutdownPromise::CreateAndResolve(true, __func__); } + mTimer = nullptr; mDecoder = nullptr; return p; diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h index 96befc0e87..0977a04b02 100644 --- a/dom/media/MediaDecoderReader.h +++ b/dom/media/MediaDecoderReader.h @@ -11,7 +11,9 @@ #include "MediaData.h" #include "MediaPromise.h" #include "MediaQueue.h" +#include "MediaTimer.h" #include "AudioCompactor.h" +#include "Intervals.h" #include "TimeUnits.h" namespace mozilla { @@ -96,8 +98,6 @@ public: // True if this reader is waiting for a Content Decryption Module to become // available. virtual bool IsWaitingOnCDMResource() { return false; } - // True when this reader need to become dormant state - virtual bool IsDormantNeeded() { return false; } // Release media resources they should be released in dormant state // The reader can be made usable again by calling ReadMetadata(). virtual void ReleaseMediaResources() {}; @@ -203,8 +203,10 @@ public: mIgnoreAudioOutputFormat = true; } - // Populates aBuffered with the time ranges which are buffered. This function - // is called on the main, decode, and state machine threads. + // Populates aBuffered with the time ranges which are buffered. This may only + // be called on the decode task queue, and should only be used internally by + // UpdateBuffered - mBuffered (or mirrors of it) should be used for everything + // else. // // This base implementation in MediaDecoderReader estimates the time ranges // buffered by interpolating the cached byte ranges with the duration @@ -219,6 +221,9 @@ public: // called. virtual media::TimeIntervals GetBuffered(); + // Recomputes mBuffered. + virtual void UpdateBuffered(); + // MediaSourceReader opts out of the start-time-guessing mechanism. virtual bool ForceZeroStartTime() const { return false; } @@ -239,9 +244,38 @@ public: virtual size_t SizeOfVideoQueueInFrames(); virtual size_t SizeOfAudioQueueInFrames(); - // Only used by WebMReader and MediaOmxReader for now, so stub here rather - // than in every reader than inherits from MediaDecoderReader. - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) {} +protected: + friend class TrackBuffer; + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { } + + void NotifyDataArrived(const media::Interval& aInfo) + { + MOZ_ASSERT(OnTaskQueue()); + NS_ENSURE_TRUE_VOID(!mShutdown); + NotifyDataArrivedInternal(aInfo.Length(), aInfo.mStart); + UpdateBuffered(); + } + + // Invokes NotifyDataArrived while throttling the calls to occur at most every mThrottleDuration ms. + void ThrottledNotifyDataArrived(const media::Interval& aInterval); + void DoThrottledNotify(); + +public: + // In situations where these notifications come from stochastic network + // activity, we can save significant recomputation by throttling the delivery + // of these updates to the reader implementation. We don't want to do this + // throttling when the update comes from MSE code, since that code needs the + // updates to be observable immediately, and is generally less + // trigger-happy with notifications anyway. + void DispatchNotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) + { + RefPtr r = + NS_NewRunnableMethodWithArg>(this, aThrottleUpdates ? &MediaDecoderReader::ThrottledNotifyDataArrived + : &MediaDecoderReader::NotifyDataArrived, + media::Interval(aOffset, aOffset + aLength)); + TaskQueue()->Dispatch(r.forget(), AbstractThread::DontAssertDispatchSuccess); + } + // Notify the reader that data from the resource was evicted (MediaSource only) virtual void NotifyDataRemoved() {} virtual int64_t GetEvictionOffset(double aTime) { return -1; } @@ -262,7 +296,20 @@ public: // Indicates if the media is seekable. // ReadMetada should be called before calling this method. virtual bool IsMediaSeekable() = 0; - void SetStartTime(int64_t aStartTime); + + void DispatchSetStartTime(int64_t aStartTime) + { + nsRefPtr self = this; + nsCOMPtr r = + NS_NewRunnableFunction([self, aStartTime] () -> void + { + MOZ_ASSERT(self->OnTaskQueue()); + MOZ_ASSERT(self->mStartTime == -1); + self->mStartTime = aStartTime; + self->UpdateBuffered(); + }); + TaskQueue()->Dispatch(r.forget()); + } MediaTaskQueue* TaskQueue() { return mTaskQueue; @@ -321,12 +368,30 @@ protected: // Decode task queue. nsRefPtr mTaskQueue; + // State-watching manager. + WatchManager mWatchManager; + + // MediaTimer. + nsRefPtr mTimer; + + // Buffered range. + Canonical mBuffered; +public: + AbstractCanonical* CanonicalBuffered() { return &mBuffered; } +protected: + // Stores presentation info required for playback. MediaInfo mInfo; // Duration, mirrored from the state machine task queue. Mirror mDuration; + // State for ThrottledNotifyDataArrived. + MediaPromiseRequestHolder mThrottledNotify; + const TimeDuration mThrottleDuration; + TimeStamp mLastThrottledNotify; + Maybe> mThrottledInterval; + // Whether we should accept media that we know we can't play // directly, because they have a number of channel higher than // what we support. diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 9fdbbcc39d..7d64793b66 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -187,6 +187,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mDelayedScheduler(this), mState(DECODER_STATE_DECODING_NONE, "MediaDecoderStateMachine::mState"), mPlayDuration(0), + mBuffered(mTaskQueue, TimeIntervals(), "MediaDecoderStateMachine::mBuffered (Mirror)"), mDuration(mTaskQueue, NullableTimeUnit(), "MediaDecoderStateMachine::mDuration (Canonical"), mEstimatedDuration(mTaskQueue, NullableTimeUnit(), "MediaDecoderStateMachine::mEstimatedDuration (Mirror)"), @@ -296,6 +297,7 @@ MediaDecoderStateMachine::InitializationTask() MOZ_ASSERT(OnTaskQueue()); // Connect mirrors. + mBuffered.Connect(mReader->CanonicalBuffered()); mEstimatedDuration.Connect(mDecoder->CanonicalEstimatedDuration()); mExplicitDuration.Connect(mDecoder->CanonicalExplicitDuration()); mPlayState.Connect(mDecoder->CanonicalPlayState()); @@ -306,6 +308,7 @@ MediaDecoderStateMachine::InitializationTask() mPreservesPitch.Connect(mDecoder->CanonicalPreservesPitch()); // Initialize watchers. + mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated); mWatchManager.Watch(mState, &MediaDecoderStateMachine::UpdateNextFrameStatus); mWatchManager.Watch(mAudioCompleted, &MediaDecoderStateMachine::UpdateNextFrameStatus); mWatchManager.Watch(mVolume, &MediaDecoderStateMachine::VolumeChanged); @@ -1414,11 +1417,6 @@ void MediaDecoderStateMachine::RecomputeDuration() mDuration = Some(duration); } -bool MediaDecoderStateMachine::IsDormantNeeded() -{ - return mReader->IsDormantNeeded(); -} - void MediaDecoderStateMachine::SetDormant(bool aDormant) { MOZ_ASSERT(OnTaskQueue()); @@ -1617,33 +1615,20 @@ void MediaDecoderStateMachine::LogicallySeekingChanged() ScheduleStateMachine(); } -void MediaDecoderStateMachine::NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) +void MediaDecoderStateMachine::BufferedRangeUpdated() { - NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); - mReader->NotifyDataArrived(aBuffer, aLength, aOffset); + MOZ_ASSERT(OnTaskQueue()); - // While playing an unseekable stream of unknown duration, mDuration is - // updated (in AdvanceFrame()) as we play. But if data is being downloaded - // faster than played, mDuration won't reflect the end of playable data + // While playing an unseekable stream of unknown duration, mObservedDuration + // is updated (in AdvanceFrame()) as we play. But if data is being downloaded + // faster than played, mObserved won't reflect the end of playable data // since we haven't played the frame at the end of buffered data. So update - // mDuration here as new data is downloaded to prevent such a lag. - // - // Make sure to only do this if we have a start time, otherwise the reader - // doesn't know how to compute GetBuffered. - if (!mDecoder->IsInfinite() || !HaveStartTime()) - { - return; - } - - media::TimeIntervals buffered{mDecoder->GetBuffered()}; - if (!buffered.IsInvalid()) { + // mObservedDuration here as new data is downloaded to prevent such a lag. + if (!mBuffered.Ref().IsInvalid()) { bool exists; - media::TimeUnit end{buffered.GetEnd(&exists)}; + media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)}; if (exists) { - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDuration = Some(std::max(Duration(), end)); + mObservedDuration = std::max(mObservedDuration.Ref(), end); } } } @@ -2038,17 +2023,16 @@ bool MediaDecoderStateMachine::HasLowUndecodedData(int64_t aUsecs) MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); NS_ASSERTION(mState > DECODER_STATE_DECODING_FIRSTFRAME, - "Must have loaded first frame for GetBuffered() to work"); + "Must have loaded first frame for mBuffered to be valid"); - // If we don't have a duration, GetBuffered is probably not going to produce + // If we don't have a duration, mBuffered is probably not going to have // a useful buffered range. Return false here so that we don't get stuck in // buffering mode for live streams. if (Duration().IsInfinite()) { return false; } - media::TimeIntervals buffered{mReader->GetBuffered()}; - if (buffered.IsInvalid()) { + if (mBuffered.Ref().IsInvalid()) { return false; } @@ -2070,7 +2054,7 @@ bool MediaDecoderStateMachine::HasLowUndecodedData(int64_t aUsecs) } media::TimeInterval interval(media::TimeUnit::FromMicroseconds(endOfDecodedData), media::TimeUnit::FromMicroseconds(std::min(endOfDecodedData + aUsecs, Duration().ToMicroseconds()))); - return endOfDecodedData != INT64_MAX && !buffered.Contains(interval); + return endOfDecodedData != INT64_MAX && !mBuffered.Ref().Contains(interval); } void @@ -2122,8 +2106,7 @@ MediaDecoderStateMachine::OnMetadataRead(MetadataHolder* aMetadata) mStartTimeRendezvous->AwaitStartTime()->Then(TaskQueue(), __func__, [self] () -> void { NS_ENSURE_TRUE_VOID(!self->IsShutdown()); - ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor()); - self->mReader->SetStartTime(self->StartTime()); + self->mReader->DispatchSetStartTime(self->StartTime()); }, [] () -> void { NS_WARNING("Setting start time on reader failed"); } ); @@ -2462,6 +2445,7 @@ MediaDecoderStateMachine::FinishShutdown() VideoQueue().ClearListeners(); // Disconnect canonicals and mirrors before shutting down our task queue. + mBuffered.DisconnectIfConnected(); mEstimatedDuration.DisconnectIfConnected(); mExplicitDuration.DisconnectIfConnected(); mPlayState.DisconnectIfConnected(); diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index d79101d8a5..296a293fa8 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -149,8 +149,6 @@ public: void AddOutputStream(ProcessedMediaStream* aStream, bool aFinishWhenEnded); - // Check if the decoder needs to become dormant state. - bool IsDormantNeeded(); // Set/Unset dormant state. void SetDormant(bool aDormant); @@ -257,11 +255,6 @@ public: return mState == DECODER_STATE_SEEKING; } - media::TimeIntervals GetBuffered() { - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - return mReader->GetBuffered(); - } - size_t SizeOfVideoQueue() { if (mReader) { return mReader->SizeOfVideoQueueInBytes(); @@ -276,7 +269,12 @@ public: return 0; } - void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset); + void DispatchNotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) + { + mReader->DispatchNotifyDataArrived(aLength, aOffset, aThrottleUpdates); + } + + AbstractCanonical* CanonicalBuffered() { return mReader->CanonicalBuffered(); } // Returns the state machine task queue. MediaTaskQueue* TaskQueue() const { return mTaskQueue; } @@ -395,6 +393,8 @@ protected: void SetState(State aState); + void BufferedRangeUpdated(); + // Inserts MediaData* samples into their respective MediaQueues. // aSample must not be null. void Push(AudioData* aSample); @@ -941,6 +941,9 @@ private: // buffering. TimeStamp mBufferingStart; + // The buffered range. Mirrored from the decoder thread. + Mirror mBuffered; + // Duration of the media. This is guaranteed to be non-null after we finish // decoding the first frame. Canonical mDuration; diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 384318d4b7..68852760fe 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -76,10 +76,6 @@ MediaFormatReader::MediaFormatReader(AbstractMediaDecoder* aDecoder, , mSeekable(false) , mIsEncrypted(false) , mTrackDemuxersMayBlock(false) - , mCachedTimeRangesStale(true) -#if defined(READER_DORMANT_HEURISTIC) - , mDormantEnabled(Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false)) -#endif { MOZ_ASSERT(aDemuxer); MOZ_COUNT_CTOR(MediaFormatReader); @@ -741,11 +737,8 @@ MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) decoder.mWaitingForData = false; bool hasLastEnd; media::TimeUnit lastEnd = decoder.mTimeRanges.GetEnd(&hasLastEnd); - { - MonitorAutoLock lock(decoder.mMonitor); - // Update our cached TimeRange. - decoder.mTimeRanges = decoder.mTrackDemuxer->GetBuffered(); - } + // Update our cached TimeRange. + decoder.mTimeRanges = decoder.mTrackDemuxer->GetBuffered(); if (decoder.mTimeRanges.Length() && (!hasLastEnd || decoder.mTimeRanges.GetEnd() < lastEnd)) { // New data was added after our previous end, we can clear the EOS flag. @@ -1373,6 +1366,7 @@ MediaFormatReader::GetEvictionOffset(double aTime) media::TimeIntervals MediaFormatReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); media::TimeIntervals videoti; media::TimeIntervals audioti; media::TimeIntervals intervals; @@ -1386,68 +1380,30 @@ MediaFormatReader::GetBuffered() NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); startTime = mStartTime; } - if (NS_IsMainThread()) { - if (mCachedTimeRangesStale) { - MOZ_ASSERT(mMainThreadDemuxer); - if (!mDataRange.IsEmpty()) { - mMainThreadDemuxer->NotifyDataArrived(mDataRange.Length(), mDataRange.mStart); - } - if (mVideoTrackDemuxer) { - videoti = mVideoTrackDemuxer->GetBuffered(); - } - if (mAudioTrackDemuxer) { - audioti = mAudioTrackDemuxer->GetBuffered(); - } - if (HasAudio() && HasVideo()) { - mCachedTimeRanges = media::Intersection(Move(videoti), Move(audioti)); - } else if (HasAudio()) { - mCachedTimeRanges = Move(audioti); - } else if (HasVideo()) { - mCachedTimeRanges = Move(videoti); - } - mDataRange = ByteInterval(); - mCachedTimeRangesStale = false; - } - intervals = mCachedTimeRanges; - } else { - if (OnTaskQueue()) { - // Ensure we have up to date buffered time range. - if (HasVideo()) { - UpdateReceivedNewData(TrackType::kVideoTrack); - } - if (HasAudio()) { - UpdateReceivedNewData(TrackType::kAudioTrack); - } - } - if (HasVideo()) { - MonitorAutoLock lock(mVideo.mMonitor); - videoti = mVideo.mTimeRanges; - } - if (HasAudio()) { - MonitorAutoLock lock(mAudio.mMonitor); - audioti = mAudio.mTimeRanges; - } + // Ensure we have up to date buffered time range. + if (HasVideo()) { + UpdateReceivedNewData(TrackType::kVideoTrack); + } + if (HasAudio()) { + UpdateReceivedNewData(TrackType::kAudioTrack); + } + if (HasVideo()) { + videoti = mVideo.mTimeRanges; + } + if (HasAudio()) { + audioti = mAudio.mTimeRanges; + } if (HasAudio() && HasVideo()) { - intervals = media::Intersection(Move(videoti), Move(audioti)); - } else if (HasAudio()) { - intervals = Move(audioti); - } else if (HasVideo()) { - intervals = Move(videoti); - } + intervals = media::Intersection(Move(videoti), Move(audioti)); + } else if (HasAudio()) { + intervals = Move(audioti); + } else if (HasVideo()) { + intervals = Move(videoti); } return intervals.Shift(media::TimeUnit::FromMicroseconds(-startTime)); } -bool MediaFormatReader::IsDormantNeeded() -{ -#if defined(READER_DORMANT_HEURISTIC) - return mDormantEnabled; -#else - return false; -#endif -} - void MediaFormatReader::ReleaseMediaResources() { // Before freeing a video codec, all video buffers needed to be released @@ -1504,17 +1460,10 @@ MediaFormatReader::NotifyDemuxer(uint32_t aLength, int64_t aOffset) } void -MediaFormatReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +MediaFormatReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - MOZ_ASSERT(NS_IsMainThread()); - - MOZ_ASSERT(aBuffer || aLength); - if (mDataRange.IsEmpty()) { - mDataRange = ByteInterval(aOffset, aOffset + aLength); - } else { - mDataRange = mDataRange.Span(ByteInterval(aOffset, aOffset + aLength)); - } - mCachedTimeRangesStale = true; + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(aLength); if (!mInitDone || mShutdown) { return; @@ -1522,35 +1471,34 @@ MediaFormatReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int6 MOZ_ASSERT(mMainThreadDemuxer); - // Queue a task to notify our main demuxer. - RefPtr task = - NS_NewRunnableMethodWithArgs( - this, &MediaFormatReader::NotifyDemuxer, + // Queue a task to notify our main thread demuxer. + nsCOMPtr task = + NS_NewRunnableMethodWithArgs( + mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataArrived, aLength, aOffset); - TaskQueue()->Dispatch(task.forget()); + AbstractThread::MainThread()->Dispatch(task.forget()); + + NotifyDemuxer(aLength, aOffset); } void MediaFormatReader::NotifyDataRemoved() { - MOZ_ASSERT(NS_IsMainThread()); - - mDataRange = ByteInterval(); - mCachedTimeRangesStale = true; + MOZ_ASSERT(OnTaskQueue()); if (!mInitDone || mShutdown) { return; } MOZ_ASSERT(mMainThreadDemuxer); - mMainThreadDemuxer->NotifyDataRemoved(); - // Queue a task to notify our main demuxer. - RefPtr task = - NS_NewRunnableMethodWithArgs( - this, &MediaFormatReader::NotifyDemuxer, - 0, 0); - TaskQueue()->Dispatch(task.forget()); + // Queue a task to notify our main thread demuxer. + nsCOMPtr task = + NS_NewRunnableMethod( + mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataRemoved); + AbstractThread::MainThread()->Dispatch(task.forget()); + + NotifyDemuxer(0, 0); } bool diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 6c1b032d45..3c6cb1c4cc 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -9,7 +9,6 @@ #include "mozilla/Atomics.h" #include "mozilla/Maybe.h" -#include "mozilla/Monitor.h" #include "MediaDataDemuxer.h" #include "MediaDecoderReader.h" #include "MediaTaskQueue.h" @@ -17,12 +16,6 @@ namespace mozilla { -#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) -#define READER_DORMANT_HEURISTIC -#else -#undef READER_DORMANT_HEURISTIC -#endif - class MediaFormatReader final : public MediaDecoderReader { typedef TrackInfo::TrackType TrackType; @@ -68,9 +61,9 @@ public: } int64_t GetEvictionOffset(double aTime) override; - void NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) override; +protected: + void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; +public: void NotifyDataRemoved() override; media::TimeIntervals GetBuffered() override; @@ -79,7 +72,6 @@ public: // For Media Resource Management void SetIdle() override; - bool IsDormantNeeded() override; void ReleaseMediaResources() override; void SetSharedDecoderManager(SharedDecoderManager* aManager) override; @@ -205,8 +197,6 @@ private: , mNumSamplesOutput(0) , mSizeOfQueue(0) , mLastStreamSourceID(UINT32_MAX) - , mMonitor(aType == MediaData::AUDIO_DATA ? "audio decoder data" - : "video decoder data") {} MediaFormatReader* mOwner; @@ -298,9 +288,6 @@ private: // Sample format monitoring. uint32_t mLastStreamSourceID; Maybe mNextStreamSourceID; - // Monitor that protects all non-threadsafe state; the primitives - // that follow. - Monitor mMonitor; media::TimeIntervals mTimeRanges; nsRefPtr mInfo; }; @@ -419,9 +406,6 @@ private: nsRefPtr mMainThreadDemuxer; nsRefPtr mAudioTrackDemuxer; nsRefPtr mVideoTrackDemuxer; - ByteInterval mDataRange; - media::TimeIntervals mCachedTimeRanges; - bool mCachedTimeRangesStale; #if defined(READER_DORMANT_HEURISTIC) const bool mDormantEnabled; diff --git a/dom/media/MediaResource.cpp b/dom/media/MediaResource.cpp index 1894684740..70abf83cc1 100644 --- a/dom/media/MediaResource.cpp +++ b/dom/media/MediaResource.cpp @@ -457,7 +457,8 @@ ChannelMediaResource::CopySegmentToCache(nsIInputStream *aInStream, { CopySegmentClosure* closure = static_cast(aClosure); - closure->mResource->mDecoder->NotifyDataArrived(aFromSegment, aCount, closure->mResource->mOffset); + closure->mResource->mDecoder->NotifyDataArrived(aCount, closure->mResource->mOffset, + /* aThrottleUpdates = */ true); // Keep track of where we're up to. RESOURCE_LOG("%p [ChannelMediaResource]: CopySegmentToCache at mOffset [%lld] add " diff --git a/dom/media/TimeUnits.h b/dom/media/TimeUnits.h index ee3918fdd3..78252a1676 100644 --- a/dom/media/TimeUnits.h +++ b/dom/media/TimeUnits.h @@ -259,7 +259,7 @@ public: return TimeIntervals(TimeInterval(TimeUnit::FromMicroseconds(INT64_MIN), TimeUnit::FromMicroseconds(INT64_MIN))); } - bool IsInvalid() + bool IsInvalid() const { return Length() == 1 && Start(0).ToMicroseconds() == INT64_MIN && End(0).ToMicroseconds() == INT64_MIN; diff --git a/dom/media/apple/AppleMP3Reader.cpp b/dom/media/apple/AppleMP3Reader.cpp index ebab09cdc9..eb06d9004a 100644 --- a/dom/media/apple/AppleMP3Reader.cpp +++ b/dom/media/apple/AppleMP3Reader.cpp @@ -387,7 +387,7 @@ AppleMP3Reader::ReadMetadata(MediaInfo* aInfo, bytes, 0 /* flags */); - mMP3FrameParser.Parse(bytes, numBytes, offset); + mMP3FrameParser.Parse(reinterpret_cast(bytes), numBytes, offset); offset += numBytes; @@ -524,16 +524,16 @@ AppleMP3Reader::Seek(int64_t aTime, int64_t aEndTime) } void -AppleMP3Reader::NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) +AppleMP3Reader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(OnTaskQueue()); if (!mMP3FrameParser.NeedsData()) { return; } - mMP3FrameParser.Parse(aBuffer, aLength, aOffset); + nsRefPtr bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength); + NS_ENSURE_TRUE_VOID(bytes); + mMP3FrameParser.Parse(bytes->Elements(), aLength, aOffset); if (!mMP3FrameParser.IsMP3()) { return; } @@ -542,9 +542,8 @@ AppleMP3Reader::NotifyDataArrived(const char* aBuffer, if (duration != mDuration) { LOGD("Updating media duration to %lluus\n", duration); MOZ_ASSERT(mDecoder); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDuration = duration; - mDecoder->UpdateEstimatedMediaDuration(duration); + mDecoder->DispatchUpdateEstimatedMediaDuration(duration); } } diff --git a/dom/media/apple/AppleMP3Reader.h b/dom/media/apple/AppleMP3Reader.h index a1542a5ce8..d65787732c 100644 --- a/dom/media/apple/AppleMP3Reader.h +++ b/dom/media/apple/AppleMP3Reader.h @@ -45,9 +45,10 @@ public: AudioFileStreamPropertyID aPropertyID, UInt32 *aFlags); - virtual void NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) override; +protected: + virtual void NotifyDataArrivedInternal(uint32_t aLength, + int64_t aOffset) override; +public: virtual bool IsMediaSeekable() override; diff --git a/dom/media/directshow/DirectShowReader.cpp b/dom/media/directshow/DirectShowReader.cpp index fdef04b8bd..5b2b2a0e77 100644 --- a/dom/media/directshow/DirectShowReader.cpp +++ b/dom/media/directshow/DirectShowReader.cpp @@ -84,7 +84,7 @@ ParseMP3Headers(MP3FrameParser *aParser, MediaResource *aResource) return NS_ERROR_FAILURE; } - aParser->Parse(buffer, bytesRead, offset); + aParser->Parse(reinterpret_cast(buffer), bytesRead, offset); offset += bytesRead; } @@ -401,14 +401,16 @@ DirectShowReader::SeekInternal(int64_t aTargetUs) } void -DirectShowReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +DirectShowReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(OnTaskQueue()); if (!mMP3FrameParser.NeedsData()) { return; } - mMP3FrameParser.Parse(aBuffer, aLength, aOffset); + nsRefPtr bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength); + NS_ENSURE_TRUE_VOID(bytes); + mMP3FrameParser.Parse(bytes->Elements(), aLength, aOffset); if (!mMP3FrameParser.IsMP3()) { return; } @@ -416,9 +418,8 @@ DirectShowReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64 int64_t duration = mMP3FrameParser.GetDuration(); if (duration != mDuration) { MOZ_ASSERT(mDecoder); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); mDuration = duration; - mDecoder->UpdateEstimatedMediaDuration(mDuration); + mDecoder->DispatchUpdateEstimatedMediaDuration(mDuration); } } diff --git a/dom/media/directshow/DirectShowReader.h b/dom/media/directshow/DirectShowReader.h index 792121a027..cc51c844ad 100644 --- a/dom/media/directshow/DirectShowReader.h +++ b/dom/media/directshow/DirectShowReader.h @@ -58,9 +58,10 @@ public: nsRefPtr Seek(int64_t aTime, int64_t aEndTime) override; - void NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) override; +protected: + void NotifyDataArrivedInternal(uint32_t aLength, + int64_t aOffset) override; +public: bool IsMediaSeekable() override; diff --git a/dom/media/fmp4/MP4Decoder.cpp b/dom/media/fmp4/MP4Decoder.cpp index afe880145f..c584b0e730 100644 --- a/dom/media/fmp4/MP4Decoder.cpp +++ b/dom/media/fmp4/MP4Decoder.cpp @@ -32,6 +32,19 @@ namespace mozilla { +#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) +#define MP4_READER_DORMANT_HEURISTIC +#else +#undef MP4_READER_DORMANT_HEURISTIC +#endif + +MP4Decoder::MP4Decoder() +{ +#if defined(MP4_READER_DORMANT_HEURISTIC) + mDormantSupported = Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false); +#endif +} + MediaDecoderStateMachine* MP4Decoder::CreateStateMachine() { bool useFormatDecoder = diff --git a/dom/media/fmp4/MP4Decoder.h b/dom/media/fmp4/MP4Decoder.h index 229a78c51c..a4bcd14d1a 100644 --- a/dom/media/fmp4/MP4Decoder.h +++ b/dom/media/fmp4/MP4Decoder.h @@ -14,6 +14,7 @@ namespace mozilla { class MP4Decoder : public MediaDecoder { public: + MP4Decoder(); virtual MediaDecoder* Clone() override { if (!IsEnabled()) { diff --git a/dom/media/fmp4/MP4Demuxer.cpp b/dom/media/fmp4/MP4Demuxer.cpp index 209eef356d..86b6b2f9fc 100644 --- a/dom/media/fmp4/MP4Demuxer.cpp +++ b/dom/media/fmp4/MP4Demuxer.cpp @@ -65,7 +65,7 @@ private: MP4Demuxer::MP4Demuxer(MediaResource* aResource) : mResource(aResource) , mStream(new mp4_demuxer::ResourceStream(aResource)) - , mInitData(new MediaLargeByteBuffer) + , mInitData(new MediaByteBuffer) { } diff --git a/dom/media/fmp4/MP4Demuxer.h b/dom/media/fmp4/MP4Demuxer.h index 5fd9b34ccb..ec85c3f359 100644 --- a/dom/media/fmp4/MP4Demuxer.h +++ b/dom/media/fmp4/MP4Demuxer.h @@ -50,7 +50,7 @@ private: friend class MP4TrackDemuxer; nsRefPtr mResource; nsRefPtr mStream; - nsRefPtr mInitData; + nsRefPtr mInitData; UniquePtr mMetadata; nsTArray> mDemuxers; }; diff --git a/dom/media/fmp4/MP4Reader.cpp b/dom/media/fmp4/MP4Reader.cpp index a4f1873b77..38e9bb0a8b 100644 --- a/dom/media/fmp4/MP4Reader.cpp +++ b/dom/media/fmp4/MP4Reader.cpp @@ -152,9 +152,6 @@ MP4Reader::MP4Reader(AbstractMediaDecoder* aDecoder, MediaTaskQueue* aBorrowedTa , mIndexReady(false) , mLastSeenEnd(-1) , mDemuxerMonitor("MP4 Demuxer") -#if defined(MP4_READER_DORMANT_HEURISTIC) - , mDormantEnabled(Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false)) -#endif { MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); MOZ_COUNT_CTOR(MP4Reader); @@ -1001,6 +998,7 @@ MP4Reader::GetEvictionOffset(double aTime) media::TimeIntervals MP4Reader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); MonitorAutoLock mon(mDemuxerMonitor); media::TimeIntervals buffered; if (!mIndexReady) { @@ -1026,15 +1024,6 @@ MP4Reader::GetBuffered() return buffered; } -bool MP4Reader::IsDormantNeeded() -{ -#if defined(MP4_READER_DORMANT_HEURISTIC) - return mDormantEnabled; -#else - return false; -#endif -} - void MP4Reader::ReleaseMediaResources() { // Before freeing a video codec, all video buffers needed to be released @@ -1066,14 +1055,9 @@ MP4Reader::SetSharedDecoderManager(SharedDecoderManager* aManager) } void -MP4Reader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +MP4Reader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - MOZ_ASSERT(NS_IsMainThread()); - - if (mShutdown) { - return; - } - + MOZ_ASSERT(OnTaskQueue()); if (mLastSeenEnd < 0) { MonitorAutoLock mon(mDemuxerMonitor); mLastSeenEnd = mDecoder->GetResource()->GetLength(); diff --git a/dom/media/fmp4/MP4Reader.h b/dom/media/fmp4/MP4Reader.h index 4d1473997a..6ec100d403 100644 --- a/dom/media/fmp4/MP4Reader.h +++ b/dom/media/fmp4/MP4Reader.h @@ -23,12 +23,6 @@ typedef std::deque> MediaSampleQueue; class MP4Stream; -#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) -#define MP4_READER_DORMANT_HEURISTIC -#else -#undef MP4_READER_DORMANT_HEURISTIC -#endif - class MP4Reader final : public MediaDecoderReader { typedef TrackInfo::TrackType TrackType; @@ -62,13 +56,15 @@ public: virtual bool IsMediaSeekable() override; virtual int64_t GetEvictionOffset(double aTime) override; - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) override; + +protected: + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; +public: virtual media::TimeIntervals GetBuffered() override; // For Media Resource Management virtual void SetIdle() override; - virtual bool IsDormantNeeded() override; virtual void ReleaseMediaResources() override; virtual void SetSharedDecoderManager(SharedDecoderManager* aManager) override; @@ -272,10 +268,6 @@ private: int64_t mLastSeenEnd; Monitor mDemuxerMonitor; nsRefPtr mSharedDecoderManager; - -#if defined(MP4_READER_DORMANT_HEURISTIC) - const bool mDormantEnabled; -#endif }; } // namespace mozilla diff --git a/dom/media/gtest/TestMP4Reader.cpp b/dom/media/gtest/TestMP4Reader.cpp index 4c7c919111..66a22810c0 100644 --- a/dom/media/gtest/TestMP4Reader.cpp +++ b/dom/media/gtest/TestMP4Reader.cpp @@ -36,12 +36,9 @@ public: decoder->SetResource(resource); reader->Init(nullptr); - { - // This needs to be done before invoking GetBuffered. This is normally - // done by MediaDecoderStateMachine. - ReentrantMonitorAutoEnter mon(decoder->GetReentrantMonitor()); - reader->SetStartTime(0); - } + // This needs to be done before invoking GetBuffered. This is normally + // done by MediaDecoderStateMachine. + reader->DispatchSetStartTime(0); } void Init() { diff --git a/dom/media/mediasource/ContainerParser.cpp b/dom/media/mediasource/ContainerParser.cpp index b2687e70ca..1beee08e92 100644 --- a/dom/media/mediasource/ContainerParser.cpp +++ b/dom/media/mediasource/ContainerParser.cpp @@ -40,7 +40,7 @@ ContainerParser::ContainerParser(const nsACString& aType) } bool -ContainerParser::IsInitSegmentPresent(MediaLargeByteBuffer* aData) +ContainerParser::IsInitSegmentPresent(MediaByteBuffer* aData) { MSE_DEBUG(ContainerParser, "aLength=%u [%x%x%x%x]", aData->Length(), @@ -52,7 +52,7 @@ return false; } bool -ContainerParser::IsMediaSegmentPresent(MediaLargeByteBuffer* aData) +ContainerParser::IsMediaSegmentPresent(MediaByteBuffer* aData) { MSE_DEBUG(ContainerParser, "aLength=%u [%x%x%x%x]", aData->Length(), @@ -64,7 +64,7 @@ ContainerParser::IsMediaSegmentPresent(MediaLargeByteBuffer* aData) } bool -ContainerParser::ParseStartAndEndTimestamps(MediaLargeByteBuffer* aData, +ContainerParser::ParseStartAndEndTimestamps(MediaByteBuffer* aData, int64_t& aStart, int64_t& aEnd) { return false; @@ -89,7 +89,7 @@ ContainerParser::HasCompleteInitData() return mHasInitData && !!mInitData->Length(); } -MediaLargeByteBuffer* +MediaByteBuffer* ContainerParser::InitData() { return mInitData; @@ -124,7 +124,7 @@ public: static const unsigned NS_PER_USEC = 1000; static const unsigned USEC_PER_SEC = 1000000; - bool IsInitSegmentPresent(MediaLargeByteBuffer* aData) override + bool IsInitSegmentPresent(MediaByteBuffer* aData) override { ContainerParser::IsInitSegmentPresent(aData); // XXX: This is overly primitive, needs to collect data as it's appended @@ -149,7 +149,7 @@ public: return false; } - bool IsMediaSegmentPresent(MediaLargeByteBuffer* aData) override + bool IsMediaSegmentPresent(MediaByteBuffer* aData) override { ContainerParser::IsMediaSegmentPresent(aData); // XXX: This is overly primitive, needs to collect data as it's appended @@ -188,7 +188,7 @@ public: return false; } - bool ParseStartAndEndTimestamps(MediaLargeByteBuffer* aData, + bool ParseStartAndEndTimestamps(MediaByteBuffer* aData, int64_t& aStart, int64_t& aEnd) override { bool initSegment = IsInitSegmentPresent(aData); @@ -196,7 +196,7 @@ public: mOffset = 0; mParser = WebMBufferedParser(0); mOverlappedMapping.Clear(); - mInitData = new MediaLargeByteBuffer(); + mInitData = new MediaByteBuffer(); mResource = new SourceBufferResource(NS_LITERAL_CSTRING("video/webm")); } @@ -281,7 +281,7 @@ public: , mMonitor("MP4ContainerParser Index Monitor") {} - bool IsInitSegmentPresent(MediaLargeByteBuffer* aData) override + bool IsInitSegmentPresent(MediaByteBuffer* aData) override { ContainerParser::IsInitSegmentPresent(aData); // Each MP4 atom has a chunk size and chunk type. The root chunk in an MP4 @@ -291,7 +291,7 @@ public: return parser.StartWithInitSegment(); } - bool IsMediaSegmentPresent(MediaLargeByteBuffer* aData) override + bool IsMediaSegmentPresent(MediaByteBuffer* aData) override { AtomParser parser(mType, aData); return parser.StartWithMediaSegment(); @@ -300,7 +300,7 @@ public: private: class AtomParser { public: - AtomParser(const nsACString& aType, const MediaLargeByteBuffer* aData) + AtomParser(const nsACString& aType, const MediaByteBuffer* aData) { const nsCString mType(aType); // for logging macro. mp4_demuxer::ByteReader reader(aData); @@ -361,7 +361,7 @@ private: }; public: - bool ParseStartAndEndTimestamps(MediaLargeByteBuffer* aData, + bool ParseStartAndEndTimestamps(MediaByteBuffer* aData, int64_t& aStart, int64_t& aEnd) override { MonitorAutoLock mon(mMonitor); // We're not actually racing against anything, @@ -375,7 +375,7 @@ public: // manually. This allows the ContainerParser to be shared across different // timestampOffsets. mParser = new mp4_demuxer::MoofParser(mStream, 0, /* aIsAudio = */ false, &mMonitor); - mInitData = new MediaLargeByteBuffer(); + mInitData = new MediaByteBuffer(); } else if (!mStream || !mParser) { return false; } diff --git a/dom/media/mediasource/ContainerParser.h b/dom/media/mediasource/ContainerParser.h index 9077081216..6b31878881 100644 --- a/dom/media/mediasource/ContainerParser.h +++ b/dom/media/mediasource/ContainerParser.h @@ -13,7 +13,7 @@ namespace mozilla { -class MediaLargeByteBuffer; +class MediaByteBuffer; class SourceBufferResource; class ContainerParser { @@ -24,17 +24,17 @@ public: // Return true if aData starts with an initialization segment. // The base implementation exists only for debug logging and is expected // to be called first from the overriding implementation. - virtual bool IsInitSegmentPresent(MediaLargeByteBuffer* aData); + virtual bool IsInitSegmentPresent(MediaByteBuffer* aData); // Return true if aData starts with a media segment. // The base implementation exists only for debug logging and is expected // to be called first from the overriding implementation. - virtual bool IsMediaSegmentPresent(MediaLargeByteBuffer* aData); + virtual bool IsMediaSegmentPresent(MediaByteBuffer* aData); // Parse aData to extract the start and end frame times from the media // segment. aData may not start on a parser sync boundary. Return true // if aStart and aEnd have been updated. - virtual bool ParseStartAndEndTimestamps(MediaLargeByteBuffer* aData, + virtual bool ParseStartAndEndTimestamps(MediaByteBuffer* aData, int64_t& aStart, int64_t& aEnd); // Compare aLhs and rHs, considering any error that may exist in the @@ -44,7 +44,7 @@ public: virtual int64_t GetRoundingError(); - MediaLargeByteBuffer* InitData(); + MediaByteBuffer* InitData(); bool HasInitData() { @@ -65,7 +65,7 @@ public: static ContainerParser* CreateForMIMEType(const nsACString& aType); protected: - nsRefPtr mInitData; + nsRefPtr mInitData; nsRefPtr mResource; bool mHasInitData; MediaByteRange mCompleteInitSegmentRange; diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp index ebeec7d83a..5b6acf9ad9 100644 --- a/dom/media/mediasource/MediaSourceDecoder.cpp +++ b/dom/media/mediasource/MediaSourceDecoder.cpp @@ -88,7 +88,7 @@ MediaSourceDecoder::GetSeekable() if (IsNaN(duration)) { // Return empty range. } else if (duration > 0 && mozilla::IsInfinite(duration)) { - media::TimeIntervals buffered = mReader->GetBuffered(); + media::TimeIntervals buffered = GetBuffered(); // 1. If live seekable range is not empty: if (mMediaSource->HasLiveSeekableRange()) { @@ -116,6 +116,44 @@ MediaSourceDecoder::GetSeekable() return seekable; } +media::TimeIntervals +MediaSourceDecoder::GetBuffered() +{ + MOZ_ASSERT(NS_IsMainThread()); + + dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers(); + media::TimeUnit highestEndTime; + nsTArray activeRanges; + media::TimeIntervals buffered; + + for (uint32_t i = 0; i < sourceBuffers->Length(); i++) { + bool found; + dom::SourceBuffer* sb = sourceBuffers->IndexedGetter(i, found); + MOZ_ASSERT(found); + + activeRanges.AppendElement(sb->GetTimeIntervals()); + highestEndTime = + std::max(highestEndTime, activeRanges.LastElement().GetEnd()); + } + + buffered += + media::TimeInterval(media::TimeUnit::FromMicroseconds(0), highestEndTime); + + for (auto& range : activeRanges) { + if (mEnded && range.Length()) { + // Set the end time on the last range to highestEndTime by adding a + // new range spanning the current end time to highestEndTime, which + // Normalize() will then merge with the old last range. + range += + media::TimeInterval(range.GetEnd(), highestEndTime); + } + buffered.Intersection(range); + } + + MSE_DEBUG("ranges=%s", DumpTimeRanges(buffered).get()); + return buffered; +} + void MediaSourceDecoder::Shutdown() { @@ -334,38 +372,6 @@ MediaSourceDecoder::CanPlayThrough() return GetBuffered().Contains(interval); } -already_AddRefed -MediaSourceDecoder::SelectDecoder(int64_t aTarget, - int64_t aTolerance, - const nsTArray>& aTrackDecoders) -{ - MOZ_ASSERT(!mIsUsingFormatReader); - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - - media::TimeUnit target{media::TimeUnit::FromMicroseconds(aTarget)}; - media::TimeUnit tolerance{media::TimeUnit::FromMicroseconds(aTolerance + aTarget)}; - - // aTolerance gives a slight bias toward the start of a range only. - // Consider decoders in order of newest to oldest, as a newer decoder - // providing a given buffered range is expected to replace an older one. - for (int32_t i = aTrackDecoders.Length() - 1; i >= 0; --i) { - nsRefPtr newDecoder = aTrackDecoders[i]; - - media::TimeIntervals ranges = newDecoder->GetBuffered(); - for (uint32_t j = 0; j < ranges.Length(); j++) { - if (target < ranges.End(j) && tolerance >= ranges.Start(j)) { - return newDecoder.forget(); - } - } - - MSE_DEBUGV("SelectDecoder(%lld fuzz:%lld) newDecoder=%p (%d/%d) target not in ranges=%s", - aTarget, aTolerance, newDecoder.get(), i+1, - aTrackDecoders.Length(), DumpTimeRanges(ranges).get()); - } - - return nullptr; -} - #undef MSE_DEBUG #undef MSE_DEBUGV diff --git a/dom/media/mediasource/MediaSourceDecoder.h b/dom/media/mediasource/MediaSourceDecoder.h index 00f8bb35c1..35e30c5270 100644 --- a/dom/media/mediasource/MediaSourceDecoder.h +++ b/dom/media/mediasource/MediaSourceDecoder.h @@ -41,6 +41,15 @@ public: virtual MediaDecoderStateMachine* CreateStateMachine() override; virtual nsresult Load(nsIStreamListener**, MediaDecoder*) override; virtual media::TimeIntervals GetSeekable() override; + media::TimeIntervals GetBuffered() override; + + // We can't do this in the constructor because we don't know what type of + // media we're dealing with by that point. + void NotifyDormantSupported(bool aSupported) + { + MOZ_ASSERT(NS_IsMainThread()); + mDormantSupported = aSupported; + } virtual void Shutdown() override; @@ -87,12 +96,6 @@ public: // reader in this decoders MediaSourceReader. bool IsActiveReader(MediaDecoderReader* aReader); - // Return a decoder from the set available in aTrackDecoders that has data - // available in the range requested by aTarget. - already_AddRefed SelectDecoder(int64_t aTarget /* microseconds */, - int64_t aTolerance /* microseconds */, - const nsTArray>& aTrackDecoders); - // Returns a string describing the state of the MediaSource internal // buffered data. Used for debugging purposes. void GetMozDebugReaderData(nsAString& aString); diff --git a/dom/media/mediasource/MediaSourceReader.cpp b/dom/media/mediasource/MediaSourceReader.cpp index bc82a3e808..6849fca400 100644 --- a/dom/media/mediasource/MediaSourceReader.cpp +++ b/dom/media/mediasource/MediaSourceReader.cpp @@ -532,10 +532,32 @@ MediaSourceReader::BreakCycles() already_AddRefed MediaSourceReader::SelectDecoder(int64_t aTarget, int64_t aTolerance, - const nsTArray>& aTrackDecoders) + TrackBuffer* aTrackBuffer) { - return static_cast(mDecoder) - ->SelectDecoder(aTarget, aTolerance, aTrackDecoders); + ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); + + media::TimeUnit target{media::TimeUnit::FromMicroseconds(aTarget)}; + media::TimeUnit tolerance{media::TimeUnit::FromMicroseconds(aTolerance + aTarget)}; + + const nsTArray>& decoders{aTrackBuffer->Decoders()}; + + // aTolerance gives a slight bias toward the start of a range only. + // Consider decoders in order of newest to oldest, as a newer decoder + // providing a given buffered range is expected to replace an older one. + for (int32_t i = decoders.Length() - 1; i >= 0; --i) { + nsRefPtr newDecoder = decoders[i]; + media::TimeIntervals ranges = aTrackBuffer->GetBuffered(newDecoder); + for (uint32_t j = 0; j < ranges.Length(); j++) { + if (target < ranges.End(j) && tolerance >= ranges.Start(j)) { + return newDecoder.forget(); + } + } + MSE_DEBUGV("SelectDecoder(%lld fuzz:%lld) newDecoder=%p (%d/%d) target not in ranges=%s", + aTarget, aTolerance, newDecoder.get(), i+1, + decoders.Length(), DumpTimeRanges(ranges).get()); + } + + return nullptr; } bool @@ -543,7 +565,7 @@ MediaSourceReader::HaveData(int64_t aTarget, MediaData::Type aType) { TrackBuffer* trackBuffer = aType == MediaData::AUDIO_DATA ? mAudioTrack : mVideoTrack; MOZ_ASSERT(trackBuffer); - nsRefPtr decoder = SelectDecoder(aTarget, EOS_FUZZ_US, trackBuffer->Decoders()); + nsRefPtr decoder = SelectDecoder(aTarget, EOS_FUZZ_US, trackBuffer); return !!decoder; } @@ -561,9 +583,9 @@ MediaSourceReader::SwitchAudioSource(int64_t* aTarget) // reader and skip the last few samples of the current one. bool usedFuzz = false; nsRefPtr newDecoder = - SelectDecoder(*aTarget, /* aTolerance = */ 0, mAudioTrack->Decoders()); + SelectDecoder(*aTarget, /* aTolerance = */ 0, mAudioTrack); if (!newDecoder) { - newDecoder = SelectDecoder(*aTarget, EOS_FUZZ_US, mAudioTrack->Decoders()); + newDecoder = SelectDecoder(*aTarget, EOS_FUZZ_US, mAudioTrack); usedFuzz = true; } if (GetAudioReader() && mAudioSourceDecoder != newDecoder) { @@ -606,9 +628,9 @@ MediaSourceReader::SwitchVideoSource(int64_t* aTarget) // reader and skip the last few samples of the current one. bool usedFuzz = false; nsRefPtr newDecoder = - SelectDecoder(*aTarget, /* aTolerance = */ 0, mVideoTrack->Decoders()); + SelectDecoder(*aTarget, /* aTolerance = */ 0, mVideoTrack); if (!newDecoder) { - newDecoder = SelectDecoder(*aTarget, EOS_FUZZ_US, mVideoTrack->Decoders()); + newDecoder = SelectDecoder(*aTarget, EOS_FUZZ_US, mVideoTrack); usedFuzz = true; } if (GetVideoReader() && mVideoSourceDecoder != newDecoder) { @@ -637,17 +659,6 @@ MediaSourceReader::SwitchVideoSource(int64_t* aTarget) return SOURCE_NEW; } -bool -MediaSourceReader::IsDormantNeeded() -{ - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - if (GetVideoReader()) { - return GetVideoReader()->IsDormantNeeded(); - } - - return false; -} - void MediaSourceReader::ReleaseMediaResources() { @@ -711,10 +722,7 @@ MediaSourceReader::CreateSubDecoder(const nsACString& aType, int64_t aTimestampO // MSE uses a start time of 0 everywhere. Set that immediately on the // subreader to make sure that it's always in a state where we can invoke // GetBuffered on it. - { - ReentrantMonitorAutoEnter mon(decoder->GetReentrantMonitor()); - reader->SetStartTime(0); - } + reader->DispatchSetStartTime(0); #ifdef MOZ_FMP4 reader->SetSharedDecoderManager(mSharedDecoderManager); @@ -987,6 +995,7 @@ MediaSourceReader::DoVideoSeek() media::TimeIntervals MediaSourceReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); media::TimeIntervals buffered; @@ -1225,25 +1234,27 @@ MediaSourceReader::GetMozDebugReaderData(nsAString& aString) if (mAudioTrack) { result += nsPrintfCString("\tDumping Audio Track Decoders: - mLastAudioTime: %f\n", double(mLastAudioTime) / USECS_PER_S); for (int32_t i = mAudioTrack->Decoders().Length() - 1; i >= 0; --i) { - nsRefPtr newReader = mAudioTrack->Decoders()[i]->GetReader(); - - media::TimeIntervals ranges = mAudioTrack->Decoders()[i]->GetBuffered(); + const nsRefPtr& newDecoder{mAudioTrack->Decoders()[i]}; + media::TimeIntervals ranges = mAudioTrack->GetBuffered(newDecoder); result += nsPrintfCString("\t\tReader %d: %p ranges=%s active=%s size=%lld\n", - i, newReader.get(), DumpTimeRanges(ranges).get(), - newReader.get() == GetAudioReader() ? "true" : "false", - mAudioTrack->Decoders()[i]->GetResource()->GetSize()); + i, + newDecoder->GetReader(), + DumpTimeRanges(ranges).get(), + newDecoder->GetReader() == GetAudioReader() ? "true" : "false", + newDecoder->GetResource()->GetSize()); } } if (mVideoTrack) { result += nsPrintfCString("\tDumping Video Track Decoders - mLastVideoTime: %f\n", double(mLastVideoTime) / USECS_PER_S); for (int32_t i = mVideoTrack->Decoders().Length() - 1; i >= 0; --i) { - nsRefPtr newReader = mVideoTrack->Decoders()[i]->GetReader(); - - media::TimeIntervals ranges = mVideoTrack->Decoders()[i]->GetBuffered(); + const nsRefPtr& newDecoder{mVideoTrack->Decoders()[i]}; + media::TimeIntervals ranges = mVideoTrack->GetBuffered(newDecoder); result += nsPrintfCString("\t\tReader %d: %p ranges=%s active=%s size=%lld\n", - i, newReader.get(), DumpTimeRanges(ranges).get(), - newReader.get() == GetVideoReader() ? "true" : "false", + i, + newDecoder->GetReader(), + DumpTimeRanges(ranges).get(), + newDecoder->GetReader() == GetVideoReader() ? "true" : "false", mVideoTrack->Decoders()[i]->GetResource()->GetSize()); } } diff --git a/dom/media/mediasource/MediaSourceReader.h b/dom/media/mediasource/MediaSourceReader.h index 475ecd7fd2..02fcdf2fce 100644 --- a/dom/media/mediasource/MediaSourceReader.h +++ b/dom/media/mediasource/MediaSourceReader.h @@ -54,7 +54,6 @@ public: virtual size_t SizeOfVideoQueueInFrames() override; virtual size_t SizeOfAudioQueueInFrames() override; - virtual bool IsDormantNeeded() override; virtual void ReleaseMediaResources() override; void OnAudioDecoded(AudioData* aSample); @@ -216,9 +215,10 @@ private: // Return a decoder from the set available in aTrackDecoders that has data // available in the range requested by aTarget. + friend class TrackBuffer; already_AddRefed SelectDecoder(int64_t aTarget /* microseconds */, int64_t aTolerance /* microseconds */, - const nsTArray>& aTrackDecoders); + TrackBuffer* aTrackBuffer); bool HaveData(int64_t aTarget, MediaData::Type aType); already_AddRefed FirstDecoder(MediaData::Type aType); diff --git a/dom/media/mediasource/ResourceQueue.cpp b/dom/media/mediasource/ResourceQueue.cpp index 84c481d074..6fff6566b3 100644 --- a/dom/media/mediasource/ResourceQueue.cpp +++ b/dom/media/mediasource/ResourceQueue.cpp @@ -20,7 +20,7 @@ extern PRLogModuleInfo* GetSourceBufferResourceLog(); namespace mozilla { -ResourceItem::ResourceItem(MediaLargeByteBuffer* aData) +ResourceItem::ResourceItem(MediaByteBuffer* aData) : mData(aData) { } @@ -82,7 +82,7 @@ ResourceQueue::CopyData(uint64_t aOffset, uint32_t aCount, char* aDest) } void -ResourceQueue::AppendItem(MediaLargeByteBuffer* aData) +ResourceQueue::AppendItem(MediaByteBuffer* aData) { mLogicalLength += aData->Length(); Push(new ResourceItem(aData)); @@ -111,7 +111,7 @@ uint32_t ResourceQueue::EvictBefore(uint64_t aOffset, ErrorResult& aRv) uint32_t offset = aOffset - mOffset; mOffset += offset; evicted += offset; - nsRefPtr data = new MediaLargeByteBuffer; + nsRefPtr data = new MediaByteBuffer; if (!data->AppendElements(item->mData->Elements() + offset, item->mData->Length() - offset, fallible)) { diff --git a/dom/media/mediasource/ResourceQueue.h b/dom/media/mediasource/ResourceQueue.h index 78d60169a8..5ca9f047ef 100644 --- a/dom/media/mediasource/ResourceQueue.h +++ b/dom/media/mediasource/ResourceQueue.h @@ -26,9 +26,9 @@ class ErrorResult; // timepoint. struct ResourceItem { - explicit ResourceItem(MediaLargeByteBuffer* aData); + explicit ResourceItem(MediaByteBuffer* aData); size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const; - nsRefPtr mData; + nsRefPtr mData; }; class ResourceQueue : private nsDeque { @@ -45,7 +45,7 @@ public: // Copies aCount bytes from aOffset in the queue into aDest. void CopyData(uint64_t aOffset, uint32_t aCount, char* aDest); - void AppendItem(MediaLargeByteBuffer* aData); + void AppendItem(MediaByteBuffer* aData); // Tries to evict at least aSizeToEvict from the queue up until // aOffset. Returns amount evicted. diff --git a/dom/media/mediasource/SourceBuffer.cpp b/dom/media/mediasource/SourceBuffer.cpp index b7d6be6697..75ed493108 100644 --- a/dom/media/mediasource/SourceBuffer.cpp +++ b/dom/media/mediasource/SourceBuffer.cpp @@ -117,6 +117,12 @@ SourceBuffer::GetBuffered(ErrorResult& aRv) return tr.forget(); } +media::TimeIntervals +SourceBuffer::GetTimeIntervals() +{ + return mContentManager->Buffered(); +} + void SourceBuffer::SetAppendWindowStart(double aAppendWindowStart, ErrorResult& aRv) { @@ -271,7 +277,7 @@ SourceBuffer::Ended() mContentManager->Ended(); // We want the MediaSourceReader to refresh its buffered range as it may // have been modified (end lined up). - mMediaSource->GetDecoder()->NotifyDataArrived(nullptr, 1, mReportedOffset++); + mMediaSource->GetDecoder()->NotifyDataArrived(1, mReportedOffset++, /* aThrottleUpdates = */ false); } SourceBuffer::SourceBuffer(MediaSource* aMediaSource, const nsACString& aType) @@ -398,7 +404,7 @@ SourceBuffer::AppendData(const uint8_t* aData, uint32_t aLength, ErrorResult& aR { MSE_DEBUG("AppendData(aLength=%u)", aLength); - nsRefPtr data = PrepareAppend(aData, aLength, aRv); + nsRefPtr data = PrepareAppend(aData, aLength, aRv); if (!data) { return; } @@ -446,7 +452,7 @@ SourceBuffer::AppendDataCompletedWithSuccess(bool aHasActiveTracks) // Tell our parent decoder that we have received new data. // The information provided do not matter much so long as it is monotonically // increasing. - mMediaSource->GetDecoder()->NotifyDataArrived(nullptr, 1, mReportedOffset++); + mMediaSource->GetDecoder()->NotifyDataArrived(1, mReportedOffset++, /* aThrottleUpdates = */ false); // Send progress event. mMediaSource->GetDecoder()->NotifyBytesDownloaded(); } @@ -493,7 +499,7 @@ SourceBuffer::AppendError(bool aDecoderError) } } -already_AddRefed +already_AddRefed SourceBuffer::PrepareAppend(const uint8_t* aData, uint32_t aLength, ErrorResult& aRv) { typedef SourceBufferContentManager::EvictDataResult Result; @@ -551,7 +557,7 @@ SourceBuffer::PrepareAppend(const uint8_t* aData, uint32_t aLength, ErrorResult& return nullptr; } - nsRefPtr data = new MediaLargeByteBuffer(); + nsRefPtr data = new MediaByteBuffer(); if (!data->AppendElements(aData, aLength, fallible)) { aRv.Throw(NS_ERROR_DOM_QUOTA_EXCEEDED_ERR); return nullptr; diff --git a/dom/media/mediasource/SourceBuffer.h b/dom/media/mediasource/SourceBuffer.h index cf95c74085..9cd9ee7ba1 100644 --- a/dom/media/mediasource/SourceBuffer.h +++ b/dom/media/mediasource/SourceBuffer.h @@ -33,7 +33,7 @@ struct JSContext; namespace mozilla { class ErrorResult; -class MediaLargeByteBuffer; +class MediaByteBuffer; template class AsyncEventRunner; class TrackBuffersManager; @@ -151,6 +151,7 @@ public: } already_AddRefed GetBuffered(ErrorResult& aRv); + TimeIntervals GetTimeIntervals(); double TimestampOffset() const { @@ -255,11 +256,11 @@ private: // http://w3c.github.io/media-source/#sourcebuffer-append-error void AppendError(bool aDecoderError); - // Implements the "Prepare Append Algorithm". Returns MediaLargeByteBuffer object + // Implements the "Prepare Append Algorithm". Returns MediaByteBuffer object // on success or nullptr (with aRv set) on error. - already_AddRefed PrepareAppend(const uint8_t* aData, - uint32_t aLength, - ErrorResult& aRv); + already_AddRefed PrepareAppend(const uint8_t* aData, + uint32_t aLength, + ErrorResult& aRv); void AppendDataCompletedWithSuccess(bool aHasActiveTracks); void AppendDataErrored(nsresult aError); diff --git a/dom/media/mediasource/SourceBufferContentManager.cpp b/dom/media/mediasource/SourceBufferContentManager.cpp index 6f8325e01b..0caf553c59 100644 --- a/dom/media/mediasource/SourceBufferContentManager.cpp +++ b/dom/media/mediasource/SourceBufferContentManager.cpp @@ -10,6 +10,12 @@ namespace mozilla { +#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) +#define MP4_READER_DORMANT_HEURISTIC +#else +#undef MP4_READER_DORMANT_HEURISTIC +#endif + already_AddRefed SourceBufferContentManager::CreateManager(dom::SourceBufferAttributes* aAttributes, MediaSourceDecoder* aParentDecoder, @@ -23,6 +29,18 @@ SourceBufferContentManager::CreateManager(dom::SourceBufferAttributes* aAttribut } else { manager = new TrackBuffer(aParentDecoder, aType); } + + // Now that we know what type we're dealing with, enable dormant as needed. +#if defined(MP4_READER_DORMANT_HEURISTIC) + if (aType.LowerCaseEqualsLiteral("video/mp4") || + aType.LowerCaseEqualsLiteral("audio/mp4") || + useFormatReader) + { + aParentDecoder->NotifyDormantSupported(Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false)); + } +#endif + + return manager.forget(); } diff --git a/dom/media/mediasource/SourceBufferContentManager.h b/dom/media/mediasource/SourceBufferContentManager.h index 938351ee55..44a51d8599 100644 --- a/dom/media/mediasource/SourceBufferContentManager.h +++ b/dom/media/mediasource/SourceBufferContentManager.h @@ -38,7 +38,7 @@ public: // Add data to the end of the input buffer. // Returns false if the append failed. virtual bool - AppendData(MediaLargeByteBuffer* aData, TimeUnit aTimestampOffset) = 0; + AppendData(MediaByteBuffer* aData, TimeUnit aTimestampOffset) = 0; // Run MSE Buffer Append Algorithm // 3.5.5 Buffer Append Algorithm. diff --git a/dom/media/mediasource/SourceBufferDecoder.cpp b/dom/media/mediasource/SourceBufferDecoder.cpp index 69d1f33e92..9a34cde288 100644 --- a/dom/media/mediasource/SourceBufferDecoder.cpp +++ b/dom/media/mediasource/SourceBufferDecoder.cpp @@ -178,12 +178,6 @@ SourceBufferDecoder::Trim(int64_t aDuration) mTrimmedOffset = (double)aDuration / USECS_PER_S; } -void -SourceBufferDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) -{ - MSE_DEBUG("UNIMPLEMENTED"); -} - void SourceBufferDecoder::SetMediaSeekable(bool aMediaSeekable) { @@ -203,15 +197,8 @@ SourceBufferDecoder::GetOwner() } void -SourceBufferDecoder::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +SourceBufferDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) { - mReader->NotifyDataArrived(aBuffer, aLength, aOffset); - - // XXX: Params make no sense to parent decoder as it relates to a - // specific SourceBufferDecoder's data stream. Pass bogus values here to - // force parent decoder's state machine to recompute end time for - // infinite length media. - mParentDecoder->NotifyDataArrived(nullptr, 0, 0); } media::TimeIntervals diff --git a/dom/media/mediasource/SourceBufferDecoder.h b/dom/media/mediasource/SourceBufferDecoder.h index 4289607be5..295258940d 100644 --- a/dom/media/mediasource/SourceBufferDecoder.h +++ b/dom/media/mediasource/SourceBufferDecoder.h @@ -44,14 +44,13 @@ public: virtual void FirstFrameLoaded(nsAutoPtr aInfo, MediaDecoderEventVisibility aEventVisibility) final override; virtual void NotifyBytesConsumed(int64_t aBytes, int64_t aOffset) final override; - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) final override; + virtual void NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) final override; virtual void NotifyDecodedFrames(uint32_t aParsed, uint32_t aDecoded, uint32_t aDropped) final override; virtual void NotifyWaitingForResourcesStatusChanged() final override; virtual void OnReadMetadataCompleted() final override; virtual void QueueMetadata(int64_t aTime, nsAutoPtr aInfo, nsAutoPtr aTags) final override; virtual void RemoveMediaTracks() final override; virtual void SetMediaSeekable(bool aMediaSeekable) final override; - virtual void UpdateEstimatedMediaDuration(int64_t aDuration) final override; virtual bool HasInitializationData() final override; // SourceBufferResource specific interface below. diff --git a/dom/media/mediasource/SourceBufferResource.cpp b/dom/media/mediasource/SourceBufferResource.cpp index 797eb3fc3b..3d0644511f 100644 --- a/dom/media/mediasource/SourceBufferResource.cpp +++ b/dom/media/mediasource/SourceBufferResource.cpp @@ -212,7 +212,7 @@ SourceBufferResource::EvictAll() } void -SourceBufferResource::AppendData(MediaLargeByteBuffer* aData) +SourceBufferResource::AppendData(MediaByteBuffer* aData) { SBR_DEBUG("AppendData(aData=%p, aLength=%u)", aData->Elements(), aData->Length()); diff --git a/dom/media/mediasource/SourceBufferResource.h b/dom/media/mediasource/SourceBufferResource.h index 46c7e1df11..fd545d7617 100644 --- a/dom/media/mediasource/SourceBufferResource.h +++ b/dom/media/mediasource/SourceBufferResource.h @@ -27,7 +27,7 @@ class nsIStreamListener; namespace mozilla { class MediaDecoder; -class MediaLargeByteBuffer; +class MediaByteBuffer; namespace dom { @@ -103,7 +103,7 @@ public: } // Used by SourceBuffer. - void AppendData(MediaLargeByteBuffer* aData); + void AppendData(MediaByteBuffer* aData); void Ended(); bool IsEnded() { diff --git a/dom/media/mediasource/TrackBuffer.cpp b/dom/media/mediasource/TrackBuffer.cpp index b589d9a997..8f80f74342 100644 --- a/dom/media/mediasource/TrackBuffer.cpp +++ b/dom/media/mediasource/TrackBuffer.cpp @@ -36,6 +36,9 @@ extern PRLogModuleInfo* GetMediaSourceLog(); #define EOS_FUZZ_US 125000 +using media::TimeIntervals; +using media::Interval; + namespace mozilla { TrackBuffer::TrackBuffer(MediaSourceDecoder* aParentDecoder, const nsACString& aType) @@ -139,7 +142,7 @@ TrackBuffer::ContinueShutdown() } bool -TrackBuffer::AppendData(MediaLargeByteBuffer* aData, TimeUnit aTimestampOffset) +TrackBuffer::AppendData(MediaByteBuffer* aData, TimeUnit aTimestampOffset) { MOZ_ASSERT(NS_IsMainThread()); mInputBuffer = aData; @@ -162,7 +165,7 @@ TrackBuffer::BufferAppend() nsRefPtr p = mInitializationPromise.Ensure(__func__); bool hadInitData = mParser->HasInitData(); bool hadCompleteInitData = mParser->HasCompleteInitData(); - nsRefPtr oldInit = mParser->InitData(); + nsRefPtr oldInit = mParser->InitData(); bool newInitData = mParser->IsInitSegmentPresent(mInputBuffer); // TODO: Run more of the buffer append algorithm asynchronously. @@ -242,44 +245,96 @@ TrackBuffer::BufferAppend() mAdjustedTimestamp = starttu; } - if (!AppendDataToCurrentResource(mInputBuffer, end - start)) { + int64_t offset = AppendDataToCurrentResource(mInputBuffer, end - start); + if (offset < 0) { mInitializationPromise.Reject(NS_ERROR_FAILURE, __func__); return p; } + mLastAppendRange = + Interval(offset, offset + int64_t(mInputBuffer->Length())); + if (decoders.Length()) { // We're going to have to wait for the decoder to initialize, the promise // will be resolved once initialization completes. return p; } - // Tell our reader that we have more data to ensure that playback starts if - // required when data is appended. - NotifyTimeRangesChanged(); + nsRefPtr self = this; + + ProxyMediaCall(mParentDecoder->GetReader()->TaskQueue(), this, __func__, + &TrackBuffer::UpdateBufferedRanges, + mLastAppendRange, /* aNotifyParent */ true) + ->Then(mParentDecoder->GetReader()->TaskQueue(), __func__, + [self] { + self->mInitializationPromise.ResolveIfExists(self->HasInitSegment(), __func__); + }, + [self] (nsresult) { MOZ_CRASH("Never called."); }); - mInitializationPromise.Resolve(HasInitSegment(), __func__); return p; } -bool -TrackBuffer::AppendDataToCurrentResource(MediaLargeByteBuffer* aData, uint32_t aDuration) +int64_t +TrackBuffer::AppendDataToCurrentResource(MediaByteBuffer* aData, uint32_t aDuration) { MOZ_ASSERT(NS_IsMainThread()); if (!mCurrentDecoder) { - return false; + return -1; } SourceBufferResource* resource = mCurrentDecoder->GetResource(); int64_t appendOffset = resource->GetLength(); resource->AppendData(aData); mCurrentDecoder->SetRealMediaDuration(mCurrentDecoder->GetRealMediaDuration() + aDuration); - // XXX: For future reference: NDA call must run on the main thread. - mCurrentDecoder->NotifyDataArrived(reinterpret_cast(aData->Elements()), - aData->Length(), appendOffset); - mParentDecoder->NotifyBytesDownloaded(); + + return appendOffset; +} + +nsRefPtr +TrackBuffer::UpdateBufferedRanges(Interval aByteRange, bool aNotifyParent) +{ + if (aByteRange.Length()) { + mCurrentDecoder->GetReader()->NotifyDataArrived(aByteRange); + } + + // Recalculate and cache our new buffered range. + { + ReentrantMonitorAutoEnter mon(mParentDecoder->GetReentrantMonitor()); + TimeIntervals buffered; + + for (auto& decoder : mInitializedDecoders) { + TimeIntervals decoderBuffered(decoder->GetBuffered()); + mReadersBuffered[decoder] = decoderBuffered; + buffered += decoderBuffered; + } + // mParser may not be initialized yet, and will only be so if we have a + // buffered range. + if (buffered.Length()) { + buffered.SetFuzz(TimeUnit::FromMicroseconds(mParser->GetRoundingError())); + } + + mBufferedRanges = buffered; + } + + if (aNotifyParent) { + nsRefPtr parent = mParentDecoder; + nsCOMPtr task = + NS_NewRunnableFunction([parent] () { + // XXX: Params make no sense to parent decoder as it relates to a + // specific SourceBufferDecoder's data stream. Pass bogus values here to + // force parent decoder's state machine to recompute end time for + // infinite length media. + parent->NotifyDataArrived(0, 0, /* aThrottleUpdates = */ false); + parent->NotifyBytesDownloaded(); + }); + AbstractThread::MainThread()->Dispatch(task.forget()); + } + + // Tell our reader that we have more data to ensure that playback starts if + // required when data is appended. NotifyTimeRangesChanged(); - return true; + return BufferedRangesUpdatedPromise::CreateAndResolve(true, __func__); } void @@ -291,24 +346,49 @@ TrackBuffer::NotifyTimeRangesChanged() mParentDecoder->GetReader()->TaskQueue()->Dispatch(task.forget()); } +void +TrackBuffer::NotifyReaderDataRemoved(MediaDecoderReader* aReader) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsRefPtr self = this; + nsRefPtr reader = aReader; + RefPtr task = + NS_NewRunnableFunction([self, reader] () { + reader->NotifyDataRemoved(); + self->UpdateBufferedRanges(Interval(), /* aNotifyParent */ false); + }); + aReader->TaskQueue()->Dispatch(task.forget()); +} + class DecoderSorter { public: + explicit DecoderSorter(const TrackBuffer::DecoderBufferedMap& aMap) + : mMap(aMap) + {} + bool LessThan(SourceBufferDecoder* aFirst, SourceBufferDecoder* aSecond) const { - TimeIntervals first = aFirst->GetBuffered(); - TimeIntervals second = aSecond->GetBuffered(); + MOZ_ASSERT(mMap.find(aFirst) != mMap.end()); + MOZ_ASSERT(mMap.find(aSecond) != mMap.end()); + const TimeIntervals& first = mMap.find(aFirst)->second; + const TimeIntervals& second = mMap.find(aSecond)->second; return first.GetStart() < second.GetStart(); } bool Equals(SourceBufferDecoder* aFirst, SourceBufferDecoder* aSecond) const { - TimeIntervals first = aFirst->GetBuffered(); - TimeIntervals second = aSecond->GetBuffered(); + MOZ_ASSERT(mMap.find(aFirst) != mMap.end()); + MOZ_ASSERT(mMap.find(aSecond) != mMap.end()); + const TimeIntervals& first = mMap.find(aFirst)->second; + const TimeIntervals& second = mMap.find(aSecond)->second; return first.GetStart() == second.GetStart(); } + + const TrackBuffer::DecoderBufferedMap& mMap; }; TrackBuffer::EvictDataResult @@ -331,14 +411,24 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, } // Get a list of initialized decoders. - nsTArray decoders; + nsTArray> decoders; decoders.AppendElements(mInitializedDecoders); const TimeUnit evictThresholdTime{TimeUnit::FromSeconds(MSE_EVICT_THRESHOLD_TIME)}; + // Find the reader currently being played with. + SourceBufferDecoder* playingDecoder = nullptr; + for (const auto& decoder : decoders) { + if (mParentDecoder->IsActiveReader(decoder->GetReader())) { + playingDecoder = decoder; + break; + } + } + TimeUnit playingDecoderStartTime{GetBuffered(playingDecoder).GetStart()}; + // First try to evict data before the current play position, starting // with the oldest decoder. for (uint32_t i = 0; i < decoders.Length() && toEvict > 0; ++i) { - TimeIntervals buffered = decoders[i]->GetBuffered(); + TimeIntervals buffered = GetBuffered(decoders[i]); MSE_DEBUG("Step1. decoder=%u/%u threshold=%u toEvict=%lld", i, decoders.Length(), aThreshold, toEvict); @@ -366,6 +456,10 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, buffered.GetEnd().ToSeconds(), aPlaybackTime.ToSeconds(), time, playbackOffset, decoders[i]->GetResource()->GetSize()); if (playbackOffset > 0) { + if (decoders[i] == playingDecoder) { + // This is an approximation only, likely pessimistic. + playingDecoderStartTime = time; + } ErrorResult rv; toEvict -= decoders[i]->GetResource()->EvictData(playbackOffset, playbackOffset, @@ -376,7 +470,7 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, } } } - decoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(decoders[i]->GetReader()); } } @@ -384,13 +478,15 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, for (uint32_t i = 0; i < decoders.Length() && toEvict > 0; ++i) { MSE_DEBUG("Step2. decoder=%u/%u threshold=%u toEvict=%lld", i, decoders.Length(), aThreshold, toEvict); - if (mParentDecoder->IsActiveReader(decoders[i]->GetReader())) { + if (decoders[i] == playingDecoder) { break; } if (decoders[i] == mCurrentDecoder) { continue; } - TimeIntervals buffered = decoders[i]->GetBuffered(); + // The buffered value is potentially stale should eviction occurred in + // step 1. However this is only used for logging. + TimeIntervals buffered = GetBuffered(decoders[i]); // Remove data from older decoders than the current one. MSE_DEBUG("evicting all " @@ -398,7 +494,7 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, buffered.GetStart().ToSeconds(), buffered.GetEnd().ToSeconds(), aPlaybackTime, decoders[i]->GetResource()->GetSize()); toEvict -= decoders[i]->GetResource()->EvictAll(); - decoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(decoders[i]->GetReader()); } // Evict all data from future decoders, starting furthest away from @@ -409,26 +505,21 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, // TODO: This step should be done using RangeRemoval: // Something like: RangeRemoval(aPlaybackTime + 60s, End); - // Find the reader currently being played with. - SourceBufferDecoder* playingDecoder = nullptr; - for (uint32_t i = 0; i < decoders.Length() && toEvict > 0; ++i) { - if (mParentDecoder->IsActiveReader(decoders[i]->GetReader())) { - playingDecoder = decoders[i]; - break; - } - } // Find the next decoder we're likely going to play with. nsRefPtr nextPlayingDecoder = nullptr; if (playingDecoder) { - TimeIntervals buffered = playingDecoder->GetBuffered(); + // The buffered value is potentially stale should eviction occurred in + // step 1. However step 1 modified the start of the range value, and now + // will use the end value. + TimeIntervals buffered = GetBuffered(playingDecoder); nextPlayingDecoder = - mParentDecoder->SelectDecoder(buffered.GetEnd().ToMicroseconds() + 1, - EOS_FUZZ_US, - mInitializedDecoders); + mParentDecoder->GetReader()->SelectDecoder(buffered.GetEnd().ToMicroseconds() + 1, + EOS_FUZZ_US, + this); } // Sort decoders by their start times. - decoders.Sort(DecoderSorter()); + decoders.Sort(DecoderSorter{mReadersBuffered}); for (int32_t i = int32_t(decoders.Length()) - 1; i >= 0 && toEvict > 0; --i) { MSE_DEBUG("Step3. decoder=%u/%u threshold=%u toEvict=%lld", @@ -437,14 +528,17 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, decoders[i] == mCurrentDecoder) { continue; } - TimeIntervals buffered = decoders[i]->GetBuffered(); + // The buffered value is potentially stale should eviction occurred in + // step 1 and 2. However step 3 is a last resort step where we will remove + // all content and the buffered value is only used for logging. + TimeIntervals buffered = GetBuffered(decoders[i]); MSE_DEBUG("evicting all " "bufferedStart=%f bufferedEnd=%f aPlaybackTime=%f size=%lld", buffered.GetStart().ToSeconds(), buffered.GetEnd().ToSeconds(), aPlaybackTime, decoders[i]->GetResource()->GetSize()); toEvict -= decoders[i]->GetResource()->EvictAll(); - decoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(decoders[i]->GetReader()); } RemoveEmptyDecoders(decoders); @@ -452,8 +546,8 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, bool evicted = toEvict < (totalSize - aThreshold); if (evicted) { if (playingDecoder) { - TimeIntervals ranges = playingDecoder->GetBuffered(); - *aBufferStartTime = std::max(TimeUnit::FromSeconds(0), ranges.GetStart()); + *aBufferStartTime = + std::max(TimeUnit::FromSeconds(0), playingDecoderStartTime); } else { // We do not currently have data to play yet. // Avoid evicting anymore data to minimize rebuffering time. @@ -461,38 +555,37 @@ TrackBuffer::EvictData(TimeUnit aPlaybackTime, } } - if (evicted) { - NotifyTimeRangesChanged(); - } - return evicted ? EvictDataResult::DATA_EVICTED : (HasOnlyIncompleteMedia() ? EvictDataResult::CANT_EVICT : EvictDataResult::NO_DATA_EVICTED); } void -TrackBuffer::RemoveEmptyDecoders(nsTArray& aDecoders) +TrackBuffer::RemoveEmptyDecoders(const nsTArray>& aDecoders) { - ReentrantMonitorAutoEnter mon(mParentDecoder->GetReentrantMonitor()); + nsRefPtr self = this; + nsTArray> decoders(aDecoders); + nsCOMPtr task = + NS_NewRunnableFunction([self, decoders] () { + if (!self->mParentDecoder) { + return; + } + ReentrantMonitorAutoEnter mon(self->mParentDecoder->GetReentrantMonitor()); - // Remove decoders that have no data in them - for (uint32_t i = 0; i < aDecoders.Length(); ++i) { - TimeIntervals buffered = aDecoders[i]->GetBuffered(); - MSE_DEBUG("maybe remove empty decoders=%d " - "size=%lld start=%f end=%f", - i, aDecoders[i]->GetResource()->GetSize(), - buffered.GetStart().ToSeconds(), buffered.GetEnd().ToSeconds()); - if (aDecoders[i] == mCurrentDecoder || - mParentDecoder->IsActiveReader(aDecoders[i]->GetReader())) { - continue; - } - - if (aDecoders[i]->GetResource()->GetSize() == 0 || !buffered.Length() || - buffered[0].IsEmpty()) { - MSE_DEBUG("remove empty decoders=%d", i); - RemoveDecoder(aDecoders[i]); - } - } + // Remove decoders that have decoders data in them + for (uint32_t i = 0; i < decoders.Length(); ++i) { + if (decoders[i] == self->mCurrentDecoder || + self->mParentDecoder->IsActiveReader(decoders[i]->GetReader())) { + continue; + } + TimeIntervals buffered = self->GetBuffered(decoders[i]); + if (decoders[i]->GetResource()->GetSize() == 0 || !buffered.Length() || + buffered[0].IsEmpty()) { + self->RemoveDecoder(decoders[i]); + } + } + }); + AbstractThread::MainThread()->Dispatch(task.forget()); } int64_t @@ -511,7 +604,7 @@ TrackBuffer::HasOnlyIncompleteMedia() if (!mCurrentDecoder) { return false; } - TimeIntervals buffered = mCurrentDecoder->GetBuffered(); + TimeIntervals buffered = GetBuffered(mCurrentDecoder); MSE_DEBUG("mCurrentDecoder.size=%lld, start=%f end=%f", mCurrentDecoder->GetResource()->GetSize(), buffered.GetStart(), buffered.GetEnd()); @@ -534,10 +627,9 @@ TrackBuffer::EvictBefore(TimeUnit aTime) rv.SuppressException(); return; } - mInitializedDecoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(mInitializedDecoders[i]->GetReader()); } } - NotifyTimeRangesChanged(); } TimeIntervals @@ -545,18 +637,20 @@ TrackBuffer::Buffered() { ReentrantMonitorAutoEnter mon(mParentDecoder->GetReentrantMonitor()); - TimeIntervals buffered; + return mBufferedRanges; +} - for (auto& decoder : mInitializedDecoders) { - buffered += decoder->GetBuffered(); - } - // mParser may not be initialized yet, and will only be so if we have a - // buffered range. - if (buffered.Length()) { - buffered.SetFuzz(TimeUnit::FromMicroseconds(mParser->GetRoundingError())); - } +TimeIntervals +TrackBuffer::GetBuffered(SourceBufferDecoder* aDecoder) +{ + ReentrantMonitorAutoEnter mon(mParentDecoder->GetReentrantMonitor()); - return buffered; + DecoderBufferedMap::const_iterator val = mReadersBuffered.find(aDecoder); + + if (val == mReadersBuffered.end()) { + return TimeIntervals::Invalid(); + } + return val->second; } already_AddRefed @@ -668,6 +762,8 @@ TrackBuffer::InitializeDecoder(SourceBufferDecoder* aDecoder) MSE_DEBUG("Initializing subdecoder %p reader %p", aDecoder, reader); + reader->NotifyDataArrived(mLastAppendRange); + // HACK WARNING: // We only reach this point once we know that we have a complete init segment. // We don't want the demuxer to do a blocking read as no more data can be @@ -726,7 +822,7 @@ TrackBuffer::OnMetadataRead(MetadataHolder* aMetadata, // Adding an empty buffer will reopen the SourceBufferResource if (!aWasEnded) { - nsRefPtr emptyBuffer = new MediaLargeByteBuffer; + nsRefPtr emptyBuffer = new MediaByteBuffer; aDecoder->GetResource()->AppendData(emptyBuffer); } // HACK END. @@ -840,7 +936,15 @@ TrackBuffer::CompleteInitializeDecoder(SourceBufferDecoder* aDecoder) MSE_DEBUG("Reader %p activated", aDecoder->GetReader()); - mInitializationPromise.ResolveIfExists(true, __func__); + nsRefPtr self = this; + ProxyMediaCall(mParentDecoder->GetReader()->TaskQueue(), this, __func__, + &TrackBuffer::UpdateBufferedRanges, + Interval(), /* aNotifyParent */ true) + ->Then(mParentDecoder->GetReader()->TaskQueue(), __func__, + [self] { + self->mInitializationPromise.ResolveIfExists(self->HasInitSegment(), __func__); + }, + [self] (nsresult) { MOZ_CRASH("Never called."); }); } bool @@ -936,7 +1040,7 @@ TrackBuffer::ContainsTime(int64_t aTime, int64_t aTolerance) ReentrantMonitorAutoEnter mon(mParentDecoder->GetReentrantMonitor()); TimeUnit time{TimeUnit::FromMicroseconds(aTime)}; for (auto& decoder : mInitializedDecoders) { - TimeIntervals r = decoder->GetBuffered(); + TimeIntervals r = GetBuffered(decoder); r.SetFuzz(TimeUnit::FromMicroseconds(aTolerance)); if (r.Contains(time)) { return true; @@ -1073,6 +1177,8 @@ TrackBuffer::RemoveDecoder(SourceBufferDecoder* aDecoder) MOZ_ASSERT(!mParentDecoder->IsActiveReader(aDecoder->GetReader())); mInitializedDecoders.RemoveElement(aDecoder); mDecoders.RemoveElement(aDecoder); + // Remove associated buffered range from our cache. + mReadersBuffered.erase(aDecoder); } aDecoder->GetReader()->TaskQueue()->Dispatch(task.forget()); } @@ -1099,13 +1205,13 @@ TrackBuffer::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) return RangeRemovalPromise::CreateAndResolve(false, __func__); } - nsTArray decoders; + nsTArray> decoders; decoders.AppendElements(mInitializedDecoders); if (aStart <= bufferedStart && aEnd < bufferedEnd) { // Evict data from beginning. for (size_t i = 0; i < decoders.Length(); ++i) { - TimeIntervals buffered = decoders[i]->GetBuffered(); + TimeIntervals buffered = GetBuffered(decoders[i]); if (buffered.GetEnd() < aEnd) { // Can be fully removed. MSE_DEBUG("remove all bufferedEnd=%f size=%lld", @@ -1126,7 +1232,7 @@ TrackBuffer::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) } } } - decoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(decoders[i]->GetReader()); } } else { // Only trimming existing buffers. @@ -1137,14 +1243,26 @@ TrackBuffer::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) } else { decoders[i]->Trim(aStart.ToMicroseconds()); } - decoders[i]->GetReader()->NotifyDataRemoved(); + NotifyReaderDataRemoved(decoders[i]->GetReader()); } } RemoveEmptyDecoders(decoders); - NotifyTimeRangesChanged(); - return RangeRemovalPromise::CreateAndResolve(true, __func__); + nsRefPtr p = mRangeRemovalPromise.Ensure(__func__); + + // Make sure our buffered ranges got updated before resolving promise. + nsRefPtr self = this; + ProxyMediaCall(mParentDecoder->GetReader()->TaskQueue(), this, __func__, + &TrackBuffer::UpdateBufferedRanges, + Interval(), /* aNotifyParent */ false) + ->Then(mParentDecoder->GetReader()->TaskQueue(), __func__, + [self] { + self->mRangeRemovalPromise.ResolveIfExists(true, __func__); + }, + [self] (nsresult) { MOZ_CRASH("Never called."); }); + + return p; } void diff --git a/dom/media/mediasource/TrackBuffer.h b/dom/media/mediasource/TrackBuffer.h index a4e9bfab8c..ada6a156eb 100644 --- a/dom/media/mediasource/TrackBuffer.h +++ b/dom/media/mediasource/TrackBuffer.h @@ -17,12 +17,13 @@ #include "nsString.h" #include "nscore.h" #include "TimeUnits.h" +#include namespace mozilla { class ContainerParser; class MediaSourceDecoder; -class MediaLargeByteBuffer; +class MediaByteBuffer; class TrackBuffer final : public SourceBufferContentManager { public: @@ -30,7 +31,7 @@ public: nsRefPtr Shutdown(); - bool AppendData(MediaLargeByteBuffer* aData, TimeUnit aTimestampOffset) override; + bool AppendData(MediaByteBuffer* aData, TimeUnit aTimestampOffset) override; // Append data to the current decoder. Also responsible for calling // NotifyDataArrived on the decoder to keep buffered range computation up @@ -102,10 +103,15 @@ public: // currently not playable. bool HasOnlyIncompleteMedia(); + // Return the buffered ranges for given decoder. + media::TimeIntervals GetBuffered(SourceBufferDecoder* aDecoder); + #if defined(DEBUG) void Dump(const char* aPath) override; #endif + typedef std::map DecoderBufferedMap; + private: friend class DecodersToInitialize; friend class MetadataRecipient; @@ -116,14 +122,19 @@ private: // for initialization. // The decoder is not considered initialized until it is added to // mInitializedDecoders. - already_AddRefed NewDecoder(TimeUnit aTimestampOffset); + already_AddRefed NewDecoder(media::TimeUnit aTimestampOffset); // Helper for AppendData, ensures NotifyDataArrived is called whenever // data is appended to the current decoder's SourceBufferResource. - bool AppendDataToCurrentResource(MediaLargeByteBuffer* aData, + int64_t AppendDataToCurrentResource(MediaByteBuffer* aData, uint32_t aDuration /* microseconds */); // Queue on the parent's decoder task queue a call to NotifyTimeRangesChanged. void NotifyTimeRangesChanged(); + // Queue on the parent's decoder task queue a call to NotifyDataRemoved. + void NotifyReaderDataRemoved(MediaDecoderReader* aReader); + + typedef MediaPromise BufferedRangesUpdatedPromise; + nsRefPtr UpdateBufferedRanges(Interval aByteRange, bool aNotifyParent); // Queue execution of InitializeDecoder on mTaskQueue. bool QueueInitializeDecoder(SourceBufferDecoder* aDecoder); @@ -154,7 +165,7 @@ private: void RemoveDecoder(SourceBufferDecoder* aDecoder); // Remove all empty decoders from the provided list; - void RemoveEmptyDecoders(nsTArray& aDecoders); + void RemoveEmptyDecoders(const nsTArray>& aDecoders); void OnMetadataRead(MetadataHolder* aMetadata, SourceBufferDecoder* aDecoder, @@ -164,7 +175,7 @@ private: SourceBufferDecoder* aDecoder); nsAutoPtr mParser; - nsRefPtr mInputBuffer; + nsRefPtr mInputBuffer; // A task queue using the shared media thread pool. Used exclusively to // initialize (i.e. call ReadMetadata on) decoders as they are created via @@ -198,9 +209,9 @@ private: void AdjustDecodersTimestampOffset(TimeUnit aOffset); // The timestamp offset used by our current decoder. - TimeUnit mLastTimestampOffset; - TimeUnit mTimestampOffset; - TimeUnit mAdjustedTimestamp; + media::TimeUnit mLastTimestampOffset; + media::TimeUnit mTimestampOffset; + media::TimeUnit mAdjustedTimestamp; // True if at least one of our decoders has encrypted content. bool mIsWaitingOnCDM; @@ -217,6 +228,15 @@ private: MediaPromiseHolder mInitializationPromise; // Track our request for metadata from the reader. MediaPromiseRequestHolder mMetadataRequest; + + MediaPromiseHolder mRangeRemovalPromise; + + Interval mLastAppendRange; + + // Protected by Parent's decoder Monitor. + media::TimeIntervals mBufferedRanges; + + DecoderBufferedMap mReadersBuffered; }; } // namespace mozilla diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index c4c0fae13c..2c7aa7e136 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -47,7 +47,7 @@ static Atomic sStreamSourceID(0u); TrackBuffersManager::TrackBuffersManager(dom::SourceBufferAttributes* aAttributes, MediaSourceDecoder* aParentDecoder, const nsACString& aType) - : mInputBuffer(new MediaLargeByteBuffer) + : mInputBuffer(new MediaByteBuffer) , mAppendState(AppendState::WAITING_FOR_SEGMENT) , mBufferFull(false) , mFirstInitializationSegmentReceived(false) @@ -73,7 +73,7 @@ TrackBuffersManager::~TrackBuffersManager() } bool -TrackBuffersManager::AppendData(MediaLargeByteBuffer* aData, +TrackBuffersManager::AppendData(MediaByteBuffer* aData, TimeUnit aTimestampOffset) { MOZ_ASSERT(NS_IsMainThread()); @@ -309,7 +309,7 @@ TrackBuffersManager::CompleteResetParserState() CreateDemuxerforMIMEType(); // Recreate our input buffer. We can't directly assign the initData buffer // to mInputBuffer as it will get modified in the Segment Parser Loop. - mInputBuffer = new MediaLargeByteBuffer; + mInputBuffer = new MediaByteBuffer; mInputBuffer->AppendElements(*mInitData, fallible); } RecreateParser(true); @@ -704,12 +704,12 @@ TrackBuffersManager::CreateDemuxerforMIMEType() } void -TrackBuffersManager::AppendDataToCurrentInputBuffer(MediaLargeByteBuffer* aData) +TrackBuffersManager::AppendDataToCurrentInputBuffer(MediaByteBuffer* aData) { MOZ_ASSERT(mCurrentInputBuffer); int64_t offset = mCurrentInputBuffer->GetLength(); mCurrentInputBuffer->AppendData(aData); - // A MediaLargeByteBuffer has a maximum size of 2GB. + // A MediaByteBuffer has a maximum size of 2GiB. mInputDemuxer->NotifyDataArrived(uint32_t(aData->Length()), offset); } @@ -978,7 +978,7 @@ TrackBuffersManager::CodedFrameProcessing() // The mediaRange is offset by the init segment position previously added. uint32_t length = mediaRange.mEnd - (mProcessedInput - mInputBuffer->Length()); - nsRefPtr segment = new MediaLargeByteBuffer; + nsRefPtr segment = new MediaByteBuffer; if (!segment->AppendElements(mInputBuffer->Elements(), length, fallible)) { return CodedFrameProcessingPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); } diff --git a/dom/media/mediasource/TrackBuffersManager.h b/dom/media/mediasource/TrackBuffersManager.h index 2def2b02af..b244046435 100644 --- a/dom/media/mediasource/TrackBuffersManager.h +++ b/dom/media/mediasource/TrackBuffersManager.h @@ -20,7 +20,7 @@ namespace mozilla { class ContainerParser; -class MediaLargeByteBuffer; +class MediaByteBuffer; class MediaRawData; class MediaSourceDemuxer; class SourceBufferResource; @@ -45,7 +45,7 @@ public: MediaSourceDecoder* aParentDecoder, const nsACString& aType); - bool AppendData(MediaLargeByteBuffer* aData, + bool AppendData(MediaByteBuffer* aData, TimeUnit aTimestampOffset) override; nsRefPtr BufferAppend() override; @@ -136,12 +136,12 @@ private: return mAudioTracks.mNumTracks > 0; } - typedef Pair, TimeUnit> IncomingBuffer; + typedef Pair, TimeUnit> IncomingBuffer; void AppendIncomingBuffer(IncomingBuffer aData); nsTArray mIncomingBuffers; // The input buffer as per http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer - nsRefPtr mInputBuffer; + nsRefPtr mInputBuffer; // The current append state as per https://w3c.github.io/media-source/#sourcebuffer-append-state // Accessed on both the main thread and the task queue. Atomic mAppendState; @@ -164,8 +164,8 @@ private: nsAutoPtr mParser; // Demuxer objects and methods. - void AppendDataToCurrentInputBuffer(MediaLargeByteBuffer* aData); - nsRefPtr mInitData; + void AppendDataToCurrentInputBuffer(MediaByteBuffer* aData); + nsRefPtr mInitData; nsRefPtr mCurrentInputBuffer; nsRefPtr mInputDemuxer; // Length already processed in current media segment. diff --git a/dom/media/mediasource/test/test_TruncatedDuration.html b/dom/media/mediasource/test/test_TruncatedDuration.html index f901217993..7410196be3 100644 --- a/dom/media/mediasource/test/test_TruncatedDuration.html +++ b/dom/media/mediasource/test/test_TruncatedDuration.html @@ -41,7 +41,6 @@ function do_seeked(e) { is(v.currentTime, duration, "current time was updated"); is(v.duration, duration, "element duration was updated"); is(v._sb.buffered.length, 1, "One buffered range"); - is(v._sb.buffered.end(0), duration, "sourcebuffer truncated"); // Truncated mediasource duration will cause the video element to seek. v.addEventListener("seeking", do_seeking, false); } diff --git a/dom/media/ogg/OggReader.cpp b/dom/media/ogg/OggReader.cpp index ea2a9058f4..b974663272 100644 --- a/dom/media/ogg/OggReader.cpp +++ b/dom/media/ogg/OggReader.cpp @@ -1844,6 +1844,7 @@ nsresult OggReader::SeekBisection(int64_t aTarget, media::TimeIntervals OggReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); { mozilla::ReentrantMonitorAutoEnter mon(mMonitor); diff --git a/dom/media/omx/MediaCodecReader.cpp b/dom/media/omx/MediaCodecReader.cpp index 3a3bea6994..8bc9ae9197 100644 --- a/dom/media/omx/MediaCodecReader.cpp +++ b/dom/media/omx/MediaCodecReader.cpp @@ -497,10 +497,12 @@ MediaCodecReader::HasVideo() } void -MediaCodecReader::NotifyDataArrived(const char* aBuffer, - uint32_t aLength, - int64_t aOffset) +MediaCodecReader::NotifyDataArrivedInternal(uint32_t aLength, + int64_t aOffset) { + nsRefPtr bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength); + NS_ENSURE_TRUE_VOID(bytes); + MonitorAutoLock monLock(mParserMonitor); if (mNextParserPosition == mParsedDataLength && mNextParserPosition >= aOffset && @@ -508,7 +510,7 @@ MediaCodecReader::NotifyDataArrived(const char* aBuffer, // No pending parsing runnable currently. And available data are adjacent to // parsed data. int64_t shift = mNextParserPosition - aOffset; - const char* buffer = aBuffer + shift; + const char* buffer = reinterpret_cast(bytes->Elements()) + shift; uint32_t length = aLength - shift; int64_t offset = mNextParserPosition; if (length > 0) { @@ -605,7 +607,7 @@ MediaCodecReader::ParseDataSegment(const char* aBuffer, return true; // NO-OP } - mMP3FrameParser->Parse(aBuffer, aLength, aOffset); + mMP3FrameParser->Parse(reinterpret_cast(aBuffer), aLength, aOffset); duration = mMP3FrameParser->GetDuration(); } @@ -627,8 +629,7 @@ MediaCodecReader::ParseDataSegment(const char* aBuffer, if (durationUpdateRequired) { MOZ_ASSERT(mDecoder); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecoder->UpdateEstimatedMediaDuration(duration); + mDecoder->DispatchUpdateEstimatedMediaDuration(duration); } return true; diff --git a/dom/media/omx/MediaCodecReader.h b/dom/media/omx/MediaCodecReader.h index 9e168419b2..0857604008 100644 --- a/dom/media/omx/MediaCodecReader.h +++ b/dom/media/omx/MediaCodecReader.h @@ -62,9 +62,6 @@ public: // on failure. virtual nsresult Init(MediaDecoderReader* aCloneDonor); - // True when this reader need to become dormant state - virtual bool IsDormantNeeded() { return true;} - // Release media resources they should be released in dormant state virtual void ReleaseMediaResources(); @@ -73,10 +70,12 @@ public: // irreversible, whereas ReleaseMediaResources() is reversible. virtual nsRefPtr Shutdown(); +protected: // Used to retrieve some special information that can only be retrieved after // all contents have been continuously parsed. (ex. total duration of some // variable-bit-rate MP3 files.) - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset); + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; +public: // Flush the MediaTaskQueue, flush MediaCodec and raise the mDiscontinuity. virtual nsresult ResetDecode() override; diff --git a/dom/media/omx/MediaOmxCommonDecoder.cpp b/dom/media/omx/MediaOmxCommonDecoder.cpp index 7e5311e61b..4acec2ee47 100644 --- a/dom/media/omx/MediaOmxCommonDecoder.cpp +++ b/dom/media/omx/MediaOmxCommonDecoder.cpp @@ -29,6 +29,7 @@ MediaOmxCommonDecoder::MediaOmxCommonDecoder() , mCanOffloadAudio(false) , mFallbackToStateMachine(false) { + mDormantSupported = true; if (!gMediaDecoderLog) { gMediaDecoderLog = PR_NewLogModule("MediaDecoder"); } diff --git a/dom/media/omx/MediaOmxReader.cpp b/dom/media/omx/MediaOmxReader.cpp index a60fe3f126..2f7c8ea2a2 100644 --- a/dom/media/omx/MediaOmxReader.cpp +++ b/dom/media/omx/MediaOmxReader.cpp @@ -43,7 +43,7 @@ public: { MOZ_ASSERT(!NS_IsMainThread()); MOZ_ASSERT(mOmxReader.get()); - mOmxReader->ProcessCachedData(mOffset, false); + mOmxReader->ProcessCachedData(mOffset); } private: @@ -70,24 +70,20 @@ class MediaOmxReader::NotifyDataArrivedRunnable : public nsRunnable { public: NotifyDataArrivedRunnable(MediaOmxReader* aOmxReader, - const char* aBuffer, uint64_t aLength, - int64_t aOffset, uint64_t aFullLength) + uint64_t aLength, + int64_t aOffset, uint64_t aFullLength) : mOmxReader(aOmxReader), - mBuffer(aBuffer), mLength(aLength), mOffset(aOffset), mFullLength(aFullLength) { MOZ_ASSERT(mOmxReader.get()); - MOZ_ASSERT(mBuffer.get() || !mLength); } NS_IMETHOD Run() { - NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); - + MOZ_ASSERT(mOmxReader->OnTaskQueue()); NotifyDataArrived(); - return NS_OK; } @@ -97,13 +93,10 @@ private: if (mOmxReader->IsShutdown()) { return; } - const char* buffer = mBuffer.get(); while (mLength) { uint32_t length = std::min(mLength, UINT32_MAX); - mOmxReader->NotifyDataArrived(buffer, length, - mOffset); - buffer += length; + mOmxReader->NotifyDataArrived(Interval(mOffset, mOffset + length)); mLength -= length; mOffset += length; } @@ -118,7 +111,6 @@ private: } nsRefPtr mOmxReader; - nsAutoArrayPtr mBuffer; uint64_t mLength; int64_t mOffset; uint64_t mFullLength; @@ -245,7 +237,7 @@ MediaOmxReader::AsyncReadMetadata() // the mDecoder->GetResource()->GetLength() would return -1. // Delay set the total duration on this function. mMP3FrameParser.SetLength(mDecoder->GetResource()->GetLength()); - ProcessCachedData(0, true); + ProcessCachedData(0); } nsRefPtr p = mMetadataPromise.Ensure(__func__); @@ -469,9 +461,9 @@ bool MediaOmxReader::DecodeVideoFrame(bool &aKeyframeSkip, return true; } -void MediaOmxReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +void MediaOmxReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(OnTaskQueue()); nsRefPtr decoder = SafeGetDecoder(); if (!decoder) { // reader has shut down return; @@ -483,16 +475,17 @@ void MediaOmxReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, in return; } - mMP3FrameParser.Parse(aBuffer, aLength, aOffset); + nsRefPtr bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength); + NS_ENSURE_TRUE_VOID(bytes); + mMP3FrameParser.Parse(bytes->Elements(), aLength, aOffset); if (!mMP3FrameParser.IsMP3()) { return; } int64_t duration = mMP3FrameParser.GetDuration(); if (duration != mLastParserDuration) { - ReentrantMonitorAutoEnter mon(decoder->GetReentrantMonitor()); mLastParserDuration = duration; - decoder->UpdateEstimatedMediaDuration(mLastParserDuration); + decoder->DispatchUpdateEstimatedMediaDuration(mLastParserDuration); } } @@ -575,7 +568,7 @@ void MediaOmxReader::EnsureActive() { NS_ASSERTION(result == NS_OK, "OmxDecoder should be in play state to continue decoding"); } -int64_t MediaOmxReader::ProcessCachedData(int64_t aOffset, bool aWaitForCompletion) +int64_t MediaOmxReader::ProcessCachedData(int64_t aOffset) { // Could run on decoder thread or IO thread. nsRefPtr decoder = SafeGetDecoder(); @@ -599,22 +592,14 @@ int64_t MediaOmxReader::ProcessCachedData(int64_t aOffset, bool aWaitForCompleti } int64_t bufferLength = std::min(resourceLength-aOffset, sReadSize); - - nsAutoArrayPtr buffer(new char[bufferLength]); - - nsresult rv = decoder->GetResource()->ReadFromCache(buffer.get(), - aOffset, bufferLength); - NS_ENSURE_SUCCESS(rv, -1); - nsRefPtr runnable( - new NotifyDataArrivedRunnable(this, buffer.forget(), bufferLength, - aOffset, resourceLength)); - if (aWaitForCompletion) { - rv = NS_DispatchToMainThread(runnable.get(), NS_DISPATCH_SYNC); + new NotifyDataArrivedRunnable(this, bufferLength, aOffset, resourceLength)); + + if (OnTaskQueue()) { + runnable->Run(); } else { - rv = NS_DispatchToMainThread(runnable.get()); + TaskQueue()->Dispatch(runnable.forget()); } - NS_ENSURE_SUCCESS(rv, -1); return resourceLength - aOffset - bufferLength; } diff --git a/dom/media/omx/MediaOmxReader.h b/dom/media/omx/MediaOmxReader.h index f0183897b7..f861b0ea88 100644 --- a/dom/media/omx/MediaOmxReader.h +++ b/dom/media/omx/MediaOmxReader.h @@ -69,7 +69,9 @@ public: virtual nsresult Init(MediaDecoderReader* aCloneDonor); - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset); +protected: + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; +public: virtual bool DecodeAudioData(); virtual bool DecodeVideoFrame(bool &aKeyframeSkip, @@ -85,7 +87,6 @@ public: return mHasVideo; } - virtual bool IsDormantNeeded() { return true;} virtual void ReleaseMediaResources(); virtual nsRefPtr AsyncReadMetadata() override; @@ -114,7 +115,7 @@ private: return mIsShutdown; } - int64_t ProcessCachedData(int64_t aOffset, bool aWaitForCompletion); + int64_t ProcessCachedData(int64_t aOffset); already_AddRefed SafeGetDecoder(); }; diff --git a/dom/media/raw/RawReader.cpp b/dom/media/raw/RawReader.cpp index 45af430516..16ea7e99a0 100644 --- a/dom/media/raw/RawReader.cpp +++ b/dom/media/raw/RawReader.cpp @@ -286,5 +286,6 @@ nsresult RawReader::SeekInternal(int64_t aTime) media::TimeIntervals RawReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); return media::TimeIntervals(); } diff --git a/dom/media/wave/WaveReader.cpp b/dom/media/wave/WaveReader.cpp index b740035ea7..7f9dbe2372 100644 --- a/dom/media/wave/WaveReader.cpp +++ b/dom/media/wave/WaveReader.cpp @@ -274,6 +274,7 @@ WaveReader::Seek(int64_t aTarget, int64_t aEndTime) media::TimeIntervals WaveReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); if (!mInfo.HasAudio()) { return media::TimeIntervals(); } diff --git a/dom/media/webaudio/BufferDecoder.cpp b/dom/media/webaudio/BufferDecoder.cpp index 1d00855233..2a119dfd96 100644 --- a/dom/media/webaudio/BufferDecoder.cpp +++ b/dom/media/webaudio/BufferDecoder.cpp @@ -85,12 +85,6 @@ BufferDecoder::NotifyDecodedFrames(uint32_t aParsed, uint32_t aDecoded, // ignore } -void -BufferDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) -{ - // ignore -} - void BufferDecoder::SetMediaSeekable(bool aMediaSeekable) { @@ -159,12 +153,6 @@ BufferDecoder::NotifyWaitingForResourcesStatusChanged() // ignore } -void -BufferDecoder::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) -{ - // ignore -} - MediaDecoderOwner* BufferDecoder::GetOwner() { diff --git a/dom/media/webaudio/BufferDecoder.h b/dom/media/webaudio/BufferDecoder.h index 35c1f1a085..cf07293ab7 100644 --- a/dom/media/webaudio/BufferDecoder.h +++ b/dom/media/webaudio/BufferDecoder.h @@ -45,8 +45,6 @@ public: virtual void NotifyDecodedFrames(uint32_t aParsed, uint32_t aDecoded, uint32_t aDropped) final override; - virtual void UpdateEstimatedMediaDuration(int64_t aDuration) final override; - virtual void SetMediaSeekable(bool aMediaSeekable) final override; virtual VideoFrameContainer* GetVideoFrameContainer() final override; @@ -71,7 +69,7 @@ public: virtual void NotifyWaitingForResourcesStatusChanged() final override; - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) final override; + virtual void NotifyDataArrived(uint32_t, int64_t, bool) final override {}; private: virtual ~BufferDecoder(); diff --git a/dom/media/webm/WebMBufferedParser.cpp b/dom/media/webm/WebMBufferedParser.cpp index 42a298f708..00b8a852d5 100644 --- a/dom/media/webm/WebMBufferedParser.cpp +++ b/dom/media/webm/WebMBufferedParser.cpp @@ -300,9 +300,8 @@ bool WebMBufferedState::GetOffsetForTime(uint64_t aTime, int64_t* aOffset) return true; } -void WebMBufferedState::NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset) +void WebMBufferedState::NotifyDataArrived(const unsigned char* aBuffer, uint32_t aLength, int64_t aOffset) { - NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); uint32_t idx = mRangeParsers.IndexOfFirstElementGt(aOffset - 1); if (idx == 0 || !(mRangeParsers[idx-1] == aOffset)) { // If the incoming data overlaps an already parsed range, adjust the @@ -329,7 +328,7 @@ void WebMBufferedState::NotifyDataArrived(const char* aBuffer, uint32_t aLength, } } - mRangeParsers[idx].Append(reinterpret_cast(aBuffer), + mRangeParsers[idx].Append(aBuffer, aLength, mTimeMapping, mReentrantMonitor); diff --git a/dom/media/webm/WebMBufferedParser.h b/dom/media/webm/WebMBufferedParser.h index 2599151ebb..d95625694d 100644 --- a/dom/media/webm/WebMBufferedParser.h +++ b/dom/media/webm/WebMBufferedParser.h @@ -228,7 +228,7 @@ public: MOZ_COUNT_CTOR(WebMBufferedState); } - void NotifyDataArrived(const char* aBuffer, uint32_t aLength, int64_t aOffset); + void NotifyDataArrived(const unsigned char* aBuffer, uint32_t aLength, int64_t aOffset); bool CalculateBufferedForRange(int64_t aStartOffset, int64_t aEndOffset, uint64_t* aStartTime, uint64_t* aEndTime); diff --git a/dom/media/webm/WebMReader.cpp b/dom/media/webm/WebMReader.cpp index feea2faf6c..2923922dcc 100644 --- a/dom/media/webm/WebMReader.cpp +++ b/dom/media/webm/WebMReader.cpp @@ -777,6 +777,7 @@ nsresult WebMReader::SeekInternal(int64_t aTarget) media::TimeIntervals WebMReader::GetBuffered() { + MOZ_ASSERT(OnTaskQueue()); NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); AutoPinned resource(mDecoder->GetResource()); @@ -830,10 +831,12 @@ media::TimeIntervals WebMReader::GetBuffered() return buffered; } -void WebMReader::NotifyDataArrived(const char* aBuffer, uint32_t aLength, - int64_t aOffset) +void WebMReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) { - mBufferedState->NotifyDataArrived(aBuffer, aLength, aOffset); + MOZ_ASSERT(OnTaskQueue()); + nsRefPtr bytes = mDecoder->GetResource()->SilentReadAt(aOffset, aLength); + NS_ENSURE_TRUE_VOID(bytes); + mBufferedState->NotifyDataArrived(bytes->Elements(), aLength, aOffset); } int64_t WebMReader::GetEvictionOffset(double aTime) diff --git a/dom/media/webm/WebMReader.h b/dom/media/webm/WebMReader.h index 9c7de6bb8c..5b11078a45 100644 --- a/dom/media/webm/WebMReader.h +++ b/dom/media/webm/WebMReader.h @@ -91,8 +91,6 @@ public: Seek(int64_t aTime, int64_t aEndTime) override; virtual media::TimeIntervals GetBuffered() override; - virtual void NotifyDataArrived(const char* aBuffer, uint32_t aLength, - int64_t aOffset) override; virtual int64_t GetEvictionOffset(double aTime) override; virtual bool IsMediaSeekable() override; @@ -122,6 +120,9 @@ public: uint64_t GetCodecDelay() { return mCodecDelay; } protected: + + virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; + // Decode a nestegg packet of audio data. Push the audio data on the // audio queue. Returns true when there's more audio to decode, // false if the audio is finished, end of file has been reached, diff --git a/media/libstagefright/binding/BufferStream.cpp b/media/libstagefright/binding/BufferStream.cpp index 01801c278d..5e30ebcc7e 100644 --- a/media/libstagefright/binding/BufferStream.cpp +++ b/media/libstagefright/binding/BufferStream.cpp @@ -13,11 +13,11 @@ namespace mp4_demuxer { BufferStream::BufferStream() : mStartOffset(0) - , mData(new mozilla::MediaLargeByteBuffer) + , mData(new mozilla::MediaByteBuffer) { } -BufferStream::BufferStream(mozilla::MediaLargeByteBuffer* aBuffer) +BufferStream::BufferStream(mozilla::MediaByteBuffer* aBuffer) : mStartOffset(0) , mData(aBuffer) { diff --git a/media/libstagefright/binding/MP4Metadata.cpp b/media/libstagefright/binding/MP4Metadata.cpp index eec762d624..1ce840273d 100644 --- a/media/libstagefright/binding/MP4Metadata.cpp +++ b/media/libstagefright/binding/MP4Metadata.cpp @@ -297,7 +297,7 @@ MP4Metadata::HasCompleteMetadata(Stream* aSource) return parser->HasMetadata(); } -/*static*/ already_AddRefed +/*static*/ already_AddRefed MP4Metadata::Metadata(Stream* aSource) { // The MoofParser requires a monitor, but we don't need one here. diff --git a/media/libstagefright/binding/MoofParser.cpp b/media/libstagefright/binding/MoofParser.cpp index e8ad89d99f..0a47f149ae 100644 --- a/media/libstagefright/binding/MoofParser.cpp +++ b/media/libstagefright/binding/MoofParser.cpp @@ -181,7 +181,7 @@ MoofParser::HasMetadata() return !!ftyp.Length() && !!moov.Length(); } -already_AddRefed +already_AddRefed MoofParser::Metadata() { MediaByteRange ftyp; @@ -190,7 +190,7 @@ MoofParser::Metadata() if (!ftyp.Length() || !moov.Length()) { return nullptr; } - nsRefPtr metadata = new MediaLargeByteBuffer(); + nsRefPtr metadata = new MediaByteBuffer(); if (!metadata->SetLength(ftyp.Length() + moov.Length(), fallible)) { // OOM return nullptr; diff --git a/media/libstagefright/binding/include/mp4_demuxer/BufferStream.h b/media/libstagefright/binding/include/mp4_demuxer/BufferStream.h index 4544d5177d..659027d22c 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/BufferStream.h +++ b/media/libstagefright/binding/include/mp4_demuxer/BufferStream.h @@ -10,7 +10,7 @@ #include "MediaResource.h" namespace mozilla { -class MediaLargeByteBuffer; +class MediaByteBuffer; } namespace mp4_demuxer { @@ -22,7 +22,7 @@ public: * Therefore BufferStream shouldn't get used after aData is destroyed. */ BufferStream(); - explicit BufferStream(mozilla::MediaLargeByteBuffer* aBuffer); + explicit BufferStream(mozilla::MediaByteBuffer* aBuffer); virtual bool ReadAt(int64_t aOffset, void* aData, size_t aLength, size_t* aBytesRead) override; @@ -39,7 +39,7 @@ public: private: ~BufferStream(); int64_t mStartOffset; - nsRefPtr mData; + nsRefPtr mData; }; } diff --git a/media/libstagefright/binding/include/mp4_demuxer/ByteReader.h b/media/libstagefright/binding/include/mp4_demuxer/ByteReader.h index abd0531ca0..8a30a21dcd 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/ByteReader.h +++ b/media/libstagefright/binding/include/mp4_demuxer/ByteReader.h @@ -33,10 +33,6 @@ public: : mPtr(aData.Elements()), mRemaining(aData.Length()), mLength(aData.Length()) { } - explicit ByteReader(const mozilla::MediaLargeByteBuffer* aData) - : mPtr(aData->Elements()), mRemaining(aData->Length()), mLength(aData->Length()) - { - } explicit ByteReader(const mozilla::MediaByteBuffer* aData) : mPtr(aData->Elements()), mRemaining(aData->Length()), mLength(aData->Length()) { diff --git a/media/libstagefright/binding/include/mp4_demuxer/MP4Metadata.h b/media/libstagefright/binding/include/mp4_demuxer/MP4Metadata.h index cec9feb1a6..f2cbfd6a34 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/MP4Metadata.h +++ b/media/libstagefright/binding/include/mp4_demuxer/MP4Metadata.h @@ -30,7 +30,7 @@ public: ~MP4Metadata(); static bool HasCompleteMetadata(Stream* aSource); - static already_AddRefed Metadata(Stream* aSource); + static already_AddRefed Metadata(Stream* aSource); uint32_t GetNumberTracks(mozilla::TrackInfo::TrackType aType) const; mozilla::UniquePtr GetTrackInfo(mozilla::TrackInfo::TrackType aType, size_t aTrackNumber) const; diff --git a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h index 8979fa9b56..850d8edd04 100644 --- a/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h +++ b/media/libstagefright/binding/include/mp4_demuxer/MoofParser.h @@ -227,7 +227,7 @@ public: bool BlockingReadNextMoof(); bool HasMetadata(); - already_AddRefed Metadata(); + already_AddRefed Metadata(); MediaByteRange FirstCompleteMediaSegment(); MediaByteRange FirstCompleteMediaHeader();