From c80d3c1b1f4f85cc74f464f596166b63fedcd934 Mon Sep 17 00:00:00 2001 From: roytam1 Date: Tue, 5 Oct 2021 09:55:30 +0800 Subject: [PATCH] import changes from `dev' branch of rmottola/Arctic-Fox: - Bug 1195185. Part 1 - rename Connect/Remove to AddOutput/RemoveOutput. r=roc. (549b92cb4) - Bug 1195185. Part 2 - Check if all output streams are from the same graph because we don't support connecting streams from different graphs. r=roc. (6b66501ed) - Bug 1195185. Part 3 - align the life cycle of mData with {Start,Stop}Playback. r=roc. (90f9a6fce) - Bug 1195185. Part 4 - remove dead code. r=roc. (1bcfb5837) - Bug 1195632. Part 1 - Let DecodedStream have a worker thread and asset some funtions on the worker thread. r=roc. (5987408e8) - Bug 1195632. Part 2 - Have DecodedStream listen to push events of the media queues and call SendData() on its own without the help of MDSM. r=roc. (d082c7c8a) - Bug 1198568 - Fix build error in non-unified build for DecodedAudioDataSink.cpp. r=kinetik. (dc1289a89) - Bug 1199104. Part 1 - create MediaSink. r=kinetik. (29657636b) - Bug 1199104. Part 2 - create AudioSinkWrapper. r=kinetik. (2a970c2a6) - Bug 1199104. Part 3 - use AudioSinkWrapper in MDSM. r=kinetik. (d31f8986d) - Bug 1195158. Part 1 - Have MediaMetadataManager listen to an event source to receive TimedMetadata events. OggReader will send TimedMetadata events through an event source. This will break OggReader's dependency on AbstractMediaDecoder::QueueMetadata which then can be removed for it is against our goal to run all MediaDecoder's methods on the main thread. r=cpearce. (4f2df907d) - Bug 1195158. Part 2 - Have OggReader send TimedMetadata events through a event source instead of direct calls to AbstractMediaDecoder::QueueMetadata. r=cpearce. (cce94d58a) - Bug 1195158. Part 3 - connect listeners. r=cpearce. (60bcc5f8b) - Bug 1195158. Part 4 - remove unused code. r=cpearce. (072432eef) - Bug 1195158. Part 5 - 1. Fix insufficient includes and sort out include order. 2. Only disconnect |mTimedMetadataListener| when the state machine is created. r=cpearce. (d552e56a8) - Bug 1204407: P1. Remove no longer used mainthread object. r=cpearce (30f6b1c86) - Bug 1185792: [webm] P1. Don't clear mNeedReIndex if GetCachedRanges is emtpy. r=jya (3169897d5) - Bug 1194884: [webm] P1. Use MediaResourceIndex. r=j^ (6dbe08ddf) - Bug 1194884: [webm] P2. Retrieve all VPX decoded frames from decoder. r=j^ (060727eca) --- dom/media/AbstractMediaDecoder.h | 42 ---- dom/media/DecodedStream.cpp | 194 +++++++++++++------ dom/media/DecodedStream.h | 38 +++- dom/media/MediaDecoder.cpp | 17 +- dom/media/MediaDecoder.h | 29 +-- dom/media/MediaDecoderReader.h | 12 +- dom/media/MediaDecoderStateMachine.cpp | 124 +++++------- dom/media/MediaDecoderStateMachine.h | 21 +- dom/media/MediaFormatReader.cpp | 50 ----- dom/media/MediaFormatReader.h | 7 - dom/media/MediaMetadataManager.h | 62 ++++-- dom/media/mediasink/AudioSink.h | 1 + dom/media/mediasink/AudioSinkWrapper.cpp | 160 +++++++++++++++ dom/media/mediasink/AudioSinkWrapper.h | 93 +++++++++ dom/media/mediasink/DecodedAudioDataSink.cpp | 4 + dom/media/mediasink/DecodedAudioDataSink.h | 6 +- dom/media/mediasink/MediaSink.h | 115 +++++++++++ dom/media/mediasink/moz.build | 1 + dom/media/ogg/OggReader.cpp | 7 +- dom/media/platforms/agnostic/VPXDecoder.cpp | 7 +- dom/media/platforms/agnostic/VPXDecoder.h | 1 - dom/media/webaudio/BufferDecoder.cpp | 12 -- dom/media/webaudio/BufferDecoder.h | 3 - dom/media/webm/WebMDemuxer.cpp | 101 ++++------ dom/media/webm/WebMDemuxer.h | 7 +- 25 files changed, 726 insertions(+), 388 deletions(-) create mode 100644 dom/media/mediasink/AudioSinkWrapper.cpp create mode 100644 dom/media/mediasink/AudioSinkWrapper.h create mode 100644 dom/media/mediasink/MediaSink.h diff --git a/dom/media/AbstractMediaDecoder.h b/dom/media/AbstractMediaDecoder.h index ecc5748034..c7fdb82ed5 100644 --- a/dom/media/AbstractMediaDecoder.h +++ b/dom/media/AbstractMediaDecoder.h @@ -100,11 +100,8 @@ public: virtual bool IsMediaSeekable() = 0; virtual void MetadataLoaded(nsAutoPtr aInfo, nsAutoPtr aTags, MediaDecoderEventVisibility aEventVisibility) = 0; - virtual void QueueMetadata(const media::TimeUnit& aTime, nsAutoPtr aInfo, nsAutoPtr aTags) = 0; virtual void FirstFrameLoaded(nsAutoPtr aInfo, MediaDecoderEventVisibility aEventVisibility) = 0; - virtual void RemoveMediaTracks() = 0; - // May be called by the reader to notify this decoder that the metadata from // the media file has been read. Call on the decode thread only. virtual void OnReadMetadataCompleted() = 0; @@ -213,45 +210,6 @@ public: } }; -class MetadataUpdatedEventRunner : public nsRunnable, private MetadataContainer -{ -public: - MetadataUpdatedEventRunner(AbstractMediaDecoder* aDecoder, - nsAutoPtr aInfo, - nsAutoPtr aTags, - MediaDecoderEventVisibility aEventVisibility = MediaDecoderEventVisibility::Observable) - : MetadataContainer(aDecoder, aInfo, aTags, aEventVisibility) - {} - - NS_IMETHOD Run() override - { - nsAutoPtr info(new MediaInfo()); - *info = *mInfo; - mDecoder->MetadataLoaded(info, mTags, mEventVisibility); - mDecoder->FirstFrameLoaded(mInfo, mEventVisibility); - return NS_OK; - } -}; - -class RemoveMediaTracksEventRunner : public nsRunnable -{ -public: - explicit RemoveMediaTracksEventRunner(AbstractMediaDecoder* aDecoder) - : mDecoder(aDecoder) - {} - - NS_IMETHOD Run() override - { - MOZ_ASSERT(NS_IsMainThread()); - - mDecoder->RemoveMediaTracks(); - return NS_OK; - } - -private: - nsRefPtr mDecoder; -}; - } // namespace mozilla #endif diff --git a/dom/media/DecodedStream.cpp b/dom/media/DecodedStream.cpp index 1271ff173a..43f604e543 100644 --- a/dom/media/DecodedStream.cpp +++ b/dom/media/DecodedStream.cpp @@ -105,7 +105,8 @@ UpdateStreamBlocking(MediaStream* aStream, bool aBlocking) */ class DecodedStreamData { public: - DecodedStreamData(SourceMediaStream* aStream, bool aPlaying); + DecodedStreamData(SourceMediaStream* aStream, bool aPlaying, + MozPromiseHolder&& aPromise); ~DecodedStreamData(); bool IsFinished() const; int64_t GetPosition() const; @@ -139,11 +140,10 @@ public: // True if we need to send a compensation video frame to ensure the // StreamTime going forward. bool mEOSVideoCompensation; - // This promise will be resolved when the SourceMediaStream is finished. - nsRefPtr mFinishPromise; }; -DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream, bool aPlaying) +DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream, bool aPlaying, + MozPromiseHolder&& aPromise) : mAudioFramesWritten(0) , mNextVideoTime(-1) , mNextAudioTime(-1) @@ -155,10 +155,8 @@ DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream, bool aPlaying) , mPlaying(aPlaying) , mEOSVideoCompensation(false) { - MozPromiseHolder promise; - mFinishPromise = promise.Ensure(__func__); // DecodedStreamGraphListener will resolve this promise. - mListener = new DecodedStreamGraphListener(mStream, Move(promise)); + mListener = new DecodedStreamGraphListener(mStream, Move(aPromise)); mStream->AddListener(mListener); // Block the stream if we are not playing. @@ -293,10 +291,19 @@ OutputStreamData::Remove() mOwner->Remove(mStream); } +MediaStreamGraph* +OutputStreamData::Graph() const +{ + return mStream->Graph(); +} + void OutputStreamManager::Add(ProcessedMediaStream* aStream, bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); + // All streams must belong to the same graph. + MOZ_ASSERT(!Graph() || Graph() == aStream->Graph()); + // Ensure that aStream finishes the moment mDecodedStream does. if (aFinishWhenEnded) { aStream->SetAutofinish(true); @@ -347,9 +354,11 @@ OutputStreamManager::Disconnect() } } -DecodedStream::DecodedStream(MediaQueue& aAudioQueue, +DecodedStream::DecodedStream(AbstractThread* aOwnerThread, + MediaQueue& aAudioQueue, MediaQueue& aVideoQueue) - : mMonitor("DecodedStream::mMonitor") + : mOwnerThread(aOwnerThread) + , mMonitor("DecodedStream::mMonitor") , mPlaying(false) , mVolume(1.0) , mAudioQueue(aAudioQueue) @@ -359,73 +368,109 @@ DecodedStream::DecodedStream(MediaQueue& aAudioQueue, DecodedStream::~DecodedStream() { + MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended."); } nsRefPtr DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo) { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); MOZ_ASSERT(mStartTime.isNothing(), "playback already started."); + mStartTime.emplace(aStartTime); mInfo = aInfo; + ConnectListener(); - // TODO: Unfortunately, current call flow of MDSM guarantees mData is non-null - // when StartPlayback() is called which imposes an obscure dependency on MDSM. - // We will align the life cycle of mData with {Start,Stop}Playback so that - // DecodedStream doesn't need to make assumptions about mData's life cycle. - return mData->mFinishPromise; + class R : public nsRunnable { + typedef MozPromiseHolder Promise; + typedef void(DecodedStream::*Method)(Promise&&); + public: + R(DecodedStream* aThis, Method aMethod, Promise&& aPromise) + : mThis(aThis), mMethod(aMethod) + { + mPromise = Move(aPromise); + } + NS_IMETHOD Run() override + { + (mThis->*mMethod)(Move(mPromise)); + return NS_OK; + } + private: + RefPtr mThis; + Method mMethod; + Promise mPromise; + }; + + MozPromiseHolder promise; + nsRefPtr rv = promise.Ensure(__func__); + nsCOMPtr r = new R(this, &DecodedStream::CreateData, Move(promise)); + AbstractThread::MainThread()->Dispatch(r.forget()); + + return rv.forget(); } void DecodedStream::StopPlayback() { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - mStartTime.reset(); -} - -void -DecodedStream::DestroyData() -{ - MOZ_ASSERT(NS_IsMainThread()); - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - - // Avoid the redundant blocking to output stream. - if (!mData) { + // Playback didn't even start at all. + if (mStartTime.isNothing()) { return; } - mOutputStreamManager.Disconnect(); - mData = nullptr; -} + mStartTime.reset(); + DisconnectListener(); + + // Clear mData immediately when this playback session ends so we won't + // send data to the wrong stream in SendData() in next playback session. + DecodedStreamData* data = mData.release(); + // mData is not yet created on the main thread. + if (!data) { + return; + } -void -DecodedStream::RecreateData() -{ nsRefPtr self = this; - nsCOMPtr r = NS_NewRunnableFunction([self] () -> void { - self->RecreateData(nullptr); + nsCOMPtr r = NS_NewRunnableFunction([=] () { + self->mOutputStreamManager.Disconnect(); + delete data; }); AbstractThread::MainThread()->Dispatch(r.forget()); } void -DecodedStream::RecreateData(MediaStreamGraph* aGraph) +DecodedStream::CreateData(MozPromiseHolder&& aPromise) { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - MOZ_ASSERT((aGraph && !mData && !HasConsumers()) || // first time - (!aGraph && mData)); // 2nd time and later + MOZ_ASSERT(!mData, "Already created."); - if (!aGraph) { - aGraph = mData->mStream->Graph(); + // No need to create a source stream when there are no output streams. This + // happens when RemoveOutput() is called immediately after StartPlayback(). + // We also bail out when the playback session has ended. This happens when + // StopPlayback() is called immediately after StartPlayback(). + if (!mOutputStreamManager.Graph() || mStartTime.isNothing()) { + // Resolve the promise to indicate the end of playback. + aPromise.Resolve(true, __func__); + return; } - auto source = aGraph->CreateSourceStream(nullptr); - DestroyData(); - mData.reset(new DecodedStreamData(source, mPlaying)); - // Note that the delay between removing ports in DestroyData - // and adding new ones won't cause a glitch since all graph operations - // between main-thread stable states take effect atomically. + auto source = mOutputStreamManager.Graph()->CreateSourceStream(nullptr); + mData.reset(new DecodedStreamData(source, mPlaying, Move(aPromise))); mOutputStreamManager.Connect(mData->mStream); + + // Start to send data to the stream immediately + nsRefPtr self = this; + nsCOMPtr r = NS_NewRunnableFunction([=] () { + ReentrantMonitorAutoEnter mon(self->GetReentrantMonitor()); + // Don't send data if playback has ended. + if (self->mStartTime.isSome()) { + self->SendData(); + } + }); + // Don't assert success because the owner thread might have begun shutdown + // while we are still dealing with jobs on the main thread. + mOwnerThread->Dispatch(r.forget(), AbstractThread::DontAssertDispatchSuccess); } bool @@ -441,20 +486,13 @@ DecodedStream::GetReentrantMonitor() const } void -DecodedStream::Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded) +DecodedStream::AddOutput(ProcessedMediaStream* aStream, bool aFinishWhenEnded) { - MOZ_ASSERT(NS_IsMainThread()); - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - - if (!mData) { - RecreateData(aStream->Graph()); - } - mOutputStreamManager.Add(aStream, aFinishWhenEnded); } void -DecodedStream::Remove(MediaStream* aStream) +DecodedStream::RemoveOutput(MediaStream* aStream) { mOutputStreamManager.Remove(aStream); } @@ -462,6 +500,7 @@ DecodedStream::Remove(MediaStream* aStream) void DecodedStream::SetPlaying(bool aPlaying) { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mPlaying = aPlaying; if (mData) { @@ -472,6 +511,7 @@ DecodedStream::SetPlaying(bool aPlaying) void DecodedStream::SetVolume(double aVolume) { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mVolume = aVolume; } @@ -479,6 +519,7 @@ DecodedStream::SetVolume(double aVolume) void DecodedStream::SetSameOrigin(bool aSameOrigin) { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mSameOrigin = aSameOrigin; } @@ -486,6 +527,7 @@ DecodedStream::SetSameOrigin(bool aSameOrigin) void DecodedStream::InitTracks() { + AssertOwnerThread(); GetReentrantMonitor().AssertCurrentThreadIn(); if (mData->mStreamInitialized) { @@ -566,6 +608,7 @@ SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, void DecodedStream::SendAudio(double aVolume, bool aIsSameOrigin) { + AssertOwnerThread(); GetReentrantMonitor().AssertCurrentThreadIn(); if (!mInfo.HasAudio()) { @@ -631,6 +674,7 @@ ZeroDurationAtLastChunk(VideoSegment& aInput) void DecodedStream::SendVideo(bool aIsSameOrigin) { + AssertOwnerThread(); GetReentrantMonitor().AssertCurrentThreadIn(); if (!mInfo.HasVideo()) { @@ -709,6 +753,7 @@ DecodedStream::SendVideo(bool aIsSameOrigin) void DecodedStream::AdvanceTracks() { + AssertOwnerThread(); GetReentrantMonitor().AssertCurrentThreadIn(); StreamTime endPosition = 0; @@ -733,9 +778,15 @@ DecodedStream::AdvanceTracks() void DecodedStream::SendData() { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); MOZ_ASSERT(mStartTime.isSome(), "Must be called after StartPlayback()"); + // Not yet created on the main thread. MDSM will try again later. + if (!mData) { + return; + } + // Nothing to do when the stream is finished. if (mData->mHaveSentFinish) { return; @@ -758,8 +809,9 @@ DecodedStream::SendData() int64_t DecodedStream::AudioEndTime() const { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - if (mStartTime.isSome() && mInfo.HasAudio()) { + if (mStartTime.isSome() && mInfo.HasAudio() && mData) { CheckedInt64 t = mStartTime.ref() + FramesToUsecs(mData->mAudioFramesWritten, mInfo.mAudio.mRate); if (t.isValid()) { @@ -772,18 +824,48 @@ DecodedStream::AudioEndTime() const int64_t DecodedStream::GetPosition() const { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); // This is only called after MDSM starts playback. So mStartTime is // guaranteed to be something. MOZ_ASSERT(mStartTime.isSome()); - return mStartTime.ref() + mData->GetPosition(); + return mStartTime.ref() + (mData ? mData->GetPosition() : 0); } bool DecodedStream::IsFinished() const { + AssertOwnerThread(); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - return mData->IsFinished(); + return mData && mData->IsFinished(); +} + +void +DecodedStream::ConnectListener() +{ + AssertOwnerThread(); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + mAudioPushListener = mAudioQueue.PushEvent().Connect( + mOwnerThread, this, &DecodedStream::SendData); + mAudioFinishListener = mAudioQueue.FinishEvent().Connect( + mOwnerThread, this, &DecodedStream::SendData); + mVideoPushListener = mVideoQueue.PushEvent().Connect( + mOwnerThread, this, &DecodedStream::SendData); + mVideoFinishListener = mVideoQueue.FinishEvent().Connect( + mOwnerThread, this, &DecodedStream::SendData); +} + +void +DecodedStream::DisconnectListener() +{ + AssertOwnerThread(); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + mAudioPushListener.Disconnect(); + mVideoPushListener.Disconnect(); + mAudioFinishListener.Disconnect(); + mVideoFinishListener.Disconnect(); } } // namespace mozilla diff --git a/dom/media/DecodedStream.h b/dom/media/DecodedStream.h index 478ec3dcef..adc7220bad 100644 --- a/dom/media/DecodedStream.h +++ b/dom/media/DecodedStream.h @@ -8,8 +8,10 @@ #define DecodedStream_h_ #include "nsTArray.h" +#include "MediaEventSource.h" #include "MediaInfo.h" +#include "mozilla/AbstractThread.h" #include "mozilla/CheckedInt.h" #include "mozilla/Maybe.h" #include "mozilla/MozPromise.h" @@ -56,6 +58,8 @@ public: { return mStream == aStream; } + // Return the graph mStream belongs to. + MediaStreamGraph* Graph() const; private: OutputStreamManager* mOwner; @@ -81,6 +85,12 @@ public: void Connect(MediaStream* aStream); // Disconnect all output streams from the input stream. void Disconnect(); + // Return the graph these streams belong to or null if empty. + MediaStreamGraph* Graph() const + { + MOZ_ASSERT(NS_IsMainThread()); + return !IsEmpty() ? mStreams[0].Graph() : nullptr; + } private: // Keep the input stream so we can connect the output streams that @@ -92,7 +102,8 @@ private: class DecodedStream { NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStream); public: - DecodedStream(MediaQueue& aAudioQueue, + DecodedStream(AbstractThread* aOwnerThread, + MediaQueue& aAudioQueue, MediaQueue& aVideoQueue); // Mimic MDSM::StartAudioThread. @@ -105,10 +116,8 @@ public: // Mimic MDSM::StopAudioThread. void StopPlayback(); - void DestroyData(); - void RecreateData(); - void Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded); - void Remove(MediaStream* aStream); + void AddOutput(ProcessedMediaStream* aStream, bool aFinishWhenEnded); + void RemoveOutput(MediaStream* aStream); void SetPlaying(bool aPlaying); void SetVolume(double aVolume); @@ -119,18 +128,26 @@ public: bool IsFinished() const; bool HasConsumers() const; - void SendData(); - protected: virtual ~DecodedStream(); private: ReentrantMonitor& GetReentrantMonitor() const; - void RecreateData(MediaStreamGraph* aGraph); + void CreateData(MozPromiseHolder&& aPromise); void InitTracks(); void AdvanceTracks(); void SendAudio(double aVolume, bool aIsSameOrigin); void SendVideo(bool aIsSameOrigin); + void SendData(); + + void AssertOwnerThread() const { + MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); + } + + void ConnectListener(); + void DisconnectListener(); + + const nsRefPtr mOwnerThread; UniquePtr mData; // Data about MediaStreams that are being fed by the decoder. @@ -154,6 +171,11 @@ private: MediaQueue& mAudioQueue; MediaQueue& mVideoQueue; + + MediaEventListener mAudioPushListener; + MediaEventListener mVideoPushListener; + MediaEventListener mAudioFinishListener; + MediaEventListener mVideoFinishListener; }; } // namespace mozilla diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 6199ee8816..7efcaca092 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -446,6 +446,7 @@ void MediaDecoder::Shutdown() // the asynchronous shutdown in nsDestroyStateMachine won't deadlock. if (mDecoderStateMachine) { mDecoderStateMachine->DispatchShutdown(); + mTimedMetadataListener.Disconnect(); } // Force any outstanding seek and byterange requests to complete @@ -528,6 +529,8 @@ void MediaDecoder::SetStateMachineParameters() if (mMinimizePreroll) { mDecoderStateMachine->DispatchMinimizePrerollUntilPlaybackStarts(); } + mTimedMetadataListener = mDecoderStateMachine->TimedMetadataEvent().Connect( + AbstractThread::MainThread(), this, &MediaDecoder::OnMetadataUpdate); } void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts() @@ -618,13 +621,15 @@ already_AddRefed MediaDecoder::GetCurrentPrincipal() return mResource ? mResource->GetCurrentPrincipal() : nullptr; } -void MediaDecoder::QueueMetadata(const TimeUnit& aPublishTime, - nsAutoPtr aInfo, - nsAutoPtr aTags) +void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) { - MOZ_ASSERT(OnDecodeTaskQueue()); - GetReentrantMonitor().AssertCurrentThreadIn(); - mDecoderStateMachine->QueueMetadata(aPublishTime, aInfo, aTags); + MOZ_ASSERT(NS_IsMainThread()); + RemoveMediaTracks(); + MetadataLoaded(nsAutoPtr(new MediaInfo(*aMetadata.mInfo)), + Move(aMetadata.mTags), + MediaDecoderEventVisibility::Observable); + FirstFrameLoaded(Move(aMetadata.mInfo), + MediaDecoderEventVisibility::Observable); } void MediaDecoder::MetadataLoaded(nsAutoPtr aInfo, diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index d301ab5bd7..36b9ddc02d 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -191,17 +191,20 @@ destroying the MediaDecoder object. #include "mozilla/dom/AudioChannelBinding.h" -#include "nsISupports.h" +#include "necko-config.h" +#include "nsAutoPtr.h" #include "nsCOMPtr.h" #include "nsIObserver.h" -#include "nsAutoPtr.h" +#include "nsISupports.h" #include "nsITimer.h" -#include "MediaResource.h" -#include "MediaDecoderOwner.h" -#include "MediaStreamGraph.h" + #include "AbstractMediaDecoder.h" #include "DecodedStream.h" -#include "necko-config.h" +#include "MediaDecoderOwner.h" +#include "MediaEventSource.h" +#include "MediaMetadataManager.h" +#include "MediaResource.h" +#include "MediaStreamGraph.h" #include "TimeUnits.h" class nsIStreamListener; @@ -576,13 +579,6 @@ public: void SetAudioChannel(dom::AudioChannel aChannel) { mAudioChannel = aChannel; } dom::AudioChannel GetAudioChannel() { return mAudioChannel; } - // Send a new set of metadata to the state machine, to be dispatched to the - // main thread to be presented when the |currentTime| of the media is greater - // or equal to aPublishTime. - void QueueMetadata(const media::TimeUnit& aPublishTime, - nsAutoPtr aInfo, - nsAutoPtr aTags) override; - /****** * The following methods must only be called on the main * thread. @@ -615,7 +611,7 @@ public: // Removes all audio tracks and video tracks that are previously added into // the track list. Call on the main thread only. - virtual void RemoveMediaTracks() override; + void RemoveMediaTracks(); // Called when the video has completed playing. // Call on the main thread only. @@ -973,6 +969,8 @@ protected: const char* PlayStateStr(); + void OnMetadataUpdate(TimedMetadata&& aMetadata); + // This should only ever be accessed from the main thread. // It is set in Init and cleared in Shutdown when the element goes away. // The decoder does not add a reference the element. @@ -1043,6 +1041,9 @@ protected: // Timer to schedule updating dormant state. nsCOMPtr mDormantTimer; + // A listener to receive metadata updates from MDSM. + MediaEventListener mTimedMetadataListener; + protected: // Whether the state machine is shut down. Mirror mStateMachineIsShutdown; diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h index 53b97f59b1..83ad28ab1e 100644 --- a/dom/media/MediaDecoderReader.h +++ b/dom/media/MediaDecoderReader.h @@ -11,6 +11,7 @@ #include "AbstractMediaDecoder.h" #include "MediaInfo.h" #include "MediaData.h" +#include "MediaMetadataManager.h" #include "MediaQueue.h" #include "MediaTimer.h" #include "AudioCompactor.h" @@ -320,12 +321,16 @@ public: // the newer async model. virtual bool IsAsync() const { return false; } - virtual void DisableHardwareAcceleration() {} - // Returns true if this decoder reader uses hardware accelerated video // decoding. virtual bool VideoIsHardwareAccelerated() const { return false; } + virtual void DisableHardwareAcceleration() {} + + TimedMetadataEventSource& TimedMetadataEvent() { + return mTimedMetadataEvent; + } + protected: virtual ~MediaDecoderReader(); @@ -417,6 +422,9 @@ protected: bool mHitAudioDecodeError; bool mShutdown; + // Used to send TimedMetadata to the listener. + TimedMetadataEventProducer mTimedMetadataEvent; + private: // Promises used only for the base-class (sync->async adapter) implementation // of Request{Audio,Video}Data. diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 801e0fddfc..9817af2506 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -15,6 +15,7 @@ #include "MediaDecoderStateMachine.h" #include "MediaTimer.h" #include "mediasink/DecodedAudioDataSink.h" +#include "mediasink/AudioSinkWrapper.h" #include "nsTArray.h" #include "MediaDecoder.h" #include "MediaDecoderReader.h" @@ -220,7 +221,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mSentLoadedMetadataEvent(false), mSentFirstFrameLoadedEvent(false), mSentPlaybackEndedEvent(false), - mDecodedStream(new DecodedStream(mAudioQueue, mVideoQueue)), + mDecodedStream(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)), mResource(aDecoder->GetResource()), mBuffered(mTaskQueue, TimeIntervals(), "MediaDecoderStateMachine::mBuffered (Mirror)"), @@ -284,6 +285,17 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped); mVideoQueueListener = VideoQueue().PopEvent().Connect( mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped); + + mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread()); + + nsRefPtr self = this; + auto audioSinkCreator = [self] () { + MOZ_ASSERT(self->OnTaskQueue()); + return new DecodedAudioDataSink( + self->mAudioQueue, self->GetMediaTime(), + self->mInfo.mAudio, self->mDecoder->GetAudioChannel()); + }; + mAudioSink = new AudioSinkWrapper(mTaskQueue, audioSinkCreator); } MediaDecoderStateMachine::~MediaDecoderStateMachine() @@ -359,19 +371,17 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); int64_t audioDecoded = AudioQueue().Duration(); - if (AudioEndTime() != -1) { + if (mAudioSink->IsStarted()) { audioDecoded += AudioEndTime() - GetMediaTime(); } return audioDecoded; } -void MediaDecoderStateMachine::SendStreamData() +void MediaDecoderStateMachine::DiscardStreamData() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - MOZ_ASSERT(!mAudioSink, "Should've been stopped in RunStateMachine()"); - - mDecodedStream->SendData(); + MOZ_ASSERT(!mAudioSink->IsStarted(), "Should've been stopped in RunStateMachine()"); const auto clockTime = GetClock(); while (true) { @@ -381,7 +391,9 @@ void MediaDecoderStateMachine::SendStreamData() // keep decoding audio samples till the end and consume a lot of memory. // Therefore we only discard those behind the stream clock to throttle // the decoding speed. - if (a && a->mTime <= clockTime) { + // Note we don't discard a sample when |a->mTime == clockTime| because that + // will discard the 1st sample when clockTime is still 0. + if (a && a->mTime < clockTime) { nsRefPtr releaseMe = AudioQueue().PopFront(); continue; } @@ -562,10 +574,6 @@ MediaDecoderStateMachine::OnAudioDecoded(AudioData* aAudioSample) if (mIsAudioPrerolling && DonePrerollingAudio()) { StopPrerollingAudio(); } - // Schedule the state machine to send stream data as soon as possible. - if (mAudioCaptured) { - ScheduleStateMachine(); - } return; } @@ -761,10 +769,6 @@ MediaDecoderStateMachine::OnNotDecoded(MediaData::Type aType, return; } CheckIfDecodeComplete(); - // Schedule the state machine to notify track ended as soon as possible. - if (mAudioCaptured) { - ScheduleStateMachine(); - } return; } case DECODER_STATE_SEEKING: { @@ -851,7 +855,7 @@ MediaDecoderStateMachine::OnVideoDecoded(VideoData* aVideoSample) // frame pushed in the queue, schedule the state machine as soon as // possible to render the video frame or delay the state machine thread // accurately. - if (mAudioCaptured || VideoQueue().GetSize() == 1) { + if (VideoQueue().GetSize() == 1) { ScheduleStateMachine(); } @@ -1091,7 +1095,7 @@ void MediaDecoderStateMachine::UpdatePlaybackPosition(int64_t aTime) UpdatePlaybackPositionInternal(aTime); bool fragmentEnded = mFragmentEndTime >= 0 && GetMediaTime() >= mFragmentEndTime; - mMetadataManager.DispatchMetadataIfNeeded(mDecoder, TimeUnit::FromMicroseconds(aTime)); + mMetadataManager.DispatchMetadataIfNeeded(TimeUnit::FromMicroseconds(aTime)); if (fragmentEnded) { StopPlayback(); @@ -1142,9 +1146,7 @@ void MediaDecoderStateMachine::VolumeChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - if (mAudioSink) { - mAudioSink->SetVolume(mVolume); - } + mAudioSink->SetVolume(mVolume); mDecodedStream->SetVolume(mVolume); } @@ -1262,6 +1264,8 @@ void MediaDecoderStateMachine::Shutdown() Reset(); + mAudioSink->Shutdown(); + // Shut down our start time rendezvous. if (mStartTimeRendezvous) { mStartTimeRendezvous->Destroy(); @@ -1456,12 +1460,11 @@ void MediaDecoderStateMachine::StopAudioSink() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (mAudioSink) { - DECODER_LOG("Shutdown audio thread"); - mAudioSink->Shutdown(); - mAudioSink = nullptr; + if (mAudioSink->IsStarted()) { + DECODER_LOG("Stop AudioSink"); + mAudioSink->Stop(); + mAudioSinkPromise.DisconnectIfExists(); } - mAudioSinkPromise.DisconnectIfExists(); } void @@ -1542,10 +1545,6 @@ MediaDecoderStateMachine::InitiateSeek() "Can only seek in range [0,duration]"); mCurrentSeek.mTarget.mTime = seekTime; - if (mAudioCaptured) { - mDecodedStream->RecreateData(); - } - mDropAudioUntilNextDiscontinuity = HasAudio(); mDropVideoUntilNextDiscontinuity = HasVideo(); mCurrentTimeBeforeSeek = GetMediaTime(); @@ -1749,25 +1748,19 @@ MediaDecoderStateMachine::StartAudioSink() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); if (mAudioCaptured) { - MOZ_ASSERT(!mAudioSink); + MOZ_ASSERT(!mAudioSink->IsStarted()); return; } - if (HasAudio() && !mAudioSink) { + if (HasAudio() && !mAudioSink->IsStarted()) { mAudioCompleted = false; - mAudioSink = new DecodedAudioDataSink(mAudioQueue, - GetMediaTime(), mInfo.mAudio, - mDecoder->GetAudioChannel()); + mAudioSink->Start(GetMediaTime(), mInfo); mAudioSinkPromise.Begin( - mAudioSink->Init()->Then( + mAudioSink->OnEnded(TrackInfo::kAudioTrack)->Then( OwnerThread(), __func__, this, &MediaDecoderStateMachine::OnAudioSinkComplete, &MediaDecoderStateMachine::OnAudioSinkError)); - - mAudioSink->SetVolume(mVolume); - mAudioSink->SetPlaybackRate(mPlaybackRate); - mAudioSink->SetPreservesPitch(mPreservesPitch); } } @@ -1805,7 +1798,7 @@ int64_t MediaDecoderStateMachine::AudioDecodedUsecs() // The amount of audio we have decoded is the amount of audio data we've // already decoded and pushed to the hardware, plus the amount of audio // data waiting to be pushed to the hardware. - int64_t pushed = (AudioEndTime() != -1) ? (AudioEndTime() - GetMediaTime()) : 0; + int64_t pushed = mAudioSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0; // Currently for real time streams, AudioQueue().Duration() produce // wrong values (Bug 1114434), so we use frame counts to calculate duration. @@ -1834,7 +1827,7 @@ bool MediaDecoderStateMachine::OutOfDecodedAudio() MOZ_ASSERT(OnTaskQueue()); return IsAudioDecoding() && !AudioQueue().IsFinished() && AudioQueue().GetSize() == 0 && - (!mAudioSink || !mAudioSink->HasUnplayedFrames()); + !mAudioSink->HasUnplayedFrames(TrackInfo::kAudioTrack); } bool MediaDecoderStateMachine::HasLowUndecodedData() @@ -2195,6 +2188,7 @@ MediaDecoderStateMachine::FinishShutdown() // Prevent dangling pointers by disconnecting the listeners. mAudioQueueListener.Disconnect(); mVideoQueueListener.Disconnect(); + mMetadataManager.Disconnect(); // Disconnect canonicals and mirrors before shutting down our task queue. mBuffered.DisconnectIfConnected(); @@ -2397,8 +2391,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine() mSentPlaybackEndedEvent = true; - // Stop audio sink after call to AudioEndTime() above, otherwise it will - // return an incorrect value due to a null mAudioSink. + // MediaSink::GetEndTime() must be called before stopping playback. StopAudioSink(); StopDecodedStream(); } @@ -2561,8 +2554,8 @@ MediaDecoderStateMachine::GetAudioClock() const AssertCurrentThreadInMonitor(); MOZ_ASSERT(HasAudio() && !mAudioCompleted && IsPlaying()); // Since this function is called while we are playing and AudioSink is - // created once playback starts, mAudioSink is guaranteed to be non-null. - MOZ_ASSERT(mAudioSink); + // started once playback starts, IsStarted() is guaranteed to be true. + MOZ_ASSERT(mAudioSink->IsStarted()); return mAudioSink->GetPosition(); } @@ -2631,7 +2624,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() } if (mAudioCaptured) { - SendStreamData(); + DiscardStreamData(); } TimeStamp nowTime; @@ -2905,9 +2898,7 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp) AssertCurrentThreadInMonitor(); mPlayStartTime = aTimeStamp; - if (mAudioSink) { - mAudioSink->SetPlaying(!mPlayStartTime.IsNull()); - } + mAudioSink->SetPlaying(!mPlayStartTime.IsNull()); // Have DecodedStream remember the playing state so it doesn't need to // ask MDSM about IsPlaying(). Note we have to do this even before capture // happens since capture could happen in the middle of playback. @@ -3000,9 +2991,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged() } mPlaybackRate = mLogicalPlaybackRate; - if (mAudioSink) { - mAudioSink->SetPlaybackRate(mPlaybackRate); - } + mAudioSink->SetPlaybackRate(mPlaybackRate); ScheduleStateMachine(); } @@ -3011,10 +3000,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - - if (mAudioSink) { - mAudioSink->SetPreservesPitch(mPreservesPitch); - } + mAudioSink->SetPreservesPitch(mPreservesPitch); } bool MediaDecoderStateMachine::IsShutdown() @@ -3023,33 +3009,17 @@ bool MediaDecoderStateMachine::IsShutdown() return mIsShutdown; } -void MediaDecoderStateMachine::QueueMetadata(const TimeUnit& aPublishTime, - nsAutoPtr aInfo, - nsAutoPtr aTags) -{ - MOZ_ASSERT(OnDecodeTaskQueue()); - AssertCurrentThreadInMonitor(); - TimedMetadata* metadata = new TimedMetadata; - metadata->mPublishTime = aPublishTime; - metadata->mInfo = aInfo.forget(); - metadata->mTags = aTags.forget(); - mMetadataManager.QueueMetadata(metadata); -} - int64_t MediaDecoderStateMachine::AudioEndTime() const { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (mAudioSink) { - return mAudioSink->GetEndTime(); + if (mAudioSink->IsStarted()) { + return mAudioSink->GetEndTime(TrackInfo::kAudioTrack); } else if (mAudioCaptured) { return mDecodedStream->AudioEndTime(); } - // Don't call this after mAudioSink becomes null since we can't distinguish - // "before StartAudioSink" and "after StopAudioSink" where mAudioSink - // is null in both cases. - MOZ_ASSERT(!mAudioCompleted); + MOZ_ASSERT(!HasAudio()); return -1; } @@ -3166,7 +3136,7 @@ void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream, { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("AddOutputStream aStream=%p!", aStream); - mDecodedStream->Connect(aStream, aFinishWhenEnded); + mDecodedStream->AddOutput(aStream, aFinishWhenEnded); DispatchAudioCaptured(); } @@ -3174,7 +3144,7 @@ void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream) { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("RemoveOutputStream=%p!", aStream); - mDecodedStream->Remove(aStream); + mDecodedStream->RemoveOutput(aStream); if (!mDecodedStream->HasConsumers()) { DispatchAudioUncaptured(); } diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index f3ca4813f9..5541caa7b7 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -100,7 +100,7 @@ hardware (via AudioStream). namespace mozilla { namespace media { -class AudioSink; +class MediaSink; } class AudioSegment; @@ -157,6 +157,10 @@ public: // Set/Unset dormant state. void SetDormant(bool aDormant); + TimedMetadataEventSource& TimedMetadataEvent() { + return mMetadataManager.TimedMetadataEvent(); + } + private: // Initialization that needs to happen on the task queue. This is the first // task that gets run on the task queue, and is dispatched from the MDSM @@ -319,15 +323,12 @@ public: if (mReader) { mReader->BreakCycles(); } - mDecodedStream->DestroyData(); mResource = nullptr; mDecoder = nullptr; } - // Copy queued audio/video data in the reader to any output MediaStreams that - // need it. - void SendStreamData(); - void FinishStreamData(); + // Discard audio/video data that are already played by MSG. + void DiscardStreamData(); bool HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs); bool HaveEnoughDecodedVideo(); @@ -335,10 +336,6 @@ public: // shutting down. The decoder monitor must be held while calling this. bool IsShutdown(); - void QueueMetadata(const media::TimeUnit& aPublishTime, - nsAutoPtr aInfo, - nsAutoPtr aTags); - // Returns true if we're currently playing. The decoder monitor must // be held. bool IsPlaying() const; @@ -1001,8 +998,8 @@ private: // Media Fragment end time in microseconds. Access controlled by decoder monitor. int64_t mFragmentEndTime; - // The audio sink resource. Used on state machine and audio threads. - RefPtr mAudioSink; + // The audio sink resource. Used on the state machine thread. + nsRefPtr mAudioSink; // The reader, don't call its methods with the decoder monitor held. // This is created in the state machine's constructor. diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 43903b5540..87b711fe0e 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -139,17 +139,6 @@ MediaFormatReader::Shutdown() mDemuxer = nullptr; - // shutdown main thread demuxer and track demuxers. - if (mAudioTrackDemuxer) { - mAudioTrackDemuxer->BreakCycles(); - mAudioTrackDemuxer = nullptr; - } - if (mVideoTrackDemuxer) { - mVideoTrackDemuxer->BreakCycles(); - mVideoTrackDemuxer = nullptr; - } - mMainThreadDemuxer = nullptr; - mPlatform = nullptr; return MediaDecoderReader::Shutdown(); @@ -293,32 +282,10 @@ MediaFormatReader::OnDemuxerInitDone(nsresult) mSeekable = mDemuxer->IsSeekable(); - // Create demuxer object for main thread. - if (mDemuxer->IsThreadSafe()) { - mMainThreadDemuxer = mDemuxer; - } else { - mMainThreadDemuxer = mDemuxer->Clone(); - } - if (!mMainThreadDemuxer) { - mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__); - NS_WARNING("Unable to clone current MediaDataDemuxer"); - return; - } - if (!videoActive && !audioActive) { mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__); return; } - if (videoActive) { - mVideoTrackDemuxer = - mMainThreadDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); - MOZ_ASSERT(mVideoTrackDemuxer); - } - if (audioActive) { - mAudioTrackDemuxer = - mMainThreadDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); - MOZ_ASSERT(mAudioTrackDemuxer); - } if (IsWaitingOnCDMResource()) { // Decoder can't be allocated before CDM resource is ready, so resolving the @@ -1552,15 +1519,6 @@ MediaFormatReader::NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) return; } - MOZ_ASSERT(mMainThreadDemuxer); - - // Queue a task to notify our main thread demuxer. - nsCOMPtr task = - NS_NewRunnableMethodWithArgs( - mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataArrived, - aLength, aOffset); - AbstractThread::MainThread()->Dispatch(task.forget()); - NotifyDemuxer(aLength, aOffset); } @@ -1573,14 +1531,6 @@ MediaFormatReader::NotifyDataRemoved() return; } - MOZ_ASSERT(mMainThreadDemuxer); - - // Queue a task to notify our main thread demuxer. - nsCOMPtr task = - NS_NewRunnableMethod( - mMainThreadDemuxer, &MediaDataDemuxer::NotifyDataRemoved); - AbstractThread::MainThread()->Dispatch(task.forget()); - NotifyDemuxer(0, 0); } diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 058ee7e150..b5810df80c 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -420,13 +420,6 @@ private: nsRefPtr mSharedDecoderManager; - // Main thread objects - // Those are only used to calculate our buffered range on the main thread. - // The cached buffered range is calculated one when required. - nsRefPtr mMainThreadDemuxer; - nsRefPtr mAudioTrackDemuxer; - nsRefPtr mVideoTrackDemuxer; - #if defined(READER_DORMANT_HEURISTIC) const bool mDormantEnabled; #endif diff --git a/dom/media/MediaMetadataManager.h b/dom/media/MediaMetadataManager.h index 3414f0a9cc..1f126c494c 100644 --- a/dom/media/MediaMetadataManager.h +++ b/dom/media/MediaMetadataManager.h @@ -7,19 +7,41 @@ #if !defined(MediaMetadataManager_h__) #define MediaMetadataManager_h__ +#include "mozilla/AbstractThread.h" #include "mozilla/LinkedList.h" #include "nsAutoPtr.h" #include "AbstractMediaDecoder.h" +#include "MediaEventSource.h" #include "TimeUnits.h" #include "VideoUtils.h" namespace mozilla { +class TimedMetadata; +typedef MediaEventProducer + TimedMetadataEventProducer; +typedef MediaEventSource + TimedMetadataEventSource; + // A struct that contains the metadata of a media, and the time at which those // metadata should start to be reported. class TimedMetadata : public LinkedListElement { public: + TimedMetadata(const media::TimeUnit& aPublishTime, + nsAutoPtr&& aTags, + nsAutoPtr&& aInfo) + : mPublishTime(aPublishTime) + , mTags(Move(aTags)) + , mInfo(Move(aInfo)) {} + + // Define our move constructor because we don't want to move the members of + // LinkedListElement to change the list. + TimedMetadata(TimedMetadata&& aOther) + : mPublishTime(aOther.mPublishTime) + , mTags(Move(aOther.mTags)) + , mInfo(Move(aOther.mInfo)) {} + // The time, in microseconds, at which those metadata should be available. media::TimeUnit mPublishTime; // The metadata. The ownership is transfered to the element when dispatching to @@ -33,8 +55,7 @@ public: // This class encapsulate the logic to give the metadata from the reader to // the content, at the right time. -class MediaMetadataManager -{ +class MediaMetadataManager { public: ~MediaMetadataManager() { TimedMetadata* element; @@ -43,30 +64,41 @@ public: } } - void QueueMetadata(TimedMetadata* aMetadata) { - mMetadataQueue.insertBack(aMetadata); + // Connect to an event source to receive TimedMetadata events. + void Connect(TimedMetadataEventSource& aEvent, AbstractThread* aThread) { + mListener = aEvent.Connect( + aThread, this, &MediaMetadataManager::OnMetadataQueued); } - void DispatchMetadataIfNeeded(AbstractMediaDecoder* aDecoder, const media::TimeUnit& aCurrentTime) { + // Stop receiving TimedMetadata events. + void Disconnect() { + mListener.Disconnect(); + } + + // Return an event source through which we will send TimedMetadata events + // when playback position reaches the publish time. + TimedMetadataEventSource& TimedMetadataEvent() { + return mTimedMetadataEvent; + } + + void DispatchMetadataIfNeeded(const media::TimeUnit& aCurrentTime) { TimedMetadata* metadata = mMetadataQueue.getFirst(); while (metadata && aCurrentTime >= metadata->mPublishTime) { - // Remove all media tracks from the list first. - nsCOMPtr removeTracksEvent = - new RemoveMediaTracksEventRunner(aDecoder); - NS_DispatchToMainThread(removeTracksEvent); - - nsCOMPtr metadataUpdatedEvent = - new MetadataUpdatedEventRunner(aDecoder, - metadata->mInfo, - metadata->mTags); - NS_DispatchToMainThread(metadataUpdatedEvent); + // Our listener will figure out what to do with TimedMetadata. + mTimedMetadataEvent.Notify(Move(*metadata)); delete mMetadataQueue.popFirst(); metadata = mMetadataQueue.getFirst(); } } protected: + void OnMetadataQueued(TimedMetadata&& aMetadata) { + mMetadataQueue.insertBack(new TimedMetadata(Move(aMetadata))); + } + LinkedList mMetadataQueue; + MediaEventListener mListener; + TimedMetadataEventProducer mTimedMetadataEvent; }; } // namespace mozilla diff --git a/dom/media/mediasink/AudioSink.h b/dom/media/mediasink/AudioSink.h index 49d4fa2cc5..2276a735e1 100644 --- a/dom/media/mediasink/AudioSink.h +++ b/dom/media/mediasink/AudioSink.h @@ -6,6 +6,7 @@ #if !defined(AudioSink_h__) #define AudioSink_h__ +#include "mozilla/MozPromise.h" #include "mozilla/nsRefPtr.h" #include "nsISupportsImpl.h" diff --git a/dom/media/mediasink/AudioSinkWrapper.cpp b/dom/media/mediasink/AudioSinkWrapper.cpp new file mode 100644 index 0000000000..db36dee8f1 --- /dev/null +++ b/dom/media/mediasink/AudioSinkWrapper.cpp @@ -0,0 +1,160 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AudioSink.h" +#include "AudioSinkWrapper.h" + +namespace mozilla { +namespace media { + +AudioSinkWrapper::~AudioSinkWrapper() +{ +} + +void +AudioSinkWrapper::Shutdown() +{ + AssertOwnerThread(); + MOZ_ASSERT(!mIsStarted, "Must be called after playback stopped."); + mCreator = nullptr; +} + +const MediaSink::PlaybackParams& +AudioSinkWrapper::GetPlaybackParams() const +{ + AssertOwnerThread(); + return mParams; +} + +void +AudioSinkWrapper::SetPlaybackParams(const PlaybackParams& aParams) +{ + AssertOwnerThread(); + if (mAudioSink) { + mAudioSink->SetVolume(aParams.volume); + mAudioSink->SetPlaybackRate(aParams.playbackRate); + mAudioSink->SetPreservesPitch(aParams.preservesPitch); + } + mParams = aParams; +} + +nsRefPtr +AudioSinkWrapper::OnEnded(TrackType aType) +{ + AssertOwnerThread(); + MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); + if (aType == TrackInfo::kAudioTrack) { + return mEndPromise; + } + return nullptr; +} + +int64_t +AudioSinkWrapper::GetEndTime(TrackType aType) const +{ + AssertOwnerThread(); + MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); + if (aType == TrackInfo::kAudioTrack && mAudioSink) { + return mAudioSink->GetEndTime(); + } + return -1; +} + +int64_t +AudioSinkWrapper::GetPosition() const +{ + AssertOwnerThread(); + MOZ_ASSERT(mIsStarted, "Must be called after playback starts."); + return mAudioSink->GetPosition(); +} + +bool +AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const +{ + AssertOwnerThread(); + return mAudioSink ? mAudioSink->HasUnplayedFrames() : false; +} + +void +AudioSinkWrapper::SetVolume(double aVolume) +{ + AssertOwnerThread(); + mParams.volume = aVolume; + if (mAudioSink) { + mAudioSink->SetVolume(aVolume); + } +} + +void +AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate) +{ + AssertOwnerThread(); + mParams.playbackRate = aPlaybackRate; + if (mAudioSink) { + mAudioSink->SetPlaybackRate(aPlaybackRate); + } +} + +void +AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch) +{ + AssertOwnerThread(); + mParams.preservesPitch = aPreservesPitch; + if (mAudioSink) { + mAudioSink->SetPreservesPitch(aPreservesPitch); + } +} + +void +AudioSinkWrapper::SetPlaying(bool aPlaying) +{ + AssertOwnerThread(); + + // Resume/pause matters only when playback started. + if (!mIsStarted) { + return; + } + + if (mAudioSink) { + mAudioSink->SetPlaying(aPlaying); + } +} + +void +AudioSinkWrapper::Start(int64_t aStartTime, const MediaInfo& aInfo) +{ + AssertOwnerThread(); + MOZ_ASSERT(!mIsStarted, "playback already started."); + + mIsStarted = true; + + mAudioSink = mCreator->Create(); + mEndPromise = mAudioSink->Init(); + SetPlaybackParams(mParams); +} + +void +AudioSinkWrapper::Stop() +{ + AssertOwnerThread(); + MOZ_ASSERT(mIsStarted, "playback not started."); + + mIsStarted = false; + mAudioSink->Shutdown(); + mAudioSink = nullptr; + mEndPromise = nullptr; +} + +bool +AudioSinkWrapper::IsStarted() const +{ + AssertOwnerThread(); + return mIsStarted; +} + +} // namespace media +} // namespace mozilla + diff --git a/dom/media/mediasink/AudioSinkWrapper.h b/dom/media/mediasink/AudioSinkWrapper.h new file mode 100644 index 0000000000..34e8e9410f --- /dev/null +++ b/dom/media/mediasink/AudioSinkWrapper.h @@ -0,0 +1,93 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef AudioSinkWrapper_h_ +#define AudioSinkWrapper_h_ + +#include "mozilla/AbstractThread.h" +#include "mozilla/dom/AudioChannelBinding.h" +#include "mozilla/nsRefPtr.h" +#include "mozilla/UniquePtr.h" + +#include "MediaSink.h" + +namespace mozilla { + +class MediaData; +template class MediaQueue; + +namespace media { + +class AudioSink; + +/** + * A wrapper around AudioSink to provide the interface of MediaSink. + */ +class AudioSinkWrapper : public MediaSink { + // An AudioSink factory. + class Creator { + public: + virtual ~Creator() {} + virtual AudioSink* Create() = 0; + }; + + // Wrap around a function object which creates AudioSinks. + template + class CreatorImpl : public Creator { + public: + explicit CreatorImpl(const Function& aFunc) : mFunction(aFunc) {} + AudioSink* Create() override { return mFunction(); } + private: + Function mFunction; + }; + +public: + template + AudioSinkWrapper(AbstractThread* aOwnerThread, const Function& aFunc) + : mOwnerThread(aOwnerThread) + , mCreator(new CreatorImpl(aFunc)) + , mIsStarted(false) + {} + + const PlaybackParams& GetPlaybackParams() const override; + void SetPlaybackParams(const PlaybackParams& aParams) override; + + nsRefPtr OnEnded(TrackType aType) override; + int64_t GetEndTime(TrackType aType) const override; + int64_t GetPosition() const override; + bool HasUnplayedFrames(TrackType aType) const override; + + void SetVolume(double aVolume) override; + void SetPlaybackRate(double aPlaybackRate) override; + void SetPreservesPitch(bool aPreservesPitch) override; + void SetPlaying(bool aPlaying) override; + + void Start(int64_t aStartTime, const MediaInfo& aInfo) override; + void Stop() override; + bool IsStarted() const override; + + void Shutdown() override; + +private: + virtual ~AudioSinkWrapper(); + + void AssertOwnerThread() const { + MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn()); + } + + const nsRefPtr mOwnerThread; + UniquePtr mCreator; + nsRefPtr mAudioSink; + nsRefPtr mEndPromise; + + bool mIsStarted; + PlaybackParams mParams; +}; + +} // namespace media +} // namespace mozilla + +#endif //AudioSinkWrapper_h_ diff --git a/dom/media/mediasink/DecodedAudioDataSink.cpp b/dom/media/mediasink/DecodedAudioDataSink.cpp index 75dd113ace..793ab6eaa3 100644 --- a/dom/media/mediasink/DecodedAudioDataSink.cpp +++ b/dom/media/mediasink/DecodedAudioDataSink.cpp @@ -45,6 +45,10 @@ DecodedAudioDataSink::DecodedAudioDataSink(MediaQueue& aAudioQueue, { } +DecodedAudioDataSink::~DecodedAudioDataSink() +{ +} + void DecodedAudioDataSink::SetState(State aState) { diff --git a/dom/media/mediasink/DecodedAudioDataSink.h b/dom/media/mediasink/DecodedAudioDataSink.h index ea3a2ec50a..ec8144809d 100644 --- a/dom/media/mediasink/DecodedAudioDataSink.h +++ b/dom/media/mediasink/DecodedAudioDataSink.h @@ -7,6 +7,7 @@ #define DecodedAudioDataSink_h__ #include "AudioSink.h" +#include "MediaEventSource.h" #include "MediaInfo.h" #include "mozilla/RefPtr.h" #include "nsISupportsImpl.h" @@ -18,6 +19,9 @@ #include "mozilla/ReentrantMonitor.h" namespace mozilla { + +class AudioStream; + namespace media { class DecodedAudioDataSink : public AudioSink { @@ -59,7 +63,7 @@ private: AUDIOSINK_STATE_ERROR }; - virtual ~DecodedAudioDataSink() {} + virtual ~DecodedAudioDataSink(); void DispatchTask(already_AddRefed&& event); void SetState(State aState); diff --git a/dom/media/mediasink/MediaSink.h b/dom/media/mediasink/MediaSink.h new file mode 100644 index 0000000000..73e8bf44ec --- /dev/null +++ b/dom/media/mediasink/MediaSink.h @@ -0,0 +1,115 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MediaSink_h_ +#define MediaSink_h_ + +#include "mozilla/nsRefPtr.h" +#include "mozilla/MozPromise.h" +#include "nsISupportsImpl.h" +#include "MediaInfo.h" + +namespace mozilla { +namespace media { + +/** + * A consumer of audio/video data which plays audio and video tracks and + * manages A/V sync between them. + * + * A typical sink sends audio/video outputs to the speaker and screen. + * However, there are also sinks which capture the output of an media element + * and send the output to a MediaStream. + * + * This class is used to move A/V sync management and audio/video rendering + * out of MDSM so it is possible for subclasses to do external rendering using + * specific hardware which is required by TV projects and CDM. + * + * Note this class is not thread-safe and should be called from the state + * machine thread only. + */ +class MediaSink { +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSink); + typedef mozilla::TrackInfo::TrackType TrackType; + + struct PlaybackParams { + PlaybackParams() + : volume(1.0) , playbackRate(1.0) , preservesPitch(true) {} + double volume; + double playbackRate; + bool preservesPitch; + }; + + // Return the playback parameters of this sink. + // Can be called in any state. + virtual const PlaybackParams& GetPlaybackParams() const = 0; + + // Set the playback parameters of this sink. + // Can be called in any state. + virtual void SetPlaybackParams(const PlaybackParams& aParams) = 0; + + // Return a promise which is resolved when the track finishes + // or null if no such track. + // Must be called after playback starts. + virtual nsRefPtr OnEnded(TrackType aType) = 0; + + // Return the end time of the audio/video data that has been consumed + // or -1 if no such track. + // Must be called after playback starts. + virtual int64_t GetEndTime(TrackType aType) const = 0; + + // Return playback position of the media. + // Since A/V sync is always maintained by this sink, there is no need to + // specify whether we want to get audio or video position. + // Must be called after playback starts. + virtual int64_t GetPosition() const = 0; + + // Return true if there are data consumed but not played yet. + // Can be called in any state. + virtual bool HasUnplayedFrames(TrackType aType) const = 0; + + // Set volume of the audio track. + // Do nothing if this sink has no audio track. + // Can be called in any state. + virtual void SetVolume(double aVolume) {} + + // Set the playback rate. + // Can be called in any state. + virtual void SetPlaybackRate(double aPlaybackRate) {} + + // Whether to preserve pitch of the audio track. + // Do nothing if this sink has no audio track. + // Can be called in any state. + virtual void SetPreservesPitch(bool aPreservesPitch) {} + + // Pause/resume the playback. Only work after playback starts. + virtual void SetPlaying(bool aPlaying) = 0; + + // Begin a playback session with the provided start time and media info. + // Must be called when playback is stopped. + virtual void Start(int64_t aStartTime, const MediaInfo& aInfo) = 0; + + // Finish a playback session. + // Must be called after playback starts. + virtual void Stop() = 0; + + // Return true if playback has started. + // Can be called in any state. + virtual bool IsStarted() const = 0; + + // Called on the state machine thread to shut down the sink. All resources + // allocated by this sink should be released. + // Must be called after playback stopped. + virtual void Shutdown() {} + +protected: + virtual ~MediaSink() {} +}; + +} // namespace media +} // namespace mozilla + +#endif //MediaSink_h_ diff --git a/dom/media/mediasink/moz.build b/dom/media/mediasink/moz.build index 6fd1f5d06d..2f3c5de119 100644 --- a/dom/media/mediasink/moz.build +++ b/dom/media/mediasink/moz.build @@ -5,6 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. UNIFIED_SOURCES += [ + 'AudioSinkWrapper.cpp', 'DecodedAudioDataSink.cpp', ] diff --git a/dom/media/ogg/OggReader.cpp b/dom/media/ogg/OggReader.cpp index 045fe42b21..af374765bb 100644 --- a/dom/media/ogg/OggReader.cpp +++ b/dom/media/ogg/OggReader.cpp @@ -814,10 +814,11 @@ bool OggReader::ReadOggChain() if (chained) { SetChained(true); { - nsAutoPtr info(new MediaInfo(mInfo)); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); auto t = mDecodedAudioFrames * USECS_PER_S / mInfo.mAudio.mRate; - mDecoder->QueueMetadata(media::TimeUnit::FromMicroseconds(t), info, tags); + mTimedMetadataEvent.Notify( + TimedMetadata(media::TimeUnit::FromMicroseconds(t), + Move(tags), + nsAutoPtr(new MediaInfo(mInfo)))); } return true; } diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp index 160acb9cfd..a89d701067 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.cpp +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -29,7 +29,6 @@ VPXDecoder::VPXDecoder(const VideoInfo& aConfig, : mImageContainer(aImageContainer) , mTaskQueue(aTaskQueue) , mCallback(aCallback) - , mIter(nullptr) , mInfo(aConfig) { MOZ_COUNT_CTOR(VPXDecoder); @@ -74,7 +73,6 @@ nsresult VPXDecoder::Flush() { mTaskQueue->Flush(); - mIter = nullptr; return NS_OK; } @@ -99,9 +97,10 @@ VPXDecoder::DoDecodeFrame(MediaRawData* aSample) return -1; } + vpx_codec_iter_t iter = nullptr; vpx_image_t *img; - if ((img = vpx_codec_get_frame(&mVPX, &mIter))) { + while ((img = vpx_codec_get_frame(&mVPX, &iter))) { NS_ASSERTION(img->fmt == VPX_IMG_FMT_I420, "WebM image format not I420"); // Chroma shifts are rounded down as per the decoding examples in the SDK @@ -143,9 +142,7 @@ VPXDecoder::DoDecodeFrame(MediaRawData* aSample) return -1; } mCallback->Output(v); - return 1; } - mIter = nullptr; return 0; } diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h index 5421834696..a0688f4c0d 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.h +++ b/dom/media/platforms/agnostic/VPXDecoder.h @@ -54,7 +54,6 @@ private: // VPx decoder state vpx_codec_ctx_t mVPX; - vpx_codec_iter_t mIter; const VideoInfo& mInfo; diff --git a/dom/media/webaudio/BufferDecoder.cpp b/dom/media/webaudio/BufferDecoder.cpp index 8018451929..88d8d7dcf5 100644 --- a/dom/media/webaudio/BufferDecoder.cpp +++ b/dom/media/webaudio/BufferDecoder.cpp @@ -129,18 +129,6 @@ BufferDecoder::FirstFrameLoaded(nsAutoPtr aInfo, MediaDecoderEventVis // ignore } -void -BufferDecoder::QueueMetadata(const media::TimeUnit& aTime, nsAutoPtr aInfo, nsAutoPtr aTags) -{ - // ignore -} - -void -BufferDecoder::RemoveMediaTracks() -{ - // ignore -} - void BufferDecoder::OnReadMetadataCompleted() { diff --git a/dom/media/webaudio/BufferDecoder.h b/dom/media/webaudio/BufferDecoder.h index 916f6853d6..736b49169c 100644 --- a/dom/media/webaudio/BufferDecoder.h +++ b/dom/media/webaudio/BufferDecoder.h @@ -58,12 +58,9 @@ public: virtual void MetadataLoaded(nsAutoPtr aInfo, nsAutoPtr aTags, MediaDecoderEventVisibility aEventVisibility) final override; - virtual void QueueMetadata(const media::TimeUnit& aTime, nsAutoPtr aInfo, nsAutoPtr aTags) final override; virtual void FirstFrameLoaded(nsAutoPtr aInfo, MediaDecoderEventVisibility aEventVisibility) final override; - virtual void RemoveMediaTracks() final override; - virtual void OnReadMetadataCompleted() final override; virtual MediaDecoderOwner* GetOwner() final override; diff --git a/dom/media/webm/WebMDemuxer.cpp b/dom/media/webm/WebMDemuxer.cpp index 338bdf3da5..93338cbf69 100644 --- a/dom/media/webm/WebMDemuxer.cpp +++ b/dom/media/webm/WebMDemuxer.cpp @@ -18,6 +18,7 @@ #include "NesteggPacketHolder.h" #include +#include #define VPX_DONT_DEFINE_STDINT_TYPES #include "vpx/vp8dx.h" @@ -33,32 +34,41 @@ extern PRLogModuleInfo* gMediaDecoderLog; extern PRLogModuleInfo* gNesteggLog; // Functions for reading and seeking using WebMDemuxer required for -// nestegg_io. The 'user data' passed to these functions is the demuxer +// nestegg_io. The 'user data' passed to these functions is the +// demuxer's MediaResourceIndex static int webmdemux_read(void* aBuffer, size_t aLength, void* aUserData) { MOZ_ASSERT(aUserData); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); + MediaResourceIndex* resource = + reinterpret_cast(aUserData); + int64_t length = resource->GetLength(); + MOZ_ASSERT(aLength < UINT32_MAX); + uint32_t count = aLength; + if (length >= 0 && count + resource->Tell() > length) { + count = uint32_t(length - resource->Tell()); + } + uint32_t bytes = 0; - bool eof = false; - char* p = static_cast(aBuffer); - nsresult rv = demuxer->Read(p, aLength, &bytes); - eof = bytes < aLength; + nsresult rv = resource->Read(static_cast(aBuffer), count, &bytes); + bool eof = !bytes; return NS_FAILED(rv) ? -1 : eof ? 0 : 1; } static int webmdemux_seek(int64_t aOffset, int aWhence, void* aUserData) { MOZ_ASSERT(aUserData); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); - nsresult rv = demuxer->Seek(aWhence, aOffset); + MediaResourceIndex* resource = + reinterpret_cast(aUserData); + nsresult rv = resource->Seek(aWhence, aOffset); return NS_SUCCEEDED(rv) ? 0 : -1; } static int64_t webmdemux_tell(void* aUserData) { MOZ_ASSERT(aUserData); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); - return demuxer->Tell(); + MediaResourceIndex* resource = + reinterpret_cast(aUserData); + return resource->Tell(); } static void webmdemux_log(nestegg* aContext, @@ -109,7 +119,6 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource) , mBufferedState(nullptr) , mInitData(nullptr) , mContext(nullptr) - , mOffset(0) , mVideoTrack(0) , mAudioTrack(0) , mSeekPreroll(0) @@ -162,7 +171,7 @@ WebMDemuxer::InitBufferedState() already_AddRefed WebMDemuxer::Clone() const { - nsRefPtr demuxer = new WebMDemuxer(mResource); + nsRefPtr demuxer = new WebMDemuxer(mResource.GetResource()); demuxer->mInitData = mInitData; if (demuxer->InitBufferedState() != NS_OK || demuxer->ReadMetadata() != NS_OK) { @@ -243,10 +252,10 @@ WebMDemuxer::ReadMetadata() io.read = webmdemux_read; io.seek = webmdemux_seek; io.tell = webmdemux_tell; - io.userdata = this; + io.userdata = &mResource; int64_t maxOffset = mBufferedState->GetInitEndOffset(); if (maxOffset == -1) { - maxOffset = mResource->GetLength(); + maxOffset = mResource.GetLength(); } int r = nestegg_init(&mContext, io, &webmdemux_log, maxOffset); if (r == -1) { @@ -394,50 +403,6 @@ WebMDemuxer::ReadMetadata() return NS_OK; } -nsresult -WebMDemuxer::Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) -{ - int64_t length = mResource->GetLength(); - if (length >= 0 && aCount + mOffset > length) { - WEBM_DEBUG("requested to large amount, trying to get %ld bytes at %ld (length: %ld)", aCount, mOffset, length); - aCount = length - mOffset; - WEBM_DEBUG("will only return %ld", aCount); - } - nsRefPtr bytes = mResource->MediaReadAt(mOffset, aCount); - if (!bytes) { - return NS_ERROR_FAILURE; - } - mOffset += bytes->Length(); - *aBytes = bytes->Length(); - memcpy(aBuffer, bytes->Elements(), bytes->Length()); - return NS_OK; -} - -nsresult -WebMDemuxer::Seek(int32_t aWhence, int64_t aOffset) -{ - if (aWhence == SEEK_CUR) { - aOffset += mOffset; - } else if (aWhence == SEEK_END) { - int64_t length = mResource->GetLength(); - if (length == -1 || length - aOffset < 0) { - return NS_ERROR_FAILURE; - } - aOffset = mResource->GetLength() - aOffset; - } - if (aOffset > mResource->GetLength()) { - return NS_ERROR_FAILURE; - } - mOffset = aOffset; - return NS_OK; -} - -int64_t -WebMDemuxer::Tell() -{ - return mOffset; -} - bool WebMDemuxer::IsSeekable() const { @@ -450,18 +415,18 @@ WebMDemuxer::EnsureUpToDateIndex() if (!mNeedReIndex) { return; } - AutoPinned resource(mResource); - nsTArray byteRanges; - nsresult rv = resource->GetCachedRanges(byteRanges); - if (NS_FAILED(rv)) { - return; - } if (mInitData && mBufferedState->GetInitEndOffset() == -1) { mBufferedState->NotifyDataArrived(mInitData->Elements(), mInitData->Length(), 0); } - mBufferedState->UpdateIndex(byteRanges, mResource); + AutoPinned resource(mResource.GetResource()); + nsTArray byteRanges; + nsresult rv = resource->GetCachedRanges(byteRanges); + if (NS_FAILED(rv) || !byteRanges.Length()) { + return; + } + mBufferedState->UpdateIndex(byteRanges, resource); if (!mInitData && mBufferedState->GetInitEndOffset() != -1) { - mInitData = mResource->MediaReadAt(0, mBufferedState->GetInitEndOffset()); + mInitData = mResource.MediaReadAt(0, mBufferedState->GetInitEndOffset()); } mNeedReIndex = false; } @@ -642,7 +607,7 @@ WebMDemuxer::DemuxPacket() return nullptr; } - int64_t offset = Tell(); + int64_t offset = mResource.Tell(); nsRefPtr holder = new NesteggPacketHolder(); if (!holder->Init(packet, offset, track, false)) { return nullptr; @@ -731,7 +696,7 @@ media::TimeIntervals WebMDemuxer::GetBuffered() { EnsureUpToDateIndex(); - AutoPinned resource(mResource); + AutoPinned resource(mResource.GetResource()); media::TimeIntervals buffered; diff --git a/dom/media/webm/WebMDemuxer.h b/dom/media/webm/WebMDemuxer.h index f464984dfd..a7ba0dd3f6 100644 --- a/dom/media/webm/WebMDemuxer.h +++ b/dom/media/webm/WebMDemuxer.h @@ -84,10 +84,6 @@ public: // Pushes a packet to the front of the video packet queue. virtual void PushVideoPacket(NesteggPacketHolder* aItem); - nsresult Read(char* aBuffer, uint32_t aCount, uint32_t * aBytes); - nsresult Seek(int32_t aWhence, int64_t aOffset); - int64_t Tell(); - private: friend class WebMTrackDemuxer; @@ -112,7 +108,7 @@ private: // is responsible for making sure it doesn't get lost. nsRefPtr DemuxPacket(); - nsRefPtr mResource; + MediaResourceIndex mResource; MediaInfo mInfo; nsTArray> mDemuxers; @@ -125,7 +121,6 @@ private: // Access on reader's thread for main demuxer, // or main thread for cloned demuxer nestegg* mContext; - int64_t mOffset; // Queue of video and audio packets that have been read but not decoded. WebMPacketQueue mVideoPackets;