diff --git a/dom/base/nsDOMWindowUtils.cpp b/dom/base/nsDOMWindowUtils.cpp index 6fa59eb8f6..5ad5756b22 100644 --- a/dom/base/nsDOMWindowUtils.cpp +++ b/dom/base/nsDOMWindowUtils.cpp @@ -28,7 +28,7 @@ #include "ClientLayerManager.h" #include "nsQueryObject.h" #ifdef MOZ_FMP4 -#include "MP4Reader.h" +#include "MP4Decoder.h" #endif #include "nsIScrollableFrame.h" @@ -2264,7 +2264,7 @@ nsDOMWindowUtils::GetSupportsHardwareH264Decoding(bool* retval) if (!mgr) return NS_ERROR_FAILURE; - *retval = MP4Reader::IsVideoAccelerated(mgr->GetCompositorBackendType()); + *retval = MP4Decoder::IsVideoAccelerated(mgr->GetCompositorBackendType()); #else *retval = false; #endif diff --git a/dom/media/AudioSink.cpp b/dom/media/AudioSink.cpp index 34ba2e1036..2e2666f18c 100644 --- a/dom/media/AudioSink.cpp +++ b/dom/media/AudioSink.cpp @@ -165,18 +165,10 @@ AudioSink::SetPreservesPitch(bool aPreservesPitch) } void -AudioSink::StartPlayback() +AudioSink::SetPlaying(bool aPlaying) { AssertCurrentThreadInMonitor(); - mPlaying = true; - GetReentrantMonitor().NotifyAll(); -} - -void -AudioSink::StopPlayback() -{ - AssertCurrentThreadInMonitor(); - mPlaying = false; + mPlaying = aPlaying; GetReentrantMonitor().NotifyAll(); } diff --git a/dom/media/AudioSink.h b/dom/media/AudioSink.h index 3e1f1a9f32..1c7723d56f 100644 --- a/dom/media/AudioSink.h +++ b/dom/media/AudioSink.h @@ -43,8 +43,7 @@ public: void SetPlaybackRate(double aPlaybackRate); void SetPreservesPitch(bool aPreservesPitch); - void StartPlayback(); - void StopPlayback(); + void SetPlaying(bool aPlaying); private: ~AudioSink() {} diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 1f012f8fcd..40bdee8bd3 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -678,7 +678,9 @@ DOMHwMediaStream::DOMHwMediaStream() mImageContainer = LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS_OVERLAY); nsRefPtr img = mImageContainer->CreateImage(ImageFormat::OVERLAY_IMAGE); mOverlayImage = static_cast(img.get()); - mImageContainer->SetCurrentImage(mOverlayImage); + nsAutoTArray images; + images.AppendElement(ImageContainer::NonOwningImage(img)); + mImageContainer->SetCurrentImages(images); #endif } diff --git a/dom/media/DecodedStream.cpp b/dom/media/DecodedStream.cpp index 3beffaa89b..e7c89fb0e3 100644 --- a/dom/media/DecodedStream.cpp +++ b/dom/media/DecodedStream.cpp @@ -6,7 +6,13 @@ #include "DecodedStream.h" #include "MediaStreamGraph.h" -#include "mozilla/ReentrantMonitor.h" +#include "AudioSegment.h" +#include "VideoSegment.h" +#include "MediaQueue.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "SharedBuffer.h" +#include "VideoUtils.h" namespace mozilla { @@ -69,7 +75,20 @@ private: bool mStreamFinishedOnMainThread; }; -DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream) +static void +UpdateStreamBlocking(MediaStream* aStream, bool aBlocking) +{ + int32_t delta = aBlocking ? 1 : -1; + if (NS_IsMainThread()) { + aStream->ChangeExplicitBlockerCount(delta); + } else { + nsCOMPtr r = NS_NewRunnableMethodWithArg( + aStream, &MediaStream::ChangeExplicitBlockerCount, delta); + AbstractThread::MainThread()->Dispatch(r.forget()); + } +} + +DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream, bool aPlaying) : mAudioFramesWritten(0) , mNextVideoTime(-1) , mNextAudioTime(-1) @@ -78,13 +97,16 @@ DecodedStreamData::DecodedStreamData(SourceMediaStream* aStream) , mHaveSentFinishAudio(false) , mHaveSentFinishVideo(false) , mStream(aStream) - , mHaveBlockedForPlayState(false) - , mHaveBlockedForStateMachineNotPlaying(false) + , mPlaying(aPlaying) + , mEOSVideoCompensation(false) { mListener = new DecodedStreamGraphListener(mStream); mStream->AddListener(mListener); - // Block the stream until the initialization is done. - mStream->ChangeExplicitBlockerCount(1); + + // Block the stream if we are not playing. + if (!aPlaying) { + UpdateStreamBlocking(mStream, true); + } } DecodedStreamData::~DecodedStreamData() @@ -105,6 +127,15 @@ DecodedStreamData::GetPosition() const return mListener->GetLastOutputTime(); } +void +DecodedStreamData::SetPlaying(bool aPlaying) +{ + if (mPlaying != aPlaying) { + mPlaying = aPlaying; + UpdateStreamBlocking(mStream, !mPlaying); + } +} + class OutputStreamListener : public MediaStreamListener { typedef MediaStreamListener::MediaStreamGraphEvent MediaStreamGraphEvent; public: @@ -130,26 +161,9 @@ private: void DoNotifyFinished() { MOZ_ASSERT(NS_IsMainThread()); - if (!mDecodedStream) { - return; - } - - // Remove the finished stream so it won't block the decoded stream. - ReentrantMonitorAutoEnter mon(mDecodedStream->GetReentrantMonitor()); - auto& streams = mDecodedStream->OutputStreams(); - // Don't read |mDecodedStream| in the loop since removing the element will lead - // to ~OutputStreamData() which will call Forget() to reset |mDecodedStream|. - for (int32_t i = streams.Length() - 1; i >= 0; --i) { - auto& os = streams[i]; - MediaStream* p = os.mStream.get(); - if (p == mStream.get()) { - if (os.mPort) { - os.mPort->Destroy(); - os.mPort = nullptr; - } - streams.RemoveElementAt(i); - break; - } + if (mDecodedStream) { + // Remove the finished stream so it won't block the decoded stream. + mDecodedStream->Remove(mStream); } } @@ -171,24 +185,18 @@ OutputStreamData::Init(DecodedStream* aDecodedStream, ProcessedMediaStream* aStr aStream->AddListener(mListener); } -DecodedStream::DecodedStream(ReentrantMonitor& aMonitor) - : mMonitor(aMonitor) +DecodedStream::DecodedStream() + : mMonitor("DecodedStream::mMonitor") + , mPlaying(false) { // } -DecodedStreamData* -DecodedStream::GetData() const -{ - GetReentrantMonitor().AssertCurrentThreadIn(); - return mData.get(); -} - void DecodedStream::DestroyData() { MOZ_ASSERT(NS_IsMainThread()); - GetReentrantMonitor().AssertCurrentThreadIn(); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); // Avoid the redundant blocking to output stream. if (!mData) { @@ -219,17 +227,27 @@ DecodedStream::DestroyData() mData = nullptr; } +void +DecodedStream::RecreateData() +{ + nsRefPtr self = this; + nsCOMPtr r = NS_NewRunnableFunction([self] () -> void { + self->RecreateData(nullptr); + }); + AbstractThread::MainThread()->Dispatch(r.forget()); +} + void DecodedStream::RecreateData(MediaStreamGraph* aGraph) { MOZ_ASSERT(NS_IsMainThread()); - GetReentrantMonitor().AssertCurrentThreadIn(); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); MOZ_ASSERT((aGraph && !mData && OutputStreams().IsEmpty()) || // first time (!aGraph && mData)); // 2nd time and later auto source = aGraph->CreateSourceStream(nullptr); DestroyData(); - mData.reset(new DecodedStreamData(source)); + mData.reset(new DecodedStreamData(source, mPlaying)); // Note that the delay between removing ports in DestroyDecodedStream // and adding new ones won't cause a glitch since all graph operations @@ -245,6 +263,7 @@ DecodedStream::RecreateData(MediaStreamGraph* aGraph) nsTArray& DecodedStream::OutputStreams() { + MOZ_ASSERT(NS_IsMainThread()); GetReentrantMonitor().AssertCurrentThreadIn(); return mOutputStreams; } @@ -275,7 +294,11 @@ void DecodedStream::Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); - GetReentrantMonitor().AssertCurrentThreadIn(); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + if (!mData) { + RecreateData(aStream->Graph()); + } OutputStreamData* os = OutputStreams().AppendElement(); os->Init(this, aStream); @@ -286,4 +309,362 @@ DecodedStream::Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded) } } +void +DecodedStream::Remove(MediaStream* aStream) +{ + MOZ_ASSERT(NS_IsMainThread()); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + auto& streams = OutputStreams(); + for (int32_t i = streams.Length() - 1; i >= 0; --i) { + auto& os = streams[i]; + MediaStream* p = os.mStream.get(); + if (p == aStream) { + if (os.mPort) { + os.mPort->Destroy(); + os.mPort = nullptr; + } + streams.RemoveElementAt(i); + break; + } + } +} + +void +DecodedStream::SetPlaying(bool aPlaying) +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + mPlaying = aPlaying; + if (mData) { + mData->SetPlaying(aPlaying); + } +} + +bool +DecodedStream::HaveEnoughAudio(const MediaInfo& aInfo) const +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + if (mData->mStreamInitialized && !mData->mHaveSentFinishAudio) { + MOZ_ASSERT(aInfo.HasAudio()); + TrackID audioTrackId = aInfo.mAudio.mTrackId; + if (!mData->mStream->HaveEnoughBuffered(audioTrackId)) { + return false; + } + } + return true; +} + +bool +DecodedStream::HaveEnoughVideo(const MediaInfo& aInfo) const +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + if (mData->mStreamInitialized && !mData->mHaveSentFinishVideo) { + MOZ_ASSERT(aInfo.HasVideo()); + TrackID videoTrackId = aInfo.mVideo.mTrackId; + if (!mData->mStream->HaveEnoughBuffered(videoTrackId)) { + return false; + } + } + return true; +} + +void +DecodedStream::InitTracks(int64_t aStartTime, const MediaInfo& aInfo) +{ + GetReentrantMonitor().AssertCurrentThreadIn(); + + if (mData->mStreamInitialized) { + return; + } + + SourceMediaStream* sourceStream = mData->mStream; + + if (aInfo.HasAudio()) { + TrackID audioTrackId = aInfo.mAudio.mTrackId; + AudioSegment* audio = new AudioSegment(); + sourceStream->AddAudioTrack(audioTrackId, aInfo.mAudio.mRate, 0, audio, + SourceMediaStream::ADDTRACK_QUEUED); + mData->mNextAudioTime = aStartTime; + } + + if (aInfo.HasVideo()) { + TrackID videoTrackId = aInfo.mVideo.mTrackId; + VideoSegment* video = new VideoSegment(); + sourceStream->AddTrack(videoTrackId, 0, video, + SourceMediaStream::ADDTRACK_QUEUED); + mData->mNextVideoTime = aStartTime; + } + + sourceStream->FinishAddTracks(); + mData->mStreamInitialized = true; +} + +static void +SendStreamAudio(DecodedStreamData* aStream, int64_t aStartTime, + AudioData* aAudio, AudioSegment* aOutput, + uint32_t aRate, double aVolume) +{ + // This logic has to mimic AudioSink closely to make sure we write + // the exact same silences + CheckedInt64 audioWrittenOffset = aStream->mAudioFramesWritten + + UsecsToFrames(aStartTime, aRate); + CheckedInt64 frameOffset = UsecsToFrames(aAudio->mTime, aRate); + + if (!audioWrittenOffset.isValid() || + !frameOffset.isValid() || + // ignore packet that we've already processed + frameOffset.value() + aAudio->mFrames <= audioWrittenOffset.value()) { + return; + } + + if (audioWrittenOffset.value() < frameOffset.value()) { + int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value(); + // Write silence to catch up + AudioSegment silence; + silence.InsertNullDataAtStart(silentFrames); + aStream->mAudioFramesWritten += silentFrames; + audioWrittenOffset += silentFrames; + aOutput->AppendFrom(&silence); + } + + MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value()); + + int64_t offset = audioWrittenOffset.value() - frameOffset.value(); + size_t framesToWrite = aAudio->mFrames - offset; + + aAudio->EnsureAudioBuffer(); + nsRefPtr buffer = aAudio->mAudioBuffer; + AudioDataValue* bufferData = static_cast(buffer->Data()); + nsAutoTArray channels; + for (uint32_t i = 0; i < aAudio->mChannels; ++i) { + channels.AppendElement(bufferData + i * aAudio->mFrames + offset); + } + aOutput->AppendFrames(buffer.forget(), channels, framesToWrite); + aStream->mAudioFramesWritten += framesToWrite; + aOutput->ApplyVolume(aVolume); + + aStream->mNextAudioTime = aAudio->GetEndTime(); +} + +void +DecodedStream::SendAudio(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aQueue, + double aVolume, bool aIsSameOrigin) +{ + GetReentrantMonitor().AssertCurrentThreadIn(); + + if (!aInfo.HasAudio()) { + return; + } + + AudioSegment output; + uint32_t rate = aInfo.mAudio.mRate; + nsAutoTArray,10> audio; + TrackID audioTrackId = aInfo.mAudio.mTrackId; + SourceMediaStream* sourceStream = mData->mStream; + + // It's OK to hold references to the AudioData because AudioData + // is ref-counted. + aQueue.GetElementsAfter(mData->mNextAudioTime, &audio); + for (uint32_t i = 0; i < audio.Length(); ++i) { + SendStreamAudio(mData.get(), aStartTime, audio[i], &output, rate, aVolume); + } + + if (!aIsSameOrigin) { + output.ReplaceWithDisabled(); + } + + // |mNextAudioTime| is updated as we process each audio sample in + // SendStreamAudio(). This is consistent with how |mNextVideoTime| + // is updated for video samples. + if (output.GetDuration() > 0) { + sourceStream->AppendToTrack(audioTrackId, &output); + } + + if (aQueue.IsFinished() && !mData->mHaveSentFinishAudio) { + sourceStream->EndTrack(audioTrackId); + mData->mHaveSentFinishAudio = true; + } +} + +static void +WriteVideoToMediaStream(MediaStream* aStream, + layers::Image* aImage, + int64_t aEndMicroseconds, + int64_t aStartMicroseconds, + const mozilla::gfx::IntSize& aIntrinsicSize, + VideoSegment* aOutput) +{ + nsRefPtr image = aImage; + StreamTime duration = + aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) - + aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds); + aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize); +} + +static bool +ZeroDurationAtLastChunk(VideoSegment& aInput) +{ + // Get the last video frame's start time in VideoSegment aInput. + // If the start time is equal to the duration of aInput, means the last video + // frame's duration is zero. + StreamTime lastVideoStratTime; + aInput.GetLastFrame(&lastVideoStratTime); + return lastVideoStratTime == aInput.GetDuration(); +} + +void +DecodedStream::SendVideo(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aQueue, + bool aIsSameOrigin) +{ + GetReentrantMonitor().AssertCurrentThreadIn(); + + if (!aInfo.HasVideo()) { + return; + } + + VideoSegment output; + TrackID videoTrackId = aInfo.mVideo.mTrackId; + nsAutoTArray, 10> video; + SourceMediaStream* sourceStream = mData->mStream; + + // It's OK to hold references to the VideoData because VideoData + // is ref-counted. + aQueue.GetElementsAfter(mData->mNextVideoTime, &video); + + for (uint32_t i = 0; i < video.Length(); ++i) { + VideoData* v = video[i]; + + if (mData->mNextVideoTime < v->mTime) { + // Write last video frame to catch up. mLastVideoImage can be null here + // which is fine, it just means there's no video. + + // TODO: |mLastVideoImage| should come from the last image rendered + // by the state machine. This will avoid the black frame when capture + // happens in the middle of playback (especially in th middle of a + // video frame). E.g. if we have a video frame that is 30 sec long + // and capture happens at 15 sec, we'll have to append a black frame + // that is 15 sec long. + WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, v->mTime, + mData->mNextVideoTime, mData->mLastVideoImageDisplaySize, &output); + mData->mNextVideoTime = v->mTime; + } + + if (mData->mNextVideoTime < v->GetEndTime()) { + WriteVideoToMediaStream(sourceStream, v->mImage, + v->GetEndTime(), mData->mNextVideoTime, v->mDisplay, &output); + mData->mNextVideoTime = v->GetEndTime(); + mData->mLastVideoImage = v->mImage; + mData->mLastVideoImageDisplaySize = v->mDisplay; + } + } + + // Check the output is not empty. + if (output.GetLastFrame()) { + mData->mEOSVideoCompensation = ZeroDurationAtLastChunk(output); + } + + if (!aIsSameOrigin) { + output.ReplaceWithDisabled(); + } + + if (output.GetDuration() > 0) { + sourceStream->AppendToTrack(videoTrackId, &output); + } + + if (aQueue.IsFinished() && !mData->mHaveSentFinishVideo) { + if (mData->mEOSVideoCompensation) { + VideoSegment endSegment; + // Calculate the deviation clock time from DecodedStream. + int64_t deviation_usec = sourceStream->StreamTimeToMicroseconds(1); + WriteVideoToMediaStream(sourceStream, mData->mLastVideoImage, + mData->mNextVideoTime + deviation_usec, mData->mNextVideoTime, + mData->mLastVideoImageDisplaySize, &endSegment); + mData->mNextVideoTime += deviation_usec; + MOZ_ASSERT(endSegment.GetDuration() > 0); + if (!aIsSameOrigin) { + endSegment.ReplaceWithDisabled(); + } + sourceStream->AppendToTrack(videoTrackId, &endSegment); + } + sourceStream->EndTrack(videoTrackId); + mData->mHaveSentFinishVideo = true; + } +} + +void +DecodedStream::AdvanceTracks(int64_t aStartTime, const MediaInfo& aInfo) +{ + GetReentrantMonitor().AssertCurrentThreadIn(); + + StreamTime endPosition = 0; + + if (aInfo.HasAudio()) { + StreamTime audioEnd = mData->mStream->TicksToTimeRoundDown( + aInfo.mAudio.mRate, mData->mAudioFramesWritten); + endPosition = std::max(endPosition, audioEnd); + } + + if (aInfo.HasVideo()) { + StreamTime videoEnd = mData->mStream->MicrosecondsToStreamTimeRoundDown( + mData->mNextVideoTime - aStartTime); + endPosition = std::max(endPosition, videoEnd); + } + + if (!mData->mHaveSentFinish) { + mData->mStream->AdvanceKnownTracksTime(endPosition); + } +} + +bool +DecodedStream::SendData(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aAudioQueue, + MediaQueue& aVideoQueue, + double aVolume, bool aIsSameOrigin) +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + + InitTracks(aStartTime, aInfo); + SendAudio(aStartTime, aInfo, aAudioQueue, aVolume, aIsSameOrigin); + SendVideo(aStartTime, aInfo, aVideoQueue, aIsSameOrigin); + AdvanceTracks(aStartTime, aInfo); + + bool finished = (!aInfo.HasAudio() || aAudioQueue.IsFinished()) && + (!aInfo.HasVideo() || aVideoQueue.IsFinished()); + + if (finished && !mData->mHaveSentFinish) { + mData->mHaveSentFinish = true; + mData->mStream->Finish(); + } + + return finished; +} + +CheckedInt64 +DecodedStream::AudioEndTime(int64_t aStartTime, uint32_t aRate) const +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + return aStartTime + FramesToUsecs(mData->mAudioFramesWritten, aRate); +} + +int64_t +DecodedStream::GetPosition() const +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + return mData->GetPosition(); +} + +bool +DecodedStream::IsFinished() const +{ + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + return mData->IsFinished(); +} + } // namespace mozilla diff --git a/dom/media/DecodedStream.h b/dom/media/DecodedStream.h index b1d7c59d1b..f3bae24cd6 100644 --- a/dom/media/DecodedStream.h +++ b/dom/media/DecodedStream.h @@ -9,11 +9,19 @@ #include "mozilla/nsRefPtr.h" #include "nsTArray.h" + #include "mozilla/UniquePtr.h" #include "mozilla/gfx/Point.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/ReentrantMonitor.h" namespace mozilla { +class AudioData; +class VideoData; +class MediaInfo; +class AudioSegment; +class MediaStream; class MediaInputPort; class SourceMediaStream; class ProcessedMediaStream; @@ -23,6 +31,8 @@ class OutputStreamListener; class ReentrantMonitor; class MediaStreamGraph; +template class MediaQueue; + namespace layers { class Image; } // namespace layers @@ -37,10 +47,11 @@ class Image; */ class DecodedStreamData { public: - explicit DecodedStreamData(SourceMediaStream* aStream); + DecodedStreamData(SourceMediaStream* aStream, bool aPlaying); ~DecodedStreamData(); bool IsFinished() const; int64_t GetPosition() const; + void SetPlaying(bool aPlaying); /* The following group of fields are protected by the decoder's monitor * and can be read or written on any thread. @@ -66,12 +77,10 @@ public: // The decoder is responsible for calling Destroy() on this stream. const nsRefPtr mStream; nsRefPtr mListener; - // True when we've explicitly blocked this stream because we're - // not in PLAY_STATE_PLAYING. Used on the main thread only. - bool mHaveBlockedForPlayState; - // We also have an explicit blocker on the stream when - // mDecoderStateMachine is non-null and MediaDecoderStateMachine is false. - bool mHaveBlockedForStateMachineNotPlaying; + bool mPlaying; + // True if we need to send a compensation video frame to ensure the + // StreamTime going forward. + bool mEOSVideoCompensation; }; class OutputStreamData { @@ -85,21 +94,62 @@ public: }; class DecodedStream { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStream); public: - explicit DecodedStream(ReentrantMonitor& aMonitor); - DecodedStreamData* GetData() const; + DecodedStream(); void DestroyData(); - void RecreateData(MediaStreamGraph* aGraph); - nsTArray& OutputStreams(); - ReentrantMonitor& GetReentrantMonitor() const; + void RecreateData(); void Connect(ProcessedMediaStream* aStream, bool aFinishWhenEnded); + void Remove(MediaStream* aStream); + void SetPlaying(bool aPlaying); + bool HaveEnoughAudio(const MediaInfo& aInfo) const; + bool HaveEnoughVideo(const MediaInfo& aInfo) const; + CheckedInt64 AudioEndTime(int64_t aStartTime, uint32_t aRate) const; + int64_t GetPosition() const; + bool IsFinished() const; + + // Return true if stream is finished. + bool SendData(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aAudioQueue, + MediaQueue& aVideoQueue, + double aVolume, bool aIsSameOrigin); + +protected: + virtual ~DecodedStream() {} private: + ReentrantMonitor& GetReentrantMonitor() const; + void RecreateData(MediaStreamGraph* aGraph); void Connect(OutputStreamData* aStream); + nsTArray& OutputStreams(); + void InitTracks(int64_t aStartTime, const MediaInfo& aInfo); + void AdvanceTracks(int64_t aStartTime, const MediaInfo& aInfo); + + void SendAudio(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aQueue, + double aVolume, bool aIsSameOrigin); + + void SendVideo(int64_t aStartTime, + const MediaInfo& aInfo, + MediaQueue& aQueue, + bool aIsSameOrigin); + UniquePtr mData; // Data about MediaStreams that are being fed by the decoder. nsTArray mOutputStreams; - ReentrantMonitor& mMonitor; + + // TODO: This is a temp solution to get rid of decoder monitor on the main + // thread in MDSM::AddOutputStream and MDSM::RecreateDecodedStream as + // required by bug 1146482. DecodedStream needs to release monitor before + // calling back into MDSM functions in order to prevent deadlocks. + // + // Please move all capture-stream related code from MDSM into DecodedStream + // and apply "dispatch + mirroring" to get rid of this monitor in the future. + mutable ReentrantMonitor mMonitor; + + bool mPlaying; }; } // namespace mozilla diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp index 49215f51a7..471d4d7187 100644 --- a/dom/media/MediaData.cpp +++ b/dom/media/MediaData.cpp @@ -113,9 +113,12 @@ VideoData::VideoData(int64_t aOffset, int64_t aDuration, bool aKeyframe, int64_t aTimecode, - IntSize aDisplay) + IntSize aDisplay, + layers::ImageContainer::FrameID aFrameID) : MediaData(VIDEO_DATA, aOffset, aTime, aDuration) , mDisplay(aDisplay) + , mFrameID(aFrameID) + , mSentToCompositor(false) { NS_ASSERTION(mDuration >= 0, "Frame must have non-negative duration."); mKeyframe = aKeyframe; @@ -152,7 +155,8 @@ VideoData::ShallowCopyUpdateDuration(const VideoData* aOther, aDuration, aOther->mKeyframe, aOther->mTimecode, - aOther->mDisplay); + aOther->mDisplay, + aOther->mFrameID); v->mDiscontinuity = aOther->mDiscontinuity; v->mImage = aOther->mImage; return v.forget(); @@ -169,7 +173,8 @@ VideoData::ShallowCopyUpdateTimestamp(const VideoData* aOther, aOther->GetEndTime() - aTimestamp, aOther->mKeyframe, aOther->mTimecode, - aOther->mDisplay); + aOther->mDisplay, + aOther->mFrameID); v->mDiscontinuity = aOther->mDiscontinuity; v->mImage = aOther->mImage; return v.forget(); @@ -187,7 +192,8 @@ VideoData::ShallowCopyUpdateTimestampAndDuration(const VideoData* aOther, aDuration, aOther->mKeyframe, aOther->mTimecode, - aOther->mDisplay); + aOther->mDisplay, + aOther->mFrameID); v->mDiscontinuity = aOther->mDiscontinuity; v->mImage = aOther->mImage; return v.forget(); @@ -252,7 +258,8 @@ VideoData::Create(const VideoInfo& aInfo, aDuration, aKeyframe, aTimecode, - aInfo.mDisplay)); + aInfo.mDisplay, + 0)); return v.forget(); } @@ -294,7 +301,8 @@ VideoData::Create(const VideoInfo& aInfo, aDuration, aKeyframe, aTimecode, - aInfo.mDisplay)); + aInfo.mDisplay, + 0)); #ifdef MOZ_WIDGET_GONK const YCbCrBuffer::Plane &Y = aBuffer.mPlanes[0]; const YCbCrBuffer::Plane &Cb = aBuffer.mPlanes[1]; @@ -396,7 +404,8 @@ VideoData::CreateFromImage(const VideoInfo& aInfo, aDuration, aKeyframe, aTimecode, - aInfo.mDisplay)); + aInfo.mDisplay, + 0)); v->mImage = aImage; return v.forget(); } @@ -422,7 +431,8 @@ VideoData::Create(const VideoInfo& aInfo, aDuration, aKeyframe, aTimecode, - aInfo.mDisplay)); + aInfo.mDisplay, + 0)); return v.forget(); } @@ -449,7 +459,8 @@ VideoData::Create(const VideoInfo& aInfo, aDuration, aKeyframe, aTimecode, - aInfo.mDisplay)); + aInfo.mDisplay, + 0)); v->mImage = aContainer->CreateImage(ImageFormat::GRALLOC_PLANAR_YCBCR); if (!v->mImage) { diff --git a/dom/media/MediaData.h b/dom/media/MediaData.h index a853b3ed70..ea92356f11 100644 --- a/dom/media/MediaData.h +++ b/dom/media/MediaData.h @@ -279,13 +279,17 @@ public: // This frame's image. nsRefPtr mImage; + int32_t mFrameID; + + bool mSentToCompositor; VideoData(int64_t aOffset, int64_t aTime, int64_t aDuration, bool aKeyframe, int64_t aTimecode, - IntSize aDisplay); + IntSize aDisplay, + int32_t aFrameID); protected: ~VideoData(); diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index c9bdfcbb6d..998b9a78e4 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -123,6 +123,7 @@ PRLogModuleInfo* gMediaSampleLog; void MediaDecoder::InitStatics() { + MOZ_ASSERT(NS_IsMainThread()); AbstractThread::InitStatics(); SharedThreadPool::InitStatics(); @@ -235,6 +236,7 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure) { + MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aClosure); MediaDecoder* decoder = static_cast(aClosure); ReentrantMonitorAutoEnter mon(decoder->GetReentrantMonitor()); @@ -244,6 +246,7 @@ void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure) void MediaDecoder::StartDormantTimer() { + MOZ_ASSERT(NS_IsMainThread()); if (!mIsHeuristicDormantSupported) { return; } @@ -269,6 +272,7 @@ void MediaDecoder::StartDormantTimer() void MediaDecoder::CancelDormantTimer() { + MOZ_ASSERT(NS_IsMainThread()); if (mDormantTimer) { mDormantTimer->Cancel(); } @@ -503,7 +507,7 @@ nsresult MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor) void MediaDecoder::SetStateMachineParameters() { - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + MOZ_ASSERT(NS_IsMainThread()); if (mMinimizePreroll) { mDecoderStateMachine->DispatchMinimizePrerollUntilPlaybackStarts(); } @@ -511,8 +515,8 @@ void MediaDecoder::SetStateMachineParameters() void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts() { - DECODER_LOG("SetMinimizePrerollUntilPlaybackStarts()"); MOZ_ASSERT(NS_IsMainThread()); + DECODER_LOG("SetMinimizePrerollUntilPlaybackStarts()"); mMinimizePreroll = true; // This needs to be called before we init the state machine, otherwise it will @@ -589,6 +593,7 @@ nsresult MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) void MediaDecoder::CallSeek(const SeekTarget& aTarget) { + MOZ_ASSERT(NS_IsMainThread()); mSeekRequest.DisconnectIfExists(); mSeekRequest.Begin(ProxyMediaCall(mDecoderStateMachine->TaskQueue(), mDecoderStateMachine.get(), __func__, @@ -668,6 +673,7 @@ void MediaDecoder::MetadataLoaded(nsAutoPtr aInfo, const char* MediaDecoder::PlayStateStr() { + MOZ_ASSERT(NS_IsMainThread()); switch (mPlayState) { case PLAY_STATE_START: return "PLAY_STATE_START"; case PLAY_STATE_LOADING: return "PLAY_STATE_LOADING"; @@ -760,6 +766,7 @@ void MediaDecoder::DecodeError() void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) { + MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mSameOriginMedia = aSameOrigin; } @@ -784,6 +791,7 @@ bool MediaDecoder::IsEndedOrShutdown() const bool MediaDecoder::IsEnded() const { + MOZ_ASSERT(NS_IsMainThread()); return mPlayState == PLAY_STATE_ENDED || (mWasEndedWhenEnteredDormant && (mPlayState != PLAY_STATE_SHUTDOWN)); } @@ -857,7 +865,7 @@ double MediaDecoder::ComputePlaybackRate(bool* aReliable) void MediaDecoder::UpdatePlaybackRate() { - MOZ_ASSERT(NS_IsMainThread() || OnStateMachineTaskQueue()); + MOZ_ASSERT(NS_IsMainThread()); GetReentrantMonitor().AssertCurrentThreadIn(); if (!mResource) return; @@ -927,6 +935,7 @@ void MediaDecoder::NotifyDownloadEnded(nsresult aStatus) void MediaDecoder::NotifyPrincipalChanged() { + MOZ_ASSERT(NS_IsMainThread()); if (mOwner) { mOwner->NotifyDecoderPrincipalChanged(); } @@ -1134,6 +1143,7 @@ bool MediaDecoder::IsMediaSeekable() media::TimeIntervals MediaDecoder::GetSeekable() { + MOZ_ASSERT(NS_IsMainThread()); // We can seek in buffered range if the media is seekable. Also, we can seek // in unbuffered ranges if the transport level is seekable (local file or the // server supports range requests, etc.) @@ -1220,6 +1230,7 @@ bool MediaDecoder::OnStateMachineTaskQueue() const void MediaDecoder::SetPlaybackRate(double aPlaybackRate) { + MOZ_ASSERT(NS_IsMainThread()); mPlaybackRate = aPlaybackRate; if (mPlaybackRate == 0.0) { mPausedForPlaybackRateNull = true; @@ -1237,6 +1248,7 @@ void MediaDecoder::SetPlaybackRate(double aPlaybackRate) void MediaDecoder::SetPreservesPitch(bool aPreservesPitch) { + MOZ_ASSERT(NS_IsMainThread()); mPreservesPitch = aPreservesPitch; } @@ -1248,6 +1260,7 @@ bool MediaDecoder::OnDecodeTaskQueue() const { void MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine) { + MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT_IF(aStateMachine, !mDecoderStateMachine); mDecoderStateMachine = aStateMachine; @@ -1292,10 +1305,12 @@ void MediaDecoder::Invalidate() // Constructs the time ranges representing what segments of the media // are buffered and playable. media::TimeIntervals MediaDecoder::GetBuffered() { + MOZ_ASSERT(NS_IsMainThread()); return mBuffered.Ref(); } size_t MediaDecoder::SizeOfVideoQueue() { + MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfVideoQueue(); } @@ -1303,6 +1318,7 @@ size_t MediaDecoder::SizeOfVideoQueue() { } size_t MediaDecoder::SizeOfAudioQueue() { + MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfAudioQueue(); } @@ -1355,6 +1371,7 @@ MediaDecoderOwner* MediaDecoder::GetMediaOwner() const void MediaDecoder::FireTimeUpdate() { + MOZ_ASSERT(NS_IsMainThread()); if (!mOwner) return; mOwner->FireTimeUpdate(true); @@ -1362,6 +1379,7 @@ void MediaDecoder::FireTimeUpdate() void MediaDecoder::PinForSeek() { + MOZ_ASSERT(NS_IsMainThread()); MediaResource* resource = GetResource(); if (!resource || mPinnedForSeek) { return; @@ -1372,6 +1390,7 @@ void MediaDecoder::PinForSeek() void MediaDecoder::UnpinForSeek() { + MOZ_ASSERT(NS_IsMainThread()); MediaResource* resource = GetResource(); if (!resource || !mPinnedForSeek) { return; diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 68b55f5c5a..0dab221541 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -336,7 +336,7 @@ public: } void SetResource(MediaResource* aResource) { - NS_ASSERTION(NS_IsMainThread(), "Should only be called on main thread"); + MOZ_ASSERT(NS_IsMainThread()); mResource = aResource; } @@ -556,17 +556,21 @@ public: virtual void UpdatePlaybackRate(); // Used to estimate rates of data passing through the decoder's channel. - // Records activity stopping on the channel. The monitor must be held. - virtual void NotifyPlaybackStarted() { - GetReentrantMonitor().AssertCurrentThreadIn(); - mPlaybackStatistics->Start(); + // Records activity stopping on the channel. + void DispatchPlaybackStarted() { + nsRefPtr self = this; + nsCOMPtr r = + NS_NewRunnableFunction([self] () { self->mPlaybackStatistics->Start(); }); + AbstractThread::MainThread()->Dispatch(r.forget()); } // Used to estimate rates of data passing through the decoder's channel. - // Records activity stopping on the channel. The monitor must be held. - virtual void NotifyPlaybackStopped() { - GetReentrantMonitor().AssertCurrentThreadIn(); - mPlaybackStatistics->Stop(); + // Records activity stopping on the channel. + void DispatchPlaybackStopped() { + nsRefPtr self = this; + nsCOMPtr r = + NS_NewRunnableFunction([self] () { self->mPlaybackStatistics->Stop(); }); + AbstractThread::MainThread()->Dispatch(r.forget()); } // The actual playback rate computation. The monitor must be held. @@ -637,6 +641,7 @@ public: void OnSeekRejected() { + MOZ_ASSERT(NS_IsMainThread()); mSeekRequest.Complete(); mLogicallySeeking = false; } @@ -647,7 +652,11 @@ public: void SeekingStarted(MediaDecoderEventVisibility aEventVisibility = MediaDecoderEventVisibility::Observable); void UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisibility); - void UpdateLogicalPosition() { UpdateLogicalPosition(MediaDecoderEventVisibility::Observable); } + void UpdateLogicalPosition() + { + MOZ_ASSERT(NS_IsMainThread()); + UpdateLogicalPosition(MediaDecoderEventVisibility::Observable); + } // Find the end of the cached data starting at the current decoder // position. diff --git a/dom/media/MediaDecoderReader.cpp b/dom/media/MediaDecoderReader.cpp index 0b5a526911..f03be14b5c 100644 --- a/dom/media/MediaDecoderReader.cpp +++ b/dom/media/MediaDecoderReader.cpp @@ -76,7 +76,6 @@ MediaDecoderReader::MediaDecoderReader(AbstractMediaDecoder* aDecoder, , mThrottleDuration(TimeDuration::FromMilliseconds(500)) , mLastThrottledNotify(TimeStamp::Now() - mThrottleDuration) , mIgnoreAudioOutputFormat(false) - , mStartTime(-1) , mHitAudioDecodeError(false) , mShutdown(false) , mTaskQueueIsBorrowed(!!aBorrowedTaskQueue) @@ -237,7 +236,7 @@ media::TimeIntervals MediaDecoderReader::GetBuffered() { MOZ_ASSERT(OnTaskQueue()); - NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); AutoPinned stream(mDecoder->GetResource()); if (!mDuration.Ref().isSome()) { @@ -296,7 +295,8 @@ public: // Make sure ResetDecode hasn't been called in the mean time. if (!mReader->mBaseVideoPromise.IsEmpty()) { - mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold); + mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold, + /* aForceDecodeAhead = */ false); } return NS_OK; @@ -333,7 +333,8 @@ private: nsRefPtr MediaDecoderReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) + int64_t aTimeThreshold, + bool aForceDecodeAhead) { nsRefPtr p = mBaseVideoPromise.Ensure(__func__); bool skip = aSkipToNextKeyframe; diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h index 47b11c1b59..6c0dabcc82 100644 --- a/dom/media/MediaDecoderReader.h +++ b/dom/media/MediaDecoderReader.h @@ -147,7 +147,7 @@ public: // If aSkipToKeyframe is true, the decode should skip ahead to the // the next keyframe at or after aTimeThreshold microseconds. virtual nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold); + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead); friend class ReRequestVideoWithSkipTask; friend class ReRequestAudioTask; @@ -303,8 +303,8 @@ public: NS_NewRunnableFunction([self, aStartTime] () -> void { MOZ_ASSERT(self->OnTaskQueue()); - MOZ_ASSERT(self->mStartTime == -1); - self->mStartTime = aStartTime; + MOZ_ASSERT(!self->HaveStartTime()); + self->mStartTime.emplace(aStartTime); self->UpdateBuffered(); }); TaskQueue()->Dispatch(r.forget()); @@ -406,7 +406,9 @@ protected: // readers to return the correct value of GetBuffered. We should refactor // things such that all GetBuffered calls go through the MDSM, which would // offset the range accordingly. - int64_t mStartTime; + Maybe mStartTime; + bool HaveStartTime() { MOZ_ASSERT(OnTaskQueue()); return mStartTime.isSome(); } + int64_t StartTime() { MOZ_ASSERT(HaveStartTime()); return mStartTime.ref(); } // This is a quick-and-dirty way for DecodeAudioData implementations to // communicate the presence of a decoding error to RequestAudioData. We should diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index e3dd655c37..0975763934 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -45,7 +45,6 @@ namespace mozilla { using namespace mozilla::dom; -using namespace mozilla::gfx; using namespace mozilla::layers; using namespace mozilla::media; @@ -100,7 +99,7 @@ const int64_t NO_VIDEO_AMPLE_AUDIO_DIVISOR = 8; // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and // we're not "prerolling video", we'll skip the video up to the next keyframe // which is at or after the current playback position. -static const uint32_t LOW_VIDEO_FRAMES = 1; +static const uint32_t LOW_VIDEO_FRAMES = 2; // Threshold in usecs that used to check if we are low on decoded video. // If the last video frame's end time |mDecodedVideoEndTime| is more than @@ -171,6 +170,7 @@ static int64_t DurationToUsecs(TimeDuration aDuration) { static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3; static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10; +static const uint32_t SCARCE_VIDEO_QUEUE_SIZE = 1; static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE; static uint32_t sVideoQueueHWAccelSize = MIN_VIDEO_QUEUE_SIZE; @@ -240,7 +240,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mSentLoadedMetadataEvent(false), mSentFirstFrameLoadedEvent(false), mSentPlaybackEndedEvent(false), - mDecodedStream(mDecoder->GetReentrantMonitor()) + mDecodedStream(new DecodedStream()) { MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); @@ -321,8 +321,6 @@ MediaDecoderStateMachine::InitializationTask() mWatchManager.Watch(mObservedDuration, &MediaDecoderStateMachine::RecomputeDuration); mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::PlayStateChanged); mWatchManager.Watch(mLogicallySeeking, &MediaDecoderStateMachine::LogicallySeekingChanged); - mWatchManager.Watch(mPlayState, &MediaDecoderStateMachine::UpdateStreamBlockingForPlayState); - mWatchManager.Watch(mLogicallySeeking, &MediaDecoderStateMachine::UpdateStreamBlockingForPlayState); } bool MediaDecoderStateMachine::HasFutureAudio() @@ -346,7 +344,7 @@ bool MediaDecoderStateMachine::HaveNextFrameData() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); return (!HasAudio() || HasFutureAudio()) && - (!HasVideo() || VideoQueue().GetSize() > 0); + (!HasVideo() || VideoQueue().GetSize() > 1); } int64_t MediaDecoderStateMachine::GetDecodedAudioDuration() @@ -360,227 +358,21 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration() return audioDecoded; } -void MediaDecoderStateMachine::SendStreamAudio(AudioData* aAudio, - DecodedStreamData* aStream, - AudioSegment* aOutput) -{ - MOZ_ASSERT(OnTaskQueue()); - AssertCurrentThreadInMonitor(); - - // This logic has to mimic AudioSink closely to make sure we write - // the exact same silences - CheckedInt64 audioWrittenOffset = aStream->mAudioFramesWritten + - UsecsToFrames(mInfo.mAudio.mRate, mStreamStartTime); - CheckedInt64 frameOffset = UsecsToFrames(mInfo.mAudio.mRate, aAudio->mTime); - - if (!audioWrittenOffset.isValid() || - !frameOffset.isValid() || - // ignore packet that we've already processed - frameOffset.value() + aAudio->mFrames <= audioWrittenOffset.value()) { - return; - } - - if (audioWrittenOffset.value() < frameOffset.value()) { - int64_t silentFrames = frameOffset.value() - audioWrittenOffset.value(); - // Write silence to catch up - VERBOSE_LOG("writing %lld frames of silence to MediaStream", silentFrames); - AudioSegment silence; - silence.InsertNullDataAtStart(silentFrames); - aStream->mAudioFramesWritten += silentFrames; - audioWrittenOffset += silentFrames; - aOutput->AppendFrom(&silence); - } - - MOZ_ASSERT(audioWrittenOffset.value() >= frameOffset.value()); - - int64_t offset = audioWrittenOffset.value() - frameOffset.value(); - size_t framesToWrite = aAudio->mFrames - offset; - - aAudio->EnsureAudioBuffer(); - nsRefPtr buffer = aAudio->mAudioBuffer; - AudioDataValue* bufferData = static_cast(buffer->Data()); - nsAutoTArray channels; - for (uint32_t i = 0; i < aAudio->mChannels; ++i) { - channels.AppendElement(bufferData + i*aAudio->mFrames + offset); - } - aOutput->AppendFrames(buffer.forget(), channels, framesToWrite); - VERBOSE_LOG("writing %u frames of data to MediaStream for AudioData at %lld", - static_cast(framesToWrite), - aAudio->mTime); - aStream->mAudioFramesWritten += framesToWrite; - aOutput->ApplyVolume(mVolume); - - aStream->mNextAudioTime = aAudio->GetEndTime(); -} - -static void WriteVideoToMediaStream(MediaStream* aStream, - layers::Image* aImage, - int64_t aEndMicroseconds, - int64_t aStartMicroseconds, - const IntSize& aIntrinsicSize, - VideoSegment* aOutput) -{ - nsRefPtr image = aImage; - StreamTime duration = - aStream->MicrosecondsToStreamTimeRoundDown(aEndMicroseconds) - - aStream->MicrosecondsToStreamTimeRoundDown(aStartMicroseconds); - aOutput->AppendFrame(image.forget(), duration, aIntrinsicSize); -} - -static void -UpdateStreamBlocking(MediaStream* aStream, bool aBlocking) -{ - int32_t delta = aBlocking ? 1 : -1; - if (NS_IsMainThread()) { - aStream->ChangeExplicitBlockerCount(delta); - } else { - nsCOMPtr r = NS_NewRunnableMethodWithArg( - aStream, &MediaStream::ChangeExplicitBlockerCount, delta); - AbstractThread::MainThread()->Dispatch(r.forget()); - } -} - void MediaDecoderStateMachine::SendStreamData() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); MOZ_ASSERT(!mAudioSink, "Should've been stopped in RunStateMachine()"); - MOZ_ASSERT(mStreamStartTime != -1); - DecodedStreamData* stream = GetDecodedStream(); + bool finished = mDecodedStream->SendData( + mStreamStartTime, mInfo, AudioQueue(), VideoQueue(), + mVolume, mDecoder->IsSameOriginMedia()); - bool finished = - (!mInfo.HasAudio() || AudioQueue().IsFinished()) && - (!mInfo.HasVideo() || VideoQueue().IsFinished()); - - { - SourceMediaStream* mediaStream = stream->mStream; - StreamTime endPosition = 0; - const bool isSameOrigin = mDecoder->IsSameOriginMedia(); - - if (!stream->mStreamInitialized) { - if (mInfo.HasAudio()) { - TrackID audioTrackId = mInfo.mAudio.mTrackId; - AudioSegment* audio = new AudioSegment(); - mediaStream->AddAudioTrack(audioTrackId, mInfo.mAudio.mRate, 0, audio, - SourceMediaStream::ADDTRACK_QUEUED); - stream->mNextAudioTime = mStreamStartTime; - } - if (mInfo.HasVideo()) { - TrackID videoTrackId = mInfo.mVideo.mTrackId; - VideoSegment* video = new VideoSegment(); - mediaStream->AddTrack(videoTrackId, 0, video, - SourceMediaStream::ADDTRACK_QUEUED); - stream->mNextVideoTime = mStreamStartTime; - } - mediaStream->FinishAddTracks(); - stream->mStreamInitialized = true; - - // Make sure stream blocking is updated before sending stream data so we - // don't 'leak' data when the stream is supposed to be blocked. - UpdateStreamBlockingForPlayState(); - UpdateStreamBlockingForStateMachinePlaying(); - UpdateStreamBlocking(mediaStream, false); - } - - if (mInfo.HasAudio()) { - MOZ_ASSERT(stream->mNextAudioTime != -1, "Should've been initialized"); - TrackID audioTrackId = mInfo.mAudio.mTrackId; - nsAutoTArray,10> audio; - // It's OK to hold references to the AudioData because AudioData - // is ref-counted. - AudioQueue().GetElementsAfter(stream->mNextAudioTime, &audio); - AudioSegment output; - for (uint32_t i = 0; i < audio.Length(); ++i) { - SendStreamAudio(audio[i], stream, &output); - } - if (!isSameOrigin) { - output.ReplaceWithDisabled(); - } - // |mNextAudioTime| is updated as we process each audio sample in - // SendStreamAudio(). This is consistent with how |mNextVideoTime| - // is updated for video samples. - if (output.GetDuration() > 0) { - mediaStream->AppendToTrack(audioTrackId, &output); - } - if (AudioQueue().IsFinished() && !stream->mHaveSentFinishAudio) { - mediaStream->EndTrack(audioTrackId); - stream->mHaveSentFinishAudio = true; - } - endPosition = std::max(endPosition, - mediaStream->TicksToTimeRoundDown(mInfo.mAudio.mRate, - stream->mAudioFramesWritten)); - - CheckedInt64 playedUsecs = mStreamStartTime + - FramesToUsecs(stream->mAudioFramesWritten, mInfo.mAudio.mRate); - if (playedUsecs.isValid()) { - OnAudioEndTimeUpdate(playedUsecs.value()); - } - } - - if (mInfo.HasVideo()) { - MOZ_ASSERT(stream->mNextVideoTime != -1, "Should've been initialized"); - TrackID videoTrackId = mInfo.mVideo.mTrackId; - nsAutoTArray,10> video; - // It's OK to hold references to the VideoData because VideoData - // is ref-counted. - VideoQueue().GetElementsAfter(stream->mNextVideoTime, &video); - VideoSegment output; - for (uint32_t i = 0; i < video.Length(); ++i) { - VideoData* v = video[i]; - if (stream->mNextVideoTime < v->mTime) { - VERBOSE_LOG("writing last video to MediaStream %p for %lldus", - mediaStream, v->mTime - stream->mNextVideoTime); - // Write last video frame to catch up. mLastVideoImage can be null here - // which is fine, it just means there's no video. - - // TODO: |mLastVideoImage| should come from the last image rendered - // by the state machine. This will avoid the black frame when capture - // happens in the middle of playback (especially in th middle of a - // video frame). E.g. if we have a video frame that is 30 sec long - // and capture happens at 15 sec, we'll have to append a black frame - // that is 15 sec long. - WriteVideoToMediaStream(mediaStream, stream->mLastVideoImage, - v->mTime, stream->mNextVideoTime, stream->mLastVideoImageDisplaySize, - &output); - stream->mNextVideoTime = v->mTime; - } - if (stream->mNextVideoTime < v->GetEndTime()) { - VERBOSE_LOG("writing video frame %lldus to MediaStream %p for %lldus", - v->mTime, mediaStream, v->GetEndTime() - stream->mNextVideoTime); - WriteVideoToMediaStream(mediaStream, v->mImage, - v->GetEndTime(), stream->mNextVideoTime, v->mDisplay, - &output); - stream->mNextVideoTime = v->GetEndTime(); - stream->mLastVideoImage = v->mImage; - stream->mLastVideoImageDisplaySize = v->mDisplay; - } else { - VERBOSE_LOG("skipping writing video frame %lldus (end %lldus) to MediaStream", - v->mTime, v->GetEndTime()); - } - } - if (!isSameOrigin) { - output.ReplaceWithDisabled(); - } - if (output.GetDuration() > 0) { - mediaStream->AppendToTrack(videoTrackId, &output); - } - if (VideoQueue().IsFinished() && !stream->mHaveSentFinishVideo) { - mediaStream->EndTrack(videoTrackId); - stream->mHaveSentFinishVideo = true; - } - endPosition = std::max(endPosition, - mediaStream->MicrosecondsToStreamTimeRoundDown( - stream->mNextVideoTime - mStreamStartTime)); - } - - if (!stream->mHaveSentFinish) { - stream->mStream->AdvanceKnownTracksTime(endPosition); - } - - if (finished && !stream->mHaveSentFinish) { - stream->mHaveSentFinish = true; - stream->mStream->Finish(); + if (mInfo.HasAudio()) { + CheckedInt64 playedUsecs = mDecodedStream->AudioEndTime( + mStreamStartTime, mInfo.mAudio.mRate); + if (playedUsecs.isValid()) { + OnAudioEndTimeUpdate(playedUsecs.value()); } } @@ -614,21 +406,8 @@ bool MediaDecoderStateMachine::HaveEnoughDecodedAudio(int64_t aAmpleAudioUSecs) GetDecodedAudioDuration() < aAmpleAudioUSecs) { return false; } - if (!mAudioCaptured) { - return true; - } - DecodedStreamData* stream = GetDecodedStream(); - - if (stream && stream->mStreamInitialized && !stream->mHaveSentFinishAudio) { - MOZ_ASSERT(mInfo.HasAudio()); - TrackID audioTrackId = mInfo.mAudio.mTrackId; - if (!stream->mStream->HaveEnoughBuffered(audioTrackId)) { - return false; - } - } - - return true; + return !mAudioCaptured || mDecodedStream->HaveEnoughAudio(mInfo); } bool MediaDecoderStateMachine::HaveEnoughDecodedVideo() @@ -636,21 +415,11 @@ bool MediaDecoderStateMachine::HaveEnoughDecodedVideo() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (static_cast(VideoQueue().GetSize()) < GetAmpleVideoFrames() * mPlaybackRate) { + if (VideoQueue().GetSize() - 1 < GetAmpleVideoFrames() * mPlaybackRate) { return false; } - DecodedStreamData* stream = GetDecodedStream(); - - if (stream && stream->mStreamInitialized && !stream->mHaveSentFinishVideo) { - MOZ_ASSERT(mInfo.HasVideo()); - TrackID videoTrackId = mInfo.mVideo.mTrackId; - if (!stream->mStream->HaveEnoughBuffered(videoTrackId)) { - return false; - } - } - - return true; + return !mAudioCaptured || mDecodedStream->HaveEnoughVideo(mInfo); } bool @@ -1262,7 +1031,7 @@ void MediaDecoderStateMachine::StopPlayback() AssertCurrentThreadInMonitor(); - mDecoder->NotifyPlaybackStopped(); + mDecoder->DispatchPlaybackStopped(); if (IsPlaying()) { mPlayDuration = GetClock(); @@ -1272,7 +1041,6 @@ void MediaDecoderStateMachine::StopPlayback() // so it can pause audio playback. mDecoder->GetReentrantMonitor().NotifyAll(); NS_ASSERTION(!IsPlaying(), "Should report not playing at end of StopPlayback()"); - UpdateStreamBlockingForStateMachinePlaying(); DispatchDecodeTasksIfNeeded(); } @@ -1302,7 +1070,7 @@ void MediaDecoderStateMachine::MaybeStartPlayback() DECODER_LOG("MaybeStartPlayback() starting playback"); - mDecoder->NotifyPlaybackStarted(); + mDecoder->DispatchPlaybackStarted(); SetPlayStartTime(TimeStamp::Now()); MOZ_ASSERT(IsPlaying()); @@ -1310,7 +1078,6 @@ void MediaDecoderStateMachine::MaybeStartPlayback() NS_ENSURE_SUCCESS_VOID(rv); mDecoder->GetReentrantMonitor().NotifyAll(); - UpdateStreamBlockingForStateMachinePlaying(); DispatchDecodeTasksIfNeeded(); } @@ -1790,9 +1557,7 @@ MediaDecoderStateMachine::InitiateSeek() mCurrentSeek.mTarget.mTime = seekTime; if (mAudioCaptured) { - nsCOMPtr r = NS_NewRunnableMethodWithArgs( - this, &MediaDecoderStateMachine::RecreateDecodedStream, nullptr); - AbstractThread::MainThread()->Dispatch(r.forget()); + mDecodedStream->RecreateData(); } mDropAudioUntilNextDiscontinuity = HasAudio(); @@ -1928,6 +1693,7 @@ MediaDecoderStateMachine::EnsureVideoDecodeTaskQueued() bool skipToNextKeyFrame = NeedToSkipToNextKeyframe(); int64_t currentTime = mState == DECODER_STATE_SEEKING ? 0 : GetMediaTime(); + bool forceDecodeAhead = static_cast(VideoQueue().GetSize()) <= SCARCE_VIDEO_QUEUE_SIZE; // Time the video decode, so that if it's slow, we can increase our low // audio threshold to reduce the chance of an audio underrun while we're @@ -1940,7 +1706,7 @@ MediaDecoderStateMachine::EnsureVideoDecodeTaskQueued() mVideoDataRequest.Begin(ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestVideoData, - skipToNextKeyFrame, currentTime) + skipToNextKeyFrame, currentTime, forceDecodeAhead) ->Then(TaskQueue(), __func__, this, &MediaDecoderStateMachine::OnVideoDecoded, &MediaDecoderStateMachine::OnVideoNotDecoded)); @@ -2258,7 +2024,7 @@ MediaDecoderStateMachine::DecodeFirstFrame() mVideoDecodeStartTime = TimeStamp::Now(); mVideoDataRequest.Begin( ProxyMediaCall(DecodeTaskQueue(), mReader.get(), __func__, - &MediaDecoderReader::RequestVideoData, false, int64_t(0)) + &MediaDecoderReader::RequestVideoData, false, int64_t(0), false) ->Then(TaskQueue(), __func__, mStartTimeRendezvous.get(), &StartTimeRendezvous::ProcessFirstSample, &StartTimeRendezvous::FirstSampleRejected) @@ -2377,7 +2143,7 @@ MediaDecoderStateMachine::SeekCompleted() SetState(DECODER_STATE_SEEKING); } else if (GetMediaTime() == Duration().ToMicroseconds() && !isLiveStream) { // Seeked to end of media, move to COMPLETED state. Note we don't do - // this if we're playing a live stream, since the end of media will advance + // this when playing a live stream, since the end of media will advance // once we download more data! DECODER_LOG("Changed state from SEEKING (to %lld) to COMPLETED", seekTime); // Explicitly set our state so we don't decode further, and so @@ -2610,9 +2376,9 @@ nsresult MediaDecoderStateMachine::RunStateMachine() // Play the remaining media. We want to run AdvanceFrame() at least // once to ensure the current playback position is advanced to the // end of the media, and so that we update the readyState. - if (VideoQueue().GetSize() > 0 || + if (VideoQueue().GetSize() > 1 || (HasAudio() && !mAudioCompleted) || - (mAudioCaptured && !GetDecodedStream()->IsFinished())) + (mAudioCaptured && !mDecodedStream->IsFinished())) { // Start playback if necessary to play the remaining media. MaybeStartPlayback(); @@ -2734,6 +2500,7 @@ void MediaDecoderStateMachine::RenderVideoFrame(VideoData* aData, } else { mCorruptFrames.insert(0); } + aData->mSentToCompositor = true; container->SetCurrentFrame(aData->mDisplay, aData->mImage, aTarget); MOZ_ASSERT(container->GetFrameDelay() >= 0 || IsRealTime()); } @@ -2766,11 +2533,10 @@ int64_t MediaDecoderStateMachine::GetStreamClock() const { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - MOZ_ASSERT(mStreamStartTime != -1); - return mStreamStartTime + GetDecodedStream()->GetPosition(); + return mStreamStartTime + mDecodedStream->GetPosition(); } -int64_t MediaDecoderStateMachine::GetVideoStreamPosition() const +int64_t MediaDecoderStateMachine::GetVideoStreamPosition(TimeStamp aTimeStamp) const { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); @@ -2780,13 +2546,13 @@ int64_t MediaDecoderStateMachine::GetVideoStreamPosition() const } // Time elapsed since we started playing. - int64_t delta = DurationToUsecs(TimeStamp::Now() - mPlayStartTime); + int64_t delta = DurationToUsecs(aTimeStamp - mPlayStartTime); // Take playback rate into account. delta *= mPlaybackRate; return mPlayDuration + delta; } -int64_t MediaDecoderStateMachine::GetClock() const +int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); @@ -2796,6 +2562,7 @@ int64_t MediaDecoderStateMachine::GetClock() const // audio, or don't have audio, use the system clock. If our output is being // fed to a MediaStream, use that stream as the source of the clock. int64_t clock_time = -1; + TimeStamp t; if (!IsPlaying()) { clock_time = mPlayDuration; } else { @@ -2804,11 +2571,15 @@ int64_t MediaDecoderStateMachine::GetClock() const } else if (HasAudio() && !mAudioCompleted) { clock_time = GetAudioClock(); } else { + t = TimeStamp::Now(); // Audio is disabled on this system. Sync to the system clock. - clock_time = GetVideoStreamPosition(); + clock_time = GetVideoStreamPosition(t); } NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards."); } + if (aTimeStamp) { + *aTimeStamp = t.IsNull() ? TimeStamp::Now() : t; + } return clock_time; } @@ -2828,34 +2599,35 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() SendStreamData(); } - const int64_t clock_time = GetClock(); - TimeStamp nowTime = TimeStamp::Now(); + TimeStamp nowTime; + const int64_t clockTime = GetClock(&nowTime); // Skip frames up to the frame at the playback position, and figure out - // the time remaining until it's time to display the next frame. + // the time remaining until it's time to display the next frame and drop + // the current frame. + NS_ASSERTION(clockTime >= 0, "Should have positive clock time."); int64_t remainingTime = AUDIO_DURATION_USECS; - NS_ASSERTION(clock_time >= 0, "Should have positive clock time."); nsRefPtr currentFrame; if (VideoQueue().GetSize() > 0) { - VideoData* frame = VideoQueue().PeekFront(); - int32_t droppedFrames = 0; - while (IsRealTime() || clock_time >= frame->mTime) { - mVideoFrameEndTime = frame->GetEndTime(); - if (currentFrame) { - mDecoder->NotifyDecodedFrames(0, 0, 1); - VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld (%d so far)", - currentFrame->mTime, clock_time, ++droppedFrames); - } - currentFrame = frame; - nsRefPtr releaseMe = VideoQueue().PopFront(); - OnPlaybackOffsetUpdate(frame->mOffset); - if (VideoQueue().GetSize() == 0) + currentFrame = VideoQueue().PopFront(); + int32_t framesRemoved = 0; + while (VideoQueue().GetSize() > 0) { + VideoData* nextFrame = VideoQueue().PeekFront(); + if (!IsRealTime() && nextFrame->mTime > clockTime) { + remainingTime = nextFrame->mTime - clockTime; break; - frame = VideoQueue().PeekFront(); + } + ++framesRemoved; + if (!currentFrame->mSentToCompositor) { + mDecoder->NotifyDecodedFrames(0, 0, 1); + VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld", + currentFrame->mTime, clockTime); + } + currentFrame = VideoQueue().PopFront(); } - // Current frame has already been presented, wait until it's time to - // present the next frame. - if (frame && !currentFrame) { - remainingTime = frame->mTime - clock_time; + VideoQueue().PushFront(currentFrame); + if (framesRemoved > 0) { + OnPlaybackOffsetUpdate(currentFrame->mOffset); + mVideoFrameEndTime = currentFrame->GetEndTime(); } } @@ -2874,9 +2646,6 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() (OutOfDecodedVideo() && mVideoWaitRequest.Exists()); } if (shouldBuffer) { - if (currentFrame) { - PushFront(currentFrame); - } StartBuffering(); // Don't go straight back to the state machine loop since that might // cause us to start decoding again and we could flip-flop between @@ -2891,7 +2660,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() // advance the clock to after the media end time. if (mVideoFrameEndTime != -1 || mAudioEndTime != -1) { // These will be non -1 if we've displayed a video frame, or played an audio frame. - int64_t t = std::min(clock_time, std::max(mVideoFrameEndTime, mAudioEndTime)); + int64_t t = std::min(clockTime, std::max(mVideoFrameEndTime, mAudioEndTime)); if (t > GetMediaTime()) { UpdatePlaybackPosition(t); } @@ -2903,7 +2672,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() if (currentFrame) { // Decode one frame and display it. - int64_t delta = currentFrame->mTime - clock_time; + int64_t delta = currentFrame->mTime - clockTime; TimeStamp presTime = nowTime + TimeDuration::FromMicroseconds(delta / mPlaybackRate); NS_ASSERTION(currentFrame->mTime >= 0, "Should have positive frame time"); { @@ -2915,32 +2684,12 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() MOZ_ASSERT(IsPlaying()); MediaDecoder::FrameStatistics& frameStats = mDecoder->GetFrameStatistics(); frameStats.NotifyPresentedFrame(); - remainingTime = currentFrame->GetEndTime() - clock_time; + remainingTime = currentFrame->GetEndTime() - clockTime; currentFrame = nullptr; } - // The remainingTime is negative (include zero): - // 1. When the clock_time is larger than the latest video frame's endtime. - // All the video frames should be rendered or dropped, nothing left in - // VideoQueue. And since the VideoQueue is empty, we don't need to wake up - // statemachine thread immediately, so set the remainingTime to default value. - // 2. Current frame's endtime is smaller than clock_time but there still exist - // newer frames in queue. Re-calculate the remainingTime. - if (remainingTime <= 0) { - VideoData* nextFrame = VideoQueue().PeekFront(); - if (nextFrame) { - remainingTime = nextFrame->mTime - clock_time; - } else { - remainingTime = AUDIO_DURATION_USECS; - } - } - - int64_t delay = remainingTime / mPlaybackRate; - if (delay > 0) { - ScheduleStateMachineIn(delay); - } else { - ScheduleStateMachine(); - } + int64_t delay = std::max(1, remainingTime / mPlaybackRate); + ScheduleStateMachineIn(delay); } nsresult @@ -3134,14 +2883,14 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp) MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); mPlayStartTime = aTimeStamp; - if (!mAudioSink) { - return; - } - if (!mPlayStartTime.IsNull()) { - mAudioSink->StartPlayback(); - } else { - mAudioSink->StopPlayback(); + + if (mAudioSink) { + mAudioSink->SetPlaying(!mPlayStartTime.IsNull()); } + // Have DecodedStream remember the playing state so it doesn't need to + // ask MDSM about IsPlaying(). Note we have to do this even before capture + // happens since capture could happen in the middle of playback. + mDecodedStream->SetPlaying(!mPlayStartTime.IsNull()); } void MediaDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder() @@ -3223,14 +2972,17 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged() if (!HasAudio() && IsPlaying()) { // Remember how much time we've spent in playing the media // for playback rate will change from now on. - mPlayDuration = GetVideoStreamPosition(); - SetPlayStartTime(TimeStamp::Now()); + TimeStamp now = TimeStamp::Now(); + mPlayDuration = GetVideoStreamPosition(now); + SetPlayStartTime(now); } mPlaybackRate = mLogicalPlaybackRate; if (mAudioSink) { mAudioSink->SetPlaybackRate(mPlaybackRate); } + + ScheduleStateMachine(); } void MediaDecoderStateMachine::PreservesPitchChanged() @@ -3312,10 +3064,13 @@ void MediaDecoderStateMachine::OnAudioSinkError() DecodeError(); } -DecodedStreamData* MediaDecoderStateMachine::GetDecodedStream() const + +uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { AssertCurrentThreadInMonitor(); - return mDecodedStream.GetData(); + return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated()) + ? std::max(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE) + : std::max(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); } void MediaDecoderStateMachine::DispatchAudioCaptured() @@ -3328,9 +3083,6 @@ void MediaDecoderStateMachine::DispatchAudioCaptured() if (!self->mAudioCaptured) { // Stop the audio sink if it's running. self->StopAudioThread(); - // GetMediaTime() could return -1 because we haven't decoded - // the 1st frame. But this is OK since we will update mStreamStartTime - // again in SetStartTime(). self->mStreamStartTime = self->GetMediaTime(); // Reset mAudioEndTime which will be updated as we send audio data to // stream. Otherwise it will remain -1 if we don't have audio. @@ -3347,63 +3099,10 @@ void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream, { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("AddOutputStream aStream=%p!", aStream); - - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - if (!GetDecodedStream()) { - RecreateDecodedStream(aStream->Graph()); - } - mDecodedStream.Connect(aStream, aFinishWhenEnded); + mDecodedStream->Connect(aStream, aFinishWhenEnded); DispatchAudioCaptured(); } -void MediaDecoderStateMachine::UpdateStreamBlockingForPlayState() -{ - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - - auto stream = GetDecodedStream(); - if (!stream) { - return; - } - - bool blocking = mPlayState != MediaDecoder::PLAY_STATE_PLAYING || - mLogicallySeeking; - if (blocking != stream->mHaveBlockedForPlayState) { - stream->mHaveBlockedForPlayState = blocking; - UpdateStreamBlocking(stream->mStream, blocking); - } -} - -void MediaDecoderStateMachine::UpdateStreamBlockingForStateMachinePlaying() -{ - AssertCurrentThreadInMonitor(); - - auto stream = GetDecodedStream(); - if (!stream) { - return; - } - - bool blocking = !IsPlaying(); - if (blocking != stream->mHaveBlockedForStateMachineNotPlaying) { - stream->mHaveBlockedForStateMachineNotPlaying = blocking; - UpdateStreamBlocking(stream->mStream, blocking); - } -} - -void MediaDecoderStateMachine::RecreateDecodedStream(MediaStreamGraph* aGraph) -{ - MOZ_ASSERT(NS_IsMainThread()); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecodedStream.RecreateData(aGraph); -} - -uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const -{ - AssertCurrentThreadInMonitor(); - return (mReader->IsAsync() && mReader->VideoIsHardwareAccelerated()) - ? std::max(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE) - : std::max(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); -} - } // namespace mozilla // avoid redefined macro in unified build diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index a305a34512..71690bfaf9 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -145,8 +145,6 @@ public: DECODER_STATE_ERROR }; - DecodedStreamData* GetDecodedStream() const; - void AddOutputStream(ProcessedMediaStream* aStream, bool aFinishWhenEnded); // Set/Unset dormant state. @@ -160,18 +158,6 @@ private: void DispatchAudioCaptured(); - // Update blocking state of mDecodedStream when mPlayState or - // mLogicallySeeking change. Decoder monitor must be held. - void UpdateStreamBlockingForPlayState(); - - // Call this IsPlaying() changes. Decoder monitor must be held. - void UpdateStreamBlockingForStateMachinePlaying(); - - // Recreates mDecodedStream. Call this to create mDecodedStream at first, - // and when seeking, to ensure a new stream is set up with fresh buffers. - // Decoder monitor must be held. - void RecreateDecodedStream(MediaStreamGraph* aGraph); - void Shutdown(); public: @@ -317,10 +303,7 @@ public: if (mReader) { mReader->BreakCycles(); } - { - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecodedStream.DestroyData(); - } + mDecodedStream->DestroyData(); mDecoder = nullptr; } @@ -433,9 +416,7 @@ protected: bool OutOfDecodedVideo() { MOZ_ASSERT(OnTaskQueue()); - // In buffering mode, we keep the last already-played frame in the queue. - int emptyVideoSize = mState == DECODER_STATE_BUFFERING ? 1 : 0; - return IsVideoDecoding() && !VideoQueue().IsFinished() && VideoQueue().GetSize() <= emptyVideoSize; + return IsVideoDecoding() && !VideoQueue().IsFinished() && VideoQueue().GetSize() <= 1; } @@ -476,13 +457,15 @@ protected: // Get the video stream position, taking the |playbackRate| change into // account. This is a position in the media, not the duration of the playback - // so far. - int64_t GetVideoStreamPosition() const; + // so far. Returns the position for the given time aTimeStamp. + int64_t GetVideoStreamPosition(TimeStamp aTimeStamp) const; // Return the current time, either the audio clock if available (if the media // has audio, and the playback is possible), or a clock for the video. // Called on the state machine thread. - int64_t GetClock() const; + // If aTimeStamp is non-null, set *aTimeStamp to the TimeStamp corresponding + // to the returned stream time. + int64_t GetClock(TimeStamp* aTimeStamp = nullptr) const; nsresult DropAudioUpToSeekTarget(AudioData* aSample); nsresult DropVideoUpToSeekTarget(VideoData* aSample); @@ -635,11 +618,6 @@ protected: // The decoder monitor must be held. void CheckIfDecodeComplete(); - // Copy audio from an AudioData packet to aOutput. This may require - // inserting silence depending on the timing of the audio packet. - void SendStreamAudio(AudioData* aAudio, DecodedStreamData* aStream, - AudioSegment* aOutput); - // Performs one "cycle" of the state machine. Polls the state, and may send // a video frame to be displayed, and generally manages the decode. Called // periodically via timer to ensure the video stays in sync. @@ -1162,7 +1140,8 @@ protected: { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - return !IsAudioDecoding() || GetDecodedAudioDuration() >= AudioPrerollUsecs() * mPlaybackRate; + return !IsAudioDecoding() || + GetDecodedAudioDuration() >= AudioPrerollUsecs() * mPlaybackRate; } bool DonePrerollingVideo() @@ -1170,7 +1149,8 @@ protected: MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); return !IsVideoDecoding() || - static_cast(VideoQueue().GetSize()) >= VideoPrerollFrames() * mPlaybackRate; + static_cast(VideoQueue().GetSize()) >= + VideoPrerollFrames() * mPlaybackRate + 1; } void StopPrerollingAudio() @@ -1360,7 +1340,7 @@ protected: // Only written on the main thread while holding the monitor. Therefore it // can be read on any thread while holding the monitor, or on the main thread // without holding the monitor. - DecodedStream mDecodedStream; + nsRefPtr mDecodedStream; }; } // namespace mozilla diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 68852760fe..43e822aa59 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -465,7 +465,8 @@ MediaFormatReader::ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThr nsRefPtr MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) + int64_t aTimeThreshold, + bool aForceDecodeAhead) { MOZ_ASSERT(OnTaskQueue()); MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking"); @@ -498,6 +499,7 @@ MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe, MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder); + mVideo.mForceDecodeAhead = aForceDecodeAhead; media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)}; if (ShouldSkip(aSkipToNextKeyframe, timeThreshold)) { Flush(TrackInfo::kVideoTrack); @@ -698,11 +700,12 @@ MediaFormatReader::NeedInput(DecoderData& aDecoder) return !aDecoder.mDraining && !aDecoder.mError && - aDecoder.HasPromise() && + (aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) && !aDecoder.mDemuxRequest.Exists() && aDecoder.mOutput.IsEmpty() && (aDecoder.mInputExhausted || !aDecoder.mQueuedSamples.IsEmpty() || aDecoder.mTimeThreshold.isSome() || + aDecoder.mForceDecodeAhead || aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead); } @@ -963,18 +966,25 @@ MediaFormatReader::Update(TrackType aTrack) needOutput = true; if (!decoder.mOutput.IsEmpty()) { // We have a decoded sample ready to be returned. - nsRefPtr output = decoder.mOutput[0]; - decoder.mOutput.RemoveElementAt(0); - decoder.mSizeOfQueue -= 1; - if (decoder.mTimeThreshold.isNothing() || - media::TimeUnit::FromMicroseconds(output->mTime) >= decoder.mTimeThreshold.ref()) { - ReturnOutput(output, aTrack); - decoder.mTimeThreshold.reset(); - } else { - LOGV("Internal Seeking: Dropping frame time:%f wanted:%f (kf:%d)", - media::TimeUnit::FromMicroseconds(output->mTime).ToSeconds(), - decoder.mTimeThreshold.ref().ToSeconds(), - output->mKeyframe); + if (aTrack == TrackType::kVideoTrack) { + mVideo.mIsHardwareAccelerated = + mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(); + } + while (decoder.mOutput.Length()) { + nsRefPtr output = decoder.mOutput[0]; + decoder.mOutput.RemoveElementAt(0); + decoder.mSizeOfQueue -= 1; + if (decoder.mTimeThreshold.isNothing() || + media::TimeUnit::FromMicroseconds(output->mTime) >= decoder.mTimeThreshold.ref()) { + ReturnOutput(output, aTrack); + decoder.mTimeThreshold.reset(); + break; + } else { + LOGV("Internal Seeking: Dropping frame time:%f wanted:%f (kf:%d)", + media::TimeUnit::FromMicroseconds(output->mTime).ToSeconds(), + decoder.mTimeThreshold.ref().ToSeconds(), + output->mKeyframe); + } } } else if (decoder.mDrainComplete) { decoder.mDrainComplete = false; @@ -1377,8 +1387,8 @@ MediaFormatReader::GetBuffered() int64_t startTime; { ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); - startTime = mStartTime; + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); + startTime = StartTime(); } // Ensure we have up to date buffered time range. if (HasVideo()) { @@ -1435,6 +1445,12 @@ MediaFormatReader::SetSharedDecoderManager(SharedDecoderManager* aManager) #endif } +bool +MediaFormatReader::VideoIsHardwareAccelerated() const +{ + return mVideo.mIsHardwareAccelerated; +} + void MediaFormatReader::NotifyDemuxer(uint32_t aLength, int64_t aOffset) { diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 3c6cb1c4cc..cd2043eee9 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -34,7 +34,7 @@ public: size_t SizeOfAudioQueueInFrames() override; nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override; + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override; nsRefPtr RequestAudioData() override; @@ -82,6 +82,8 @@ public: bool IsAsync() const override { return true; } + bool VideoIsHardwareAccelerated() const override; + void DisableHardwareAcceleration() override; bool IsWaitForDataSupported() override { return true; } @@ -182,6 +184,7 @@ private: : mOwner(aOwner) , mType(aType) , mDecodeAhead(aDecodeAhead) + , mForceDecodeAhead(false) , mUpdateScheduled(false) , mDemuxEOS(false) , mWaitingForData(false) @@ -196,6 +199,7 @@ private: , mNumSamplesInput(0) , mNumSamplesOutput(0) , mSizeOfQueue(0) + , mIsHardwareAccelerated(false) , mLastStreamSourceID(UINT32_MAX) {} @@ -213,6 +217,7 @@ private: // Only accessed from reader's task queue. uint32_t mDecodeAhead; + bool mForceDecodeAhead; bool mUpdateScheduled; bool mDemuxEOS; bool mWaitingForData; @@ -266,6 +271,7 @@ private: void ResetState() { MOZ_ASSERT(mOwner->OnTaskQueue()); + mForceDecodeAhead = false; mDemuxEOS = false; mWaitingForData = false; mReceivedNewData = false; @@ -285,6 +291,9 @@ private: // Used by the MDSM for logging purposes. Atomic mSizeOfQueue; + // Used by the MDSM to determine if video decoding is hardware accelerated. + // This value is updated after a frame is successfully decoded. + Atomic mIsHardwareAccelerated; // Sample format monitoring. uint32_t mLastStreamSourceID; Maybe mNextStreamSourceID; diff --git a/dom/media/MediaPromise.h b/dom/media/MediaPromise.h index ded986659e..dc76925409 100644 --- a/dom/media/MediaPromise.h +++ b/dom/media/MediaPromise.h @@ -911,6 +911,22 @@ protected: Arg2Type mArg2; }; +template +class MethodCallWithThreeArgs : public MethodCallBase +{ +public: + typedef nsRefPtr(ThisType::*Type)(Arg1Type, Arg2Type, Arg3Type); + MethodCallWithThreeArgs(ThisType* aThisVal, Type aMethod, Arg1Type aArg1, Arg2Type aArg2, Arg3Type aArg3) + : mThisVal(aThisVal), mMethod(aMethod), mArg1(aArg1), mArg2(aArg2), mArg3(aArg3) {} + nsRefPtr Invoke() override { return ((*mThisVal).*mMethod)(mArg1, mArg2, mArg3); } +protected: + nsRefPtr mThisVal; + Type mMethod; + Arg1Type mArg1; + Arg2Type mArg2; + Arg3Type mArg3; +}; + template class ProxyRunnable : public nsRunnable { @@ -974,6 +990,16 @@ ProxyMediaCall(AbstractThread* aTarget, ThisType* aThisVal, const char* aCallerN return detail::ProxyInternal(aTarget, methodCall, aCallerName); } +template +static nsRefPtr +ProxyMediaCall(AbstractThread* aTarget, ThisType* aThisVal, const char* aCallerName, + nsRefPtr(ThisType::*aMethod)(Arg1Type, Arg2Type, Arg3Type), Arg1Type aArg1, Arg2Type aArg2, Arg3Type aArg3) +{ + typedef detail::MethodCallWithThreeArgs MethodCallType; + MethodCallType* methodCall = new MethodCallType(aThisVal, aMethod, aArg1, aArg2, aArg3); + return detail::ProxyInternal(aTarget, methodCall, aCallerName); +} + #undef PROMISE_LOG } // namespace mozilla diff --git a/dom/media/VideoFrameContainer.cpp b/dom/media/VideoFrameContainer.cpp index 4a722d462a..774e7739d6 100644 --- a/dom/media/VideoFrameContainer.cpp +++ b/dom/media/VideoFrameContainer.cpp @@ -41,10 +41,6 @@ void VideoFrameContainer::SetCurrentFrame(const gfxIntSize& aIntrinsicSize, } gfx::IntSize oldFrameSize = mImageContainer->GetCurrentSize(); - TimeStamp lastPaintTime = mImageContainer->GetPaintTime(); - if (!lastPaintTime.IsNull() && !mPaintTarget.IsNull()) { - mPaintDelay = lastPaintTime - mPaintTarget; - } // When using the OMX decoder, destruction of the current image can indirectly // block on main thread I/O. If we let this happen while holding onto @@ -55,13 +51,18 @@ void VideoFrameContainer::SetCurrentFrame(const gfxIntSize& aIntrinsicSize, nsTArray kungFuDeathGrip; mImageContainer->GetCurrentImages(&kungFuDeathGrip); - mImageContainer->SetCurrentImage(aImage); + if (aImage) { + nsAutoTArray imageList; + imageList.AppendElement( + ImageContainer::NonOwningImage(aImage, aTargetTime)); + mImageContainer->SetCurrentImages(imageList); + } else { + mImageContainer->ClearAllImages(); + } gfx::IntSize newFrameSize = mImageContainer->GetCurrentSize(); if (oldFrameSize != newFrameSize) { mImageSizeChanged = true; } - - mPaintTarget = aTargetTime; } void VideoFrameContainer::ClearCurrentFrame() @@ -84,8 +85,7 @@ ImageContainer* VideoFrameContainer::GetImageContainer() { double VideoFrameContainer::GetFrameDelay() { - MutexAutoLock lock(mMutex); - return mPaintDelay.ToSeconds(); + return mImageContainer->GetPaintDelay().ToSeconds(); } void VideoFrameContainer::InvalidateWithFlags(uint32_t aFlags) diff --git a/dom/media/VideoFrameContainer.h b/dom/media/VideoFrameContainer.h index 310d64118c..a8cecd8c04 100644 --- a/dom/media/VideoFrameContainer.h +++ b/dom/media/VideoFrameContainer.h @@ -77,13 +77,6 @@ protected: // specifies that the Image should be stretched to have the correct aspect // ratio. gfxIntSize mIntrinsicSize; - // The time at which the current video frame should have been painted. - // Access protected by mVideoUpdateLock. - TimeStamp mPaintTarget; - // The delay between the last video frame being presented and it being - // painted. This is time elapsed after mPaintTarget until the most recently - // painted frame appeared on screen. - TimeDuration mPaintDelay; // True when the intrinsic size has been changed by SetCurrentFrame() since // the last call to Invalidate(). // The next call to Invalidate() will recalculate diff --git a/dom/media/fmp4/MP4Decoder.cpp b/dom/media/fmp4/MP4Decoder.cpp index c584b0e730..d3548fb1ae 100644 --- a/dom/media/fmp4/MP4Decoder.cpp +++ b/dom/media/fmp4/MP4Decoder.cpp @@ -29,6 +29,7 @@ #ifdef MOZ_FFMPEG #include "FFmpegRuntimeLinker.h" #endif +#include "mozilla/layers/LayersTypes.h" namespace mozilla { @@ -66,11 +67,6 @@ IsWhitelistedH264Codec(const nsAString& aCodec) } #if 0 - if (!Preferences::GetBool("media.use-blank-decoder") && - !WMFDecoderModule::HasH264()) { - return false; - } - // Disable 4k video on windows vista since it performs poorly. if (!IsWin7OrLater() && level >= H264_LEVEL_5) { @@ -263,5 +259,133 @@ MP4Decoder::IsEnabled() HavePlatformMPEGDecoders(); } -} // namespace mozilla +static const uint8_t sTestH264ExtraData[] = { + 0x01, 0x64, 0x00, 0x0a, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x64, + 0x00, 0x0a, 0xac, 0xd9, 0x44, 0x26, 0x84, 0x00, 0x00, 0x03, + 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x3c, 0x48, 0x96, + 0x58, 0x01, 0x00, 0x06, 0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0 +}; +static already_AddRefed +CreateTestH264Decoder(layers::LayersBackend aBackend, + VideoInfo& aConfig) +{ + aConfig.mMimeType = "video/avc"; + aConfig.mId = 1; + aConfig.mDuration = 40000; + aConfig.mMediaTime = 0; + aConfig.mDisplay = aConfig.mImage = nsIntSize(64, 64); + aConfig.mExtraData = new MediaByteBuffer(); + aConfig.mExtraData->AppendElements(sTestH264ExtraData, + MOZ_ARRAY_LENGTH(sTestH264ExtraData)); + + PlatformDecoderModule::Init(); + + nsRefPtr platform = PlatformDecoderModule::Create(); + if (!platform) { + return nullptr; + } + + nsRefPtr decoder( + platform->CreateDecoder(aConfig, nullptr, nullptr, aBackend, nullptr)); + if (!decoder) { + return nullptr; + } + nsresult rv = decoder->Init(); + NS_ENSURE_SUCCESS(rv, nullptr); + + return decoder.forget(); +} + +/* static */ bool +MP4Decoder::IsVideoAccelerated(layers::LayersBackend aBackend) +{ + VideoInfo config; + nsRefPtr decoder(CreateTestH264Decoder(aBackend, config)); + if (!decoder) { + return false; + } + bool result = decoder->IsHardwareAccelerated(); + decoder->Shutdown(); + return result; +} + +/* static */ bool +MP4Decoder::CanCreateH264Decoder() +{ + static bool haveCachedResult = false; + static bool result = false; + if (haveCachedResult) { + return result; + } + VideoInfo config; + nsRefPtr decoder( + CreateTestH264Decoder(layers::LayersBackend::LAYERS_BASIC, config)); + if (decoder) { + decoder->Shutdown(); + result = true; + } + haveCachedResult = true; + return result; +} + +static already_AddRefed +CreateTestAACDecoder(AudioInfo& aConfig) +{ + PlatformDecoderModule::Init(); + + nsRefPtr platform = PlatformDecoderModule::Create(); + if (!platform) { + return nullptr; + } + + nsRefPtr decoder( + platform->CreateDecoder(aConfig, nullptr, nullptr)); + if (!decoder) { + return nullptr; + } + nsresult rv = decoder->Init(); + NS_ENSURE_SUCCESS(rv, nullptr); + + return decoder.forget(); +} + +// bipbop.mp4's extradata/config... +static const uint8_t sTestAACExtraData[] = { + 0x03, 0x80, 0x80, 0x80, 0x22, 0x00, 0x02, 0x00, 0x04, 0x80, + 0x80, 0x80, 0x14, 0x40, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x51, 0x00, 0x00, 0x11, 0x51, 0x05, 0x80, 0x80, 0x80, + 0x02, 0x13, 0x90, 0x06, 0x80, 0x80, 0x80, 0x01, 0x02 +}; + +static const uint8_t sTestAACConfig[] = { 0x13, 0x90 }; + +/* static */ bool +MP4Decoder::CanCreateAACDecoder() +{ + static bool haveCachedResult = false; + static bool result = false; + if (haveCachedResult) { + return result; + } + AudioInfo config; + config.mMimeType = "audio/mp4a-latm"; + config.mRate = 22050; + config.mChannels = 2; + config.mBitDepth = 16; + config.mProfile = 2; + config.mExtendedProfile = 2; + config.mCodecSpecificConfig->AppendElements(sTestAACConfig, + MOZ_ARRAY_LENGTH(sTestAACConfig)); + config.mExtraData->AppendElements(sTestAACExtraData, + MOZ_ARRAY_LENGTH(sTestAACExtraData)); + nsRefPtr decoder(CreateTestAACDecoder(config)); + if (decoder) { + decoder->Shutdown(); + result = true; + } + haveCachedResult = true; + return result; +} + +} // namespace mozilla diff --git a/dom/media/fmp4/MP4Decoder.h b/dom/media/fmp4/MP4Decoder.h index a4bcd14d1a..3b8243c4fb 100644 --- a/dom/media/fmp4/MP4Decoder.h +++ b/dom/media/fmp4/MP4Decoder.h @@ -36,6 +36,10 @@ public: // Returns true if the MP4 backend is preffed on. static bool IsEnabled(); + + static bool IsVideoAccelerated(layers::LayersBackend aBackend); + static bool CanCreateAACDecoder(); + static bool CanCreateH264Decoder(); }; } // namespace mozilla diff --git a/dom/media/fmp4/MP4Reader.cpp b/dom/media/fmp4/MP4Reader.cpp index 38e9bb0a8b..627d1d5bde 100644 --- a/dom/media/fmp4/MP4Reader.cpp +++ b/dom/media/fmp4/MP4Reader.cpp @@ -61,43 +61,6 @@ TrackTypeToStr(TrackInfo::TrackType aTrack) } } -uint8_t sTestExtraData[40] = { 0x01, 0x64, 0x00, 0x0a, 0xff, 0xe1, 0x00, 0x17, 0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x44, 0x26, 0x84, 0x00, 0x00, 0x03, - 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x3c, 0x48, 0x96, 0x58, 0x01, 0x00, 0x06, 0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0 }; - -/* static */ bool -MP4Reader::IsVideoAccelerated(LayersBackend aBackend) -{ - VideoInfo config; - config.mMimeType = "video/avc"; - config.mId = 1; - config.mDuration = 40000; - config.mMediaTime = 0; - config.mDisplay = config.mImage = nsIntSize(64, 64); - config.mExtraData = new MediaByteBuffer(); - config.mExtraData->AppendElements(sTestExtraData, 40); - - PlatformDecoderModule::Init(); - - nsRefPtr platform = PlatformDecoderModule::Create(); - if (!platform) { - return false; - } - - nsRefPtr decoder = - platform->CreateDecoder(config, nullptr, nullptr, aBackend, nullptr); - if (!decoder) { - return false; - } - nsresult rv = decoder->Init(); - NS_ENSURE_SUCCESS(rv, false); - - bool result = decoder->IsHardwareAccelerated(); - - decoder->Shutdown(); - - return result; -} - // MP4Demuxer wants to do various blocking reads, which cause deadlocks while // mDemuxerMonitor is held. This stuff should really be redesigned, but we don't // have time for that right now. So in order to get proper synchronization while @@ -538,7 +501,8 @@ MP4Reader::ShouldSkip(bool aSkipToNextKeyframe, int64_t aTimeThreshold) nsRefPtr MP4Reader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) + int64_t aTimeThreshold, + bool aForceDecodeAhead) { MOZ_ASSERT(OnTaskQueue()); VLOG("skip=%d time=%lld", aSkipToNextKeyframe, aTimeThreshold); @@ -556,6 +520,7 @@ MP4Reader::RequestVideoData(bool aSkipToNextKeyframe, MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder); bool eos = false; + mVideo.mForceDecodeAhead = aForceDecodeAhead; if (ShouldSkip(aSkipToNextKeyframe, aTimeThreshold)) { uint32_t parsed = 0; eos = !SkipVideoDemuxToNextKeyFrame(aTimeThreshold, parsed); @@ -627,9 +592,10 @@ MP4Reader::NeedInput(DecoderData& aDecoder) return !aDecoder.mError && !aDecoder.mDemuxEOS && - aDecoder.HasPromise() && + (aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) && aDecoder.mOutput.IsEmpty() && (aDecoder.mInputExhausted || + aDecoder.mForceDecodeAhead || aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead); } @@ -891,12 +857,13 @@ MP4Reader::Flush(TrackType aTrack) MonitorAutoLock mon(data.mMonitor); data.mIsFlushing = true; data.mDemuxEOS = false; - data.mDrainComplete = false; } data.mDecoder->Flush(); { MonitorAutoLock mon(data.mMonitor); + data.mForceDecodeAhead = false; data.mIsFlushing = false; + data.mDrainComplete = false; data.mOutput.Clear(); data.mNumSamplesInput = 0; data.mNumSamplesOutput = 0; @@ -1005,7 +972,7 @@ MP4Reader::GetBuffered() return buffered; } UpdateIndex(); - NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); AutoPinned resource(mDecoder->GetResource()); nsTArray ranges; @@ -1016,8 +983,8 @@ MP4Reader::GetBuffered() mDemuxer->ConvertByteRangesToTime(ranges, &timeRanges); for (size_t i = 0; i < timeRanges.Length(); i++) { buffered += media::TimeInterval( - media::TimeUnit::FromMicroseconds(timeRanges[i].start - mStartTime), - media::TimeUnit::FromMicroseconds(timeRanges[i].end - mStartTime)); + media::TimeUnit::FromMicroseconds(timeRanges[i].start - StartTime()), + media::TimeUnit::FromMicroseconds(timeRanges[i].end - StartTime())); } } diff --git a/dom/media/fmp4/MP4Reader.h b/dom/media/fmp4/MP4Reader.h index 6ec100d403..f8b0836d67 100644 --- a/dom/media/fmp4/MP4Reader.h +++ b/dom/media/fmp4/MP4Reader.h @@ -38,7 +38,7 @@ public: virtual size_t SizeOfAudioQueueInFrames() override; virtual nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override; + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override; virtual nsRefPtr RequestAudioData() override; @@ -77,8 +77,6 @@ public: virtual void DisableHardwareAcceleration() override; - static bool IsVideoAccelerated(layers::LayersBackend aBackend); - virtual bool VideoIsHardwareAccelerated() const override; private: @@ -168,6 +166,7 @@ private: , mNumSamplesInput(0) , mNumSamplesOutput(0) , mDecodeAhead(aDecodeAhead) + , mForceDecodeAhead(false) , mActive(false) , mInputExhausted(false) , mError(false) @@ -204,6 +203,7 @@ private: uint64_t mNumSamplesInput; uint64_t mNumSamplesOutput; uint32_t mDecodeAhead; + bool mForceDecodeAhead; // Whether this stream exists in the media. bool mActive; bool mInputExhausted; diff --git a/dom/media/mediasource/MediaSourceReader.cpp b/dom/media/mediasource/MediaSourceReader.cpp index 7d2ef844b7..302d6e9f42 100644 --- a/dom/media/mediasource/MediaSourceReader.cpp +++ b/dom/media/mediasource/MediaSourceReader.cpp @@ -45,6 +45,7 @@ MediaSourceReader::MediaSourceReader(MediaSourceDecoder* aDecoder) : MediaDecoderReader(aDecoder) , mLastAudioTime(0) , mLastVideoTime(0) + , mForceVideoDecodeAhead(false) , mOriginalSeekTime(-1) , mPendingSeekTime(-1) , mWaitingForSeekData(false) @@ -284,7 +285,9 @@ MediaSourceReader::OnAudioNotDecoded(NotDecodedReason aReason) } nsRefPtr -MediaSourceReader::RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) +MediaSourceReader::RequestVideoData(bool aSkipToNextKeyframe, + int64_t aTimeThreshold, + bool aForceDecodeAhead) { MOZ_ASSERT(OnTaskQueue()); MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking"); @@ -308,6 +311,7 @@ MediaSourceReader::RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThres return p; } MOZ_DIAGNOSTIC_ASSERT(!mVideoSeekRequest.Exists()); + mForceVideoDecodeAhead = aForceDecodeAhead; SwitchSourceResult ret = SwitchVideoSource(&mLastVideoTime); switch (ret) { @@ -340,7 +344,9 @@ MediaSourceReader::RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThres void MediaSourceReader::DoVideoRequest() { - mVideoRequest.Begin(GetVideoReader()->RequestVideoData(mDropVideoBeforeThreshold, GetReaderVideoTime(mTimeThreshold)) + mVideoRequest.Begin(GetVideoReader()->RequestVideoData(mDropVideoBeforeThreshold, + GetReaderVideoTime(mTimeThreshold), + mForceVideoDecodeAhead) ->Then(TaskQueue(), __func__, this, &MediaSourceReader::OnVideoDecoded, &MediaSourceReader::OnVideoNotDecoded)); @@ -860,6 +866,9 @@ MediaSourceReader::ResetDecode() mWaitingForSeekData = false; mPendingSeekTime = -1; + // Reset force video decode ahead. + mForceVideoDecodeAhead = false; + // Reset all the readers. if (GetAudioReader()) { GetAudioReader()->ResetDecode(); diff --git a/dom/media/mediasource/MediaSourceReader.h b/dom/media/mediasource/MediaSourceReader.h index 72e448c618..f0fcc06413 100644 --- a/dom/media/mediasource/MediaSourceReader.h +++ b/dom/media/mediasource/MediaSourceReader.h @@ -49,7 +49,7 @@ public: nsRefPtr RequestAudioData() override; nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override; + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override; virtual size_t SizeOfVideoQueueInFrames() override; virtual size_t SizeOfAudioQueueInFrames() override; @@ -248,6 +248,8 @@ private: int64_t mLastAudioTime; int64_t mLastVideoTime; + bool mForceVideoDecodeAhead; + MediaPromiseRequestHolder mAudioSeekRequest; MediaPromiseRequestHolder mVideoSeekRequest; MediaPromiseHolder mSeekPromise; diff --git a/dom/media/mediasource/TrackBuffer.cpp b/dom/media/mediasource/TrackBuffer.cpp index 8f80f74342..577bab2462 100644 --- a/dom/media/mediasource/TrackBuffer.cpp +++ b/dom/media/mediasource/TrackBuffer.cpp @@ -205,6 +205,8 @@ TrackBuffer::BufferAppend() decoders.AppendElement(mCurrentDecoder); } + mLastAppendRange = Interval(); + if (gotMedia) { if (mParser->IsMediaSegmentPresent(mInputBuffer) && mLastEndTimestamp && (!mParser->TimestampsFuzzyEqual(start, mLastEndTimestamp.value()) || @@ -225,6 +227,7 @@ TrackBuffer::BufferAppend() } MSE_DEBUG("Decoder marked as initialized."); AppendDataToCurrentResource(oldInit, 0); + mLastAppendRange = Interval(0, int64_t(oldInit->Length())); } mLastStartTimestamp = start; } else { @@ -251,8 +254,10 @@ TrackBuffer::BufferAppend() return p; } - mLastAppendRange = - Interval(offset, offset + int64_t(mInputBuffer->Length())); + mLastAppendRange = mLastAppendRange.IsEmpty() + ? Interval(offset, offset + int64_t(mInputBuffer->Length())) + : mLastAppendRange.Span( + Interval(offset, offset + int64_t(mInputBuffer->Length()))); if (decoders.Length()) { // We're going to have to wait for the decoder to initialize, the promise diff --git a/dom/media/ogg/OggReader.cpp b/dom/media/ogg/OggReader.cpp index 13675577c5..cb1b470d22 100644 --- a/dom/media/ogg/OggReader.cpp +++ b/dom/media/ogg/OggReader.cpp @@ -1428,6 +1428,7 @@ OggReader::Seek(int64_t aTarget, int64_t aEndTime) nsresult OggReader::SeekInternal(int64_t aTarget, int64_t aEndTime) { MOZ_ASSERT(OnTaskQueue()); + NS_ENSURE_TRUE(HaveStartTime(), NS_ERROR_FAILURE); if (mIsChained) return NS_ERROR_FAILURE; LOG(LogLevel::Debug, ("%p About to seek to %lld", mDecoder, aTarget)); @@ -1436,10 +1437,10 @@ nsresult OggReader::SeekInternal(int64_t aTarget, int64_t aEndTime) NS_ENSURE_TRUE(resource != nullptr, NS_ERROR_FAILURE); int64_t adjustedTarget = aTarget; if (HasAudio() && mOpusState){ - adjustedTarget = std::max(mStartTime, aTarget - SEEK_OPUS_PREROLL); + adjustedTarget = std::max(StartTime(), aTarget - SEEK_OPUS_PREROLL); } - if (adjustedTarget == mStartTime) { + if (adjustedTarget == StartTime()) { // We've seeked to the media start. Just seek to the offset of the first // content page. res = resource->Seek(nsISeekableStream::NS_SEEK_SET, 0); @@ -1462,18 +1463,18 @@ nsresult OggReader::SeekInternal(int64_t aTarget, int64_t aEndTime) NS_ENSURE_SUCCESS(res,res); // Figure out if the seek target lies in a buffered range. - SeekRange r = SelectSeekRange(ranges, aTarget, mStartTime, aEndTime, true); + SeekRange r = SelectSeekRange(ranges, aTarget, StartTime(), aEndTime, true); if (!r.IsNull()) { // We know the buffered range in which the seek target lies, do a // bisection search in that buffered range. - res = SeekInBufferedRange(aTarget, adjustedTarget, mStartTime, aEndTime, ranges, r); + res = SeekInBufferedRange(aTarget, adjustedTarget, StartTime(), aEndTime, ranges, r); NS_ENSURE_SUCCESS(res,res); } else { // The target doesn't lie in a buffered range. Perform a bisection // search over the whole media, using the known buffered ranges to // reduce the search space. - res = SeekInUnbuffered(aTarget, mStartTime, aEndTime, ranges); + res = SeekInUnbuffered(aTarget, StartTime(), aEndTime, ranges); NS_ENSURE_SUCCESS(res,res); } } @@ -1841,7 +1842,7 @@ nsresult OggReader::SeekBisection(int64_t aTarget, media::TimeIntervals OggReader::GetBuffered() { MOZ_ASSERT(OnTaskQueue()); - NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); { mozilla::ReentrantMonitorAutoEnter mon(mMonitor); if (mIsChained) { @@ -1880,7 +1881,7 @@ media::TimeIntervals OggReader::GetBuffered() // we special-case (startOffset == 0) so that the first // buffered range always appears to be buffered from the media start // time, rather than from the end-time of the first page. - int64_t startTime = (startOffset == 0) ? mStartTime : -1; + int64_t startTime = (startOffset == 0) ? StartTime() : -1; // Find the start time of the range. Read pages until we find one with a // granulepos which we can convert into a timestamp to use as the time of @@ -1947,8 +1948,8 @@ media::TimeIntervals OggReader::GetBuffered() int64_t endTime = RangeEndTime(startOffset, endOffset, true); if (endTime > startTime) { buffered += media::TimeInterval( - media::TimeUnit::FromMicroseconds(startTime - mStartTime), - media::TimeUnit::FromMicroseconds(endTime - mStartTime)); + media::TimeUnit::FromMicroseconds(startTime - StartTime()), + media::TimeUnit::FromMicroseconds(endTime - StartTime())); } } } diff --git a/dom/media/omx/MediaCodecReader.cpp b/dom/media/omx/MediaCodecReader.cpp index 8bc9ae9197..ad40899e77 100644 --- a/dom/media/omx/MediaCodecReader.cpp +++ b/dom/media/omx/MediaCodecReader.cpp @@ -353,7 +353,8 @@ MediaCodecReader::RequestAudioData() nsRefPtr MediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) + int64_t aTimeThreshold, + bool aForceDecodeAhead) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasVideo()); @@ -396,9 +397,6 @@ MediaCodecReader::DecodeAudioDataSync() } else if (status == -EAGAIN) { if (TimeStamp::Now() > timeout) { // Don't let this loop run for too long. Try it again later. - if (CheckAudioResources()) { - DispatchAudioTask(); - } return; } continue; // Try it again now. @@ -413,8 +411,11 @@ MediaCodecReader::DecodeAudioDataSync() } } - if (bufferInfo.mBuffer != nullptr && bufferInfo.mSize > 0 && - bufferInfo.mBuffer->data() != nullptr) { + if ((bufferInfo.mFlags & MediaCodec::BUFFER_FLAG_EOS) || + (status == ERROR_END_OF_STREAM)) { + AudioQueue().Finish(); + } else if (bufferInfo.mBuffer != nullptr && bufferInfo.mSize > 0 && + bufferInfo.mBuffer->data() != nullptr) { // This is the approximate byte position in the stream. int64_t pos = mDecoder->GetResource()->Tell(); @@ -432,23 +433,28 @@ MediaCodecReader::DecodeAudioDataSync() bufferInfo.mSize, mInfo.mAudio.mChannels)); } - - if ((bufferInfo.mFlags & MediaCodec::BUFFER_FLAG_EOS) || - (status == ERROR_END_OF_STREAM)) { - AudioQueue().Finish(); - } mAudioTrack.mCodec->releaseOutputBuffer(bufferInfo.mIndex); - } void MediaCodecReader::DecodeAudioDataTask() { - if (AudioQueue().GetSize() == 0 && !AudioQueue().IsFinished()) { - DecodeAudioDataSync(); - } + MOZ_ASSERT(mAudioTrack.mTaskQueue->IsCurrentThreadIn()); MonitorAutoLock al(mAudioTrack.mTrackMonitor); if (mAudioTrack.mAudioPromise.IsEmpty()) { + // Clear the data in queue because the promise might be canceled by + // ResetDecode(). + AudioQueue().Reset(); + return; + } + if (AudioQueue().GetSize() == 0 && !AudioQueue().IsFinished()) { + MonitorAutoUnlock ul(mAudioTrack.mTrackMonitor); + DecodeAudioDataSync(); + } + // Since we unlock the monitor above, we should check the promise again + // because the promise might be canceled by ResetDecode(). + if (mAudioTrack.mAudioPromise.IsEmpty()) { + AudioQueue().Reset(); return; } if (AudioQueue().GetSize() > 0) { @@ -462,14 +468,32 @@ MediaCodecReader::DecodeAudioDataTask() } } else if (AudioQueue().AtEndOfStream()) { mAudioTrack.mAudioPromise.Reject(END_OF_STREAM, __func__); + } else if (AudioQueue().GetSize() == 0) { + DispatchAudioTask(); } } void MediaCodecReader::DecodeVideoFrameTask(int64_t aTimeThreshold) { - DecodeVideoFrameSync(aTimeThreshold); + MOZ_ASSERT(mVideoTrack.mTaskQueue->IsCurrentThreadIn()); MonitorAutoLock al(mVideoTrack.mTrackMonitor); + if (mVideoTrack.mVideoPromise.IsEmpty()) { + // Clear the data in queue because the promise might be canceled by + // ResetDecode(). + VideoQueue().Reset(); + return; + } + { + MonitorAutoUnlock ul(mVideoTrack.mTrackMonitor); + DecodeVideoFrameSync(aTimeThreshold); + } + // Since we unlock the monitor above, we should check the promise again + // because the promise might be canceled by ResetDecode(). + if (mVideoTrack.mVideoPromise.IsEmpty()) { + VideoQueue().Reset(); + return; + } if (VideoQueue().GetSize() > 0) { nsRefPtr v = VideoQueue().PopFront(); if (v) { @@ -481,6 +505,8 @@ MediaCodecReader::DecodeVideoFrameTask(int64_t aTimeThreshold) } } else if (VideoQueue().AtEndOfStream()) { mVideoTrack.mVideoPromise.Reject(END_OF_STREAM, __func__); + } else if (VideoQueue().GetSize() == 0) { + DispatchVideoTask(aTimeThreshold); } } @@ -737,7 +763,6 @@ nsresult MediaCodecReader::ResetDecode() { if (CheckAudioResources()) { - mAudioTrack.mTaskQueue->Flush(); MonitorAutoLock al(mAudioTrack.mTrackMonitor); if (!mAudioTrack.mAudioPromise.IsEmpty()) { mAudioTrack.mAudioPromise.Reject(CANCELED, __func__); @@ -746,7 +771,6 @@ MediaCodecReader::ResetDecode() mAudioTrack.mDiscontinuity = true; } if (CheckVideoResources()) { - mVideoTrack.mTaskQueue->Flush(); MonitorAutoLock al(mVideoTrack.mTrackMonitor); if (!mVideoTrack.mVideoPromise.IsEmpty()) { mVideoTrack.mVideoPromise.Reject(CANCELED, __func__); @@ -890,9 +914,6 @@ MediaCodecReader::DecodeVideoFrameSync(int64_t aTimeThreshold) } else if (status == -EAGAIN) { if (TimeStamp::Now() > timeout) { // Don't let this loop run for too long. Try it again later. - if (CheckVideoResources()) { - DispatchVideoTask(aTimeThreshold); - } return; } continue; // Try it again now. @@ -907,6 +928,13 @@ MediaCodecReader::DecodeVideoFrameSync(int64_t aTimeThreshold) } } + if ((bufferInfo.mFlags & MediaCodec::BUFFER_FLAG_EOS) || + (status == ERROR_END_OF_STREAM)) { + VideoQueue().Finish(); + mVideoTrack.mCodec->releaseOutputBuffer(bufferInfo.mIndex); + return; + } + nsRefPtr v; RefPtr textureClient; sp graphicBuffer; @@ -1003,11 +1031,6 @@ MediaCodecReader::DecodeVideoFrameSync(int64_t aTimeThreshold) } } - if ((bufferInfo.mFlags & MediaCodec::BUFFER_FLAG_EOS) || - (status == ERROR_END_OF_STREAM)) { - VideoQueue().Finish(); - } - if (v != nullptr && textureClient != nullptr && graphicBuffer != nullptr) { MutexAutoLock al(mTextureClientIndexesLock); mTextureClientIndexes.Put(textureClient.get(), bufferInfo.mIndex); @@ -1022,27 +1045,17 @@ MediaCodecReader::Seek(int64_t aTime, int64_t aEndTime) { MOZ_ASSERT(OnTaskQueue()); - mVideoTrack.mSeekTimeUs = aTime; - mAudioTrack.mSeekTimeUs = aTime; - mVideoTrack.mInputEndOfStream = false; - mVideoTrack.mOutputEndOfStream = false; - mAudioTrack.mInputEndOfStream = false; - mAudioTrack.mOutputEndOfStream = false; - mAudioTrack.mFlushed = false; - mVideoTrack.mFlushed = false; + int64_t timestamp = sInvalidTimestampUs; if (CheckVideoResources()) { - VideoFrameContainer* videoframe = mDecoder->GetVideoFrameContainer(); - if (videoframe) { - layers::ImageContainer* image = videoframe->GetImageContainer(); - if (image) { - image->ClearAllImagesExceptFront(); - } - } + MonitorAutoLock al(mVideoTrack.mTrackMonitor); + mVideoTrack.mSeekTimeUs = aTime; + mVideoTrack.mInputEndOfStream = false; + mVideoTrack.mOutputEndOfStream = false; + mVideoTrack.mFlushed = false; MediaBuffer* source_buffer = nullptr; MediaSource::ReadOptions options; - int64_t timestamp = sInvalidTimestampUs; options.setSeekTo(aTime, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC); if (mVideoTrack.mSource->read(&source_buffer, &options) != OK || source_buffer == nullptr) { @@ -1053,12 +1066,25 @@ MediaCodecReader::Seek(int64_t aTime, int64_t aEndTime) if (format->findInt64(kKeyTime, ×tamp) && IsValidTimestampUs(timestamp)) { mVideoTrack.mSeekTimeUs = timestamp; - mAudioTrack.mSeekTimeUs = timestamp; } format = nullptr; } source_buffer->release(); } + + { + MonitorAutoLock al(mAudioTrack.mTrackMonitor); + mAudioTrack.mInputEndOfStream = false; + mAudioTrack.mOutputEndOfStream = false; + mAudioTrack.mFlushed = false; + + if (IsValidTimestampUs(timestamp)) { + mAudioTrack.mSeekTimeUs = timestamp; + } else { + mAudioTrack.mSeekTimeUs = aTime; + } + } + return SeekPromise::CreateAndResolve(aTime, __func__); } @@ -1279,11 +1305,11 @@ bool MediaCodecReader::CreateTaskQueues() { if (mAudioTrack.mSource != nullptr && !mAudioTrack.mTaskQueue) { - mAudioTrack.mTaskQueue = CreateFlushableMediaDecodeTaskQueue(); + mAudioTrack.mTaskQueue = CreateMediaDecodeTaskQueue(); NS_ENSURE_TRUE(mAudioTrack.mTaskQueue, false); } if (mVideoTrack.mSource != nullptr && !mVideoTrack.mTaskQueue) { - mVideoTrack.mTaskQueue = CreateFlushableMediaDecodeTaskQueue(); + mVideoTrack.mTaskQueue = CreateMediaDecodeTaskQueue(); NS_ENSURE_TRUE(mVideoTrack.mTaskQueue, false); mVideoTrack.mReleaseBufferTaskQueue = CreateMediaDecodeTaskQueue(); NS_ENSURE_TRUE(mVideoTrack.mReleaseBufferTaskQueue, false); @@ -1832,7 +1858,7 @@ MediaCodecReader::GetCodecOutputData(Track& aTrack, if (status == OK) { // Notify mDecoder that we have parsed a video frame. - if (&aTrack == &mVideoTrack) { + if (aTrack.mType == Track::kVideo) { mDecoder->NotifyDecodedFrames(1, 0, 0); } if (!IsValidTimestampUs(aThreshold) || info.mTimeUs >= aThreshold) { @@ -1905,9 +1931,9 @@ MediaCodecReader::EnsureCodecFormatParsed(Track& aTrack) } else if (status != -EAGAIN) { return false; // something wrong!!! } - FillCodecInputData(aTrack); } + aTrack.mCodec->releaseOutputBuffer(index); return aTrack.mCodec->getOutputFormat(&format) == OK; } diff --git a/dom/media/omx/MediaCodecReader.h b/dom/media/omx/MediaCodecReader.h index 0857604008..edae351e4c 100644 --- a/dom/media/omx/MediaCodecReader.h +++ b/dom/media/omx/MediaCodecReader.h @@ -83,7 +83,8 @@ public: // Disptach a DecodeVideoFrameTask to decode video data. virtual nsRefPtr RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) override; + int64_t aTimeThreshold, + bool aForceDecodeAhead) override; // Disptach a DecodeAduioDataTask to decode video data. virtual nsRefPtr RequestAudioData() override; @@ -147,15 +148,13 @@ protected: // playback parameters CheckedUint32 mInputIndex; - // mDiscontinuity, mFlushed, mInputEndOfStream, mInputEndOfStream, - // mSeekTimeUs don't be protected by a lock because the - // mTaskQueue->Flush() will flush all tasks. + bool mInputEndOfStream; bool mOutputEndOfStream; int64_t mSeekTimeUs; bool mFlushed; // meaningless when mSeekTimeUs is invalid. bool mDiscontinuity; - nsRefPtr mTaskQueue; + nsRefPtr mTaskQueue; Monitor mTrackMonitor; private: diff --git a/dom/media/omx/MediaOmxCommonDecoder.cpp b/dom/media/omx/MediaOmxCommonDecoder.cpp index 4acec2ee47..895706236a 100644 --- a/dom/media/omx/MediaOmxCommonDecoder.cpp +++ b/dom/media/omx/MediaOmxCommonDecoder.cpp @@ -28,6 +28,7 @@ MediaOmxCommonDecoder::MediaOmxCommonDecoder() , mReader(nullptr) , mCanOffloadAudio(false) , mFallbackToStateMachine(false) + , mIsCaptured(false) { mDormantSupported = true; if (!gMediaDecoderLog) { @@ -48,8 +49,7 @@ bool MediaOmxCommonDecoder::CheckDecoderCanOffloadAudio() { return (mCanOffloadAudio && !mFallbackToStateMachine && - !(GetStateMachine() && GetStateMachine()->GetDecodedStream()) && - mPlaybackRate == 1.0); + !mIsCaptured && mPlaybackRate == 1.0); } void @@ -176,6 +176,8 @@ MediaOmxCommonDecoder::AddOutputStream(ProcessedMediaStream* aStream, { MOZ_ASSERT(NS_IsMainThread()); + mIsCaptured = true; + if (mAudioOffloadPlayer) { ResumeStateMachine(); } diff --git a/dom/media/omx/MediaOmxCommonDecoder.h b/dom/media/omx/MediaOmxCommonDecoder.h index c06fd90e5d..eadf2eb43d 100644 --- a/dom/media/omx/MediaOmxCommonDecoder.h +++ b/dom/media/omx/MediaOmxCommonDecoder.h @@ -64,6 +64,9 @@ protected: // Set when offload playback of current track fails in the middle and need to // fallback to state machine bool mFallbackToStateMachine; + + // True if the media element is captured. + bool mIsCaptured; }; } // namespace mozilla diff --git a/dom/media/omx/MediaOmxReader.cpp b/dom/media/omx/MediaOmxReader.cpp index c8cd7231e7..53ddf1cc92 100644 --- a/dom/media/omx/MediaOmxReader.cpp +++ b/dom/media/omx/MediaOmxReader.cpp @@ -530,11 +530,6 @@ MediaOmxReader::Seek(int64_t aTarget, int64_t aEndTime) EnsureActive(); nsRefPtr p = mSeekPromise.Ensure(__func__); - VideoFrameContainer* container = mDecoder->GetVideoFrameContainer(); - if (container && container->GetImageContainer()) { - container->GetImageContainer()->ClearAllImagesExceptFront(); - } - if (mHasAudio && mHasVideo) { // The OMXDecoder seeks/demuxes audio and video streams separately. So if // we seek both audio and video to aTarget, the audio stream can typically diff --git a/dom/media/omx/RtspMediaCodecReader.cpp b/dom/media/omx/RtspMediaCodecReader.cpp index 35dd4844fe..7f2abad734 100644 --- a/dom/media/omx/RtspMediaCodecReader.cpp +++ b/dom/media/omx/RtspMediaCodecReader.cpp @@ -82,10 +82,13 @@ RtspMediaCodecReader::RequestAudioData() nsRefPtr RtspMediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) + int64_t aTimeThreshold, + bool aForceDecodeAhead) { EnsureActive(); - return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe, aTimeThreshold); + return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe, + aTimeThreshold, + aForceDecodeAhead); } nsRefPtr diff --git a/dom/media/omx/RtspMediaCodecReader.h b/dom/media/omx/RtspMediaCodecReader.h index 3bbf3ca162..86fe3ea4ea 100644 --- a/dom/media/omx/RtspMediaCodecReader.h +++ b/dom/media/omx/RtspMediaCodecReader.h @@ -53,7 +53,8 @@ public: // Disptach a DecodeVideoFrameTask to decode video data. virtual nsRefPtr RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold) override; + int64_t aTimeThreshold, + bool aForceDecodeAhead) override; // Disptach a DecodeAudioDataTask to decode audio data. virtual nsRefPtr RequestAudioData() override; diff --git a/dom/media/webm/WebMReader.cpp b/dom/media/webm/WebMReader.cpp index 2923922dcc..64a787b791 100644 --- a/dom/media/webm/WebMReader.cpp +++ b/dom/media/webm/WebMReader.cpp @@ -736,6 +736,7 @@ WebMReader::Seek(int64_t aTarget, int64_t aEndTime) nsresult WebMReader::SeekInternal(int64_t aTarget) { MOZ_ASSERT(OnTaskQueue()); + NS_ENSURE_TRUE(HaveStartTime(), NS_ERROR_FAILURE); if (mVideoDecoder) { nsresult rv = mVideoDecoder->Flush(); NS_ENSURE_SUCCESS(rv, rv); @@ -750,7 +751,7 @@ nsresult WebMReader::SeekInternal(int64_t aTarget) uint64_t target = aTarget * NS_PER_USEC; if (mSeekPreroll) { - target = std::max(uint64_t(mStartTime * NS_PER_USEC), + target = std::max(uint64_t(StartTime() * NS_PER_USEC), target - mSeekPreroll); } int r = nestegg_track_seek(mContext, trackToSeek, target); @@ -778,7 +779,7 @@ nsresult WebMReader::SeekInternal(int64_t aTarget) media::TimeIntervals WebMReader::GetBuffered() { MOZ_ASSERT(OnTaskQueue()); - NS_ENSURE_TRUE(mStartTime >= 0, media::TimeIntervals()); + NS_ENSURE_TRUE(HaveStartTime(), media::TimeIntervals()); AutoPinned resource(mDecoder->GetResource()); media::TimeIntervals buffered; @@ -805,7 +806,7 @@ media::TimeIntervals WebMReader::GetBuffered() ranges[index].mEnd, &start, &end); if (rv) { - int64_t startOffset = mStartTime * NS_PER_USEC; + int64_t startOffset = StartTime() * NS_PER_USEC; NS_ASSERTION(startOffset >= 0 && uint64_t(startOffset) <= start, "startOffset negative or larger than start time"); if (!(startOffset >= 0 && uint64_t(startOffset) <= start)) { diff --git a/dom/plugins/ipc/PluginInstanceParent.cpp b/dom/plugins/ipc/PluginInstanceParent.cpp index d3d1ce2a64..aff40dbb32 100644 --- a/dom/plugins/ipc/PluginInstanceParent.cpp +++ b/dom/plugins/ipc/PluginInstanceParent.cpp @@ -633,7 +633,10 @@ PluginInstanceParent::RecvShow(const NPRect& updatedRect, cairoData.mSourceSurface = gfxPlatform::GetPlatform()->GetSourceSurfaceForSurface(nullptr, surface); cairoImage->SetData(cairoData); - container->SetCurrentImage(cairoImage); + nsAutoTArray imageList; + imageList.AppendElement( + ImageContainer::NonOwningImage(image, TimeStamp())); + container->SetCurrentImages(imageList); } else if (mImageContainer) { mImageContainer->ClearAllImages(); diff --git a/gfx/layers/ImageContainer.cpp b/gfx/layers/ImageContainer.cpp index ac09877c70..1712ea67b1 100644 --- a/gfx/layers/ImageContainer.cpp +++ b/gfx/layers/ImageContainer.cpp @@ -16,6 +16,7 @@ #include "mozilla/layers/ImageBridgeChild.h" // for ImageBridgeChild #include "mozilla/layers/PImageContainerChild.h" #include "mozilla/layers/ImageClient.h" // for ImageClient +#include "mozilla/layers/LayersMessages.h" #include "nsISupportsUtils.h" // for NS_IF_ADDREF #include "YCbCrUtils.h" // for YCbCr conversions #ifdef MOZ_WIDGET_GONK @@ -170,7 +171,9 @@ ImageContainer::ImageContainer(Mode flag) : mReentrantMonitor("ImageContainer.mReentrantMonitor"), mGenerationCounter(++sGenerationCounter), mPaintCount(0), + mDroppedImageCount(0), mPreviousImagePainted(false), + mCurrentImageComposited(false), mImageFactory(new ImageFactory()), mRecycleBin(new BufferRecycleBin()), mImageClient(nullptr), @@ -234,59 +237,55 @@ ImageContainer::CreateImage(ImageFormat aFormat) } void -ImageContainer::SetCurrentImageInternal(Image *aImage) +ImageContainer::SetCurrentImageInternal(Image *aImage, + const TimeStamp& aTimeStamp) { ReentrantMonitorAutoEnter mon(mReentrantMonitor); if (mActiveImage != aImage) { + if (!mCurrentImageComposited && !mCurrentImageTimeStamp.IsNull() && + (aTimeStamp.IsNull() || aTimeStamp > mCurrentImageTimeStamp)) { + mFrameIDsNotYetComposited.AppendElement(mGenerationCounter); + } mGenerationCounter = ++sGenerationCounter; + mCurrentImageComposited = false; + mActiveImage = aImage; + mCurrentImageTimeStamp = aTimeStamp; } - mActiveImage = aImage; CurrentImageChanged(); } void -ImageContainer::ClearCurrentImage() +ImageContainer::ClearImagesFromImageBridge() { ReentrantMonitorAutoEnter mon(mReentrantMonitor); - SetCurrentImageInternal(nullptr); + SetCurrentImageInternal(nullptr, TimeStamp()); } void -ImageContainer::SetCurrentImage(Image *aImage) +ImageContainer::SetCurrentImages(const nsTArray& aImages) { - if (!aImage) { - ClearAllImages(); - return; - } - + MOZ_ASSERT(!aImages.IsEmpty()); ReentrantMonitorAutoEnter mon(mReentrantMonitor); if (IsAsync()) { ImageBridgeChild::DispatchImageClientUpdate(mImageClient, this); } - SetCurrentImageInternal(aImage); + MOZ_ASSERT(aImages.Length() == 1); + SetCurrentImageInternal(aImages[0].mImage, aImages[0].mTimeStamp); } void ImageContainer::ClearAllImages() { if (IsAsync()) { - // Let ImageClient release all TextureClients. - ImageBridgeChild::FlushAllImages(mImageClient, this, false); + // Let ImageClient release all TextureClients. This doesn't return + // until ImageBridge has called ClearCurrentImageFromImageBridge. + ImageBridgeChild::FlushAllImages(mImageClient, this); return; } ReentrantMonitorAutoEnter mon(mReentrantMonitor); - SetCurrentImageInternal(nullptr); -} - -void -ImageContainer::ClearAllImagesExceptFront() -{ - if (IsAsync()) { - // Let ImageClient release all TextureClients except front one. - ImageBridgeChild::FlushAllImages(mImageClient, this, true); - } + SetCurrentImageInternal(nullptr, TimeStamp()); } void @@ -295,10 +294,11 @@ ImageContainer::SetCurrentImageInTransaction(Image *aImage) NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); NS_ASSERTION(!mImageClient, "Should use async image transfer with ImageBridge."); - SetCurrentImageInternal(aImage); + SetCurrentImageInternal(aImage, TimeStamp()); } -bool ImageContainer::IsAsync() const { +bool ImageContainer::IsAsync() const +{ return mImageClient != nullptr; } @@ -349,6 +349,31 @@ ImageContainer::GetCurrentSize() return mActiveImage->GetSize(); } +void +ImageContainer::NotifyCompositeInternal(const ImageCompositeNotification& aNotification) +{ + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + + while (!mFrameIDsNotYetComposited.IsEmpty()) { + if (mFrameIDsNotYetComposited[0] <= aNotification.frameID()) { + if (mFrameIDsNotYetComposited[0] < aNotification.frameID()) { + ++mDroppedImageCount; + } + mFrameIDsNotYetComposited.RemoveElementAt(0); + } else { + break; + } + } + if (aNotification.frameID() == mGenerationCounter) { + mCurrentImageComposited = true; + } + + if (!aNotification.imageTimeStamp().IsNull()) { + mPaintDelay = aNotification.firstCompositeTimeStamp() - + aNotification.imageTimeStamp(); + } +} + PlanarYCbCrImage::PlanarYCbCrImage(BufferRecycleBin *aRecycleBin) : Image(nullptr, ImageFormat::PLANAR_YCBCR) , mBufferSize(0) diff --git a/gfx/layers/ImageContainer.h b/gfx/layers/ImageContainer.h index c49d4d8d0e..0b58a166ad 100644 --- a/gfx/layers/ImageContainer.h +++ b/gfx/layers/ImageContainer.h @@ -267,11 +267,11 @@ protected: * An ImageContainer can operate in one of these modes: * 1) Normal. Triggered by constructing the ImageContainer with * DISABLE_ASYNC or when compositing is happening on the main thread. - * SetCurrentImage changes ImageContainer state but nothing is sent to the + * SetCurrentImages changes ImageContainer state but nothing is sent to the * compositor until the next layer transaction. * 2) Asynchronous. Initiated by constructing the ImageContainer with * ENABLE_ASYNC when compositing is happening on the main thread. - * SetCurrentImage sends a message through the ImageBridge to the compositor + * SetCurrentImages sends a message through the ImageBridge to the compositor * thread to update the image, without going through the main thread or * a layer transaction. * The ImageContainer uses a shared memory block containing a cross-process mutex @@ -302,24 +302,32 @@ public: */ B2G_ACL_EXPORT already_AddRefed CreateImage(ImageFormat aFormat); + struct NonOwningImage { + NonOwningImage(Image* aImage, TimeStamp aTimeStamp) + : mImage(aImage), mTimeStamp(aTimeStamp) {} + Image* mImage; + TimeStamp mTimeStamp; + }; /** - * Set an Image as the current image to display. The Image must have + * Set aImages as the list of timestamped to display. The Images must have * been created by this ImageContainer. * Can be called on any thread. This method takes mReentrantMonitor * when accessing thread-shared state. - * aImage can be null. While it's null, nothing will be painted. + * aImages must be non-empty. The first timestamp in the list may be + * null but the others must not be, and the timestamps must increase. + * Every element of aImages must have non-null mImage. * * The Image data must not be modified after this method is called! * Note that this must not be called if ENABLE_ASYNC has not been set. * - * Implementations must call CurrentImageChanged() while holding + * The implementation calls CurrentImageChanged() while holding * mReentrantMonitor. * * If this ImageContainer has an ImageClient for async video: - * Schelude a task to send the image to the compositor using the + * Schedule a task to send the image to the compositor using the * PImageBridge protcol without using the main thread. */ - void SetCurrentImage(Image* aImage); + void SetCurrentImages(const nsTArray& aImages); /** * Clear all images. Let ImageClient release all TextureClients. @@ -327,18 +335,12 @@ public: void ClearAllImages(); /** - * Clear all images except current one. - * Let ImageClient release all TextureClients except front one. - */ - void ClearAllImagesExceptFront(); - - /** - * Clear the current image. + * Clear the current images. * This function is expect to be called only from a CompositableClient * that belongs to ImageBridgeChild. Created to prevent dead lock. * See Bug 901224. */ - void ClearCurrentImage(); + void ClearImagesFromImageBridge(); /** * Set an Image as the current image to display. The Image must have @@ -429,14 +431,17 @@ public: } /** - * Returns the time at which the currently contained image was first - * painted. This is reset every time a new image is set as the current - * image. Note this may return a null timestamp if the current image - * has not yet been painted. Can be called from any thread. + * Returns the delay between the last composited image's presentation + * timestamp and when it was first composited. It's possible for the delay + * to be negative if the first image in the list passed to SetCurrentImages + * has a presentation timestamp greater than "now". + * Returns 0 if the composited image had a null timestamp, or if no + * image has been composited yet. */ - TimeStamp GetPaintTime() { + TimeDuration GetPaintDelay() + { ReentrantMonitorAutoEnter mon(mReentrantMonitor); - return mPaintTime; + return mPaintDelay; } /** @@ -448,6 +453,19 @@ public: return mPaintCount; } + /** + * An image in the current image list "expires" when the image has an + * associated timestamp, and in a SetCurrentImages call the timestamp of the + * first new image is non-null and greater than the timestamp associated + * with the image. Every expired image that is never composited is counted + * as dropped. + */ + uint32_t GetDroppedImageCount() + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + return mDroppedImageCount; + } + /** * Increments mPaintCount if this is the first time aPainted has been * painted, and sets mPaintTime if the painted image is the current image. @@ -472,7 +490,6 @@ public: } PImageContainerChild* GetPImageContainerChild(); - static void NotifyComposite(const ImageCompositeNotification& aNotification); private: @@ -481,7 +498,7 @@ private: // Private destructor, to discourage deletion outside of Release(): B2G_ACL_EXPORT ~ImageContainer(); - void SetCurrentImageInternal(Image* aImage); + void SetCurrentImageInternal(Image* aImage, const TimeStamp& aTimeStamp); // This is called to ensure we have an active image, this may not be true // when we're storing image information in a RemoteImageData structure. @@ -489,9 +506,7 @@ private: // calling this function! void EnsureActiveImage(); - // ReentrantMonitor to protect thread safe access to the "current - // image", and any other state which is shared between threads. - ReentrantMonitor mReentrantMonitor; + void NotifyCompositeInternal(const ImageCompositeNotification& aNotification); // Performs necessary housekeeping to ensure the painted frame statistics // are accurate. Must be called by SetCurrentImage() implementations with @@ -502,9 +517,13 @@ private: mPaintTime = TimeStamp(); } - void NotifyCompositeInternal(const ImageCompositeNotification& aNotification) {} + // ReentrantMonitor to protect thread safe access to the "current + // image", and any other state which is shared between threads. + ReentrantMonitor mReentrantMonitor; nsRefPtr mActiveImage; + TimeStamp mCurrentImageTimeStamp; + // Updates every time mActiveImage changes uint32_t mGenerationCounter; @@ -517,9 +536,17 @@ private: // ImageContainer implementation to ensure accesses to this are threadsafe. TimeStamp mPaintTime; + // See GetPaintDelay. Accessed only with mReentrantMonitor held. + TimeDuration mPaintDelay; + + // See GetDroppedImageCount. Accessed only with mReentrantMonitor held. + uint32_t mDroppedImageCount; + // Denotes whether the previous image was painted. bool mPreviousImagePainted; + bool mCurrentImageComposited; + // This is the image factory used by this container, layer managers using // this container can set an alternative image factory that will be used to // create images for this container. @@ -538,6 +565,8 @@ private: // asynchronusly using the ImageBridge IPDL protocol. ImageClient* mImageClient; + nsTArray mFrameIDsNotYetComposited; + // Object must be released on the ImageBridge thread. Field is immutable // after creation of the ImageContainer. ImageContainerChild* mIPDLChild; diff --git a/gfx/layers/LayerTreeInvalidation.cpp b/gfx/layers/LayerTreeInvalidation.cpp index 50ed176a44..0cce0958da 100644 --- a/gfx/layers/LayerTreeInvalidation.cpp +++ b/gfx/layers/LayerTreeInvalidation.cpp @@ -4,6 +4,7 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "LayerTreeInvalidation.h" + #include // for uint32_t #include "ImageContainer.h" // for ImageContainer #include "ImageLayers.h" // for ImageLayer, etc @@ -23,6 +24,8 @@ #include "nsISupportsImpl.h" // for Layer::AddRef, etc #include "nsRect.h" // for IntRect #include "nsTArray.h" // for nsAutoTArray, nsTArray_Impl +#include "mozilla/layers/ImageHost.h" +#include "mozilla/layers/LayerManagerComposite.h" using namespace mozilla::gfx; @@ -383,16 +386,27 @@ struct ColorLayerProperties : public LayerPropertiesBase IntRect mBounds; }; +static ImageHost* GetImageHost(ImageLayer* aLayer) +{ + LayerComposite* composite = aLayer->AsLayerComposite(); + if (composite) { + return static_cast(composite->GetCompositableHost()); + } + return nullptr; +} + struct ImageLayerProperties : public LayerPropertiesBase { explicit ImageLayerProperties(ImageLayer* aImage, bool aIsMask) : LayerPropertiesBase(aImage) , mContainer(aImage->GetContainer()) + , mImageHost(GetImageHost(aImage)) , mFilter(aImage->GetFilter()) , mScaleToSize(aImage->GetScaleToSize()) , mScaleMode(aImage->GetScaleMode()) , mIsMask(aIsMask) { + mFrameID = mImageHost ? mImageHost->GetFrameID() : -1; } virtual nsIntRegion ComputeChangeInternal(NotifySubDocInvalidationFunc aCallback, @@ -408,30 +422,39 @@ struct ImageLayerProperties : public LayerPropertiesBase } ImageContainer* container = imageLayer->GetContainer(); + ImageHost* host = GetImageHost(imageLayer); if (mContainer != container || mFilter != imageLayer->GetFilter() || mScaleToSize != imageLayer->GetScaleToSize() || - mScaleMode != imageLayer->GetScaleMode()) { + mScaleMode != imageLayer->GetScaleMode() || + host != mImageHost || + (host && host->GetFrameID() != mFrameID)) { aGeometryChanged = true; if (mIsMask) { // Mask layers have an empty visible region, so we have to // use the image size instead. - IntSize size = container->GetCurrentSize(); + IntSize size; + if (container) { + size = container->GetCurrentSize(); + } + if (host) { + size = host->GetImageSize(); + } IntRect rect(0, 0, size.width, size.height); return TransformRect(rect, mLayer->GetLocalTransform()); - - } else { - return NewTransformedBounds(); } + return NewTransformedBounds(); } return IntRect(); } nsRefPtr mContainer; + nsRefPtr mImageHost; GraphicsFilter mFilter; gfx::IntSize mScaleToSize; + int32_t mFrameID; ScaleMode mScaleMode; bool mIsMask; }; diff --git a/gfx/layers/client/ImageClient.cpp b/gfx/layers/client/ImageClient.cpp index 134324936e..b22c18c474 100644 --- a/gfx/layers/client/ImageClient.cpp +++ b/gfx/layers/client/ImageClient.cpp @@ -110,15 +110,12 @@ TextureInfo ImageClientSingle::GetTextureInfo() const } void -ImageClientSingle::FlushAllImages(bool aExceptFront, - AsyncTransactionWaiter* aAsyncTransactionWaiter) +ImageClientSingle::FlushAllImages(AsyncTransactionWaiter* aAsyncTransactionWaiter) { - if (!aExceptFront) { - for (auto& b : mBuffers) { - RemoveTextureWithWaiter(b.mTextureClient, aAsyncTransactionWaiter); - } - mBuffers.Clear(); + for (auto& b : mBuffers) { + RemoveTextureWithWaiter(b.mTextureClient, aAsyncTransactionWaiter); } + mBuffers.Clear(); } bool @@ -140,6 +137,10 @@ ImageClientSingle::UpdateImage(ImageContainer* aContainer, uint32_t aContentFlag } } if (images.IsEmpty()) { + // This can happen if a ClearAllImages raced with SetCurrentImages from + // another thread and ClearImagesFromImageBridge ran after the + // SetCurrentImages call but before UpdateImageClientNow. + // This can also happen if all images in the list are invalid. // We return true because the caller would attempt to recreate the // ImageClient otherwise, and that isn't going to help. return true; diff --git a/gfx/layers/client/ImageClient.h b/gfx/layers/client/ImageClient.h index 6d5890192a..3751c7338e 100644 --- a/gfx/layers/client/ImageClient.h +++ b/gfx/layers/client/ImageClient.h @@ -65,8 +65,7 @@ public: * asynchronously remove all the textures used by the image client. * */ - virtual void FlushAllImages(bool aExceptFront, - AsyncTransactionWaiter* aAsyncTransactionWaiter) {} + virtual void FlushAllImages(AsyncTransactionWaiter* aAsyncTransactionWaiter) {} virtual void RemoveTexture(TextureClient* aTexture) override; @@ -102,8 +101,7 @@ public: virtual already_AddRefed CreateImage(ImageFormat aFormat) override; - virtual void FlushAllImages(bool aExceptFront, - AsyncTransactionWaiter* aAsyncTransactionWaiter) override; + virtual void FlushAllImages(AsyncTransactionWaiter* aAsyncTransactionWaiter) override; protected: struct Buffer { diff --git a/gfx/layers/composite/ImageHost.h b/gfx/layers/composite/ImageHost.h index 5c2c9465b2..ea956b79cb 100644 --- a/gfx/layers/composite/ImageHost.h +++ b/gfx/layers/composite/ImageHost.h @@ -85,6 +85,12 @@ public: virtual already_AddRefed GenEffect(const gfx::Filter& aFilter) override; + int32_t GetFrameID() + { + const TimedImage* img = ChooseImage(); + return img ? img->mFrameID : -1; + } + protected: struct TimedImage { CompositableTextureHostRef mFrontBuffer; diff --git a/gfx/layers/composite/LayerManagerComposite.cpp b/gfx/layers/composite/LayerManagerComposite.cpp index 5e17010a22..245d4ae819 100644 --- a/gfx/layers/composite/LayerManagerComposite.cpp +++ b/gfx/layers/composite/LayerManagerComposite.cpp @@ -282,7 +282,12 @@ LayerManagerComposite::EndTransaction(const TimeStamp& aTimeStamp, mInvalidRegion.Or(mInvalidRegion, mRenderBounds); } - if (mRoot && !(aFlags & END_NO_IMMEDIATE_REDRAW)) { + if (mInvalidRegion.IsEmpty() && !mTarget) { + // Composition requested, but nothing has changed. Don't do any work. + return; + } + + if (mRoot && !(aFlags & END_NO_IMMEDIATE_REDRAW)) { MOZ_ASSERT(!aTimeStamp.IsNull()); // Set composition timestamp here because we need it in // ComputeEffectiveTransforms (so the correct video frame size is picked) diff --git a/gfx/layers/ipc/ImageBridgeChild.cpp b/gfx/layers/ipc/ImageBridgeChild.cpp index d72ba0958e..301c6b5784 100644 --- a/gfx/layers/ipc/ImageBridgeChild.cpp +++ b/gfx/layers/ipc/ImageBridgeChild.cpp @@ -433,14 +433,14 @@ void ImageBridgeChild::DispatchImageClientUpdate(ImageClient* aClient, } static void FlushAllImagesSync(ImageClient* aClient, ImageContainer* aContainer, - bool aExceptFront, AsyncTransactionWaiter* aWaiter) + AsyncTransactionWaiter* aWaiter) { MOZ_ASSERT(aClient); sImageBridgeChildSingleton->BeginTransaction(); - if (aContainer && !aExceptFront) { - aContainer->ClearCurrentImage(); + if (aContainer) { + aContainer->ClearImagesFromImageBridge(); } - aClient->FlushAllImages(aExceptFront, aWaiter); + aClient->FlushAllImages(aWaiter); sImageBridgeChildSingleton->EndTransaction(); // This decrement is balanced by the increment in FlushAllImages. // If any AsyncTransactionTrackers were created by FlushAllImages and attached @@ -451,7 +451,7 @@ static void FlushAllImagesSync(ImageClient* aClient, ImageContainer* aContainer, //static void ImageBridgeChild::FlushAllImages(ImageClient* aClient, - ImageContainer* aContainer, bool aExceptFront) + ImageContainer* aContainer) { if (!IsCreated()) { return; @@ -470,7 +470,7 @@ void ImageBridgeChild::FlushAllImages(ImageClient* aClient, sImageBridgeChildSingleton->GetMessageLoop()->PostTask( FROM_HERE, - NewRunnableFunction(&FlushAllImagesSync, aClient, aContainer, aExceptFront, waiter)); + NewRunnableFunction(&FlushAllImagesSync, aClient, aContainer, waiter)); waiter->WaitComplete(); } diff --git a/gfx/layers/ipc/ImageBridgeChild.h b/gfx/layers/ipc/ImageBridgeChild.h index 2fabdeec74..8d4f6c087b 100644 --- a/gfx/layers/ipc/ImageBridgeChild.h +++ b/gfx/layers/ipc/ImageBridgeChild.h @@ -219,7 +219,7 @@ public: /** * Flush all Images sent to CompositableHost. */ - static void FlushAllImages(ImageClient* aClient, ImageContainer* aContainer, bool aExceptFront); + static void FlushAllImages(ImageClient* aClient, ImageContainer* aContainer); // CompositableForwarder diff --git a/image/RasterImage.cpp b/image/RasterImage.cpp index 8190e02e1b..629e040b77 100644 --- a/image/RasterImage.cpp +++ b/image/RasterImage.cpp @@ -905,7 +905,10 @@ RasterImage::UpdateImageContainer() } mLastImageContainerDrawResult = result.first(); - container->SetCurrentImage(result.second()); + nsAutoTArray imageList; + imageList.AppendElement( + ImageContainer::NonOwningImage(result.second(), TimeStamp())); + container->SetCurrentImages(imageList); } size_t