diff --git a/dom/media/FileBlockCache.cpp b/dom/media/FileBlockCache.cpp index 8121b5e4d5..ce6a34cce2 100644 --- a/dom/media/FileBlockCache.cpp +++ b/dom/media/FileBlockCache.cpp @@ -4,6 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/SharedThreadPool.h" #include "FileBlockCache.h" #include "VideoUtils.h" #include "prio.h" diff --git a/dom/media/FlushableTaskQueue.cpp b/dom/media/FlushableTaskQueue.cpp deleted file mode 100644 index b28340d964..0000000000 --- a/dom/media/FlushableTaskQueue.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "FlushableTaskQueue.h" - -namespace mozilla { - -void -FlushableTaskQueue::Flush() -{ - MonitorAutoLock mon(mQueueMonitor); - AutoSetFlushing autoFlush(this); - FlushLocked(); - AwaitIdleLocked(); -} - -nsresult -FlushableTaskQueue::FlushAndDispatch(already_AddRefed aRunnable) -{ - nsCOMPtr r = aRunnable; - { - MonitorAutoLock mon(mQueueMonitor); - AutoSetFlushing autoFlush(this); - FlushLocked(); - nsresult rv = DispatchLocked(/* passed by ref */r, IgnoreFlushing, AssertDispatchSuccess); - NS_ENSURE_SUCCESS(rv, rv); - AwaitIdleLocked(); - } - // If the ownership of |r| is not transferred in DispatchLocked() due to - // dispatch failure, it will be deleted here outside the lock. We do so - // since the destructor of the runnable might access TaskQueue and result - // in deadlocks. - return NS_OK; -} - -void -FlushableTaskQueue::FlushLocked() -{ - // Make sure there are no tasks for this queue waiting in the caller's tail - // dispatcher. - MOZ_ASSERT_IF(AbstractThread::GetCurrent(), - !AbstractThread::GetCurrent()->TailDispatcher().HasTasksFor(this)); - - mQueueMonitor.AssertCurrentThreadOwns(); - MOZ_ASSERT(mIsFlushing); - - // Clear the tasks. If this strikes you as awful, stop using a - // FlushableTaskQueue. - while (!mTasks.empty()) { - mTasks.pop(); - } -} - -} // namespace mozilla diff --git a/dom/media/FlushableTaskQueue.h b/dom/media/FlushableTaskQueue.h deleted file mode 100644 index 9d56db03b1..0000000000 --- a/dom/media/FlushableTaskQueue.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef FlushableTaskQueue_h_ -#define FlushableTaskQueue_h_ - -#include "mozilla/TaskQueue.h" - -// -// WARNING: THIS CLASS IS DEPRECATED AND GOING AWAY. DO NOT USE IT! -// - -namespace mozilla { - -class FlushableTaskQueue : public TaskQueue -{ -public: - explicit FlushableTaskQueue(already_AddRefed aPool) : TaskQueue(Move(aPool)) {} - nsresult FlushAndDispatch(already_AddRefed aRunnable); - void Flush(); - - bool IsDispatchReliable() override { return false; } - -private: - - class MOZ_STACK_CLASS AutoSetFlushing - { - public: - explicit AutoSetFlushing(FlushableTaskQueue* aTaskQueue) : mTaskQueue(aTaskQueue) - { - mTaskQueue->mQueueMonitor.AssertCurrentThreadOwns(); - mTaskQueue->mIsFlushing = true; - } - ~AutoSetFlushing() - { - mTaskQueue->mQueueMonitor.AssertCurrentThreadOwns(); - mTaskQueue->mIsFlushing = false; - } - - private: - FlushableTaskQueue* mTaskQueue; - }; - - void FlushLocked(); - -}; - -} // namespace mozilla - -#endif // FlushableTaskQueue_h_ diff --git a/dom/media/MediaDataDemuxer.h b/dom/media/MediaDataDemuxer.h index 59c7268996..4d22ec2b96 100644 --- a/dom/media/MediaDataDemuxer.h +++ b/dom/media/MediaDataDemuxer.h @@ -201,6 +201,13 @@ public: virtual media::TimeIntervals GetBuffered() = 0; + // By default, it is assumed that the entire resource can be evicted once + // all samples have been demuxed. + virtual int64_t GetEvictionOffset(const media::TimeUnit& aTime) + { + return INT64_MAX; + } + // If the MediaTrackDemuxer and MediaDataDemuxer hold cross references. // BreakCycles must be overridden. virtual void BreakCycles() diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index b5a8acdeda..7fd03db480 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -174,9 +174,9 @@ MediaFormatReader::Init() InitLayersBackendType(); mAudio.mTaskQueue = - new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); + new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); mVideo.mTaskQueue = - new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); + new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); return NS_OK; } @@ -1336,7 +1336,6 @@ MediaFormatReader::ResetDecode(TargetQueues aQueues) if (HasVideo()) { mVideo.ResetDemuxer(); - mVideo.ResetState(); Reset(TrackInfo::kVideoTrack); if (mVideo.HasPromise()) { mVideo.RejectPromise(CANCELED, __func__); @@ -1345,7 +1344,6 @@ MediaFormatReader::ResetDecode(TargetQueues aQueues) if (HasAudio() && aQueues == AUDIO_VIDEO) { mAudio.ResetDemuxer(); - mAudio.ResetState(); Reset(TrackInfo::kAudioTrack); if (mAudio.HasPromise()) { mAudio.RejectPromise(CANCELED, __func__); diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 4c61fcab8b..8aca346635 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -257,7 +257,7 @@ private: RefPtr mTrackDemuxer; // TaskQueue on which decoder can choose to decode. // Only non-null up until the decoder is created. - RefPtr mTaskQueue; + RefPtr mTaskQueue; // Callback that receives output and error notifications from the decoder. nsAutoPtr mCallback; diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index a796e04c0f..aa2f953513 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -205,12 +205,58 @@ MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream, for (int32_t i = aStream->mUpdateTracks.Length() - 1; i >= 0; --i) { SourceMediaStream::TrackData* data = &aStream->mUpdateTracks[i]; aStream->ApplyTrackDisabling(data->mID, data->mData); + // Dealing with NotifyQueuedTrackChanges and NotifyQueuedAudioData part. + + // The logic is different from the manipulating of aStream->mTracks part. + // So it is not combined with the manipulating of aStream->mTracks part. StreamTime offset = (data->mCommands & SourceMediaStream::TRACK_CREATE) ? data->mStart : aStream->mTracks.FindTrack(data->mID)->GetSegment()->GetDuration(); - for (MediaStreamListener* l : aStream->mListeners) { - l->NotifyQueuedTrackChanges(this, data->mID, - offset, data->mCommands, *data->mData); + + // Audio case. + if (data->mData->GetType() == MediaSegment::AUDIO) { + if (data->mCommands) { + MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED)); + for (MediaStreamListener* l : aStream->mListeners) { + if (data->mCommands & SourceMediaStream::TRACK_END) { + l->NotifyQueuedAudioData(this, data->mID, + offset, *(static_cast(data->mData.get()))); + } + l->NotifyQueuedTrackChanges(this, data->mID, + offset, data->mCommands, *data->mData); + if (data->mCommands & SourceMediaStream::TRACK_CREATE) { + l->NotifyQueuedAudioData(this, data->mID, + offset, *(static_cast(data->mData.get()))); + } + } + } else { + for (MediaStreamListener* l : aStream->mListeners) { + l->NotifyQueuedAudioData(this, data->mID, + offset, *(static_cast(data->mData.get()))); + } + } } + + // Video case. + if (data->mData->GetType() == MediaSegment::VIDEO) { + if (data->mCommands) { + MOZ_ASSERT(!(data->mCommands & SourceMediaStream::TRACK_UNUSED)); + for (MediaStreamListener* l : aStream->mListeners) { + l->NotifyQueuedTrackChanges(this, data->mID, + offset, data->mCommands, *data->mData); + } + } else { + // Fixme: This part will be removed in the bug 1201363. It will be + // removed in changeset "Do not copy video segment to StreamTracks in + // TrackUnionStream." + + // Dealing with video and not TRACK_CREATE and TRACK_END case. + for (MediaStreamListener* l : aStream->mListeners) { + l->NotifyQueuedTrackChanges(this, data->mID, + offset, data->mCommands, *data->mData); + } + } + } + for (TrackBound& b : aStream->mTrackListeners) { if (b.mTrackID != data->mID) { continue; diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 640e5edfe3..729345e590 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -159,9 +159,11 @@ public: */ virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) {} + // maskable flags, not a simple enumerated value enum { TRACK_EVENT_CREATED = 0x01, - TRACK_EVENT_ENDED = 0x02 + TRACK_EVENT_ENDED = 0x02, + TRACK_EVENT_UNUSED = ~(TRACK_EVENT_ENDED | TRACK_EVENT_CREATED), }; /** * Notify that changes to one of the stream tracks have been queued. @@ -179,6 +181,16 @@ public: MediaStream* aInputStream = nullptr, TrackID aInputTrackID = TRACK_INVALID) {} + /** + * Notify queued audio data. Only audio data need to be queued. The video data + * will be notified by MediaStreamVideoSink::SetCurrentFrame. + */ + virtual void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID, + StreamTime aTrackOffset, + const AudioSegment& aQueuedMedia, + MediaStream* aInputStream = nullptr, + TrackID aInputTrackID = TRACK_INVALID) {} + /** * Notify that all new tracks this iteration have been created. * This is to ensure that tracks added atomically to MediaStreamGraph @@ -1078,19 +1090,10 @@ public: friend class MediaStreamGraphImpl; protected: - struct ThreadAndRunnable { - void Init(TaskQueue* aTarget, nsIRunnable* aRunnable) - { - mTarget = aTarget; - mRunnable = aRunnable; - } - - RefPtr mTarget; - nsCOMPtr mRunnable; - }; enum TrackCommands { TRACK_CREATE = MediaStreamListener::TRACK_EVENT_CREATED, - TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED + TRACK_END = MediaStreamListener::TRACK_EVENT_ENDED, + TRACK_UNUSED = MediaStreamListener::TRACK_EVENT_UNUSED, }; /** * Data for each track that hasn't ended. @@ -1455,7 +1458,6 @@ public: // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that // never blocks and has a track with the ideal audio rate will produce audio // in multiples of the block size. - // // Initializing an graph that outputs audio can be quite long on some // platforms. Code that want to output audio at some point can express the diff --git a/dom/media/TrackUnionStream.cpp b/dom/media/TrackUnionStream.cpp index 4f12473820..56f20cc603 100644 --- a/dom/media/TrackUnionStream.cpp +++ b/dom/media/TrackUnionStream.cpp @@ -324,10 +324,17 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : ApplyTrackDisabling(outputTrack->GetID(), segment); for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; - l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), - outputStart, 0, *segment, - map->mInputPort->GetSource(), - map->mInputTrackID); + // Separate Audio and Video. + if (segment->GetType() == MediaSegment::AUDIO) { + l->NotifyQueuedAudioData(Graph(), outputTrack->GetID(), + outputStart, *static_cast(segment)); + } else { + // This part will be removed in bug 1201363. + l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), + outputStart, 0, *segment, + map->mInputPort->GetSource(), + map->mInputTrackID); + } } for (TrackBound& b : mTrackListeners) { if (b.mTrackID != outputTrack->GetID()) { diff --git a/dom/media/VideoUtils.cpp b/dom/media/VideoUtils.cpp index 8d4e3f21e0..0644658615 100644 --- a/dom/media/VideoUtils.cpp +++ b/dom/media/VideoUtils.cpp @@ -295,14 +295,6 @@ CreateMediaDecodeTaskQueue() return queue.forget(); } -already_AddRefed -CreateFlushableMediaDecodeTaskQueue() -{ - RefPtr queue = new FlushableTaskQueue( - GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); - return queue.forget(); -} - void SimpleTimer::Cancel() { if (mTimer) { diff --git a/dom/media/VideoUtils.h b/dom/media/VideoUtils.h index 2d905a18e1..3dc8967f81 100644 --- a/dom/media/VideoUtils.h +++ b/dom/media/VideoUtils.h @@ -7,7 +7,6 @@ #ifndef VideoUtils_h #define VideoUtils_h -#include "FlushableTaskQueue.h" #include "mozilla/Attributes.h" #include "mozilla/CheckedInt.h" #include "mozilla/MozPromise.h" @@ -268,9 +267,6 @@ GenerateRandomPathName(nsCString& aOutSalt, uint32_t aLength); already_AddRefed CreateMediaDecodeTaskQueue(); -already_AddRefed -CreateFlushableMediaDecodeTaskQueue(); - // Iteratively invokes aWork until aCondition returns true, or aWork returns false. // Use this rather than a while loop to avoid bogarting the task queue. template diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp index effc95f6f5..8519d0308f 100644 --- a/dom/media/encoder/MediaEncoder.cpp +++ b/dom/media/encoder/MediaEncoder.cpp @@ -105,6 +105,17 @@ MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, } } +void +MediaEncoder::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID, + StreamTime aTrackOffset, + const AudioSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) +{ + mAudioEncoder->NotifyQueuedTrackChanges(aGraph, aID, aTrackOffset, 0, + aQueuedMedia); +} + void MediaEncoder::NotifyEvent(MediaStreamGraph* aGraph, MediaStreamListener::MediaStreamGraphEvent event) diff --git a/dom/media/encoder/MediaEncoder.h b/dom/media/encoder/MediaEncoder.h index 43d311808f..b57f523642 100644 --- a/dom/media/encoder/MediaEncoder.h +++ b/dom/media/encoder/MediaEncoder.h @@ -130,7 +130,17 @@ public : TrackID aInputTrackID) override; /** - * Notified the stream is being removed. + * Notifed by the control loop of MediaStreamGraph; aQueueMedia is the audio + * data in the form of an AudioSegment. + */ + void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID, + StreamTime aTrackOffset, + const AudioSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override; + + /** + * * Notified the stream is being removed. */ void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamListener::MediaStreamGraphEvent event) override; diff --git a/dom/media/fmp4/MP4Decoder.cpp b/dom/media/fmp4/MP4Decoder.cpp index 2cf296e6e3..edcbeec002 100644 --- a/dom/media/fmp4/MP4Decoder.cpp +++ b/dom/media/fmp4/MP4Decoder.cpp @@ -220,7 +220,7 @@ static const uint8_t sTestH264ExtraData[] = { static already_AddRefed CreateTestH264Decoder(layers::LayersBackend aBackend, VideoInfo& aConfig, - FlushableTaskQueue* aTaskQueue) + TaskQueue* aTaskQueue) { aConfig.mMimeType = "video/avc"; aConfig.mId = 1; @@ -253,8 +253,8 @@ MP4Decoder::IsVideoAccelerated(layers::LayersBackend aBackend, nsIGlobalObject* return nullptr; } - RefPtr taskQueue = - new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); + RefPtr taskQueue = + new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER)); VideoInfo config; RefPtr decoder(CreateTestH264Decoder(aBackend, config, taskQueue)); if (!decoder) { diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index 2973a443f6..864204e254 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -1301,9 +1301,21 @@ TrackBuffersManager::CompleteCodedFrameProcessing() // 6. Remove the media segment bytes from the beginning of the input buffer. // Clear our demuxer from any already processed data. - // As we have handled a complete media segment, it is safe to evict all data - // from the resource. - mCurrentInputBuffer->EvictAll(); + int64_t safeToEvict = std::min( + HasVideo() + ? mVideoTracks.mDemuxer->GetEvictionOffset(mVideoTracks.mLastParsedEndTime) + : INT64_MAX, + HasAudio() + ? mAudioTracks.mDemuxer->GetEvictionOffset(mAudioTracks.mLastParsedEndTime) + : INT64_MAX); + ErrorResult rv; + mCurrentInputBuffer->EvictBefore(safeToEvict, rv); + if (rv.Failed()) { + rv.SuppressException(); + RejectProcessing(NS_ERROR_OUT_OF_MEMORY, __func__); + return; + } + mInputDemuxer->NotifyDataRemoved(); RecreateParser(true); @@ -1666,9 +1678,11 @@ TrackBuffersManager::InsertFrames(TrackBuffer& aSamples, // We allow a fuzz factor in our interval of half a frame length, // as fuzz is +/- value, giving an effective leeway of a full frame // length. - TimeIntervals range(aIntervals); - range.SetFuzz(trackBuffer.mLongestFrameDuration / 2); - trackBuffer.mSanitizedBufferedRanges += range; + if (aIntervals.Length()) { + TimeIntervals range(aIntervals); + range.SetFuzz(trackBuffer.mLongestFrameDuration / 2); + trackBuffer.mSanitizedBufferedRanges += range; + } } void diff --git a/dom/media/moz.build b/dom/media/moz.build index 2ddfb7bcf1..75ccebbe80 100644 --- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -99,7 +99,6 @@ EXPORTS += [ 'DOMMediaStream.h', 'EncodedBufferCache.h', 'FileBlockCache.h', - 'FlushableTaskQueue.h', 'FrameStatistics.h', 'Intervals.h', 'Latency.h', @@ -209,7 +208,6 @@ UNIFIED_SOURCES += [ 'DOMMediaStream.cpp', 'EncodedBufferCache.cpp', 'FileBlockCache.cpp', - 'FlushableTaskQueue.cpp', 'GetUserMediaRequest.cpp', 'GraphDriver.cpp', 'Latency.cpp', diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index 061699599b..7ec4216583 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -19,11 +19,11 @@ namespace mozilla { AppleATDecoder::AppleATDecoder(const AudioInfo& aConfig, - FlushableTaskQueue* aAudioTaskQueue, + TaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback) : mConfig(aConfig) , mFileStreamError(false) - , mTaskQueue(aAudioTaskQueue) + , mTaskQueue(aTaskQueue) , mCallback(aCallback) , mConverter(nullptr) , mStream(nullptr) diff --git a/dom/media/platforms/apple/AppleATDecoder.h b/dom/media/platforms/apple/AppleATDecoder.h index 4d901071d0..669ce2961f 100644 --- a/dom/media/platforms/apple/AppleATDecoder.h +++ b/dom/media/platforms/apple/AppleATDecoder.h @@ -16,13 +16,13 @@ namespace mozilla { -class FlushableTaskQueue; +class TaskQueue; class MediaDataDecoderCallback; class AppleATDecoder : public MediaDataDecoder { public: AppleATDecoder(const AudioInfo& aConfig, - FlushableTaskQueue* aVideoTaskQueue, + TaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback); virtual ~AppleATDecoder(); @@ -47,7 +47,7 @@ public: bool mFileStreamError; private: - RefPtr mTaskQueue; + const RefPtr mTaskQueue; MediaDataDecoderCallback* mCallback; AudioConverterRef mConverter; AudioStreamBasicDescription mOutputFormat; diff --git a/dom/media/platforms/gonk/GonkDecoderModule.cpp b/dom/media/platforms/gonk/GonkDecoderModule.cpp index 72088c9a7d..bc7424410e 100644 --- a/dom/media/platforms/gonk/GonkDecoderModule.cpp +++ b/dom/media/platforms/gonk/GonkDecoderModule.cpp @@ -28,7 +28,7 @@ GonkDecoderModule::CreateVideoDecoder(const VideoInfo& aConfig, { RefPtr decoder = new GonkMediaDataDecoder(new GonkVideoDecoderManager(aImageContainer, aConfig), - aVideoTaskQueue, aCallback); + aCallback); return decoder.forget(); } @@ -40,7 +40,7 @@ GonkDecoderModule::CreateAudioDecoder(const AudioInfo& aConfig, { RefPtr decoder = new GonkMediaDataDecoder(new GonkAudioDecoderManager(aConfig), - aAudioTaskQueue, aCallback); + aCallback); return decoder.forget(); } diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp index 4eb6c2aa87..70bfbf558b 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp @@ -330,7 +330,6 @@ GonkDecoderManager::OnTaskLooper() #endif GonkMediaDataDecoder::GonkMediaDataDecoder(GonkDecoderManager* aManager, - FlushableTaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback) : mManager(aManager) { diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.h b/dom/media/platforms/gonk/GonkMediaDataDecoder.h index 2f440526d9..37ecaf84af 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.h +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.h @@ -186,7 +186,6 @@ private: class GonkMediaDataDecoder : public MediaDataDecoder { public: GonkMediaDataDecoder(GonkDecoderManager* aDecoderManager, - FlushableTaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback); ~GonkMediaDataDecoder(); diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.h b/dom/media/platforms/wmf/WMFMediaDataDecoder.h index 7356b45ce8..a50da030ec 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.h @@ -69,7 +69,7 @@ protected: class WMFMediaDataDecoder : public MediaDataDecoder { public: WMFMediaDataDecoder(MFTManager* aOutputSource, - TaskQueue* aAudioTaskQueue, + TaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback); ~WMFMediaDataDecoder(); @@ -116,7 +116,7 @@ private: // different configuration (typically resolution change). void ProcessConfigurationChanged(UniquePtr&& aConfig); - RefPtr mTaskQueue; + const RefPtr mTaskQueue; MediaDataDecoderCallback* mCallback; nsAutoPtr mMFTManager; diff --git a/dom/media/webm/NesteggPacketHolder.h b/dom/media/webm/NesteggPacketHolder.h index f786e66a17..c1d0b646f2 100644 --- a/dom/media/webm/NesteggPacketHolder.h +++ b/dom/media/webm/NesteggPacketHolder.h @@ -19,7 +19,12 @@ namespace mozilla { class NesteggPacketHolder { public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(NesteggPacketHolder) - NesteggPacketHolder() : mPacket(nullptr), mOffset(-1), mTimestamp(-1), mIsKeyframe(false) {} + NesteggPacketHolder() + : mPacket(nullptr) + , mOffset(-1) + , mTimestamp(-1) + , mDuration(-1) + , mIsKeyframe(false) {} bool Init(nestegg_packet* aPacket, int64_t aOffset, unsigned aTrack, bool aIsKeyframe) { @@ -36,12 +41,17 @@ public: mTrack = aTrack; mIsKeyframe = aIsKeyframe; + uint64_t duration_ns; + if (!nestegg_packet_duration(aPacket, &duration_ns)) { + mDuration = duration_ns / 1000; + } return true; } nestegg_packet* Packet() { MOZ_ASSERT(IsInitialized()); return mPacket; } int64_t Offset() { MOZ_ASSERT(IsInitialized()); return mOffset; } int64_t Timestamp() { MOZ_ASSERT(IsInitialized()); return mTimestamp; } + int64_t Duration() { MOZ_ASSERT(IsInitialized()); return mDuration; } unsigned Track() { MOZ_ASSERT(IsInitialized()); return mTrack; } bool IsKeyframe() { MOZ_ASSERT(IsInitialized()); return mIsKeyframe; } @@ -62,6 +72,9 @@ private: // Packet presentation timestamp in microseconds. int64_t mTimestamp; + // Packet duration in microseconds; -1 if unknown or retrieval failed. + int64_t mDuration; + // Track ID. unsigned mTrack; diff --git a/dom/media/webm/WebMDemuxer.cpp b/dom/media/webm/WebMDemuxer.cpp index 8230a03d1c..cfa577e792 100644 --- a/dom/media/webm/WebMDemuxer.cpp +++ b/dom/media/webm/WebMDemuxer.cpp @@ -51,12 +51,13 @@ static int webmdemux_read(void* aBuffer, size_t aLength, void* aUserData) { MOZ_ASSERT(aUserData); MOZ_ASSERT(aLength < UINT32_MAX); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); + WebMDemuxer::NestEggContext* context = + reinterpret_cast(aUserData); uint32_t count = aLength; - if (demuxer->IsMediaSource()) { - int64_t length = demuxer->GetEndDataOffset(); - int64_t position = demuxer->GetResource()->Tell(); - MOZ_ASSERT(position <= demuxer->GetResource()->GetLength()); + if (context->IsMediaSource()) { + int64_t length = context->GetEndDataOffset(); + int64_t position = context->GetResource()->Tell(); + MOZ_ASSERT(position <= context->GetResource()->GetLength()); MOZ_ASSERT(position <= length); if (length >= 0 && count + position > length) { count = length - position; @@ -65,7 +66,7 @@ static int webmdemux_read(void* aBuffer, size_t aLength, void* aUserData) } uint32_t bytes = 0; nsresult rv = - demuxer->GetResource()->Read(static_cast(aBuffer), count, &bytes); + context->GetResource()->Read(static_cast(aBuffer), count, &bytes); bool eof = bytes < aLength; return NS_FAILED(rv) ? -1 : eof ? 0 : 1; } @@ -73,16 +74,16 @@ static int webmdemux_read(void* aBuffer, size_t aLength, void* aUserData) static int webmdemux_seek(int64_t aOffset, int aWhence, void* aUserData) { MOZ_ASSERT(aUserData); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); - nsresult rv = demuxer->GetResource()->Seek(aWhence, aOffset); + WebMDemuxer::NestEggContext* context = reinterpret_cast(aUserData); + nsresult rv = context->GetResource()->Seek(aWhence, aOffset); return NS_SUCCEEDED(rv) ? 0 : -1; } static int64_t webmdemux_tell(void* aUserData) { MOZ_ASSERT(aUserData); - WebMDemuxer* demuxer = reinterpret_cast(aUserData); - return demuxer->GetResource()->Tell(); + WebMDemuxer::NestEggContext* context = reinterpret_cast(aUserData); + return context->GetResource()->Tell(); } static void webmdemux_log(nestegg* aContext, @@ -127,6 +128,28 @@ static void webmdemux_log(nestegg* aContext, va_end(args); } +WebMDemuxer::NestEggContext::~NestEggContext() +{ + if (mContext) { + nestegg_destroy(mContext); + } +} + +int +WebMDemuxer::NestEggContext::Init() +{ + nestegg_io io; + io.read = webmdemux_read; + io.seek = webmdemux_seek; + io.tell = webmdemux_tell; + io.userdata = this; + + // While reading the metadata, we do not really care about which nestegg + // context is being used so long that they are both initialised. + // For reading the metadata however, we will use mVideoContext. + return nestegg_init(&mContext, io, &webmdemux_log, + mParent->IsMediaSource() ? mResource.GetLength() : -1); +} WebMDemuxer::WebMDemuxer(MediaResource* aResource) : WebMDemuxer(aResource, false) @@ -134,10 +157,10 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource) } WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) - : mResource(aResource) + : mVideoContext(this, aResource) + , mAudioContext(this, aResource) , mBufferedState(nullptr) , mInitData(nullptr) - , mContext(nullptr) , mVideoTrack(0) , mAudioTrack(0) , mSeekPreroll(0) @@ -153,8 +176,8 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) WebMDemuxer::~WebMDemuxer() { - Reset(); - Cleanup(); + Reset(TrackInfo::kVideoTrack); + Reset(TrackInfo::kAudioTrack); } RefPtr @@ -228,39 +251,34 @@ WebMDemuxer::GetTrackDemuxer(TrackInfo::TrackType aType, uint32_t aTrackNumber) } nsresult -WebMDemuxer::Reset() +WebMDemuxer::Reset(TrackInfo::TrackType aType) { - mVideoPackets.Reset(); - mAudioPackets.Reset(); - return NS_OK; -} - -void -WebMDemuxer::Cleanup() -{ - if (mContext) { - nestegg_destroy(mContext); - mContext = nullptr; + if (aType == TrackInfo::kVideoTrack) { + mVideoPackets.Reset(); + } else { + mAudioPackets.Reset(); } - mBufferedState = nullptr; + return NS_OK; } nsresult WebMDemuxer::ReadMetadata() { - nestegg_io io; - io.read = webmdemux_read; - io.seek = webmdemux_seek; - io.tell = webmdemux_tell; - io.userdata = this; - int r = nestegg_init(&mContext, io, &webmdemux_log, - IsMediaSource() ? mResource.GetLength() : -1); + int r = mVideoContext.Init(); if (r == -1) { return NS_ERROR_FAILURE; } + if (mAudioContext.Init() == -1) { + return NS_ERROR_FAILURE; + } + + // For reading the metadata we can only use the video resource/context. + MediaResourceIndex& resource = Resource(TrackInfo::kVideoTrack); + nestegg* context = Context(TrackInfo::kVideoTrack); + { // Check how much data nestegg read and force feed it to BufferedState. - RefPtr buffer = mResource.MediaReadAt(0, mResource.Tell()); + RefPtr buffer = resource.MediaReadAt(0, resource.Tell()); if (!buffer) { return NS_ERROR_FAILURE; } @@ -268,33 +286,33 @@ WebMDemuxer::ReadMetadata() if (mBufferedState->GetInitEndOffset() < 0) { return NS_ERROR_FAILURE; } - MOZ_ASSERT(mBufferedState->GetInitEndOffset() <= mResource.Tell()); + MOZ_ASSERT(mBufferedState->GetInitEndOffset() <= resource.Tell()); } - mInitData = mResource.MediaReadAt(0, mBufferedState->GetInitEndOffset()); + mInitData = resource.MediaReadAt(0, mBufferedState->GetInitEndOffset()); if (!mInitData || mInitData->Length() != size_t(mBufferedState->GetInitEndOffset())) { return NS_ERROR_FAILURE; } unsigned int ntracks = 0; - r = nestegg_track_count(mContext, &ntracks); + r = nestegg_track_count(context, &ntracks); if (r == -1) { return NS_ERROR_FAILURE; } for (unsigned int track = 0; track < ntracks; ++track) { - int id = nestegg_track_codec_id(mContext, track); + int id = nestegg_track_codec_id(context, track); if (id == -1) { return NS_ERROR_FAILURE; } - int type = nestegg_track_type(mContext, track); + int type = nestegg_track_type(context, track); if (type == NESTEGG_TRACK_VIDEO && !mHasVideo) { nestegg_video_params params; - r = nestegg_track_video_params(mContext, track, ¶ms); + r = nestegg_track_video_params(context, track, ¶ms); if (r == -1) { return NS_ERROR_FAILURE; } - mVideoCodec = nestegg_track_codec_id(mContext, track); + mVideoCodec = nestegg_track_codec_id(context, track); switch(mVideoCodec) { case NESTEGG_CODEC_VP8: mInfo.mVideo.mMimeType = "video/webm; codecs=vp8"; @@ -360,13 +378,13 @@ WebMDemuxer::ReadMetadata() break; } uint64_t duration = 0; - r = nestegg_duration(mContext, &duration); + r = nestegg_duration(context, &duration); if (!r) { mInfo.mVideo.mDuration = media::TimeUnit::FromNanoseconds(duration).ToMicroseconds(); } } else if (type == NESTEGG_TRACK_AUDIO && !mHasAudio) { nestegg_audio_params params; - r = nestegg_track_audio_params(mContext, track, ¶ms); + r = nestegg_track_audio_params(context, track, ¶ms); if (r == -1) { return NS_ERROR_FAILURE; } @@ -374,7 +392,7 @@ WebMDemuxer::ReadMetadata() mAudioTrack = track; mHasAudio = true; mCodecDelay = media::TimeUnit::FromNanoseconds(params.codec_delay).ToMicroseconds(); - mAudioCodec = nestegg_track_codec_id(mContext, track); + mAudioCodec = nestegg_track_codec_id(context, track); if (mAudioCodec == NESTEGG_CODEC_VORBIS) { mInfo.mAudio.mMimeType = "audio/webm; codecs=vorbis"; } else if (mAudioCodec == NESTEGG_CODEC_OPUS) { @@ -388,7 +406,7 @@ WebMDemuxer::ReadMetadata() mInfo.mAudio.mChannels = params.channels; unsigned int nheaders = 0; - r = nestegg_track_codec_data_count(mContext, track, &nheaders); + r = nestegg_track_codec_data_count(context, track, &nheaders); if (r == -1) { return NS_ERROR_FAILURE; } @@ -398,7 +416,7 @@ WebMDemuxer::ReadMetadata() for (uint32_t header = 0; header < nheaders; ++header) { unsigned char* data = 0; size_t length = 0; - r = nestegg_track_codec_data(mContext, track, header, &data, &length); + r = nestegg_track_codec_data(context, track, header, &data, &length); if (r == -1) { return NS_ERROR_FAILURE; } @@ -422,7 +440,7 @@ WebMDemuxer::ReadMetadata() headerLens[0]); } uint64_t duration = 0; - r = nestegg_duration(mContext, &duration); + r = nestegg_duration(context, &duration); if (!r) { mInfo.mAudio.mDuration = media::TimeUnit::FromNanoseconds(duration).ToMicroseconds(); } @@ -434,13 +452,15 @@ WebMDemuxer::ReadMetadata() bool WebMDemuxer::IsSeekable() const { - return mContext && nestegg_has_cues(mContext); + return Context(TrackInfo::kVideoTrack) && + nestegg_has_cues(Context(TrackInfo::kVideoTrack)); } bool WebMDemuxer::IsSeekableOnlyInBufferedRanges() const { - return mContext && !nestegg_has_cues(mContext); + return Context(TrackInfo::kVideoTrack) && + !nestegg_has_cues(Context(TrackInfo::kVideoTrack)); } void @@ -449,7 +469,8 @@ WebMDemuxer::EnsureUpToDateIndex() if (!mNeedReIndex || !mInitData) { return; } - AutoPinned resource(mResource.GetResource()); + AutoPinned resource( + Resource(TrackInfo::kVideoTrack).GetResource()); MediaByteRangeSet byteRanges; nsresult rv = resource->GetCachedRanges(byteRanges); if (NS_FAILED(rv) || !byteRanges.Length()) { @@ -463,7 +484,7 @@ WebMDemuxer::EnsureUpToDateIndex() return; } mLastWebMBlockOffset = mBufferedState->GetLastBlockOffset(); - MOZ_ASSERT(mLastWebMBlockOffset <= mResource.GetLength()); + MOZ_ASSERT(mLastWebMBlockOffset <= resource->GetLength()); } void @@ -510,8 +531,9 @@ WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, MediaRawDataQueue *aSampl return false; } int64_t tstamp = holder->Timestamp(); + int64_t duration = holder->Duration(); - // The end time of this frame is the start time of the next frame. Fetch + // The end time of this frame is the start time of the next frame. Fetch // the timestamp of the next packet for this track. If we've reached the // end of the resource, use the file's duration as the end time of this // video frame. @@ -521,6 +543,8 @@ WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, MediaRawDataQueue *aSampl if (next_holder) { next_tstamp = next_holder->Timestamp(); PushAudioPacket(next_holder); + } else if (duration >= 0) { + next_tstamp = tstamp + duration; } else if (!mIsMediaSource || (mIsMediaSource && mLastAudioFrameTime.isSome())) { next_tstamp = tstamp; @@ -534,6 +558,8 @@ WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, MediaRawDataQueue *aSampl if (next_holder) { next_tstamp = next_holder->Timestamp(); PushVideoPacket(next_holder); + } else if (duration >= 0) { + next_tstamp = tstamp + duration; } else if (!mIsMediaSource || (mIsMediaSource && mLastVideoFrameTime.isSome())) { next_tstamp = tstamp; @@ -616,56 +642,45 @@ WebMDemuxer::NextPacket(TrackInfo::TrackType aType) { bool isVideo = aType == TrackInfo::kVideoTrack; - // The packet queue that packets will be pushed on if they - // are not the type we are interested in. - WebMPacketQueue& otherPackets = isVideo ? mAudioPackets : mVideoPackets; - - // The packet queue for the type that we are interested in. - WebMPacketQueue &packets = isVideo ? mVideoPackets : mAudioPackets; - // Flag to indicate that we do need to playback these types of // packets. bool hasType = isVideo ? mHasVideo : mHasAudio; - // Flag to indicate that we do need to playback the other type - // of track. - bool hasOtherType = isVideo ? mHasAudio : mHasVideo; + if (!hasType) { + return nullptr; + } - // Track we are interested in - uint32_t ourTrack = isVideo ? mVideoTrack : mAudioTrack; - - // Value of other track - uint32_t otherTrack = isVideo ? mAudioTrack : mVideoTrack; + // The packet queue for the type that we are interested in. + WebMPacketQueue &packets = isVideo ? mVideoPackets : mAudioPackets; if (packets.GetSize() > 0) { return packets.PopFront(); } + // Track we are interested in + uint32_t ourTrack = isVideo ? mVideoTrack : mAudioTrack; + do { - RefPtr holder = DemuxPacket(); + RefPtr holder = DemuxPacket(aType); if (!holder) { return nullptr; } - if (hasOtherType && otherTrack == holder->Track()) { - // Save the packet for when we want these packets - otherPackets.Push(holder); - continue; - } - - // The packet is for the track we want to play - if (hasType && ourTrack == holder->Track()) { + if (ourTrack == holder->Track()) { return holder; } } while (true); } RefPtr -WebMDemuxer::DemuxPacket() +WebMDemuxer::DemuxPacket(TrackInfo::TrackType aType) { nestegg_packet* packet; - int r = nestegg_read_packet(mContext, &packet); - if (r <= 0) { + int r = nestegg_read_packet(Context(aType), &packet); + if (r == 0) { + nestegg_read_reset(Context(aType)); + return nullptr; + } else if (r < 0) { return nullptr; } @@ -675,7 +690,7 @@ WebMDemuxer::DemuxPacket() return nullptr; } - int64_t offset = mResource.Tell(); + int64_t offset = Resource(aType).Tell(); RefPtr holder = new NesteggPacketHolder(); if (!holder->Init(packet, offset, track, false)) { return nullptr; @@ -697,13 +712,14 @@ WebMDemuxer::PushVideoPacket(NesteggPacketHolder* aItem) } nsresult -WebMDemuxer::SeekInternal(const media::TimeUnit& aTarget) +WebMDemuxer::SeekInternal(TrackInfo::TrackType aType, + const media::TimeUnit& aTarget) { EnsureUpToDateIndex(); uint32_t trackToSeek = mHasVideo ? mVideoTrack : mAudioTrack; uint64_t target = aTarget.ToNanoseconds(); - if (NS_FAILED(Reset())) { + if (NS_FAILED(Reset(aType))) { return NS_ERROR_FAILURE; } @@ -724,7 +740,7 @@ WebMDemuxer::SeekInternal(const media::TimeUnit& aTarget) media::TimeUnit::FromNanoseconds(startTime).ToSeconds(), media::TimeUnit::FromNanoseconds(target).ToSeconds()); } - int r = nestegg_track_seek(mContext, trackToSeek, target); + int r = nestegg_track_seek(Context(aType), trackToSeek, target); if (r == -1) { WEBM_DEBUG("track_seek for track %u to %f failed, r=%d", trackToSeek, media::TimeUnit::FromNanoseconds(target).ToSeconds(), r); @@ -736,7 +752,7 @@ WebMDemuxer::SeekInternal(const media::TimeUnit& aTarget) return NS_ERROR_FAILURE; } - r = nestegg_offset_seek(mContext, offset); + r = nestegg_offset_seek(Context(aType), offset); if (r == -1) { WEBM_DEBUG("and nestegg_offset_seek to %" PRIu64 " failed", offset); return NS_ERROR_FAILURE; @@ -744,8 +760,11 @@ WebMDemuxer::SeekInternal(const media::TimeUnit& aTarget) WEBM_DEBUG("got offset from buffered state: %" PRIu64 "", offset); } - mLastAudioFrameTime.reset(); - mLastVideoFrameTime.reset(); + if (aType == TrackInfo::kAudioTrack) { + mLastAudioFrameTime.reset(); + } else { + mLastVideoFrameTime.reset(); + } return NS_OK; } @@ -754,7 +773,8 @@ media::TimeIntervals WebMDemuxer::GetBuffered() { EnsureUpToDateIndex(); - AutoPinned resource(mResource.GetResource()); + AutoPinned resource( + Resource(TrackInfo::kVideoTrack).GetResource()); media::TimeIntervals buffered; @@ -765,7 +785,7 @@ WebMDemuxer::GetBuffered() } uint64_t duration = 0; uint64_t startOffset = 0; - if (!nestegg_duration(mContext, &duration)) { + if (!nestegg_duration(Context(TrackInfo::kVideoTrack), &duration)) { if(mBufferedState->GetStartTime(&startOffset)) { duration += startOffset; } @@ -835,7 +855,7 @@ WebMTrackDemuxer::Seek(media::TimeUnit aTime) media::TimeUnit seekTime = aTime; mSamples.Reset(); - mParent->SeekInternal(aTime); + mParent->SeekInternal(mType, aTime); mParent->GetNextPacket(mType, &mSamples); mNeedKeyframe = true; @@ -892,7 +912,7 @@ WebMTrackDemuxer::GetSamples(int32_t aNumSamples) void WebMTrackDemuxer::SetNextKeyFrameTime() { - if (mType != TrackInfo::kVideoTrack) { + if (mType != TrackInfo::kVideoTrack || mParent->IsMediaSource()) { return; } @@ -956,7 +976,7 @@ WebMTrackDemuxer::Reset() mNeedKeyframe = true; if (buffered.Length()) { WEBM_DEBUG("Seek to start point: %f", buffered.Start(0).ToSeconds()); - mParent->SeekInternal(buffered.Start(0)); + mParent->SeekInternal(mType, buffered.Start(0)); SetNextKeyFrameTime(); } else { mNextKeyframeTime.reset(); @@ -991,11 +1011,13 @@ WebMTrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold) uint32_t parsed = 0; bool found = false; RefPtr sample; + int64_t sampleTime; WEBM_DEBUG("TimeThreshold: %f", aTimeThreshold.ToSeconds()); while (!found && (sample = NextSample())) { parsed++; - if (sample->mKeyframe && sample->mTime >= aTimeThreshold.ToMicroseconds()) { + sampleTime = sample->mTime; + if (sample->mKeyframe && sampleTime >= aTimeThreshold.ToMicroseconds()) { found = true; mSamples.Reset(); mSamples.PushFront(sample.forget()); @@ -1004,7 +1026,7 @@ WebMTrackDemuxer::SkipToNextRandomAccessPoint(media::TimeUnit aTimeThreshold) SetNextKeyFrameTime(); if (found) { WEBM_DEBUG("next sample: %f (parsed: %d)", - media::TimeUnit::FromMicroseconds(sample->mTime).ToSeconds(), + media::TimeUnit::FromMicroseconds(sampleTime).ToSeconds(), parsed); return SkipAccessPointPromise::CreateAndResolve(parsed, __func__); } else { @@ -1025,5 +1047,16 @@ WebMTrackDemuxer::BreakCycles() mParent = nullptr; } +int64_t +WebMTrackDemuxer::GetEvictionOffset(const media::TimeUnit& aTime) +{ + int64_t offset; + if (!mParent->GetOffsetForTime(aTime.ToNanoseconds(), &offset)) { + return 0; + } + + return offset; +} + #undef WEBM_DEBUG } // namespace mozilla diff --git a/dom/media/webm/WebMDemuxer.h b/dom/media/webm/WebMDemuxer.h index d54043bc96..4346594b8f 100644 --- a/dom/media/webm/WebMDemuxer.h +++ b/dom/media/webm/WebMDemuxer.h @@ -107,7 +107,7 @@ public: // Demux next WebM packet and append samples to MediaRawDataQueue bool GetNextPacket(TrackInfo::TrackType aType, MediaRawDataQueue *aSamples); - nsresult Reset(); + nsresult Reset(TrackInfo::TrackType aType); // Pushes a packet to the front of the audio packet queue. void PushAudioPacket(NesteggPacketHolder* aItem); @@ -116,33 +116,54 @@ public: void PushVideoPacket(NesteggPacketHolder* aItem); // Public accessor for nestegg callbacks - MediaResourceIndex* GetResource() - { - return &mResource; - } - - int64_t GetEndDataOffset() const - { - return (!mIsMediaSource || mLastWebMBlockOffset < 0) - ? mResource.GetLength() : mLastWebMBlockOffset; - } - int64_t IsMediaSource() const + bool IsMediaSource() const { return mIsMediaSource; } + int64_t LastWebMBlockOffset() const + { + return mLastWebMBlockOffset; + } + + struct NestEggContext { + NestEggContext(WebMDemuxer* aParent, MediaResource* aResource) + : mParent(aParent) + , mResource(aResource) + , mContext(nullptr) {} + + ~NestEggContext(); + + int Init(); + + // Public accessor for nestegg callbacks + + bool IsMediaSource() const { return mParent->IsMediaSource(); } + MediaResourceIndex* GetResource() { return &mResource; } + + int64_t GetEndDataOffset() const + { + return (!mParent->IsMediaSource() || mParent->LastWebMBlockOffset() < 0) + ? mResource.GetLength() : mParent->LastWebMBlockOffset(); + } + + WebMDemuxer* mParent; + MediaResourceIndex mResource; + nestegg* mContext; + }; + private: friend class WebMTrackDemuxer; ~WebMDemuxer(); - void Cleanup(); void InitBufferedState(); nsresult ReadMetadata(); void NotifyDataArrived() override; void NotifyDataRemoved() override; void EnsureUpToDateIndex(); media::TimeIntervals GetBuffered(); - nsresult SeekInternal(const media::TimeUnit& aTarget); + nsresult SeekInternal(TrackInfo::TrackType aType, + const media::TimeUnit& aTarget); // Read a packet from the nestegg file. Returns nullptr if all packets for // the particular track have been read. Pass TrackInfo::kVideoTrack or @@ -151,9 +172,23 @@ private: // Internal method that demuxes the next packet from the stream. The caller // is responsible for making sure it doesn't get lost. - RefPtr DemuxPacket(); + RefPtr DemuxPacket(TrackInfo::TrackType aType); + + // libnestegg audio and video context for webm container. + // Access on reader's thread only. + NestEggContext mVideoContext; + NestEggContext mAudioContext; + MediaResourceIndex& Resource(TrackInfo::TrackType aType) + { + return aType == TrackInfo::kVideoTrack + ? mVideoContext.mResource : mAudioContext.mResource; + } + nestegg* Context(TrackInfo::TrackType aType) const + { + return aType == TrackInfo::kVideoTrack + ? mVideoContext.mContext : mAudioContext.mContext; + } - MediaResourceIndex mResource; MediaInfo mInfo; nsTArray> mDemuxers; @@ -162,10 +197,6 @@ private: RefPtr mBufferedState; RefPtr mInitData; - // libnestegg context for webm container. - // Access on reader's thread for main demuxer, - // or main thread for cloned demuxer - nestegg* mContext; // Queue of video and audio packets that have been read but not decoded. WebMPacketQueue mVideoPackets; @@ -230,6 +261,8 @@ public: media::TimeIntervals GetBuffered() override; + int64_t GetEvictionOffset(const media::TimeUnit& aTime) override; + void BreakCycles() override; private: diff --git a/dom/media/webspeech/recognition/SpeechStreamListener.cpp b/dom/media/webspeech/recognition/SpeechStreamListener.cpp index 773ec27a2c..d1d0c4e898 100644 --- a/dom/media/webspeech/recognition/SpeechStreamListener.cpp +++ b/dom/media/webspeech/recognition/SpeechStreamListener.cpp @@ -26,13 +26,11 @@ SpeechStreamListener::~SpeechStreamListener() } void -SpeechStreamListener::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, - TrackID aID, - StreamTime aTrackOffset, - uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia, - MediaStream* aInputStream, - TrackID aInputTrackID) +SpeechStreamListener::NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID, + StreamTime aTrackOffset, + const AudioSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) { AudioSegment* audio = const_cast( static_cast(&aQueuedMedia)); diff --git a/dom/media/webspeech/recognition/SpeechStreamListener.h b/dom/media/webspeech/recognition/SpeechStreamListener.h index 7dd596118e..a3e5efcf08 100644 --- a/dom/media/webspeech/recognition/SpeechStreamListener.h +++ b/dom/media/webspeech/recognition/SpeechStreamListener.h @@ -24,12 +24,11 @@ public: explicit SpeechStreamListener(SpeechRecognition* aRecognition); ~SpeechStreamListener(); - void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, - StreamTime aTrackOffset, - uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia, - MediaStream* aInputStream, - TrackID aInputTrackID) override; + void NotifyQueuedAudioData(MediaStreamGraph* aGraph, TrackID aID, + StreamTime aTrackOffset, + const AudioSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override; void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamListener::MediaStreamGraphEvent event) override; diff --git a/layout/media/symbols.def.in b/layout/media/symbols.def.in index 3994f11425..9113fca608 100644 --- a/layout/media/symbols.def.in +++ b/layout/media/symbols.def.in @@ -10,11 +10,13 @@ nestegg_free_packet nestegg_init nestegg_offset_seek nestegg_packet_count -nestegg_packet_discard_padding nestegg_packet_data +nestegg_packet_discard_padding +nestegg_packet_duration nestegg_packet_track nestegg_packet_tstamp nestegg_read_packet +nestegg_read_reset nestegg_track_audio_params nestegg_track_codec_data nestegg_track_codec_data_count diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 618858ae46..d4d51e46bc 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -1976,13 +1976,6 @@ public: } // Implement MediaStreamListener - void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, - StreamTime offset, - uint32_t events, - const MediaSegment& queued_media, - MediaStream* input_stream, - TrackID input_tid) override {} - void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override { MOZ_ASSERT(source_); @@ -2138,14 +2131,7 @@ public: #endif } - // Implement MediaStreamListener - void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, - StreamTime offset, - uint32_t events, - const MediaSegment& queued_media, - MediaStream* input_stream, - TrackID input_tid) override {} void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override { ReentrantMonitorAutoEnter enter(monitor_); diff --git a/media/webrtc/signaling/test/FakeMediaStreams.h b/media/webrtc/signaling/test/FakeMediaStreams.h index 5ee5979b31..ad78e646f8 100644 --- a/media/webrtc/signaling/test/FakeMediaStreams.h +++ b/media/webrtc/signaling/test/FakeMediaStreams.h @@ -93,8 +93,13 @@ public: uint32_t aTrackEvents, const mozilla::MediaSegment& aQueuedMedia, Fake_MediaStream* aInputStream, - mozilla::TrackID aInputTrackID) = 0; + mozilla::TrackID aInputTrackID) {} virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0; + virtual void NotifyQueuedAudioData(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID, + mozilla::StreamTime aTrackOffset, + const mozilla::AudioSegment& aQueuedMedia, + Fake_MediaStream* aInputStream, + mozilla::TrackID aInputTrackID) {} NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener) };