From 3ef89dda8788dd778c8490243cb7276fa4cc46e1 Mon Sep 17 00:00:00 2001 From: trav90 Date: Sun, 18 Sep 2016 19:21:40 -0500 Subject: [PATCH] Fire loadedmetadata before encrypted event --- dom/html/HTMLMediaElement.cpp | 8 +- dom/media/MediaDecoderStateMachine.cpp | 25 +++- dom/media/MediaDecoderStateMachine.h | 1 + dom/media/MediaInfo.h | 26 +++- dom/media/fmp4/MP4Reader.cpp | 133 ++++++++++++-------- dom/media/fmp4/MP4Reader.h | 5 +- dom/media/mediasource/MediaSourceReader.cpp | 26 +++- dom/media/mediasource/MediaSourceReader.h | 1 + dom/media/mediasource/TrackBuffer.cpp | 10 +- dom/media/mediasource/TrackBuffer.h | 9 +- 10 files changed, 174 insertions(+), 70 deletions(-) diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index b35635b781..ab0cb924a3 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -3142,7 +3142,7 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, nsAutoPtr aTags) { mMediaInfo = *aInfo; - mIsEncrypted = aInfo->mIsEncrypted; + mIsEncrypted = aInfo->IsEncrypted(); mTags = aTags.forget(); mLoadedDataFired = false; ChangeReadyState(nsIDOMHTMLMediaElement::HAVE_METADATA); @@ -3162,6 +3162,12 @@ void HTMLMediaElement::MetadataLoaded(const MediaInfo* aInfo, ProcessMediaFragmentURI(); mDecoder->SetFragmentEndTime(mFragmentEnd); } + if (mIsEncrypted) { + if (!mMediaSource) { + DecodeError(); + return; + } + } // Expose the tracks to JS directly. for (OutputMediaStream& out : mOutputStreams) { diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index eb67ba7fc7..74f0f53fc5 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -1320,6 +1320,7 @@ static const char* const gMachineStateStr[] = { "NONE", "DECODING_METADATA", "WAIT_FOR_RESOURCES", + "WAIT_FOR_RESOURCES", "DECODING_FIRSTFRAME", "DORMANT", "DECODING", @@ -1627,13 +1628,17 @@ void MediaDecoderStateMachine::DoNotifyWaitingForResourcesStatusChanged() { NS_ASSERTION(OnDecodeThread(), "Should be on decode thread."); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - if (mState != DECODER_STATE_WAIT_FOR_RESOURCES) { - return; - } DECODER_LOG("DoNotifyWaitingForResourcesStatusChanged"); - // The reader is no longer waiting for resources (say a hardware decoder), - // we can now proceed to decode metadata. - SetState(DECODER_STATE_DECODING_NONE); + + if (mState == DECODER_STATE_WAIT_FOR_RESOURCES) { + // The reader is no longer waiting for resources (say a hardware decoder), + // we can now proceed to decode metadata. + SetState(DECODER_STATE_DECODING_NONE); + } else if (mState == DECODER_STATE_WAIT_FOR_CDM && + !mReader->IsWaitingOnCDMResource()) { + SetState(DECODER_STATE_DECODING_FIRSTFRAME); + EnqueueDecodeFirstFrameTask(); + } ScheduleStateMachine(); } @@ -2300,6 +2305,13 @@ nsresult MediaDecoderStateMachine::DecodeMetadata() } if (mState == DECODER_STATE_DECODING_METADATA) { + if (mReader->IsWaitingOnCDMResource()) { + // Metadata parsing was successful but we're still waiting for CDM caps + // to become available so that we can build the correct decryptor/decoder. + SetState(DECODER_STATE_WAIT_FOR_CDM); + return NS_OK; + } + SetState(DECODER_STATE_DECODING_FIRSTFRAME); res = EnqueueDecodeFirstFrameTask(); if (NS_FAILED(res)) { @@ -2717,6 +2729,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine() return NS_OK; } + case DECODER_STATE_WAIT_FOR_CDM: case DECODER_STATE_WAIT_FOR_RESOURCES: { return NS_OK; } diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index be9515eb2a..dedbcf43d6 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -139,6 +139,7 @@ public: DECODER_STATE_DECODING_NONE, DECODER_STATE_DECODING_METADATA, DECODER_STATE_WAIT_FOR_RESOURCES, + DECODER_STATE_WAIT_FOR_CDM, DECODER_STATE_DECODING_FIRSTFRAME, DECODER_STATE_DORMANT, DECODER_STATE_DECODING, diff --git a/dom/media/MediaInfo.h b/dom/media/MediaInfo.h index 5cc7aa40c3..990d43eab7 100644 --- a/dom/media/MediaInfo.h +++ b/dom/media/MediaInfo.h @@ -10,6 +10,7 @@ #include "nsRect.h" #include "ImageTypes.h" #include "nsString.h" +#include "nsTArray.h" #include "StreamBuffer.h" // for TrackID namespace mozilla { @@ -105,10 +106,22 @@ public: TrackInfo mTrackInfo; }; +class EncryptionInfo { +public: + EncryptionInfo() : mIsEncrypted(false) {} + + // Encryption type to be passed to JS. Usually `cenc'. + nsString mType; + + // Encryption data. + nsTArray mInitData; + + // True if the stream has encryption metadata + bool mIsEncrypted; +}; + class MediaInfo { public: - MediaInfo() : mIsEncrypted(false) {} - bool HasVideo() const { return mVideo.mHasVideo; @@ -119,16 +132,21 @@ public: return mAudio.mHasAudio; } + bool IsEncrypted() const + { + return mCrypto.mIsEncrypted; + } + bool HasValidMedia() const { return HasVideo() || HasAudio(); } - bool mIsEncrypted; - // TODO: Store VideoInfo and AudioIndo in arrays to support multi-tracks. VideoInfo mVideo; AudioInfo mAudio; + + EncryptionInfo mCrypto; }; } // namespace mozilla diff --git a/dom/media/fmp4/MP4Reader.cpp b/dom/media/fmp4/MP4Reader.cpp index c6bc6144d3..94eb38aba9 100644 --- a/dom/media/fmp4/MP4Reader.cpp +++ b/dom/media/fmp4/MP4Reader.cpp @@ -151,6 +151,7 @@ MP4Reader::MP4Reader(AbstractMediaDecoder* aDecoder) , mDemuxerInitialized(false) , mFoundSPSForTelemetry(false) , mIsEncrypted(false) + , mAreDecodersSetup(false) , mIndexReady(false) , mLastSeenEnd(-1) , mDemuxerMonitor("MP4 Demuxer") @@ -265,7 +266,7 @@ void MP4Reader::RequestCodecResource() { } } -bool MP4Reader::IsWaitingOnCodecResource() { +bool MP4Reader::IsWaitingMediaResources() { return mVideo.mDecoder && mVideo.mDecoder->IsWaitingMediaResources(); } @@ -274,15 +275,6 @@ bool MP4Reader::IsWaitingOnCDMResource() { return false; } -bool MP4Reader::IsWaitingMediaResources() -{ - // IsWaitingOnCDMResource() *must* come first, because we don't know whether - // we can create a decoder until the CDM is initialized and it has told us - // whether *it* will decode, or whether we need to create a PDM to do the - // decoding - return IsWaitingOnCDMResource() || IsWaitingOnCodecResource(); -} - void MP4Reader::ExtractCryptoInitData(nsTArray& aInitData) { @@ -344,7 +336,7 @@ MP4Reader::ReadMetadata(MediaInfo* aInfo, { MonitorAutoUnlock unlock(mDemuxerMonitor); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mInfo.mIsEncrypted = mIsEncrypted = mDemuxer->Crypto().valid; + mInfo.mCrypto.mIsEncrypted = mIsEncrypted = mCrypto.valid; } // Remember that we've initialized the demuxer, so that if we're decoding @@ -357,57 +349,18 @@ MP4Reader::ReadMetadata(MediaInfo* aInfo, return NS_OK; } - if (mDemuxer->Crypto().valid) { - // EME not supported. - return NS_ERROR_FAILURE; - } else { - mPlatform = PlatformDecoderModule::Create(); - NS_ENSURE_TRUE(mPlatform, NS_ERROR_FAILURE); - } - if (HasAudio()) { const AudioDecoderConfig& audio = mDemuxer->AudioConfig(); - if (mInfo.mAudio.mHasAudio && !IsSupportedAudioMimeType(audio.mime_type)) { - return NS_ERROR_FAILURE; - } mInfo.mAudio.mRate = audio.samples_per_second; mInfo.mAudio.mChannels = audio.channel_count; mAudio.mCallback = new DecoderCallback(this, kAudio); - mAudio.mDecoder = mPlatform->CreateAudioDecoder(audio, - mAudio.mTaskQueue, - mAudio.mCallback); - NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, NS_ERROR_FAILURE); - nsresult rv = mAudio.mDecoder->Init(); - NS_ENSURE_SUCCESS(rv, rv); } if (HasVideo()) { const VideoDecoderConfig& video = mDemuxer->VideoConfig(); - if (mInfo.mVideo.mHasVideo && !IsSupportedVideoMimeType(video.mime_type)) { - return NS_ERROR_FAILURE; - } mInfo.mVideo.mDisplay = nsIntSize(video.display_width, video.display_height); mVideo.mCallback = new DecoderCallback(this, kVideo); - if (mSharedDecoderManager && mPlatform->SupportsSharedDecoders(video)) { - mVideo.mDecoder = - mSharedDecoderManager->CreateVideoDecoder(mPlatform, - video, - mLayersBackendType, - mDecoder->GetImageContainer(), - mVideo.mTaskQueue, - mVideo.mCallback); - } else { - mVideo.mDecoder = mPlatform->CreateVideoDecoder(video, - mLayersBackendType, - mDecoder->GetImageContainer(), - mVideo.mTaskQueue, - mVideo.mCallback); - } - NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, NS_ERROR_FAILURE); - nsresult rv = mVideo.mDecoder->Init(); - NS_ENSURE_SUCCESS(rv, rv); - mInfo.mVideo.mIsHardwareAccelerated = mVideo.mDecoder->IsHardwareAccelerated(); // Collect telemetry from h264 AVCC SPS. if (!mFoundSPSForTelemetry) { @@ -415,6 +368,16 @@ MP4Reader::ReadMetadata(MediaInfo* aInfo, } } + if (mIsEncrypted) { + nsTArray initData; + ExtractCryptoInitData(initData); + if (initData.Length() == 0) { + return NS_ERROR_FAILURE; + } + mInfo.mCrypto.mInitData = initData; + mInfo.mCrypto.mType = NS_LITERAL_STRING("cenc"); + } + // Get the duration, and report it to the decoder if we have it. Microseconds duration; { @@ -429,12 +392,71 @@ MP4Reader::ReadMetadata(MediaInfo* aInfo, *aInfo = mInfo; *aTags = nullptr; + if (!IsWaitingMediaResources() && !IsWaitingOnCDMResource()) { + NS_ENSURE_TRUE(EnsureDecodersSetup(), NS_ERROR_FAILURE); + } + MonitorAutoLock mon(mDemuxerMonitor); UpdateIndex(); return NS_OK; } +bool +MP4Reader::EnsureDecodersSetup() +{ + if (mAreDecodersSetup) { + return !!mPlatform; + } + + if (mIsEncrypted) { + // EME not supported. + return false; + } else { + mPlatform = PlatformDecoderModule::Create(); + NS_ENSURE_TRUE(mPlatform, false); + } + + if (HasAudio()) { + NS_ENSURE_TRUE(IsSupportedAudioMimeType(mDemuxer->AudioConfig().mime_type), + false); + + mAudio.mDecoder = mPlatform->CreateAudioDecoder(mDemuxer->AudioConfig(), + mAudio.mTaskQueue, + mAudio.mCallback); + NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, false); + nsresult rv = mAudio.mDecoder->Init(); + NS_ENSURE_SUCCESS(rv, false); + } + + if (HasVideo()) { + NS_ENSURE_TRUE(IsSupportedVideoMimeType(mDemuxer->VideoConfig().mime_type), + false); + + if (mSharedDecoderManager && mPlatform->SupportsSharedDecoders(mDemuxer->VideoConfig())) { + mVideo.mDecoder = + mSharedDecoderManager->CreateVideoDecoder(mPlatform, + mDemuxer->VideoConfig(), + mLayersBackendType, + mDecoder->GetImageContainer(), + mVideo.mTaskQueue, + mVideo.mCallback); + } else { + mVideo.mDecoder = mPlatform->CreateVideoDecoder(mDemuxer->VideoConfig(), + mLayersBackendType, + mDecoder->GetImageContainer(), + mVideo.mTaskQueue, + mVideo.mCallback); + } + NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, false); + nsresult rv = mVideo.mDecoder->Init(); + NS_ENSURE_SUCCESS(rv, false); + } + + mAreDecodersSetup = true; + return true; +} + void MP4Reader::ReadUpdatedMetadata(MediaInfo* aInfo) { @@ -522,6 +544,11 @@ MP4Reader::RequestVideoData(bool aSkipToNextKeyframe, MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn()); VLOG("skip=%d time=%lld", aSkipToNextKeyframe, aTimeThreshold); + if (!EnsureDecodersSetup()) { + NS_WARNING("Error constructing MP4 decoders"); + return VideoDataPromise::CreateAndReject(DECODE_ERROR, __func__); + } + if (mShutdown) { NS_WARNING("RequestVideoData on shutdown MP4Reader!"); return VideoDataPromise::CreateAndReject(CANCELED, __func__); @@ -557,6 +584,12 @@ MP4Reader::RequestAudioData() { MOZ_ASSERT(GetTaskQueue()->IsCurrentThreadIn()); VLOG(""); + + if (!EnsureDecodersSetup()) { + NS_WARNING("Error constructing MP4 decoders"); + return AudioDataPromise::CreateAndReject(DECODE_ERROR, __func__); + } + if (mShutdown) { NS_WARNING("RequestAudioData on shutdown MP4Reader!"); return AudioDataPromise::CreateAndReject(CANCELED, __func__); diff --git a/dom/media/fmp4/MP4Reader.h b/dom/media/fmp4/MP4Reader.h index 271cb7315b..3d8acb397c 100644 --- a/dom/media/fmp4/MP4Reader.h +++ b/dom/media/fmp4/MP4Reader.h @@ -101,6 +101,8 @@ private: bool InitDemuxer(); void ReturnOutput(MediaData* aData, TrackType aTrack); + bool EnsureDecodersSetup(); + // Sends input to decoder for aTrack, and output to the state machine, // if necessary. void Update(TrackType aTrack); @@ -133,7 +135,6 @@ private: bool IsSupportedVideoMimeType(const nsACString& aMimeType); void NotifyResourcesStatusChanged(); void RequestCodecResource(); - bool IsWaitingOnCodecResource(); virtual bool IsWaitingOnCDMResource() override; Microseconds GetNextKeyframeTime(); @@ -281,6 +282,8 @@ private: // Synchronized by decoder monitor. bool mIsEncrypted; + bool mAreDecodersSetup; + bool mIndexReady; int64_t mLastSeenEnd; Monitor mDemuxerMonitor; diff --git a/dom/media/mediasource/MediaSourceReader.cpp b/dom/media/mediasource/MediaSourceReader.cpp index 934c7453b1..ed5f045e6b 100644 --- a/dom/media/mediasource/MediaSourceReader.cpp +++ b/dom/media/mediasource/MediaSourceReader.cpp @@ -88,6 +88,12 @@ MediaSourceReader::IsWaitingMediaResources() return !mHasEssentialTrackBuffers; } +MediaSourceReader::IsWaitingOnCDMResource() +{ + // EME not supported + return false; +} + size_t MediaSourceReader::SizeOfVideoQueueInFrames() { @@ -1079,6 +1085,22 @@ MediaSourceReader::MaybeNotifyHaveData() IsSeeking(), haveAudio, haveVideo, ended); } +static void +CombineEncryptionData(EncryptionInfo& aTo, const EncryptionInfo& aFrom) +{ + if (!aFrom.mIsEncrypted) { + return; + } + aTo.mIsEncrypted = true; + + if (!aTo.mType.IsEmpty() && !aTo.mType.Equals(aFrom.mType)) { + NS_WARNING("mismatched encryption types"); + } + + aTo.mType = aFrom.mType; + aTo.mInitData.AppendElements(aFrom.mInitData); +} + nsresult MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) { @@ -1102,7 +1124,7 @@ MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) const MediaInfo& info = GetAudioReader()->GetMediaInfo(); MOZ_ASSERT(info.HasAudio()); mInfo.mAudio = info.mAudio; - mInfo.mIsEncrypted = mInfo.mIsEncrypted || info.mIsEncrypted; + CombineEncryptionData(mInfo.mCrypto, info.mCrypto); MSE_DEBUG("audio reader=%p duration=%lld", mAudioSourceDecoder.get(), mAudioSourceDecoder->GetReader()->GetDecoder()->GetMediaDuration()); @@ -1115,7 +1137,7 @@ MediaSourceReader::ReadMetadata(MediaInfo* aInfo, MetadataTags** aTags) const MediaInfo& info = GetVideoReader()->GetMediaInfo(); MOZ_ASSERT(info.HasVideo()); mInfo.mVideo = info.mVideo; - mInfo.mIsEncrypted = mInfo.mIsEncrypted || info.mIsEncrypted; + CombineEncryptionData(mInfo.mCrypto, info.mCrypto); MSE_DEBUG("video reader=%p duration=%lld", GetVideoReader(), GetVideoReader()->GetDecoder()->GetMediaDuration()); diff --git a/dom/media/mediasource/MediaSourceReader.h b/dom/media/mediasource/MediaSourceReader.h index b5e3482c08..9bddc341a9 100644 --- a/dom/media/mediasource/MediaSourceReader.h +++ b/dom/media/mediasource/MediaSourceReader.h @@ -45,6 +45,7 @@ public: void PrepareInitialization(); bool IsWaitingMediaResources() override; + bool IsWaitingOnCDMResource() override; nsRefPtr RequestAudioData() override; nsRefPtr diff --git a/dom/media/mediasource/TrackBuffer.cpp b/dom/media/mediasource/TrackBuffer.cpp index cfa304776f..207ed03740 100644 --- a/dom/media/mediasource/TrackBuffer.cpp +++ b/dom/media/mediasource/TrackBuffer.cpp @@ -47,6 +47,7 @@ TrackBuffer::TrackBuffer(MediaSourceDecoder* aParentDecoder, const nsACString& a , mLastStartTimestamp(0) , mLastTimestampOffset(0) , mAdjustedTimestamp(0) + , mIsWaitingOnCDM(false) , mShutdown(false) { MOZ_COUNT_CTOR(TrackBuffer); @@ -645,8 +646,7 @@ TrackBuffer::InitializeDecoder(SourceBufferDecoder* aDecoder) } if (NS_SUCCEEDED(rv) && reader->IsWaitingOnCDMResource()) { - mWaitingDecoders.AppendElement(aDecoder); - return; + mIsWaitingOnCDM = true; } aDecoder->SetTaskQueue(nullptr); @@ -807,6 +807,12 @@ TrackBuffer::IsReady() return mInfo.HasAudio() || mInfo.HasVideo(); } +bool +TrackBuffer::IsWaitingOnCDMResource() +{ + return mIsWaitingOnCDM; +} + bool TrackBuffer::ContainsTime(int64_t aTime, int64_t aTolerance) { diff --git a/dom/media/mediasource/TrackBuffer.h b/dom/media/mediasource/TrackBuffer.h index ffa0b80af8..8a5933361c 100644 --- a/dom/media/mediasource/TrackBuffer.h +++ b/dom/media/mediasource/TrackBuffer.h @@ -76,6 +76,8 @@ public: // segment has successfully initialized by setting mHas{Audio,Video}.. bool IsReady(); + bool IsWaitingOnCDMResource(); + // Returns true if any of the decoders managed by this track buffer // contain aTime in their buffered ranges. bool ContainsTime(int64_t aTime, int64_t aTolerance); @@ -181,10 +183,6 @@ private: // Access protected by mParentDecoder's monitor. nsTArray> mInitializedDecoders; - // Decoders which are waiting on a Content Decryption Module to be able to - // finish ReadMetadata. - nsTArray> mWaitingDecoders; - // The decoder that the owning SourceBuffer is currently appending data to. // Modified on the main thread only. nsRefPtr mCurrentDecoder; @@ -202,6 +200,9 @@ private: int64_t mLastTimestampOffset; int64_t mAdjustedTimestamp; + // True if at least one of our decoders has encrypted content. + bool mIsWaitingOnCDM; + // Set when the first decoder used by this TrackBuffer is initialized. // Protected by mParentDecoder's monitor. MediaInfo mInfo;