diff --git a/dom/base/nsDOMWindowUtils.cpp b/dom/base/nsDOMWindowUtils.cpp index b048c309c2..4d0a33159c 100644 --- a/dom/base/nsDOMWindowUtils.cpp +++ b/dom/base/nsDOMWindowUtils.cpp @@ -2254,7 +2254,7 @@ nsDOMWindowUtils::GetLayerManagerRemote(bool* retval) } NS_IMETHODIMP -nsDOMWindowUtils::GetSupportsHardwareH264Decoding(bool* retval) +nsDOMWindowUtils::GetSupportsHardwareH264Decoding(nsAString& aRetval) { MOZ_RELEASE_ASSERT(nsContentUtils::IsCallerChrome()); @@ -2267,9 +2267,15 @@ nsDOMWindowUtils::GetSupportsHardwareH264Decoding(bool* retval) if (!mgr) return NS_ERROR_FAILURE; - *retval = MP4Decoder::IsVideoAccelerated(mgr->GetCompositorBackendType()); + nsCString failureReason; + if (MP4Decoder::IsVideoAccelerated(mgr->GetCompositorBackendType(), failureReason)) { + aRetval.AssignLiteral("Yes"); + } else { + aRetval.AssignLiteral("No; "); + AppendUTF8toUTF16(failureReason, aRetval); + } #else - *retval = false; + aRetval.AssignLiteral("No; Compiled without MP4 support."); #endif return NS_OK; } diff --git a/dom/interfaces/base/nsIDOMWindowUtils.idl b/dom/interfaces/base/nsIDOMWindowUtils.idl index 5e40a6d246..3ea31d210f 100644 --- a/dom/interfaces/base/nsIDOMWindowUtils.idl +++ b/dom/interfaces/base/nsIDOMWindowUtils.idl @@ -49,7 +49,7 @@ interface nsIJSRAIIHelper; interface nsIContentPermissionRequest; interface nsIObserver; -[scriptable, uuid(e6d3ced6-fbce-4ea5-902a-c2055680d250)] +[scriptable, uuid(47fa312b-2ad1-4b80-8a0a-c9822e2d1ec9)] interface nsIDOMWindowUtils : nsISupports { /** @@ -1335,7 +1335,7 @@ interface nsIDOMWindowUtils : nsISupports { * test video, does not mean that all h264 video decoding will be done * in hardware. */ - readonly attribute boolean supportsHardwareH264Decoding; + readonly attribute AString supportsHardwareH264Decoding; /** * Record (and return) frame-intervals for frames which were presented diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 1ef5e5343f..d96c02f496 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -92,6 +92,7 @@ MediaFormatReader::Shutdown() MOZ_ASSERT(OnTaskQueue()); mDemuxerInitRequest.DisconnectIfExists(); + mDecodersInitRequest.DisconnectIfExists(); mMetadataPromise.RejectIfExists(ReadMetadataFailureReason::METADATA_ERROR, __func__); mSeekPromise.RejectIfExists(NS_ERROR_FAILURE, __func__); mSkipRequest.DisconnectIfExists(); @@ -224,19 +225,19 @@ MediaFormatReader::AsyncReadMetadata() { MOZ_ASSERT(OnTaskQueue()); + nsRefPtr p = mMetadataPromise.Ensure(__func__); + if (mInitDone) { // We are returning from dormant. - if (!EnsureDecodersSetup()) { - return MetadataPromise::CreateAndReject(ReadMetadataFailureReason::METADATA_ERROR, __func__); + if (!EnsureDecodersCreated()) { + mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__); + return p; } - nsRefPtr metadata = new MetadataHolder(); - metadata->mInfo = mInfo; - metadata->mTags = nullptr; - return MetadataPromise::CreateAndResolve(metadata, __func__); + MOZ_ASSERT(!mDecodersInitRequest.Exists()); + EnsureDecodersInitialized(); + return p; } - nsRefPtr p = mMetadataPromise.Ensure(__func__); - mDemuxerInitRequest.Begin(mDemuxer->Init() ->Then(OwnerThread(), __func__, this, &MediaFormatReader::OnDemuxerInitDone, @@ -319,16 +320,24 @@ MediaFormatReader::OnDemuxerInitDone(nsresult) MOZ_ASSERT(mAudioTrackDemuxer); } - mInitDone = true; - - if (!IsWaitingOnCDMResource() && !EnsureDecodersSetup()) { - mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__); - } else { + if (IsWaitingOnCDMResource()) { + // Decoder can't be allocated before CDM resource is ready, so resolving the + // mMetadataPromise here to wait for CDM on MDSM. + mInitDone = true; nsRefPtr metadata = new MetadataHolder(); metadata->mInfo = mInfo; metadata->mTags = nullptr; mMetadataPromise.Resolve(metadata, __func__); + return; } + + if (!EnsureDecodersCreated()) { + mMetadataPromise.Reject(ReadMetadataFailureReason::METADATA_ERROR, __func__); + return; + } + + MOZ_ASSERT(!mDecodersInitRequest.Exists()); + EnsureDecodersInitialized(); } void @@ -343,10 +352,9 @@ MediaFormatReader::OnDemuxerInitFailed(DemuxerFailureReason aFailure) } bool -MediaFormatReader::EnsureDecodersSetup() +MediaFormatReader::EnsureDecodersCreated() { MOZ_ASSERT(OnTaskQueue()); - MOZ_ASSERT(mInitDone); if (!mPlatform) { if (IsEncrypted()) { @@ -364,6 +372,7 @@ MediaFormatReader::EnsureDecodersSetup() NS_ENSURE_TRUE(IsSupportedAudioMimeType(mInfo.mAudio.mMimeType), false); + mAudio.mDecoderInitialized = false; mAudio.mDecoder = mPlatform->CreateDecoder(mAudio.mInfo ? *mAudio.mInfo->GetAsAudioInfo() : @@ -371,14 +380,13 @@ MediaFormatReader::EnsureDecodersSetup() mAudio.mTaskQueue, mAudio.mCallback); NS_ENSURE_TRUE(mAudio.mDecoder != nullptr, false); - nsresult rv = mAudio.mDecoder->Init(); - NS_ENSURE_SUCCESS(rv, false); } if (HasVideo() && !mVideo.mDecoder) { NS_ENSURE_TRUE(IsSupportedVideoMimeType(mInfo.mVideo.mMimeType), false); + mVideo.mDecoderInitialized = false; if (mSharedDecoderManager && mPlatform->SupportsSharedDecoders(mInfo.mVideo)) { mVideo.mDecoder = @@ -401,13 +409,86 @@ MediaFormatReader::EnsureDecodersSetup() mDecoder->GetImageContainer()); } NS_ENSURE_TRUE(mVideo.mDecoder != nullptr, false); - nsresult rv = mVideo.mDecoder->Init(); - NS_ENSURE_SUCCESS(rv, false); } return true; } +bool +MediaFormatReader::EnsureDecodersInitialized() +{ + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(mVideo.mDecoder || mAudio.mDecoder); + + // DecodeDemuxedSample() could call this function before mDecodersInitRequest + // is completed. And it is ok to return false here because DecodeDemuxedSample + // will call ScheduleUpdate() again. + // It also avoids calling decoder->Init() multiple times. + if (mDecodersInitRequest.Exists()) { + return false; + } + + nsTArray> promises; + + if (mVideo.mDecoder && !mVideo.mDecoderInitialized) { + promises.AppendElement(mVideo.mDecoder->Init()); + } + + if (mAudio.mDecoder && !mAudio.mDecoderInitialized) { + promises.AppendElement(mAudio.mDecoder->Init()); + } + + if (promises.Length()) { + mDecodersInitRequest.Begin(MediaDataDecoder::InitPromise::All(OwnerThread(), promises) + ->Then(OwnerThread(), __func__, this, + &MediaFormatReader::OnDecoderInitDone, + &MediaFormatReader::OnDecoderInitFailed)); + } + + LOG("Init decoders: audio: %p, audio init: %d, video: %p, video init: %d", + mAudio.mDecoder.get(), mAudio.mDecoderInitialized, + mVideo.mDecoder.get(), mVideo.mDecoderInitialized); + + // Return false if any decoder is under initialization. + return !promises.Length(); +} + +void +MediaFormatReader::OnDecoderInitDone(const nsTArray& aTrackTypes) +{ + MOZ_ASSERT(OnTaskQueue()); + mDecodersInitRequest.Complete(); + + for (const auto& track : aTrackTypes) { + auto& decoder = GetDecoderData(track); + decoder.mDecoderInitialized = true; + + ScheduleUpdate(track); + } + + if (!mMetadataPromise.IsEmpty()) { + mInitDone = true; + nsRefPtr metadata = new MetadataHolder(); + metadata->mInfo = mInfo; + metadata->mTags = nullptr; + mMetadataPromise.Resolve(metadata, __func__); + } +} + +void +MediaFormatReader::OnDecoderInitFailed(MediaDataDecoder::DecoderFailureReason aReason) +{ + MOZ_ASSERT(OnTaskQueue()); + mDecodersInitRequest.Complete(); + + NS_WARNING("Failed to init decoder"); + + mMetadataPromise.RejectIfExists(ReadMetadataFailureReason::METADATA_ERROR, __func__); + + NotifyError(TrackType::kAudioTrack); + NotifyError(TrackType::kVideoTrack); +} + void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) { @@ -434,7 +515,7 @@ MediaFormatReader::DisableHardwareAcceleration() Flush(TrackInfo::kVideoTrack); mVideo.mDecoder->Shutdown(); mVideo.mDecoder = nullptr; - if (!EnsureDecodersSetup()) { + if (!EnsureDecodersCreated()) { LOG("Unable to re-create decoder, aborting"); NotifyError(TrackInfo::kVideoTrack); return; @@ -484,13 +565,6 @@ MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe, return VideoDataPromise::CreateAndReject(CANCELED, __func__); } - if (!EnsureDecodersSetup()) { - NS_WARNING("Error constructing decoders"); - return VideoDataPromise::CreateAndReject(DECODE_ERROR, __func__); - } - - MOZ_ASSERT(HasVideo() && mPlatform && mVideo.mDecoder); - mVideo.mForceDecodeAhead = aForceDecodeAhead; media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)}; if (ShouldSkip(aSkipToNextKeyframe, timeThreshold)) { @@ -582,13 +656,9 @@ MediaFormatReader::RequestAudioData() return AudioDataPromise::CreateAndReject(CANCELED, __func__); } - if (!EnsureDecodersSetup()) { - NS_WARNING("Error constructing decoders"); - return AudioDataPromise::CreateAndReject(DECODE_ERROR, __func__); - } - nsRefPtr p = mAudio.mPromise.Ensure(__func__); - ScheduleUpdate(TrackInfo::kAudioTrack); + ScheduleUpdate(TrackType::kAudioTrack); + return p; } @@ -801,6 +871,17 @@ MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack, return; } + if (!EnsureDecodersCreated()) { + NS_WARNING("Error constructing decoders"); + NotifyError(aTrack); + return; + } + + if (!EnsureDecodersInitialized()) { + ScheduleUpdate(aTrack); + return; + } + // Decode all our demuxed frames. bool samplesPending = false; while (decoder.mQueuedSamples.Length()) { @@ -834,15 +915,9 @@ MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack, Flush(aTrack); decoder.mDecoder->Shutdown(); decoder.mDecoder = nullptr; - if (!EnsureDecodersSetup()) { - LOG("Unable to re-create decoder, aborting"); - NotifyError(aTrack); - return; - } - LOGV("%s decoder:%p created for sid:%u", - TrackTypeToStr(aTrack), decoder.mDecoder.get(), info->GetID()); if (sample->mKeyframe) { decoder.mQueuedSamples.MoveElementsFrom(samples); + ScheduleUpdate(aTrack); } else { MOZ_ASSERT(decoder.mTimeThreshold.isNothing()); LOG("Stream change occurred on a non-keyframe. Seeking to:%lld", @@ -875,8 +950,8 @@ MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack, } decoder.mTimeThreshold.reset(); })); - return; } + return; } LOGV("Input:%lld (dts:%lld kf:%d)", @@ -967,8 +1042,9 @@ MediaFormatReader::Update(TrackType aTrack) if (!decoder.mOutput.IsEmpty()) { // We have a decoded sample ready to be returned. if (aTrack == TrackType::kVideoTrack) { + nsCString error; mVideo.mIsHardwareAccelerated = - mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(); + mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error); } while (decoder.mOutput.Length()) { nsRefPtr output = decoder.mOutput[0]; diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index 1063da1731..5a3f2f54a1 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -106,7 +106,10 @@ private: void NotifyDemuxer(uint32_t aLength, int64_t aOffset); void ReturnOutput(MediaData* aData, TrackType aTrack); - bool EnsureDecodersSetup(); + bool EnsureDecodersCreated(); + // It returns true when all decoders are initialized. False when there is pending + // initialization. + bool EnsureDecodersInitialized(); // Enqueues a task to call Update(aTrack) on the decoder task queue. // Lock for corresponding track must be held. @@ -191,6 +194,7 @@ private: , mWaitingForData(false) , mReceivedNewData(false) , mDiscontinuity(true) + , mDecoderInitialized(false) , mOutputRequested(false) , mInputExhausted(false) , mError(false) @@ -239,6 +243,8 @@ private: } // MediaDataDecoder handler's variables. + // False when decoder is created. True when decoder Init() promise is resolved. + bool mDecoderInitialized; bool mOutputRequested; bool mInputExhausted; bool mError; @@ -333,6 +339,9 @@ private: DecoderData& GetDecoderData(TrackType aTrack); + void OnDecoderInitDone(const nsTArray& aTrackTypes); + void OnDecoderInitFailed(MediaDataDecoder::DecoderFailureReason aReason); + // Demuxer objects. void OnDemuxerInitDone(nsresult); void OnDemuxerInitFailed(DemuxerFailureReason aFailure); @@ -407,6 +416,9 @@ private: Maybe mPendingSeekTime; MozPromiseHolder mSeekPromise; + // Pending decoders initialization. + MozPromiseRequestHolder mDecodersInitRequest; + nsRefPtr mSharedDecoderManager; // Main thread objects diff --git a/dom/media/fmp4/MP4Decoder.cpp b/dom/media/fmp4/MP4Decoder.cpp index 158d2f6424..ebca7356a1 100644 --- a/dom/media/fmp4/MP4Decoder.cpp +++ b/dom/media/fmp4/MP4Decoder.cpp @@ -287,22 +287,20 @@ CreateTestH264Decoder(layers::LayersBackend aBackend, if (!decoder) { return nullptr; } - nsresult rv = decoder->Init(); - NS_ENSURE_SUCCESS(rv, nullptr); return decoder.forget(); } /* static */ bool -MP4Decoder::IsVideoAccelerated(layers::LayersBackend aBackend) +MP4Decoder::IsVideoAccelerated(layers::LayersBackend aBackend, nsACString& aFailureReason) { VideoInfo config; nsRefPtr decoder(CreateTestH264Decoder(aBackend, config)); if (!decoder) { + aFailureReason.AssignLiteral("Failed to create H264 decoder"); return false; } - bool result = decoder->IsHardwareAccelerated(); - decoder->Shutdown(); + bool result = decoder->IsHardwareAccelerated(aFailureReason); return result; } @@ -340,8 +338,6 @@ CreateTestAACDecoder(AudioInfo& aConfig) if (!decoder) { return nullptr; } - nsresult rv = decoder->Init(); - NS_ENSURE_SUCCESS(rv, nullptr); return decoder.forget(); } @@ -377,7 +373,6 @@ MP4Decoder::CanCreateAACDecoder() MOZ_ARRAY_LENGTH(sTestAACExtraData)); nsRefPtr decoder(CreateTestAACDecoder(config)); if (decoder) { - decoder->Shutdown(); result = true; } haveCachedResult = true; diff --git a/dom/media/fmp4/MP4Decoder.h b/dom/media/fmp4/MP4Decoder.h index 3b8243c4fb..aceec851ae 100644 --- a/dom/media/fmp4/MP4Decoder.h +++ b/dom/media/fmp4/MP4Decoder.h @@ -37,7 +37,7 @@ public: // Returns true if the MP4 backend is preffed on. static bool IsEnabled(); - static bool IsVideoAccelerated(layers::LayersBackend aBackend); + static bool IsVideoAccelerated(layers::LayersBackend aBackend, nsACString& aReason); static bool CanCreateAACDecoder(); static bool CanCreateH264Decoder(); }; diff --git a/dom/media/platforms/PlatformDecoderModule.h b/dom/media/platforms/PlatformDecoderModule.h index 7846fcb39c..47e338e2d2 100644 --- a/dom/media/platforms/PlatformDecoderModule.h +++ b/dom/media/platforms/PlatformDecoderModule.h @@ -8,6 +8,7 @@ #define PlatformDecoderModule_h_ #include "MediaDecoderReader.h" +#include "mozilla/MozPromise.h" #include "mozilla/layers/LayersTypes.h" #include "nsTArray.h" #include "mozilla/RefPtr.h" @@ -201,16 +202,24 @@ protected: virtual ~MediaDataDecoder() {}; public: + enum DecoderFailureReason { + INIT_ERROR, + CANCELED + }; + + typedef TrackInfo::TrackType TrackType; + typedef MozPromise InitPromise; + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataDecoder) - // Initialize the decoder. The decoder should be ready to decode after - // this returns. The decoder should do any initialization here, rather + // Initialize the decoder. The decoder should be ready to decode once + // promise resolves. The decoder should do any initialization here, rather // than in its constructor or PlatformDecoderModule::Create*Decoder(), // so that if the MP4Reader needs to shutdown during initialization, // it can call Shutdown() to cancel this operation. Any initialization // that requires blocking the calling thread in this function *must* // be done here so that it can be canceled by calling Shutdown()! - virtual nsresult Init() = 0; + virtual nsRefPtr Init() = 0; // Inserts a sample into the decoder's decode pipeline. virtual nsresult Input(MediaRawData* aSample) = 0; @@ -246,7 +255,9 @@ public: virtual nsresult Shutdown() = 0; // Called from the state machine task queue or main thread. - virtual bool IsHardwareAccelerated() const { return false; } + // Decoder needs to decide whether or not hardware accelearation is supported + // after creating. It doesn't need to call Init() before calling this function. + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const { return false; } // ConfigurationChanged will be called to inform the video or audio decoder // that the format of the next input sample is about to change. diff --git a/dom/media/platforms/SharedDecoderManager.cpp b/dom/media/platforms/SharedDecoderManager.cpp index 5dafe6fa90..c1fde63d32 100644 --- a/dom/media/platforms/SharedDecoderManager.cpp +++ b/dom/media/platforms/SharedDecoderManager.cpp @@ -54,6 +54,7 @@ SharedDecoderManager::SharedDecoderManager() : mTaskQueue(new FlushableTaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER))) , mActiveProxy(nullptr) , mActiveCallback(nullptr) + , mInit(false) , mWaitForInternalDrain(false) , mMonitor("SharedDecoderManager") , mDecoderReleasedResources(false) @@ -92,11 +93,7 @@ SharedDecoderManager::CreateVideoDecoder( mPDM = nullptr; return nullptr; } - nsresult rv = mDecoder->Init(); - if (NS_FAILED(rv)) { - mDecoder = nullptr; - return nullptr; - } + mPDM = aPDM; } @@ -124,8 +121,8 @@ SharedDecoderManager::Recreate(const VideoInfo& aConfig) if (!mDecoder) { return false; } - nsresult rv = mDecoder->Init(); - return rv == NS_OK; + mInit = false; + return true; } void @@ -162,6 +159,35 @@ SharedDecoderManager::SetIdle(MediaDataDecoder* aProxy) } } +nsRefPtr +SharedDecoderManager::InitDecoder() +{ + if (!mInit && mDecoder) { + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + + nsRefPtr self = this; + nsRefPtr p = mDecoderInitPromise.Ensure(__func__); + + // The mTaskQueue is flushable which can't be used in MediaPromise. So we get + // the current AbstractThread instead of it. The MOZ_ASSERT above ensures + // we are running in AbstractThread so we won't get a nullptr. + mDecoderInitPromiseRequest.Begin( + mDecoder->Init()->Then(AbstractThread::GetCurrent(), __func__, + [self] (TrackInfo::TrackType aType) -> void { + self->mDecoderInitPromiseRequest.Complete(); + self->mInit = true; + self->mDecoderInitPromise.ResolveIfExists(aType, __func__); + }, + [self] (MediaDataDecoder::DecoderFailureReason aReason) -> void { + self->mDecoderInitPromiseRequest.Complete(); + self->mDecoderInitPromise.RejectIfExists(aReason, __func__); + })); + return p; + } + + return MediaDataDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); +} + void SharedDecoderManager::DrainComplete() { @@ -189,6 +215,7 @@ SharedDecoderManager::Shutdown() mTaskQueue->AwaitShutdownAndIdle(); mTaskQueue = nullptr; } + mDecoderInitPromiseRequest.DisconnectIfExists(); } SharedDecoderProxy::SharedDecoderProxy(SharedDecoderManager* aManager, @@ -203,10 +230,14 @@ SharedDecoderProxy::~SharedDecoderProxy() Shutdown(); } -nsresult +nsRefPtr SharedDecoderProxy::Init() { - return NS_OK; + if (mManager->mActiveProxy != this) { + mManager->Select(this); + } + + return mManager->InitDecoder(); } nsresult @@ -246,9 +277,9 @@ SharedDecoderProxy::Shutdown() } bool -SharedDecoderProxy::IsHardwareAccelerated() const +SharedDecoderProxy::IsHardwareAccelerated(nsACString& aFailureReason) const { - return mManager->mDecoder->IsHardwareAccelerated(); + return mManager->mDecoder->IsHardwareAccelerated(aFailureReason); } } // namespace mozilla diff --git a/dom/media/platforms/SharedDecoderManager.h b/dom/media/platforms/SharedDecoderManager.h index 15d9d83b2b..43944e040c 100644 --- a/dom/media/platforms/SharedDecoderManager.h +++ b/dom/media/platforms/SharedDecoderManager.h @@ -48,6 +48,8 @@ private: virtual ~SharedDecoderManager(); void DrainComplete(); + nsRefPtr InitDecoder(); + nsRefPtr mPDM; nsRefPtr mDecoder; layers::LayersBackend mLayersBackend; @@ -56,6 +58,9 @@ private: SharedDecoderProxy* mActiveProxy; MediaDataDecoderCallback* mActiveCallback; nsAutoPtr mCallback; + MozPromiseHolder mDecoderInitPromise; + MozPromiseRequestHolder mDecoderInitPromiseRequest; + bool mInit; // access protected by mMonitor bool mWaitForInternalDrain; Monitor mMonitor; @@ -69,12 +74,12 @@ public: MediaDataDecoderCallback* aCallback); virtual ~SharedDecoderProxy(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; virtual nsresult Shutdown() override; - virtual bool IsHardwareAccelerated() const override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override; friend class SharedDecoderManager; diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.cpp b/dom/media/platforms/agnostic/BlankDecoderModule.cpp index 62bfef72f0..9319e46333 100644 --- a/dom/media/platforms/agnostic/BlankDecoderModule.cpp +++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp @@ -25,15 +25,17 @@ public: BlankMediaDataDecoder(BlankMediaDataCreator* aCreator, FlushableTaskQueue* aTaskQueue, - MediaDataDecoderCallback* aCallback) + MediaDataDecoderCallback* aCallback, + TrackInfo::TrackType aType) : mCreator(aCreator) , mTaskQueue(aTaskQueue) , mCallback(aCallback) + , mType(aType) { } - virtual nsresult Init() override { - return NS_OK; + virtual nsRefPtr Init() override { + return InitPromise::CreateAndResolve(mType, __func__); } virtual nsresult Shutdown() override { @@ -89,6 +91,7 @@ private: nsAutoPtr mCreator; RefPtr mTaskQueue; MediaDataDecoderCallback* mCallback; + TrackInfo::TrackType mType; }; class BlankVideoDataCreator { @@ -221,7 +224,8 @@ public: nsRefPtr decoder = new BlankMediaDataDecoder(creator, aVideoTaskQueue, - aCallback); + aCallback, + TrackInfo::kVideoTrack); return decoder.forget(); } @@ -236,7 +240,8 @@ public: nsRefPtr decoder = new BlankMediaDataDecoder(creator, aAudioTaskQueue, - aCallback); + aCallback, + TrackInfo::kAudioTrack); return decoder.forget(); } diff --git a/dom/media/platforms/agnostic/OpusDecoder.cpp b/dom/media/platforms/agnostic/OpusDecoder.cpp index 69393e9a19..65fd613d81 100644 --- a/dom/media/platforms/agnostic/OpusDecoder.cpp +++ b/dom/media/platforms/agnostic/OpusDecoder.cpp @@ -47,19 +47,19 @@ OpusDataDecoder::Shutdown() return NS_OK; } -nsresult +nsRefPtr OpusDataDecoder::Init() { size_t length = mInfo.mCodecSpecificConfig->Length(); uint8_t *p = mInfo.mCodecSpecificConfig->Elements(); if (length < sizeof(uint64_t)) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } int64_t codecDelay = BigEndian::readUint64(p); length -= sizeof(uint64_t); p += sizeof(uint64_t); if (NS_FAILED(DecodeHeader(p, length))) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } int r; @@ -75,7 +75,7 @@ OpusDataDecoder::Init() if (codecDelay != FramesToUsecs(mOpusParser->mPreSkip, mOpusParser->mRate).value()) { NS_WARNING("Invalid Opus header: CodecDelay and pre-skip do not match!"); - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } if (mInfo.mRate != (uint32_t)mOpusParser->mRate) { @@ -85,7 +85,8 @@ OpusDataDecoder::Init() NS_WARNING("Invalid Opus header: container and codec channels do not match!"); } - return r == OPUS_OK ? NS_OK : NS_ERROR_FAILURE; + return r == OPUS_OK ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__) + : InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } nsresult diff --git a/dom/media/platforms/agnostic/OpusDecoder.h b/dom/media/platforms/agnostic/OpusDecoder.h index a8e05523d5..679102dd3b 100644 --- a/dom/media/platforms/agnostic/OpusDecoder.h +++ b/dom/media/platforms/agnostic/OpusDecoder.h @@ -21,7 +21,7 @@ public: MediaDataDecoderCallback* aCallback); ~OpusDataDecoder(); - nsresult Init() override; + nsRefPtr Init() override; nsresult Input(MediaRawData* aSample) override; nsresult Flush() override; nsresult Drain() override; diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp index 7a758a4e4d..160acb9cfd 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.cpp +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -55,7 +55,7 @@ VPXDecoder::Shutdown() return NS_OK; } -nsresult +nsRefPtr VPXDecoder::Init() { vpx_codec_iface_t* dx = nullptr; @@ -65,9 +65,9 @@ VPXDecoder::Init() dx = vpx_codec_vp9_dx(); } if (!dx || vpx_codec_dec_init(&mVPX, dx, nullptr, 0)) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - return NS_OK; + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); } nsresult diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h index 8deb0624ed..5421834696 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.h +++ b/dom/media/platforms/agnostic/VPXDecoder.h @@ -28,7 +28,7 @@ public: ~VPXDecoder(); - nsresult Init() override; + nsRefPtr Init() override; nsresult Input(MediaRawData* aSample) override; nsresult Flush() override; nsresult Drain() override; diff --git a/dom/media/platforms/agnostic/VorbisDecoder.cpp b/dom/media/platforms/agnostic/VorbisDecoder.cpp index 647e5b9109..12e6ffb733 100644 --- a/dom/media/platforms/agnostic/VorbisDecoder.cpp +++ b/dom/media/platforms/agnostic/VorbisDecoder.cpp @@ -63,7 +63,7 @@ VorbisDataDecoder::Shutdown() return NS_OK; } -nsresult +nsRefPtr VorbisDataDecoder::Init() { vorbis_info_init(&mVorbisInfo); @@ -75,17 +75,17 @@ VorbisDataDecoder::Init() uint8_t *p = mInfo.mCodecSpecificConfig->Elements(); for(int i = 0; i < 3; i++) { if (available < 2) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } available -= 2; size_t length = BigEndian::readUint16(p); p += 2; if (available < length) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } available -= length; if (NS_FAILED(DecodeHeader((const unsigned char*)p, length))) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } p += length; } @@ -94,12 +94,12 @@ VorbisDataDecoder::Init() int r = vorbis_synthesis_init(&mVorbisDsp, &mVorbisInfo); if (r) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } r = vorbis_block_init(&mVorbisDsp, &mVorbisBlock); if (r) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } if (mInfo.mRate != (uint32_t)mVorbisDsp.vi->rate) { @@ -111,7 +111,7 @@ VorbisDataDecoder::Init() ("Invalid Vorbis header: container and codec channels do not match!")); } - return NS_OK; + return InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__); } nsresult diff --git a/dom/media/platforms/agnostic/VorbisDecoder.h b/dom/media/platforms/agnostic/VorbisDecoder.h index dde83736ae..e9594367c9 100644 --- a/dom/media/platforms/agnostic/VorbisDecoder.h +++ b/dom/media/platforms/agnostic/VorbisDecoder.h @@ -24,7 +24,7 @@ public: MediaDataDecoderCallback* aCallback); ~VorbisDataDecoder(); - nsresult Init() override; + nsRefPtr Init() override; nsresult Input(MediaRawData* aSample) override; nsresult Flush() override; nsresult Drain() override; diff --git a/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.cpp b/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.cpp index 6ae0a21bce..c24bfa7cf5 100644 --- a/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.cpp +++ b/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.cpp @@ -167,7 +167,7 @@ GMPAudioDecoder::GMPInitDone(GMPAudioDecoderProxy* aGMP) } } -nsresult +nsRefPtr GMPAudioDecoder::Init() { MOZ_ASSERT(IsOnGMPThread()); @@ -188,7 +188,8 @@ GMPAudioDecoder::Init() NS_ProcessNextEvent(gmpThread, true); } - return mGMP ? NS_OK : NS_ERROR_FAILURE; + return mGMP ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__) + : InitPromise::CreateAndReject(MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); } nsresult diff --git a/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.h b/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.h index f4737575b3..6fad355982 100644 --- a/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.h +++ b/dom/media/platforms/agnostic/gmp/GMPAudioDecoder.h @@ -69,7 +69,7 @@ public: { } - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp index b315bc03a7..4c1827941c 100644 --- a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp +++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp @@ -211,7 +211,7 @@ GMPVideoDecoder::GMPInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost) } } -nsresult +nsRefPtr GMPVideoDecoder::Init() { MOZ_ASSERT(IsOnGMPThread()); @@ -232,7 +232,8 @@ GMPVideoDecoder::Init() NS_ProcessNextEvent(gmpThread, true); } - return mGMP ? NS_OK : NS_ERROR_FAILURE; + return mGMP ? InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__) + : InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } nsresult diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h index eff48b5464..9e1d208565 100644 --- a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h +++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h @@ -84,7 +84,7 @@ public: { } - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; diff --git a/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.cpp b/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.cpp index d7aa0b950b..a1897e215f 100644 --- a/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.cpp +++ b/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.cpp @@ -21,16 +21,21 @@ MediaDataDecoderCallbackProxy::FlushComplete() mProxyDecoder->FlushComplete(); } -nsresult +nsRefPtr +MediaDataDecoderProxy::InternalInit() +{ + MOZ_ASSERT(!mIsShutdown); + + return mProxyDecoder->Init(); +} + +nsRefPtr MediaDataDecoderProxy::Init() { MOZ_ASSERT(!mIsShutdown); - nsRefPtr task(new InitTask(mProxyDecoder)); - nsresult rv = mProxyThread->Dispatch(task, NS_DISPATCH_SYNC); - NS_ENSURE_SUCCESS(rv, rv); - NS_ENSURE_SUCCESS(task->Result(), task->Result()); - return NS_OK; + return ProxyMediaCall(mProxyThreadWrapper, this, __func__, + &MediaDataDecoderProxy::InternalInit); } nsresult diff --git a/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.h b/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.h index 3a6658d8d7..2ae730b452 100644 --- a/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.h +++ b/dom/media/platforms/agnostic/gmp/MediaDataDecoderProxy.h @@ -32,30 +32,6 @@ private: nsRefPtr mSample; }; -class InitTask : public nsRunnable { -public: - explicit InitTask(MediaDataDecoder* aDecoder) - : mDecoder(aDecoder) - , mResultValid(false) - {} - - NS_IMETHOD Run() { - mResult = mDecoder->Init(); - mResultValid = true; - return NS_OK; - } - - nsresult Result() { - MOZ_ASSERT(mResultValid); - return mResult; - } - -private: - MediaDataDecoder* mDecoder; - nsresult mResult; - bool mResultValid; -}; - template class Condition { public: @@ -127,6 +103,7 @@ public: , mIsShutdown(false) #endif { + mProxyThreadWrapper = CreateXPCOMAbstractThreadWrapper(aProxyThread, false); } // Ideally, this would return a regular MediaDataDecoderCallback pointer @@ -150,7 +127,7 @@ public: // Init and Shutdown run synchronously on the proxy thread, all others are // asynchronously and responded to via the MediaDataDecoderCallback. // Note: the nsresults returned by the proxied decoder are lost. - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; @@ -160,6 +137,8 @@ public: void FlushComplete(); private: + nsRefPtr InternalInit(); + #ifdef DEBUG bool IsOnProxyThread() { return NS_GetCurrentThread() == mProxyThread; @@ -171,6 +150,7 @@ private: nsRefPtr mProxyDecoder; nsCOMPtr mProxyThread; + nsRefPtr mProxyThreadWrapper; MediaDataDecoderCallbackProxy mProxyCallback; diff --git a/dom/media/platforms/android/AndroidDecoderModule.cpp b/dom/media/platforms/android/AndroidDecoderModule.cpp index 3612fe6e5e..f315d83164 100644 --- a/dom/media/platforms/android/AndroidDecoderModule.cpp +++ b/dom/media/platforms/android/AndroidDecoderModule.cpp @@ -47,14 +47,18 @@ public: } - nsresult Init() override { + nsRefPtr Init() override { mSurfaceTexture = AndroidSurfaceTexture::Create(); if (!mSurfaceTexture) { NS_WARNING("Failed to create SurfaceTexture for video decode\n"); - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - return InitDecoder(mSurfaceTexture->JavaSurface()); + if (NS_FAILED(InitDecoder(mSurfaceTexture->JavaSurface()))) { + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); + } + + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); } void Cleanup() override { @@ -336,9 +340,17 @@ MediaCodecDataDecoder::~MediaCodecDataDecoder() Shutdown(); } -nsresult MediaCodecDataDecoder::Init() +nsRefPtr MediaCodecDataDecoder::Init() { - return InitDecoder(nullptr); + nsresult rv = InitDecoder(nullptr); + + TrackInfo::TrackType type = + (mType == MediaData::AUDIO_DATA ? TrackInfo::TrackType::kAudioTrack + : TrackInfo::TrackType::kVideoTrack); + + return NS_SUCCEEDED(rv) ? + InitPromise::CreateAndResolve(type, __func__) : + InitPromise::CreateAndReject(MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); } nsresult MediaCodecDataDecoder::InitDecoder(Surface::Param aSurface) diff --git a/dom/media/platforms/android/AndroidDecoderModule.h b/dom/media/platforms/android/AndroidDecoderModule.h index 68b23555cf..44baf84742 100644 --- a/dom/media/platforms/android/AndroidDecoderModule.h +++ b/dom/media/platforms/android/AndroidDecoderModule.h @@ -52,7 +52,7 @@ public: virtual ~MediaCodecDataDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Flush() override; virtual nsresult Drain() override; virtual nsresult Shutdown() override; diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index 768264a213..9c0251e58d 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -50,14 +50,15 @@ AppleATDecoder::~AppleATDecoder() MOZ_ASSERT(!mConverter); } -nsresult +nsRefPtr AppleATDecoder::Init() { if (!mFormatID) { NS_ERROR("Non recognised format"); - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - return NS_OK; + + return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); } nsresult diff --git a/dom/media/platforms/apple/AppleATDecoder.h b/dom/media/platforms/apple/AppleATDecoder.h index 9447142fca..d2b4df38ca 100644 --- a/dom/media/platforms/apple/AppleATDecoder.h +++ b/dom/media/platforms/apple/AppleATDecoder.h @@ -25,7 +25,7 @@ public: MediaDataDecoderCallback* aCallback); virtual ~AppleATDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; diff --git a/dom/media/platforms/apple/AppleUtils.h b/dom/media/platforms/apple/AppleUtils.h index 684bb42228..9e30aff86e 100644 --- a/dom/media/platforms/apple/AppleUtils.h +++ b/dom/media/platforms/apple/AppleUtils.h @@ -43,6 +43,56 @@ private: T mRef; }; +// CFRefPtr: A CoreFoundation smart pointer. +template +class CFRefPtr { +public: + explicit CFRefPtr(T aRef) + : mRef(aRef) + { + if (mRef) { + CFRetain(mRef); + } + } + // Copy constructor. + CFRefPtr(const CFRefPtr& aCFRefPtr) + : mRef(aCFRefPtr.mRef) + { + if (mRef) { + CFRetain(mRef); + } + } + // Copy operator + CFRefPtr& operator=(const CFRefPtr& aCFRefPtr) + { + if (mRef == aCFRefPtr.mRef) { + return; + } + if (mRef) { + CFRelease(mRef); + } + mRef = aCFRefPtr.mRef; + if (mRef) { + CFRetain(mRef); + } + return *this; + } + ~CFRefPtr() + { + if (mRef) { + CFRelease(mRef); + } + } + // Return the wrapped ref so it can be used as an in parameter. + operator T() + { + return mRef; + } + +private: + T mRef; +}; + } // namespace mozilla #endif // mozilla_AppleUtils_h diff --git a/dom/media/platforms/apple/AppleVDADecoder.cpp b/dom/media/platforms/apple/AppleVDADecoder.cpp index 8005aa8d82..76dbf8fc1a 100644 --- a/dom/media/platforms/apple/AppleVDADecoder.cpp +++ b/dom/media/platforms/apple/AppleVDADecoder.cpp @@ -40,8 +40,12 @@ AppleVDADecoder::AppleVDADecoder(const VideoInfo& aConfig, , mPictureHeight(aConfig.mImage.height) , mDisplayWidth(aConfig.mDisplay.width) , mDisplayHeight(aConfig.mDisplay.height) + , mInputIncoming(0) + , mIsShutDown(false) , mUseSoftwareImages(false) , mIs106(!nsCocoaFeatures::OnLionOrLater()) + , mMonitor("AppleVideoDecoder") + , mIsFlushing(false) , mDecoder(nullptr) { MOZ_COUNT_CTOR(AppleVDADecoder); @@ -73,35 +77,42 @@ AppleVDADecoder::~AppleVDADecoder() MOZ_COUNT_DTOR(AppleVDADecoder); } -nsresult +nsRefPtr AppleVDADecoder::Init() { - if (!gfxPlatform::GetPlatform()->CanUseHardwareVideoDecoding()) { - // This GPU is blacklisted for hardware decoding. - return NS_ERROR_FAILURE; - } - - if (mDecoder) { - return NS_OK; - } - nsresult rv = InitializeSession(); - return rv; + return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__); } nsresult AppleVDADecoder::Shutdown() +{ + MOZ_DIAGNOSTIC_ASSERT(!mIsShutDown); + mIsShutDown = true; + if (mTaskQueue) { + nsCOMPtr runnable = + NS_NewRunnableMethod(this, &AppleVDADecoder::ProcessShutdown); + mTaskQueue->Dispatch(runnable.forget()); + } else { + ProcessShutdown(); + } + return NS_OK; +} + +void +AppleVDADecoder::ProcessShutdown() { if (mDecoder) { LOG("%s: cleaning up decoder %p", __func__, mDecoder); VDADecoderDestroy(mDecoder); mDecoder = nullptr; } - return NS_OK; } nsresult AppleVDADecoder::Input(MediaRawData* aSample) { + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + LOG("mp4 input sample %p pts %lld duration %lld us%s %d bytes", aSample, aSample->mTime, @@ -109,6 +120,8 @@ AppleVDADecoder::Input(MediaRawData* aSample) aSample->mKeyframe ? " keyframe" : "", aSample->Size()); + mInputIncoming++; + nsCOMPtr runnable = NS_NewRunnableMethodWithArg>( this, @@ -121,21 +134,51 @@ AppleVDADecoder::Input(MediaRawData* aSample) nsresult AppleVDADecoder::Flush() { + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + mIsFlushing = true; mTaskQueue->Flush(); - OSStatus rv = VDADecoderFlush(mDecoder, 0 /*dont emit*/); - if (rv != noErr) { - LOG("AppleVDADecoder::Flush failed waiting for platform decoder " - "with error:%d.", rv); + nsCOMPtr runnable = + NS_NewRunnableMethod(this, &AppleVDADecoder::ProcessFlush); + MonitorAutoLock mon(mMonitor); + mTaskQueue->Dispatch(runnable.forget()); + while (mIsFlushing) { + mon.Wait(); } - ClearReorderedFrames(); - + mInputIncoming = 0; return NS_OK; } nsresult AppleVDADecoder::Drain() { - mTaskQueue->AwaitIdle(); + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + nsCOMPtr runnable = + NS_NewRunnableMethod(this, &AppleVDADecoder::ProcessDrain); + mTaskQueue->Dispatch(runnable.forget()); + return NS_OK; +} + +void +AppleVDADecoder::ProcessFlush() +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + OSStatus rv = VDADecoderFlush(mDecoder, 0 /*dont emit*/); + if (rv != noErr) { + LOG("AppleVDADecoder::Flush failed waiting for platform decoder " + "with error:%d.", rv); + } + ClearReorderedFrames(); + MonitorAutoLock mon(mMonitor); + mIsFlushing = false; + mon.NotifyAll(); +} + +void +AppleVDADecoder::ProcessDrain() +{ + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + OSStatus rv = VDADecoderFlush(mDecoder, kVDADecoderFlush_EmitFrames); if (rv != noErr) { LOG("AppleVDADecoder::Drain failed waiting for platform decoder " @@ -143,7 +186,6 @@ AppleVDADecoder::Drain() } DrainReorderedFrames(); mCallback->DrainComplete(); - return NS_OK; } // @@ -208,17 +250,19 @@ PlatformCallback(void* decompressionOutputRefCon, CFNumberGetValue(boref, kCFNumberSInt64Type, &byte_offset); CFNumberGetValue(kfref, kCFNumberSInt8Type, &is_sync_point); - nsAutoPtr frameRef( - new AppleVDADecoder::AppleFrameRef( + AppleVDADecoder::AppleFrameRef frameRef( media::TimeUnit::FromMicroseconds(dts), media::TimeUnit::FromMicroseconds(pts), media::TimeUnit::FromMicroseconds(duration), byte_offset, - is_sync_point == 1)); + is_sync_point == 1); // Forward the data back to an object method which can access - // the correct MP4Reader callback. - decoder->OutputFrame(image, frameRef); + // the correct reader's callback. + nsCOMPtr task = + NS_NewRunnableMethodWithArgs, AppleVDADecoder::AppleFrameRef>( + decoder, &AppleVDADecoder::OutputFrame, image, frameRef); + decoder->DispatchOutputTask(task.forget()); } AppleVDADecoder::AppleFrameRef* @@ -246,15 +290,22 @@ AppleVDADecoder::ClearReorderedFrames() // Copy and return a decoded frame. nsresult -AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, - nsAutoPtr aFrameRef) +AppleVDADecoder::OutputFrame(CFRefPtr aImage, + AppleVDADecoder::AppleFrameRef aFrameRef) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + if (mIsFlushing) { + // We are in the process of flushing; ignore frame. + return NS_OK; + } + LOG("mp4 output frame %lld dts %lld pts %lld duration %lld us%s", - aFrameRef->byte_offset, - aFrameRef->decode_timestamp.ToMicroseconds(), - aFrameRef->composition_timestamp.ToMicroseconds(), - aFrameRef->duration.ToMicroseconds(), - aFrameRef->is_sync_point ? " keyframe" : "" + aFrameRef.byte_offset, + aFrameRef.decode_timestamp.ToMicroseconds(), + aFrameRef.composition_timestamp.ToMicroseconds(), + aFrameRef.duration.ToMicroseconds(), + aFrameRef.is_sync_point ? " keyframe" : "" ); // Where our resulting image will end up. @@ -312,12 +363,12 @@ AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, VideoData::Create(info, mImageContainer, nullptr, - aFrameRef->byte_offset, - aFrameRef->composition_timestamp.ToMicroseconds(), - aFrameRef->duration.ToMicroseconds(), + aFrameRef.byte_offset, + aFrameRef.composition_timestamp.ToMicroseconds(), + aFrameRef.duration.ToMicroseconds(), buffer, - aFrameRef->is_sync_point, - aFrameRef->decode_timestamp.ToMicroseconds(), + aFrameRef.is_sync_point, + aFrameRef.decode_timestamp.ToMicroseconds(), visible); // Unlock the returned image data. CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); @@ -336,12 +387,12 @@ AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, data = VideoData::CreateFromImage(info, mImageContainer, - aFrameRef->byte_offset, - aFrameRef->composition_timestamp.ToMicroseconds(), - aFrameRef->duration.ToMicroseconds(), + aFrameRef.byte_offset, + aFrameRef.composition_timestamp.ToMicroseconds(), + aFrameRef.duration.ToMicroseconds(), image.forget(), - aFrameRef->is_sync_point, - aFrameRef->decode_timestamp.ToMicroseconds(), + aFrameRef.is_sync_point, + aFrameRef.decode_timestamp.ToMicroseconds(), visible); } @@ -366,6 +417,10 @@ AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, nsresult AppleVDADecoder::SubmitFrame(MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + mInputIncoming--; + AutoCFRelease block = CFDataCreate(kCFAllocatorDefault, aSample->Data(), aSample->Size()); if (!block) { @@ -438,7 +493,7 @@ AppleVDADecoder::SubmitFrame(MediaRawData* aSample) } // Ask for more data. - if (mTaskQueue->IsEmpty()) { + if (!mInputIncoming) { LOG("AppleVDADecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } @@ -579,11 +634,18 @@ AppleVDADecoder::CreateVDADecoder( MediaDataDecoderCallback* aCallback, layers::ImageContainer* aImageContainer) { - nsRefPtr decoder = - new AppleVDADecoder(aConfig, aVideoTaskQueue, aCallback, aImageContainer); - if (NS_FAILED(decoder->Init())) { + if (!gfxPlatform::GetPlatform()->CanUseHardwareVideoDecoding()) { + // This GPU is blacklisted for hardware decoding. return nullptr; } + + nsRefPtr decoder = + new AppleVDADecoder(aConfig, aVideoTaskQueue, aCallback, aImageContainer); + + if (NS_FAILED(decoder->InitializeSession())) { + return nullptr; + } + return decoder.forget(); } diff --git a/dom/media/platforms/apple/AppleVDADecoder.h b/dom/media/platforms/apple/AppleVDADecoder.h index 8ce6ab35ff..ee7d2f10f5 100644 --- a/dom/media/platforms/apple/AppleVDADecoder.h +++ b/dom/media/platforms/apple/AppleVDADecoder.h @@ -8,6 +8,7 @@ #define mozilla_AppleVDADecoder_h #include "PlatformDecoderModule.h" +#include "mozilla/Atomics.h" #include "mozilla/ReentrantMonitor.h" #include "MP4Decoder.h" #include "nsIThread.h" @@ -70,16 +71,34 @@ public: MediaDataDecoderCallback* aCallback, layers::ImageContainer* aImageContainer); virtual ~AppleVDADecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; virtual nsresult Shutdown() override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override + { + return true; + } - nsresult OutputFrame(CVPixelBufferRef aImage, - nsAutoPtr aFrameRef); + void DispatchOutputTask(already_AddRefed aTask) + { + nsCOMPtr task = aTask; + if (mIsShutDown || mIsFlushing) { + return; + } + mTaskQueue->Dispatch(task.forget()); + } + + nsresult OutputFrame(CFRefPtr aImage, + AppleFrameRef aFrameRef); + +protected: + // Flush and Drain operation, always run + virtual void ProcessFlush(); + virtual void ProcessDrain(); + virtual void ProcessShutdown(); - protected: AppleFrameRef* CreateAppleFrameRef(const MediaRawData* aSample); void DrainReorderedFrames(); void ClearReorderedFrames(); @@ -95,16 +114,29 @@ public: uint32_t mDisplayWidth; uint32_t mDisplayHeight; uint32_t mMaxRefFrames; + // Increased when Input is called, and decreased when ProcessFrame runs. + // Reaching 0 indicates that there's no pending Input. + Atomic mInputIncoming; + Atomic mIsShutDown; + bool mUseSoftwareImages; bool mIs106; + // For wait on mIsFlushing during Shutdown() process. + Monitor mMonitor; + // Set on reader/decode thread calling Flush() to indicate that output is + // not required and so input samples on mTaskQueue need not be processed. + // Cleared on mTaskQueue in ProcessDrain(). + Atomic mIsFlushing; + private: VDADecoder mDecoder; - // Method to pass a frame to VideoToolbox for decoding. - nsresult SubmitFrame(MediaRawData* aSample); // Method to set up the decompression session. nsresult InitializeSession(); + + // Method to pass a frame to VideoToolbox for decoding. + nsresult SubmitFrame(MediaRawData* aSample); CFDictionaryRef CreateDecoderSpecification(); }; diff --git a/dom/media/platforms/apple/AppleVTDecoder.cpp b/dom/media/platforms/apple/AppleVTDecoder.cpp index c3ee89a2b7..b65f776da7 100644 --- a/dom/media/platforms/apple/AppleVTDecoder.cpp +++ b/dom/media/platforms/apple/AppleVTDecoder.cpp @@ -36,6 +36,7 @@ AppleVTDecoder::AppleVTDecoder(const VideoInfo& aConfig, : AppleVDADecoder(aConfig, aVideoTaskQueue, aCallback, aImageContainer) , mFormat(nullptr) , mSession(nullptr) + , mIsHardwareAccelerated(false) { MOZ_COUNT_CTOR(AppleVTDecoder); // TODO: Verify aConfig.mime_type. @@ -50,15 +51,20 @@ AppleVTDecoder::~AppleVTDecoder() MOZ_COUNT_DTOR(AppleVTDecoder); } -nsresult +nsRefPtr AppleVTDecoder::Init() { nsresult rv = InitializeSession(); - return rv; + + if (NS_SUCCEEDED(rv)) { + return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__); + } + + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } -nsresult -AppleVTDecoder::Shutdown() +void +AppleVTDecoder::ProcessShutdown() { if (mSession) { LOG("%s: cleaning up session %p", __func__, mSession); @@ -71,12 +77,13 @@ AppleVTDecoder::Shutdown() CFRelease(mFormat); mFormat = nullptr; } - return NS_OK; } nsresult AppleVTDecoder::Input(MediaRawData* aSample) { + MOZ_ASSERT(mCallback->OnReaderTaskQueue()); + LOG("mp4 input sample %p pts %lld duration %lld us%s %d bytes", aSample, aSample->mTime, @@ -96,33 +103,34 @@ AppleVTDecoder::Input(MediaRawData* aSample) LOG(" sha1 %s", digest.get()); #endif // LOG_MEDIA_SHA1 + mInputIncoming++; + nsCOMPtr runnable = NS_NewRunnableMethodWithArg>( - this, - &AppleVTDecoder::SubmitFrame, - nsRefPtr(aSample)); + this, &AppleVTDecoder::SubmitFrame, aSample); mTaskQueue->Dispatch(runnable.forget()); return NS_OK; } -nsresult -AppleVTDecoder::Flush() +void +AppleVTDecoder::ProcessFlush() { - mTaskQueue->Flush(); + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); nsresult rv = WaitForAsynchronousFrames(); if (NS_FAILED(rv)) { LOG("AppleVTDecoder::Flush failed waiting for platform decoder " "with error:%d.", rv); } ClearReorderedFrames(); - - return rv; + MonitorAutoLock mon(mMonitor); + mIsFlushing = false; + mon.NotifyAll(); } -nsresult -AppleVTDecoder::Drain() +void +AppleVTDecoder::ProcessDrain() { - mTaskQueue->AwaitIdle(); + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); nsresult rv = WaitForAsynchronousFrames(); if (NS_FAILED(rv)) { LOG("AppleVTDecoder::Drain failed waiting for platform decoder " @@ -130,7 +138,6 @@ AppleVTDecoder::Drain() } DrainReorderedFrames(); mCallback->DrainComplete(); - return NS_OK; } // @@ -168,9 +175,10 @@ PlatformCallback(void* decompressionOutputRefCon, MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(), "VideoToolbox returned an unexpected image type"); - // Forward the data back to an object method which can access - // the correct MP4Reader callback. - decoder->OutputFrame(image, frameRef); + nsCOMPtr task = + NS_NewRunnableMethodWithArgs, AppleVTDecoder::AppleFrameRef>( + decoder, &AppleVTDecoder::OutputFrame, image, *frameRef); + decoder->DispatchOutputTask(task.forget()); } nsresult @@ -202,6 +210,8 @@ TimingInfoFromSample(MediaRawData* aSample) nsresult AppleVTDecoder::SubmitFrame(MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + mInputIncoming--; // For some reason this gives me a double-free error with stagefright. AutoCFRelease block = nullptr; AutoCFRelease sample = nullptr; @@ -245,7 +255,7 @@ AppleVTDecoder::SubmitFrame(MediaRawData* aSample) } // Ask for more data. - if (mTaskQueue->IsEmpty()) { + if (!mInputIncoming) { LOG("AppleVTDecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } @@ -313,8 +323,9 @@ AppleVTDecoder::InitializeSession() if (rv != noErr) { LOG("AppleVTDecoder: system doesn't support hardware acceleration"); } + mIsHardwareAccelerated = rv == noErr && isUsingHW == kCFBooleanTrue; LOG("AppleVTDecoder: %s hardware accelerated decoding", - (rv == noErr && isUsingHW == kCFBooleanTrue) ? "using" : "not using"); + mIsHardwareAccelerated ? "using" : "not using"); } else { LOG("AppleVTDecoder: couldn't determine hardware acceleration status."); } diff --git a/dom/media/platforms/apple/AppleVTDecoder.h b/dom/media/platforms/apple/AppleVTDecoder.h index ab8febf8f8..b0ab88e33c 100644 --- a/dom/media/platforms/apple/AppleVTDecoder.h +++ b/dom/media/platforms/apple/AppleVTDecoder.h @@ -20,11 +20,17 @@ public: MediaDataDecoderCallback* aCallback, layers::ImageContainer* aImageContainer); virtual ~AppleVTDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; - virtual nsresult Flush() override; - virtual nsresult Drain() override; - virtual nsresult Shutdown() override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override + { + return mIsHardwareAccelerated; + } + +protected: + void ProcessFlush() override; + void ProcessDrain() override; + void ProcessShutdown() override; private: CMVideoFormatDescriptionRef mFormat; @@ -37,6 +43,7 @@ private: nsresult WaitForAsynchronousFrames(); CFDictionaryRef CreateDecoderSpecification(); CFDictionaryRef CreateDecoderExtensions(); + bool mIsHardwareAccelerated; }; } // namespace mozilla diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp index c3276edf73..d728d792d9 100644 --- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.cpp @@ -31,16 +31,18 @@ FFmpegAudioDecoder::FFmpegAudioDecoder( mExtraData->AppendElements(*aConfig.mCodecSpecificConfig); } -nsresult +nsRefPtr FFmpegAudioDecoder::Init() { - nsresult rv = FFmpegDataDecoder::Init(); - NS_ENSURE_SUCCESS(rv, rv); + nsresult rv = InitDecoder(); - avcodec_decode_audio4 = (decltype(avcodec_decode_audio4))FFmpegRuntimeLinker::avc_ptr[_decode_audio4]; - av_init_packet1 = (decltype(av_init_packet1))FFmpegRuntimeLinker::avc_ptr[_init_packet]; + if(rv == NS_OK) { + avcodec_decode_audio4 = (decltype(avcodec_decode_audio4))FFmpegRuntimeLinker::avc_ptr[_decode_audio4]; + av_init_packet1 = (decltype(av_init_packet1))FFmpegRuntimeLinker::avc_ptr[_init_packet]; + } - return NS_OK; + return rv == NS_OK ? InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__) + : InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } void diff --git a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h index 46b2f88113..732695268d 100644 --- a/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegAudioDecoder.h @@ -25,7 +25,7 @@ public: const AudioInfo& aConfig); virtual ~FFmpegAudioDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Drain() override; void InitCodecContext() override; diff --git a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp index 77da356e74..d417f53d97 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.cpp @@ -40,7 +40,7 @@ FFmpegDataDecoder::~FFmpegDataDecoder() } nsresult -FFmpegDataDecoder::Init() +FFmpegDataDecoder::InitDecoder() { FFMPEG_LOG("Initialising FFmpeg decoder."); diff --git a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h index a0ff001169..6a9948b0bd 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegDataDecoder.h @@ -28,7 +28,7 @@ public: static bool Link(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override = 0; virtual nsresult Input(MediaRawData* aSample) override = 0; virtual nsresult Flush() override; virtual nsresult Drain() override = 0; @@ -39,6 +39,7 @@ public: protected: virtual void InitCodecContext() {} AVFrame* PrepareFrame(); + nsresult InitDecoder(); FlushableTaskQueue* mTaskQueue; AVCodecContext* mCodecContext; diff --git a/dom/media/platforms/ffmpeg/FFmpegH264Decoder.cpp b/dom/media/platforms/ffmpeg/FFmpegH264Decoder.cpp index c844e8e135..2d59da713c 100644 --- a/dom/media/platforms/ffmpeg/FFmpegH264Decoder.cpp +++ b/dom/media/platforms/ffmpeg/FFmpegH264Decoder.cpp @@ -128,14 +128,17 @@ FFmpegH264Decoder::FFmpegH264Decoder( mExtraData->AppendElements(*aConfig.mExtraData); } -nsresult +nsRefPtr FFmpegH264Decoder::Init() { - nsresult rv = FFmpegDataDecoder::Init(); - NS_ENSURE_SUCCESS(rv, rv); + if (NS_FAILED(InitDecoder())) { + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); + } + avcodec_decode_video2 = (decltype(avcodec_decode_video2))FFmpegRuntimeLinker::avc_ptr[_decode_video2]; av_init_packet = (decltype(av_init_packet))FFmpegRuntimeLinker::avc_ptr[_init_packet]; - return NS_OK; + + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); } void diff --git a/dom/media/platforms/ffmpeg/FFmpegH264Decoder.h b/dom/media/platforms/ffmpeg/FFmpegH264Decoder.h index e68756b6b1..af3f484834 100644 --- a/dom/media/platforms/ffmpeg/FFmpegH264Decoder.h +++ b/dom/media/platforms/ffmpeg/FFmpegH264Decoder.h @@ -38,7 +38,7 @@ public: ImageContainer* aImageContainer); virtual ~FFmpegH264Decoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Drain() override; virtual nsresult Flush() override; diff --git a/dom/media/platforms/gonk/GonkAudioDecoderManager.h b/dom/media/platforms/gonk/GonkAudioDecoderManager.h index 8674648732..c951201c4c 100644 --- a/dom/media/platforms/gonk/GonkAudioDecoderManager.h +++ b/dom/media/platforms/gonk/GonkAudioDecoderManager.h @@ -37,6 +37,8 @@ public: virtual bool HasQueuedSample() override; + virtual TrackType GetTrackType() override { return TrackType::kAudioTrack; } + private: nsresult CreateAudioData(int64_t aStreamOffset, AudioData** aOutData); diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp index 6d643cbd62..500865a44f 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp @@ -37,7 +37,7 @@ GonkMediaDataDecoder::~GonkMediaDataDecoder() MOZ_COUNT_DTOR(GonkMediaDataDecoder); } -nsresult +nsRefPtr GonkMediaDataDecoder::Init() { sp decoder; @@ -45,7 +45,7 @@ GonkMediaDataDecoder::Init() mDecoder = decoder; mDrainComplete = false; - return NS_OK; + return InitPromise::CreateAndResolve(mManager->GetTrackType(), __func__); } nsresult diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.h b/dom/media/platforms/gonk/GonkMediaDataDecoder.h index 740762feb1..66b7066d0c 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.h +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.h @@ -18,6 +18,8 @@ class MediaRawData; // Manage the data flow from inputting encoded data and outputting decode data. class GonkDecoderManager { public: + typedef TrackInfo::TrackType TrackType; + virtual ~GonkDecoderManager() {} // Creates and initializs the GonkDecoder. @@ -39,6 +41,8 @@ public: // Flush the queued sample. virtual nsresult Flush() = 0; + virtual TrackType GetTrackType() = 0; + void ClearQueuedSample() { MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); mQueueSample.Clear(); @@ -60,9 +64,9 @@ public: ~GonkMediaDataDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; - virtual nsresult Input(MediaRawData* aSample); + virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; diff --git a/dom/media/platforms/gonk/GonkVideoDecoderManager.h b/dom/media/platforms/gonk/GonkVideoDecoderManager.h index 3f215c83c8..0be83e9a13 100644 --- a/dom/media/platforms/gonk/GonkVideoDecoderManager.h +++ b/dom/media/platforms/gonk/GonkVideoDecoderManager.h @@ -53,6 +53,8 @@ public: virtual bool HasQueuedSample() override; + virtual TrackType GetTrackType() override { return TrackType::kVideoTrack; } + static void RecycleCallback(TextureClient* aClient, void* aClosure); private: diff --git a/dom/media/platforms/wmf/DXVA2Manager.cpp b/dom/media/platforms/wmf/DXVA2Manager.cpp index a9bbcfcb79..d2b3c73549 100644 --- a/dom/media/platforms/wmf/DXVA2Manager.cpp +++ b/dom/media/platforms/wmf/DXVA2Manager.cpp @@ -15,6 +15,7 @@ #include "mfapi.h" #include "MFTDecoder.h" #include "DriverCrashGuard.h" +#include "nsPrintfCString.h" const CLSID CLSID_VideoProcessorMFT = { @@ -48,7 +49,7 @@ public: D3D9DXVA2Manager(); virtual ~D3D9DXVA2Manager(); - HRESULT Init(); + HRESULT Init(nsACString& aFailureReason); IUnknown* GetDXVADeviceManager() override; @@ -87,13 +88,14 @@ D3D9DXVA2Manager::GetDXVADeviceManager() } HRESULT -D3D9DXVA2Manager::Init() +D3D9DXVA2Manager::Init(nsACString& aFailureReason) { MOZ_ASSERT(NS_IsMainThread()); gfx::D3D9VideoCrashGuard crashGuard; if (crashGuard.Crashed()) { NS_WARNING("DXVA2D3D9 crash detected"); + aFailureReason.AssignLiteral("DXVA2D3D9 crashes detected in the past"); return E_FAIL; } @@ -106,6 +108,7 @@ D3D9DXVA2Manager::Init() HRESULT hr = d3d9Create(D3D_SDK_VERSION, getter_AddRefs(d3d9Ex)); if (!d3d9Ex) { NS_WARNING("Direct3DCreate9 failed"); + aFailureReason.AssignLiteral("Direct3DCreate9 failed"); return E_FAIL; } @@ -115,7 +118,10 @@ D3D9DXVA2Manager::Init() D3DDEVTYPE_HAL, (D3DFORMAT)MAKEFOURCC('N','V','1','2'), D3DFMT_X8R8G8B8); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("CheckDeviceFormatConversion failed with error %X", hr); + return hr; + } // Create D3D9DeviceEx. D3DPRESENT_PARAMETERS params = {0}; @@ -138,7 +144,10 @@ D3D9DXVA2Manager::Init() ¶ms, nullptr, getter_AddRefs(device)); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("CreateDeviceEx failed with error %X", hr); + return hr; + } // Ensure we can create queries to synchronize operations between devices. // Without this, when we make a copy of the frame in order to share it with @@ -147,7 +156,10 @@ D3D9DXVA2Manager::Init() nsRefPtr query; hr = device->CreateQuery(D3DQUERYTYPE_EVENT, getter_AddRefs(query)); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("CreateQuery failed with error %X", hr); + return hr; + } // Create and initialize IDirect3DDeviceManager9. UINT resetToken = 0; @@ -155,9 +167,15 @@ D3D9DXVA2Manager::Init() hr = wmf::DXVA2CreateDirect3DDeviceManager9(&resetToken, getter_AddRefs(deviceManager)); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("DXVA2CreateDirect3DDeviceManager9 failed with error %X", hr); + return hr; + } hr = deviceManager->ResetDevice(device, resetToken); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("IDirect3DDeviceManager9::ResetDevice failed with error %X", hr); + return hr; + } mResetToken = resetToken; mD3D9 = d3d9Ex; @@ -203,7 +221,7 @@ static uint32_t sDXVAVideosCount = 0; /* static */ DXVA2Manager* -DXVA2Manager::CreateD3D9DXVA() +DXVA2Manager::CreateD3D9DXVA(nsACString& aFailureReason) { MOZ_ASSERT(NS_IsMainThread()); HRESULT hr; @@ -213,11 +231,12 @@ DXVA2Manager::CreateD3D9DXVA() const uint32_t dxvaLimit = Preferences::GetInt("media.windows-media-foundation.max-dxva-videos", 8); if (sDXVAVideosCount == dxvaLimit) { + aFailureReason.AssignLiteral("Too many DXVA videos playing"); return nullptr; } nsAutoPtr d3d9Manager(new D3D9DXVA2Manager()); - hr = d3d9Manager->Init(); + hr = d3d9Manager->Init(aFailureReason); if (SUCCEEDED(hr)) { return d3d9Manager.forget(); } @@ -232,7 +251,7 @@ public: D3D11DXVA2Manager(); virtual ~D3D11DXVA2Manager(); - HRESULT Init(); + HRESULT Init(nsACString& aFailureReason); IUnknown* GetDXVADeviceManager() override; @@ -281,28 +300,46 @@ D3D11DXVA2Manager::GetDXVADeviceManager() } HRESULT -D3D11DXVA2Manager::Init() +D3D11DXVA2Manager::Init(nsACString& aFailureReason) { HRESULT hr; mDevice = gfxWindowsPlatform::GetPlatform()->CreateD3D11DecoderDevice(); - NS_ENSURE_TRUE(mDevice, E_FAIL); + if (!mDevice) { + aFailureReason.AssignLiteral("Failed to create D3D11 device for decoder"); + return E_FAIL; + } mDevice->GetImmediateContext(byRef(mContext)); - NS_ENSURE_TRUE(mContext, E_FAIL); + if (!mContext) { + aFailureReason.AssignLiteral("Failed to get immediate context for d3d11 device"); + return E_FAIL; + } hr = wmf::MFCreateDXGIDeviceManager(&mDeviceManagerToken, byRef(mDXGIDeviceManager)); - NS_ENSURE_TRUE(SUCCEEDED(hr),hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("MFCreateDXGIDeviceManager failed with code %X", hr); + return hr; + } hr = mDXGIDeviceManager->ResetDevice(mDevice, mDeviceManagerToken); - NS_ENSURE_TRUE(SUCCEEDED(hr),hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("IMFDXGIDeviceManager::ResetDevice failed with code %X", hr); + return hr; + } mTransform = new MFTDecoder(); hr = mTransform->Create(CLSID_VideoProcessorMFT); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("MFTDecoder::Create(CLSID_VideoProcessorMFT) failed with code %X", hr); + return hr; + } hr = mTransform->SendMFTMessage(MFT_MESSAGE_SET_D3D_MANAGER, ULONG_PTR(mDXGIDeviceManager.get())); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); + if (!SUCCEEDED(hr)) { + aFailureReason = nsPrintfCString("MFTDecoder::SendMFTMessage(MFT_MESSAGE_SET_D3D_MANAGER) failed with code %X", hr); + return hr; + } return S_OK; } @@ -458,18 +495,19 @@ D3D11DXVA2Manager::ConfigureForSize(uint32_t aWidth, uint32_t aHeight) /* static */ DXVA2Manager* -DXVA2Manager::CreateD3D11DXVA() +DXVA2Manager::CreateD3D11DXVA(nsACString& aFailureReason) { // DXVA processing takes up a lot of GPU resources, so limit the number of // videos we use DXVA with at any one time. const uint32_t dxvaLimit = Preferences::GetInt("media.windows-media-foundation.max-dxva-videos", 8); if (sDXVAVideosCount == dxvaLimit) { + aFailureReason.AssignLiteral("Too many DXVA videos playing"); return nullptr; } nsAutoPtr manager(new D3D11DXVA2Manager()); - HRESULT hr = manager->Init(); + HRESULT hr = manager->Init(aFailureReason); NS_ENSURE_TRUE(SUCCEEDED(hr), nullptr); return manager.forget(); diff --git a/dom/media/platforms/wmf/DXVA2Manager.h b/dom/media/platforms/wmf/DXVA2Manager.h index 9908827650..c7dc995a9b 100644 --- a/dom/media/platforms/wmf/DXVA2Manager.h +++ b/dom/media/platforms/wmf/DXVA2Manager.h @@ -23,8 +23,8 @@ public: // Creates and initializes a DXVA2Manager. We can use DXVA2 via either // D3D9Ex or D3D11. - static DXVA2Manager* CreateD3D9DXVA(); - static DXVA2Manager* CreateD3D11DXVA(); + static DXVA2Manager* CreateD3D9DXVA(nsACString& aFailureReason); + static DXVA2Manager* CreateD3D11DXVA(nsACString& aFailureReason); // Returns a pointer to the D3D device manager responsible for managing the // device we're using for hardware accelerated video decoding. If we're using diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.h b/dom/media/platforms/wmf/WMFAudioMFTManager.h index 95d6aab469..e142f595f6 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.h +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.h @@ -31,6 +31,10 @@ public: virtual void Shutdown() override; + virtual TrackInfo::TrackType GetType() override { + return TrackInfo::kAudioTrack; + } + private: HRESULT UpdateOutputType(); diff --git a/dom/media/platforms/wmf/WMFDecoderModule.cpp b/dom/media/platforms/wmf/WMFDecoderModule.cpp index 4722b87a28..f4aacda4a0 100644 --- a/dom/media/platforms/wmf/WMFDecoderModule.cpp +++ b/dom/media/platforms/wmf/WMFDecoderModule.cpp @@ -8,6 +8,7 @@ #include "WMFDecoderModule.h" #include "WMFVideoMFTManager.h" #include "WMFAudioMFTManager.h" +#include "MFTDecoder.h" #include "mozilla/Preferences.h" #include "mozilla/DebugOnly.h" #include "mozilla/Services.h" @@ -98,13 +99,21 @@ WMFDecoderModule::CreateVideoDecoder(const VideoInfo& aConfig, FlushableTaskQueue* aVideoTaskQueue, MediaDataDecoderCallback* aCallback) { + nsAutoPtr manager = + new WMFVideoMFTManager(aConfig, + aLayersBackend, + aImageContainer, + sDXVAEnabled && ShouldUseDXVA(aConfig)); + + nsRefPtr mft = manager->Init(); + + if (!mft) { + return nullptr; + } + nsRefPtr decoder = - new WMFMediaDataDecoder(new WMFVideoMFTManager(aConfig, - aLayersBackend, - aImageContainer, - sDXVAEnabled && ShouldUseDXVA(aConfig)), - aVideoTaskQueue, - aCallback); + new WMFMediaDataDecoder(manager.forget(), mft, aVideoTaskQueue, aCallback); + return decoder.forget(); } @@ -113,10 +122,15 @@ WMFDecoderModule::CreateAudioDecoder(const AudioInfo& aConfig, FlushableTaskQueue* aAudioTaskQueue, MediaDataDecoderCallback* aCallback) { + nsAutoPtr manager = new WMFAudioMFTManager(aConfig); + nsRefPtr mft = manager->Init(); + + if (!mft) { + return nullptr; + } + nsRefPtr decoder = - new WMFMediaDataDecoder(new WMFAudioMFTManager(aConfig), - aAudioTaskQueue, - aCallback); + new WMFMediaDataDecoder(manager.forget(), mft, aAudioTaskQueue, aCallback); return decoder.forget(); } diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp index d5f2c9c706..35776cf2e7 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp @@ -17,10 +17,12 @@ PRLogModuleInfo* GetDemuxerLog(); namespace mozilla { WMFMediaDataDecoder::WMFMediaDataDecoder(MFTManager* aMFTManager, + MFTDecoder* aDecoder, FlushableTaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback) : mTaskQueue(aTaskQueue) , mCallback(aCallback) + , mDecoder(aDecoder) , mMFTManager(aMFTManager) , mMonitor("WMFMediaDataDecoder") , mIsFlushing(false) @@ -32,16 +34,14 @@ WMFMediaDataDecoder::~WMFMediaDataDecoder() { } -nsresult +nsRefPtr WMFMediaDataDecoder::Init() { - MOZ_ASSERT(!mDecoder); MOZ_ASSERT(!mIsShutDown); - mDecoder = mMFTManager->Init(); - NS_ENSURE_TRUE(mDecoder, NS_ERROR_FAILURE); - - return NS_OK; + return mDecoder ? + InitPromise::CreateAndResolve(mMFTManager->GetType(), __func__) : + InitPromise::CreateAndReject(MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); } nsresult @@ -188,10 +188,10 @@ WMFMediaDataDecoder::Drain() } bool -WMFMediaDataDecoder::IsHardwareAccelerated() const { +WMFMediaDataDecoder::IsHardwareAccelerated(nsACString& aFailureReason) const { MOZ_ASSERT(!mIsShutDown); - return mMFTManager && mMFTManager->IsHardwareAccelerated(); + return mMFTManager && mMFTManager->IsHardwareAccelerated(aFailureReason); } } // namespace mozilla diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.h b/dom/media/platforms/wmf/WMFMediaDataDecoder.h index b4528d0fff..7549e25fe0 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.h +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.h @@ -43,7 +43,9 @@ public: // Destroys all resources. virtual void Shutdown() = 0; - virtual bool IsHardwareAccelerated() const { return false; } + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const { return false; } + + virtual TrackInfo::TrackType GetType() = 0; }; @@ -55,11 +57,12 @@ public: class WMFMediaDataDecoder : public MediaDataDecoder { public: WMFMediaDataDecoder(MFTManager* aOutputSource, + MFTDecoder* aDecoder, FlushableTaskQueue* aAudioTaskQueue, MediaDataDecoderCallback* aCallback); ~WMFMediaDataDecoder(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample); @@ -69,7 +72,7 @@ public: virtual nsresult Shutdown() override; - virtual bool IsHardwareAccelerated() const override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override; private: diff --git a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp index c7779e7cb2..003fd5ee84 100644 --- a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp @@ -21,6 +21,7 @@ #include "IMFYCbCrImage.h" #include "mozilla/WindowsVersion.h" #include "mozilla/Preferences.h" +#include "nsPrintfCString.h" PRLogModuleInfo* GetDemuxerLog(); #define LOG(...) MOZ_LOG(GetDemuxerLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) @@ -125,8 +126,9 @@ WMFVideoMFTManager::GetMediaSubtypeGUID() class CreateDXVAManagerEvent : public nsRunnable { public: - CreateDXVAManagerEvent(LayersBackend aBackend) + CreateDXVAManagerEvent(LayersBackend aBackend, nsCString& aFailureReason) : mBackend(aBackend) + , mFailureReason(aFailureReason) {} NS_IMETHOD Run() { @@ -134,14 +136,15 @@ public: if (mBackend == LayersBackend::LAYERS_D3D11 && Preferences::GetBool("media.windows-media-foundation.allow-d3d11-dxva", false) && IsWin8OrLater()) { - mDXVA2Manager = DXVA2Manager::CreateD3D11DXVA(); + mDXVA2Manager = DXVA2Manager::CreateD3D11DXVA(mFailureReason); } else { - mDXVA2Manager = DXVA2Manager::CreateD3D9DXVA(); + mDXVA2Manager = DXVA2Manager::CreateD3D9DXVA(mFailureReason); } return NS_OK; } nsAutoPtr mDXVA2Manager; LayersBackend mBackend; + nsACString& mFailureReason; }; bool @@ -152,15 +155,19 @@ WMFVideoMFTManager::InitializeDXVA(bool aForceD3D9) // If we use DXVA but aren't running with a D3D layer manager then the // readback of decoded video frames from GPU to CPU memory grinds painting // to a halt, and makes playback performance *worse*. - if (!mDXVAEnabled || - (mLayersBackend != LayersBackend::LAYERS_D3D9 && - mLayersBackend != LayersBackend::LAYERS_D3D11)) { + if (!mDXVAEnabled) { + mDXVAFailureReason.AssignLiteral("Hardware video decoding disabled or blacklisted"); + return false; + } + if (mLayersBackend != LayersBackend::LAYERS_D3D9 && + mLayersBackend != LayersBackend::LAYERS_D3D11) { + mDXVAFailureReason.AssignLiteral("Unsupported layers backend"); return false; } // The DXVA manager must be created on the main thread. nsRefPtr event = - new CreateDXVAManagerEvent(aForceD3D9 ? LayersBackend::LAYERS_D3D9 : mLayersBackend); + new CreateDXVAManagerEvent(aForceD3D9 ? LayersBackend::LAYERS_D3D9 : mLayersBackend, mDXVAFailureReason); if (NS_IsMainThread()) { event->Run(); @@ -181,7 +188,10 @@ WMFVideoMFTManager::Init() // to d3d9. if (!decoder && mDXVA2Manager && mDXVA2Manager->IsD3D11()) { mDXVA2Manager = nullptr; + nsCString d3d11Failure = mDXVAFailureReason; decoder = InitInternal(true); + mDXVAFailureReason.Append(NS_LITERAL_CSTRING("; ")); + mDXVAFailureReason.Append(d3d11Failure); } return decoder.forget(); @@ -216,7 +226,11 @@ WMFVideoMFTManager::InitInternal(bool aForceD3D9) hr = decoder->SendMFTMessage(MFT_MESSAGE_SET_D3D_MANAGER, manager); if (SUCCEEDED(hr)) { mUseHwAccel = true; + } else { + mDXVAFailureReason = nsPrintfCString("MFT_MESSAGE_SET_D3D_MANAGER failed with code %X", hr); } + } else { + mDXVAFailureReason.AssignLiteral("Decoder returned false for MF_SA_D3D_AWARE"); } } @@ -540,8 +554,9 @@ WMFVideoMFTManager::Shutdown() } bool -WMFVideoMFTManager::IsHardwareAccelerated() const +WMFVideoMFTManager::IsHardwareAccelerated(nsACString& aFailureReason) const { + aFailureReason = mDXVAFailureReason; return mDecoder && mUseHwAccel; } diff --git a/dom/media/platforms/wmf/WMFVideoMFTManager.h b/dom/media/platforms/wmf/WMFVideoMFTManager.h index aad1fa3c9b..2d3fa0cdbd 100644 --- a/dom/media/platforms/wmf/WMFVideoMFTManager.h +++ b/dom/media/platforms/wmf/WMFVideoMFTManager.h @@ -34,7 +34,11 @@ public: virtual void Shutdown() override; - virtual bool IsHardwareAccelerated() const override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override; + + virtual TrackInfo::TrackType GetType() override { + return TrackInfo::kVideoTrack; + } private: @@ -67,6 +71,8 @@ private: const layers::LayersBackend mLayersBackend; bool mUseHwAccel; + nsCString mDXVAFailureReason; + enum StreamType { Unknown, H264, diff --git a/dom/media/platforms/wrappers/H264Converter.cpp b/dom/media/platforms/wrappers/H264Converter.cpp index c80d7f7175..a470ad1cd1 100644 --- a/dom/media/platforms/wrappers/H264Converter.cpp +++ b/dom/media/platforms/wrappers/H264Converter.cpp @@ -29,6 +29,7 @@ H264Converter::H264Converter(PlatformDecoderModule* aPDM, , mCallback(aCallback) , mDecoder(nullptr) , mNeedAVCC(aPDM->DecoderNeedsConversion(aConfig) == PlatformDecoderModule::kNeedAVCC) + , mDecoderInitializing(false) , mLastError(NS_OK) { CreateDecoder(); @@ -38,13 +39,15 @@ H264Converter::~H264Converter() { } -nsresult +nsRefPtr H264Converter::Init() { if (mDecoder) { return mDecoder->Init(); } - return mLastError; + + return MediaDataDecoder::InitPromise::CreateAndReject( + MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); } nsresult @@ -59,6 +62,12 @@ H264Converter::Input(MediaRawData* aSample) return NS_ERROR_FAILURE; } } + + if (mDecoderInitializing) { + mMediaRawSamples.AppendElement(aSample); + return NS_OK; + } + nsresult rv; if (!mDecoder) { // It is not possible to create an AVCC H264 decoder without SPS. @@ -104,6 +113,7 @@ H264Converter::Shutdown() { if (mDecoder) { nsresult rv = mDecoder->Shutdown(); + mInitPromiseRequest.DisconnectIfExists(); mDecoder = nullptr; return rv; } @@ -111,12 +121,12 @@ H264Converter::Shutdown() } bool -H264Converter::IsHardwareAccelerated() const +H264Converter::IsHardwareAccelerated(nsACString& aFailureReason) const { if (mDecoder) { - return mDecoder->IsHardwareAccelerated(); + return mDecoder->IsHardwareAccelerated(aFailureReason); } - return MediaDataDecoder::IsHardwareAccelerated(); + return MediaDataDecoder::IsHardwareAccelerated(aFailureReason); } nsresult @@ -151,8 +161,40 @@ H264Converter::CreateDecoderAndInit(MediaRawData* aSample) UpdateConfigFromExtraData(extra_data); nsresult rv = CreateDecoder(); - NS_ENSURE_SUCCESS(rv, rv); - return Init(); + + if (NS_SUCCEEDED(rv)) { + mDecoderInitializing = true; + nsRefPtr self = this; + + // The mVideoTaskQueue is flushable which can't be used in MediaPromise. So + // we get the current AbstractThread instead of it. The MOZ_ASSERT above + // ensures we are running in AbstractThread so we won't get a nullptr. + mInitPromiseRequest.Begin(mDecoder->Init() + ->Then(AbstractThread::GetCurrent(), __func__, this, + &H264Converter::OnDecoderInitDone, + &H264Converter::OnDecoderInitFailed)); + } + return rv; +} + +void +H264Converter::OnDecoderInitDone(const TrackType aTrackType) +{ + mInitPromiseRequest.Complete(); + for (uint32_t i = 0 ; i < mMediaRawSamples.Length(); i++) { + if (NS_FAILED(mDecoder->Input(mMediaRawSamples[i]))) { + mCallback->Error(); + } + } + mMediaRawSamples.Clear(); + mDecoderInitializing = false; +} + +void +H264Converter::OnDecoderInitFailed(MediaDataDecoder::DecoderFailureReason aReason) +{ + mInitPromiseRequest.Complete(); + mCallback->Error(); } nsresult diff --git a/dom/media/platforms/wrappers/H264Converter.h b/dom/media/platforms/wrappers/H264Converter.h index 948f516e1b..c218ee3cd8 100644 --- a/dom/media/platforms/wrappers/H264Converter.h +++ b/dom/media/platforms/wrappers/H264Converter.h @@ -29,12 +29,12 @@ public: MediaDataDecoderCallback* aCallback); virtual ~H264Converter(); - virtual nsresult Init() override; + virtual nsRefPtr Init() override; virtual nsresult Input(MediaRawData* aSample) override; virtual nsresult Flush() override; virtual nsresult Drain() override; virtual nsresult Shutdown() override; - virtual bool IsHardwareAccelerated() const override; + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const override; // Return true if mimetype is H.264. static bool IsH264(const TrackInfo& aConfig); @@ -49,14 +49,20 @@ private: nsresult CheckForSPSChange(MediaRawData* aSample); void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData); + void OnDecoderInitDone(const TrackType aTrackType); + void OnDecoderInitFailed(MediaDataDecoder::DecoderFailureReason aReason); + nsRefPtr mPDM; VideoInfo mCurrentConfig; layers::LayersBackend mLayersBackend; nsRefPtr mImageContainer; nsRefPtr mVideoTaskQueue; + nsTArray> mMediaRawSamples; MediaDataDecoderCallback* mCallback; nsRefPtr mDecoder; + MozPromiseRequestHolder mInitPromiseRequest; bool mNeedAVCC; + bool mDecoderInitializing; nsresult mLastError; }; diff --git a/dom/media/webm/AudioDecoder.cpp b/dom/media/webm/AudioDecoder.cpp index c629728c16..2e1eadfa64 100644 --- a/dom/media/webm/AudioDecoder.cpp +++ b/dom/media/webm/AudioDecoder.cpp @@ -47,7 +47,7 @@ ogg_packet InitOggPacket(const unsigned char* aData, size_t aLength, class VorbisDecoder : public WebMAudioDecoder { public: - nsresult Init(); + nsRefPtr Init() override; void Shutdown(); nsresult ResetDecode(); nsresult DecodeHeader(const unsigned char* aData, size_t aLength); @@ -94,14 +94,14 @@ VorbisDecoder::Shutdown() mReader = nullptr; } -nsresult +nsRefPtr VorbisDecoder::Init() { vorbis_info_init(&mVorbisInfo); vorbis_comment_init(&mVorbisComment); PodZero(&mVorbisDsp); PodZero(&mVorbisBlock); - return NS_OK; + return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); } nsresult @@ -229,7 +229,7 @@ VorbisDecoder::Decode(const unsigned char* aData, size_t aLength, class OpusDecoder : public WebMAudioDecoder { public: - nsresult Init(); + nsRefPtr Init() override; void Shutdown(); nsresult ResetDecode(); nsresult DecodeHeader(const unsigned char* aData, size_t aLength); @@ -277,10 +277,10 @@ OpusDecoder::Shutdown() mReader = nullptr; } -nsresult +nsRefPtr OpusDecoder::Init() { - return NS_OK; + return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); } nsresult diff --git a/dom/media/webm/IntelWebMVideoDecoder.cpp b/dom/media/webm/IntelWebMVideoDecoder.cpp index 72a0fe1cdc..1f70abf85a 100644 --- a/dom/media/webm/IntelWebMVideoDecoder.cpp +++ b/dom/media/webm/IntelWebMVideoDecoder.cpp @@ -106,12 +106,12 @@ IntelWebMVideoDecoder::IsSupportedVideoMimeType(const nsACString& aMimeType) mPlatform->SupportsMimeType(aMimeType); } -nsresult +nsRefPtr IntelWebMVideoDecoder::Init(unsigned int aWidth, unsigned int aHeight) { mPlatform = PlatformDecoderModule::Create(); if (!mPlatform) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } mDecoderConfig = new VideoInfo(); @@ -127,12 +127,12 @@ IntelWebMVideoDecoder::Init(unsigned int aWidth, unsigned int aHeight) mDecoderConfig->mMimeType = "video/webm; codecs=vp9"; break; default: - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } const VideoInfo& video = *mDecoderConfig; if (!IsSupportedVideoMimeType(video.mMimeType)) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } mMediaDataDecoder = mPlatform->CreateDecoder(video, @@ -141,11 +141,10 @@ IntelWebMVideoDecoder::Init(unsigned int aWidth, unsigned int aHeight) mReader->GetLayersBackendType(), mReader->GetDecoder()->GetImageContainer()); if (!mMediaDataDecoder) { - return NS_ERROR_FAILURE; + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - nsresult rv = mMediaDataDecoder->Init(); - NS_ENSURE_SUCCESS(rv, rv); - return NS_OK; + + return mMediaDataDecoder->Init(); } bool diff --git a/dom/media/webm/IntelWebMVideoDecoder.h b/dom/media/webm/IntelWebMVideoDecoder.h index bdb38fee57..508a934c39 100644 --- a/dom/media/webm/IntelWebMVideoDecoder.h +++ b/dom/media/webm/IntelWebMVideoDecoder.h @@ -28,7 +28,8 @@ class IntelWebMVideoDecoder : public WebMVideoDecoder, public MediaDataDecoderCa { public: static WebMVideoDecoder* Create(WebMReader* aReader); - virtual nsresult Init(unsigned int aWidth, unsigned int aHeight) override; + virtual nsRefPtr Init(unsigned int aWidth = 0, + unsigned int aHeight = 0) override; virtual nsresult Flush() override; virtual void Shutdown() override; diff --git a/dom/media/webm/SoftwareWebMVideoDecoder.cpp b/dom/media/webm/SoftwareWebMVideoDecoder.cpp index 6548eae996..1fd9fcc61d 100644 --- a/dom/media/webm/SoftwareWebMVideoDecoder.cpp +++ b/dom/media/webm/SoftwareWebMVideoDecoder.cpp @@ -54,11 +54,23 @@ SoftwareWebMVideoDecoder::Create(WebMReader* aReader) return new SoftwareWebMVideoDecoder(aReader); } -nsresult +nsRefPtr SoftwareWebMVideoDecoder::Init(unsigned int aWidth, unsigned int aHeight) +{ + nsresult rv = InitDecoder(aWidth, aHeight); + + if (NS_SUCCEEDED(rv)) { + return InitPromise::CreateAndResolve(TrackType::kVideoTrack, __func__); + } + + return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); +} + +nsresult +SoftwareWebMVideoDecoder::InitDecoder(unsigned int aWidth, unsigned int aHeight) { int decode_threads = 2; //Default to 2 threads for small sizes or VP8 - + vpx_codec_iface_t* dx = nullptr; switch(mReader->GetVideoCodec()) { case NESTEGG_CODEC_VP8: diff --git a/dom/media/webm/SoftwareWebMVideoDecoder.h b/dom/media/webm/SoftwareWebMVideoDecoder.h index cccfacf274..defa631638 100644 --- a/dom/media/webm/SoftwareWebMVideoDecoder.h +++ b/dom/media/webm/SoftwareWebMVideoDecoder.h @@ -17,7 +17,8 @@ class SoftwareWebMVideoDecoder : public WebMVideoDecoder public: static WebMVideoDecoder* Create(WebMReader* aReader); - virtual nsresult Init(unsigned int aWidth, unsigned int aHeight) override; + virtual nsRefPtr Init(unsigned int aWidth = 0, + unsigned int aHeight = 0) override; virtual bool DecodeVideoFrame(bool &aKeyframeSkip, int64_t aTimeThreshold) override; @@ -28,6 +29,7 @@ public: ~SoftwareWebMVideoDecoder(); private: + nsresult InitDecoder(unsigned int aWidth, unsigned int aHeight); nsRefPtr mReader; // VPx decoder state diff --git a/dom/media/webm/WebMReader.cpp b/dom/media/webm/WebMReader.cpp index 1eef3c3374..dda30f216f 100644 --- a/dom/media/webm/WebMReader.cpp +++ b/dom/media/webm/WebMReader.cpp @@ -276,8 +276,22 @@ void WebMReader::Cleanup() } } -nsresult WebMReader::ReadMetadata(MediaInfo* aInfo, - MetadataTags** aTags) +nsRefPtr +WebMReader::AsyncReadMetadata() +{ + nsRefPtr metadata = new MetadataHolder(); + + if (NS_FAILED(RetrieveWebMMetadata(&metadata->mInfo)) || + !metadata->mInfo.HasValidMedia()) { + return MetadataPromise::CreateAndReject(ReadMetadataFailureReason::METADATA_ERROR, + __func__); + } + + return MetadataPromise::CreateAndResolve(metadata, __func__); +} + +nsresult +WebMReader::RetrieveWebMMetadata(MediaInfo* aInfo) { // We can't use OnTaskQueue() here because of the wacky initialization task // queue that TrackBuffer uses. We should be able to fix this when we do @@ -330,20 +344,17 @@ nsresult WebMReader::ReadMetadata(MediaInfo* aInfo, #if defined(MOZ_PDM_VPX) if (sIsIntelDecoderEnabled) { mVideoDecoder = IntelWebMVideoDecoder::Create(this); - if (mVideoDecoder && - NS_FAILED(mVideoDecoder->Init(params.display_width, params.display_height))) { - mVideoDecoder = nullptr; - } } #endif // If there's no decoder yet (e.g. HW decoder not available), use the software decoder. if (!mVideoDecoder) { mVideoDecoder = SoftwareWebMVideoDecoder::Create(this); - if (mVideoDecoder && - NS_FAILED(mVideoDecoder->Init(params.display_width, params.display_height))) { - mVideoDecoder = nullptr; - } + } + + if (mVideoDecoder) { + mInitPromises.AppendElement(mVideoDecoder->Init(params.display_width, + params.display_height)); } if (!mVideoDecoder) { @@ -426,7 +437,10 @@ nsresult WebMReader::ReadMetadata(MediaInfo* aInfo, Cleanup(); return NS_ERROR_FAILURE; } - if (NS_FAILED(mAudioDecoder->Init())) { + + if (mAudioDecoder) { + mInitPromises.AppendElement(mAudioDecoder->Init()); + } else { Cleanup(); return NS_ERROR_FAILURE; } @@ -461,8 +475,6 @@ nsresult WebMReader::ReadMetadata(MediaInfo* aInfo, *aInfo = mInfo; - *aTags = nullptr; - return NS_OK; } diff --git a/dom/media/webm/WebMReader.h b/dom/media/webm/WebMReader.h index 7956d864f0..0e6e6f7d3d 100644 --- a/dom/media/webm/WebMReader.h +++ b/dom/media/webm/WebMReader.h @@ -9,6 +9,7 @@ #include #include "MediaDecoderReader.h" +#include "PlatformDecoderModule.h" #include "nsAutoRef.h" #include "nestegg/nestegg.h" @@ -23,6 +24,10 @@ namespace mozilla { static const unsigned NS_PER_USEC = 1000; static const double NS_PER_S = 1e9; +typedef MediaDataDecoder::InitPromise InitPromise; +typedef TrackInfo::TrackType TrackType; +typedef MediaDataDecoder::DecoderFailureReason DecoderFailureReason; + class WebMBufferedState; class WebMPacketQueue; @@ -32,7 +37,7 @@ class WebMReader; class WebMVideoDecoder { public: - virtual nsresult Init(unsigned int aWidth = 0, unsigned int aHeight = 0) = 0; + virtual nsRefPtr Init(unsigned int aWidth = 0, unsigned int aHeight = 0) = 0; virtual nsresult Flush() { return NS_OK; } virtual void Shutdown() = 0; virtual bool DecodeVideoFrame(bool &aKeyframeSkip, @@ -45,7 +50,7 @@ public: class WebMAudioDecoder { public: - virtual nsresult Init() = 0; + virtual nsRefPtr Init() = 0; virtual void Shutdown() = 0; virtual nsresult ResetDecode() = 0; virtual nsresult DecodeHeader(const unsigned char* aData, size_t aLength) = 0; @@ -85,8 +90,8 @@ public: return mHasVideo; } - virtual nsresult ReadMetadata(MediaInfo* aInfo, - MetadataTags** aTags) override; + virtual nsRefPtr AsyncReadMetadata() override; + virtual nsRefPtr Seek(int64_t aTime, int64_t aEndTime) override; @@ -120,7 +125,6 @@ public: uint64_t GetCodecDelay() { return mCodecDelay; } protected: - virtual void NotifyDataArrivedInternal(uint32_t aLength, int64_t aOffset) override; // Decode a nestegg packet of audio data. Push the audio data on the @@ -143,6 +147,8 @@ protected: bool ShouldSkipVideoFrame(int64_t aTimeThreshold); private: + nsresult RetrieveWebMMetadata(MediaInfo* aInfo); + // Get the timestamp of keyframe greater than aTimeThreshold. int64_t GetNextKeyframeTime(int64_t aTimeThreshold); // Push the packets into aOutput which's timestamp is less than aEndTime. @@ -160,6 +166,8 @@ private: nsAutoPtr mAudioDecoder; nsAutoPtr mVideoDecoder; + nsTArray> mInitPromises; + // Queue of video and audio packets that have been read but not decoded. These // must only be accessed from the decode thread. WebMPacketQueue mVideoPackets; diff --git a/dom/plugins/base/npapi.h b/dom/plugins/base/npapi.h index 5995e5bd29..2c773b096b 100644 --- a/dom/plugins/base/npapi.h +++ b/dom/plugins/base/npapi.h @@ -351,6 +351,8 @@ typedef enum { /* In the NPDrawingModelCoreAnimation drawing model, the browser asks the plug-in for a Core Animation layer. */ , NPPVpluginCoreAnimationLayer = 1003 #endif + /* Notification that the plugin just started or stopped playing audio */ + , NPPVpluginIsPlayingAudio = 4000 } NPPVariable; @@ -409,6 +411,9 @@ typedef enum { , NPNVsupportsCocoaBool = 3001 /* TRUE if the browser supports the Cocoa event model */ , NPNVsupportsUpdatedCocoaTextInputBool = 3002 /* TRUE if the browser supports the updated Cocoa text input specification. */ +#endif + , NPNVmuteAudioBool = 4000 /* Request that the browser wants to mute or unmute the plugin */ +#if defined(XP_MACOSX) , NPNVsupportsCompositingCoreAnimationPluginsBool = 74656 /* TRUE if the browser supports CA model compositing */ #endif diff --git a/dom/plugins/base/nsNPAPIPlugin.cpp b/dom/plugins/base/nsNPAPIPlugin.cpp index d27de8bb0d..b51cf33b67 100644 --- a/dom/plugins/base/nsNPAPIPlugin.cpp +++ b/dom/plugins/base/nsNPAPIPlugin.cpp @@ -106,6 +106,8 @@ using mozilla::plugins::PluginModuleContentParent; #define LOG(args...) __android_log_print(ANDROID_LOG_INFO, "GeckoPlugins" , ## args) #endif +#include "nsIAudioChannelAgent.h" + using namespace mozilla; using namespace mozilla::plugins::parent; @@ -2411,6 +2413,46 @@ _setvalue(NPP npp, NPPVariable variable, void *result) return inst->SetUsesDOMForCursor(useDOMForCursor); } + case NPPVpluginIsPlayingAudio: { + bool isMuted = !result; + + nsNPAPIPluginInstance* inst = (nsNPAPIPluginInstance*) npp->ndata; + MOZ_ASSERT(inst); + + if (isMuted && !inst->HasAudioChannelAgent()) { + return NPERR_NO_ERROR; + } + + nsCOMPtr agent; + nsresult rv = inst->GetOrCreateAudioChannelAgent(getter_AddRefs(agent)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return NPERR_NO_ERROR; + } + + MOZ_ASSERT(agent); + + if (isMuted) { + rv = agent->NotifyStoppedPlaying(); + if (NS_WARN_IF(NS_FAILED(rv))) { + return NPERR_NO_ERROR; + } + } else { + float volume = 0.0; + bool muted = true; + rv = agent->NotifyStartedPlaying(&volume, &muted); + if (NS_WARN_IF(NS_FAILED(rv))) { + return NPERR_NO_ERROR; + } + + rv = inst->WindowVolumeChanged(volume, muted); + if (NS_WARN_IF(NS_FAILED(rv))) { + return NPERR_NO_ERROR; + } + } + + return NPERR_NO_ERROR; + } + #ifndef MOZ_WIDGET_ANDROID // On android, their 'drawing model' uses the same constant! case NPPVpluginDrawingModel: { diff --git a/dom/plugins/base/nsNPAPIPluginInstance.cpp b/dom/plugins/base/nsNPAPIPluginInstance.cpp index 919fc5be06..a735d37fdf 100644 --- a/dom/plugins/base/nsNPAPIPluginInstance.cpp +++ b/dom/plugins/base/nsNPAPIPluginInstance.cpp @@ -40,6 +40,7 @@ #include "mozilla/unused.h" #include "nsILoadContext.h" #include "mozilla/dom/HTMLObjectElementBinding.h" +#include "AudioChannelService.h" using namespace mozilla; using namespace mozilla::dom; @@ -168,7 +169,7 @@ using namespace mozilla::layers; static NS_DEFINE_IID(kIOutputStreamIID, NS_IOUTPUTSTREAM_IID); -NS_IMPL_ISUPPORTS0(nsNPAPIPluginInstance) +NS_IMPL_ISUPPORTS(nsNPAPIPluginInstance, nsIAudioChannelAgentCallback) nsNPAPIPluginInstance::nsNPAPIPluginInstance() : mDrawingModel(kDefaultDrawingModel) @@ -252,6 +253,7 @@ nsNPAPIPluginInstance::Destroy() { Stop(); mPlugin = nullptr; + mAudioChannelAgent = nullptr; #if MOZ_WIDGET_ANDROID if (mContentSurface) @@ -1787,3 +1789,71 @@ nsNPAPIPluginInstance::GetRunID(uint32_t* aRunID) return library->GetRunID(aRunID); } + +nsresult +nsNPAPIPluginInstance::GetOrCreateAudioChannelAgent(nsIAudioChannelAgent** aAgent) +{ + if (!mAudioChannelAgent) { + nsresult rv; + mAudioChannelAgent = do_CreateInstance("@mozilla.org/audiochannelagent;1", &rv); + if (NS_WARN_IF(!mAudioChannelAgent)) { + return NS_ERROR_FAILURE; + } + + nsCOMPtr window = GetDOMWindow(); + if (NS_WARN_IF(!window)) { + return NS_ERROR_FAILURE; + } + + rv = mAudioChannelAgent->Init(window->GetCurrentInnerWindow(), + (int32_t)AudioChannelService::GetDefaultAudioChannel(), + this); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + } + + nsCOMPtr agent = mAudioChannelAgent; + agent.forget(aAgent); + return NS_OK; +} + +NS_IMETHODIMP +nsNPAPIPluginInstance::WindowVolumeChanged(float aVolume, bool aMuted) +{ + // We just support mute/unmute + nsresult rv = SetMuted(aMuted); + NS_WARN_IF(NS_FAILED(rv)); + return rv; +} + +NS_IMETHODIMP +nsNPAPIPluginInstance::WindowAudioCaptureChanged() +{ + return NS_OK; +} + +nsresult +nsNPAPIPluginInstance::SetMuted(bool aIsMuted) +{ + if (RUNNING != mRunning) + return NS_OK; + + PLUGIN_LOG(PLUGIN_LOG_NORMAL, ("nsNPAPIPluginInstance informing plugin of mute state change this=%p\n",this)); + + if (!mPlugin || !mPlugin->GetLibrary()) + return NS_ERROR_FAILURE; + + NPPluginFuncs* pluginFunctions = mPlugin->PluginFuncs(); + + if (!pluginFunctions->setvalue) + return NS_ERROR_FAILURE; + + PluginDestructionGuard guard(this); + + NPError error; + NPBool value = static_cast(aIsMuted); + NS_TRY_SAFE_CALL_RETURN(error, (*pluginFunctions->setvalue)(&mNPP, NPNVmuteAudioBool, &value), this, + NS_PLUGIN_CALL_UNSAFE_TO_REENTER_GECKO); + return (error == NPERR_NO_ERROR) ? NS_OK : NS_ERROR_FAILURE; +} diff --git a/dom/plugins/base/nsNPAPIPluginInstance.h b/dom/plugins/base/nsNPAPIPluginInstance.h index 41dc4c839f..03e2fc00ef 100644 --- a/dom/plugins/base/nsNPAPIPluginInstance.h +++ b/dom/plugins/base/nsNPAPIPluginInstance.h @@ -17,6 +17,7 @@ #include "nsHashKeys.h" #include #include "js/TypeDecls.h" +#include "nsIAudioChannelAgent.h" #ifdef MOZ_WIDGET_ANDROID #include "nsAutoPtr.h" #include "nsIRunnable.h" @@ -75,13 +76,14 @@ public: bool needUnschedule; }; -class nsNPAPIPluginInstance : public nsISupports +class nsNPAPIPluginInstance final : public nsIAudioChannelAgentCallback { private: typedef mozilla::PluginLibrary PluginLibrary; public: NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIAUDIOCHANNELAGENTCALLBACK nsresult Initialize(nsNPAPIPlugin *aPlugin, nsPluginInstanceOwner* aOwner, const nsACString& aMIMEType); nsresult Start(); @@ -118,6 +120,15 @@ public: nsPluginInstanceOwner* GetOwner(); void SetOwner(nsPluginInstanceOwner *aOwner); + bool HasAudioChannelAgent() const + { + return !!mAudioChannelAgent; + } + + nsresult GetOrCreateAudioChannelAgent(nsIAudioChannelAgent** aAgent); + + nsresult SetMuted(bool aIsMuted); + nsNPAPIPlugin* GetPlugin(); nsresult GetNPP(NPP * aNPP); @@ -405,6 +416,8 @@ private: uint32_t mCachedParamLength; char **mCachedParamNames; char **mCachedParamValues; + + nsCOMPtr mAudioChannelAgent; }; // On Android, we need to guard against plugin code leaking entries in the local diff --git a/dom/plugins/ipc/PPluginInstance.ipdl b/dom/plugins/ipc/PPluginInstance.ipdl index 3a4a3d521d..4033813866 100644 --- a/dom/plugins/ipc/PPluginInstance.ipdl +++ b/dom/plugins/ipc/PPluginInstance.ipdl @@ -87,6 +87,8 @@ child: intr NPP_GetValue_NPPVpluginNativeAccessibleAtkPlugId() returns (nsCString plug_id, NPError result); + intr NPP_SetValue_NPNVmuteAudioBool(bool muted) returns (NPError result); + intr NPP_HandleEvent(NPRemoteEvent event) returns (int16_t handled); // special cases where we need to a shared memory buffer @@ -150,6 +152,8 @@ parent: returns (NPError result); intr NPN_SetValue_NPPVpluginEventModel(int eventModel) returns (NPError result); + intr NPN_SetValue_NPPVpluginIsPlayingAudio(bool isAudioPlaying) + returns (NPError result); intr NPN_GetURL(nsCString url, nsCString target) returns (NPError result); diff --git a/dom/plugins/ipc/PluginInstanceChild.cpp b/dom/plugins/ipc/PluginInstanceChild.cpp index 0765072859..d64364c406 100644 --- a/dom/plugins/ipc/PluginInstanceChild.cpp +++ b/dom/plugins/ipc/PluginInstanceChild.cpp @@ -624,6 +624,14 @@ PluginInstanceChild::NPN_SetValue(NPPVariable aVar, void* aValue) } #endif + case NPPVpluginIsPlayingAudio: { + NPError rv = NPERR_GENERIC_ERROR; + if (!CallNPN_SetValue_NPPVpluginIsPlayingAudio((NPBool)(intptr_t)aValue, &rv)) { + return NPERR_GENERIC_ERROR; + } + return rv; + } + default: MOZ_LOG(GetPluginLog(), LogLevel::Warning, ("In PluginInstanceChild::NPN_SetValue: Unhandled NPPVariable %i (%s)", @@ -763,6 +771,20 @@ PluginInstanceChild::AnswerNPP_SetValue_NPNVprivateModeBool(const bool& value, return true; } +bool +PluginInstanceChild::AnswerNPP_SetValue_NPNVmuteAudioBool(const bool& value, + NPError* result) +{ + if (!mPluginIface->setvalue) { + *result = NPERR_GENERIC_ERROR; + return true; + } + + NPBool v = value; + *result = mPluginIface->setvalue(GetNPP(), NPNVmuteAudioBool, &v); + return true; +} + bool PluginInstanceChild::AnswerNPP_HandleEvent(const NPRemoteEvent& event, int16_t* handled) diff --git a/dom/plugins/ipc/PluginInstanceChild.h b/dom/plugins/ipc/PluginInstanceChild.h index b4dc88779b..fa9c9b9cdd 100644 --- a/dom/plugins/ipc/PluginInstanceChild.h +++ b/dom/plugins/ipc/PluginInstanceChild.h @@ -80,6 +80,8 @@ protected: NPError* aResult) override; virtual bool AnswerNPP_SetValue_NPNVprivateModeBool(const bool& value, NPError* result) override; + virtual bool + AnswerNPP_SetValue_NPNVmuteAudioBool(const bool& value, NPError* result) override; virtual bool AnswerNPP_HandleEvent(const NPRemoteEvent& event, int16_t* handled) override; diff --git a/dom/plugins/ipc/PluginInstanceParent.cpp b/dom/plugins/ipc/PluginInstanceParent.cpp index 8e8756db3a..0753d62961 100644 --- a/dom/plugins/ipc/PluginInstanceParent.cpp +++ b/dom/plugins/ipc/PluginInstanceParent.cpp @@ -440,6 +440,15 @@ PluginInstanceParent::AnswerNPN_SetValue_NPPVpluginEventModel( #endif } +bool +PluginInstanceParent::AnswerNPN_SetValue_NPPVpluginIsPlayingAudio( + const bool& isAudioPlaying, NPError* result) +{ + *result = mNPNIface->setvalue(mNPP, NPPVpluginIsPlayingAudio, + (void*)(intptr_t)isAudioPlaying); + return true; +} + bool PluginInstanceParent::AnswerNPN_GetURL(const nsCString& url, const nsCString& target, @@ -1142,15 +1151,22 @@ PluginInstanceParent::NPP_GetValue(NPPVariable aVariable, NPError PluginInstanceParent::NPP_SetValue(NPNVariable variable, void* value) { + NPError result; switch (variable) { case NPNVprivateModeBool: - NPError result; if (!CallNPP_SetValue_NPNVprivateModeBool(*static_cast(value), &result)) return NPERR_GENERIC_ERROR; return result; + case NPNVmuteAudioBool: + if (!CallNPP_SetValue_NPNVmuteAudioBool(*static_cast(value), + &result)) + return NPERR_GENERIC_ERROR; + + return result; + default: NS_ERROR("Unhandled NPNVariable in NPP_SetValue"); MOZ_LOG(GetPluginLog(), LogLevel::Warning, diff --git a/dom/plugins/ipc/PluginInstanceParent.h b/dom/plugins/ipc/PluginInstanceParent.h index b85788b284..b90cfbf54f 100644 --- a/dom/plugins/ipc/PluginInstanceParent.h +++ b/dom/plugins/ipc/PluginInstanceParent.h @@ -128,6 +128,9 @@ public: virtual bool AnswerNPN_SetValue_NPPVpluginEventModel(const int& eventModel, NPError* result) override; + virtual bool + AnswerNPN_SetValue_NPPVpluginIsPlayingAudio(const bool& isAudioPlaying, + NPError* result) override; virtual bool AnswerNPN_GetURL(const nsCString& url, const nsCString& target, diff --git a/dom/plugins/test/testplugin/README b/dom/plugins/test/testplugin/README index e31f11cd36..1e7cc04736 100644 --- a/dom/plugins/test/testplugin/README +++ b/dom/plugins/test/testplugin/README @@ -419,3 +419,14 @@ x86-only on some OSes: Returns the contents scale factor. On platforms without support for this query always returns 1.0 (a double value). Likewise on hardware without HiDPI mode support. + +== Plugin audio channel support == + +* startAudioPlayback() +Simulates the plugin starting to play back audio. + +* stopAudioPlayback() +Simulates the plugin stopping to play back audio. + +* audioMuted() +Returns the last value set by NPP_SetValue(NPNVmuteAudioBool). diff --git a/dom/plugins/test/testplugin/nptest.cpp b/dom/plugins/test/testplugin/nptest.cpp index 9378b6f946..fa4951aa52 100644 --- a/dom/plugins/test/testplugin/nptest.cpp +++ b/dom/plugins/test/testplugin/nptest.cpp @@ -168,6 +168,9 @@ static bool getNPNVdocumentOrigin(NPObject* npobj, const NPVariant* args, uint32 static bool getMouseUpEventCount(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); static bool queryContentsScaleFactor(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); static bool echoString(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); +static bool startAudioPlayback(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); +static bool stopAudioPlayback(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); +static bool getAudioMuted(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result); static const NPUTF8* sPluginMethodIdentifierNames[] = { "npnEvaluateTest", @@ -234,6 +237,9 @@ static const NPUTF8* sPluginMethodIdentifierNames[] = { "getMouseUpEventCount", "queryContentsScaleFactor", "echoString", + "startAudioPlayback", + "stopAudioPlayback", + "audioMuted", }; static NPIdentifier sPluginMethodIdentifiers[ARRAY_LENGTH(sPluginMethodIdentifierNames)]; static const ScriptableFunction sPluginMethodFunctions[] = { @@ -301,6 +307,9 @@ static const ScriptableFunction sPluginMethodFunctions[] = { getMouseUpEventCount, queryContentsScaleFactor, echoString, + startAudioPlayback, + stopAudioPlayback, + getAudioMuted, }; static_assert(ARRAY_LENGTH(sPluginMethodIdentifierNames) == @@ -784,6 +793,8 @@ NPP_New(NPMIMEType pluginType, NPP instance, uint16_t mode, int16_t argc, char* instanceData->npnNewStream = false; instanceData->invalidateDuringPaint = false; instanceData->slowPaint = false; + instanceData->playingAudio = false; + instanceData->audioMuted = false; instanceData->writeCount = 0; instanceData->writeReadyCount = 0; memset(&instanceData->window, 0, sizeof(instanceData->window)); @@ -1440,6 +1451,11 @@ NPP_SetValue(NPP instance, NPNVariable variable, void* value) instanceData->lastReportedPrivateModeState = bool(*static_cast(value)); return NPERR_NO_ERROR; } + if (variable == NPNVmuteAudioBool) { + InstanceData* instanceData = (InstanceData*)(instance->pdata); + instanceData->audioMuted = bool(*static_cast(value)); + return NPERR_NO_ERROR; + } return NPERR_GENERIC_ERROR; } @@ -3707,3 +3723,45 @@ bool echoString(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVar return true; } +static bool +toggleAudioPlayback(NPObject* npobj, uint32_t argCount, bool playingAudio, NPVariant* result) +{ + if (argCount != 0) { + return false; + } + + NPP npp = static_cast(npobj)->npp; + InstanceData* id = static_cast(npp->pdata); + id->playingAudio = playingAudio; + + NPN_SetValue(npp, NPPVpluginIsPlayingAudio, (void*)playingAudio); + + VOID_TO_NPVARIANT(*result); + return true; +} + +static bool +startAudioPlayback(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result) +{ + return toggleAudioPlayback(npobj, argCount, true, result); +} + +static bool +stopAudioPlayback(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result) +{ + return toggleAudioPlayback(npobj, argCount, false, result); +} + +static bool +getAudioMuted(NPObject* npobj, const NPVariant* args, uint32_t argCount, NPVariant* result) +{ + if (argCount != 0) { + return false; + } + + NPP npp = static_cast(npobj)->npp; + InstanceData* id = static_cast(npp->pdata); + BOOLEAN_TO_NPVARIANT(id->audioMuted, *result); + return true; +} + diff --git a/dom/plugins/test/testplugin/nptest.h b/dom/plugins/test/testplugin/nptest.h index c929a38d7a..fd4fb9018c 100644 --- a/dom/plugins/test/testplugin/nptest.h +++ b/dom/plugins/test/testplugin/nptest.h @@ -106,6 +106,8 @@ typedef struct InstanceData { bool asyncCallbackResult; bool invalidateDuringPaint; bool slowPaint; + bool playingAudio; + bool audioMuted; int32_t winX; int32_t winY; int32_t lastMouseX; diff --git a/toolkit/modules/tests/browser/browser_Troubleshoot.js b/toolkit/modules/tests/browser/browser_Troubleshoot.js index 34f88d85d9..bae13b7918 100644 --- a/toolkit/modules/tests/browser/browser_Troubleshoot.js +++ b/toolkit/modules/tests/browser/browser_Troubleshoot.js @@ -207,7 +207,7 @@ const SNAPSHOT_SCHEMA = { type: "boolean", }, supportsHardwareH264: { - type: "boolean", + type: "string", }, numAcceleratedWindowsMessage: { type: "array", diff --git a/xpcom/threads/AbstractThread.cpp b/xpcom/threads/AbstractThread.cpp index 46ba787e38..0dcb89ec1e 100644 --- a/xpcom/threads/AbstractThread.cpp +++ b/xpcom/threads/AbstractThread.cpp @@ -141,4 +141,11 @@ AbstractThread::DispatchDirectTask(already_AddRefed aRunnable) GetCurrent()->TailDispatcher().AddDirectTask(Move(aRunnable)); } +already_AddRefed +CreateXPCOMAbstractThreadWrapper(nsIThread* aThread, bool aRequireTailDispatch) +{ + nsRefPtr wrapper = new XPCOMThreadWrapper(aThread, aRequireTailDispatch); + return wrapper.forget(); +} + } // namespace mozilla diff --git a/xpcom/threads/AbstractThread.h b/xpcom/threads/AbstractThread.h index 650968d2e7..75544950ac 100644 --- a/xpcom/threads/AbstractThread.h +++ b/xpcom/threads/AbstractThread.h @@ -92,6 +92,9 @@ protected: const bool mSupportsTailDispatch; }; +already_AddRefed CreateXPCOMAbstractThreadWrapper(nsIThread* aThread, + bool aRequireTailDispatch); + } // namespace mozilla #endif