diff --git a/dom/media/AbstractMediaDecoder.h b/dom/media/AbstractMediaDecoder.h index 9be616c924..99693ff2c8 100644 --- a/dom/media/AbstractMediaDecoder.h +++ b/dom/media/AbstractMediaDecoder.h @@ -49,9 +49,6 @@ public: // state. virtual ReentrantMonitor& GetReentrantMonitor() = 0; - // Returns true if the decoder is shut down. - virtual bool IsShutdown() const = 0; - // A special version of the above for the ogg decoder that is allowed to be // called cross-thread. virtual bool IsOggDecoderShutdown() { return false; } diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 874233ef16..48ca500ff2 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -37,14 +37,6 @@ static const int DEFAULT_HEURISTIC_DORMANT_TIMEOUT_MSECS = 60000; namespace mozilla { -// Number of estimated seconds worth of data we need to have buffered -// ahead of the current playback position before we allow the media decoder -// to report that it can play through the entire media without the decode -// catching up with the download. Having this margin make the -// MediaDecoder::CanPlayThrough() calculation more stable in the case of -// fluctuating bitrates. -static const int64_t CAN_PLAY_THROUGH_MARGIN = 1; - // The amount of instability we tollerate in calls to // MediaDecoder::UpdateEstimatedMediaDuration(); changes of duration // less than this are ignored, as they're assumed to be the result of @@ -58,15 +50,20 @@ PRLogModuleInfo* gMediaDecoderLog; #define DECODER_LOG(x, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, ("Decoder=%p " x, this, ##__VA_ARGS__)) -static const char* const gPlayStateStr[] = { - "START", - "LOADING", - "PAUSED", - "PLAYING", - "SEEKING", - "ENDED", - "SHUTDOWN" -}; +static const char* +ToPlayStateStr(MediaDecoder::PlayState aState) +{ + switch (aState) { + case MediaDecoder::PLAY_STATE_START: return "START"; + case MediaDecoder::PLAY_STATE_LOADING: return "LOADING"; + case MediaDecoder::PLAY_STATE_PAUSED: return "PAUSED"; + case MediaDecoder::PLAY_STATE_PLAYING: return "PLAYING"; + case MediaDecoder::PLAY_STATE_ENDED: return "ENDED"; + case MediaDecoder::PLAY_STATE_SHUTDOWN: return "SHUTDOWN"; + default: MOZ_ASSERT_UNREACHABLE("Invalid playState."); + } + return "UNKNOWN"; +} class MediaMemoryTracker : public nsIMemoryReporter { @@ -138,7 +135,8 @@ NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter) NS_IMPL_ISUPPORTS0(MediaDecoder) -void MediaDecoder::NotifyOwnerActivityChanged() +void +MediaDecoder::NotifyOwnerActivityChanged() { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -153,7 +151,20 @@ void MediaDecoder::NotifyOwnerActivityChanged() StartDormantTimer(); } -void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) +bool +MediaDecoder::IsHeuristicDormantSupported() const +{ + return +#if defined(MOZ_EME) + // We disallow dormant for encrypted media until bug 1181864 is fixed. + mInfo && + !mInfo->IsEncrypted() && +#endif + mIsHeuristicDormantSupported; +} + +void +MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) { MOZ_ASSERT(NS_IsMainThread()); GetReentrantMonitor().AssertCurrentThreadIn(); @@ -168,9 +179,11 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) } DECODER_LOG("UpdateDormantState aTimeout=%d aActivity=%d mIsDormant=%d " - "ownerActive=%d ownerHidden=%d mIsHeuristicDormant=%d mPlayState=%s", + "ownerActive=%d ownerHidden=%d mIsHeuristicDormant=%d " + "mPlayState=%s encrypted=%s", aDormantTimeout, aActivity, mIsDormant, mOwner->IsActive(), - mOwner->IsHidden(), mIsHeuristicDormant, PlayStateStr()); + mOwner->IsHidden(), mIsHeuristicDormant, PlayStateStr(), + (!mInfo ? "Unknown" : (mInfo->IsEncrypted() ? "1" : "0"))); bool prevDormant = mIsDormant; mIsDormant = false; @@ -182,10 +195,11 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) mIsDormant = true; } #endif + // Try to enable dormant by idle heuristic, when the owner is hidden. bool prevHeuristicDormant = mIsHeuristicDormant; mIsHeuristicDormant = false; - if (mIsHeuristicDormantSupported && mOwner->IsHidden()) { + if (IsHeuristicDormantSupported() && mOwner->IsHidden()) { if (aDormantTimeout && !aActivity && (mPlayState == PLAY_STATE_PAUSED || IsEnded())) { // Enable heuristic dormant @@ -208,13 +222,7 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) if (mIsDormant) { DECODER_LOG("UpdateDormantState() entering DORMANT state"); // enter dormant state - RefPtr event = - NS_NewRunnableMethodWithArg( - mDecoderStateMachine, - &MediaDecoderStateMachine::SetDormant, - true); - mDecoderStateMachine->OwnerThread()->Dispatch(event.forget()); - + mDecoderStateMachine->DispatchSetDormant(true); if (IsEnded()) { mWasEndedWhenEnteredDormant = true; } @@ -223,17 +231,12 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity) } else { DECODER_LOG("UpdateDormantState() leaving DORMANT state"); // exit dormant state - // trigger to state machine. - RefPtr event = - NS_NewRunnableMethodWithArg( - mDecoderStateMachine, - &MediaDecoderStateMachine::SetDormant, - false); - mDecoderStateMachine->OwnerThread()->Dispatch(event.forget()); + mDecoderStateMachine->DispatchSetDormant(false); } } -void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure) +void +MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aClosure); @@ -243,10 +246,11 @@ void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure) false /* aActivity */); } -void MediaDecoder::StartDormantTimer() +void +MediaDecoder::StartDormantTimer() { MOZ_ASSERT(NS_IsMainThread()); - if (!mIsHeuristicDormantSupported) { + if (!IsHeuristicDormantSupported()) { return; } @@ -269,7 +273,8 @@ void MediaDecoder::StartDormantTimer() nsITimer::TYPE_ONE_SHOT); } -void MediaDecoder::CancelDormantTimer() +void +MediaDecoder::CancelDormantTimer() { MOZ_ASSERT(NS_IsMainThread()); if (mDormantTimer) { @@ -277,7 +282,8 @@ void MediaDecoder::CancelDormantTimer() } } -void MediaDecoder::Pause() +void +MediaDecoder::Pause() { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -290,28 +296,32 @@ void MediaDecoder::Pause() ChangeState(PLAY_STATE_PAUSED); } -void MediaDecoder::SetVolume(double aVolume) +void +MediaDecoder::SetVolume(double aVolume) { MOZ_ASSERT(NS_IsMainThread()); mVolume = aVolume; } -void MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream, - bool aFinishWhenEnded) +void +MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream, + bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); mDecoderStateMachine->AddOutputStream(aStream, aFinishWhenEnded); } -void MediaDecoder::RemoveOutputStream(MediaStream* aStream) +void +MediaDecoder::RemoveOutputStream(MediaStream* aStream) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); mDecoderStateMachine->RemoveOutputStream(aStream); } -double MediaDecoder::GetDuration() +double +MediaDecoder::GetDuration() { MOZ_ASSERT(NS_IsMainThread()); return mDuration; @@ -324,14 +334,16 @@ MediaDecoder::CanonicalDurationOrNull() return mDecoderStateMachine->CanonicalDuration(); } -void MediaDecoder::SetInfinite(bool aInfinite) +void +MediaDecoder::SetInfinite(bool aInfinite) { MOZ_ASSERT(NS_IsMainThread()); mInfiniteStream = aInfinite; DurationChanged(); } -bool MediaDecoder::IsInfinite() +bool +MediaDecoder::IsInfinite() { MOZ_ASSERT(NS_IsMainThread()); return mInfiniteStream; @@ -340,11 +352,8 @@ bool MediaDecoder::IsInfinite() MediaDecoder::MediaDecoder() : mWatchManager(this, AbstractThread::MainThread()), mDormantSupported(false), - mDecoderPosition(0), - mPlaybackPosition(0), mLogicalPosition(0.0), mDuration(std::numeric_limits::quiet_NaN()), - mMediaSeekable(true), mReentrantMonitor("media.decoder"), mIgnoreProgressData(false), mInfiniteStream(false), @@ -375,6 +384,8 @@ MediaDecoder::MediaDecoder() : "MediaDecoder::mCurrentPosition (Mirror)"), mStateMachineDuration(AbstractThread::MainThread(), NullableTimeUnit(), "MediaDecoder::mStateMachineDuration (Mirror)"), + mPlaybackPosition(AbstractThread::MainThread(), 0, + "MediaDecoder::mPlaybackPosition (Mirror)"), mVolume(AbstractThread::MainThread(), 0.0, "MediaDecoder::mVolume (Canonical)"), mPlaybackRate(AbstractThread::MainThread(), 1.0, @@ -392,7 +403,15 @@ MediaDecoder::MediaDecoder() : mLogicallySeeking(AbstractThread::MainThread(), false, "MediaDecoder::mLogicallySeeking (Canonical)"), mSameOriginMedia(AbstractThread::MainThread(), false, - "MediaDecoder::mSameOriginMedia (Canonical)") + "MediaDecoder::mSameOriginMedia (Canonical)"), + mPlaybackBytesPerSecond(AbstractThread::MainThread(), 0.0, + "MediaDecoder::mPlaybackBytesPerSecond (Canonical)"), + mPlaybackRateReliable(AbstractThread::MainThread(), true, + "MediaDecoder::mPlaybackRateReliable (Canonical)"), + mDecoderPosition(AbstractThread::MainThread(), 0, + "MediaDecoder::mDecoderPosition (Canonical)"), + mMediaSeekable(AbstractThread::MainThread(), true, + "MediaDecoder::mMediaSeekable (Canonical)") { MOZ_COUNT_CTOR(MediaDecoder); MOZ_ASSERT(NS_IsMainThread()); @@ -423,7 +442,8 @@ MediaDecoder::MediaDecoder() : mWatchManager.Watch(mLogicallySeeking, &MediaDecoder::SeekingChanged); } -bool MediaDecoder::Init(MediaDecoderOwner* aOwner) +bool +MediaDecoder::Init(MediaDecoderOwner* aOwner) { MOZ_ASSERT(NS_IsMainThread()); mOwner = aOwner; @@ -432,7 +452,8 @@ bool MediaDecoder::Init(MediaDecoderOwner* aOwner) return true; } -void MediaDecoder::Shutdown() +void +MediaDecoder::Shutdown() { MOZ_ASSERT(NS_IsMainThread()); @@ -472,7 +493,8 @@ MediaDecoder::~MediaDecoder() MOZ_COUNT_DTOR(MediaDecoder); } -nsresult MediaDecoder::OpenResource(nsIStreamListener** aStreamListener) +nsresult +MediaDecoder::OpenResource(nsIStreamListener** aStreamListener) { MOZ_ASSERT(NS_IsMainThread()); if (aStreamListener) { @@ -491,8 +513,9 @@ nsresult MediaDecoder::OpenResource(nsIStreamListener** aStreamListener) return NS_OK; } -nsresult MediaDecoder::Load(nsIStreamListener** aStreamListener, - MediaDecoder* aCloneDonor) +nsresult +MediaDecoder::Load(nsIStreamListener** aStreamListener, + MediaDecoder* aCloneDonor) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mResource, "Can't load without a MediaResource"); @@ -506,7 +529,8 @@ nsresult MediaDecoder::Load(nsIStreamListener** aStreamListener, return InitializeStateMachine(aCloneDonor); } -nsresult MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor) +nsresult +MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor) { MOZ_ASSERT(NS_IsMainThread()); NS_ASSERTION(mDecoderStateMachine, "Cannot initialize null state machine!"); @@ -523,7 +547,8 @@ nsresult MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor) return NS_OK; } -void MediaDecoder::SetStateMachineParameters() +void +MediaDecoder::SetStateMachineParameters() { MOZ_ASSERT(NS_IsMainThread()); if (mMinimizePreroll) { @@ -533,7 +558,8 @@ void MediaDecoder::SetStateMachineParameters() AbstractThread::MainThread(), this, &MediaDecoder::OnMetadataUpdate); } -void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts() +void +MediaDecoder::SetMinimizePrerollUntilPlaybackStarts() { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("SetMinimizePrerollUntilPlaybackStarts()"); @@ -544,7 +570,8 @@ void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts() MOZ_DIAGNOSTIC_ASSERT(!mDecoderStateMachine); } -nsresult MediaDecoder::Play() +nsresult +MediaDecoder::Play() { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -566,7 +593,8 @@ nsresult MediaDecoder::Play() return NS_OK; } -nsresult MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) +nsresult +MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -598,30 +626,33 @@ nsresult MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) return NS_OK; } -void MediaDecoder::CallSeek(const SeekTarget& aTarget) +void +MediaDecoder::CallSeek(const SeekTarget& aTarget) { MOZ_ASSERT(NS_IsMainThread()); mSeekRequest.DisconnectIfExists(); - mSeekRequest.Begin(InvokeAsync(mDecoderStateMachine->OwnerThread(), - mDecoderStateMachine.get(), __func__, - &MediaDecoderStateMachine::Seek, aTarget) + mSeekRequest.Begin( + mDecoderStateMachine->InvokeSeek(aTarget) ->Then(AbstractThread::MainThread(), __func__, this, &MediaDecoder::OnSeekResolved, &MediaDecoder::OnSeekRejected)); } -double MediaDecoder::GetCurrentTime() +double +MediaDecoder::GetCurrentTime() { MOZ_ASSERT(NS_IsMainThread()); return mLogicalPosition; } -already_AddRefed MediaDecoder::GetCurrentPrincipal() +already_AddRefed +MediaDecoder::GetCurrentPrincipal() { MOZ_ASSERT(NS_IsMainThread()); return mResource ? mResource->GetCurrentPrincipal() : nullptr; } -void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) +void +MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) { MOZ_ASSERT(NS_IsMainThread()); RemoveMediaTracks(); @@ -632,9 +663,10 @@ void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) MediaDecoderEventVisibility::Observable); } -void MediaDecoder::MetadataLoaded(nsAutoPtr aInfo, - nsAutoPtr aTags, - MediaDecoderEventVisibility aEventVisibility) +void +MediaDecoder::MetadataLoaded(nsAutoPtr aInfo, + nsAutoPtr aTags, + MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); @@ -675,8 +707,9 @@ MediaDecoder::PlayStateStr() } } -void MediaDecoder::FirstFrameLoaded(nsAutoPtr aInfo, - MediaDecoderEventVisibility aEventVisibility) +void +MediaDecoder::FirstFrameLoaded(nsAutoPtr aInfo, + MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); @@ -713,7 +746,8 @@ void MediaDecoder::FirstFrameLoaded(nsAutoPtr aInfo, NotifySuspendedStatusChanged(); } -void MediaDecoder::ResetConnectionState() +void +MediaDecoder::ResetConnectionState() { MOZ_ASSERT(NS_IsMainThread()); if (mShuttingDown) @@ -730,7 +764,8 @@ void MediaDecoder::ResetConnectionState() Shutdown(); } -void MediaDecoder::NetworkError() +void +MediaDecoder::NetworkError() { MOZ_ASSERT(NS_IsMainThread()); if (mShuttingDown) @@ -742,7 +777,8 @@ void MediaDecoder::NetworkError() Shutdown(); } -void MediaDecoder::DecodeError() +void +MediaDecoder::DecodeError() { MOZ_ASSERT(NS_IsMainThread()); if (mShuttingDown) @@ -754,33 +790,38 @@ void MediaDecoder::DecodeError() Shutdown(); } -void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) +void +MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mSameOriginMedia = aSameOrigin; } -bool MediaDecoder::IsSeeking() const +bool +MediaDecoder::IsSeeking() const { MOZ_ASSERT(NS_IsMainThread()); return mLogicallySeeking; } -bool MediaDecoder::IsEndedOrShutdown() const +bool +MediaDecoder::IsEndedOrShutdown() const { MOZ_ASSERT(NS_IsMainThread()); return IsEnded() || mPlayState == PLAY_STATE_SHUTDOWN; } -bool MediaDecoder::IsEnded() const +bool +MediaDecoder::IsEnded() const { MOZ_ASSERT(NS_IsMainThread()); return mPlayState == PLAY_STATE_ENDED || (mWasEndedWhenEnteredDormant && (mPlayState != PLAY_STATE_SHUTDOWN)); } -void MediaDecoder::PlaybackEnded() +void +MediaDecoder::PlaybackEnded() { MOZ_ASSERT(NS_IsMainThread()); @@ -804,70 +845,67 @@ void MediaDecoder::PlaybackEnded() } } -MediaDecoder::Statistics +MediaStatistics MediaDecoder::GetStatistics() { - Statistics result; - + MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); - if (mResource) { - result.mDownloadRate = - mResource->GetDownloadRate(&result.mDownloadRateReliable); - result.mDownloadPosition = - mResource->GetCachedDataEnd(mDecoderPosition); - result.mTotalBytes = mResource->GetLength(); - result.mPlaybackRate = ComputePlaybackRate(&result.mPlaybackRateReliable); - result.mDecoderPosition = mDecoderPosition; - result.mPlaybackPosition = mPlaybackPosition; - } - else { - result.mDownloadRate = 0; - result.mDownloadRateReliable = true; - result.mPlaybackRate = 0; - result.mPlaybackRateReliable = true; - result.mDecoderPosition = 0; - result.mPlaybackPosition = 0; - result.mDownloadPosition = 0; - result.mTotalBytes = 0; - } + MOZ_ASSERT(mResource); + MediaStatistics result; + result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable); + result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition); + result.mTotalBytes = mResource->GetLength(); + result.mPlaybackRate = mPlaybackBytesPerSecond; + result.mPlaybackRateReliable = mPlaybackRateReliable; + result.mDecoderPosition = mDecoderPosition; + result.mPlaybackPosition = mPlaybackPosition; return result; } -double MediaDecoder::ComputePlaybackRate(bool* aReliable) +void +MediaDecoder::ComputePlaybackRate() { - GetReentrantMonitor().AssertCurrentThreadIn(); - MOZ_ASSERT(NS_IsMainThread() || OnStateMachineTaskQueue() || OnDecodeTaskQueue()); + MOZ_ASSERT(NS_IsMainThread()); + ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + MOZ_ASSERT(mResource); - int64_t length = mResource ? mResource->GetLength() : -1; + int64_t length = mResource->GetLength(); if (!IsNaN(mDuration) && !mozilla::IsInfinite(mDuration) && length >= 0) { - *aReliable = true; - return length * mDuration; + mPlaybackRateReliable = true; + mPlaybackBytesPerSecond = length / mDuration; + return; } - return mPlaybackStatistics->GetRateAtLastStop(aReliable); + + bool reliable = false; + mPlaybackBytesPerSecond = mPlaybackStatistics->GetRateAtLastStop(&reliable); + mPlaybackRateReliable = reliable; } -void MediaDecoder::UpdatePlaybackRate() +void +MediaDecoder::UpdatePlaybackRate() { MOZ_ASSERT(NS_IsMainThread()); GetReentrantMonitor().AssertCurrentThreadIn(); - if (!mResource) - return; - bool reliable; - uint32_t rate = uint32_t(ComputePlaybackRate(&reliable)); - if (reliable) { + MOZ_ASSERT(mResource); + + ComputePlaybackRate(); + uint32_t rate = mPlaybackBytesPerSecond; + + if (mPlaybackRateReliable) { // Avoid passing a zero rate rate = std::max(rate, 1u); - } - else { + } else { // Set a minimum rate of 10,000 bytes per second ... sometimes we just // don't have good data rate = std::max(rate, 10000u); } + mResource->SetPlaybackRate(rate); } -void MediaDecoder::NotifySuspendedStatusChanged() +void +MediaDecoder::NotifySuspendedStatusChanged() { MOZ_ASSERT(NS_IsMainThread()); if (mResource && mOwner) { @@ -876,7 +914,8 @@ void MediaDecoder::NotifySuspendedStatusChanged() } } -void MediaDecoder::NotifyBytesDownloaded() +void +MediaDecoder::NotifyBytesDownloaded() { MOZ_ASSERT(NS_IsMainThread()); { @@ -888,7 +927,8 @@ void MediaDecoder::NotifyBytesDownloaded() } } -void MediaDecoder::NotifyDownloadEnded(nsresult aStatus) +void +MediaDecoder::NotifyDownloadEnded(nsresult aStatus) { MOZ_ASSERT(NS_IsMainThread()); @@ -917,7 +957,8 @@ void MediaDecoder::NotifyDownloadEnded(nsresult aStatus) } } -void MediaDecoder::NotifyPrincipalChanged() +void +MediaDecoder::NotifyPrincipalChanged() { MOZ_ASSERT(NS_IsMainThread()); if (mOwner) { @@ -925,7 +966,8 @@ void MediaDecoder::NotifyPrincipalChanged() } } -void MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset) +void +MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset) { MOZ_ASSERT(NS_IsMainThread()); @@ -941,7 +983,8 @@ void MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset) mDecoderPosition = aOffset + aBytes; } -void MediaDecoder::OnSeekResolved(SeekResolveValue aVal) +void +MediaDecoder::OnSeekResolved(SeekResolveValue aVal) { MOZ_ASSERT(NS_IsMainThread()); mSeekRequest.Complete(); @@ -976,7 +1019,8 @@ void MediaDecoder::OnSeekResolved(SeekResolveValue aVal) } } -void MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility) +void +MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); if (mShuttingDown) @@ -989,7 +1033,8 @@ void MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility) } } -void MediaDecoder::ChangeState(PlayState aState) +void +MediaDecoder::ChangeState(PlayState aState) { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -1003,7 +1048,7 @@ void MediaDecoder::ChangeState(PlayState aState) } DECODER_LOG("ChangeState %s => %s", - gPlayStateStr[mPlayState], gPlayStateStr[aState]); + ToPlayStateStr(mPlayState), ToPlayStateStr(aState)); mPlayState = aState; if (mPlayState == PLAY_STATE_PLAYING) { @@ -1017,7 +1062,8 @@ void MediaDecoder::ChangeState(PlayState aState) StartDormantTimer(); } -void MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisibility) +void +MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); if (mShuttingDown) @@ -1044,7 +1090,8 @@ void MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisib } } -void MediaDecoder::DurationChanged() +void +MediaDecoder::DurationChanged() { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); @@ -1079,7 +1126,8 @@ void MediaDecoder::DurationChanged() } } -void MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) +void +MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) { MOZ_ASSERT(NS_IsMainThread()); @@ -1099,7 +1147,8 @@ void MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration) mEstimatedDuration = Some(TimeUnit::FromMicroseconds(aDuration)); } -void MediaDecoder::SetMediaSeekable(bool aMediaSeekable) { +void +MediaDecoder::SetMediaSeekable(bool aMediaSeekable) { MOZ_ASSERT(NS_IsMainThread()); ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); mMediaSeekable = aMediaSeekable; @@ -1108,18 +1157,20 @@ void MediaDecoder::SetMediaSeekable(bool aMediaSeekable) { bool MediaDecoder::IsTransportSeekable() { - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); + MOZ_ASSERT(NS_IsMainThread()); return GetResource()->IsTransportSeekable(); } -bool MediaDecoder::IsMediaSeekable() +bool +MediaDecoder::IsMediaSeekable() { + MOZ_ASSERT(NS_IsMainThread()); NS_ENSURE_TRUE(GetStateMachine(), false); - ReentrantMonitorAutoEnter mon(GetReentrantMonitor()); return mMediaSeekable; } -media::TimeIntervals MediaDecoder::GetSeekable() +media::TimeIntervals +MediaDecoder::GetSeekable() { MOZ_ASSERT(NS_IsMainThread()); // We can seek in buffered range if the media is seekable. Also, we can seek @@ -1138,7 +1189,8 @@ media::TimeIntervals MediaDecoder::GetSeekable() } } -void MediaDecoder::SetFragmentEndTime(double aTime) +void +MediaDecoder::SetFragmentEndTime(double aTime) { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { @@ -1146,7 +1198,8 @@ void MediaDecoder::SetFragmentEndTime(double aTime) } } -void MediaDecoder::Suspend() +void +MediaDecoder::Suspend() { MOZ_ASSERT(NS_IsMainThread()); if (mResource) { @@ -1154,7 +1207,8 @@ void MediaDecoder::Suspend() } } -void MediaDecoder::Resume(bool aForceBuffering) +void +MediaDecoder::Resume(bool aForceBuffering) { MOZ_ASSERT(NS_IsMainThread()); if (mResource) { @@ -1167,7 +1221,8 @@ void MediaDecoder::Resume(bool aForceBuffering) } } -void MediaDecoder::SetLoadInBackground(bool aLoadInBackground) +void +MediaDecoder::SetLoadInBackground(bool aLoadInBackground) { MOZ_ASSERT(NS_IsMainThread()); if (mResource) { @@ -1175,18 +1230,14 @@ void MediaDecoder::SetLoadInBackground(bool aLoadInBackground) } } -void MediaDecoder::UpdatePlaybackOffset(int64_t aOffset) -{ - GetReentrantMonitor().AssertCurrentThreadIn(); - mPlaybackPosition = aOffset; -} - -bool MediaDecoder::OnStateMachineTaskQueue() const +bool +MediaDecoder::OnStateMachineTaskQueue() const { return mDecoderStateMachine->OnTaskQueue(); } -void MediaDecoder::SetPlaybackRate(double aPlaybackRate) +void +MediaDecoder::SetPlaybackRate(double aPlaybackRate) { MOZ_ASSERT(NS_IsMainThread()); mPlaybackRate = aPlaybackRate; @@ -1204,13 +1255,15 @@ void MediaDecoder::SetPlaybackRate(double aPlaybackRate) } } -void MediaDecoder::SetPreservesPitch(bool aPreservesPitch) +void +MediaDecoder::SetPreservesPitch(bool aPreservesPitch) { MOZ_ASSERT(NS_IsMainThread()); mPreservesPitch = aPreservesPitch; } -bool MediaDecoder::OnDecodeTaskQueue() const { +bool +MediaDecoder::OnDecodeTaskQueue() const { NS_WARN_IF_FALSE(mDecoderStateMachine, "mDecoderStateMachine is null"); return mDecoderStateMachine ? mDecoderStateMachine->OnDecodeTaskQueue() : false; } @@ -1228,32 +1281,38 @@ MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine) mStateMachineIsShutdown.Connect(mDecoderStateMachine->CanonicalIsShutdown()); mNextFrameStatus.Connect(mDecoderStateMachine->CanonicalNextFrameStatus()); mCurrentPosition.Connect(mDecoderStateMachine->CanonicalCurrentPosition()); + mPlaybackPosition.Connect(mDecoderStateMachine->CanonicalPlaybackOffset()); } else { mStateMachineDuration.DisconnectIfConnected(); mBuffered.DisconnectIfConnected(); mStateMachineIsShutdown.DisconnectIfConnected(); mNextFrameStatus.DisconnectIfConnected(); mCurrentPosition.DisconnectIfConnected(); + mPlaybackPosition.DisconnectIfConnected(); } } -ReentrantMonitor& MediaDecoder::GetReentrantMonitor() { +ReentrantMonitor& +MediaDecoder::GetReentrantMonitor() { return mReentrantMonitor; } -ImageContainer* MediaDecoder::GetImageContainer() +ImageContainer* +MediaDecoder::GetImageContainer() { return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer() : nullptr; } -void MediaDecoder::InvalidateWithFlags(uint32_t aFlags) +void +MediaDecoder::InvalidateWithFlags(uint32_t aFlags) { if (mVideoFrameContainer) { mVideoFrameContainer->InvalidateWithFlags(aFlags); } } -void MediaDecoder::Invalidate() +void +MediaDecoder::Invalidate() { if (mVideoFrameContainer) { mVideoFrameContainer->Invalidate(); @@ -1262,12 +1321,14 @@ void MediaDecoder::Invalidate() // Constructs the time ranges representing what segments of the media // are buffered and playable. -media::TimeIntervals MediaDecoder::GetBuffered() { +media::TimeIntervals +MediaDecoder::GetBuffered() { MOZ_ASSERT(NS_IsMainThread()); return mBuffered.Ref(); } -size_t MediaDecoder::SizeOfVideoQueue() { +size_t +MediaDecoder::SizeOfVideoQueue() { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfVideoQueue(); @@ -1275,7 +1336,8 @@ size_t MediaDecoder::SizeOfVideoQueue() { return 0; } -size_t MediaDecoder::SizeOfAudioQueue() { +size_t +MediaDecoder::SizeOfAudioQueue() { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfAudioQueue(); @@ -1283,7 +1345,10 @@ size_t MediaDecoder::SizeOfAudioQueue() { return 0; } -void MediaDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) { +void +MediaDecoder::NotifyDataArrived(uint32_t aLength, + int64_t aOffset, + bool aThrottleUpdates) { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { @@ -1296,7 +1361,8 @@ void MediaDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aTh } // Provide access to the state machine object -MediaDecoderStateMachine* MediaDecoder::GetStateMachine() const { +MediaDecoderStateMachine* +MediaDecoder::GetStateMachine() const { return mDecoderStateMachine; } @@ -1304,30 +1370,24 @@ void MediaDecoder::NotifyWaitingForResourcesStatusChanged() { if (mDecoderStateMachine) { - RefPtr task = - NS_NewRunnableMethod(mDecoderStateMachine, - &MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged); - mDecoderStateMachine->OwnerThread()->Dispatch(task.forget()); + mDecoderStateMachine->DispatchWaitingForResourcesStatusChanged(); } } -bool MediaDecoder::IsShutdown() const { - MOZ_ASSERT(NS_IsMainThread()); - NS_ENSURE_TRUE(GetStateMachine(), true); - return mStateMachineIsShutdown; -} - // Drop reference to state machine. Only called during shutdown dance. -void MediaDecoder::BreakCycles() { +void +MediaDecoder::BreakCycles() { SetStateMachine(nullptr); } -MediaDecoderOwner* MediaDecoder::GetMediaOwner() const +MediaDecoderOwner* +MediaDecoder::GetMediaOwner() const { return mOwner; } -void MediaDecoder::FireTimeUpdate() +void +MediaDecoder::FireTimeUpdate() { MOZ_ASSERT(NS_IsMainThread()); if (!mOwner) @@ -1335,7 +1395,8 @@ void MediaDecoder::FireTimeUpdate() mOwner->FireTimeUpdate(true); } -void MediaDecoder::PinForSeek() +void +MediaDecoder::PinForSeek() { MOZ_ASSERT(NS_IsMainThread()); MediaResource* resource = GetResource(); @@ -1346,7 +1407,8 @@ void MediaDecoder::PinForSeek() resource->Pin(); } -void MediaDecoder::UnpinForSeek() +void +MediaDecoder::UnpinForSeek() { MOZ_ASSERT(NS_IsMainThread()); MediaResource* resource = GetResource(); @@ -1357,40 +1419,12 @@ void MediaDecoder::UnpinForSeek() resource->Unpin(); } -bool MediaDecoder::CanPlayThrough() +bool +MediaDecoder::CanPlayThrough() { - Statistics stats = GetStatistics(); + MOZ_ASSERT(NS_IsMainThread()); NS_ENSURE_TRUE(mDecoderStateMachine, false); - - if (mDecoderStateMachine->IsRealTime() || - (stats.mTotalBytes < 0 && stats.mDownloadRateReliable) || - (stats.mTotalBytes >= 0 && stats.mTotalBytes == stats.mDownloadPosition)) { - return true; - } - if (!stats.mDownloadRateReliable || !stats.mPlaybackRateReliable) { - return false; - } - int64_t bytesToDownload = stats.mTotalBytes - stats.mDownloadPosition; - int64_t bytesToPlayback = stats.mTotalBytes - stats.mPlaybackPosition; - double timeToDownload = bytesToDownload / stats.mDownloadRate; - double timeToPlay = bytesToPlayback / stats.mPlaybackRate; - - if (timeToDownload > timeToPlay) { - // Estimated time to download is greater than the estimated time to play. - // We probably can't play through without having to stop to buffer. - return false; - } - - // Estimated time to download is less than the estimated time to play. - // We can probably play through without having to buffer, but ensure that - // we've got a reasonable amount of data buffered after the current - // playback position, so that if the bitrate of the media fluctuates, or if - // our download rate or decode rate estimation is otherwise inaccurate, - // we don't suddenly discover that we need to buffer. This is particularly - // required near the start of the media, when not much data is downloaded. - int64_t readAheadMargin = - static_cast(stats.mPlaybackRate * CAN_PLAY_THROUGH_MARGIN); - return stats.mDownloadPosition > stats.mPlaybackPosition + readAheadMargin; + return mDecoderStateMachine->IsRealTime() || GetStatistics().CanPlayThrough(); } #ifdef MOZ_RAW diff --git a/dom/media/MediaDecoder.h b/dom/media/MediaDecoder.h index 36b9ddc02d..44bc419029 100644 --- a/dom/media/MediaDecoder.h +++ b/dom/media/MediaDecoder.h @@ -199,11 +199,11 @@ destroying the MediaDecoder object. #include "nsITimer.h" #include "AbstractMediaDecoder.h" -#include "DecodedStream.h" #include "MediaDecoderOwner.h" #include "MediaEventSource.h" #include "MediaMetadataManager.h" #include "MediaResource.h" +#include "MediaStatistics.h" #include "MediaStreamGraph.h" #include "TimeUnits.h" @@ -518,16 +518,13 @@ public: bool OnDecodeTaskQueue() const override; - MediaDecoderStateMachine* GetStateMachine() { return mDecoderStateMachine; } + MediaDecoderStateMachine* GetStateMachine() const; void SetStateMachine(MediaDecoderStateMachine* aStateMachine); // Returns the monitor for other threads to synchronise access to // state. ReentrantMonitor& GetReentrantMonitor() override; - // Returns true if the decoder is shut down - bool IsShutdown() const final override; - // Constructs the time ranges representing what segments of the media // are buffered and playable. virtual media::TimeIntervals GetBuffered(); @@ -564,13 +561,15 @@ public: // Records activity stopping on the channel. void DispatchPlaybackStopped() { nsRefPtr self = this; - nsCOMPtr r = - NS_NewRunnableFunction([self] () { self->mPlaybackStatistics->Stop(); }); + nsCOMPtr r = NS_NewRunnableFunction([self] () { + self->mPlaybackStatistics->Stop(); + self->ComputePlaybackRate(); + }); AbstractThread::MainThread()->Dispatch(r.forget()); } // The actual playback rate computation. The monitor must be held. - virtual double ComputePlaybackRate(bool* aReliable); + void ComputePlaybackRate(); // Returns true if we can play the entire media through without stopping // to buffer, given the current download and playback rates. @@ -648,13 +647,6 @@ public: // position. int64_t GetDownloadPosition(); - // Updates the approximate byte offset which playback has reached. This is - // used to calculate the readyState transitions. - void UpdatePlaybackOffset(int64_t aOffset); - - // Provide access to the state machine object - MediaDecoderStateMachine* GetStateMachine() const; - // Drop reference to state machine. Only called during shutdown dance. virtual void BreakCycles(); @@ -701,37 +693,11 @@ public: static bool IsAppleMP3Enabled(); #endif - struct Statistics { - // Estimate of the current playback rate (bytes/second). - double mPlaybackRate; - // Estimate of the current download rate (bytes/second). This - // ignores time that the channel was paused by Gecko. - double mDownloadRate; - // Total length of media stream in bytes; -1 if not known - int64_t mTotalBytes; - // Current position of the download, in bytes. This is the offset of - // the first uncached byte after the decoder position. - int64_t mDownloadPosition; - // Current position of decoding, in bytes (how much of the stream - // has been consumed) - int64_t mDecoderPosition; - // Current position of playback, in bytes - int64_t mPlaybackPosition; - // If false, then mDownloadRate cannot be considered a reliable - // estimate (probably because the download has only been running - // a short time). - bool mDownloadRateReliable; - // If false, then mPlaybackRate cannot be considered a reliable - // estimate (probably because playback has only been running - // a short time). - bool mPlaybackRateReliable; - }; - // Return statistics. This is used for progress events and other things. // This can be called from any thread. It's only a snapshot of the // current state, since other threads might be changing the state // at any time. - virtual Statistics GetStatistics(); + MediaStatistics GetStatistics(); // Frame decoding/painting related performance counters. // Threadsafe. @@ -890,17 +856,6 @@ protected: // Whether the decoder implementation supports dormant mode. bool mDormantSupported; - // Current decoding position in the stream. This is where the decoder - // is up to consuming the stream. This is not adjusted during decoder - // seek operations, but it's updated at the end when we start playing - // back again. - int64_t mDecoderPosition; - // Current playback position in the stream. This is (approximately) - // where we're up to playing back the stream. This is not adjusted - // during decoder seek operations, but it's updated at the end when we - // start playing back again. - int64_t mPlaybackPosition; - // The logical playback position of the media resource in units of // seconds. This corresponds to the "official position" in HTML5. Note that // we need to store this as a double, rather than an int64_t (like @@ -917,9 +872,6 @@ protected: // Official duration of the media resource as observed by script. double mDuration; - // True if the media is seekable (i.e. supports random access). - bool mMediaSeekable; - /****** * The following member variables can be accessed from any thread. ******/ @@ -948,8 +900,12 @@ private: ReentrantMonitor mReentrantMonitor; protected: + virtual void CallSeek(const SeekTarget& aTarget); + // Returns true if heuristic dormant is supported. + bool IsHeuristicDormantSupported() const; + MozPromiseRequestHolder mSeekRequest; // True when seeking or otherwise moving the play position around in @@ -1060,6 +1016,12 @@ protected: // Duration of the media resource according to the state machine. Mirror mStateMachineDuration; + // Current playback position in the stream. This is (approximately) + // where we're up to playing back the stream. This is not adjusted + // during decoder seek operations, but it's updated at the end when we + // start playing back again. + Mirror mPlaybackPosition; + // Volume of playback. 0.0 = muted. 1.0 = full volume. Canonical mVolume; @@ -1097,6 +1059,21 @@ protected: // passed to MediaStreams when this is true. Canonical mSameOriginMedia; + // Estimate of the current playback rate (bytes/second). + Canonical mPlaybackBytesPerSecond; + + // True if mPlaybackBytesPerSecond is a reliable estimate. + Canonical mPlaybackRateReliable; + + // Current decoding position in the stream. This is where the decoder + // is up to consuming the stream. This is not adjusted during decoder + // seek operations, but it's updated at the end when we start playing + // back again. + Canonical mDecoderPosition; + + // True if the media is seekable (i.e. supports random access). + Canonical mMediaSeekable; + public: AbstractCanonical* CanonicalDurationOrNull() override; AbstractCanonical* CanonicalVolume() { @@ -1126,6 +1103,18 @@ public: AbstractCanonical* CanonicalSameOriginMedia() { return &mSameOriginMedia; } + AbstractCanonical* CanonicalPlaybackBytesPerSecond() { + return &mPlaybackBytesPerSecond; + } + AbstractCanonical* CanonicalPlaybackRateReliable() { + return &mPlaybackRateReliable; + } + AbstractCanonical* CanonicalDecoderPosition() { + return &mDecoderPosition; + } + AbstractCanonical* CanonicalMediaSeekable() { + return &mMediaSeekable; + } }; } // namespace mozilla diff --git a/dom/media/MediaDecoderReader.cpp b/dom/media/MediaDecoderReader.cpp index c076ef920f..016d914093 100644 --- a/dom/media/MediaDecoderReader.cpp +++ b/dom/media/MediaDecoderReader.cpp @@ -295,8 +295,7 @@ public: // Make sure ResetDecode hasn't been called in the mean time. if (!mReader->mBaseVideoPromise.IsEmpty()) { - mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold, - /* aForceDecodeAhead = */ false); + mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold); } return NS_OK; @@ -333,8 +332,7 @@ private: nsRefPtr MediaDecoderReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) + int64_t aTimeThreshold) { nsRefPtr p = mBaseVideoPromise.Ensure(__func__); bool skip = aSkipToNextKeyframe; diff --git a/dom/media/MediaDecoderReader.h b/dom/media/MediaDecoderReader.h index 83ad28ab1e..3cd4f65c1a 100644 --- a/dom/media/MediaDecoderReader.h +++ b/dom/media/MediaDecoderReader.h @@ -149,7 +149,7 @@ public: // If aSkipToKeyframe is true, the decode should skip ahead to the // the next keyframe at or after aTimeThreshold microseconds. virtual nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead); + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold); friend class ReRequestVideoWithSkipTask; friend class ReRequestAudioTask; diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index 3cf3ee111d..8634e7e57e 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -9,40 +9,42 @@ #include "mmsystem.h" #endif -#include "mozilla/DebugOnly.h" +#include #include -#include "MediaDecoderStateMachine.h" -#include "MediaTimer.h" +#include "gfx2DGlue.h" + #include "mediasink/DecodedAudioDataSink.h" #include "mediasink/AudioSinkWrapper.h" -#include "nsTArray.h" -#include "MediaDecoder.h" -#include "MediaDecoderReader.h" -#include "mozilla/MathAlgorithms.h" +#include "mediasink/DecodedStream.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" #include "mozilla/mozalloc.h" -#include "VideoUtils.h" -#include "TimeUnits.h" -#include "nsDeque.h" -#include "AudioSegment.h" -#include "VideoSegment.h" -#include "ImageContainer.h" -#include "nsComponentManagerUtils.h" -#include "nsITimer.h" -#include "nsContentUtils.h" -#include "MediaShutdownManager.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/Preferences.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/TaskQueue.h" -#include "nsIEventTarget.h" -#include "prenv.h" -#include "mozilla/Preferences.h" -#include "gfx2DGlue.h" -#include "nsPrintfCString.h" -#include "DOMMediaStream.h" -#include "DecodedStream.h" -#include "mozilla/Logging.h" -#include +#include "nsComponentManagerUtils.h" +#include "nsContentUtils.h" +#include "nsIEventTarget.h" +#include "nsITimer.h" +#include "nsPrintfCString.h" +#include "nsTArray.h" +#include "nsDeque.h" +#include "prenv.h" + +#include "AudioSegment.h" +#include "DOMMediaStream.h" +#include "ImageContainer.h" +#include "MediaDecoder.h" +#include "MediaDecoderReader.h" +#include "MediaDecoderStateMachine.h" +#include "MediaShutdownManager.h" +#include "MediaTimer.h" +#include "TimeUnits.h" +#include "VideoSegment.h" +#include "VideoUtils.h" namespace mozilla { @@ -172,10 +174,11 @@ static int64_t DurationToUsecs(TimeDuration aDuration) { static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3; static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10; -static const uint32_t SCARCE_VIDEO_QUEUE_SIZE = 1; +static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999; static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE; static uint32_t sVideoQueueHWAccelSize = MIN_VIDEO_QUEUE_SIZE; +static uint32_t sVideoQueueSendToCompositorSize = VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE; MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, MediaDecoderReader* aReader, @@ -215,12 +218,12 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mDropVideoUntilNextDiscontinuity(false), mDecodeToSeekTarget(false), mCurrentTimeBeforeSeek(0), - mCorruptFrames(30), + mCorruptFrames(60), mDecodingFirstFrame(true), mSentLoadedMetadataEvent(false), mSentFirstFrameLoadedEvent(false), mSentPlaybackEndedEvent(false), - mDecodedStream(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)), + mStreamSink(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)), mResource(aDecoder->GetResource()), mBuffered(mTaskQueue, TimeIntervals(), "MediaDecoderStateMachine::mBuffered (Mirror)"), @@ -241,6 +244,14 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, "MediaDecoderStateMachine::mPreservesPitch (Mirror)"), mSameOriginMedia(mTaskQueue, false, "MediaDecoderStateMachine::mSameOriginMedia (Mirror)"), + mPlaybackBytesPerSecond(mTaskQueue, 0.0, + "MediaDecoderStateMachine::mPlaybackBytesPerSecond (Mirror)"), + mPlaybackRateReliable(mTaskQueue, true, + "MediaDecoderStateMachine::mPlaybackRateReliable (Mirror)"), + mDecoderPosition(mTaskQueue, 0, + "MediaDecoderStateMachine::mDecoderPosition (Mirror)"), + mMediaSeekable(mTaskQueue, true, + "MediaDecoderStateMachine::mMediaSeekable (Mirror)"), mDuration(mTaskQueue, NullableTimeUnit(), "MediaDecoderStateMachine::mDuration (Canonical"), mIsShutdown(mTaskQueue, false, @@ -248,7 +259,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mNextFrameStatus(mTaskQueue, MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED, "MediaDecoderStateMachine::mNextFrameStatus (Canonical)"), mCurrentPosition(mTaskQueue, 0, - "MediaDecoderStateMachine::mCurrentPosition (Canonical)") + "MediaDecoderStateMachine::mCurrentPosition (Canonical)"), + mPlaybackOffset(mTaskQueue, 0, + "MediaDecoderStateMachine::mPlaybackOffset (Canonical)") { MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); @@ -266,6 +279,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, Preferences::AddUintVarCache(&sVideoQueueHWAccelSize, "media.video-queue.hw-accel-size", MIN_VIDEO_QUEUE_SIZE); + Preferences::AddUintVarCache(&sVideoQueueSendToCompositorSize, + "media.video-queue.send-to-compositor-size", + VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE); } mBufferingWait = IsRealTime() ? 0 : 15; @@ -287,14 +303,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread()); - nsRefPtr self = this; - auto audioSinkCreator = [self] () { - MOZ_ASSERT(self->OnTaskQueue()); - return new DecodedAudioDataSink( - self->mAudioQueue, self->GetMediaTime(), - self->mInfo.mAudio, self->mDecoder->GetAudioChannel()); - }; - mAudioSink = new AudioSinkWrapper(mTaskQueue, audioSinkCreator); + mMediaSink = CreateAudioSink(); } MediaDecoderStateMachine::~MediaDecoderStateMachine() @@ -325,6 +334,10 @@ MediaDecoderStateMachine::InitializationTask() mLogicalPlaybackRate.Connect(mDecoder->CanonicalPlaybackRate()); mPreservesPitch.Connect(mDecoder->CanonicalPreservesPitch()); mSameOriginMedia.Connect(mDecoder->CanonicalSameOriginMedia()); + mPlaybackBytesPerSecond.Connect(mDecoder->CanonicalPlaybackBytesPerSecond()); + mPlaybackRateReliable.Connect(mDecoder->CanonicalPlaybackRateReliable()); + mDecoderPosition.Connect(mDecoder->CanonicalDecoderPosition()); + mMediaSeekable.Connect(mDecoder->CanonicalMediaSeekable()); // Initialize watchers. mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated); @@ -344,6 +357,19 @@ MediaDecoderStateMachine::InitializationTask() SameOriginMediaChanged(); } +media::MediaSink* +MediaDecoderStateMachine::CreateAudioSink() +{ + nsRefPtr self = this; + auto audioSinkCreator = [self] () { + MOZ_ASSERT(self->OnTaskQueue()); + return new DecodedAudioDataSink( + self->mAudioQueue, self->GetMediaTime(), + self->mInfo.mAudio, self->mDecoder->GetAudioChannel()); + }; + return new AudioSinkWrapper(mTaskQueue, audioSinkCreator); +} + bool MediaDecoderStateMachine::HasFutureAudio() { MOZ_ASSERT(OnTaskQueue()); @@ -373,7 +399,7 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration() MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); int64_t audioDecoded = AudioQueue().Duration(); - if (mAudioSink->IsStarted()) { + if (mMediaSink->IsStarted()) { audioDecoded += AudioEndTime() - GetMediaTime(); } return audioDecoded; @@ -383,7 +409,6 @@ void MediaDecoderStateMachine::DiscardStreamData() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - MOZ_ASSERT(!mAudioSink->IsStarted(), "Should've been stopped in RunStateMachine()"); const auto clockTime = GetClock(); while (true) { @@ -678,7 +703,7 @@ MediaDecoderStateMachine::OnAudioPopped(const nsRefPtr& aSample) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecoder->UpdatePlaybackOffset(std::max(0, aSample->mOffset)); + mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset); UpdateNextFrameStatus(); DispatchAudioDecodeTaskIfNeeded(); } @@ -688,7 +713,7 @@ MediaDecoderStateMachine::OnVideoPopped(const nsRefPtr& aSample) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecoder->UpdatePlaybackOffset(aSample->mOffset); + mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset); UpdateNextFrameStatus(); DispatchVideoDecodeTaskIfNeeded(); } @@ -1073,8 +1098,7 @@ void MediaDecoderStateMachine::MaybeStartPlayback() SetPlayStartTime(TimeStamp::Now()); MOZ_ASSERT(IsPlaying()); - StartAudioSink(); - StartDecodedStream(); + StartMediaSink(); DispatchDecodeTasksIfNeeded(); } @@ -1148,8 +1172,7 @@ void MediaDecoderStateMachine::VolumeChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mAudioSink->SetVolume(mVolume); - mDecodedStream->SetVolume(mVolume); + mMediaSink->SetVolume(mVolume); } void MediaDecoderStateMachine::RecomputeDuration() @@ -1184,7 +1207,16 @@ void MediaDecoderStateMachine::RecomputeDuration() mDuration = Some(duration); } -void MediaDecoderStateMachine::SetDormant(bool aDormant) +void +MediaDecoderStateMachine::DispatchSetDormant(bool aDormant) +{ + nsCOMPtr r = NS_NewRunnableMethodWithArg( + this, &MediaDecoderStateMachine::SetDormant, aDormant); + OwnerThread()->Dispatch(r.forget()); +} + +void +MediaDecoderStateMachine::SetDormant(bool aDormant) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); @@ -1266,7 +1298,7 @@ void MediaDecoderStateMachine::Shutdown() Reset(); - mAudioSink->Shutdown(); + mMediaSink->Shutdown(); // Shut down our start time rendezvous. if (mStartTimeRendezvous) { @@ -1329,7 +1361,16 @@ void MediaDecoderStateMachine::StartDecoding() ScheduleStateMachine(); } -void MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged() +void +MediaDecoderStateMachine::DispatchWaitingForResourcesStatusChanged() +{ + nsCOMPtr r = NS_NewRunnableMethod( + this, &MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged); + OwnerThread()->Dispatch(r.forget()); +} + +void +MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); @@ -1398,7 +1439,7 @@ void MediaDecoderStateMachine::SameOriginMediaChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mDecodedStream->SetSameOrigin(mSameOriginMedia); + mStreamSink->SetSameOrigin(mSameOriginMedia); } void MediaDecoderStateMachine::BufferedRangeUpdated() @@ -1431,7 +1472,7 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget) // We need to be able to seek both at a transport level and at a media level // to seek. - if (!mDecoder->IsMediaSeekable()) { + if (!mMediaSeekable) { DECODER_WARN("Seek() function should not be called on a non-seekable state machine"); return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__); } @@ -1457,15 +1498,22 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget) return mPendingSeek.mPromise.Ensure(__func__); } -void MediaDecoderStateMachine::StopAudioSink() +nsRefPtr +MediaDecoderStateMachine::InvokeSeek(SeekTarget aTarget) +{ + return InvokeAsync(OwnerThread(), this, __func__, + &MediaDecoderStateMachine::Seek, aTarget); +} + +void MediaDecoderStateMachine::StopMediaSink() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (mAudioSink->IsStarted()) { - DECODER_LOG("Stop AudioSink"); - mAudioSink->Stop(); - mAudioSinkPromise.DisconnectIfExists(); + if (mMediaSink->IsStarted()) { + DECODER_LOG("Stop MediaSink"); + mMediaSink->Stop(); + mMediaSinkPromise.DisconnectIfExists(); } } @@ -1714,8 +1762,6 @@ MediaDecoderStateMachine::RequestVideoData() bool skipToNextKeyFrame = mSentFirstFrameLoadedEvent && NeedToSkipToNextKeyframe(); int64_t currentTime = mState == DECODER_STATE_SEEKING ? 0 : GetMediaTime(); - bool forceDecodeAhead = mSentFirstFrameLoadedEvent && - static_cast(VideoQueue().GetSize()) <= SCARCE_VIDEO_QUEUE_SIZE; SAMPLE_LOG("Queueing video task - queued=%i, decoder-queued=%o, skip=%i, time=%lld", VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), skipToNextKeyFrame, @@ -1725,7 +1771,7 @@ MediaDecoderStateMachine::RequestVideoData() mVideoDataRequest.Begin( InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestVideoData, - skipToNextKeyFrame, currentTime, forceDecodeAhead) + skipToNextKeyFrame, currentTime) ->Then(OwnerThread(), __func__, this, &MediaDecoderStateMachine::OnVideoDecoded, &MediaDecoderStateMachine::OnVideoNotDecoded)); @@ -1733,7 +1779,7 @@ MediaDecoderStateMachine::RequestVideoData() mVideoDataRequest.Begin( InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__, &MediaDecoderReader::RequestVideoData, - skipToNextKeyFrame, currentTime, forceDecodeAhead) + skipToNextKeyFrame, currentTime) ->Then(OwnerThread(), __func__, mStartTimeRendezvous.get(), &StartTimeRendezvous::ProcessFirstSample, &StartTimeRendezvous::FirstSampleRejected) @@ -1745,55 +1791,25 @@ MediaDecoderStateMachine::RequestVideoData() } void -MediaDecoderStateMachine::StartAudioSink() +MediaDecoderStateMachine::StartMediaSink() { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (mAudioCaptured) { - MOZ_ASSERT(!mAudioSink->IsStarted()); - return; - } - if (!mAudioSink->IsStarted()) { + if (!mMediaSink->IsStarted()) { mAudioCompleted = false; - mAudioSink->Start(GetMediaTime(), mInfo); + mMediaSink->Start(GetMediaTime(), mInfo); - auto promise = mAudioSink->OnEnded(TrackInfo::kAudioTrack); + auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack); if (promise) { - mAudioSinkPromise.Begin(promise->Then( + mMediaSinkPromise.Begin(promise->Then( OwnerThread(), __func__, this, - &MediaDecoderStateMachine::OnAudioSinkComplete, - &MediaDecoderStateMachine::OnAudioSinkError)); + &MediaDecoderStateMachine::OnMediaSinkComplete, + &MediaDecoderStateMachine::OnMediaSinkError)); } } } -void -MediaDecoderStateMachine::StopDecodedStream() -{ - MOZ_ASSERT(OnTaskQueue()); - AssertCurrentThreadInMonitor(); - mDecodedStream->StopPlayback(); - mDecodedStreamPromise.DisconnectIfExists(); -} - -void -MediaDecoderStateMachine::StartDecodedStream() -{ - MOZ_ASSERT(OnTaskQueue()); - AssertCurrentThreadInMonitor(); - - // Tell DecodedStream to start playback with specified start time and media - // info. This is consistent with how we create AudioSink in StartAudioThread(). - if (mAudioCaptured && !mDecodedStreamPromise.Exists()) { - mDecodedStreamPromise.Begin( - mDecodedStream->StartPlayback(GetMediaTime(), mInfo)->Then( - OwnerThread(), __func__, this, - &MediaDecoderStateMachine::OnDecodedStreamFinish, - &MediaDecoderStateMachine::OnDecodedStreamError)); - } -} - int64_t MediaDecoderStateMachine::AudioDecodedUsecs() { MOZ_ASSERT(OnTaskQueue()); @@ -1802,7 +1818,7 @@ int64_t MediaDecoderStateMachine::AudioDecodedUsecs() // The amount of audio we have decoded is the amount of audio data we've // already decoded and pushed to the hardware, plus the amount of audio // data waiting to be pushed to the hardware. - int64_t pushed = mAudioSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0; + int64_t pushed = mMediaSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0; // Currently for real time streams, AudioQueue().Duration() produce // wrong values (Bug 1114434), so we use frame counts to calculate duration. @@ -1831,7 +1847,7 @@ bool MediaDecoderStateMachine::OutOfDecodedAudio() MOZ_ASSERT(OnTaskQueue()); return IsAudioDecoding() && !AudioQueue().IsFinished() && AudioQueue().GetSize() == 0 && - !mAudioSink->HasUnplayedFrames(TrackInfo::kAudioTrack); + !mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack); } bool MediaDecoderStateMachine::HasLowUndecodedData() @@ -2047,7 +2063,7 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame() DECODER_LOG("Media duration %lld, " "transportSeekable=%d, mediaSeekable=%d", - Duration().ToMicroseconds(), mDecoder->IsTransportSeekable(), mDecoder->IsMediaSeekable()); + Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable.Ref()); if (HasAudio() && !HasVideo() && !mSentFirstFrameLoadedEvent) { // We're playing audio only. We don't need to worry about slow video @@ -2179,6 +2195,15 @@ private: nsRefPtr mStateMachine; }; +void +MediaDecoderStateMachine::DispatchShutdown() +{ + mStreamSink->BeginShutdown(); + nsCOMPtr runnable = + NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown); + OwnerThread()->Dispatch(runnable.forget()); +} + void MediaDecoderStateMachine::FinishShutdown() { @@ -2205,10 +2230,16 @@ MediaDecoderStateMachine::FinishShutdown() mLogicalPlaybackRate.DisconnectIfConnected(); mPreservesPitch.DisconnectIfConnected(); mSameOriginMedia.DisconnectIfConnected(); + mPlaybackBytesPerSecond.DisconnectIfConnected(); + mPlaybackRateReliable.DisconnectIfConnected(); + mDecoderPosition.DisconnectIfConnected(); + mMediaSeekable.DisconnectIfConnected(); + mDuration.DisconnectAll(); mIsShutdown.DisconnectAll(); mNextFrameStatus.DisconnectAll(); mCurrentPosition.DisconnectAll(); + mPlaybackOffset.DisconnectAll(); // Shut down the watch manager before shutting down our task queue. mWatchManager.Shutdown(); @@ -2310,7 +2341,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine() if (mReader->UseBufferingHeuristics()) { TimeDuration elapsed = now - mBufferingStart; bool isLiveStream = resource->IsLiveStream(); - if ((isLiveStream || !mDecoder->CanPlayThrough()) && + if ((isLiveStream || !CanPlayThrough()) && elapsed < TimeDuration::FromSeconds(mBufferingWait * mPlaybackRate) && (mQuickBuffering ? HasLowDecodedData(mQuickBufferingLowDataThresholdUsecs) : HasLowUndecodedData(mBufferingWait * USECS_PER_S)) && @@ -2359,7 +2390,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine() // end of the media, and so that we update the readyState. if (VideoQueue().GetSize() > 1 || (HasAudio() && !mAudioCompleted) || - (mAudioCaptured && !mDecodedStream->IsFinished())) + (mAudioCaptured && !mStreamSink->IsFinished())) { // Start playback if necessary to play the remaining media. MaybeStartPlayback(); @@ -2396,8 +2427,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine() mSentPlaybackEndedEvent = true; // MediaSink::GetEndTime() must be called before stopping playback. - StopAudioSink(); - StopDecodedStream(); + StopMediaSink(); } return NS_OK; @@ -2423,11 +2453,10 @@ MediaDecoderStateMachine::Reset() mState == DECODER_STATE_DORMANT || mState == DECODER_STATE_DECODING_NONE); - // Stop the audio thread. Otherwise, AudioSink might be accessing AudioQueue + // Stop the audio thread. Otherwise, MediaSink might be accessing AudioQueue // outside of the decoder monitor while we are clearing the queue and causes // crash for no samples to be popped. - StopAudioSink(); - StopDecodedStream(); + StopMediaSink(); mVideoFrameEndTime = -1; mDecodedVideoEndTime = -1; @@ -2447,6 +2476,8 @@ MediaDecoderStateMachine::Reset() mVideoWaitRequest.DisconnectIfExists(); mSeekRequest.DisconnectIfExists(); + mPlaybackOffset = 0; + nsCOMPtr resetTask = NS_NewRunnableMethod(mReader, &MediaDecoderReader::ResetDecode); DecodeTaskQueue()->Dispatch(resetTask.forget()); @@ -2472,8 +2503,8 @@ bool MediaDecoderStateMachine::CheckFrameValidity(VideoData* aData) // only supports integer types. mCorruptFrames.insert(10); if (mReader->VideoIsHardwareAccelerated() && - frameStats.GetPresentedFrames() > 30 && - mCorruptFrames.mean() >= 1 /* 10% */) { + frameStats.GetPresentedFrames() > 60 && + mCorruptFrames.mean() >= 2 /* 20% */) { nsCOMPtr task = NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration); DecodeTaskQueue()->Dispatch(task.forget()); @@ -2552,13 +2583,6 @@ void MediaDecoderStateMachine::RenderVideoFrames(int32_t aMaxFrames, container->SetCurrentFrames(frames[0]->As()->mDisplay, images); } -int64_t MediaDecoderStateMachine::GetStreamClock() const -{ - MOZ_ASSERT(OnTaskQueue()); - AssertCurrentThreadInMonitor(); - return mDecodedStream->GetPosition(); -} - int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const { MOZ_ASSERT(OnTaskQueue()); @@ -2573,11 +2597,7 @@ int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const if (!IsPlaying()) { clock_time = mPlayDuration; } else { - if (mAudioCaptured) { - clock_time = GetStreamClock(); - } else { - clock_time = mAudioSink->GetPosition(&t); - } + clock_time = mMediaSink->GetPosition(&t); NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards."); } if (aTimeStamp) { @@ -2633,7 +2653,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames() } } - RenderVideoFrames(1, clockTime, nowTime); + RenderVideoFrames(sVideoQueueSendToCompositorSize, clockTime, nowTime); // Check to see if we don't have enough data to play up to the next frame. // If we don't, switch to buffering mode. @@ -2828,6 +2848,28 @@ bool MediaDecoderStateMachine::JustExitedQuickBuffering() (TimeStamp::Now() - mDecodeStartTime) < TimeDuration::FromMicroseconds(QUICK_BUFFER_THRESHOLD_USECS); } +bool +MediaDecoderStateMachine::CanPlayThrough() +{ + MOZ_ASSERT(OnTaskQueue()); + return IsRealTime() || GetStatistics().CanPlayThrough(); +} + +MediaStatistics +MediaDecoderStateMachine::GetStatistics() +{ + MOZ_ASSERT(OnTaskQueue()); + MediaStatistics result; + result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable); + result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition); + result.mTotalBytes = mResource->GetLength(); + result.mPlaybackRate = mPlaybackBytesPerSecond; + result.mPlaybackRateReliable = mPlaybackRateReliable; + result.mDecoderPosition = mDecoderPosition; + result.mPlaybackPosition = mPlaybackOffset; + return result; +} + void MediaDecoderStateMachine::StartBuffering() { MOZ_ASSERT(OnTaskQueue()); @@ -2858,7 +2900,7 @@ void MediaDecoderStateMachine::StartBuffering() SetState(DECODER_STATE_BUFFERING); DECODER_LOG("Changed state from DECODING to BUFFERING, decoded for %.3lfs", decodeDuration.ToSeconds()); - MediaDecoder::Statistics stats = mDecoder->GetStatistics(); + MediaStatistics stats = GetStatistics(); DECODER_LOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s", stats.mPlaybackRate/1024, stats.mPlaybackRateReliable ? "" : " (unreliable)", stats.mDownloadRate/1024, stats.mDownloadRateReliable ? "" : " (unreliable)"); @@ -2869,12 +2911,7 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp) MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); mPlayStartTime = aTimeStamp; - - mAudioSink->SetPlaying(!mPlayStartTime.IsNull()); - // Have DecodedStream remember the playing state so it doesn't need to - // ask MDSM about IsPlaying(). Note we have to do this even before capture - // happens since capture could happen in the middle of playback. - mDecodedStream->SetPlaying(!mPlayStartTime.IsNull()); + mMediaSink->SetPlaying(!mPlayStartTime.IsNull()); } void MediaDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder() @@ -2952,7 +2989,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged() } mPlaybackRate = mLogicalPlaybackRate; - mAudioSink->SetPlaybackRate(mPlaybackRate); + mMediaSink->SetPlaybackRate(mPlaybackRate); ScheduleStateMachine(); } @@ -2961,7 +2998,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - mAudioSink->SetPreservesPitch(mPreservesPitch); + mMediaSink->SetPreservesPitch(mPreservesPitch); } bool MediaDecoderStateMachine::IsShutdown() @@ -2975,33 +3012,33 @@ MediaDecoderStateMachine::AudioEndTime() const { MOZ_ASSERT(OnTaskQueue()); AssertCurrentThreadInMonitor(); - if (mAudioSink->IsStarted()) { - return mAudioSink->GetEndTime(TrackInfo::kAudioTrack); - } else if (mAudioCaptured) { - return mDecodedStream->AudioEndTime(); + if (mMediaSink->IsStarted()) { + return mMediaSink->GetEndTime(TrackInfo::kAudioTrack); } MOZ_ASSERT(!HasAudio()); return -1; } -void MediaDecoderStateMachine::OnAudioSinkComplete() +void MediaDecoderStateMachine::OnMediaSinkComplete() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio."); - mAudioSinkPromise.Complete(); - mAudioCompleted = true; + mMediaSinkPromise.Complete(); + // Set true only when we have audio. + mAudioCompleted = mInfo.HasAudio(); + // To notify PlaybackEnded as soon as possible. + ScheduleStateMachine(); } -void MediaDecoderStateMachine::OnAudioSinkError() +void MediaDecoderStateMachine::OnMediaSinkError() { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio."); - mAudioSinkPromise.Complete(); - mAudioCompleted = true; + mMediaSinkPromise.Complete(); + // Set true only when we have audio. + mAudioCompleted = mInfo.HasAudio(); // Make the best effort to continue playback when there is video. if (HasVideo()) { @@ -3014,31 +3051,41 @@ void MediaDecoderStateMachine::OnAudioSinkError() } void -MediaDecoderStateMachine::OnDecodedStreamFinish() +MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured) { MOZ_ASSERT(OnTaskQueue()); ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - MOZ_ASSERT(mAudioCaptured, "Audio should be captured."); - mDecodedStreamPromise.Complete(); - if (mInfo.HasAudio()) { - mAudioCompleted = true; + if (aCaptured == mAudioCaptured) { + return; } - // To notify PlaybackEnded as soon as possible. + + // Backup current playback parameters. + MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams(); + + // Stop and shut down the existing sink. + StopMediaSink(); + mMediaSink->Shutdown(); + + // Create a new sink according to whether audio is captured. + // TODO: We can't really create a new DecodedStream until OutputStreamManager + // is extracted. It is tricky that the implementation of DecodedStream + // happens to allow reuse after shutdown without creating a new one. + mMediaSink = aCaptured ? mStreamSink : CreateAudioSink(); + + // Restore playback parameters. + mMediaSink->SetPlaybackParams(params); + + // Start the sink if we are already playing. Otherwise it will be + // handled in MaybeStartPlayback(). + if (IsPlaying()) { + StartMediaSink(); + } + + mAudioCaptured = aCaptured; ScheduleStateMachine(); } -void -MediaDecoderStateMachine::OnDecodedStreamError() -{ - MOZ_ASSERT(OnTaskQueue()); - ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor()); - MOZ_ASSERT(mAudioCaptured, "Audio should be captured."); - - mDecodedStreamPromise.Complete(); - DecodeError(); -} - uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { MOZ_ASSERT(OnTaskQueue()); @@ -3048,64 +3095,26 @@ uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const : std::max(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); } -void MediaDecoderStateMachine::DispatchAudioCaptured() -{ - nsRefPtr self = this; - nsCOMPtr r = NS_NewRunnableFunction([self] () -> void - { - MOZ_ASSERT(self->OnTaskQueue()); - ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor()); - if (!self->mAudioCaptured) { - // Stop the audio sink if it's running. - self->StopAudioSink(); - self->mAudioCaptured = true; - // Start DecodedStream if we are already playing. Otherwise it will be - // handled in MaybeStartPlayback(). - if (self->IsPlaying()) { - self->StartDecodedStream(); - } - self->ScheduleStateMachine(); - } - }); - OwnerThread()->Dispatch(r.forget()); -} - -void MediaDecoderStateMachine::DispatchAudioUncaptured() -{ - nsRefPtr self = this; - nsCOMPtr r = NS_NewRunnableFunction([self] () -> void - { - MOZ_ASSERT(self->OnTaskQueue()); - ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor()); - if (self->mAudioCaptured) { - self->StopDecodedStream(); - // Start again the audio sink. - self->mAudioCaptured = false; - if (self->IsPlaying()) { - self->StartAudioSink(); - } - self->ScheduleStateMachine(); - } - }); - OwnerThread()->Dispatch(r.forget()); -} - void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream, bool aFinishWhenEnded) { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("AddOutputStream aStream=%p!", aStream); - mDecodedStream->AddOutput(aStream, aFinishWhenEnded); - DispatchAudioCaptured(); + mStreamSink->AddOutput(aStream, aFinishWhenEnded); + nsCOMPtr r = NS_NewRunnableMethodWithArg( + this, &MediaDecoderStateMachine::SetAudioCaptured, true); + OwnerThread()->Dispatch(r.forget()); } void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream) { MOZ_ASSERT(NS_IsMainThread()); DECODER_LOG("RemoveOutputStream=%p!", aStream); - mDecodedStream->RemoveOutput(aStream); - if (!mDecodedStream->HasConsumers()) { - DispatchAudioUncaptured(); + mStreamSink->RemoveOutput(aStream); + if (!mStreamSink->HasConsumers()) { + nsCOMPtr r = NS_NewRunnableMethodWithArg( + this, &MediaDecoderStateMachine::SetAudioCaptured, false); + OwnerThread()->Dispatch(r.forget()); } } diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index 9cdb159008..c6dda6d0a6 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -93,8 +93,8 @@ hardware (via AudioStream). #include "MediaDecoderOwner.h" #include "MediaEventSource.h" #include "MediaMetadataManager.h" +#include "MediaStatistics.h" #include "MediaTimer.h" -#include "DecodedStream.h" #include "ImageContainer.h" namespace mozilla { @@ -104,6 +104,7 @@ class MediaSink; } class AudioSegment; +class DecodedStream; class TaskQueue; extern PRLogModuleInfo* gMediaDecoderLog; @@ -155,7 +156,7 @@ public: void RemoveOutputStream(MediaStream* aStream); // Set/Unset dormant state. - void SetDormant(bool aDormant); + void DispatchSetDormant(bool aDormant); TimedMetadataEventSource& TimedMetadataEvent() { return mMetadataManager.TimedMetadataEvent(); @@ -167,19 +168,18 @@ private: // constructor immediately after the task queue is created. void InitializationTask(); - void DispatchAudioCaptured(); - void DispatchAudioUncaptured(); + void SetDormant(bool aDormant); + + void SetAudioCaptured(bool aCaptured); + + void NotifyWaitingForResourcesStatusChanged(); + + nsRefPtr Seek(SeekTarget aTarget); void Shutdown(); -public: - void DispatchShutdown() - { - mDecodedStream->Shutdown(); - nsCOMPtr runnable = - NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown); - OwnerThread()->Dispatch(runnable.forget()); - } +public: + void DispatchShutdown(); void FinishShutdown(); @@ -192,8 +192,7 @@ public: bool OnTaskQueue() const; // Seeks to the decoder to aTarget asynchronously. - // Must be called on the state machine thread. - nsRefPtr Seek(SeekTarget aTarget); + nsRefPtr InvokeSeek(SeekTarget aTarget); // Clear the flag indicating that a playback position change event // is currently queued. This is called from the main thread and must @@ -212,8 +211,12 @@ private: // immediately stop playback and buffer downloaded data. Called on // the state machine thread. void StartBuffering(); -public: + bool CanPlayThrough(); + + MediaStatistics GetStatistics(); + +public: void DispatchStartBuffering() { nsCOMPtr runnable = @@ -343,7 +346,7 @@ public: // Called when the reader may have acquired the hardware resources required // to begin decoding. - void NotifyWaitingForResourcesStatusChanged(); + void DispatchWaitingForResourcesStatusChanged(); // Notifies the state machine that should minimize the number of samples // decoded we preroll, until playback starts. The first time playback starts @@ -457,8 +460,6 @@ protected: // parties. void UpdateNextFrameStatus(); - int64_t GetStreamClock() const; - // Return the current time, either the audio clock if available (if the media // has audio, and the playback is possible), or a clock for the video. // Called on the state machine thread. @@ -497,19 +498,17 @@ protected: // state machine thread. void UpdateRenderedVideoFrames(); - // Stops the audio sink and shut it down. + media::MediaSink* CreateAudioSink(); + + // Stops the media sink and shut it down. // The decoder monitor must be held with exactly one lock count. // Called on the state machine thread. - void StopAudioSink(); + void StopMediaSink(); - // Create and start the audio sink. + // Create and start the media sink. // The decoder monitor must be held with exactly one lock count. // Called on the state machine thread. - void StartAudioSink(); - - void StopDecodedStream(); - - void StartDecodedStream(); + void StartMediaSink(); // Notification method invoked when mPlayState changes. void PlayStateChanged(); @@ -656,16 +655,12 @@ protected: void SetPlayStartTime(const TimeStamp& aTimeStamp); private: - // Resolved by the AudioSink to signal that all outstanding work is complete + // Resolved by the MediaSink to signal that all outstanding work is complete // and the sink is shutting down. - void OnAudioSinkComplete(); + void OnMediaSinkComplete(); - // Rejected by the AudioSink to signal errors. - void OnAudioSinkError(); - - void OnDecodedStreamFinish(); - - void OnDecodedStreamError(); + // Rejected by the MediaSink to signal errors. + void OnMediaSinkError(); // Return true if the video decoder's decode speed can not catch up the // play time. @@ -985,15 +980,15 @@ private: // Media Fragment end time in microseconds. Access controlled by decoder monitor. int64_t mFragmentEndTime; - // The audio sink resource. Used on the state machine thread. - nsRefPtr mAudioSink; + // The media sink resource. Used on the state machine thread. + nsRefPtr mMediaSink; // The reader, don't call its methods with the decoder monitor held. // This is created in the state machine's constructor. nsRefPtr mReader; - // The end time of the last audio frame that's been pushed onto the audio sink - // or DecodedStream in microseconds. This will approximately be the end time + // The end time of the last audio frame that's been pushed onto the media sink + // in microseconds. This will approximately be the end time // of the audio stream, unless another frame is pushed to the hardware. int64_t AudioEndTime() const; @@ -1272,13 +1267,12 @@ private: // Only written on the main thread while holding the monitor. Therefore it // can be read on any thread while holding the monitor, or on the main thread // without holding the monitor. - nsRefPtr mDecodedStream; + nsRefPtr mStreamSink; // Media data resource from the decoder. nsRefPtr mResource; - MozPromiseRequestHolder mAudioSinkPromise; - MozPromiseRequestHolder mDecodedStreamPromise; + MozPromiseRequestHolder mMediaSinkPromise; MediaEventListener mAudioQueueListener; MediaEventListener mVideoQueueListener; @@ -1313,6 +1307,18 @@ private: // passed to MediaStreams when this is true. Mirror mSameOriginMedia; + // Estimate of the current playback rate (bytes/second). + Mirror mPlaybackBytesPerSecond; + + // True if mPlaybackBytesPerSecond is a reliable estimate. + Mirror mPlaybackRateReliable; + + // Current decoding position in the stream. + Mirror mDecoderPosition; + + // True if the media is seekable (i.e. supports random access). + Mirror mMediaSeekable; + // Duration of the media. This is guaranteed to be non-null after we finish // decoding the first frame. Canonical mDuration; @@ -1329,6 +1335,9 @@ private: // playback position. Canonical mCurrentPosition; + // Current playback position in the stream in bytes. + Canonical mPlaybackOffset; + public: AbstractCanonical* CanonicalBuffered() { return mReader->CanonicalBuffered(); @@ -1345,6 +1354,9 @@ public: AbstractCanonical* CanonicalCurrentPosition() { return &mCurrentPosition; } + AbstractCanonical* CanonicalPlaybackOffset() { + return &mPlaybackOffset; + } }; } // namespace mozilla diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 3d1a9a59b7..fa54c0cfae 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -508,8 +508,7 @@ MediaFormatReader::ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThr nsRefPtr MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) + int64_t aTimeThreshold) { MOZ_ASSERT(OnTaskQueue()); MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking"); @@ -535,7 +534,6 @@ MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe, return VideoDataPromise::CreateAndReject(CANCELED, __func__); } - mVideo.mForceDecodeAhead = aForceDecodeAhead; media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)}; if (ShouldSkip(aSkipToNextKeyframe, timeThreshold)) { Flush(TrackInfo::kVideoTrack); @@ -735,12 +733,11 @@ MediaFormatReader::NeedInput(DecoderData& aDecoder) return !aDecoder.mDraining && !aDecoder.mError && - (aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) && + aDecoder.HasPromise() && !aDecoder.mDemuxRequest.Exists() && aDecoder.mOutput.IsEmpty() && (aDecoder.mInputExhausted || !aDecoder.mQueuedSamples.IsEmpty() || aDecoder.mTimeThreshold.isSome() || - aDecoder.mForceDecodeAhead || aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead); } diff --git a/dom/media/MediaFormatReader.h b/dom/media/MediaFormatReader.h index ce5cf7093c..d7eaf90a94 100644 --- a/dom/media/MediaFormatReader.h +++ b/dom/media/MediaFormatReader.h @@ -35,7 +35,7 @@ public: size_t SizeOfAudioQueueInFrames() override; nsRefPtr - RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override; + RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override; nsRefPtr RequestAudioData() override; @@ -187,7 +187,6 @@ private: : mOwner(aOwner) , mType(aType) , mDecodeAhead(aDecodeAhead) - , mForceDecodeAhead(false) , mUpdateScheduled(false) , mDemuxEOS(false) , mWaitingForData(false) @@ -221,7 +220,6 @@ private: // Only accessed from reader's task queue. uint32_t mDecodeAhead; - bool mForceDecodeAhead; bool mUpdateScheduled; bool mDemuxEOS; bool mWaitingForData; @@ -276,7 +274,6 @@ private: void ResetState() { MOZ_ASSERT(mOwner->OnTaskQueue()); - mForceDecodeAhead = false; mDemuxEOS = false; mWaitingForData = false; mReceivedNewData = false; diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index e9bfeeecdb..f4aaf7be41 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -1382,6 +1382,8 @@ MediaManager::IsInMediaThread() // NOTE: never Dispatch(....,NS_DISPATCH_SYNC) to the MediaManager // thread from the MainThread, as we NS_DISPATCH_SYNC to MainThread // from MediaManager thread. + +// Guaranteed never to return nullptr. /* static */ MediaManager* MediaManager::Get() { if (!sSingleton) { @@ -2038,7 +2040,14 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow, nsCOMPtr onFailure(aOnFailure); uint64_t windowId = aWindow->WindowID(); - AddWindowID(windowId); + StreamListeners* listeners = AddWindowID(windowId); + + // Create a disabled listener to act as a placeholder + nsRefPtr listener = + new GetUserMediaCallbackMediaStreamListener(mMediaThread, windowId); + + // No need for locking because we always do this in the main thread. + listeners->AppendElement(listener); bool fake = Preferences::GetBool("media.navigator.streams.fake"); @@ -2046,11 +2055,15 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow, dom::MediaSourceEnum::Camera, dom::MediaSourceEnum::Microphone, fake); - p->Then([onSuccess](SourceSet*& aDevices) mutable { + p->Then([onSuccess, windowId, listener](SourceSet*& aDevices) mutable { ScopedDeletePtr devices(aDevices); // grab result + nsRefPtr mgr = MediaManager_GetInstance(); + mgr->RemoveFromWindowList(windowId, listener); nsCOMPtr array = MediaManager_ToJSArray(*devices); onSuccess->OnSuccess(array); - }, [onFailure](MediaStreamError& reason) mutable { + }, [onFailure, windowId, listener](MediaStreamError& reason) mutable { + nsRefPtr mgr = MediaManager_GetInstance(); + mgr->RemoveFromWindowList(windowId, listener); onFailure->OnError(&reason); }); return NS_OK; diff --git a/dom/media/MediaStatistics.h b/dom/media/MediaStatistics.h new file mode 100644 index 0000000000..c67c7e2bde --- /dev/null +++ b/dom/media/MediaStatistics.h @@ -0,0 +1,82 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MediaStatistics_h_ +#define MediaStatistics_h_ + +namespace mozilla { + +struct MediaStatistics { + // Estimate of the current playback rate (bytes/second). + double mPlaybackRate; + // Estimate of the current download rate (bytes/second). This + // ignores time that the channel was paused by Gecko. + double mDownloadRate; + // Total length of media stream in bytes; -1 if not known + int64_t mTotalBytes; + // Current position of the download, in bytes. This is the offset of + // the first uncached byte after the decoder position. + int64_t mDownloadPosition; + // Current position of decoding, in bytes (how much of the stream + // has been consumed) + int64_t mDecoderPosition; + // Current position of playback, in bytes + int64_t mPlaybackPosition; + // If false, then mDownloadRate cannot be considered a reliable + // estimate (probably because the download has only been running + // a short time). + bool mDownloadRateReliable; + // If false, then mPlaybackRate cannot be considered a reliable + // estimate (probably because playback has only been running + // a short time). + bool mPlaybackRateReliable; + + bool CanPlayThrough() + { + // Number of estimated seconds worth of data we need to have buffered + // ahead of the current playback position before we allow the media decoder + // to report that it can play through the entire media without the decode + // catching up with the download. Having this margin make the + // CanPlayThrough() calculation more stable in the case of + // fluctuating bitrates. + static const int64_t CAN_PLAY_THROUGH_MARGIN = 1; + + if ((mTotalBytes < 0 && mDownloadRateReliable) || + (mTotalBytes >= 0 && mTotalBytes == mDownloadPosition)) { + return true; + } + + if (!mDownloadRateReliable || !mPlaybackRateReliable) { + return false; + } + + int64_t bytesToDownload = mTotalBytes - mDownloadPosition; + int64_t bytesToPlayback = mTotalBytes - mPlaybackPosition; + double timeToDownload = bytesToDownload / mDownloadRate; + double timeToPlay = bytesToPlayback / mPlaybackRate; + + if (timeToDownload > timeToPlay) { + // Estimated time to download is greater than the estimated time to play. + // We probably can't play through without having to stop to buffer. + return false; + } + + // Estimated time to download is less than the estimated time to play. + // We can probably play through without having to buffer, but ensure that + // we've got a reasonable amount of data buffered after the current + // playback position, so that if the bitrate of the media fluctuates, or if + // our download rate or decode rate estimation is otherwise inaccurate, + // we don't suddenly discover that we need to buffer. This is particularly + // required near the start of the media, when not much data is downloaded. + int64_t readAheadMargin = + static_cast(mPlaybackRate * CAN_PLAY_THROUGH_MARGIN); + return mDownloadPosition > mPlaybackPosition + readAheadMargin; + } +}; + +} // namespace mozilla + +#endif // MediaStatistics_h_ diff --git a/dom/media/encoder/OmxTrackEncoder.cpp b/dom/media/encoder/OmxTrackEncoder.cpp index 2203dd0c24..f78a982662 100644 --- a/dom/media/encoder/OmxTrackEncoder.cpp +++ b/dom/media/encoder/OmxTrackEncoder.cpp @@ -134,11 +134,8 @@ OmxVideoTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) layers::Image* img = (!mLastFrame.GetImage() || mLastFrame.GetForceBlack()) ? nullptr : mLastFrame.GetImage(); rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs, - OMXCodecWrapper::BUFFER_EOS); + OMXCodecWrapper::BUFFER_EOS, &mEosSetInEncoder); NS_ENSURE_SUCCESS(rv, rv); - - // Keep sending EOS signal until OMXVideoEncoder gets it. - mEosSetInEncoder = true; } // Dequeue an encoded frame from the output buffers of OMXCodecWrapper. @@ -219,6 +216,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) PROFILER_LABEL("OmxAACAudioTrackEncoder", "GetEncodedTrack", js::ProfileEntry::Category::OTHER); AudioSegment segment; + bool EOS; // Move all the samples from mRawSegment to segment. We only hold // the monitor in this block. { @@ -234,14 +232,15 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) } segment.AppendFrom(&mRawSegment); + EOS = mEndOfStream; } nsresult rv; if (segment.GetDuration() == 0) { // Notify EOS at least once, even if segment is empty. - if (mEndOfStream && !mEosSetInEncoder) { - mEosSetInEncoder = true; - rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS); + if (EOS && !mEosSetInEncoder) { + rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS, + &mEosSetInEncoder); NS_ENSURE_SUCCESS(rv, rv); } // Nothing to encode but encoder could still have encoded data for earlier @@ -252,8 +251,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) // OMX encoder has limited input buffers only so we have to feed input and get // output more than once if there are too many samples pending in segment. while (segment.GetDuration() > 0) { - rv = mEncoder->Encode(segment, - mEndOfStream ? OMXCodecWrapper::BUFFER_EOS : 0); + rv = mEncoder->Encode(segment, 0); NS_ENSURE_SUCCESS(rv, rv); rv = AppendEncodedFrames(aData); diff --git a/dom/media/encoder/VP8TrackEncoder.cpp b/dom/media/encoder/VP8TrackEncoder.cpp index fd1c525ce8..75120524c5 100644 --- a/dom/media/encoder/VP8TrackEncoder.cpp +++ b/dom/media/encoder/VP8TrackEncoder.cpp @@ -448,6 +448,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) { PROFILER_LABEL("VP8TrackEncoder", "GetEncodedTrack", js::ProfileEntry::Category::OTHER); + bool EOS; { // Move all the samples from mRawSegment to mSourceSegment. We only hold // the monitor in this block. @@ -463,6 +464,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) return NS_ERROR_FAILURE; } mSourceSegment.AppendFrom(&mRawSegment); + EOS = mEndOfStream; } VideoSegment::ChunkIterator iter(mSourceSegment); @@ -536,7 +538,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) VP8LOG("RemoveLeading %lld\n",totalProcessedDuration); // End of stream, pull the rest frames in encoder. - if (mEndOfStream) { + if (EOS) { VP8LOG("mEndOfStream is true\n"); mEncodingComplete = true; if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp, diff --git a/dom/media/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp similarity index 93% rename from dom/media/DecodedStream.cpp rename to dom/media/mediasink/DecodedStream.cpp index 3da84c2ec2..c2d34ffa1d 100644 --- a/dom/media/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -4,6 +4,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "mozilla/CheckedInt.h" +#include "mozilla/gfx/Point.h" + #include "AudioSegment.h" #include "DecodedStream.h" #include "MediaData.h" @@ -358,7 +361,6 @@ DecodedStream::DecodedStream(AbstractThread* aOwnerThread, : mOwnerThread(aOwnerThread) , mShuttingDown(false) , mPlaying(false) - , mVolume(1.0) , mSameOrigin(false) , mAudioQueue(aAudioQueue) , mVideoQueue(aVideoQueue) @@ -370,21 +372,52 @@ DecodedStream::~DecodedStream() MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended."); } +const media::MediaSink::PlaybackParams& +DecodedStream::GetPlaybackParams() const +{ + AssertOwnerThread(); + return mParams; +} + void -DecodedStream::Shutdown() +DecodedStream::SetPlaybackParams(const PlaybackParams& aParams) +{ + AssertOwnerThread(); + mParams = aParams; +} + +nsRefPtr +DecodedStream::OnEnded(TrackType aType) +{ + AssertOwnerThread(); + MOZ_ASSERT(mStartTime.isSome()); + + if (aType == TrackInfo::kAudioTrack) { + // TODO: we should return a promise which is resolved when the audio track + // is finished. For now this promise is resolved when the whole stream is + // finished. + return mFinishPromise; + } + // TODO: handle video track. + return nullptr; +} + +void +DecodedStream::BeginShutdown() { MOZ_ASSERT(NS_IsMainThread()); mShuttingDown = true; } -nsRefPtr -DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo) +void +DecodedStream::Start(int64_t aStartTime, const MediaInfo& aInfo) { AssertOwnerThread(); MOZ_ASSERT(mStartTime.isNothing(), "playback already started."); mStartTime.emplace(aStartTime); mInfo = aInfo; + mPlaying = true; ConnectListener(); class R : public nsRunnable { @@ -408,30 +441,33 @@ DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo) }; MozPromiseHolder promise; - nsRefPtr rv = promise.Ensure(__func__); + mFinishPromise = promise.Ensure(__func__); nsCOMPtr r = new R(this, &DecodedStream::CreateData, Move(promise)); AbstractThread::MainThread()->Dispatch(r.forget()); - - return rv.forget(); } -void DecodedStream::StopPlayback() +void +DecodedStream::Stop() { AssertOwnerThread(); - - // Playback didn't even start at all. - if (mStartTime.isNothing()) { - return; - } + MOZ_ASSERT(mStartTime.isSome(), "playback not started."); mStartTime.reset(); DisconnectListener(); + mFinishPromise = nullptr; // Clear mData immediately when this playback session ends so we won't // send data to the wrong stream in SendData() in next playback session. DestroyData(Move(mData)); } +bool +DecodedStream::IsStarted() const +{ + AssertOwnerThread(); + return mStartTime.isSome(); +} + void DecodedStream::DestroyData(UniquePtr aData) { @@ -531,6 +567,12 @@ void DecodedStream::SetPlaying(bool aPlaying) { AssertOwnerThread(); + + // Resume/pause matters only when playback started. + if (mStartTime.isNothing()) { + return; + } + mPlaying = aPlaying; if (mData) { mData->SetPlaying(aPlaying); @@ -541,7 +583,21 @@ void DecodedStream::SetVolume(double aVolume) { AssertOwnerThread(); - mVolume = aVolume; + mParams.volume = aVolume; +} + +void +DecodedStream::SetPlaybackRate(double aPlaybackRate) +{ + AssertOwnerThread(); + mParams.playbackRate = aPlaybackRate; +} + +void +DecodedStream::SetPreservesPitch(bool aPreservesPitch) +{ + AssertOwnerThread(); + mParams.preservesPitch = aPreservesPitch; } void @@ -815,7 +871,7 @@ DecodedStream::SendData() } InitTracks(); - SendAudio(mVolume, mSameOrigin); + SendAudio(mParams.volume, mSameOrigin); SendVideo(mSameOrigin); AdvanceTracks(); @@ -829,26 +885,31 @@ DecodedStream::SendData() } int64_t -DecodedStream::AudioEndTime() const +DecodedStream::GetEndTime(TrackType aType) const { AssertOwnerThread(); - if (mStartTime.isSome() && mInfo.HasAudio() && mData) { + if (aType == TrackInfo::kAudioTrack && mInfo.HasAudio() && mData) { CheckedInt64 t = mStartTime.ref() + FramesToUsecs(mData->mAudioFramesWritten, mInfo.mAudio.mRate); if (t.isValid()) { return t.value(); } + } else if (aType == TrackInfo::kVideoTrack && mData) { + return mData->mNextVideoTime; } return -1; } int64_t -DecodedStream::GetPosition() const +DecodedStream::GetPosition(TimeStamp* aTimeStamp) const { AssertOwnerThread(); // This is only called after MDSM starts playback. So mStartTime is // guaranteed to be something. MOZ_ASSERT(mStartTime.isSome()); + if (aTimeStamp) { + *aTimeStamp = TimeStamp::Now(); + } return mStartTime.ref() + (mData ? mData->GetPosition() : 0); } diff --git a/dom/media/DecodedStream.h b/dom/media/mediasink/DecodedStream.h similarity index 80% rename from dom/media/DecodedStream.h rename to dom/media/mediasink/DecodedStream.h index 463bd4ae2f..5b55d68517 100644 --- a/dom/media/DecodedStream.h +++ b/dom/media/mediasink/DecodedStream.h @@ -10,15 +10,13 @@ #include "nsTArray.h" #include "MediaEventSource.h" #include "MediaInfo.h" +#include "MediaSink.h" #include "mozilla/AbstractThread.h" -#include "mozilla/CheckedInt.h" #include "mozilla/Maybe.h" #include "mozilla/MozPromise.h" #include "mozilla/nsRefPtr.h" -#include "mozilla/ReentrantMonitor.h" #include "mozilla/UniquePtr.h" -#include "mozilla/gfx/Point.h" namespace mozilla { @@ -31,7 +29,7 @@ class MediaStreamGraph; class OutputStreamListener; class OutputStreamManager; class ProcessedMediaStream; -class ReentrantMonitor; +class TimeStamp; template class MediaQueue; @@ -99,34 +97,41 @@ private: nsTArray mStreams; }; -class DecodedStream { - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStream); +class DecodedStream : public media::MediaSink { + using media::MediaSink::PlaybackParams; + public: DecodedStream(AbstractThread* aOwnerThread, MediaQueue& aAudioQueue, MediaQueue& aVideoQueue); - void Shutdown(); + // MediaSink functions. + const PlaybackParams& GetPlaybackParams() const override; + void SetPlaybackParams(const PlaybackParams& aParams) override; - // Mimic MDSM::StartAudioThread. - // Must be called before any calls to SendData(). - // - // Return a promise which will be resolved when the stream is finished - // or rejected if any error. - nsRefPtr StartPlayback(int64_t aStartTime, - const MediaInfo& aInfo); - // Mimic MDSM::StopAudioThread. - void StopPlayback(); + nsRefPtr OnEnded(TrackType aType) override; + int64_t GetEndTime(TrackType aType) const override; + int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override; + bool HasUnplayedFrames(TrackType aType) const override + { + // TODO: implement this. + return false; + } + void SetVolume(double aVolume) override; + void SetPlaybackRate(double aPlaybackRate) override; + void SetPreservesPitch(bool aPreservesPitch) override; + void SetPlaying(bool aPlaying) override; + + void Start(int64_t aStartTime, const MediaInfo& aInfo) override; + void Stop() override; + bool IsStarted() const override; + + // TODO: fix these functions that don't fit into the interface of MediaSink. + void BeginShutdown(); void AddOutput(ProcessedMediaStream* aStream, bool aFinishWhenEnded); void RemoveOutput(MediaStream* aStream); - - void SetPlaying(bool aPlaying); - void SetVolume(double aVolume); void SetSameOrigin(bool aSameOrigin); - - int64_t AudioEndTime() const; - int64_t GetPosition() const; bool IsFinished() const; bool HasConsumers() const; @@ -164,10 +169,11 @@ private: * Worker thread only members. */ UniquePtr mData; + nsRefPtr mFinishPromise; bool mPlaying; - double mVolume; bool mSameOrigin; + PlaybackParams mParams; Maybe mStartTime; MediaInfo mInfo; diff --git a/dom/media/mediasink/moz.build b/dom/media/mediasink/moz.build index 2f3c5de119..931bf0fd59 100644 --- a/dom/media/mediasink/moz.build +++ b/dom/media/mediasink/moz.build @@ -7,8 +7,7 @@ UNIFIED_SOURCES += [ 'AudioSinkWrapper.cpp', 'DecodedAudioDataSink.cpp', + 'DecodedStream.cpp', ] FINAL_LIBRARY = 'xul' - -FAIL_ON_WARNINGS = True diff --git a/dom/media/moz.build b/dom/media/moz.build index 7c5c8c0a03..01ddb4aa2f 100644 --- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -102,7 +102,6 @@ EXPORTS += [ 'AudioStream.h', 'BufferMediaResource.h', 'CubebUtils.h', - 'DecodedStream.h', 'DecoderTraits.h', 'DOMMediaStream.h', 'EncodedBufferCache.h', @@ -125,6 +124,7 @@ EXPORTS += [ 'MediaRecorder.h', 'MediaResource.h', 'MediaSegment.h', + 'MediaStatistics.h', 'MediaStreamGraph.h', 'MediaTimer.h', 'MediaTrack.h', @@ -200,7 +200,6 @@ UNIFIED_SOURCES += [ 'AudioTrackList.cpp', 'CanvasCaptureMediaStream.cpp', 'CubebUtils.cpp', - 'DecodedStream.cpp', 'DOMMediaStream.cpp', 'EncodedBufferCache.cpp', 'FileBlockCache.cpp', diff --git a/dom/media/omx/MediaCodecReader.cpp b/dom/media/omx/MediaCodecReader.cpp index 4e2ef947cc..aee16b3eb6 100644 --- a/dom/media/omx/MediaCodecReader.cpp +++ b/dom/media/omx/MediaCodecReader.cpp @@ -353,8 +353,7 @@ MediaCodecReader::RequestAudioData() nsRefPtr MediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) + int64_t aTimeThreshold) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasVideo()); diff --git a/dom/media/omx/MediaCodecReader.h b/dom/media/omx/MediaCodecReader.h index 30538ade92..7ceea2c477 100644 --- a/dom/media/omx/MediaCodecReader.h +++ b/dom/media/omx/MediaCodecReader.h @@ -83,8 +83,7 @@ public: // Disptach a DecodeVideoFrameTask to decode video data. virtual nsRefPtr RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) override; + int64_t aTimeThreshold) override; // Disptach a DecodeAduioDataTask to decode video data. virtual nsRefPtr RequestAudioData() override; diff --git a/dom/media/omx/MediaOmxCommonDecoder.cpp b/dom/media/omx/MediaOmxCommonDecoder.cpp index 1eaaff9e39..8914f88b83 100644 --- a/dom/media/omx/MediaOmxCommonDecoder.cpp +++ b/dom/media/omx/MediaOmxCommonDecoder.cpp @@ -117,12 +117,7 @@ MediaOmxCommonDecoder::PauseStateMachine() return; } // enter dormant state - RefPtr event = - NS_NewRunnableMethodWithArg( - GetStateMachine(), - &MediaDecoderStateMachine::SetDormant, - true); - GetStateMachine()->OwnerThread()->Dispatch(event.forget()); + GetStateMachine()->DispatchSetDormant(true); } void @@ -146,22 +141,12 @@ MediaOmxCommonDecoder::ResumeStateMachine() SeekTarget::Accurate, MediaDecoderEventVisibility::Suppressed); // Call Seek of MediaDecoderStateMachine to suppress seek events. - RefPtr event = - NS_NewRunnableMethodWithArg( - GetStateMachine(), - &MediaDecoderStateMachine::Seek, - target); - GetStateMachine()->OwnerThread()->Dispatch(event.forget()); + GetStateMachine()->InvokeSeek(target); mNextState = mPlayState; ChangeState(PLAY_STATE_LOADING); // exit dormant state - event = - NS_NewRunnableMethodWithArg( - GetStateMachine(), - &MediaDecoderStateMachine::SetDormant, - false); - GetStateMachine()->OwnerThread()->Dispatch(event.forget()); + GetStateMachine()->DispatchSetDormant(false); UpdateLogicalPosition(); } diff --git a/dom/media/omx/OMXCodecWrapper.cpp b/dom/media/omx/OMXCodecWrapper.cpp index c19025b7a5..0ea79c928c 100644 --- a/dom/media/omx/OMXCodecWrapper.cpp +++ b/dom/media/omx/OMXCodecWrapper.cpp @@ -382,7 +382,7 @@ ConvertGrallocImageToNV12(GrallocImage* aSource, uint8_t* aDestination) nsresult OMXVideoEncoder::Encode(const Image* aImage, int aWidth, int aHeight, - int64_t aTimestamp, int aInputFlags) + int64_t aTimestamp, int aInputFlags, bool* aSendEOS) { MOZ_ASSERT(mStarted, "Configure() should be called before Encode()."); @@ -456,6 +456,9 @@ OMXVideoEncoder::Encode(const Image* aImage, int aWidth, int aHeight, // Queue this input buffer. result = mCodec->queueInputBuffer(index, 0, dstSize, aTimestamp, aInputFlags); + if (aSendEOS && (aInputFlags & BUFFER_EOS) && result == OK) { + *aSendEOS = true; + } return result == OK ? NS_OK : NS_ERROR_FAILURE; } @@ -822,7 +825,8 @@ OMXAudioEncoder::~OMXAudioEncoder() } nsresult -OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags) +OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags, + bool* aSendEOS) { #ifndef MOZ_SAMPLE_TYPE_S16 #error MediaCodec accepts only 16-bit PCM data. @@ -877,7 +881,9 @@ OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags) } result = buffer.Enqueue(mTimestamp, flags); NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE); - + if (aSendEOS && (aInputFlags & BUFFER_EOS)) { + *aSendEOS = true; + } return NS_OK; } diff --git a/dom/media/omx/OMXCodecWrapper.h b/dom/media/omx/OMXCodecWrapper.h index 4e087c20bc..23f9fc66f6 100644 --- a/dom/media/omx/OMXCodecWrapper.h +++ b/dom/media/omx/OMXCodecWrapper.h @@ -224,8 +224,13 @@ public: * stream, set aInputFlags to BUFFER_EOS. Since encoder has limited buffers, * this function might not be able to encode all chunks in one call, however * it will remove chunks it consumes from aSegment. + * aSendEOS is the output to tell the caller EOS signal sent into MediaCodec + * because the signal might not be sent due to the dequeueInputBuffer timeout. + * And the value of aSendEOS won't be set to any default value, only set to + * true when EOS signal sent into MediaCodec. */ - nsresult Encode(mozilla::AudioSegment& aSegment, int aInputFlags = 0); + nsresult Encode(mozilla::AudioSegment& aSegment, int aInputFlags = 0, + bool* aSendEOS = nullptr); ~OMXAudioEncoder(); protected: @@ -299,9 +304,14 @@ public: * semi-planar YUV420 format stored in the buffer of aImage. aTimestamp gives * the frame timestamp/presentation time (in microseconds). To notify end of * stream, set aInputFlags to BUFFER_EOS. + * aSendEOS is the output to tell the caller EOS signal sent into MediaCodec + * because the signal might not be sent due to the dequeueInputBuffer timeout. + * And the value of aSendEOS won't be set to any default value, only set to + * true when EOS signal sent into MediaCodec. */ nsresult Encode(const mozilla::layers::Image* aImage, int aWidth, int aHeight, - int64_t aTimestamp, int aInputFlags = 0); + int64_t aTimestamp, int aInputFlags = 0, + bool* aSendEOS = nullptr); #if ANDROID_VERSION >= 18 /** Set encoding bitrate (in kbps). */ diff --git a/dom/media/omx/RtspMediaCodecReader.cpp b/dom/media/omx/RtspMediaCodecReader.cpp index 7f2abad734..35dd4844fe 100644 --- a/dom/media/omx/RtspMediaCodecReader.cpp +++ b/dom/media/omx/RtspMediaCodecReader.cpp @@ -82,13 +82,10 @@ RtspMediaCodecReader::RequestAudioData() nsRefPtr RtspMediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) + int64_t aTimeThreshold) { EnsureActive(); - return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe, - aTimeThreshold, - aForceDecodeAhead); + return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe, aTimeThreshold); } nsRefPtr diff --git a/dom/media/omx/RtspMediaCodecReader.h b/dom/media/omx/RtspMediaCodecReader.h index 86fe3ea4ea..3bbf3ca162 100644 --- a/dom/media/omx/RtspMediaCodecReader.h +++ b/dom/media/omx/RtspMediaCodecReader.h @@ -53,8 +53,7 @@ public: // Disptach a DecodeVideoFrameTask to decode video data. virtual nsRefPtr RequestVideoData(bool aSkipToNextKeyframe, - int64_t aTimeThreshold, - bool aForceDecodeAhead) override; + int64_t aTimeThreshold) override; // Disptach a DecodeAudioDataTask to decode audio data. virtual nsRefPtr RequestAudioData() override; diff --git a/dom/media/systemservices/CamerasChild.cpp b/dom/media/systemservices/CamerasChild.cpp new file mode 100644 index 0000000000..d80126db00 --- /dev/null +++ b/dom/media/systemservices/CamerasChild.cpp @@ -0,0 +1,741 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CamerasChild.h" +#include "CamerasUtils.h" + +#include "webrtc/video_engine/include/vie_capture.h" +#undef FF + +#include "mozilla/Assertions.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/Logging.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/WeakPtr.h" +#include "mozilla/unused.h" +#include "MediaUtils.h" +#include "nsThreadUtils.h" + +#undef LOG +#undef LOG_ENABLED +PRLogModuleInfo *gCamerasChildLog; +#define LOG(args) MOZ_LOG(gCamerasChildLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasChildLog, mozilla::LogLevel::Debug) + +namespace mozilla { +namespace camera { + +// We emulate the sync webrtc.org API with the help of singleton +// CamerasSingleton, which manages a pointer to an IPC object, a thread +// where IPC operations should run on, and a mutex. +// The static function Cameras() will use that Singleton to set up, +// if needed, both the thread and the associated IPC objects and return +// a pointer to the IPC object. Users can then do IPC calls on that object +// after dispatching them to aforementioned thread. + +// 2 Threads are involved in this code: +// - the MediaManager thread, which will call the (static, sync API) functions +// through MediaEngineRemoteVideoSource +// - the Cameras IPC thread, which will be doing our IPC to the parent process +// via PBackground + +// Our main complication is that we emulate a sync API while (having to do) +// async messaging. We dispatch the messages to another thread to send them +// async and hold a Monitor to wait for the result to be asynchronously received +// again. The requirement for async messaging originates on the parent side: +// it's not reasonable to block all PBackground IPC there while waiting for +// something like device enumeration to complete. + +class CamerasSingleton { +public: + CamerasSingleton() + : mCamerasMutex("CamerasSingleton::mCamerasMutex"), + mCameras(nullptr), + mCamerasChildThread(nullptr) { + if (!gCamerasChildLog) { + gCamerasChildLog = PR_NewLogModule("CamerasChild"); + } + LOG(("CamerasSingleton: %p", this)); + } + + ~CamerasSingleton() { + LOG(("~CamerasSingleton: %p", this)); + } + + static CamerasSingleton& GetInstance() { + static CamerasSingleton instance; + return instance; + } + + static OffTheBooksMutex& Mutex() { + return GetInstance().mCamerasMutex; + } + + static CamerasChild*& Child() { + GetInstance().Mutex().AssertCurrentThreadOwns(); + return GetInstance().mCameras; + } + + static nsCOMPtr& Thread() { + GetInstance().Mutex().AssertCurrentThreadOwns(); + return GetInstance().mCamerasChildThread; + } + +private: + // Reinitializing CamerasChild will change the pointers below. + // We don't want this to happen in the middle of preparing IPC. + // We will be alive on destruction, so this needs to be off the books. + mozilla::OffTheBooksMutex mCamerasMutex; + + // This is owned by the IPC code, and the same code controls the lifetime. + // It will set and clear this pointer as appropriate in setup/teardown. + // We'd normally make this a WeakPtr but unfortunately the IPC code already + // uses the WeakPtr mixin in a protected base class of CamerasChild, and in + // any case the object becomes unusable as soon as IPC is tearing down, which + // will be before actual destruction. + CamerasChild* mCameras; + nsCOMPtr mCamerasChildThread; +}; + +class InitializeIPCThread : public nsRunnable +{ +public: + InitializeIPCThread() + : mCamerasChild(nullptr) {} + + NS_IMETHOD Run() override { + // Try to get the PBackground handle + ipc::PBackgroundChild* existingBackgroundChild = + ipc::BackgroundChild::GetForCurrentThread(); + // If it's not spun up yet, block until it is, and retry + if (!existingBackgroundChild) { + LOG(("No existingBackgroundChild")); + SynchronouslyCreatePBackground(); + existingBackgroundChild = + ipc::BackgroundChild::GetForCurrentThread(); + LOG(("BackgroundChild: %p", existingBackgroundChild)); + } + // By now PBackground is guaranteed to be up + MOZ_RELEASE_ASSERT(existingBackgroundChild); + + // Create CamerasChild + // We will be returning the resulting pointer (synchronously) to our caller. + mCamerasChild = + static_cast(existingBackgroundChild->SendPCamerasConstructor()); + + return NS_OK; + } + + CamerasChild* GetCamerasChild() { + MOZ_ASSERT(mCamerasChild); + return mCamerasChild; + } + +private: + CamerasChild* mCamerasChild; +}; + +static CamerasChild* +Cameras() { + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + if (!CamerasSingleton::Child()) { + MOZ_ASSERT(!NS_IsMainThread(), "Should not be on the main Thread"); + if (!gCamerasChildLog) { + gCamerasChildLog = PR_NewLogModule("CamerasChild"); + } + + MOZ_ASSERT(!CamerasSingleton::Thread()); + LOG(("No sCameras, setting up IPC Thread")); + nsresult rv = NS_NewNamedThread("Cameras IPC", + getter_AddRefs(CamerasSingleton::Thread())); + if (NS_FAILED(rv)) { + LOG(("Error launching IPC Thread")); + return nullptr; + } + + // At this point we are in the MediaManager thread, and the thread we are + // dispatching to is the specific Cameras IPC thread that was just made + // above, so now we will fire off a runnable to run + // SynchronouslyCreatePBackground there, while we block in this thread. + // We block until the following happens in the Cameras IPC thread: + // 1) Creation of PBackground finishes + // 2) Creation of PCameras finishes by sending a message to the parent + nsRefPtr runnable = new InitializeIPCThread(); + nsRefPtr sr = new SyncRunnable(runnable); + sr->DispatchToThread(CamerasSingleton::Thread()); + CamerasSingleton::Child() = runnable->GetCamerasChild(); + } + MOZ_ASSERT(CamerasSingleton::Child()); + return CamerasSingleton::Child(); +} + +bool +CamerasChild::RecvReplyFailure(void) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = false; + monitor.Notify(); + return true; +} + +bool +CamerasChild::RecvReplySuccess(void) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + monitor.Notify(); + return true; +} + +int NumberOfCapabilities(CaptureEngine aCapEngine, const char* deviceUniqueIdUTF8) +{ + return Cameras()->NumberOfCapabilities(aCapEngine, deviceUniqueIdUTF8); +} + +bool +CamerasChild::RecvReplyNumberOfCapabilities(const int& numdev) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = numdev; + monitor.Notify(); + return true; +} + +bool +CamerasChild::DispatchToParent(nsIRunnable* aRunnable, + MonitorAutoLock& aMonitor) +{ + { + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + CamerasSingleton::Thread()->Dispatch(aRunnable, NS_DISPATCH_NORMAL); + } + // We can't see if the send worked, so we need to be able to bail + // out on shutdown (when it failed and we won't get a reply). + if (!mIPCIsAlive) { + return false; + } + // Guard against spurious wakeups. + mReceivedReply = false; + // Wait for a reply + do { + aMonitor.Wait(); + } while (!mReceivedReply && mIPCIsAlive); + if (!mReplySuccess) { + return false; + } + return true; +} + +int +CamerasChild::NumberOfCapabilities(CaptureEngine aCapEngine, + const char* deviceUniqueIdUTF8) +{ + // Prevents multiple outstanding requests from happening. + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + LOG(("NumberOfCapabilities for %s", deviceUniqueIdUTF8)); + nsCString unique_id(deviceUniqueIdUTF8); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, unique_id]() -> nsresult { + if (this->SendNumberOfCapabilities(aCapEngine, unique_id)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + // Prevent concurrent use of the reply variables. Note + // that this is unlocked while waiting for the reply to be + // filled in, necessitating the first Mutex above. + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + LOG(("Get capture capability count failed")); + return 0; + } + LOG(("Capture capability count: %d", mReplyInteger)); + return mReplyInteger; +} + +int NumberOfCaptureDevices(CaptureEngine aCapEngine) +{ + return Cameras()->NumberOfCaptureDevices(aCapEngine); +} + +int +CamerasChild::NumberOfCaptureDevices(CaptureEngine aCapEngine) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine]() -> nsresult { + if (this->SendNumberOfCaptureDevices(aCapEngine)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + LOG(("Get NumberOfCaptureDevices failed")); + return 0; + } + LOG(("Capture Devices: %d", mReplyInteger)); + return mReplyInteger; +} + +bool +CamerasChild::RecvReplyNumberOfCaptureDevices(const int& numdev) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = numdev; + monitor.Notify(); + return true; +} + +int GetCaptureCapability(CaptureEngine aCapEngine, const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::CaptureCapability& capability) +{ + return Cameras()->GetCaptureCapability(aCapEngine, + unique_idUTF8, + capability_number, + capability); +} + +int +CamerasChild::GetCaptureCapability(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::CaptureCapability& capability) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG(("GetCaptureCapability: %s %d", unique_idUTF8, capability_number)); + nsCString unique_id(unique_idUTF8); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, unique_id, capability_number]() -> nsresult { + if (this->SendGetCaptureCapability(aCapEngine, unique_id, capability_number)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + return -1; + } + capability = mReplyCapability; + return 0; +} + +bool +CamerasChild::RecvReplyGetCaptureCapability(const CaptureCapability& ipcCapability) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyCapability.width = ipcCapability.width(); + mReplyCapability.height = ipcCapability.height(); + mReplyCapability.maxFPS = ipcCapability.maxFPS(); + mReplyCapability.expectedCaptureDelay = ipcCapability.expectedCaptureDelay(); + mReplyCapability.rawType = static_cast(ipcCapability.rawType()); + mReplyCapability.codecType = static_cast(ipcCapability.codecType()); + mReplyCapability.interlaced = ipcCapability.interlaced(); + monitor.Notify(); + return true; +} + + +int GetCaptureDevice(CaptureEngine aCapEngine, + unsigned int list_number, char* device_nameUTF8, + const unsigned int device_nameUTF8Length, + char* unique_idUTF8, + const unsigned int unique_idUTF8Length) +{ + return Cameras()->GetCaptureDevice(aCapEngine, + list_number, + device_nameUTF8, + device_nameUTF8Length, + unique_idUTF8, + unique_idUTF8Length); +} + +int +CamerasChild::GetCaptureDevice(CaptureEngine aCapEngine, + unsigned int list_number, char* device_nameUTF8, + const unsigned int device_nameUTF8Length, + char* unique_idUTF8, + const unsigned int unique_idUTF8Length) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, list_number]() -> nsresult { + if (this->SendGetCaptureDevice(aCapEngine, list_number)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + LOG(("GetCaptureDevice failed")); + return -1; + } + base::strlcpy(device_nameUTF8, mReplyDeviceName.get(), device_nameUTF8Length); + base::strlcpy(unique_idUTF8, mReplyDeviceID.get(), unique_idUTF8Length); + LOG(("Got %s name %s id", device_nameUTF8, unique_idUTF8)); + return 0; +} + +bool +CamerasChild::RecvReplyGetCaptureDevice(const nsCString& device_name, + const nsCString& device_id) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyDeviceName = device_name; + mReplyDeviceID = device_id; + monitor.Notify(); + return true; +} + +int AllocateCaptureDevice(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int unique_idUTF8Length, + int& capture_id) +{ + return Cameras()->AllocateCaptureDevice(aCapEngine, + unique_idUTF8, + unique_idUTF8Length, + capture_id); +} + +int +CamerasChild::AllocateCaptureDevice(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int unique_idUTF8Length, + int& capture_id) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + nsCString unique_id(unique_idUTF8); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, unique_id]() -> nsresult { + if (this->SendAllocateCaptureDevice(aCapEngine, unique_id)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + LOG(("AllocateCaptureDevice failed")); + return -1; + } + LOG(("Capture Device allocated: %d", mReplyInteger)); + capture_id = mReplyInteger; + return 0; +} + + +bool +CamerasChild::RecvReplyAllocateCaptureDevice(const int& numdev) +{ + LOG((__PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = numdev; + monitor.Notify(); + return true; +} + +int ReleaseCaptureDevice(CaptureEngine aCapEngine, const int capture_id) +{ + return Cameras()->ReleaseCaptureDevice(aCapEngine, capture_id); +} + +int +CamerasChild::ReleaseCaptureDevice(CaptureEngine aCapEngine, + const int capture_id) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, capture_id]() -> nsresult { + if (this->SendReleaseCaptureDevice(aCapEngine, capture_id)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + return -1; + } + return 0; +} + +void +CamerasChild::AddCallback(const CaptureEngine aCapEngine, const int capture_id, + webrtc::ExternalRenderer* render) +{ + MutexAutoLock lock(mCallbackMutex); + CapturerElement ce; + ce.engine = aCapEngine; + ce.id = capture_id; + ce.callback = render; + mCallbacks.AppendElement(ce); +} + +void +CamerasChild::RemoveCallback(const CaptureEngine aCapEngine, const int capture_id) +{ + MutexAutoLock lock(mCallbackMutex); + for (unsigned int i = 0; i < mCallbacks.Length(); i++) { + CapturerElement ce = mCallbacks[i]; + if (ce.engine == aCapEngine && ce.id == capture_id) { + mCallbacks.RemoveElementAt(i); + break; + } + } +} + +int StartCapture(CaptureEngine aCapEngine, + const int capture_id, + webrtc::CaptureCapability& webrtcCaps, + webrtc::ExternalRenderer* cb) +{ + return Cameras()->StartCapture(aCapEngine, + capture_id, + webrtcCaps, + cb); +} + +int +CamerasChild::StartCapture(CaptureEngine aCapEngine, + const int capture_id, + webrtc::CaptureCapability& webrtcCaps, + webrtc::ExternalRenderer* cb) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + AddCallback(aCapEngine, capture_id, cb); + CaptureCapability capCap(webrtcCaps.width, + webrtcCaps.height, + webrtcCaps.maxFPS, + webrtcCaps.expectedCaptureDelay, + webrtcCaps.rawType, + webrtcCaps.codecType, + webrtcCaps.interlaced); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, capture_id, capCap]() -> nsresult { + if (this->SendStartCapture(aCapEngine, capture_id, capCap)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + return -1; + } + return 0; +} + +int StopCapture(CaptureEngine aCapEngine, const int capture_id) +{ + return Cameras()->StopCapture(aCapEngine, capture_id); +} + +int +CamerasChild::StopCapture(CaptureEngine aCapEngine, const int capture_id) +{ + MutexAutoLock requestLock(mRequestMutex); + LOG((__PRETTY_FUNCTION__)); + nsCOMPtr runnable = + media::NewRunnableFrom([this, aCapEngine, capture_id]() -> nsresult { + if (this->SendStopCapture(aCapEngine, capture_id)) { + return NS_OK; + } + return NS_ERROR_FAILURE; + }); + MonitorAutoLock monitor(mReplyMonitor); + if (!DispatchToParent(runnable, monitor)) { + return -1; + } + RemoveCallback(aCapEngine, capture_id); + return 0; +} + +void +Shutdown(void) +{ + { + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + if (!CamerasSingleton::Child()) { + // We don't want to cause everything to get fired up if we're + // really already shut down. + LOG(("Shutdown when already shut down")); + return; + } + } + Cameras()->Shutdown(); +} + +class ShutdownRunnable : public nsRunnable { +public: + ShutdownRunnable(nsRefPtr aReplyEvent, + nsIThread* aReplyThread) + : mReplyEvent(aReplyEvent), mReplyThread(aReplyThread) {}; + + NS_IMETHOD Run() override { + LOG(("Closing BackgroundChild")); + ipc::BackgroundChild::CloseForCurrentThread(); + + LOG(("PBackground thread exists, shutting down thread")); + mReplyThread->Dispatch(mReplyEvent, NS_DISPATCH_NORMAL); + + return NS_OK; + } + +private: + nsRefPtr mReplyEvent; + nsIThread* mReplyThread; +}; + +void +CamerasChild::Shutdown() +{ + { + MonitorAutoLock monitor(mReplyMonitor); + mIPCIsAlive = false; + monitor.NotifyAll(); + } + + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + if (CamerasSingleton::Thread()) { + LOG(("Dispatching actor deletion")); + // Delete the parent actor. + nsRefPtr deleteRunnable = + // CamerasChild (this) will remain alive and is only deleted by the + // IPC layer when SendAllDone returns. + media::NewRunnableFrom([this]() -> nsresult { + unused << this->SendAllDone(); + return NS_OK; + }); + CamerasSingleton::Thread()->Dispatch(deleteRunnable, NS_DISPATCH_NORMAL); + LOG(("PBackground thread exists, dispatching close")); + // Dispatch closing the IPC thread back to us when the + // BackgroundChild is closed. + nsRefPtr event = + new ThreadDestructor(CamerasSingleton::Thread()); + nsRefPtr runnable = + new ShutdownRunnable(event, NS_GetCurrentThread()); + CamerasSingleton::Thread()->Dispatch(runnable, NS_DISPATCH_NORMAL); + } else { + LOG(("Shutdown called without PBackground thread")); + } + LOG(("Erasing sCameras & thread refs (original thread)")); + CamerasSingleton::Child() = nullptr; + CamerasSingleton::Thread() = nullptr; +} + +bool +CamerasChild::RecvDeliverFrame(const int& capEngine, + const int& capId, + mozilla::ipc::Shmem&& shmem, + const int& size, + const uint32_t& time_stamp, + const int64_t& ntp_time, + const int64_t& render_time) +{ + MutexAutoLock lock(mCallbackMutex); + CaptureEngine capEng = static_cast(capEngine); + if (Callback(capEng, capId)) { + unsigned char* image = shmem.get(); + Callback(capEng, capId)->DeliverFrame(image, size, + time_stamp, + ntp_time, render_time, + nullptr); + } else { + LOG(("DeliverFrame called with dead callback")); + } + SendReleaseFrame(shmem); + return true; +} + +bool +CamerasChild::RecvFrameSizeChange(const int& capEngine, + const int& capId, + const int& w, const int& h) +{ + LOG((__PRETTY_FUNCTION__)); + MutexAutoLock lock(mCallbackMutex); + CaptureEngine capEng = static_cast(capEngine); + if (Callback(capEng, capId)) { + Callback(capEng, capId)->FrameSizeChange(w, h, 0); + } else { + LOG(("Frame size change with dead callback")); + } + return true; +} + +void +CamerasChild::ActorDestroy(ActorDestroyReason aWhy) +{ + MonitorAutoLock monitor(mReplyMonitor); + mIPCIsAlive = false; + // Hopefully prevent us from getting stuck + // on replies that'll never come. + monitor.NotifyAll(); +} + +CamerasChild::CamerasChild() + : mCallbackMutex("mozilla::cameras::CamerasChild::mCallbackMutex"), + mIPCIsAlive(true), + mRequestMutex("mozilla::cameras::CamerasChild::mRequestMutex"), + mReplyMonitor("mozilla::cameras::CamerasChild::mReplyMonitor") +{ + if (!gCamerasChildLog) { + gCamerasChildLog = PR_NewLogModule("CamerasChild"); + } + + LOG(("CamerasChild: %p", this)); + + MOZ_COUNT_CTOR(CamerasChild); +} + +CamerasChild::~CamerasChild() +{ + LOG(("~CamerasChild: %p", this)); + + Shutdown(); + + MOZ_COUNT_DTOR(CamerasChild); +} + +webrtc::ExternalRenderer* CamerasChild::Callback(CaptureEngine aCapEngine, + int capture_id) +{ + for (unsigned int i = 0; i < mCallbacks.Length(); i++) { + CapturerElement ce = mCallbacks[i]; + if (ce.engine == aCapEngine && ce.id == capture_id) { + return ce.callback; + } + } + + return nullptr; +} + +} +} diff --git a/dom/media/systemservices/CamerasChild.h b/dom/media/systemservices/CamerasChild.h new file mode 100644 index 0000000000..0ead231447 --- /dev/null +++ b/dom/media/systemservices/CamerasChild.h @@ -0,0 +1,175 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CamerasChild_h +#define mozilla_CamerasChild_h + +#include "mozilla/Pair.h" +#include "mozilla/dom/ContentChild.h" +#include "mozilla/camera/PCamerasChild.h" +#include "mozilla/camera/PCamerasParent.h" +#include "mozilla/Mutex.h" +#include "nsCOMPtr.h" + +// conflicts with #include of scoped_ptr.h +#undef FF +#include "webrtc/common.h" +// Video Engine +#include "webrtc/video_engine/include/vie_base.h" +#include "webrtc/video_engine/include/vie_capture.h" +#include "webrtc/video_engine/include/vie_render.h" + +namespace mozilla { + +namespace ipc { +class BackgroundChildImpl; +} + +namespace camera { + +enum CaptureEngine : int { + InvalidEngine = 0, + ScreenEngine, + BrowserEngine, + WinEngine, + AppEngine, + CameraEngine, + MaxEngine +}; + +struct CapturerElement { + CaptureEngine engine; + int id; + webrtc::ExternalRenderer* callback; +}; + +// statically mirror webrtc.org ViECapture API +// these are called via MediaManager->MediaEngineRemoteVideoSource +// on the MediaManager thread +int NumberOfCapabilities(CaptureEngine aCapEngine, + const char* deviceUniqueIdUTF8); +int GetCaptureCapability(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::CaptureCapability& capability); +int NumberOfCaptureDevices(CaptureEngine aCapEngine); +int GetCaptureDevice(CaptureEngine aCapEngine, + unsigned int list_number, char* device_nameUTF8, + const unsigned int device_nameUTF8Length, + char* unique_idUTF8, + const unsigned int unique_idUTF8Length); +int AllocateCaptureDevice(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int unique_idUTF8Length, + int& capture_id); +int ReleaseCaptureDevice(CaptureEngine aCapEngine, + const int capture_id); +int StartCapture(CaptureEngine aCapEngine, + const int capture_id, webrtc::CaptureCapability& capability, + webrtc::ExternalRenderer* func); +int StopCapture(CaptureEngine aCapEngine, const int capture_id); +void Shutdown(void); + +class CamerasChild final : public PCamerasChild +{ + friend class mozilla::ipc::BackgroundChildImpl; + +public: + // We are owned by the PBackground thread only. CamerasSingleton + // takes a non-owning reference. + NS_INLINE_DECL_REFCOUNTING(CamerasChild) + + // IPC messages recevied, received on the PBackground thread + // these are the actual callbacks with data + virtual bool RecvDeliverFrame(const int&, const int&, mozilla::ipc::Shmem&&, + const int&, const uint32_t&, const int64_t&, + const int64_t&) override; + virtual bool RecvFrameSizeChange(const int&, const int&, + const int& w, const int& h) override; + + // these are response messages to our outgoing requests + virtual bool RecvReplyNumberOfCaptureDevices(const int&) override; + virtual bool RecvReplyNumberOfCapabilities(const int&) override; + virtual bool RecvReplyAllocateCaptureDevice(const int&) override; + virtual bool RecvReplyGetCaptureCapability(const CaptureCapability& capability) override; + virtual bool RecvReplyGetCaptureDevice(const nsCString& device_name, + const nsCString& device_id) override; + virtual bool RecvReplyFailure(void) override; + virtual bool RecvReplySuccess(void) override; + virtual void ActorDestroy(ActorDestroyReason aWhy) override; + + // the webrtc.org ViECapture calls are mirrored here, but with access + // to a specific PCameras instance to communicate over. These also + // run on the MediaManager thread + int NumberOfCaptureDevices(CaptureEngine aCapEngine); + int NumberOfCapabilities(CaptureEngine aCapEngine, + const char* deviceUniqueIdUTF8); + int ReleaseCaptureDevice(CaptureEngine aCapEngine, + const int capture_id); + int StartCapture(CaptureEngine aCapEngine, + const int capture_id, webrtc::CaptureCapability& capability, + webrtc::ExternalRenderer* func); + int StopCapture(CaptureEngine aCapEngine, const int capture_id); + int AllocateCaptureDevice(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int unique_idUTF8Length, + int& capture_id); + int GetCaptureCapability(CaptureEngine aCapEngine, + const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::CaptureCapability& capability); + int GetCaptureDevice(CaptureEngine aCapEngine, + unsigned int list_number, char* device_nameUTF8, + const unsigned int device_nameUTF8Length, + char* unique_idUTF8, + const unsigned int unique_idUTF8Length); + void Shutdown(); + + webrtc::ExternalRenderer* Callback(CaptureEngine aCapEngine, int capture_id); + void AddCallback(const CaptureEngine aCapEngine, const int capture_id, + webrtc::ExternalRenderer* render); + void RemoveCallback(const CaptureEngine aCapEngine, const int capture_id); + + +private: + CamerasChild(); + ~CamerasChild(); + // Dispatch a Runnable to the PCamerasParent, by executing it on the + // decidecated Cameras IPC/PBackground thread. + bool DispatchToParent(nsIRunnable* aRunnable, + MonitorAutoLock& aMonitor); + + nsTArray mCallbacks; + // Protects the callback arrays + Mutex mCallbackMutex; + + bool mIPCIsAlive; + + // Hold to prevent multiple outstanding requests. We don't use + // request IDs so we only support one at a time. Don't want try + // to use the webrtc.org API from multiple threads simultanously. + // The monitor below isn't sufficient for this, as it will drop + // the lock when Wait-ing for a response, allowing us to send a new + // request. The Notify on receiving the response will then unblock + // both waiters and one will be guaranteed to get the wrong result. + // Take this one before taking mReplyMonitor. + Mutex mRequestMutex; + // Hold to wait for an async response to our calls + Monitor mReplyMonitor; + // Async resposne valid? + bool mReceivedReply; + // Aynsc reponses data contents; + bool mReplySuccess; + int mReplyInteger; + webrtc::CaptureCapability mReplyCapability; + nsCString mReplyDeviceName; + nsCString mReplyDeviceID; +}; + +} // namespace camera +} // namespace mozilla + +#endif // mozilla_CamerasChild_h diff --git a/dom/media/systemservices/CamerasParent.cpp b/dom/media/systemservices/CamerasParent.cpp new file mode 100644 index 0000000000..1a589cce15 --- /dev/null +++ b/dom/media/systemservices/CamerasParent.cpp @@ -0,0 +1,866 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CamerasParent.h" +#include "CamerasUtils.h" +#include "MediaEngine.h" +#include "MediaUtils.h" + +#include "mozilla/Assertions.h" +#include "mozilla/unused.h" +#include "mozilla/Logging.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "nsThreadUtils.h" + +#undef LOG +#undef LOG_ENABLED +PRLogModuleInfo *gCamerasParentLog; +#define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) + +namespace mozilla { +namespace camera { + +// 3 threads are involved in this code: +// - the main thread for some setups, and occassionally for video capture setup +// calls that don't work correctly elsewhere. +// - the IPC thread on which PBackground is running and which receives and +// sends messages +// - a thread which will execute the actual (possibly slow) camera access +// called "VideoCapture". On Windows this is a thread with an event loop +// suitable for UI access. + +class FrameSizeChangeRunnable : public nsRunnable { +public: + FrameSizeChangeRunnable(CamerasParent *aParent, CaptureEngine capEngine, + int cap_id, unsigned int aWidth, unsigned int aHeight) + : mParent(aParent), mCapEngine(capEngine), mCapId(cap_id), + mWidth(aWidth), mHeight(aHeight) {} + + NS_IMETHOD Run() { + if (mParent->IsShuttingDown()) { + // Communication channel is being torn down + LOG(("FrameSizeChangeRunnable is active without active Child")); + mResult = 0; + return NS_OK; + } + if (!mParent->SendFrameSizeChange(mCapEngine, mCapId, mWidth, mHeight)) { + mResult = -1; + } else { + mResult = 0; + } + return NS_OK; + } + + int GetResult() { + return mResult; + } + +private: + nsRefPtr mParent; + CaptureEngine mCapEngine; + int mCapId; + unsigned int mWidth; + unsigned int mHeight; + int mResult; +}; + +int +CallbackHelper::FrameSizeChange(unsigned int w, unsigned int h, + unsigned int streams) +{ + LOG(("CallbackHelper Video FrameSizeChange: %ux%u", w, h)); + nsRefPtr runnable = + new FrameSizeChangeRunnable(mParent, mCapEngine, mCapturerId, w, h); + MOZ_ASSERT(mParent); + nsIThread * thread = mParent->GetBackgroundThread(); + MOZ_ASSERT(thread != nullptr); + thread->Dispatch(runnable, NS_DISPATCH_NORMAL); + return 0; +} + +class DeliverFrameRunnable : public nsRunnable { +public: + DeliverFrameRunnable(CamerasParent *aParent, + CaptureEngine engine, + int cap_id, + ShmemBuffer buffer, + unsigned char* altbuffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time) + : mParent(aParent), mCapEngine(engine), mCapId(cap_id), mBuffer(Move(buffer)), + mSize(size), mTimeStamp(time_stamp), mNtpTime(ntp_time), + mRenderTime(render_time) { + // No ShmemBuffer (of the right size) was available, so make an + // extra buffer here. We have no idea when we are going to run and + // it will be potentially long after the webrtc frame callback has + // returned, so the copy needs to be no later than here. + // We will need to copy this back into a Shmem later on so we prefer + // using ShmemBuffers to avoid the extra copy. + if (altbuffer != nullptr) { + mAlternateBuffer.reset(new unsigned char[size]); + memcpy(mAlternateBuffer.get(), altbuffer, size); + } + }; + + NS_IMETHOD Run() { + if (mParent->IsShuttingDown()) { + // Communication channel is being torn down + mResult = 0; + return NS_OK; + } + if (!mParent->DeliverFrameOverIPC(mCapEngine, mCapId, + Move(mBuffer), mAlternateBuffer.get(), + mSize, mTimeStamp, + mNtpTime, mRenderTime)) { + mResult = -1; + } else { + mResult = 0; + } + return NS_OK; + } + + int GetResult() { + return mResult; + } + +private: + nsRefPtr mParent; + CaptureEngine mCapEngine; + int mCapId; + ShmemBuffer mBuffer; + mozilla::UniquePtr mAlternateBuffer; + int mSize; + uint32_t mTimeStamp; + int64_t mNtpTime; + int64_t mRenderTime; + int mResult; +}; + +int +CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine, + int cap_id, + ShmemBuffer buffer, + unsigned char* altbuffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time) +{ + // No ShmemBuffers were available, so construct one now of the right size + // and copy into it. That is an extra copy, but we expect this to be + // the exceptional case, because we just assured the next call *will* have a + // buffer of the right size. + if (altbuffer != nullptr) { + // Get a shared memory buffer from the pool, at least size big + ShmemBuffer shMemBuff = mShmemPool.Get(this, size); + + if (!shMemBuff.Valid()) { + LOG(("Video shmem is not writeable in DeliverFrame")); + // We can skip this frame if we run out of buffers, it's not a real error. + return 0; + } + + // get() and Size() check for proper alignment of the segment + memcpy(shMemBuff.GetBytes(), altbuffer, size); + + if (!SendDeliverFrame(cap_engine, cap_id, + shMemBuff.Get(), size, + time_stamp, ntp_time, render_time)) { + return -1; + } + } else { + // ShmemBuffer was available, we're all good. A single copy happened + // in the original webrtc callback. + if (!SendDeliverFrame(cap_engine, cap_id, + buffer.Get(), size, + time_stamp, ntp_time, render_time)) { + return -1; + } + } + + return 0; +} + +ShmemBuffer +CamerasParent::GetBuffer(size_t aSize) +{ + return mShmemPool.GetIfAvailable(aSize); +} + +int +CallbackHelper::DeliverFrame(unsigned char* buffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time, + void *handle) +{ + // Get a shared memory buffer to copy the frame data into + ShmemBuffer shMemBuffer = mParent->GetBuffer(size); + if (!shMemBuffer.Valid()) { + // Either we ran out of buffers or they're not the right size yet + LOG(("Video shmem is not available in DeliverFrame")); + // We will do the copy into a(n extra) temporary buffer inside + // the DeliverFrameRunnable constructor. + } else { + // Shared memory buffers of the right size are available, do the copy here. + memcpy(shMemBuffer.GetBytes(), buffer, size); + // Mark the original buffer as cleared. + buffer = nullptr; + } + nsRefPtr runnable = + new DeliverFrameRunnable(mParent, mCapEngine, mCapturerId, + Move(shMemBuffer), buffer, size, time_stamp, + ntp_time, render_time); + MOZ_ASSERT(mParent); + nsIThread* thread = mParent->GetBackgroundThread(); + MOZ_ASSERT(thread != nullptr); + thread->Dispatch(runnable, NS_DISPATCH_NORMAL); + return 0; +} + +bool +CamerasParent::RecvReleaseFrame(mozilla::ipc::Shmem&& s) { + mShmemPool.Put(ShmemBuffer(s)); + return true; +} + +bool +CamerasParent::SetupEngine(CaptureEngine aCapEngine) +{ + EngineHelper *helper = &mEngines[aCapEngine]; + + // Already initialized + if (helper->mEngine) { + return true; + } + + webrtc::CaptureDeviceInfo *captureDeviceInfo = nullptr; + + switch (aCapEngine) { + case ScreenEngine: + captureDeviceInfo = + new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen); + break; + case BrowserEngine: + captureDeviceInfo = + new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Browser); + break; + case WinEngine: + captureDeviceInfo = + new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Window); + break; + case AppEngine: + captureDeviceInfo = + new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Application); + break; + case CameraEngine: + captureDeviceInfo = + new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Camera); + break; + default: + LOG(("Invalid webrtc Video engine")); + MOZ_CRASH(); + break; + } + + helper->mConfig.Set(captureDeviceInfo); + helper->mEngine = webrtc::VideoEngine::Create(helper->mConfig); + + if (!helper->mEngine) { + LOG(("VideoEngine::Create failed")); + return false; + } + + helper->mPtrViEBase = webrtc::ViEBase::GetInterface(helper->mEngine); + if (!helper->mPtrViEBase) { + LOG(("ViEBase::GetInterface failed")); + return false; + } + + if (helper->mPtrViEBase->Init() < 0) { + LOG(("ViEBase::Init failed")); + return false; + } + + helper->mPtrViECapture = webrtc::ViECapture::GetInterface(helper->mEngine); + if (!helper->mPtrViECapture) { + LOG(("ViECapture::GetInterface failed")); + return false; + } + + helper->mPtrViERender = webrtc::ViERender::GetInterface(helper->mEngine); + if (!helper->mPtrViERender) { + LOG(("ViERender::GetInterface failed")); + return false; + } + + return true; +} + +void +CamerasParent::CloseEngines() +{ + { + MutexAutoLock lock(mCallbackMutex); + // Stop the callers + while (mCallbacks.Length()) { + auto capEngine = mCallbacks[0]->mCapEngine; + auto capNum = mCallbacks[0]->mCapturerId; + LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, capNum)); + { + MutexAutoUnlock unlock(mCallbackMutex); + RecvStopCapture(capEngine, capNum); + RecvReleaseCaptureDevice(capEngine, capNum); + } + // The callbacks list might have changed while we released the lock, + // but note that due to the loop construct this will not break us. + } + } + + { + MutexAutoLock lock(mEngineMutex); + for (int i = 0; i < CaptureEngine::MaxEngine; i++) { + if (mEngines[i].mEngineIsRunning) { + LOG(("Being closed down while engine %d is running!", i)); + } + if (mEngines[i].mPtrViERender) { + mEngines[i].mPtrViERender->Release(); + mEngines[i].mPtrViERender = nullptr; + } + if (mEngines[i].mPtrViECapture) { + mEngines[i].mPtrViECapture->Release(); + mEngines[i].mPtrViECapture = nullptr; + } + if(mEngines[i].mPtrViEBase) { + mEngines[i].mPtrViEBase->Release(); + mEngines[i].mPtrViEBase = nullptr; + } + } + } +} + +bool +CamerasParent::EnsureInitialized(int aEngine) +{ + LOG((__PRETTY_FUNCTION__)); + CaptureEngine capEngine = static_cast(aEngine); + if (!SetupEngine(capEngine)) { + return false; + } + + return true; +} + +// Dispatch the runnable to do the camera operation on the +// specific Cameras thread, preventing us from blocking, and +// chain a runnable to send back the result on the IPC thread. +// It would be nice to get rid of the code duplication here, +// perhaps via Promises. +bool +CamerasParent::RecvNumberOfCaptureDevices(const int& aCapEngine) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("RecvNumberOfCaptureDevices fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine]() -> nsresult { + MutexAutoLock lock(self->mEngineMutex); + int num = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + num = self->mEngines[aCapEngine].mPtrViECapture->NumberOfCaptureDevices(); + } + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, num]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (num < 0) { + LOG(("RecvNumberOfCaptureDevices couldn't find devices")); + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } else { + LOG(("RecvNumberOfCaptureDevices: %d", num)); + unused << self->SendReplyNumberOfCaptureDevices(num); + return NS_OK; + } + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + + return true; +} + +bool +CamerasParent::RecvNumberOfCapabilities(const int& aCapEngine, + const nsCString& unique_id) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("RecvNumberOfCapabilities fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + LOG(("Getting caps for %s", unique_id.get())); + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, unique_id, aCapEngine]() -> nsresult { + MutexAutoLock lock(self->mEngineMutex); + int num = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + num = + self->mEngines[aCapEngine].mPtrViECapture->NumberOfCapabilities( + unique_id.get(), + MediaEngineSource::kMaxUniqueIdLength); + } + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, num]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (num < 0) { + LOG(("RecvNumberOfCapabilities couldn't find capabilities")); + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } else { + LOG(("RecvNumberOfCapabilities: %d", num)); + } + unused << self->SendReplyNumberOfCapabilities(num); + return NS_OK; + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return true; +} + +bool +CamerasParent::RecvGetCaptureCapability(const int &aCapEngine, + const nsCString& unique_id, + const int& num) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + LOG(("RecvGetCaptureCapability: %s %d", unique_id.get(), num)); + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, unique_id, aCapEngine, num]() -> nsresult { + webrtc::CaptureCapability webrtcCaps; + MutexAutoLock lock(self->mEngineMutex); + int error = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureCapability( + unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, num, webrtcCaps); + } + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, webrtcCaps, error]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + CaptureCapability capCap(webrtcCaps.width, + webrtcCaps.height, + webrtcCaps.maxFPS, + webrtcCaps.expectedCaptureDelay, + webrtcCaps.rawType, + webrtcCaps.codecType, + webrtcCaps.interlaced); + LOG(("Capability: %u %u %u %u %d %d", + webrtcCaps.width, + webrtcCaps.height, + webrtcCaps.maxFPS, + webrtcCaps.expectedCaptureDelay, + webrtcCaps.rawType, + webrtcCaps.codecType)); + if (error) { + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } + unused << self->SendReplyGetCaptureCapability(capCap); + return NS_OK; + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return true; +} + +bool +CamerasParent::RecvGetCaptureDevice(const int& aCapEngine, + const int& aListNumber) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + LOG(("RecvGetCaptureDevice")); + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine, aListNumber]() -> nsresult { + char deviceName[MediaEngineSource::kMaxDeviceNameLength]; + char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength]; + nsCString name; + nsCString uniqueId; + MutexAutoLock lock(self->mEngineMutex); + int error = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureDevice(aListNumber, + deviceName, + sizeof(deviceName), + deviceUniqueId, + sizeof(deviceUniqueId)); + } + if (!error) { + name.Assign(deviceName); + uniqueId.Assign(deviceUniqueId); + } + + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, error, name, uniqueId]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (error) { + LOG(("GetCaptureDevice failed: %d", error)); + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } + + LOG(("Returning %s name %s id", name.get(), uniqueId.get())); + unused << self->SendReplyGetCaptureDevice(name, uniqueId); + return NS_OK; + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return true; +} + +bool +CamerasParent::RecvAllocateCaptureDevice(const int& aCapEngine, + const nsCString& unique_id) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine, unique_id]() -> nsresult { + int numdev = -1; + MutexAutoLock lock(self->mEngineMutex); + int error = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + error = self->mEngines[aCapEngine].mPtrViECapture->AllocateCaptureDevice( + unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, numdev); + } + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, numdev, error]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (error) { + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } else { + LOG(("Allocated device nr %d", numdev)); + unused << self->SendReplyAllocateCaptureDevice(numdev); + return NS_OK; + } + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return true; +} + +bool +CamerasParent::RecvReleaseCaptureDevice(const int& aCapEngine, + const int& numdev) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Fails to initialize")); + unused << SendReplyFailure(); + return false; + } + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine, numdev]() -> nsresult { + LOG(("RecvReleaseCamera device nr %d", numdev)); + MutexAutoLock lock(self->mEngineMutex); + int error = -1; + if (self->mEngines[aCapEngine].mPtrViECapture) { + error = self->mEngines[aCapEngine].mPtrViECapture->ReleaseCaptureDevice(numdev); + } + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, error, numdev]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (error) { + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } else { + unused << self->SendReplySuccess(); + LOG(("Freed device nr %d", numdev)); + return NS_OK; + } + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); +#ifndef XP_MACOSX + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); +#else + // Mac OS X hangs on shutdown if we don't do this on the main thread. + NS_DispatchToMainThread(webrtc_runnable); +#endif + return true; +} + +bool +CamerasParent::RecvStartCapture(const int& aCapEngine, + const int& capnum, + const CaptureCapability& ipcCaps) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Failure to initialize")); + unused << SendReplyFailure(); + return false; + } + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine, capnum, ipcCaps]() -> nsresult { + CallbackHelper** cbh; + webrtc::ExternalRenderer* render; + EngineHelper* helper = nullptr; + int error; + { + MutexAutoLock lockCallback(self->mCallbackMutex); + cbh = self->mCallbacks.AppendElement( + new CallbackHelper(static_cast(aCapEngine), capnum, self)); + render = static_cast(*cbh); + } + { + MutexAutoLock lockEngine(self->mEngineMutex); + if (self->mEngines[aCapEngine].mPtrViECapture) { + helper = &self->mEngines[aCapEngine]; + error = + helper->mPtrViERender->AddRenderer(capnum, webrtc::kVideoI420, render); + } else { + error = -1; + } + + if (!error) { + error = helper->mPtrViERender->StartRender(capnum); + } + + webrtc::CaptureCapability capability; + capability.width = ipcCaps.width(); + capability.height = ipcCaps.height(); + capability.maxFPS = ipcCaps.maxFPS(); + capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay(); + capability.rawType = static_cast(ipcCaps.rawType()); + capability.codecType = static_cast(ipcCaps.codecType()); + capability.interlaced = ipcCaps.interlaced(); + + if (!error) { + error = helper->mPtrViECapture->StartCapture(capnum, capability); + } + if (!error) { + helper->mEngineIsRunning = true; + } + } + + nsRefPtr ipc_runnable = + media::NewRunnableFrom([self, error]() -> nsresult { + if (self->IsShuttingDown()) { + return NS_ERROR_FAILURE; + } + if (!error) { + unused << self->SendReplySuccess(); + return NS_OK; + } else { + unused << self->SendReplyFailure(); + return NS_ERROR_FAILURE; + } + }); + self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL); + return NS_OK; + }); + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return true; +} + +bool +CamerasParent::RecvStopCapture(const int& aCapEngine, + const int& capnum) +{ + LOG((__PRETTY_FUNCTION__)); + if (!EnsureInitialized(aCapEngine)) { + LOG(("Failure to initialize")); + unused << SendReplyFailure(); + return false; + } + + nsRefPtr self(this); + nsRefPtr webrtc_runnable = + media::NewRunnableFrom([self, aCapEngine, capnum]() -> nsresult { + { + MutexAutoLock lock(self->mEngineMutex); + // We only need to check mPtrViECapture as all other engines are guaranteed + // to be nulled under the same lock. + if (self->mEngines[aCapEngine].mPtrViECapture) { + self->mEngines[aCapEngine].mPtrViECapture->StopCapture(capnum); + self->mEngines[aCapEngine].mPtrViERender->StopRender(capnum); + self->mEngines[aCapEngine].mPtrViERender->RemoveRenderer(capnum); + self->mEngines[aCapEngine].mEngineIsRunning = false; + } + } + MutexAutoLock lock(self->mCallbackMutex); + for (unsigned int i = 0; i < self->mCallbacks.Length(); i++) { + if (self->mCallbacks[i]->mCapEngine == aCapEngine + && self->mCallbacks[i]->mCapturerId == capnum) { + delete self->mCallbacks[i]; + self->mCallbacks.RemoveElementAt(i); + break; + } + } + return NS_OK; + }); + + mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable)); + return SendReplySuccess(); +} + +bool +CamerasParent::RecvAllDone() +{ + LOG((__PRETTY_FUNCTION__)); + // Don't try to send anything to the child now + mChildIsAlive = false; + return Send__delete__(this); +} + +void CamerasParent::DoShutdown() +{ + LOG((__PRETTY_FUNCTION__)); + CloseEngines(); + + { + MutexAutoLock lock(mEngineMutex); + for (int i = 0; i < CaptureEngine::MaxEngine; i++) { + if (mEngines[i].mEngine) { + mEngines[i].mEngine->SetTraceCallback(nullptr); + webrtc::VideoEngine::Delete(mEngines[i].mEngine); + mEngines[i].mEngine = nullptr; + } + } + } + + mShmemPool.Cleanup(this); + + mPBackgroundThread = nullptr; + + if (mVideoCaptureThread) { + if (mVideoCaptureThread->IsRunning()) { + mVideoCaptureThread->Stop(); + } + delete mVideoCaptureThread; + mVideoCaptureThread = nullptr; + } +} + +void +CamerasParent::ActorDestroy(ActorDestroyReason aWhy) +{ + // No more IPC from here + LOG((__PRETTY_FUNCTION__)); + // We don't want to receive callbacks or anything if we can't + // forward them anymore anyway. + mChildIsAlive = false; + mDestroyed = true; + CloseEngines(); +} + +CamerasParent::CamerasParent() + : mCallbackMutex("CamerasParent.mCallbackMutex"), + mEngineMutex("CamerasParent.mEngineMutex"), + mShmemPool(CaptureEngine::MaxEngine), + mVideoCaptureThread(nullptr), + mChildIsAlive(true), + mDestroyed(false) +{ + if (!gCamerasParentLog) { + gCamerasParentLog = PR_NewLogModule("CamerasParent"); + } + LOG(("CamerasParent: %p", this)); + + mPBackgroundThread = NS_GetCurrentThread(); + MOZ_ASSERT(mPBackgroundThread != nullptr, "GetCurrentThread failed"); + + LOG(("Spinning up WebRTC Cameras Thread")); + mVideoCaptureThread = new base::Thread("VideoCapture"); + base::Thread::Options options; +#if defined(_WIN32) + options.message_loop_type = MessageLoop::TYPE_MOZILLA_NONMAINUITHREAD; +#else + options.message_loop_type = MessageLoop::TYPE_MOZILLA_NONMAINTHREAD; +#endif + if (!mVideoCaptureThread->StartWithOptions(options)) { + MOZ_CRASH(); + } + + MOZ_COUNT_CTOR(CamerasParent); +} + +CamerasParent::~CamerasParent() +{ + LOG(("~CamerasParent: %p", this)); + + MOZ_COUNT_DTOR(CamerasParent); + DoShutdown(); +} + +already_AddRefed +CamerasParent::Create() { + mozilla::ipc::AssertIsOnBackgroundThread(); + nsRefPtr camerasParent = new CamerasParent(); + return camerasParent.forget(); +} + +} +} diff --git a/dom/media/systemservices/CamerasParent.h b/dom/media/systemservices/CamerasParent.h new file mode 100644 index 0000000000..4770cde09c --- /dev/null +++ b/dom/media/systemservices/CamerasParent.h @@ -0,0 +1,147 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CamerasParent_h +#define mozilla_CamerasParent_h + +#include "mozilla/dom/ContentParent.h" +#include "mozilla/camera/PCamerasParent.h" +#include "mozilla/ipc/Shmem.h" +#include "mozilla/ShmemPool.h" + +// conflicts with #include of scoped_ptr.h +#undef FF +#include "webrtc/common.h" +// Video Engine +#include "webrtc/video_engine/include/vie_base.h" +#include "webrtc/video_engine/include/vie_capture.h" +#include "webrtc/video_engine/include/vie_render.h" +#include "CamerasChild.h" + +#include "base/thread.h" + +namespace mozilla { +namespace camera { + +class CamerasParent; + +class CallbackHelper : public webrtc::ExternalRenderer +{ +public: + CallbackHelper(CaptureEngine aCapEng, int aCapId, CamerasParent *aParent) + : mCapEngine(aCapEng), mCapturerId(aCapId), mParent(aParent) {}; + + // ViEExternalRenderer implementation. These callbacks end up + // running on the VideoCapture thread. + virtual int FrameSizeChange(unsigned int w, unsigned int h, + unsigned int streams) override; + virtual int DeliverFrame(unsigned char* buffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time, + void *handle) override; + virtual bool IsTextureSupported() override { return false; }; + + friend CamerasParent; + +private: + CaptureEngine mCapEngine; + int mCapturerId; + CamerasParent *mParent; +}; + +class EngineHelper +{ +public: + EngineHelper() : + mEngine(nullptr), mPtrViEBase(nullptr), mPtrViECapture(nullptr), + mPtrViERender(nullptr), mEngineIsRunning(false) {}; + + webrtc::VideoEngine *mEngine; + webrtc::ViEBase *mPtrViEBase; + webrtc::ViECapture *mPtrViECapture; + webrtc::ViERender *mPtrViERender; + + // The webrtc code keeps a reference to this one. + webrtc::Config mConfig; + + // Engine alive + bool mEngineIsRunning; +}; + +class CamerasParent : public PCamerasParent +{ + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CamerasParent); + +public: + static already_AddRefed Create(); + + // Messages received form the child. These run on the IPC/PBackground thread. + virtual bool RecvAllocateCaptureDevice(const int&, const nsCString&) override; + virtual bool RecvReleaseCaptureDevice(const int&, const int &) override; + virtual bool RecvNumberOfCaptureDevices(const int&) override; + virtual bool RecvNumberOfCapabilities(const int&, const nsCString&) override; + virtual bool RecvGetCaptureCapability(const int&, const nsCString&, const int&) override; + virtual bool RecvGetCaptureDevice(const int&, const int&) override; + virtual bool RecvStartCapture(const int&, const int&, const CaptureCapability&) override; + virtual bool RecvStopCapture(const int&, const int&) override; + virtual bool RecvReleaseFrame(mozilla::ipc::Shmem&&) override; + virtual bool RecvAllDone() override; + virtual void ActorDestroy(ActorDestroyReason aWhy) override; + + nsIThread* GetBackgroundThread() { return mPBackgroundThread; }; + bool IsShuttingDown() { return !mChildIsAlive || mDestroyed; }; + ShmemBuffer GetBuffer(size_t aSize); + + // helper to forward to the PBackground thread + int DeliverFrameOverIPC(CaptureEngine capEng, + int cap_id, + ShmemBuffer buffer, + unsigned char* altbuffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time); + + + CamerasParent(); + +protected: + virtual ~CamerasParent(); + + bool SetupEngine(CaptureEngine aCapEngine); + void CloseEngines(); + bool EnsureInitialized(int aEngine); + void DoShutdown(); + + EngineHelper mEngines[CaptureEngine::MaxEngine]; + nsTArray mCallbacks; + // Protects the callback arrays + Mutex mCallbackMutex; + // Protects the engines array + Mutex mEngineMutex; + + // image buffers + mozilla::ShmemPool mShmemPool; + + // PBackground parent thread + nsCOMPtr mPBackgroundThread; + + // video processing thread - where webrtc.org capturer code runs + base::Thread* mVideoCaptureThread; + + // Shutdown handling + bool mChildIsAlive; + bool mDestroyed; +}; + +PCamerasParent* CreateCamerasParent(); + +} // namespace camera +} // namespace mozilla + +#endif // mozilla_CameraParent_h diff --git a/dom/media/systemservices/CamerasUtils.cpp b/dom/media/systemservices/CamerasUtils.cpp new file mode 100644 index 0000000000..c4c10a8929 --- /dev/null +++ b/dom/media/systemservices/CamerasUtils.cpp @@ -0,0 +1,81 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/unused.h" +#include "nsThreadUtils.h" +#include "nsCOMPtr.h" +#include "mozilla/Assertions.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "nsIIPCBackgroundChildCreateCallback.h" + +namespace mozilla { +namespace camera { + +class WorkerBackgroundChildCallback final : + public nsIIPCBackgroundChildCreateCallback +{ + bool* mDone; + +public: + explicit WorkerBackgroundChildCallback(bool* aDone) + : mDone(aDone) + { + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(mDone); + } + + NS_DECL_ISUPPORTS + +private: + ~WorkerBackgroundChildCallback() { } + + virtual void + ActorCreated(PBackgroundChild* aActor) override + { + *mDone = true; + } + + virtual void + ActorFailed() override + { + *mDone = true; + } +}; +NS_IMPL_ISUPPORTS(WorkerBackgroundChildCallback, nsIIPCBackgroundChildCreateCallback) + +nsresult +SynchronouslyCreatePBackground() +{ + using mozilla::ipc::BackgroundChild; + + MOZ_ASSERT(!BackgroundChild::GetForCurrentThread()); + + bool done = false; + nsCOMPtr callback = + new WorkerBackgroundChildCallback(&done); + + if (NS_WARN_IF(!BackgroundChild::GetOrCreateForCurrentThread(callback))) { + return NS_ERROR_FAILURE; + } + + nsIThread *thread = NS_GetCurrentThread(); + + while (!done) { + if (NS_WARN_IF(!NS_ProcessNextEvent(thread, true /* aMayWait */))) { + return NS_ERROR_FAILURE; + } + } + + if (NS_WARN_IF(!BackgroundChild::GetForCurrentThread())) { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +} +} diff --git a/dom/media/systemservices/CamerasUtils.h b/dom/media/systemservices/CamerasUtils.h new file mode 100644 index 0000000000..b58b21fac3 --- /dev/null +++ b/dom/media/systemservices/CamerasUtils.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CameraUtils_h +#define mozilla_CameraUtils_h + +#include "nsThreadUtils.h" +#include "nsCOMPtr.h" +#include "mozilla/UniquePtr.h" + +#include "base/thread.h" + +namespace mozilla { +namespace camera { + +nsresult SynchronouslyCreatePBackground(); + +class ThreadDestructor : public nsRunnable +{ + DISALLOW_COPY_AND_ASSIGN(ThreadDestructor); + +public: + explicit ThreadDestructor(nsIThread* aThread) + : mThread(aThread) {} + + NS_IMETHOD Run() override + { + if (mThread) { + mThread->Shutdown(); + } + return NS_OK; + } + +private: + ~ThreadDestructor() {} + nsCOMPtr mThread; +}; + +class RunnableTask : public Task +{ +public: + explicit RunnableTask(nsRunnable* aRunnable) + : mRunnable(aRunnable) {} + + void Run() override { + mRunnable->Run(); + } + +private: + ~RunnableTask() {} + nsRefPtr mRunnable; +}; + +} +} + +#endif // mozilla_CameraUtils_h diff --git a/dom/media/systemservices/PCameras.ipdl b/dom/media/systemservices/PCameras.ipdl new file mode 100644 index 0000000000..637b04a95a --- /dev/null +++ b/dom/media/systemservices/PCameras.ipdl @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PContent; +include protocol PBackground; + +namespace mozilla { +namespace camera { + +struct CaptureCapability +{ + int width; + int height; + int maxFPS; + int expectedCaptureDelay; + int rawType; + int codecType; + bool interlaced; +}; + +async protocol PCameras +{ + manager PBackground; + +child: + async FrameSizeChange(int capEngine, int cap_id, int w, int h); + // transfers ownership of |buffer| from parent to child + async DeliverFrame(int capEngine, int cap_id, + Shmem buffer, int size, uint32_t time_stamp, + int64_t ntp_time, int64_t render_time); + async ReplyNumberOfCaptureDevices(int numdev); + async ReplyNumberOfCapabilities(int numdev); + async ReplyAllocateCaptureDevice(int numdev); + async ReplyGetCaptureCapability(CaptureCapability cap); + async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id); + async ReplyFailure(); + async ReplySuccess(); + async __delete__(); + +parent: + async NumberOfCaptureDevices(int engine); + async NumberOfCapabilities(int engine, nsCString deviceUniqueIdUTF8); + + async GetCaptureCapability(int engine, nsCString unique_idUTF8, int capability_number); + async GetCaptureDevice(int engine, int num); + + async AllocateCaptureDevice(int engine, nsCString unique_idUTF8); + async ReleaseCaptureDevice(int engine, int numdev); + async StartCapture(int engine, int numdev, CaptureCapability capability); + async StopCapture(int engine, int numdev); + // transfers frame back + async ReleaseFrame(Shmem s); + + // Ask parent to delete us + async AllDone(); +}; + +} // namespace camera +} // namespace mozilla diff --git a/dom/media/systemservices/ShmemPool.cpp b/dom/media/systemservices/ShmemPool.cpp new file mode 100644 index 0000000000..648842ddb4 --- /dev/null +++ b/dom/media/systemservices/ShmemPool.cpp @@ -0,0 +1,129 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/ShmemPool.h" +#include "mozilla/Move.h" + +#undef LOG +#undef LOG_ENABLED +extern PRLogModuleInfo *gCamerasParentLog; +#define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) + +namespace mozilla { + +ShmemPool::ShmemPool(size_t aPoolSize) + : mMutex("mozilla::ShmemPool"), + mPoolFree(aPoolSize) +#ifdef DEBUG + ,mMaxPoolUse(0) +#endif +{ + mShmemPool.SetLength(aPoolSize); +} + +mozilla::ShmemBuffer ShmemPool::GetIfAvailable(size_t aSize) +{ + MutexAutoLock lock(mMutex); + + // Pool is empty, don't block caller. + if (mPoolFree == 0) { + // This isn't initialized, so will be understood as an error. + return ShmemBuffer(); + } + + ShmemBuffer& res = mShmemPool[mPoolFree - 1]; + + if (!res.mInitialized) { + return ShmemBuffer(); + } + + MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?"); + + if (res.mShmem.Size() < aSize) { + return ShmemBuffer(); + } + + mPoolFree--; +#ifdef DEBUG + size_t poolUse = mShmemPool.Length() - mPoolFree; + if (poolUse > mMaxPoolUse) { + mMaxPoolUse = poolUse; + LOG(("Maximum ShmemPool use increased: %d buffers", mMaxPoolUse)); + } +#endif + return Move(res); +} + +template +mozilla::ShmemBuffer ShmemPool::Get(T* aInstance, size_t aSize) +{ + MutexAutoLock lock(mMutex); + + // Pool is empty, don't block caller. + if (mPoolFree == 0) { + // This isn't initialized, so will be understood as an error. + return ShmemBuffer(); + } + + ShmemBuffer res = Move(mShmemPool[mPoolFree - 1]); + + if (!res.mInitialized) { + LOG(("Initiaizing new Shmem in pool")); + aInstance->AllocShmem(aSize, SharedMemory::TYPE_BASIC, &res.mShmem); + res.mInitialized = true; + } + + MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?"); + + // Prepare buffer, increase size if needed (we never shrink as we don't + // maintain seperate sized pools and we don't want to keep reallocating) + if (res.mShmem.Size() < aSize) { + LOG(("Size change/increase in Shmem Pool")); + aInstance->DeallocShmem(res.mShmem); + // this may fail; always check return value + if (!aInstance->AllocShmem(aSize, SharedMemory::TYPE_BASIC, &res.mShmem)) { + LOG(("Failure allocating new size Shmem buffer")); + return ShmemBuffer(); + } + } + + mPoolFree--; + return res; +} + +void ShmemPool::Put(ShmemBuffer&& aShmem) +{ + MutexAutoLock lock(mMutex); + MOZ_ASSERT(mPoolFree < mShmemPool.Length()); + mShmemPool[mPoolFree] = Move(aShmem); + mPoolFree++; +} + +template +void ShmemPool::Cleanup(T* aInstance) +{ + MutexAutoLock lock(mMutex); + for (size_t i = 0; i < mShmemPool.Length(); i++) { + if (mShmemPool[i].mInitialized) { + aInstance->DeallocShmem(mShmemPool[i].Get()); + mShmemPool[i].mInitialized = false; + } + } +} + +ShmemPool::~ShmemPool() +{ +#ifdef DEBUG + for (size_t i = 0; i < mShmemPool.Length(); i++) { + MOZ_ASSERT(!mShmemPool[i].Valid()); + } +#endif +} + +} // namespace mozilla diff --git a/dom/media/systemservices/ShmemPool.h b/dom/media/systemservices/ShmemPool.h new file mode 100644 index 0000000000..c815ef5ab9 --- /dev/null +++ b/dom/media/systemservices/ShmemPool.h @@ -0,0 +1,85 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_ShmemPool_h +#define mozilla_ShmemPool_h + +#include "mozilla/ipc/Shmem.h" +#include "mozilla/Mutex.h" + +namespace mozilla { + +class ShmemPool; + +class ShmemBuffer { +public: + ShmemBuffer() : mInitialized(false) {} + explicit ShmemBuffer(mozilla::ipc::Shmem aShmem) { + mInitialized = true; + mShmem = aShmem; + } + + ShmemBuffer(ShmemBuffer&& rhs) { + mInitialized = rhs.mInitialized; + mShmem = Move(rhs.mShmem); + } + + ShmemBuffer& operator=(ShmemBuffer&& rhs) { + MOZ_ASSERT(&rhs != this, "self-moves are prohibited"); + mInitialized = rhs.mInitialized; + mShmem = Move(rhs.mShmem); + return *this; + } + + // No copies allowed + ShmemBuffer(const ShmemBuffer&) = delete; + ShmemBuffer& operator=(const ShmemBuffer&) = delete; + + bool Valid() { + return mInitialized; + } + + char* GetBytes() { + return mShmem.get(); + } + + mozilla::ipc::Shmem& Get() { + return mShmem; + } + +private: + friend class ShmemPool; + + bool mInitialized; + mozilla::ipc::Shmem mShmem; +}; + +class ShmemPool { +public: + explicit ShmemPool(size_t aPoolSize); + ~ShmemPool(); + // We need to use the allocation/deallocation functions + // of a specific IPC child/parent instance. + template void Cleanup(T* aInstance); + // These 2 differ in what thread they can run on. GetIfAvailable + // can run anywhere but won't allocate if the right size isn't available. + ShmemBuffer GetIfAvailable(size_t aSize); + template ShmemBuffer Get(T* aInstance, size_t aSize); + void Put(ShmemBuffer&& aShmem); + +private: + Mutex mMutex; + size_t mPoolFree; +#ifdef DEBUG + size_t mMaxPoolUse; +#endif + nsTArray mShmemPool; +}; + + +} // namespace mozilla + +#endif // mozilla_ShmemPool_h diff --git a/dom/media/systemservices/moz.build b/dom/media/systemservices/moz.build index 8b9892cca6..1da2910fca 100644 --- a/dom/media/systemservices/moz.build +++ b/dom/media/systemservices/moz.build @@ -5,15 +5,25 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. if CONFIG['MOZ_WEBRTC']: - EXPORTS += ['LoadManager.h', - 'LoadManagerFactory.h', - 'LoadMonitor.h', + EXPORTS += [ + 'CamerasChild.h', + 'CamerasParent.h', + 'CamerasUtils.h', + 'LoadManager.h', + 'LoadManagerFactory.h', + 'LoadMonitor.h', ] - UNIFIED_SOURCES += ['LoadManager.cpp', - 'LoadManagerFactory.cpp', - 'LoadMonitor.cpp', + UNIFIED_SOURCES += [ + 'CamerasChild.cpp', + 'CamerasParent.cpp', + 'CamerasUtils.cpp', + 'LoadManager.cpp', + 'LoadManagerFactory.cpp', + 'LoadMonitor.cpp', + 'ShmemPool.cpp', ] LOCAL_INCLUDES += [ + '/media/webrtc/signaling', '/media/webrtc/trunk', ] @@ -37,6 +47,11 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk': ] ] +if CONFIG['_MSC_VER']: + DEFINES['__PRETTY_FUNCTION__'] = '__FUNCSIG__' + +EXPORTS.mozilla += ['ShmemPool.h',] + EXPORTS.mozilla.media += ['MediaChild.h', 'MediaParent.h', 'MediaSystemResourceClient.h', @@ -46,18 +61,20 @@ EXPORTS.mozilla.media += ['MediaChild.h', 'MediaSystemResourceMessageUtils.h', 'MediaSystemResourceService.h', 'MediaSystemResourceTypes.h', - 'MediaUtils.h' + 'MediaUtils.h', ] -UNIFIED_SOURCES += ['MediaChild.cpp', - 'MediaParent.cpp', - 'MediaSystemResourceClient.cpp', - 'MediaSystemResourceManager.cpp', - 'MediaSystemResourceManagerChild.cpp', - 'MediaSystemResourceManagerParent.cpp', - 'MediaSystemResourceService.cpp', - 'MediaUtils.cpp', +UNIFIED_SOURCES += [ + 'MediaChild.cpp', + 'MediaParent.cpp', + 'MediaSystemResourceClient.cpp', + 'MediaSystemResourceManager.cpp', + 'MediaSystemResourceManagerChild.cpp', + 'MediaSystemResourceManagerParent.cpp', + 'MediaSystemResourceService.cpp', + 'MediaUtils.cpp', ] IPDL_SOURCES += [ + 'PCameras.ipdl', 'PMedia.ipdl', 'PMediaSystemResourceManager.ipdl', ] diff --git a/dom/media/test/mochitest.ini b/dom/media/test/mochitest.ini index da05b8ade8..7dadb25c7d 100644 --- a/dom/media/test/mochitest.ini +++ b/dom/media/test/mochitest.ini @@ -392,7 +392,6 @@ tags=msg tags=msg [test_mediarecorder_record_gum_video_timeslice.html] tags=msg -skip-if = buildapp == 'b2g' || toolkit == 'android' # mimetype check, bug 969289 [test_mediarecorder_record_immediate_stop.html] tags=msg [test_mediarecorder_record_no_timeslice.html] diff --git a/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html b/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html index 142aabaed5..efdd983037 100644 --- a/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html +++ b/dom/media/test/test_mediarecorder_record_gum_video_timeslice.html @@ -40,12 +40,10 @@ function startTest() { is(evt.type, 'dataavailable', 'Event type should dataavailable'); ok(evt.data.size >= 0, - 'Blob data size received is greater than or equal to zero'); - - is(evt.data.type, 'video/webm', - 'Blob data received should have type = ' + 'video/webm'); - is(mediaRecorder.mimeType, 'video/webm', - 'Mime type in ondataavailable = ' + mediaRecorder.mimeType); + 'Blob data size ' + evt.data.size + ' received is greater than or equal to zero'); + is(mediaRecorder.mimeType, evt.data.type, + 'Mime type in MediaRecorder and ondataavailable : ' + + mediaRecorder.mimeType + ' == ' + evt.data.type); // We'll stop recording upon the 1st blob being received if (dataAvailableCount === 1) { diff --git a/dom/media/webaudio/AudioContext.cpp b/dom/media/webaudio/AudioContext.cpp index 085b2a88bf..3fa6d5a212 100644 --- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -944,8 +944,11 @@ AudioContext::UpdateNodeCount(int32_t aDelta) bool firstNode = mNodeCount == 0; mNodeCount += aDelta; MOZ_ASSERT(mNodeCount >= 0); - // mDestinationNode may be null when we're destroying nodes unlinked by CC - if (!firstNode && mDestination) { + // mDestinationNode may be null when we're destroying nodes unlinked by CC. + // Skipping unnecessary calls after shutdown avoids RunInStableState events + // getting stuck in CycleCollectedJSRuntime during final cycle collection + // (bug 1200514). + if (!firstNode && mDestination && !mIsShutDown) { mDestination->SetIsOnlyNodeForContext(mNodeCount == 1); } } diff --git a/dom/media/webaudio/BufferDecoder.cpp b/dom/media/webaudio/BufferDecoder.cpp index 88d8d7dcf5..d9115e5170 100644 --- a/dom/media/webaudio/BufferDecoder.cpp +++ b/dom/media/webaudio/BufferDecoder.cpp @@ -45,13 +45,6 @@ BufferDecoder::GetReentrantMonitor() return mReentrantMonitor; } -bool -BufferDecoder::IsShutdown() const -{ - // BufferDecoder cannot be shut down. - return false; -} - bool BufferDecoder::OnStateMachineTaskQueue() const { diff --git a/dom/media/webaudio/BufferDecoder.h b/dom/media/webaudio/BufferDecoder.h index 736b49169c..24ab1baefe 100644 --- a/dom/media/webaudio/BufferDecoder.h +++ b/dom/media/webaudio/BufferDecoder.h @@ -33,8 +33,6 @@ public: virtual ReentrantMonitor& GetReentrantMonitor() final override; - virtual bool IsShutdown() const final override; - virtual bool OnStateMachineTaskQueue() const final override; virtual bool OnDecodeTaskQueue() const final override; diff --git a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp index 68b456b9e6..9cd1acff4e 100644 --- a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp @@ -162,6 +162,49 @@ MediaEngineCameraVideoSource::LogConstraints( c.mFrameRate.mIdeal.WasPassed()? c.mFrameRate.mIdeal.Value() : 0)); } +void +MediaEngineCameraVideoSource::LogCapability(const char* aHeader, + const webrtc::CaptureCapability &aCapability, uint32_t aDistance) +{ + // RawVideoType and VideoCodecType media/webrtc/trunk/webrtc/common_types.h + static const char* const types[] = { + "I420", + "YV12", + "YUY2", + "UYVY", + "IYUV", + "ARGB", + "RGB24", + "RGB565", + "ARGB4444", + "ARGB1555", + "MJPEG", + "NV12", + "NV21", + "BGRA", + "Unknown type" + }; + + static const char* const codec[] = { + "VP8", + "VP9", + "H264", + "I420", + "RED", + "ULPFEC", + "Generic codec", + "Unknown codec" + }; + + LOG(("%s: %4u x %4u x %2u maxFps, %s, %s. Distance = %lu", + aHeader, aCapability.width, aCapability.height, aCapability.maxFPS, + types[std::min(std::max(uint32_t(0), uint32_t(aCapability.rawType)), + uint32_t(sizeof(types) / sizeof(*types) - 1))], + codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.codecType)), + uint32_t(sizeof(codec) / sizeof(*codec) - 1))], + aDistance)); +} + bool MediaEngineCameraVideoSource::ChooseCapability( const MediaTrackConstraints &aConstraints, @@ -195,6 +238,7 @@ MediaEngineCameraVideoSource::ChooseCapability( webrtc::CaptureCapability cap; GetCapability(candidate.mIndex, cap); candidate.mDistance = GetFitnessDistance(cap, aConstraints, false, aDeviceId); + LogCapability("Capability", cap, candidate.mDistance); if (candidate.mDistance == UINT32_MAX) { candidateSet.RemoveElementAt(i); } else { @@ -234,6 +278,7 @@ MediaEngineCameraVideoSource::ChooseCapability( // Any remaining multiples all have the same distance. A common case of this // occurs when no ideal is specified. Lean toward defaults. + uint32_t sameDistance = candidateSet[0].mDistance; { MediaTrackConstraintSet prefs; prefs.mWidth.SetAsLong() = aPrefs.GetWidth(); @@ -268,9 +313,7 @@ MediaEngineCameraVideoSource::ChooseCapability( GetCapability(candidateSet[0].mIndex, mCapability); } - LOG(("chose cap %dx%d @%dfps codec %d raw %d", - mCapability.width, mCapability.height, mCapability.maxFPS, - mCapability.codecType, mCapability.rawType)); + LogCapability("Chosen capability", mCapability, sameDistance); return true; } diff --git a/dom/media/webrtc/MediaEngineCameraVideoSource.h b/dom/media/webrtc/MediaEngineCameraVideoSource.h index d5edb3111b..8b822a88ad 100644 --- a/dom/media/webrtc/MediaEngineCameraVideoSource.h +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.h @@ -30,7 +30,6 @@ public: , mHasDirectListeners(false) , mCaptureIndex(aIndex) , mTrackID(0) - , mFps(-1) {} @@ -50,10 +49,6 @@ public: return false; } - virtual const dom::MediaSourceEnum GetMediaSource() override { - return dom::MediaSourceEnum::Camera; - } - virtual nsresult TakePhoto(PhotoCallback* aCallback) override { return NS_ERROR_NOT_IMPLEMENTED; @@ -89,6 +84,9 @@ protected: static void TrimLessFitCandidates(CapabilitySet& set); static void LogConstraints(const dom::MediaTrackConstraintSet& aConstraints, bool aAdvanced); +static void LogCapability(const char* aHeader, + const webrtc::CaptureCapability &aCapability, + uint32_t aDistance); virtual size_t NumCapabilities(); virtual void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut); bool ChooseCapability(const dom::MediaTrackConstraints &aConstraints, @@ -119,7 +117,6 @@ protected: bool mHasDirectListeners; int mCaptureIndex; TrackID mTrackID; - int mFps; // Track rate (30 fps by default) webrtc::CaptureCapability mCapability; // Doesn't work on OS X. diff --git a/dom/media/webrtc/MediaEngineGonkVideoSource.h b/dom/media/webrtc/MediaEngineGonkVideoSource.h index 3145803ec1..2f57fb5465 100644 --- a/dom/media/webrtc/MediaEngineGonkVideoSource.h +++ b/dom/media/webrtc/MediaEngineGonkVideoSource.h @@ -70,6 +70,9 @@ public: SourceMediaStream* aSource, TrackID aId, StreamTime aDesiredTime) override; + virtual const dom::MediaSourceEnum GetMediaSource() override { + return dom::MediaSourceEnum::Camera; + } void OnHardwareStateChange(HardwareState aState, nsresult aReason) override; void GetRotation(); diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp new file mode 100644 index 0000000000..2193949373 --- /dev/null +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -0,0 +1,398 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaEngineRemoteVideoSource.h" + +#include "mozilla/RefPtr.h" +#include "VideoUtils.h" +#include "nsIPrefService.h" +#include "MediaTrackConstraints.h" +#include "CamerasChild.h" + +extern PRLogModuleInfo* GetMediaManagerLog(); +#define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) +#define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) + +namespace mozilla { + +using dom::ConstrainLongRange; + +NS_IMPL_ISUPPORTS0(MediaEngineRemoteVideoSource) + +MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource( + int aIndex, mozilla::camera::CaptureEngine aCapEngine, + dom::MediaSourceEnum aMediaSource, const char* aMonitorName) + : MediaEngineCameraVideoSource(aIndex, aMonitorName), + mMediaSource(aMediaSource), + mCapEngine(aCapEngine) +{ + MOZ_ASSERT(aMediaSource != dom::MediaSourceEnum::Other); + Init(); +} + +void +MediaEngineRemoteVideoSource::Init() +{ + LOG((__PRETTY_FUNCTION__)); + char deviceName[kMaxDeviceNameLength]; + char uniqueId[kMaxUniqueIdLength]; + if (mozilla::camera::GetCaptureDevice(mCapEngine, + mCaptureIndex, + deviceName, kMaxDeviceNameLength, + uniqueId, kMaxUniqueIdLength)) { + LOG(("Error initializing RemoteVideoSource (GetCaptureDevice)")); + return; + } + + SetName(NS_ConvertUTF8toUTF16(deviceName)); + SetUUID(uniqueId); + + mInitDone = true; + + return; +} + +void +MediaEngineRemoteVideoSource::Shutdown() +{ + LOG((__PRETTY_FUNCTION__)); + if (!mInitDone) { + return; + } + if (mState == kStarted) { + SourceMediaStream *source; + bool empty; + + while (1) { + { + MonitorAutoLock lock(mMonitor); + empty = mSources.IsEmpty(); + if (empty) { + break; + } + source = mSources[0]; + } + Stop(source, kVideoTrack); // XXX change to support multiple tracks + } + MOZ_ASSERT(mState == kStopped); + } + + if (mState == kAllocated || mState == kStopped) { + Deallocate(); + } + + mozilla::camera::Shutdown(); + + mState = kReleased; + mInitDone = false; + return; +} + +nsresult +MediaEngineRemoteVideoSource::Allocate(const dom::MediaTrackConstraints& aConstraints, + const MediaEnginePrefs& aPrefs, + const nsString& aDeviceId) +{ + LOG((__PRETTY_FUNCTION__)); + + if (mState == kReleased && mInitDone) { + // Note: if shared, we don't allow a later opener to affect the resolution. + // (This may change depending on spec changes for Constraints/settings) + + if (!ChooseCapability(aConstraints, aPrefs, aDeviceId)) { + return NS_ERROR_UNEXPECTED; + } + + if (mozilla::camera::AllocateCaptureDevice(mCapEngine, + GetUUID().get(), + kMaxUniqueIdLength, mCaptureIndex)) { + return NS_ERROR_FAILURE; + } + mState = kAllocated; + LOG(("Video device %d allocated", mCaptureIndex)); + } else if (MOZ_LOG_TEST(GetMediaManagerLog(), mozilla::LogLevel::Debug)) { + MonitorAutoLock lock(mMonitor); + if (mSources.IsEmpty()) { + LOG(("Video device %d reallocated", mCaptureIndex)); + } else { + LOG(("Video device %d allocated shared", mCaptureIndex)); + } + } + + return NS_OK; +} + +nsresult +MediaEngineRemoteVideoSource::Deallocate() +{ + LOG((__FUNCTION__)); + bool empty; + { + MonitorAutoLock lock(mMonitor); + empty = mSources.IsEmpty(); + } + if (empty) { + if (mState != kStopped && mState != kAllocated) { + return NS_ERROR_FAILURE; + } + mozilla::camera::ReleaseCaptureDevice(mCapEngine, mCaptureIndex); + mState = kReleased; + LOG(("Video device %d deallocated", mCaptureIndex)); + } else { + LOG(("Video device %d deallocated but still in use", mCaptureIndex)); + } + return NS_OK; +} + +nsresult +MediaEngineRemoteVideoSource::Start(SourceMediaStream* aStream, TrackID aID) +{ + LOG((__PRETTY_FUNCTION__)); + if (!mInitDone || !aStream) { + LOG(("No stream or init not done")); + return NS_ERROR_FAILURE; + } + + { + MonitorAutoLock lock(mMonitor); + mSources.AppendElement(aStream); + } + + aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED); + + if (mState == kStarted) { + return NS_OK; + } + mImageContainer = layers::LayerManager::CreateImageContainer(); + + mState = kStarted; + mTrackID = aID; + + if (mozilla::camera::StartCapture(mCapEngine, + mCaptureIndex, mCapability, this)) { + LOG(("StartCapture failed")); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +nsresult +MediaEngineRemoteVideoSource::Stop(mozilla::SourceMediaStream* aSource, + mozilla::TrackID aID) +{ + LOG((__PRETTY_FUNCTION__)); + { + MonitorAutoLock lock(mMonitor); + + if (!mSources.RemoveElement(aSource)) { + // Already stopped - this is allowed + return NS_OK; + } + + aSource->EndTrack(aID); + + if (!mSources.IsEmpty()) { + return NS_OK; + } + if (mState != kStarted) { + return NS_ERROR_FAILURE; + } + + mState = kStopped; + // Drop any cached image so we don't start with a stale image on next + // usage + mImage = nullptr; + } + + mozilla::camera::StopCapture(mCapEngine, mCaptureIndex); + + return NS_OK; +} + +void +MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph, + SourceMediaStream* aSource, + TrackID aID, StreamTime aDesiredTime) +{ + VideoSegment segment; + + MonitorAutoLock lock(mMonitor); + StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID); + + if (delta > 0) { + // nullptr images are allowed + AppendToTrack(aSource, mImage, aID, delta); + } +} + +int +MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h, + unsigned int streams) +{ + mWidth = w; + mHeight = h; + LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u", w, h)); + return 0; +} + +int +MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time, + void *handle) +{ + // Check for proper state. + if (mState != kStarted) { + LOG(("DeliverFrame: video not started")); + return 0; + } + + if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) { + MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!"); + return 0; + } + + // Create a video frame and append it to the track. + nsRefPtr image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); + layers::PlanarYCbCrImage* videoImage = static_cast(image.get()); + + uint8_t* frame = static_cast (buffer); + const uint8_t lumaBpp = 8; + const uint8_t chromaBpp = 4; + + // Take lots of care to round up! + layers::PlanarYCbCrData data; + data.mYChannel = frame; + data.mYSize = IntSize(mWidth, mHeight); + data.mYStride = (mWidth * lumaBpp + 7)/ 8; + data.mCbCrStride = (mWidth * chromaBpp + 7) / 8; + data.mCbChannel = frame + mHeight * data.mYStride; + data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride; + data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2); + data.mPicX = 0; + data.mPicY = 0; + data.mPicSize = IntSize(mWidth, mHeight); + data.mStereoMode = StereoMode::MONO; + + videoImage->SetData(data); + +#ifdef DEBUG + static uint32_t frame_num = 0; + LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++, + mWidth, mHeight, time_stamp, ntp_time, render_time)); +#endif + + // we don't touch anything in 'this' until here (except for snapshot, + // which has it's own lock) + MonitorAutoLock lock(mMonitor); + + // implicitly releases last image + mImage = image.forget(); + + // Push the frame into the MSG with a minimal duration. This will likely + // mean we'll still get NotifyPull calls which will then return the same + // frame again with a longer duration. However, this means we won't + // fail to get the frame in and drop frames. + + // XXX The timestamp for the frame should be based on the Capture time, + // not the MSG time, and MSG should never, ever block on a (realtime) + // video frame (or even really for streaming - audio yes, video probably no). + // Note that MediaPipeline currently ignores the timestamps from MSG + uint32_t len = mSources.Length(); + for (uint32_t i = 0; i < len; i++) { + if (mSources[i]) { + AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration + } + } + + return 0; +} + +size_t +MediaEngineRemoteVideoSource::NumCapabilities() +{ + int num = mozilla::camera::NumberOfCapabilities(mCapEngine, GetUUID().get()); + if (num > 0) { + return num; + } + + switch(mMediaSource) { + case dom::MediaSourceEnum::Camera: +#ifdef XP_MACOSX + // Mac doesn't support capabilities. + // + // Hardcode generic desktop capabilities modeled on OSX camera. + // Note: Values are empirically picked to be OSX friendly, as on OSX, values + // other than these cause the source to not produce. + + if (mHardcodedCapabilities.IsEmpty()) { + for (int i = 0; i < 9; i++) { + webrtc::CaptureCapability c; + c.width = 1920 - i*128; + c.height = 1080 - i*72; + c.maxFPS = 30; + mHardcodedCapabilities.AppendElement(c); + } + for (int i = 0; i < 16; i++) { + webrtc::CaptureCapability c; + c.width = 640 - i*40; + c.height = 480 - i*30; + c.maxFPS = 30; + mHardcodedCapabilities.AppendElement(c); + } + } + break; +#endif + default: + webrtc::CaptureCapability c; + // The default for devices that don't return discrete capabilities: treat + // them as supporting all capabilities orthogonally. E.g. screensharing. + c.width = 0; // 0 = accept any value + c.height = 0; + c.maxFPS = 0; + mHardcodedCapabilities.AppendElement(c); + break; + } + + return mHardcodedCapabilities.Length(); +} + +void +MediaEngineRemoteVideoSource::GetCapability(size_t aIndex, + webrtc::CaptureCapability& aOut) +{ + if (!mHardcodedCapabilities.IsEmpty()) { + MediaEngineCameraVideoSource::GetCapability(aIndex, aOut); + } + mozilla::camera::GetCaptureCapability(mCapEngine, + GetUUID().get(), + aIndex, + aOut); +} + +void MediaEngineRemoteVideoSource::Refresh(int aIndex) { + // NOTE: mCaptureIndex might have changed when allocated! + // Use aIndex to update information, but don't change mCaptureIndex!! + // Caller looked up this source by uniqueId, so it shouldn't change + char deviceName[kMaxDeviceNameLength]; + char uniqueId[kMaxUniqueIdLength]; + + if (mozilla::camera::GetCaptureDevice(mCapEngine, + aIndex, + deviceName, sizeof(deviceName), + uniqueId, sizeof(uniqueId))) { + return; + } + + SetName(NS_ConvertUTF8toUTF16(deviceName)); +#ifdef DEBUG + MOZ_ASSERT(GetUUID().Equals(uniqueId)); +#endif +} + +} diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.h b/dom/media/webrtc/MediaEngineRemoteVideoSource.h new file mode 100644 index 0000000000..6e373225d8 --- /dev/null +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.h @@ -0,0 +1,101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_ +#define MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_ + +#include "prcvar.h" +#include "prthread.h" +#include "nsIThread.h" +#include "nsIRunnable.h" + +#include "mozilla/Mutex.h" +#include "mozilla/Monitor.h" +#include "nsCOMPtr.h" +#include "nsThreadUtils.h" +#include "DOMMediaStream.h" +#include "nsDirectoryServiceDefs.h" +#include "nsComponentManagerUtils.h" + +#include "VideoUtils.h" +#include "MediaEngineCameraVideoSource.h" +#include "VideoSegment.h" +#include "AudioSegment.h" +#include "StreamBuffer.h" +#include "MediaStreamGraph.h" + +#include "MediaEngineWrapper.h" +#include "mozilla/dom/MediaStreamTrackBinding.h" + +// WebRTC library includes follow +#include "webrtc/common.h" +#include "webrtc/video_engine/include/vie_capture.h" +#include "webrtc/video_engine/include/vie_render.h" +#include "CamerasChild.h" + +#include "NullTransport.h" + +namespace mozilla { + +/** + * The WebRTC implementation of the MediaEngine interface. + */ +class MediaEngineRemoteVideoSource : public MediaEngineCameraVideoSource, + public webrtc::ExternalRenderer +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + + // ExternalRenderer + virtual int FrameSizeChange(unsigned int w, unsigned int h, + unsigned int streams) override; + virtual int DeliverFrame(unsigned char* buffer, + int size, + uint32_t time_stamp, + int64_t ntp_time, + int64_t render_time, + void *handle) override; + virtual bool IsTextureSupported() override { return false; }; + + // MediaEngineCameraVideoSource + MediaEngineRemoteVideoSource(int aIndex, mozilla::camera::CaptureEngine aCapEngine, + dom::MediaSourceEnum aMediaSource, + const char* aMonitorName = "RemoteVideo.Monitor"); + + virtual nsresult Allocate(const dom::MediaTrackConstraints& aConstraints, + const MediaEnginePrefs& aPrefs, + const nsString& aDeviceId) override; + virtual nsresult Deallocate() override;; + virtual nsresult Start(SourceMediaStream*, TrackID) override; + virtual nsresult Stop(SourceMediaStream*, TrackID) override; + virtual void NotifyPull(MediaStreamGraph* aGraph, + SourceMediaStream* aSource, + TrackID aId, + StreamTime aDesiredTime) override; + virtual const dom::MediaSourceEnum GetMediaSource() override { + return mMediaSource; + } + + void Refresh(int aIndex); + + virtual void Shutdown() override; + +protected: + ~MediaEngineRemoteVideoSource() { Shutdown(); } + +private: + // Initialize the needed Video engine interfaces. + void Init(); + size_t NumCapabilities() override; + void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) override; + + dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen) + mozilla::camera::CaptureEngine mCapEngine; +}; + +} + +#endif /* MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_ */ diff --git a/dom/media/webrtc/MediaEngineWebRTC.cpp b/dom/media/webrtc/MediaEngineWebRTC.cpp index 207e97acb3..96b4193b47 100644 --- a/dom/media/webrtc/MediaEngineWebRTC.cpp +++ b/dom/media/webrtc/MediaEngineWebRTC.cpp @@ -1,9 +1,12 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "nsIPrefService.h" #include "nsIPrefBranch.h" +#include "CamerasUtils.h" #include "CSFLog.h" #include "prenv.h" @@ -23,6 +26,8 @@ GetUserMediaLog() #include "ImageContainer.h" #include "nsIComponentRegistrar.h" #include "MediaEngineTabVideoSource.h" +#include "MediaEngineRemoteVideoSource.h" +#include "CamerasChild.h" #include "nsITabSource.h" #include "MediaTrackConstraints.h" @@ -42,18 +47,9 @@ GetUserMediaLog() namespace mozilla { MediaEngineWebRTC::MediaEngineWebRTC(MediaEnginePrefs &aPrefs) - : mMutex("mozilla::MediaEngineWebRTC") - , mScreenEngine(nullptr) - , mBrowserEngine(nullptr) - , mWinEngine(nullptr) - , mAppEngine(nullptr) - , mVideoEngine(nullptr) - , mVoiceEngine(nullptr) - , mVideoEngineInit(false) - , mAudioEngineInit(false) - , mScreenEngineInit(false) - , mBrowserEngineInit(false) - , mAppEngineInit(false) + : mMutex("mozilla::MediaEngineWebRTC"), + mVoiceEngine(nullptr), + mAudioEngineInit(false) { #ifndef MOZ_B2G_CAMERA nsCOMPtr compMgr; @@ -122,11 +118,7 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource, return; #else - ScopedCustomReleasePtr ptrViEBase; - ScopedCustomReleasePtr ptrViECapture; - webrtc::Config configSet; - webrtc::VideoEngine *videoEngine = nullptr; - bool *videoEngineInit = nullptr; + mozilla::camera::CaptureEngine capEngine = mozilla::camera::InvalidEngine; #ifdef MOZ_WIDGET_ANDROID // get the JVM @@ -140,74 +132,24 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource, switch (aMediaSource) { case dom::MediaSourceEnum::Window: - mWinEngineConfig.Set( - new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Window)); - if (!mWinEngine) { - if (!(mWinEngine = webrtc::VideoEngine::Create(mWinEngineConfig))) { - return; - } - } - videoEngine = mWinEngine; - videoEngineInit = &mWinEngineInit; + capEngine = mozilla::camera::WinEngine; break; case dom::MediaSourceEnum::Application: - mAppEngineConfig.Set( - new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Application)); - if (!mAppEngine) { - if (!(mAppEngine = webrtc::VideoEngine::Create(mAppEngineConfig))) { - return; - } - } - videoEngine = mAppEngine; - videoEngineInit = &mAppEngineInit; + capEngine = mozilla::camera::AppEngine; break; case dom::MediaSourceEnum::Screen: - mScreenEngineConfig.Set( - new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen)); - if (!mScreenEngine) { - if (!(mScreenEngine = webrtc::VideoEngine::Create(mScreenEngineConfig))) { - return; - } - } - videoEngine = mScreenEngine; - videoEngineInit = &mScreenEngineInit; + capEngine = mozilla::camera::ScreenEngine; break; case dom::MediaSourceEnum::Browser: - mBrowserEngineConfig.Set( - new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Browser)); - if (!mBrowserEngine) { - if (!(mBrowserEngine = webrtc::VideoEngine::Create(mBrowserEngineConfig))) { - return; - } - } - videoEngine = mBrowserEngine; - videoEngineInit = &mBrowserEngineInit; + capEngine = mozilla::camera::BrowserEngine; break; case dom::MediaSourceEnum::Camera: - // fall through - default: - if (!mVideoEngine) { - if (!(mVideoEngine = webrtc::VideoEngine::Create())) { - return; - } - } - videoEngine = mVideoEngine; - videoEngineInit = &mVideoEngineInit; + capEngine = mozilla::camera::CameraEngine; + break; + default: + // BOOM + MOZ_CRASH("No valid video engine"); break; - } - - ptrViEBase = webrtc::ViEBase::GetInterface(videoEngine); - if (!ptrViEBase) { - return; - } - if (ptrViEBase->Init() < 0) { - return; - } - *videoEngineInit = true; - - ptrViECapture = webrtc::ViECapture::GetInterface(videoEngine); - if (!ptrViECapture) { - return; } /** @@ -218,7 +160,8 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource, * for a given instance of the engine. Likewise, if a device was plugged out, * mVideoSources must be updated. */ - int num = ptrViECapture->NumberOfCaptureDevices(); + int num; + num = mozilla::camera::NumberOfCaptureDevices(capEngine); if (num <= 0) { return; } @@ -230,27 +173,29 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource, // paranoia deviceName[0] = '\0'; uniqueId[0] = '\0'; - int error = ptrViECapture->GetCaptureDevice(i, deviceName, - sizeof(deviceName), uniqueId, - sizeof(uniqueId)); + int error; + + error = mozilla::camera::GetCaptureDevice(capEngine, + i, deviceName, + sizeof(deviceName), uniqueId, + sizeof(uniqueId)); if (error) { - LOG((" VieCapture:GetCaptureDevice: Failed %d", - ptrViEBase->LastError() )); + LOG(("camera:GetCaptureDevice: Failed %d", error )); continue; } #ifdef DEBUG LOG((" Capture Device Index %d, Name %s", i, deviceName)); webrtc::CaptureCapability cap; - int numCaps = ptrViECapture->NumberOfCapabilities(uniqueId, - MediaEngineSource::kMaxUniqueIdLength); + int numCaps = mozilla::camera::NumberOfCapabilities(capEngine, + uniqueId); LOG(("Number of Capabilities %d", numCaps)); for (int j = 0; j < numCaps; j++) { - if (ptrViECapture->GetCaptureCapability(uniqueId, - MediaEngineSource::kMaxUniqueIdLength, - j, cap ) != 0 ) { - break; + if (mozilla::camera::GetCaptureCapability(capEngine, + uniqueId, + j, cap ) != 0 ) { + break; } LOG(("type=%d width=%d height=%d maxFPS=%d", cap.rawType, cap.width, cap.height, cap.maxFPS )); @@ -267,10 +212,10 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource, NS_ConvertUTF8toUTF16 uuid(uniqueId); if (mVideoSources.Get(uuid, getter_AddRefs(vSource))) { // We've already seen this device, just refresh and append. - static_cast(vSource.get())->Refresh(i); + static_cast(vSource.get())->Refresh(i); aVSources->AppendElement(vSource.get()); } else { - vSource = new MediaEngineWebRTCVideoSource(videoEngine, i, aMediaSource); + vSource = new MediaEngineRemoteVideoSource(i, capEngine, aMediaSource); mVideoSources.Put(uuid, vSource); // Hashtable takes ownership. aVSources->AppendElement(vSource); } @@ -414,40 +359,14 @@ MediaEngineWebRTC::Shutdown() mVideoSources.Clear(); mAudioSources.Clear(); - // Clear callbacks before we go away since the engines may outlive us - if (mVideoEngine) { - mVideoEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mVideoEngine); - } - - if (mScreenEngine) { - mScreenEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mScreenEngine); - } - if (mWinEngine) { - mWinEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mWinEngine); - } - if (mBrowserEngine) { - mBrowserEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mBrowserEngine); - } - if (mAppEngine) { - mAppEngine->SetTraceCallback(nullptr); - webrtc::VideoEngine::Delete(mAppEngine); - } - if (mVoiceEngine) { mVoiceEngine->SetTraceCallback(nullptr); webrtc::VoiceEngine::Delete(mVoiceEngine); } - mVideoEngine = nullptr; mVoiceEngine = nullptr; - mScreenEngine = nullptr; - mWinEngine = nullptr; - mBrowserEngine = nullptr; - mAppEngine = nullptr; + + mozilla::camera::Shutdown(); if (mThread) { mThread->Shutdown(); diff --git a/dom/media/webrtc/MediaEngineWebRTC.h b/dom/media/webrtc/MediaEngineWebRTC.h index 112c5134d6..2ec0797059 100644 --- a/dom/media/webrtc/MediaEngineWebRTC.h +++ b/dom/media/webrtc/MediaEngineWebRTC.h @@ -48,91 +48,13 @@ #include "webrtc/video_engine/include/vie_codec.h" #include "webrtc/video_engine/include/vie_render.h" #include "webrtc/video_engine/include/vie_capture.h" +#include "CamerasChild.h" #include "NullTransport.h" #include "AudioOutputObserver.h" namespace mozilla { -/** - * The WebRTC implementation of the MediaEngine interface. - */ -class MediaEngineWebRTCVideoSource : public MediaEngineCameraVideoSource - , public webrtc::ExternalRenderer -{ -public: - NS_DECL_THREADSAFE_ISUPPORTS - - // ViEExternalRenderer. - virtual int FrameSizeChange(unsigned int w, unsigned int h, unsigned int streams) override; - virtual int DeliverFrame(unsigned char* buffer, - int size, - uint32_t time_stamp, - int64_t ntp_time_ms, - int64_t render_time, - void *handle) override; - /** - * Does DeliverFrame() support a null buffer and non-null handle - * (video texture)? - * XXX Investigate! Especially for Android/B2G - */ - virtual bool IsTextureSupported() override { return false; } - - MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex, - dom::MediaSourceEnum aMediaSource = dom::MediaSourceEnum::Camera) - : MediaEngineCameraVideoSource(aIndex, "WebRTCCamera.Monitor") - , mVideoEngine(aVideoEnginePtr) - , mMinFps(-1) - , mMediaSource(aMediaSource) - { - MOZ_ASSERT(aVideoEnginePtr); - MOZ_ASSERT(aMediaSource != dom::MediaSourceEnum::Other); - Init(); - } - - virtual nsresult Allocate(const dom::MediaTrackConstraints& aConstraints, - const MediaEnginePrefs& aPrefs, - const nsString& aDeviceId) override; - virtual nsresult Deallocate() override; - virtual nsresult Start(SourceMediaStream*, TrackID) override; - virtual nsresult Stop(SourceMediaStream*, TrackID) override; - virtual void NotifyPull(MediaStreamGraph* aGraph, - SourceMediaStream* aSource, - TrackID aId, - StreamTime aDesiredTime) override; - - virtual const dom::MediaSourceEnum GetMediaSource() override { - return mMediaSource; - } - virtual nsresult TakePhoto(PhotoCallback* aCallback) override - { - return NS_ERROR_NOT_IMPLEMENTED; - } - - void Refresh(int aIndex); - - virtual void Shutdown() override; - -protected: - ~MediaEngineWebRTCVideoSource() { Shutdown(); } - -private: - // Initialize the needed Video engine interfaces. - void Init(); - - // Engine variables. - webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free. - ScopedCustomReleasePtr mViEBase; - ScopedCustomReleasePtr mViECapture; - ScopedCustomReleasePtr mViERender; - - int mMinFps; // Min rate we want to accept - dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen) - - size_t NumCapabilities() override; - void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) override; -}; - class MediaEngineWebRTCAudioCaptureSource : public MediaEngineAudioSource { public: @@ -266,10 +188,10 @@ public: int16_t audio10ms[], int length, int samplingFreq, bool isStereo) override; - NS_DECL_THREADSAFE_ISUPPORTS - virtual void Shutdown() override; + NS_DECL_THREADSAFE_ISUPPORTS + protected: ~MediaEngineWebRTCMicrophoneSource() { Shutdown(); } @@ -330,29 +252,11 @@ private: nsCOMPtr mThread; + // gUM runnables can e.g. Enumerate from multiple threads Mutex mMutex; - - // protected with mMutex: - webrtc::VideoEngine* mScreenEngine; - webrtc::VideoEngine* mBrowserEngine; - webrtc::VideoEngine* mWinEngine; - webrtc::VideoEngine* mAppEngine; - webrtc::VideoEngine* mVideoEngine; webrtc::VoiceEngine* mVoiceEngine; - - // specialized configurations - webrtc::Config mAppEngineConfig; - webrtc::Config mWinEngineConfig; - webrtc::Config mScreenEngineConfig; - webrtc::Config mBrowserEngineConfig; - - // Need this to avoid unneccesary WebRTC calls while enumerating. - bool mVideoEngineInit; bool mAudioEngineInit; - bool mScreenEngineInit; - bool mBrowserEngineInit; - bool mWinEngineInit; - bool mAppEngineInit; + bool mHasTabVideoSource; // Store devices we've already seen in a hashtable for quick return. diff --git a/dom/media/webrtc/MediaEngineWebRTCVideo.cpp b/dom/media/webrtc/MediaEngineWebRTCVideo.cpp deleted file mode 100644 index 32f1d4e5c6..0000000000 --- a/dom/media/webrtc/MediaEngineWebRTCVideo.cpp +++ /dev/null @@ -1,449 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "MediaEngineWebRTC.h" -#include "Layers.h" -#include "ImageTypes.h" -#include "ImageContainer.h" -#include "mozilla/layers/GrallocTextureClient.h" -#include "nsMemory.h" -#include "mtransport/runnable_utils.h" -#include "MediaTrackConstraints.h" - -namespace mozilla { - -using namespace mozilla::gfx; -using dom::ConstrainLongRange; -using dom::ConstrainDoubleRange; -using dom::MediaTrackConstraintSet; - -extern PRLogModuleInfo* GetMediaManagerLog(); -#define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) -#define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) - -/** - * Webrtc video source. - */ - -NS_IMPL_ISUPPORTS0(MediaEngineWebRTCVideoSource) - -int -MediaEngineWebRTCVideoSource::FrameSizeChange( - unsigned int w, unsigned int h, unsigned int streams) -{ - mWidth = w; - mHeight = h; - LOG(("Video FrameSizeChange: %ux%u", w, h)); - return 0; -} - -// ViEExternalRenderer Callback. Process every incoming frame here. -int -MediaEngineWebRTCVideoSource::DeliverFrame( - unsigned char* buffer, int size, uint32_t time_stamp, - int64_t ntp_time_ms, int64_t render_time, void *handle) -{ - // Check for proper state. - if (mState != kStarted) { - LOG(("DeliverFrame: video not started")); - return 0; - } - - if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) { - MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!"); - return 0; - } - - // Create a video frame and append it to the track. - nsRefPtr image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); - - layers::PlanarYCbCrImage* videoImage = static_cast(image.get()); - - uint8_t* frame = static_cast (buffer); - const uint8_t lumaBpp = 8; - const uint8_t chromaBpp = 4; - - // Take lots of care to round up! - layers::PlanarYCbCrData data; - data.mYChannel = frame; - data.mYSize = IntSize(mWidth, mHeight); - data.mYStride = (mWidth * lumaBpp + 7)/ 8; - data.mCbCrStride = (mWidth * chromaBpp + 7) / 8; - data.mCbChannel = frame + mHeight * data.mYStride; - data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride; - data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2); - data.mPicX = 0; - data.mPicY = 0; - data.mPicSize = IntSize(mWidth, mHeight); - data.mStereoMode = StereoMode::MONO; - - videoImage->SetData(data); - -#ifdef DEBUG - static uint32_t frame_num = 0; - LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++, - mWidth, mHeight, time_stamp, ntp_time_ms, render_time)); -#endif - - // we don't touch anything in 'this' until here (except for snapshot, - // which has it's own lock) - MonitorAutoLock lock(mMonitor); - - // implicitly releases last image - mImage = image.forget(); - - // Push the frame into the MSG with a minimal duration. This will likely - // mean we'll still get NotifyPull calls which will then return the same - // frame again with a longer duration. However, this means we won't - // fail to get the frame in and drop frames. - - // XXX The timestamp for the frame should be based on the Capture time, - // not the MSG time, and MSG should never, ever block on a (realtime) - // video frame (or even really for streaming - audio yes, video probably no). - // Note that MediaPipeline currently ignores the timestamps from MSG - uint32_t len = mSources.Length(); - for (uint32_t i = 0; i < len; i++) { - if (mSources[i]) { - AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration - } - } - - return 0; -} - -// Called if the graph thinks it's running out of buffered video; repeat -// the last frame for whatever minimum period it think it needs. Note that -// this means that no *real* frame can be inserted during this period. -void -MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph, - SourceMediaStream* aSource, - TrackID aID, - StreamTime aDesiredTime) -{ - VideoSegment segment; - - MonitorAutoLock lock(mMonitor); - // B2G does AddTrack, but holds kStarted until the hardware changes state. - // So mState could be kReleased here. We really don't care about the state, - // though. - - StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID); - LOGFRAME(("NotifyPull, desired = %ld, delta = %ld %s", (int64_t) aDesiredTime, - (int64_t) delta, mImage.get() ? "" : "")); - - // Bug 846188 We may want to limit incoming frames to the requested frame rate - // mFps - if you want 30FPS, and the camera gives you 60FPS, this could - // cause issues. - // We may want to signal if the actual frame rate is below mMinFPS - - // cameras often don't return the requested frame rate especially in low - // light; we should consider surfacing this so that we can switch to a - // lower resolution (which may up the frame rate) - - // Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime - // Doing so means a negative delta and thus messes up handling of the graph - if (delta > 0) { - // nullptr images are allowed - AppendToTrack(aSource, mImage, aID, delta); - } -} - -size_t -MediaEngineWebRTCVideoSource::NumCapabilities() -{ - int num = mViECapture->NumberOfCapabilities(GetUUID().get(), kMaxUniqueIdLength); - if (num > 0) { - return num; - } - - switch (mMediaSource) { - case dom::MediaSourceEnum::Camera: -#ifdef XP_MACOSX - // Mac doesn't support capabilities. - // - // Hardcode generic desktop capabilities modeled on OSX camera. - // Note: Values are empirically picked to be OSX friendly, as on OSX, - // values other than these cause the source to not produce. - - if (mHardcodedCapabilities.IsEmpty()) { - for (int i = 0; i < 9; i++) { - webrtc::CaptureCapability c; - c.width = 1920 - i*128; - c.height = 1080 - i*72; - c.maxFPS = 30; - mHardcodedCapabilities.AppendElement(c); - } - for (int i = 0; i < 16; i++) { - webrtc::CaptureCapability c; - c.width = 640 - i*40; - c.height = 480 - i*30; - c.maxFPS = 30; - mHardcodedCapabilities.AppendElement(c); - } - } - break; -#endif - default: - // The default for devices that don't return discrete capabilities: treat - // them as supporting all capabilities orthogonally. E.g. screensharing. - webrtc::CaptureCapability c; - c.width = 0; // 0 = accept any value - c.height = 0; - c.maxFPS = 0; - mHardcodedCapabilities.AppendElement(c); - break; - } - return mHardcodedCapabilities.Length(); -} - -void -MediaEngineWebRTCVideoSource::GetCapability(size_t aIndex, - webrtc::CaptureCapability& aOut) -{ - if (!mHardcodedCapabilities.IsEmpty()) { - MediaEngineCameraVideoSource::GetCapability(aIndex, aOut); - } - mViECapture->GetCaptureCapability(GetUUID().get(), kMaxUniqueIdLength, aIndex, aOut); -} - -nsresult -MediaEngineWebRTCVideoSource::Allocate(const dom::MediaTrackConstraints &aConstraints, - const MediaEnginePrefs &aPrefs, - const nsString& aDeviceId) -{ - LOG((__FUNCTION__)); - if (mState == kReleased && mInitDone) { - // Note: if shared, we don't allow a later opener to affect the resolution. - // (This may change depending on spec changes for Constraints/settings) - - if (!ChooseCapability(aConstraints, aPrefs, aDeviceId)) { - return NS_ERROR_UNEXPECTED; - } - if (mViECapture->AllocateCaptureDevice(GetUUID().get(), - kMaxUniqueIdLength, mCaptureIndex)) { - return NS_ERROR_FAILURE; - } - mState = kAllocated; - LOG(("Video device %d allocated", mCaptureIndex)); - } else if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) { - MonitorAutoLock lock(mMonitor); - if (mSources.IsEmpty()) { - LOG(("Video device %d reallocated", mCaptureIndex)); - } else { - LOG(("Video device %d allocated shared", mCaptureIndex)); - } - } - - return NS_OK; -} - -nsresult -MediaEngineWebRTCVideoSource::Deallocate() -{ - LOG((__FUNCTION__)); - bool empty; - { - MonitorAutoLock lock(mMonitor); - empty = mSources.IsEmpty(); - } - if (empty) { - // If empty, no callbacks to deliver data should be occuring - if (mState != kStopped && mState != kAllocated) { - return NS_ERROR_FAILURE; - } -#ifdef XP_MACOSX - // Bug 829907 - on mac, in shutdown, the mainthread stops processing - // 'native' events, and the QTKit code uses events to the main native CFRunLoop - // in order to provide thread safety. In order to avoid this locking us up, - // release the ViE capture device synchronously on MainThread (so the native - // event isn't needed). - // XXX Note if MainThread Dispatch()es NS_DISPATCH_SYNC to us we can deadlock. - // XXX It might be nice to only do this if we're in shutdown... Hard to be - // sure when that is though. - // Thread safety: a) we call this synchronously, and don't use ViECapture from - // another thread anywhere else, b) ViEInputManager::DestroyCaptureDevice() grabs - // an exclusive object lock and deletes it in a critical section, so all in all - // this should be safe threadwise. - NS_DispatchToMainThread(WrapRunnable(mViECapture.get(), - &webrtc::ViECapture::ReleaseCaptureDevice, - mCaptureIndex), - NS_DISPATCH_SYNC); -#else - mViECapture->ReleaseCaptureDevice(mCaptureIndex); -#endif - mState = kReleased; - LOG(("Video device %d deallocated", mCaptureIndex)); - } else { - LOG(("Video device %d deallocated but still in use", mCaptureIndex)); - } - return NS_OK; -} - -nsresult -MediaEngineWebRTCVideoSource::Start(SourceMediaStream* aStream, TrackID aID) -{ - LOG((__FUNCTION__)); - int error = 0; - if (!mInitDone || !aStream) { - return NS_ERROR_FAILURE; - } - - { - MonitorAutoLock lock(mMonitor); - mSources.AppendElement(aStream); - } - - aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED); - - if (mState == kStarted) { - return NS_OK; - } - mImageContainer = layers::LayerManager::CreateImageContainer(); - - mState = kStarted; - mTrackID = aID; - - error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this); - if (error == -1) { - return NS_ERROR_FAILURE; - } - - error = mViERender->StartRender(mCaptureIndex); - if (error == -1) { - return NS_ERROR_FAILURE; - } - - if (mViECapture->StartCapture(mCaptureIndex, mCapability) < 0) { - return NS_ERROR_FAILURE; - } - - return NS_OK; -} - -nsresult -MediaEngineWebRTCVideoSource::Stop(SourceMediaStream *aSource, TrackID aID) -{ - LOG((__FUNCTION__)); - { - MonitorAutoLock lock(mMonitor); - - if (!mSources.RemoveElement(aSource)) { - // Already stopped - this is allowed - return NS_OK; - } - - aSource->EndTrack(aID); - - if (!mSources.IsEmpty()) { - return NS_OK; - } - if (mState != kStarted) { - return NS_ERROR_FAILURE; - } - - mState = kStopped; - // Drop any cached image so we don't start with a stale image on next - // usage - mImage = nullptr; - } - mViERender->StopRender(mCaptureIndex); - mViERender->RemoveRenderer(mCaptureIndex); - mViECapture->StopCapture(mCaptureIndex); - - return NS_OK; -} - -void -MediaEngineWebRTCVideoSource::Init() -{ - // fix compile warning for these being unused. (remove once used) - (void) mFps; - (void) mMinFps; - - LOG((__FUNCTION__)); - if (mVideoEngine == nullptr) { - return; - } - - mViEBase = webrtc::ViEBase::GetInterface(mVideoEngine); - if (mViEBase == nullptr) { - return; - } - - // Get interfaces for capture, render for now - mViECapture = webrtc::ViECapture::GetInterface(mVideoEngine); - mViERender = webrtc::ViERender::GetInterface(mVideoEngine); - - if (mViECapture == nullptr || mViERender == nullptr) { - return; - } - - char deviceName[kMaxDeviceNameLength]; - char uniqueId[kMaxUniqueIdLength]; - if (mViECapture->GetCaptureDevice(mCaptureIndex, - deviceName, kMaxDeviceNameLength, - uniqueId, kMaxUniqueIdLength)) { - return; - } - SetName(NS_ConvertUTF8toUTF16(deviceName)); - SetUUID(uniqueId); - - mInitDone = true; -} - -void -MediaEngineWebRTCVideoSource::Shutdown() -{ - LOG((__FUNCTION__)); - if (!mInitDone) { - return; - } - if (mState == kStarted) { - SourceMediaStream *source; - bool empty; - - while (1) { - { - MonitorAutoLock lock(mMonitor); - empty = mSources.IsEmpty(); - if (empty) { - break; - } - source = mSources[0]; - } - Stop(source, kVideoTrack); // XXX change to support multiple tracks - } - MOZ_ASSERT(mState == kStopped); - } - - if (mState == kAllocated || mState == kStopped) { - Deallocate(); - } - mViECapture = nullptr; - mViERender = nullptr; - mViEBase = nullptr; - - mState = kReleased; - mInitDone = false; -} - -void MediaEngineWebRTCVideoSource::Refresh(int aIndex) { - // NOTE: mCaptureIndex might have changed when allocated! - // Use aIndex to update information, but don't change mCaptureIndex!! - // Caller looked up this source by uniqueId, so it shouldn't change - char deviceName[kMaxDeviceNameLength]; - char uniqueId[kMaxUniqueIdLength]; - - if (mViECapture->GetCaptureDevice(aIndex, - deviceName, sizeof(deviceName), - uniqueId, sizeof(uniqueId))) { - return; - } - - SetName(NS_ConvertUTF8toUTF16(deviceName)); -#ifdef DEBUG - MOZ_ASSERT(GetUUID().Equals(uniqueId)); -#endif -} - -} // namespace mozilla diff --git a/dom/media/webrtc/moz.build b/dom/media/webrtc/moz.build index 0d3e5fe2ec..7789514afc 100644 --- a/dom/media/webrtc/moz.build +++ b/dom/media/webrtc/moz.build @@ -21,15 +21,16 @@ EXPORTS += [ if CONFIG['MOZ_WEBRTC']: EXPORTS += ['AudioOutputObserver.h', + 'MediaEngineRemoteVideoSource.h', 'MediaEngineWebRTC.h'] EXPORTS.mozilla.dom += [ 'RTCIdentityProviderRegistrar.h' ] UNIFIED_SOURCES += [ 'MediaEngineCameraVideoSource.cpp', + 'MediaEngineRemoteVideoSource.cpp', 'MediaEngineTabVideoSource.cpp', 'MediaEngineWebRTCAudio.cpp', - 'MediaEngineWebRTCVideo.cpp', 'MediaTrackConstraints.cpp', - 'RTCCertificate.cpp', + 'RTCCertificate.cpp', 'RTCIdentityProviderRegistrar.cpp', ] # MediaEngineWebRTC.cpp needs to be built separately. @@ -78,9 +79,9 @@ include('/ipc/chromium/chromium-config.mozbuild') # defined, which complains about important MOZ_EXPORT attributes for # android API types if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']: - CXXFLAGS += [ - '-Wno-error=attributes' - ] + CXXFLAGS += [ + '-Wno-error=attributes' + ] FINAL_LIBRARY = 'xul' if CONFIG['OS_ARCH'] == 'WINNT': @@ -88,8 +89,7 @@ if CONFIG['OS_ARCH'] == 'WINNT': if CONFIG['_MSC_VER']: - CXXFLAGS += [ - '-wd4275', # non dll-interface class used as base for dll-interface class - ] - -FAIL_ON_WARNINGS = True + CXXFLAGS += [ + '-wd4275', # non dll-interface class used as base for dll-interface class + ] + DEFINES['__PRETTY_FUNCTION__'] = '__FUNCSIG__' diff --git a/ipc/glue/BackgroundChildImpl.cpp b/ipc/glue/BackgroundChildImpl.cpp index 055d1e17f4..184f4e34a9 100644 --- a/ipc/glue/BackgroundChildImpl.cpp +++ b/ipc/glue/BackgroundChildImpl.cpp @@ -8,6 +8,10 @@ #include "BroadcastChannelChild.h" #include "ServiceWorkerManagerChild.h" #include "FileDescriptorSetChild.h" +#ifdef MOZ_WEBRTC +#include "CamerasChild.h" +#endif +#include "mozilla/media/MediaChild.h" #include "mozilla/Assertions.h" #include "mozilla/dom/PBlobChild.h" #include "mozilla/dom/asmjscache/AsmJSCache.h" @@ -265,6 +269,29 @@ BackgroundChildImpl::DeallocPBroadcastChannelChild( return true; } +camera::PCamerasChild* +BackgroundChildImpl::AllocPCamerasChild() +{ +#ifdef MOZ_WEBRTC + nsRefPtr agent = + new camera::CamerasChild(); + return agent.forget().take(); +#else + return nullptr; +#endif +} + +bool +BackgroundChildImpl::DeallocPCamerasChild(camera::PCamerasChild *aActor) +{ +#ifdef MOZ_WEBRTC + nsRefPtr child = + dont_AddRef(static_cast(aActor)); + MOZ_ASSERT(aActor); +#endif + return true; +} + // ----------------------------------------------------------------------------- // ServiceWorkerManager // ----------------------------------------------------------------------------- diff --git a/ipc/glue/BackgroundChildImpl.h b/ipc/glue/BackgroundChildImpl.h index 892444d822..1b449a3059 100644 --- a/ipc/glue/BackgroundChildImpl.h +++ b/ipc/glue/BackgroundChildImpl.h @@ -71,6 +71,12 @@ protected: virtual bool DeallocPFileDescriptorSetChild(PFileDescriptorSetChild* aActor) override; + virtual PCamerasChild* + AllocPCamerasChild() override; + + virtual bool + DeallocPCamerasChild(PCamerasChild* aActor) override; + virtual PVsyncChild* AllocPVsyncChild() override; diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp index a9ef9b730a..26e1b9b018 100644 --- a/ipc/glue/BackgroundParentImpl.cpp +++ b/ipc/glue/BackgroundParentImpl.cpp @@ -6,6 +6,10 @@ #include "BroadcastChannelParent.h" #include "FileDescriptorSetParent.h" +#ifdef MOZ_WEBRTC +#include "CamerasParent.h" +#endif +#include "mozilla/media/MediaParent.h" #include "mozilla/AppProcessChecker.h" #include "mozilla/Assertions.h" #include "mozilla/dom/ContentParent.h" @@ -282,6 +286,35 @@ BackgroundParentImpl::DeallocPVsyncParent(PVsyncParent* aActor) return true; } +camera::PCamerasParent* +BackgroundParentImpl::AllocPCamerasParent() +{ + AssertIsInMainProcess(); + AssertIsOnBackgroundThread(); + +#ifdef MOZ_WEBRTC + nsRefPtr actor = + mozilla::camera::CamerasParent::Create(); + return actor.forget().take(); +#else + return nullptr; +#endif +} + +bool +BackgroundParentImpl::DeallocPCamerasParent(camera::PCamerasParent *aActor) +{ + AssertIsInMainProcess(); + AssertIsOnBackgroundThread(); + MOZ_ASSERT(aActor); + +#ifdef MOZ_WEBRTC + nsRefPtr actor = + dont_AddRef(static_cast(aActor)); +#endif + return true; +} + namespace { class InitUDPSocketParentCallback final : public nsRunnable diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h index 154ff04374..618cab98e3 100644 --- a/ipc/glue/BackgroundParentImpl.h +++ b/ipc/glue/BackgroundParentImpl.h @@ -101,6 +101,12 @@ protected: virtual bool DeallocPServiceWorkerManagerParent(PServiceWorkerManagerParent* aActor) override; + virtual PCamerasParent* + AllocPCamerasParent() override; + + virtual bool + DeallocPCamerasParent(PCamerasParent* aActor) override; + virtual bool RecvShutdownServiceWorkerRegistrar() override; diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl index 7308895bc5..74af077a26 100644 --- a/ipc/glue/PBackground.ipdl +++ b/ipc/glue/PBackground.ipdl @@ -12,6 +12,7 @@ include protocol PCacheStorage; include protocol PCacheStreamControl; include protocol PFileDescriptorSet; include protocol PMessagePort; +include protocol PCameras; include protocol PNuwa; include protocol PServiceWorkerManager; include protocol PUDPSocket; @@ -47,6 +48,7 @@ sync protocol PBackground manages PCacheStreamControl; manages PFileDescriptorSet; manages PMessagePort; + manages PCameras; manages PNuwa; manages PServiceWorkerManager; manages PUDPSocket; @@ -60,6 +62,8 @@ parent: PVsync(); + PCameras(); + PUDPSocket(OptionalPrincipalInfo pInfo, nsCString filter); PBroadcastChannel(PrincipalInfo pInfo, nsCString origin, nsString channel, bool privateBrowsing); diff --git a/ipc/glue/moz.build b/ipc/glue/moz.build index 42ee0345a9..95c7cdeda8 100644 --- a/ipc/glue/moz.build +++ b/ipc/glue/moz.build @@ -139,6 +139,7 @@ LOCAL_INCLUDES += [ '/dom/broadcastchannel', '/dom/indexedDB', '/dom/workers', + '/media/webrtc/trunk', '/xpcom/build', ] diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 4c76c0a259..4da9c99876 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -515,6 +515,10 @@ pref("media.autoplay.allowscripted", true); // MediaDecoderReader's mVideoQueue. pref("media.video-queue.default-size", 10); +// The maximum number of queued frames to send to the compositor. +// By default, send all of them. +pref("media.video-queue.send-to-compositor-size", 9999); + // Whether to disable the video stats to prevent fingerprinting pref("media.video_stats.enabled", true);