diff --git a/dom/camera/CameraPreviewMediaStream.cpp b/dom/camera/CameraPreviewMediaStream.cpp index 8eb66c17f5..0695fd721d 100644 --- a/dom/camera/CameraPreviewMediaStream.cpp +++ b/dom/camera/CameraPreviewMediaStream.cpp @@ -36,7 +36,9 @@ CameraPreviewMediaStream::CameraPreviewMediaStream(DOMMediaStream* aWrapper) , mRateLimit(false) , mTrackCreated(false) { - SetGraphImpl(MediaStreamGraph::GetInstance()); + SetGraphImpl( + MediaStreamGraph::GetInstance( + MediaStreamGraph::SYSTEM_THREAD_DRIVER, AudioChannel::Normal)); mFakeMediaStreamGraph = new FakeMediaStreamGraph(); mIsConsumed = false; } diff --git a/dom/events/NotifyPaintEvent.cpp b/dom/events/NotifyPaintEvent.cpp index f1fdf7eb92..424d34ca19 100644 --- a/dom/events/NotifyPaintEvent.cpp +++ b/dom/events/NotifyPaintEvent.cpp @@ -26,7 +26,7 @@ NotifyPaintEvent::NotifyPaintEvent(EventTarget* aOwner, mEvent->message = aEventType; } if (aInvalidateRequests) { - mInvalidateRequests.MoveElementsFrom(aInvalidateRequests->mRequests); + mInvalidateRequests.AppendElements(Move(aInvalidateRequests->mRequests)); } } diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 64ac63ed2e..29a13a642c 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -462,7 +462,6 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(HTMLMediaElement, nsGenericHTM NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mMediaSource) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcMediaSource) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcStream) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlaybackStream) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSrcAttrStream) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mSourcePointer) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mLoadBlockedDoc) @@ -584,8 +583,8 @@ NS_IMETHODIMP HTMLMediaElement::GetError(nsIDOMMediaError * *aError) bool HTMLMediaElement::Ended() { - if (mSrcStream) { - return GetSrcMediaStream()->IsFinished(); + if (MediaStream* stream = GetSrcMediaStream()) { + return stream->IsFinished(); } if (mDecoder) { @@ -1345,11 +1344,11 @@ NS_IMETHODIMP HTMLMediaElement::GetSeeking(bool* aSeeking) double HTMLMediaElement::CurrentTime() const { - if (mSrcStream) { - MediaStream* stream = GetSrcMediaStream(); - if (stream) { - return stream->StreamTimeToSeconds(stream->GetCurrentTime()); + if (MediaStream* stream = GetSrcMediaStream()) { + if (mSrcStreamPausedCurrentTime >= 0) { + return mSrcStreamPausedCurrentTime; } + return stream->StreamTimeToSeconds(stream->GetCurrentTime()); } if (mDefaultPlaybackStartPosition == 0.0 && mDecoder) { @@ -1649,14 +1648,9 @@ HTMLMediaElement::Pause(ErrorResult& aRv) mAutoplaying = false; // We changed mPaused and mAutoplaying which can affect AddRemoveSelfReference AddRemoveSelfReference(); + UpdateSrcMediaStreamPlaying(); if (!oldPaused) { - if (mSrcStream) { - MediaStream* stream = GetSrcMediaStream(); - if (stream) { - stream->ChangeExplicitBlockerCount(1); - } - } FireTimeUpdate(false); DispatchAsyncEvent(NS_LITERAL_STRING("pause")); } @@ -1797,8 +1791,10 @@ void HTMLMediaElement::SetVolumeInternal() if (mDecoder) { mDecoder->SetVolume(effectiveVolume); - } else if (mSrcStream) { - GetSrcMediaStream()->SetAudioOutputVolume(this, effectiveVolume); + } else if (MediaStream* stream = GetSrcMediaStream()) { + if (mSrcStreamIsPlaying) { + stream->SetAudioOutputVolume(this, effectiveVolume); + } } } @@ -1826,6 +1822,14 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, if (!window) { return nullptr; } + + if (!aGraph) { + MediaStreamGraph::GraphDriverType graphDriverType = + HasAudio() ? MediaStreamGraph::AUDIO_THREAD_DRIVER + : MediaStreamGraph::SYSTEM_THREAD_DRIVER; + aGraph = MediaStreamGraph::GetInstance(graphDriverType, mAudioChannel); + } + OutputMediaStream* out = mOutputStreams.AppendElement(); out->mStream = DOMMediaStream::CreateTrackUnionStream(window, aGraph); nsRefPtr principal = GetCurrentPrincipal(); @@ -1995,6 +1999,7 @@ HTMLMediaElement::LookupMediaElementURITable(nsIURI* aURI) HTMLMediaElement::HTMLMediaElement(already_AddRefed& aNodeInfo) : nsGenericHTMLElement(aNodeInfo), mWatchManager(this, AbstractThread::MainThread()), + mSrcStreamPausedCurrentTime(-1), mCurrentLoadID(0), mNetworkState(nsIDOMHTMLMediaElement::NETWORK_EMPTY), mReadyState(nsIDOMHTMLMediaElement::HAVE_NOTHING, "HTMLMediaElement::mReadyState"), @@ -2037,6 +2042,7 @@ HTMLMediaElement::HTMLMediaElement(already_AddRefed& aNo mHasSelfReference(false), mShuttingDown(false), mSuspendedForPreloadNone(false), + mSrcStreamIsPlaying(false), mMediaSecurityVerified(false), mCORSMode(CORS_NONE), mIsEncrypted(false), @@ -2233,9 +2239,6 @@ HTMLMediaElement::PlayInternal(bool aCallerIsChrome) // TODO: If the playback has ended, then the user agent must set // seek to the effective start. if (mPaused) { - if (mSrcStream) { - GetSrcMediaStream()->ChangeExplicitBlockerCount(-1); - } DispatchAsyncEvent(NS_LITERAL_STRING("play")); switch (mReadyState) { case nsIDOMHTMLMediaElement::HAVE_NOTHING: @@ -2259,6 +2262,7 @@ HTMLMediaElement::PlayInternal(bool aCallerIsChrome) // and our preload status. AddRemoveSelfReference(); UpdatePreloadAction(); + UpdateSrcMediaStreamPlaying(); return NS_OK; } @@ -2854,6 +2858,7 @@ public: mElement(aElement), mHaveCurrentData(false), mBlocked(false), + mFinished(false), mMutex(aName), mPendingNotifyOutput(false) {} @@ -2862,23 +2867,29 @@ public: // Main thread void DoNotifyFinished() { + mFinished = true; if (mElement) { nsRefPtr deathGrip = mElement; - mElement->PlaybackEnded(); + // Update NextFrameStatus() to move to NEXT_FRAME_UNAVAILABLE and // HAVE_CURRENT_DATA. mElement = nullptr; + // NotifyWatchers before calling PlaybackEnded since PlaybackEnded + // can remove watchers. NotifyWatchers(); + + deathGrip->PlaybackEnded(); } } MediaDecoderOwner::NextFrameStatus NextFrameStatus() { - if (!mElement || !mHaveCurrentData) { + if (!mElement || !mHaveCurrentData || mFinished) { return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE; } - return mBlocked ? MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING - : MediaDecoderOwner::NEXT_FRAME_AVAILABLE; + return mBlocked + ? MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING + : MediaDecoderOwner::NEXT_FRAME_AVAILABLE; } void DoNotifyBlocked() @@ -2958,6 +2969,7 @@ private: HTMLMediaElement* mElement; bool mHaveCurrentData; bool mBlocked; + bool mFinished; // mMutex protects the fields below; they can be accessed on any thread Mutex mMutex; @@ -3039,6 +3051,82 @@ private: WeakPtr mElement; }; +void HTMLMediaElement::UpdateSrcMediaStreamPlaying(uint32_t aFlags) +{ + if (!mSrcStream) { + return; + } + // We might be in cycle collection with mSrcStream->GetStream() already + // returning null due to unlinking. + + MediaStream* stream = mSrcStream->GetStream(); + bool shouldPlay = !(aFlags & REMOVING_SRC_STREAM) && !mPaused && + !mPausedForInactiveDocumentOrChannel && stream; + if (shouldPlay == mSrcStreamIsPlaying) { + return; + } + mSrcStreamIsPlaying = shouldPlay; + + if (shouldPlay) { + mSrcStreamPausedCurrentTime = -1; + + mMediaStreamListener = new StreamListener(this, + "HTMLMediaElement::mMediaStreamListener"); + mMediaStreamSizeListener = new StreamSizeListener(this); + stream->AddListener(mMediaStreamListener); + stream->AddListener(mMediaStreamSizeListener); + + mWatchManager.Watch(*mMediaStreamListener, + &HTMLMediaElement::UpdateReadyStateInternal); + + nsRefPtr stream = mSrcStream->GetStream(); + if (stream) { + stream->SetAudioChannelType(mAudioChannel); + } + + stream->AddAudioOutput(this); + SetVolumeInternal(); + + #ifdef MOZ_WIDGET_GONK + bool bUseOverlayImage = mSrcStream->AsDOMHwMediaStream() != nullptr; + #else + bool bUseOverlayImage = false; + #endif + VideoFrameContainer* container; + if (bUseOverlayImage) { + container = GetOverlayImageVideoFrameContainer(); + } else { + container = GetVideoFrameContainer(); + } + if (container) { + stream->AddVideoOutput(container); + } + } else { + if (stream) { + mSrcStreamPausedCurrentTime = CurrentTime(); + + stream->RemoveListener(mMediaStreamListener); + stream->RemoveListener(mMediaStreamSizeListener); + + stream->RemoveAudioOutput(this); + VideoFrameContainer* container = GetVideoFrameContainer(); + if (container) { + stream->RemoveVideoOutput(container); + } + } + // If stream is null, then DOMMediaStream::Destroy must have been + // called and that will remove all listeners/outputs. + + mWatchManager.Unwatch(*mMediaStreamListener, + &HTMLMediaElement::UpdateReadyStateInternal); + + mMediaStreamListener->Forget(); + mMediaStreamListener = nullptr; + mMediaStreamSizeListener->Forget(); + mMediaStreamSizeListener = nullptr; + } +} + void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) { NS_ASSERTION(!mSrcStream && !mMediaStreamListener && !mMediaStreamSizeListener, @@ -3051,116 +3139,38 @@ void HTMLMediaElement::SetupSrcMediaStreamPlayback(DOMMediaStream* aStream) return; } - // XXX Remove this if with CameraPreviewMediaStream per bug 1124630. - if (!mSrcStream->GetStream()->AsCameraPreviewStream()) { - // Now that we have access to |mSrcStream| we can pipe it to our shadow - // version |mPlaybackStream|. If two media elements are playing the - // same realtime DOMMediaStream, this allows them to pause playback - // independently of each other. - mPlaybackStream = DOMMediaStream::CreateTrackUnionStream(window); - mPlaybackStreamInputPort = mPlaybackStream->GetStream()->AsProcessedStream()-> - AllocateInputPort(mSrcStream->GetStream(), MediaInputPort::FLAG_BLOCK_OUTPUT); - - nsRefPtr principal = GetCurrentPrincipal(); - mPlaybackStream->CombineWithPrincipal(principal); - - // Let |mSrcStream| decide when the stream has finished. - GetSrcMediaStream()->AsProcessedStream()->SetAutofinish(true); - } - nsRefPtr stream = mSrcStream->GetStream(); if (stream) { stream->SetAudioChannelType(mAudioChannel); } - // XXX if we ever support capturing the output of a media element which is - // playing a stream, we'll need to add a CombineWithPrincipal call here. - mMediaStreamListener = new StreamListener(this, "HTMLMediaElement::mMediaStreamListener"); - mMediaStreamSizeListener = new StreamSizeListener(this); - mWatchManager.Watch(*mMediaStreamListener, &HTMLMediaElement::UpdateReadyStateInternal); - - GetSrcMediaStream()->AddListener(mMediaStreamListener); - // Listen for an initial image size on mSrcStream so we can get results even - // if we block the mPlaybackStream. - stream->AddListener(mMediaStreamSizeListener); - if (mPaused) { - GetSrcMediaStream()->ChangeExplicitBlockerCount(1); - } - if (mPausedForInactiveDocumentOrChannel) { - GetSrcMediaStream()->ChangeExplicitBlockerCount(1); - } - - ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE); - - ChangeDelayLoadStatus(false); - GetSrcMediaStream()->AddAudioOutput(this); - SetVolumeInternal(); - - bool bUseOverlayImage = mSrcStream->AsDOMHwMediaStream() != nullptr; - VideoFrameContainer* container; - - if (bUseOverlayImage) { - container = GetOverlayImageVideoFrameContainer(); - } - else { - container = GetVideoFrameContainer(); - } - - if (container) { - GetSrcMediaStream()->AddVideoOutput(container); - } - - CheckAutoplayDataReady(); + UpdateSrcMediaStreamPlaying(); // Note: we must call DisconnectTrackListListeners(...) before dropping - // mSrcStream + // mSrcStream. + // If we pause this media element, track changes in the underlying stream + // will continue to fire events at this element and alter its track list. + // That's simpler than delaying the events, but probably confusing... mSrcStream->ConstructMediaTracks(AudioTracks(), VideoTracks()); mSrcStream->OnTracksAvailable(new MediaStreamTracksAvailableCallback(this)); + ChangeNetworkState(nsIDOMHTMLMediaElement::NETWORK_IDLE); + ChangeDelayLoadStatus(false); + CheckAutoplayDataReady(); + // FirstFrameLoaded() will be called when the stream has current data. } void HTMLMediaElement::EndSrcMediaStreamPlayback() { - MediaStream* stream = GetSrcMediaStream(); - if (stream) { - stream->RemoveListener(mMediaStreamListener); - } - if (mSrcStream->GetStream()) { - mSrcStream->GetStream()->RemoveListener(mMediaStreamSizeListener); - } + MOZ_ASSERT(mSrcStream); + + UpdateSrcMediaStreamPlaying(REMOVING_SRC_STREAM); + mSrcStream->DisconnectTrackListListeners(AudioTracks(), VideoTracks()); - if (mPlaybackStreamInputPort) { - mPlaybackStreamInputPort->Destroy(); - } - - // Kill its reference to this element - mWatchManager.Unwatch(*mMediaStreamListener, &HTMLMediaElement::UpdateReadyStateInternal); - mMediaStreamListener->Forget(); - mMediaStreamListener = nullptr; - mMediaStreamSizeListener->Forget(); - mMediaStreamSizeListener = nullptr; - if (stream) { - stream->RemoveAudioOutput(this); - } - VideoFrameContainer* container = GetVideoFrameContainer(); - if (container) { - if (stream) { - stream->RemoveVideoOutput(container); - } - container->ClearCurrentFrame(); - } - if (mPaused && stream) { - stream->ChangeExplicitBlockerCount(-1); - } - if (mPausedForInactiveDocumentOrChannel && stream) { - stream->ChangeExplicitBlockerCount(-1); - } mSrcStream = nullptr; - mPlaybackStreamInputPort = nullptr; - mPlaybackStream = nullptr; } void HTMLMediaElement::ProcessMediaFragmentURI() @@ -3760,6 +3770,7 @@ void HTMLMediaElement::CheckAutoplayDataReady() mPaused = false; // We changed mPaused which can affect AddRemoveSelfReference AddRemoveSelfReference(); + UpdateSrcMediaStreamPlaying(); if (mDecoder) { SetPlayedOrSeeked(true); @@ -3769,7 +3780,6 @@ void HTMLMediaElement::CheckAutoplayDataReady() mDecoder->Play(); } else if (mSrcStream) { SetPlayedOrSeeked(true); - GetSrcMediaStream()->ChangeExplicitBlockerCount(-1); } DispatchAsyncEvent(NS_LITERAL_STRING("play")); @@ -3960,6 +3970,7 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE if (aPauseElement != mPausedForInactiveDocumentOrChannel) { mPausedForInactiveDocumentOrChannel = aPauseElement; + UpdateSrcMediaStreamPlaying(); if (aPauseElement) { if (mMediaSource) { ReportMSETelemetry(); @@ -3968,8 +3979,6 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE if (mDecoder) { mDecoder->Pause(); mDecoder->Suspend(); - } else if (mSrcStream) { - GetSrcMediaStream()->ChangeExplicitBlockerCount(1); } mEventDeliveryPaused = aSuspendEvents; } else { @@ -3978,8 +3987,6 @@ void HTMLMediaElement::SuspendOrResumeElement(bool aPauseElement, bool aSuspendE if (!mPaused && !mDecoder->IsEndedOrShutdown()) { mDecoder->Play(); } - } else if (mSrcStream) { - GetSrcMediaStream()->ChangeExplicitBlockerCount(-1); } if (mEventDeliveryPaused) { mEventDeliveryPaused = false; @@ -4564,13 +4571,15 @@ NS_IMETHODIMP HTMLMediaElement::WindowAudioCaptureChanged() nsCOMPtr window = do_QueryInterface(OwnerDoc()->GetParentObject()); uint64_t id = window->WindowID(); - MediaStreamGraph* msg = MediaStreamGraph::GetInstance(); + MediaStreamGraph* msg = + MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER, + AudioChannel::Normal); - if (!mPlaybackStream) { + if (mSrcStream) { + mCaptureStreamPort = msg->ConnectToCaptureStream(id, mSrcStream->GetStream()); + } else { nsRefPtr stream = CaptureStreamInternal(false, msg); mCaptureStreamPort = msg->ConnectToCaptureStream(id, stream->GetStream()); - } else { - mCaptureStreamPort = msg->ConnectToCaptureStream(id, mPlaybackStream->GetStream()); } } else { mAudioCapturedByWindow = false; diff --git a/dom/html/HTMLMediaElement.h b/dom/html/HTMLMediaElement.h index af89d05843..b1e699a4bd 100644 --- a/dom/html/HTMLMediaElement.h +++ b/dom/html/HTMLMediaElement.h @@ -348,14 +348,14 @@ public: */ virtual void FireTimeUpdate(bool aPeriodic) final override; + /** + * This will return null if mSrcStream is null, or if mSrcStream is not + * null but its GetStream() returns null --- which can happen during + * cycle collection unlinking! + */ MediaStream* GetSrcMediaStream() const { - NS_ASSERTION(mSrcStream, "Don't call this when not playing a stream"); - if (!mPlaybackStream) { - // XXX Remove this check with CameraPreviewMediaStream per bug 1124630. - return mSrcStream->GetStream(); - } - return mPlaybackStream->GetStream(); + return mSrcStream ? mSrcStream->GetStream() : nullptr; } // WebIDL @@ -725,6 +725,11 @@ protected: * Stop playback on mSrcStream. */ void EndSrcMediaStreamPlayback(); + /** + * Ensure we're playing mSrcStream if and only if we're not paused. + */ + enum { REMOVING_SRC_STREAM = 0x1 }; + void UpdateSrcMediaStreamPlaying(uint32_t aFlags = 0); /** * Returns an nsDOMMediaStream containing the played contents of this @@ -1056,8 +1061,9 @@ protected: // At most one of mDecoder and mSrcStream can be non-null. nsRefPtr mSrcStream; - // Holds a reference to a MediaInputPort connecting mSrcStream to mPlaybackStream. - nsRefPtr mPlaybackStreamInputPort; + // If non-negative, the time we should return for currentTime while playing + // mSrcStream. + double mSrcStreamPausedCurrentTime; // Holds a reference to the stream connecting this stream to the capture sink. nsRefPtr mCaptureStreamPort; @@ -1340,6 +1346,9 @@ protected: // stored in mPreloadURI. bool mSuspendedForPreloadNone; + // True if we've connected mSrcStream to the media element output. + bool mSrcStreamIsPlaying; + // True if a same-origin check has been done for the media element and resource. bool mMediaSecurityVerified; diff --git a/dom/inputport/InputPort.cpp b/dom/inputport/InputPort.cpp index 210e0b17fb..9d0cab37eb 100644 --- a/dom/inputport/InputPort.cpp +++ b/dom/inputport/InputPort.cpp @@ -4,6 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "MediaStreamGraph.h" #include "DOMMediaStream.h" #include "InputPortData.h" #include "InputPortListeners.h" @@ -60,7 +61,10 @@ InputPort::Init(nsIInputPortData* aData, nsIInputPortListener* aListener, ErrorR mInputPortListener = static_cast(aListener); mInputPortListener->RegisterInputPort(this); - mStream = DOMMediaStream::CreateSourceStream(GetOwner()); + MediaStreamGraph* graph = + MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER, + AudioChannel::Normal); + mStream = DOMMediaStream::CreateSourceStream(GetOwner(), graph); } void diff --git a/dom/media/CanvasCaptureMediaStream.cpp b/dom/media/CanvasCaptureMediaStream.cpp index d8f06b790e..7ad05d9f51 100644 --- a/dom/media/CanvasCaptureMediaStream.cpp +++ b/dom/media/CanvasCaptureMediaStream.cpp @@ -328,7 +328,10 @@ CanvasCaptureMediaStream::CreateSourceStream(nsIDOMWindow* aWindow, HTMLCanvasElement* aCanvas) { nsRefPtr stream = new CanvasCaptureMediaStream(aCanvas); - stream->InitSourceStream(aWindow); + MediaStreamGraph* graph = + MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER, + AudioChannel::Normal); + stream->InitSourceStream(aWindow, graph); return stream.forget(); } diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index e75d5fbfad..26209a21a1 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -10,6 +10,7 @@ #include "mozilla/dom/MediaStreamBinding.h" #include "mozilla/dom/LocalMediaStreamBinding.h" #include "mozilla/dom/AudioNode.h" +#include "AudioChannelAgent.h" #include "mozilla/dom/AudioTrack.h" #include "mozilla/dom/AudioTrackList.h" #include "mozilla/dom/VideoTrack.h" @@ -283,9 +284,6 @@ DOMMediaStream::InitSourceStream(nsIDOMWindow* aWindow, MediaStreamGraph* aGraph) { mWindow = aWindow; - if (!aGraph) { - aGraph = MediaStreamGraph::GetInstance(); - } InitStreamCommon(aGraph->CreateSourceStream(this)); } @@ -295,9 +293,6 @@ DOMMediaStream::InitTrackUnionStream(nsIDOMWindow* aWindow, { mWindow = aWindow; - if (!aGraph) { - aGraph = MediaStreamGraph::GetInstance(); - } InitStreamCommon(aGraph->CreateTrackUnionStream(this)); } @@ -307,9 +302,6 @@ DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow, { mWindow = aWindow; - if (!aGraph) { - aGraph = MediaStreamGraph::GetInstance(); - } InitStreamCommon(aGraph->CreateAudioCaptureStream(this)); } @@ -722,7 +714,11 @@ already_AddRefed DOMHwMediaStream::CreateHwStream(nsIDOMWindow* aWindow) { nsRefPtr stream = new DOMHwMediaStream(); - stream->InitSourceStream(aWindow); + + MediaStreamGraph* graph = + MediaStreamGraph::GetInstance(MediaStreamGraph::SYSTEM_THREAD_DRIVER, + AudioChannel::Normal); + stream->InitSourceStream(aWindow, graph); stream->Init(stream->GetStream()); return stream.forget(); diff --git a/dom/media/DOMMediaStream.h b/dom/media/DOMMediaStream.h index 92ede0dfa1..1fbc9741cc 100644 --- a/dom/media/DOMMediaStream.h +++ b/dom/media/DOMMediaStream.h @@ -190,20 +190,20 @@ public: * Create an nsDOMMediaStream whose underlying stream is a SourceMediaStream. */ static already_AddRefed CreateSourceStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); /** * Create an nsDOMMediaStream whose underlying stream is a TrackUnionStream. */ static already_AddRefed CreateTrackUnionStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); /** * Create an nsDOMMediaStream whose underlying stream is an * AudioCaptureStream */ static already_AddRefed CreateAudioCaptureStream( - nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + nsIDOMWindow* aWindow, MediaStreamGraph* aGraph); void SetLogicalStreamStartTime(StreamTime aTime) { @@ -266,11 +266,11 @@ protected: void Destroy(); void InitSourceStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); void InitTrackUnionStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); void InitAudioCaptureStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); void InitStreamCommon(MediaStream* aStream); already_AddRefed CreateAudioTrack(AudioStreamTrack* aStreamTrack); already_AddRefed CreateVideoTrack(VideoStreamTrack* aStreamTrack); @@ -352,20 +352,20 @@ public: */ static already_AddRefed CreateSourceStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); /** * Create an nsDOMLocalMediaStream whose underlying stream is a TrackUnionStream. */ static already_AddRefed CreateTrackUnionStream(nsIDOMWindow* aWindow, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); /** * Create an nsDOMLocalMediaStream whose underlying stream is an * AudioCaptureStream. */ static already_AddRefed CreateAudioCaptureStream( - nsIDOMWindow* aWindow, MediaStreamGraph* aGraph = nullptr); + nsIDOMWindow* aWindow, MediaStreamGraph* aGraph); protected: virtual ~DOMLocalMediaStream(); @@ -389,7 +389,7 @@ public: static already_AddRefed CreateTrackUnionStream(nsIDOMWindow* aWindow, AudioNode* aNode, - MediaStreamGraph* aGraph = nullptr); + MediaStreamGraph* aGraph); protected: ~DOMAudioNodeMediaStream(); diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp index d7f64bd902..e1aa2b3726 100644 --- a/dom/media/GraphDriver.cpp +++ b/dom/media/GraphDriver.cpp @@ -285,6 +285,15 @@ ThreadedDriver::RunThread() GraphTime nextStateComputedTime = mGraphImpl->RoundUpToNextAudioBlock( mIterationEnd + mGraphImpl->MillisecondsToMediaTime(AUDIO_TARGET_MS)); + if (nextStateComputedTime < stateComputedTime) { + // A previous driver may have been processing further ahead of + // iterationEnd. + STREAM_LOG(LogLevel::Warning, + ("Prevent state from going backwards. interval[%ld; %ld] state[%ld; %ld]", + (long)mIterationStart, (long)mIterationEnd, + (long)stateComputedTime, (long)nextStateComputedTime)); + nextStateComputedTime = stateComputedTime; + } STREAM_LOG(LogLevel::Debug, ("interval[%ld; %ld] state[%ld; %ld]", (long)mIterationStart, (long)mIterationEnd, @@ -292,10 +301,7 @@ ThreadedDriver::RunThread() mGraphImpl->mFlushSourcesNow = mGraphImpl->mFlushSourcesOnNextIteration; mGraphImpl->mFlushSourcesOnNextIteration = false; - stillProcessing = mGraphImpl->OneIteration(mIterationStart, - mIterationEnd, - stateComputedTime, - nextStateComputedTime); + stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); if (mNextDriver && stillProcessing) { STREAM_LOG(LogLevel::Debug, ("Switching to AudioCallbackDriver")); @@ -838,10 +844,7 @@ AudioCallbackDriver::DataCallback(AudioDataValue* aBuffer, long aFrames) mIterationEnd = stateComputedTime; } - stillProcessing = mGraphImpl->OneIteration(mIterationStart, - mIterationEnd, - stateComputedTime, - nextStateComputedTime); + stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); } else { NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration."); stillProcessing = true; diff --git a/dom/media/Intervals.h b/dom/media/Intervals.h index 5fb6dd2bc4..e8ca7ae16f 100644 --- a/dom/media/Intervals.h +++ b/dom/media/Intervals.h @@ -269,7 +269,7 @@ public: IntervalSet(SelfType&& aOther) { - mIntervals.MoveElementsFrom(Move(aOther.mIntervals)); + mIntervals.AppendElements(Move(aOther.mIntervals)); } explicit IntervalSet(const ElemType& aOther) @@ -379,7 +379,7 @@ public: normalized.AppendElement(Move(mIntervals[i])); } mIntervals.Clear(); - mIntervals.MoveElementsFrom(Move(normalized)); + mIntervals.AppendElements(Move(normalized)); return *this; } @@ -481,9 +481,8 @@ public: j++; } } - mIntervals.Clear(); - mIntervals.MoveElementsFrom(Move(intersection)); + mIntervals.AppendElements(Move(intersection)); return *this; } @@ -683,7 +682,7 @@ private: normalized.AppendElement(Move(current)); mIntervals.Clear(); - mIntervals.MoveElementsFrom(Move(normalized)); + mIntervals.AppendElements(Move(normalized)); } } diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index 2fe0314269..5528ffb5ea 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -879,7 +879,7 @@ MediaFormatReader::DecodeDemuxedSamples(TrackType aTrack, decoder.mDecoder->Shutdown(); decoder.mDecoder = nullptr; if (sample->mKeyframe) { - decoder.mQueuedSamples.MoveElementsFrom(samples); + decoder.mQueuedSamples.AppendElements(Move(samples)); ScheduleUpdate(aTrack); } else { MOZ_ASSERT(decoder.mTimeThreshold.isNothing()); diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 22daeb65ec..e9bfeeecdb 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -484,12 +484,13 @@ public: CreateTrackUnionStream(nsIDOMWindow* aWindow, GetUserMediaCallbackMediaStreamListener* aListener, MediaEngineSource* aAudioSource, - MediaEngineSource* aVideoSource) + MediaEngineSource* aVideoSource, + MediaStreamGraph* aMSG) { nsRefPtr stream = new nsDOMUserMediaStream(aListener, aAudioSource, aVideoSource); - stream->InitTrackUnionStream(aWindow); + stream->InitTrackUnionStream(aWindow, aMSG); return stream.forget(); } @@ -788,7 +789,13 @@ public: } #endif - MediaStreamGraph* msg = MediaStreamGraph::GetInstance(); + MediaStreamGraph::GraphDriverType graphDriverType = + mAudioSource ? MediaStreamGraph::AUDIO_THREAD_DRIVER + : MediaStreamGraph::SYSTEM_THREAD_DRIVER; + MediaStreamGraph* msg = + MediaStreamGraph::GetInstance(graphDriverType, + dom::AudioChannel::Normal); + nsRefPtr stream = msg->CreateSourceStream(nullptr); nsRefPtr domStream; @@ -798,7 +805,7 @@ public: // them down instead. if (mAudioSource && mAudioSource->GetMediaSource() == dom::MediaSourceEnum::AudioCapture) { - domStream = DOMLocalMediaStream::CreateAudioCaptureStream(window); + domStream = DOMLocalMediaStream::CreateAudioCaptureStream(window, msg); // It should be possible to pipe the capture stream to anything. CORS is // not a problem here, we got explicit user content. domStream->SetPrincipal(window->GetExtantDoc()->NodePrincipal()); @@ -810,7 +817,8 @@ public: // avoid us blocking nsRefPtr trackunion = nsDOMUserMediaStream::CreateTrackUnionStream(window, mListener, - mAudioSource, mVideoSource); + mAudioSource, mVideoSource, + msg); trackunion->GetStream()->AsProcessedStream()->SetAutofinish(true); nsRefPtr port = trackunion->GetStream()->AsProcessedStream()-> AllocateInputPort(stream, MediaInputPort::FLAG_BLOCK_OUTPUT); @@ -996,7 +1004,7 @@ ApplyConstraints(const MediaTrackConstraints &aConstraints, } } if (!aSources.Length()) { - aSources.MoveElementsFrom(rejects); + aSources.AppendElements(Move(rejects)); aggregateConstraints.RemoveElementAt(aggregateConstraints.Length() - 1); } } @@ -2796,7 +2804,10 @@ GetUserMediaCallbackMediaStreamListener::StopSharing() nsCOMPtr window = nsGlobalWindow::GetInnerWindowWithId(mWindowID); MOZ_ASSERT(window); window->SetAudioCapture(false); - MediaStreamGraph::GetInstance()->UnregisterCaptureStreamForWindow(mWindowID); + MediaStreamGraph* graph = + MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER, + dom::AudioChannel::Normal); + graph->UnregisterCaptureStreamForWindow(mWindowID); mStream->Destroy(); } } diff --git a/dom/media/MediaSegment.h b/dom/media/MediaSegment.h index 82b1d150a5..423ad91b1a 100644 --- a/dom/media/MediaSegment.h +++ b/dom/media/MediaSegment.h @@ -284,6 +284,26 @@ public: uint32_t mIndex; }; + Chunk* FindChunkContaining(StreamTime aOffset, StreamTime* aStart = nullptr) + { + if (aOffset < 0) { + return nullptr; + } + StreamTime offset = 0; + for (uint32_t i = 0; i < mChunks.Length(); ++i) { + Chunk& c = mChunks[i]; + StreamTime nextOffset = offset + c.GetDuration(); + if (aOffset < nextOffset) { + if (aStart) { + *aStart = offset; + } + return &c; + } + offset = nextOffset; + } + return nullptr; + } + void RemoveLeading(StreamTime aDuration) { RemoveLeading(aDuration, 0); @@ -325,7 +345,7 @@ protected: mChunks[mChunks.Length() - 1].mDuration += aSource->mChunks[0].mDuration; aSource->mChunks.RemoveElementAt(0); } - mChunks.MoveElementsFrom(aSource->mChunks); + mChunks.AppendElements(Move(aSource->mChunks)); } void AppendSliceInternal(const MediaSegmentBase& aSource, @@ -356,26 +376,6 @@ protected: return c; } - Chunk* FindChunkContaining(StreamTime aOffset, StreamTime* aStart = nullptr) - { - if (aOffset < 0) { - return nullptr; - } - StreamTime offset = 0; - for (uint32_t i = 0; i < mChunks.Length(); ++i) { - Chunk& c = mChunks[i]; - StreamTime nextOffset = offset + c.GetDuration(); - if (aOffset < nextOffset) { - if (aStart) { - *aStart = offset; - } - return &c; - } - offset = nextOffset; - } - return nullptr; - } - Chunk* GetLastChunk() { if (mChunks.IsEmpty()) { diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index cf72aeb925..60edb1d9b5 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -79,7 +79,7 @@ private: }; /** - * The singleton graph instance. + * A hash table containing the graph instances, one per AudioChannel. */ static nsDataHashtable gGraphs; @@ -99,10 +99,6 @@ MediaStreamGraphImpl::FinishStream(MediaStream* aStream) STREAM_LOG(LogLevel::Debug, ("MediaStream %p will finish", aStream)); aStream->mFinished = true; aStream->mBuffer.AdvanceKnownTracksTime(STREAM_TIME_MAX); - // Force at least one more iteration of the control loop, since we rely - // on UpdateCurrentTimeForStreams to notify our listeners once the stream end - // has been reached. - EnsureNextIteration(); SetStreamOrderDirty(); } @@ -128,7 +124,7 @@ MediaStreamGraphImpl::AddStream(MediaStream* aStream) mSuspendedStreams.AppendElement(aStream); STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to the graph, in the suspended stream array", aStream)); } else { - aStream->mBufferStartTime = IterationEnd(); + aStream->mBufferStartTime = mProcessedTime; mStreams.AppendElement(aStream); STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to the graph", aStream)); } @@ -276,10 +272,10 @@ MediaStreamGraphImpl::GraphTimeToStreamTime(MediaStream* aStream, { MOZ_ASSERT(aTime <= mStateComputedTime, "Don't ask about times where we haven't made blocking decisions yet"); - if (aTime <= IterationEnd()) { + if (aTime <= mProcessedTime) { return std::max(0, aTime - aStream->mBufferStartTime); } - GraphTime t = IterationEnd(); + GraphTime t = mProcessedTime; StreamTime s = t - aStream->mBufferStartTime; while (t < aTime) { GraphTime end; @@ -307,7 +303,8 @@ MediaStreamGraphImpl::StreamTimeToGraphTime(MediaStream* aStream, if (aTime >= STREAM_TIME_MAX) { return GRAPH_TIME_MAX; } - MediaTime bufferElapsedToCurrentTime = IterationEnd() - aStream->mBufferStartTime; + MediaTime bufferElapsedToCurrentTime = + mProcessedTime - aStream->mBufferStartTime; if (aTime < bufferElapsedToCurrentTime || (aTime == bufferElapsedToCurrentTime && !(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL))) { return aTime + aStream->mBufferStartTime; @@ -316,7 +313,7 @@ MediaStreamGraphImpl::StreamTimeToGraphTime(MediaStream* aStream, MediaTime streamAmount = aTime - bufferElapsedToCurrentTime; NS_ASSERTION(streamAmount >= 0, "Can't answer queries before current time"); - GraphTime t = IterationEnd(); + GraphTime t = mProcessedTime; while (t < GRAPH_TIME_MAX) { if (!(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL) && streamAmount == 0) { return t; @@ -346,17 +343,43 @@ MediaStreamGraphImpl::StreamTimeToGraphTime(MediaStream* aStream, } GraphTime -MediaStreamGraphImpl::IterationEnd() +MediaStreamGraphImpl::IterationEnd() const { return CurrentDriver()->IterationEnd(); } void -MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, GraphTime aNextCurrentTime) +MediaStreamGraphImpl::StreamNotifyOutput(MediaStream* aStream) { - nsAutoTArray streamsReadyToFinish; - nsAutoTArray streamsWithOutput; + for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) { + MediaStreamListener* l = aStream->mListeners[j]; + l->NotifyOutput(this, mProcessedTime); + } +} +void +MediaStreamGraphImpl::StreamReadyToFinish(MediaStream* aStream) +{ + MOZ_ASSERT(aStream->mFinished); + MOZ_ASSERT(!aStream->mNotifiedFinished); + + // The stream is fully finished when all of its track data has been played + // out. + if (mProcessedTime >= + aStream->StreamTimeToGraphTime(aStream->GetStreamBuffer().GetAllTracksEnd())) { + aStream->mNotifiedFinished = true; + SetStreamOrderDirty(); + for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) { + MediaStreamListener* l = aStream->mListeners[j]; + l->NotifyEvent(this, MediaStreamListener::EVENT_FINISHED); + } + } +} + +void +MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, + GraphTime aNextCurrentTime) +{ nsTArray* runningAndSuspendedPair[2]; runningAndSuspendedPair[0] = &mStreams; runningAndSuspendedPair[1] = &mSuspendedStreams; @@ -397,17 +420,15 @@ MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, Gr if (runningAndSuspendedPair[array] == &mStreams) { bool streamHasOutput = blockedTime < aNextCurrentTime - aPrevCurrentTime; - // Make this an assertion when bug 957832 is fixed. - NS_WARN_IF_FALSE( - !streamHasOutput || !stream->mNotifiedFinished, + NS_ASSERTION(!streamHasOutput || !stream->mNotifiedFinished, "Shouldn't have already notified of finish *and* have output!"); if (streamHasOutput) { - streamsWithOutput.AppendElement(stream); + StreamNotifyOutput(stream); } if (stream->mFinished && !stream->mNotifiedFinished) { - streamsReadyToFinish.AppendElement(stream); + StreamReadyToFinish(stream); } } STREAM_LOG(LogLevel::Verbose, @@ -416,32 +437,6 @@ MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, Gr MediaTimeToSeconds(blockedTime))); } } - - for (uint32_t i = 0; i < streamsWithOutput.Length(); ++i) { - MediaStream* stream = streamsWithOutput[i]; - for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { - MediaStreamListener* l = stream->mListeners[j]; - l->NotifyOutput(this, IterationEnd()); - } - } - - for (uint32_t i = 0; i < streamsReadyToFinish.Length(); ++i) { - MediaStream* stream = streamsReadyToFinish[i]; - // The stream is fully finished when all of its track data has been played - // out. - if (IterationEnd() >= - stream->StreamTimeToGraphTime(stream->GetStreamBuffer().GetAllTracksEnd())) { - NS_WARN_IF_FALSE(stream->mNotifiedBlocked, - "Should've notified blocked=true for a fully finished stream"); - stream->mNotifiedFinished = true; - stream->mLastPlayedVideoFrame.SetNull(); - SetStreamOrderDirty(); - for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { - MediaStreamListener* l = stream->mListeners[j]; - l->NotifyEvent(this, MediaStreamListener::EVENT_FINISHED); - } - } - } } bool @@ -458,13 +453,13 @@ MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime, StreamTimeToGraphTime(aStream, aStream->GetBufferEnd(), INCLUDE_TRAILING_BLOCKED_INTERVAL); #ifdef DEBUG - if (bufferEnd < IterationEnd()) { + if (bufferEnd < mProcessedTime) { STREAM_LOG(LogLevel::Error, ("MediaStream %p underrun, " - "bufferEnd %f < IterationEnd() %f (%lld < %lld), Streamtime %lld", - aStream, MediaTimeToSeconds(bufferEnd), MediaTimeToSeconds(IterationEnd()), - bufferEnd, IterationEnd(), aStream->GetBufferEnd())); + "bufferEnd %f < mProcessedTime %f (%lld < %lld), Streamtime %lld", + aStream, MediaTimeToSeconds(bufferEnd), MediaTimeToSeconds(mProcessedTime), + bufferEnd, mProcessedTime, aStream->GetBufferEnd())); aStream->DumpTrackInfo(); - NS_ASSERTION(bufferEnd >= IterationEnd(), "Buffer underran"); + NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran"); } #endif // We should block after bufferEnd. @@ -766,8 +761,6 @@ MediaStreamGraphImpl::UpdateStreamOrder() void MediaStreamGraphImpl::RecomputeBlocking(GraphTime aEndBlockingDecisions) { - bool blockingDecisionsWillChange = false; - STREAM_LOG(LogLevel::Verbose, ("Media graph %p computing blocking for time %f", this, MediaTimeToSeconds(mStateComputedTime))); nsTArray* runningAndSuspendedPair[2]; @@ -789,37 +782,20 @@ MediaStreamGraphImpl::RecomputeBlocking(GraphTime aEndBlockingDecisions) t < aEndBlockingDecisions; t = end) { end = GRAPH_TIME_MAX; RecomputeBlockingAt(streamSet, t, aEndBlockingDecisions, &end); - if (end < GRAPH_TIME_MAX) { - blockingDecisionsWillChange = true; - } } } - - GraphTime end; - stream->mBlocked.GetAt(IterationEnd(), &end); - if (end < GRAPH_TIME_MAX) { - blockingDecisionsWillChange = true; - } } } STREAM_LOG(LogLevel::Verbose, ("Media graph %p computed blocking for interval %f to %f", this, MediaTimeToSeconds(mStateComputedTime), MediaTimeToSeconds(aEndBlockingDecisions))); - MOZ_ASSERT(aEndBlockingDecisions >= IterationEnd()); + MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime); // The next state computed time can be the same as the previous: it // means the driver would be have been blocking indefinitly, but the graph has // been woken up right after having been to sleep. - if (aEndBlockingDecisions < mStateComputedTime) { - printf("State time can't go backward %ld < %ld.\n", static_cast(aEndBlockingDecisions), static_cast(mStateComputedTime)); - } - + MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime); mStateComputedTime = aEndBlockingDecisions; - - if (blockingDecisionsWillChange) { - // Make sure we wake up to notify listeners about these changes. - EnsureNextIteration(); - } } void @@ -876,24 +852,6 @@ MediaStreamGraphImpl::RecomputeBlockingAt(const nsTArray& aStreams GraphTime aEndBlockingDecisions, GraphTime* aEnd) { - class MOZ_STACK_CLASS AfterLoop - { - public: - AfterLoop(MediaStream* aStream, GraphTime& aTime) - : mStream(aStream) - , mTime(aTime) - {} - - ~AfterLoop() - { - mStream->mBlocked.SetAtAndAfter(mTime, mStream->mBlockInThisPhase); - } - - private: - MediaStream* mStream; - GraphTime& mTime; - }; - for (uint32_t i = 0; i < aStreams.Length(); ++i) { MediaStream* stream = aStreams[i]; stream->mBlockInThisPhase = false; @@ -901,7 +859,6 @@ MediaStreamGraphImpl::RecomputeBlockingAt(const nsTArray& aStreams for (uint32_t i = 0; i < aStreams.Length(); ++i) { MediaStream* stream = aStreams[i]; - AfterLoop al(stream, aTime); if (stream->mFinished) { GraphTime endTime = StreamTimeToGraphTime(stream, @@ -937,8 +894,12 @@ MediaStreamGraphImpl::RecomputeBlockingAt(const nsTArray& aStreams continue; } } - NS_ASSERTION(*aEnd > aTime, "Failed to advance!"); + + for (uint32_t i = 0; i < aStreams.Length(); ++i) { + MediaStream* stream = aStreams[i]; + stream->mBlocked.SetAtAndAfter(aTime, stream->mBlockInThisPhase); + } } void @@ -959,6 +920,11 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTim { MOZ_ASSERT(mRealtime, "Should only attempt to create audio streams in real-time mode"); + if (aStream->mAudioOutputs.IsEmpty()) { + aStream->mAudioOutputStreams.Clear(); + return; + } + nsAutoTArray audioOutputStreamsFound; for (uint32_t i = 0; i < aStream->mAudioOutputStreams.Length(); ++i) { audioOutputStreamsFound.AppendElement(false); @@ -1156,69 +1122,140 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) if (aStream->mVideoOutputs.IsEmpty()) return; - // Display the next frame a bit early. This is better than letting the current - // frame be displayed for too long. Because depending on the GraphDriver in - // use, we can't really estimate the graph interval duration, we clamp it to - // the current state computed time. - GraphTime framePosition = IterationEnd() + MillisecondsToMediaTime(CurrentDriver()->IterationDuration()); - if (framePosition > mStateComputedTime) { -#ifdef DEBUG - if (std::abs(framePosition - mStateComputedTime) >= MillisecondsToMediaTime(5)) { - STREAM_LOG(LogLevel::Debug, ("Graph thread slowdown?")); - } -#endif - framePosition = mStateComputedTime; - } - MOZ_ASSERT(framePosition >= aStream->mBufferStartTime, "frame position before buffer?"); - StreamTime frameBufferTime = GraphTimeToStreamTime(aStream, framePosition); + TimeStamp currentTimeStamp = CurrentDriver()->GetCurrentTimeStamp(); + // Collect any new frames produced in this iteration. + nsAutoTArray newImages; + nsRefPtr blackImage; + + MOZ_ASSERT(mProcessedTime >= aStream->mBufferStartTime, "frame position before buffer?"); + StreamTime frameBufferTime = GraphTimeToStreamTime(aStream, mProcessedTime); + StreamTime bufferEndTime = GraphTimeToStreamTime(aStream, mStateComputedTime); StreamTime start; - const VideoFrame* frame = nullptr; - for (StreamBuffer::TrackIter tracks(aStream->GetStreamBuffer(), MediaSegment::VIDEO); - !tracks.IsEnded(); tracks.Next()) { - VideoSegment* segment = tracks->Get(); - StreamTime thisStart; - const VideoFrame* thisFrame = - segment->GetFrameAt(frameBufferTime, &thisStart); - if (thisFrame && thisFrame->GetImage()) { - start = thisStart; - frame = thisFrame; + const VideoChunk* chunk; + for ( ; + frameBufferTime < bufferEndTime; + frameBufferTime = start + chunk->GetDuration()) { + // Pick the last track that has a video chunk for the time, and + // schedule its frame. + chunk = nullptr; + for (StreamBuffer::TrackIter tracks(aStream->GetStreamBuffer(), + MediaSegment::VIDEO); + !tracks.IsEnded(); + tracks.Next()) { + VideoSegment* segment = tracks->Get(); + StreamTime thisStart; + const VideoChunk* thisChunk = + segment->FindChunkContaining(frameBufferTime, &thisStart); + if (thisChunk && thisChunk->mFrame.GetImage()) { + start = thisStart; + chunk = thisChunk; + } } + if (!chunk) + break; + + const VideoFrame* frame = &chunk->mFrame; + if (*frame == aStream->mLastPlayedVideoFrame) { + continue; + } + + Image* image = frame->GetImage(); + STREAM_LOG(LogLevel::Verbose, + ("MediaStream %p writing video frame %p (%dx%d)", + aStream, image, frame->GetIntrinsicSize().width, + frame->GetIntrinsicSize().height)); + // Schedule this frame after the previous frame finishes, instead of at + // its start time. These times only differ in the case of multiple + // tracks. + GraphTime frameTime = + StreamTimeToGraphTime(aStream, frameBufferTime, + INCLUDE_TRAILING_BLOCKED_INTERVAL); + TimeStamp targetTime = currentTimeStamp + + TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd())); + + if (frame->GetForceBlack()) { + if (!blackImage) { + blackImage = aStream->mVideoOutputs[0]-> + GetImageContainer()->CreateImage(ImageFormat::PLANAR_YCBCR); + if (blackImage) { + // Sets the image to a single black pixel, which will be scaled to + // fill the rendered size. + SetImageToBlackPixel(static_cast + (blackImage.get())); + } + } + if (blackImage) { + image = blackImage; + } + } + newImages.AppendElement(ImageContainer::NonOwningImage(image, targetTime)); + + aStream->mLastPlayedVideoFrame = *frame; } - if (!frame || *frame == aStream->mLastPlayedVideoFrame) + + if (!aStream->mLastPlayedVideoFrame.GetImage()) return; - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p writing video frame %p (%dx%d)", - aStream, frame->GetImage(), frame->GetIntrinsicSize().width, - frame->GetIntrinsicSize().height)); - GraphTime startTime = StreamTimeToGraphTime(aStream, - start, INCLUDE_TRAILING_BLOCKED_INTERVAL); - TimeStamp targetTime = CurrentDriver()->GetCurrentTimeStamp() + - TimeDuration::FromMilliseconds(double(startTime - IterationEnd())); + nsAutoTArray images; + bool haveMultipleImages = false; + for (uint32_t i = 0; i < aStream->mVideoOutputs.Length(); ++i) { VideoFrameContainer* output = aStream->mVideoOutputs[i]; - if (frame->GetForceBlack()) { - nsRefPtr image = - output->GetImageContainer()->CreateImage(ImageFormat::PLANAR_YCBCR); - if (image) { - // Sets the image to a single black pixel, which will be scaled to fill - // the rendered size. - SetImageToBlackPixel(static_cast(image.get())); - } - output->SetCurrentFrame(frame->GetIntrinsicSize(), image, - targetTime); - } else { - output->SetCurrentFrame(frame->GetIntrinsicSize(), frame->GetImage(), - targetTime); + // Find previous frames that may still be valid. + nsAutoTArray previousImages; + output->GetImageContainer()->GetCurrentImages(&previousImages); + uint32_t j = previousImages.Length(); + if (j) { + // Re-use the most recent frame before currentTimeStamp and subsequent, + // always keeping at least one frame. + do { + --j; + } while (j > 0 && previousImages[j].mTimeStamp > currentTimeStamp); } + if (previousImages.Length() - j + newImages.Length() > 1) { + haveMultipleImages = true; + } + + // Don't update if there are no changes. + if (j == 0 && newImages.IsEmpty()) + continue; + + for ( ; j < previousImages.Length(); ++j) { + const auto& image = previousImages[j]; + // Cope with potential clock skew with AudioCallbackDriver. + if (newImages.Length() && image.mTimeStamp > newImages[0].mTimeStamp) { + STREAM_LOG(LogLevel::Warning, + ("Dropping %u video frames due to clock skew", + unsigned(previousImages.Length() - j))); + break; + } + + images.AppendElement(ImageContainer:: + NonOwningImage(image.mImage, + image.mTimeStamp, image.mFrameID)); + } + + // Add the frames from this iteration. + for (auto& image : newImages) { + image.mFrameID = output->NewFrameID(); + images.AppendElement(image); + } + output->SetCurrentFrames(aStream->mLastPlayedVideoFrame.GetIntrinsicSize(), + images); nsCOMPtr event = new VideoFrameContainerInvalidateRunnable(output); DispatchToMainThreadAfterStreamStateUpdate(event.forget()); + + images.ClearAndRetainStorage(); } - if (!aStream->mNotifiedFinished) { - aStream->mLastPlayedVideoFrame = *frame; + + // If the stream has finished and the timestamps of all frames have expired + // then no more updates are required. + if (aStream->mFinished && !haveMultipleImages) { + aStream->mLastPlayedVideoFrame.SetNull(); } } @@ -1254,11 +1291,11 @@ MediaStreamGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) StreamUpdate* update = mStreamUpdates.AppendElement(); update->mStream = stream; update->mNextMainThreadCurrentTime = - GraphTimeToStreamTime(stream, IterationEnd()); + GraphTimeToStreamTime(stream, mProcessedTime); update->mNextMainThreadFinished = stream->mNotifiedFinished; } if (!mPendingUpdateRunnables.IsEmpty()) { - mUpdateRunnables.MoveElementsFrom(mPendingUpdateRunnables); + mUpdateRunnables.AppendElements(Move(mPendingUpdateRunnables)); } } @@ -1459,8 +1496,7 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) } bool -MediaStreamGraphImpl::OneIteration(GraphTime aFrom, GraphTime aTo, - GraphTime aStateFrom, GraphTime aStateEnd) +MediaStreamGraphImpl::OneIteration(GraphTime aStateEnd) { { MonitorAutoLock lock(mMemoryReportMonitor); @@ -1480,12 +1516,14 @@ MediaStreamGraphImpl::OneIteration(GraphTime aFrom, GraphTime aTo, } } - UpdateCurrentTimeForStreams(aFrom, aTo); - + GraphTime stateFrom = mStateComputedTime; GraphTime stateEnd = std::min(aStateEnd, mEndTime); UpdateGraph(stateEnd); - Process(aStateFrom, stateEnd); + Process(stateFrom, stateEnd); + mProcessedTime = stateEnd; + + UpdateCurrentTimeForStreams(stateFrom, stateEnd); // Send updates to the main thread and wait for the next control loop // iteration. @@ -1630,7 +1668,6 @@ public: virtual void Run() override { mStream->GraphImpl()->AddStream(mStream); - mStream->Init(); } virtual void RunDuringShutdown() override { @@ -1774,7 +1811,7 @@ MediaStreamGraphImpl::RunInStableState(bool aSourceIsMSG) // Defer calls to RunDuringShutdown() to happen while mMonitor is not held. for (uint32_t i = 0; i < mBackMessageQueue.Length(); ++i) { MessageBlock& mb = mBackMessageQueue[i]; - controlMessagesToRunDuringShutdown.MoveElementsFrom(mb.mMessages); + controlMessagesToRunDuringShutdown.AppendElements(Move(mb.mMessages)); } mBackMessageQueue.Clear(); MOZ_ASSERT(mCurrentTaskMessageQueue.IsEmpty()); @@ -1944,15 +1981,6 @@ MediaStream::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } -void -MediaStream::Init() -{ - MediaStreamGraphImpl* graph = GraphImpl(); - mBlocked.SetAtAndAfter(graph->IterationEnd(), true); - mExplicitBlockerCount.SetAtAndAfter(graph->IterationEnd(), true); - mExplicitBlockerCount.SetAtAndAfter(graph->mStateComputedTime, false); -} - MediaStreamGraphImpl* MediaStream::GraphImpl() { @@ -2456,7 +2484,7 @@ void SourceMediaStream::FinishAddTracks() { MutexAutoLock lock(mMutex); - mUpdateTracks.MoveElementsFrom(mPendingTracks); + mUpdateTracks.AppendElements(Move(mPendingTracks)); if (GraphImpl()) { GraphImpl()->EnsureNextIteration(); } @@ -2861,9 +2889,8 @@ ProcessedMediaStream::DestroyImpl() // SetStreamOrderDirty(), for other reasons. } -MediaStreamGraphImpl::MediaStreamGraphImpl(bool aRealtime, +MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested, TrackRate aSampleRate, - bool aStartWithAudioDriver, dom::AudioChannel aChannel) : MediaStreamGraph(aSampleRate) , mPortCount(0) @@ -2878,7 +2905,7 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(bool aRealtime, , mFlushSourcesOnNextIteration(false) , mDetectedNotRunning(false) , mPostedRunInStableState(false) - , mRealtime(aRealtime) + , mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER) , mNonRealtimeProcessing(false) , mStreamOrderDirty(false) , mLatencyLog(AsyncLatencyLogger::Get()) @@ -2899,7 +2926,7 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(bool aRealtime, } if (mRealtime) { - if (aStartWithAudioDriver) { + if (aDriverRequested == AUDIO_THREAD_DRIVER) { AudioCallbackDriver* driver = new AudioCallbackDriver(this, aChannel); mDriver = driver; mMixer.AddCallback(driver); @@ -2956,7 +2983,7 @@ MediaStreamGraphShutdownObserver::Observe(nsISupports *aSubject, } MediaStreamGraph* -MediaStreamGraph::GetInstance(bool aStartWithAudioDriver, +MediaStreamGraph::GetInstance(MediaStreamGraph::GraphDriverType aGraphDriverRequested, dom::AudioChannel aChannel) { NS_ASSERTION(NS_IsMainThread(), "Main thread only"); @@ -2972,10 +2999,15 @@ MediaStreamGraph::GetInstance(bool aStartWithAudioDriver, CubebUtils::InitPreferredSampleRate(); - graph = new MediaStreamGraphImpl(true, CubebUtils::PreferredSampleRate(), aStartWithAudioDriver, aChannel); + graph = new MediaStreamGraphImpl(aGraphDriverRequested, + CubebUtils::PreferredSampleRate(), + aChannel); + gGraphs.Put(channel, graph); - STREAM_LOG(LogLevel::Debug, ("Starting up MediaStreamGraph %p", graph)); + STREAM_LOG(LogLevel::Debug, + ("Starting up MediaStreamGraph %p for channel %s", + graph, AudioChannelValues::strings[channel])); } return graph; @@ -2986,7 +3018,10 @@ MediaStreamGraph::CreateNonRealtimeInstance(TrackRate aSampleRate) { NS_ASSERTION(NS_IsMainThread(), "Main thread only"); - MediaStreamGraphImpl* graph = new MediaStreamGraphImpl(false, aSampleRate); + MediaStreamGraphImpl* graph = + new MediaStreamGraphImpl(OFFLINE_THREAD_DRIVER, + aSampleRate, + AudioChannel::Normal); STREAM_LOG(LogLevel::Debug, ("Starting up Offline MediaStreamGraph %p", graph)); @@ -3299,7 +3334,7 @@ MediaStreamGraphImpl::MoveStreams(AudioContextOperation aAudioContextOperation, // set their buffer start time to the appropriate value now: if (aAudioContextOperation == AudioContextOperation::Resume && stream->mBufferStartTime == START_TIME_DELAYED) { - stream->mBufferStartTime = IterationEnd(); + stream->mBufferStartTime = mProcessedTime; } stream->remove(); diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 8e23eaabec..08b26a77f5 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -313,7 +313,8 @@ class CameraPreviewMediaStream; * for those objects in arbitrary order and the MediaStreamGraph has to be able * to handle this. */ -class MediaStream : public mozilla::LinkedListElement { +class MediaStream : public mozilla::LinkedListElement +{ public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream) @@ -432,8 +433,6 @@ public: virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; } virtual CameraPreviewMediaStream* AsCameraPreviewStream() { return nullptr; } - // media graph thread only - void Init(); // These Impl methods perform the core functionality of the control methods // above, on the media graph thread. /** @@ -462,6 +461,9 @@ public: } void RemoveVideoOutputImpl(VideoFrameContainer* aContainer) { + // Ensure that any frames currently queued for playback by the compositor + // are removed. + aContainer->ClearFutureFrames(); mVideoOutputs.RemoveElement(aContainer); } void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta) @@ -631,8 +633,8 @@ protected: }; nsTArray mAudioOutputs; nsTArray > mVideoOutputs; - // We record the last played video frame to avoid redundant setting - // of the current video frame. + // We record the last played video frame to avoid playing the frame again + // with a different frame id. VideoFrame mLastPlayedVideoFrame; // The number of times this stream has been explicitly blocked by the control // API, minus the number of times it has been explicitly unblocked. @@ -654,7 +656,8 @@ protected: // Where audio output is going. There is one AudioOutputStream per // audio track. - struct AudioOutputStream { + struct AudioOutputStream + { // When we started audio playback for this track. // Add mStream->GetPosition() to find the current audio playback position. GraphTime mAudioPlaybackStartTime; @@ -724,7 +727,8 @@ protected: * Audio and video can be written on any thread, but you probably want to * always write from the same thread to avoid unexpected interleavings. */ -class SourceMediaStream : public MediaStream { +class SourceMediaStream : public MediaStream +{ public: explicit SourceMediaStream(DOMMediaStream* aWrapper) : MediaStream(aWrapper), @@ -971,7 +975,8 @@ protected: * the Destroy message is processed on the graph manager thread we disconnect * the port and drop the graph's reference, destroying the object. */ -class MediaInputPort final { +class MediaInputPort final +{ private: // Do not call this constructor directly. Instead call aDest->AllocateInputPort. MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest, @@ -1088,7 +1093,8 @@ private: * its output. The details of how the output is produced are handled by * subclasses overriding the ProcessInput method. */ -class ProcessedMediaStream : public MediaStream { +class ProcessedMediaStream : public MediaStream +{ public: explicit ProcessedMediaStream(DOMMediaStream* aWrapper) : MediaStream(aWrapper), mAutofinish(false) @@ -1192,21 +1198,33 @@ protected: }; /** - * Initially, at least, we will have a singleton MediaStreamGraph per - * process. Each OfflineAudioContext object creates its own MediaStreamGraph - * object too. + * There can be multiple MediaStreamGraph per process: one per AudioChannel. + * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph + * object too.. */ -class MediaStreamGraph { +class MediaStreamGraph +{ public: + // We ensure that the graph current time advances in multiples of // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that // never blocks and has a track with the ideal audio rate will produce audio // in multiples of the block size. // + // Initializing an graph that outputs audio can be quite long on some + // platforms. Code that want to output audio at some point can express the + // fact that they will need an audio stream at some point by passing + // AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that + // the graph starts with the right driver. + enum GraphDriverType { + AUDIO_THREAD_DRIVER, + SYSTEM_THREAD_DRIVER, + OFFLINE_THREAD_DRIVER + }; // Main thread only - static MediaStreamGraph* GetInstance(bool aStartWithAudioDriver = false, - dom::AudioChannel aChannel = dom::AudioChannel::Normal); + static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested, + dom::AudioChannel aChannel); static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate); // Idempotent static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph); diff --git a/dom/media/MediaStreamGraphImpl.h b/dom/media/MediaStreamGraphImpl.h index 89f465bfbc..b9a8dfc1f8 100644 --- a/dom/media/MediaStreamGraphImpl.h +++ b/dom/media/MediaStreamGraphImpl.h @@ -40,7 +40,8 @@ struct StreamUpdate { * This represents a message passed from the main thread to the graph thread. * A ControlMessage always has a weak reference a particular affected stream. */ -class ControlMessage { +class ControlMessage +{ public: explicit ControlMessage(MediaStream* aStream) : mStream(aStream) { @@ -52,8 +53,8 @@ public: MOZ_COUNT_DTOR(ControlMessage); } // Do the action of this message on the MediaStreamGraph thread. Any actions - // affecting graph processing should take effect at mStateComputedTime. - // All stream data for times < mStateComputedTime has already been + // affecting graph processing should take effect at mProcessedTime. + // All stream data for times < mProcessedTime has already been // computed. virtual void Run() = 0; // When we're shutting down the application, most messages are ignored but @@ -69,7 +70,8 @@ protected: MediaStream* mStream; }; -class MessageBlock { +class MessageBlock +{ public: nsTArray > mMessages; }; @@ -83,22 +85,23 @@ public: * OfflineAudioContext object. */ class MediaStreamGraphImpl : public MediaStreamGraph, - public nsIMemoryReporter { + public nsIMemoryReporter +{ public: NS_DECL_THREADSAFE_ISUPPORTS NS_DECL_NSIMEMORYREPORTER /** - * Set aRealtime to true in order to create a MediaStreamGraph which provides - * support for real-time audio and video. Set it to false in order to create - * a non-realtime instance which just churns through its inputs and produces - * output. Those objects currently only support audio, and are used to - * implement OfflineAudioContext. They do not support MediaStream inputs. + * Use aGraphDriverRequested with SYSTEM_THREAD_DRIVER or AUDIO_THREAD_DRIVER + * to create a MediaStreamGraph which provides support for real-time audio + * and/or video. Set it to false in order to create a non-realtime instance + * which just churns through its inputs and produces output. Those objects + * currently only support audio, and are used to implement + * OfflineAudioContext. They do not support MediaStream inputs. */ - explicit MediaStreamGraphImpl(bool aRealtime, + explicit MediaStreamGraphImpl(GraphDriverType aGraphDriverRequested, TrackRate aSampleRate, - bool aStartWithAudioDriver = false, - dom::AudioChannel aChannel = dom::AudioChannel::Normal); + dom::AudioChannel aChannel); /** * Unregisters memory reporting and deletes this instance. This should be @@ -148,7 +151,8 @@ public: void Init(); // The following methods run on the graph thread (or possibly the main thread if // mLifecycleState > LIFECYCLE_RUNNING) - void AssertOnGraphThreadOrNotRunning() { + void AssertOnGraphThreadOrNotRunning() const + { // either we're on the right thread (and calling CurrentDriver() is safe), // or we're going to assert anyways, so don't cross-check CurrentDriver #ifdef DEBUG @@ -168,23 +172,25 @@ public: */ void DoIteration(); - bool OneIteration(GraphTime aFrom, GraphTime aTo, - GraphTime aStateFrom, GraphTime aStateEnd); + bool OneIteration(GraphTime aStateEnd); - bool Running() { + bool Running() const + { mMonitor.AssertCurrentThreadOwns(); return mLifecycleState == LIFECYCLE_RUNNING; } // Get the message queue, from the current GraphDriver thread. - nsTArray& MessageQueue() { + nsTArray& MessageQueue() + { mMonitor.AssertCurrentThreadOwns(); return mFrontMessageQueue; } /* This is the end of the current iteration, that is, the current time of the * graph. */ - GraphTime IterationEnd(); + GraphTime IterationEnd() const; + /** * Ensure there is an event posted to the main thread to run RunInStableState. * mMonitor must be held. @@ -219,7 +225,8 @@ public: */ void UpdateGraph(GraphTime aEndBlockingDecisions); - void SwapMessageQueues() { + void SwapMessageQueues() + { mMonitor.AssertCurrentThreadOwns(); mFrontMessageQueue.SwapElements(mBackMessageQueue); } @@ -302,6 +309,7 @@ public: * for all streams. */ void RecomputeBlocking(GraphTime aEndBlockingDecisions); + // The following methods are used to help RecomputeBlocking. /** * If aStream isn't already in aStreams, add it and recursively call @@ -360,9 +368,11 @@ public: * when we don't know whether it's blocked or not, we assume it's not blocked. */ StreamTime GraphTimeToStreamTimeOptimistic(MediaStream* aStream, GraphTime aTime); - enum { + enum + { INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01 }; + /** * Given a stream time aTime, convert it to a graph time taking into * account the time during which aStream is scheduled to be blocked. @@ -405,7 +415,7 @@ public: /** * Returns true when there are no active streams. */ - bool IsEmpty() + bool IsEmpty() const { return mStreams.IsEmpty() && mSuspendedStreams.IsEmpty() && mPortCount == 0; } @@ -432,20 +442,23 @@ public: } // Always stereo for now. - uint32_t AudioChannelCount() { return 2; } + uint32_t AudioChannelCount() const { return 2; } - double MediaTimeToSeconds(GraphTime aTime) + double MediaTimeToSeconds(GraphTime aTime) const { - NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time"); + NS_ASSERTION(aTime > -STREAM_TIME_MAX && aTime <= STREAM_TIME_MAX, + "Bad time"); return static_cast(aTime)/GraphRate(); } - GraphTime SecondsToMediaTime(double aS) + + GraphTime SecondsToMediaTime(double aS) const { NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX, "Bad seconds"); return GraphRate() * aS; } - GraphTime MillisecondsToMediaTime(int32_t aMS) + + GraphTime MillisecondsToMediaTime(int32_t aMS) const { return RateConvertTicksRoundDown(GraphRate(), 1000, aMS); } @@ -460,7 +473,8 @@ public: /** * Not safe to call off the MediaStreamGraph thread unless monitor is held! */ - GraphDriver* CurrentDriver() { + GraphDriver* CurrentDriver() const + { AssertOnGraphThreadOrNotRunning(); return mDriver; } @@ -478,16 +492,19 @@ public: * We can also switch from Revive() (on MainThread), in which case the * monitor is held */ - void SetCurrentDriver(GraphDriver* aDriver) { + void SetCurrentDriver(GraphDriver* aDriver) + { AssertOnGraphThreadOrNotRunning(); mDriver = aDriver; } - Monitor& GetMonitor() { + Monitor& GetMonitor() + { return mMonitor; } - void EnsureNextIteration() { + void EnsureNextIteration() + { mNeedAnotherIteration = true; // atomic if (mGraphDriverAsleep) { // atomic MonitorAutoLock mon(mMonitor); @@ -495,7 +512,8 @@ public: } } - void EnsureNextIterationLocked() { + void EnsureNextIterationLocked() + { mNeedAnotherIteration = true; // atomic if (mGraphDriverAsleep) { // atomic CurrentDriver()->WakeUp(); // Might not be the same driver; might have woken already @@ -543,11 +561,16 @@ public: */ uint32_t mFirstCycleBreaker; /** - * Blocking decisions and all stream contents have been computed up to this - * time. The next batch of updates from the main thread will be processed - * at this time. + * Blocking decisions have been computed up to this time. + * Between each iteration, this is the same as mProcessedTime. */ GraphTime mStateComputedTime = 0; + /** + * All stream contents have been computed up to this time. + * The next batch of updates from the main thread will be processed + * at this time. This is behind mStateComputedTime during processing. + */ + GraphTime mProcessedTime = 0; /** * Date of the last time we updated the main thread with the graph state. */ @@ -590,7 +613,8 @@ public: nsTArray mBackMessageQueue; /* True if there will messages to process if we swap the message queues. */ - bool MessagesQueued() { + bool MessagesQueued() const + { mMonitor.AssertCurrentThreadOwns(); return !mBackMessageQueue.IsEmpty(); } @@ -617,7 +641,8 @@ public: * This should be kept in sync with the LifecycleState_str array in * MediaStreamGraph.cpp */ - enum LifecycleState { + enum LifecycleState + { // The graph thread hasn't started yet. LIFECYCLE_THREAD_NOT_STARTED, // RunThread() is running normally. @@ -710,6 +735,10 @@ public: private: virtual ~MediaStreamGraphImpl(); + void StreamNotifyOutput(MediaStream* aStream); + + void StreamReadyToFinish(MediaStream* aStream); + MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf) /** diff --git a/dom/media/VideoFrameContainer.cpp b/dom/media/VideoFrameContainer.cpp index 280a73fd3b..5cf3c868c5 100644 --- a/dom/media/VideoFrameContainer.cpp +++ b/dom/media/VideoFrameContainer.cpp @@ -93,7 +93,24 @@ void VideoFrameContainer::ClearCurrentFrame() mImageContainer->GetCurrentImages(&kungFuDeathGrip); mImageContainer->ClearAllImages(); - mImageSizeChanged = false; +} + +void VideoFrameContainer::ClearFutureFrames() +{ + MutexAutoLock lock(mMutex); + + // See comment in SetCurrentFrame for the reasoning behind + // using a kungFuDeathGrip here. + nsTArray kungFuDeathGrip; + mImageContainer->GetCurrentImages(&kungFuDeathGrip); + + if (!kungFuDeathGrip.IsEmpty()) { + nsTArray currentFrame; + const ImageContainer::OwningImage& img = kungFuDeathGrip[0]; + currentFrame.AppendElement(ImageContainer::NonOwningImage(img.mImage, + img.mTimeStamp, img.mFrameID, img.mProducerID)); + mImageContainer->SetCurrentImages(currentFrame); + } } ImageContainer* VideoFrameContainer::GetImageContainer() { diff --git a/dom/media/VideoFrameContainer.h b/dom/media/VideoFrameContainer.h index e890419737..795def76c1 100644 --- a/dom/media/VideoFrameContainer.h +++ b/dom/media/VideoFrameContainer.h @@ -52,10 +52,22 @@ public: } void ClearCurrentFrame(); + // Make the current frame the only frame in the container, i.e. discard + // all future frames. + void ClearFutureFrames(); // Time in seconds by which the last painted video frame was late by. // E.g. if the last painted frame should have been painted at time t, // but was actually painted at t+n, this returns n in seconds. Threadsafe. double GetFrameDelay(); + + // Returns a new frame ID for SetCurrentFrames(). The client must either + // call this on only one thread or provide barriers. Do not use together + // with SetCurrentFrame(). + ImageContainer::FrameID NewFrameID() + { + return ++mFrameID; + } + // Call on main thread enum { INVALIDATE_DEFAULT, @@ -83,8 +95,8 @@ protected: // specifies that the Image should be stretched to have the correct aspect // ratio. gfxIntSize mIntrinsicSize; - // For SetCurrentFrame callers we maintain our own mFrameID which is auto- - // incremented at every SetCurrentFrame. + // We maintain our own mFrameID which is auto-incremented at every + // SetCurrentFrame() or NewFrameID() call. ImageContainer::FrameID mFrameID; // True when the intrinsic size has been changed by SetCurrentFrame() since // the last call to Invalidate(). diff --git a/dom/media/VideoSegment.h b/dom/media/VideoSegment.h index fed071c1de..36def06d9b 100644 --- a/dom/media/VideoSegment.h +++ b/dom/media/VideoSegment.h @@ -117,14 +117,6 @@ public: StreamTime aDuration, const IntSize& aIntrinsicSize, bool aForceBlack = false); - const VideoFrame* GetFrameAt(StreamTime aOffset, StreamTime* aStart = nullptr) - { - VideoChunk* c = FindChunkContaining(aOffset, aStart); - if (!c) { - return nullptr; - } - return &c->mFrame; - } const VideoFrame* GetLastFrame(StreamTime* aStart = nullptr) { VideoChunk* c = GetLastChunk(); diff --git a/dom/media/test/manifest.js b/dom/media/test/manifest.js index 0c634b28be..82ce7928a3 100644 --- a/dom/media/test/manifest.js +++ b/dom/media/test/manifest.js @@ -891,6 +891,12 @@ function mediaTestCleanup(callback) { SpecialPowers.exactGC(window, cb); } +// B2G emulator and Android 2.3 are condidered slow platforms +function isSlowPlatform() { + return SpecialPowers.Services.appinfo.name == "B2G" || + navigator.userAgent.indexOf("Mobile") != -1 && androidVersion == 10; +} + (function() { SimpleTest.requestFlakyTimeout("untriaged"); diff --git a/dom/media/test/mochitest.ini b/dom/media/test/mochitest.ini index 5916ef893d..da05b8ade8 100644 --- a/dom/media/test/mochitest.ini +++ b/dom/media/test/mochitest.ini @@ -496,7 +496,9 @@ skip-if = (toolkit == 'android' && processor == 'x86') #timeout x86 only bug 914 [test_standalone.html] skip-if = (toolkit == 'android' && processor == 'x86') #x86 only bug 914439 [test_streams_autoplay.html] -tags=msg +tags=msg capturestream +[test_streams_capture_origin.html] +tags=msg capturestream [test_streams_element_capture.html] #x86 only bug 914439, b2g desktop bug 752796 skip-if = (toolkit == 'android' && processor == 'x86') || (buildapp == 'b2g' && toolkit != 'gonk') diff --git a/dom/media/test/test_fragment_noplay.html b/dom/media/test/test_fragment_noplay.html index b24724816a..802ec08c95 100644 --- a/dom/media/test/test_fragment_noplay.html +++ b/dom/media/test/test_fragment_noplay.html @@ -12,6 +12,11 @@
 
+  
+  
+
+
+Mozilla Bug 1189506
+

+ + + + +
+
+
+ + diff --git a/dom/media/test/test_streams_element_capture.html b/dom/media/test/test_streams_element_capture.html index 944d42b0d8..9e1f579a12 100644 --- a/dom/media/test/test_streams_element_capture.html +++ b/dom/media/test/test_streams_element_capture.html @@ -9,7 +9,13 @@
 
 
diff --git a/dom/media/test/test_streams_element_capture_createObjectURL.html b/dom/media/test/test_streams_element_capture_createObjectURL.html index d33a8f080e..b70f351603 100644 --- a/dom/media/test/test_streams_element_capture_createObjectURL.html +++ b/dom/media/test/test_streams_element_capture_createObjectURL.html @@ -21,6 +21,11 @@ function checkDrawImage(vout) { is(imgData.data[3], 255, "Check video frame pixel has been drawn"); } +function isGreaterThanOrEqualEps(a, b, msg) { + ok(a >= b - 0.01, + "Got " + a + ", expected at least " + b + "; " + msg); +} + function startTest(test, token) { manager.started(token); @@ -36,8 +41,8 @@ function startTest(test, token) { var checkEnded = function(test, vout, stream) { return function() { is(stream.currentTime, vout.currentTime, test.name + " stream final currentTime"); if (test.duration) { - ok(Math.abs(vout.currentTime - test.duration) < 0.1, - test.name + " current time at end: " + vout.currentTime + " should be: " + test.duration); + isGreaterThanOrEqualEps(vout.currentTime, test.duration, + test.name + " current time at end"); } is(vout.readyState, vout.HAVE_CURRENT_DATA, test.name + " checking readyState"); ok(vout.ended, test.name + " checking playback has ended"); diff --git a/dom/media/test/test_streams_element_capture_reset.html b/dom/media/test/test_streams_element_capture_reset.html index 1ef454b2d5..c1ec501b3e 100644 --- a/dom/media/test/test_streams_element_capture_reset.html +++ b/dom/media/test/test_streams_element_capture_reset.html @@ -39,19 +39,24 @@ function isWithinEps(a, b, msg) { "Got " + a + ", expected " + b + "; " + msg); } +function isGreaterThanOrEqualEps(a, b, msg) { + ok(a >= b - 0.01, + "Got " + a + ", expected at least " + b + "; " + msg); +} + function startTest(test) { var seekTime = test.duration/2; function endedAfterReplay() { - isWithinEps(v.currentTime, test.duration, "checking v.currentTime at third 'ended' event"); - isWithinEps(vout.currentTime, (test.duration - seekTime) + test.duration*2, + isGreaterThanOrEqualEps(v.currentTime, test.duration, "checking v.currentTime at third 'ended' event"); + isGreaterThanOrEqualEps(vout.currentTime, (test.duration - seekTime) + test.duration*2, "checking vout.currentTime after seeking, playing through and reloading"); SimpleTest.finish(); }; function endedAfterSeek() { - isWithinEps(v.currentTime, test.duration, "checking v.currentTime at second 'ended' event"); - isWithinEps(vout.currentTime, (test.duration - seekTime) + test.duration, + isGreaterThanOrEqualEps(v.currentTime, test.duration, "checking v.currentTime at second 'ended' event"); + isGreaterThanOrEqualEps(vout.currentTime, (test.duration - seekTime) + test.duration, "checking vout.currentTime after seeking and playing through again"); v.removeEventListener("ended", endedAfterSeek, false); v.addEventListener("ended", endedAfterReplay, false); @@ -60,8 +65,8 @@ function startTest(test) { }; function seeked() { - isWithinEps(v.currentTime, seekTime, "Finished seeking"); - isWithinEps(vout.currentTime, test.duration, + isGreaterThanOrEqualEps(v.currentTime, seekTime, "Finished seeking"); + isGreaterThanOrEqualEps(vout.currentTime, test.duration, "checking vout.currentTime has not changed after seeking"); v.removeEventListener("seeked", seeked, false); function dontPlayAgain() { @@ -80,8 +85,8 @@ function startTest(test) { return; } - isWithinEps(vout.currentTime, test.duration, "checking vout.currentTime at first 'ended' event"); - isWithinEps(v.currentTime, test.duration, "checking v.currentTime at first 'ended' event"); + isGreaterThanOrEqualEps(vout.currentTime, test.duration, "checking vout.currentTime at first 'ended' event"); + isGreaterThanOrEqualEps(v.currentTime, test.duration, "checking v.currentTime at first 'ended' event"); is(vout.ended, false, "checking vout has not ended"); is(vout_untilended.ended, true, "checking vout_untilended has actually ended"); diff --git a/dom/media/webaudio/AudioDestinationNode.cpp b/dom/media/webaudio/AudioDestinationNode.cpp index bb041e9775..24a68fa178 100644 --- a/dom/media/webaudio/AudioDestinationNode.cpp +++ b/dom/media/webaudio/AudioDestinationNode.cpp @@ -325,10 +325,9 @@ AudioDestinationNode::AudioDestinationNode(AudioContext* aContext, , mExtraCurrentTimeUpdatedSinceLastStableState(false) , mCaptured(false) { - bool startWithAudioDriver = true; MediaStreamGraph* graph = aIsOffline ? MediaStreamGraph::CreateNonRealtimeInstance(aSampleRate) : - MediaStreamGraph::GetInstance(startWithAudioDriver, aChannel); + MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER, aChannel); AudioNodeEngine* engine = aIsOffline ? new OfflineDestinationNodeEngine(this, aNumberOfChannels, aLength, aSampleRate) : diff --git a/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp b/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp index f471711843..0172e1e3d0 100644 --- a/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp +++ b/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp @@ -28,7 +28,10 @@ MediaStreamAudioDestinationNode::MediaStreamAudioDestinationNode(AudioContext* a 2, ChannelCountMode::Explicit, ChannelInterpretation::Speakers) - , mDOMStream(DOMAudioNodeMediaStream::CreateTrackUnionStream(GetOwner(), this)) + , mDOMStream( + DOMAudioNodeMediaStream::CreateTrackUnionStream(GetOwner(), + this, + aContext->Graph())) { // Ensure an audio track with the correct ID is exposed to JS mDOMStream->CreateDOMTrack(AudioNodeStream::AUDIO_TRACK, MediaSegment::AUDIO); diff --git a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp index e216633c6c..68b456b9e6 100644 --- a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp @@ -219,7 +219,7 @@ MediaEngineCameraVideoSource::ChooseCapability( } } if (!candidateSet.Length()) { - candidateSet.MoveElementsFrom(rejects); + candidateSet.AppendElements(Move(rejects)); } } } diff --git a/dom/media/webspeech/synth/SpeechSynthesis.cpp b/dom/media/webspeech/synth/SpeechSynthesis.cpp index e09537685a..c72600b42c 100644 --- a/dom/media/webspeech/synth/SpeechSynthesis.cpp +++ b/dom/media/webspeech/synth/SpeechSynthesis.cpp @@ -72,6 +72,7 @@ NS_IMPL_CYCLE_COLLECTING_RELEASE(SpeechSynthesis) SpeechSynthesis::SpeechSynthesis(nsPIDOMWindow* aParent) : mParent(aParent) + , mHoldQueue(false) { MOZ_ASSERT(aParent->IsInnerWindow()); } @@ -110,21 +111,20 @@ SpeechSynthesis::Pending() const bool SpeechSynthesis::Speaking() const { - if (mSpeechQueue.IsEmpty()) { - return false; + if (!mSpeechQueue.IsEmpty() && + mSpeechQueue.ElementAt(0)->GetState() == SpeechSynthesisUtterance::STATE_SPEAKING) { + return true; } - return mSpeechQueue.ElementAt(0)->GetState() == SpeechSynthesisUtterance::STATE_SPEAKING; + // Returns global speaking state if global queue is enabled. Or false. + return nsSynthVoiceRegistry::GetInstance()->IsSpeaking(); } bool SpeechSynthesis::Paused() const { - if (mSpeechQueue.IsEmpty()) { - return false; - } - - return mSpeechQueue.ElementAt(0)->IsPaused(); + return mHoldQueue || (mCurrentTask && mCurrentTask->IsPrePaused()) || + (!mSpeechQueue.IsEmpty() && mSpeechQueue.ElementAt(0)->IsPaused()); } void @@ -138,7 +138,7 @@ SpeechSynthesis::Speak(SpeechSynthesisUtterance& aUtterance) mSpeechQueue.AppendElement(&aUtterance); aUtterance.mState = SpeechSynthesisUtterance::STATE_PENDING; - if (mSpeechQueue.Length() == 1 && !mCurrentTask) { + if (mSpeechQueue.Length() == 1 && !mCurrentTask && !mHoldQueue) { AdvanceQueue(); } } @@ -180,29 +180,47 @@ SpeechSynthesis::AdvanceQueue() void SpeechSynthesis::Cancel() { - if (mCurrentTask) { - if (mSpeechQueue.Length() > 1) { - // Remove all queued utterances except for current one. - mSpeechQueue.RemoveElementsAt(1, mSpeechQueue.Length() - 1); - } + if (!mSpeechQueue.IsEmpty() && + mSpeechQueue.ElementAt(0)->GetState() == SpeechSynthesisUtterance::STATE_SPEAKING) { + // Remove all queued utterances except for current one, we will remove it + // in OnEnd + mSpeechQueue.RemoveElementsAt(1, mSpeechQueue.Length() - 1); + } else { + mSpeechQueue.Clear(); + } - mCurrentTask->Cancel(); + if (mCurrentTask) { + mCurrentTask->Cancel(); } } void SpeechSynthesis::Pause() { - if (mCurrentTask && !Paused() && (Speaking() || Pending())) { + if (Paused()) { + return; + } + + if (mCurrentTask && + mSpeechQueue.ElementAt(0)->GetState() != SpeechSynthesisUtterance::STATE_ENDED) { mCurrentTask->Pause(); + } else { + mHoldQueue = true; } } void SpeechSynthesis::Resume() { - if (mCurrentTask && Paused()) { + if (!Paused()) { + return; + } + + if (mCurrentTask) { mCurrentTask->Resume(); + } else { + mHoldQueue = false; + AdvanceQueue(); } } @@ -256,5 +274,13 @@ SpeechSynthesis::GetVoices(nsTArray< nsRefPtr >& aResult) } } +// For testing purposes, allows us to drop anything in the global queue from +// content, and bring the browser to initial state. +void +SpeechSynthesis::DropGlobalQueue() +{ + nsSynthVoiceRegistry::GetInstance()->DropGlobalQueue(); +} + } // namespace dom } // namespace mozilla diff --git a/dom/media/webspeech/synth/SpeechSynthesis.h b/dom/media/webspeech/synth/SpeechSynthesis.h index 4b779f9616..81b2275cb5 100644 --- a/dom/media/webspeech/synth/SpeechSynthesis.h +++ b/dom/media/webspeech/synth/SpeechSynthesis.h @@ -54,6 +54,8 @@ public: void GetVoices(nsTArray< nsRefPtr >& aResult); + void DropGlobalQueue(); + private: virtual ~SpeechSynthesis(); @@ -66,9 +68,10 @@ private: nsRefPtr mCurrentTask; nsRefPtrHashtable mVoiceCache; + + bool mHoldQueue; }; } // namespace dom } // namespace mozilla - #endif diff --git a/dom/media/webspeech/synth/SpeechSynthesisUtterance.cpp b/dom/media/webspeech/synth/SpeechSynthesisUtterance.cpp index e01f2b6376..61d96abd74 100644 --- a/dom/media/webspeech/synth/SpeechSynthesisUtterance.cpp +++ b/dom/media/webspeech/synth/SpeechSynthesisUtterance.cpp @@ -55,7 +55,7 @@ already_AddRefed SpeechSynthesisUtterance::Constructor(GlobalObject& aGlobal, ErrorResult& aRv) { - return Constructor(aGlobal, NS_LITERAL_STRING(""), aRv); + return Constructor(aGlobal, EmptyString(), aRv); } already_AddRefed diff --git a/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl b/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl index f1de1f102c..7939e1abf9 100644 --- a/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl +++ b/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl @@ -15,6 +15,7 @@ struct RemoteVoice { nsString name; nsString lang; bool localService; + bool queued; }; sync protocol PSpeechSynthesis @@ -30,12 +31,18 @@ child: SetDefaultVoice(nsString aUri, bool aIsDefault); + IsSpeakingChanged(bool aIsSpeaking); + parent: __delete__(); PSpeechSynthesisRequest(nsString aText, nsString aUri, nsString aLang, float aVolume, float aRate, float aPitch); - sync ReadVoiceList() returns (RemoteVoice[] aVoices, nsString[] aDefaults); + + sync ReadVoicesAndState() returns (RemoteVoice[] aVoices, + nsString[] aDefaults, bool aIsSpeaking); + + DropGlobalQueue(); }; } // namespace dom diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp index 81d950f5db..56b8aee1b8 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp @@ -40,6 +40,13 @@ SpeechSynthesisChild::RecvSetDefaultVoice(const nsString& aUri, return true; } +bool +SpeechSynthesisChild::RecvIsSpeakingChanged(const bool& aIsSpeaking) +{ + nsSynthVoiceRegistry::RecvIsSpeakingChanged(aIsSpeaking); + return true; +} + PSpeechSynthesisRequestChild* SpeechSynthesisChild::AllocPSpeechSynthesisRequestChild(const nsString& aText, const nsString& aLang, diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h index 9cdfafb245..99b4284599 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h @@ -28,6 +28,8 @@ public: bool RecvSetDefaultVoice(const nsString& aUri, const bool& aIsDefault) override; + bool RecvIsSpeakingChanged(const bool& aIsSpeaking) override; + protected: SpeechSynthesisChild(); virtual ~SpeechSynthesisChild(); diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp index 1e62441a69..24a7c0958a 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp @@ -25,10 +25,20 @@ SpeechSynthesisParent::ActorDestroy(ActorDestroyReason aWhy) } bool -SpeechSynthesisParent::RecvReadVoiceList(InfallibleTArray* aVoices, - InfallibleTArray* aDefaults) +SpeechSynthesisParent::RecvReadVoicesAndState(InfallibleTArray* aVoices, + InfallibleTArray* aDefaults, + bool* aIsSpeaking) { - nsSynthVoiceRegistry::GetInstance()->SendVoices(aVoices, aDefaults); + nsSynthVoiceRegistry::GetInstance()->SendVoicesAndState(aVoices, aDefaults, + aIsSpeaking); + return true; +} + +bool +SpeechSynthesisParent::RecvDropGlobalQueue() +{ + nsSynthVoiceRegistry::GetInstance()->DropGlobalQueue(); + return true; } diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h index 3044b19acf..137119d7bf 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h @@ -24,8 +24,11 @@ class SpeechSynthesisParent : public PSpeechSynthesisParent public: virtual void ActorDestroy(ActorDestroyReason aWhy) override; - bool RecvReadVoiceList(InfallibleTArray* aVoices, - InfallibleTArray* aDefaults) override; + bool RecvReadVoicesAndState(InfallibleTArray* aVoices, + InfallibleTArray* aDefaults, + bool* aIsSpeaking) override; + + bool RecvDropGlobalQueue() override; protected: SpeechSynthesisParent(); diff --git a/dom/media/webspeech/synth/moz.build b/dom/media/webspeech/synth/moz.build index 7ec01dfb95..f29c13e5e3 100644 --- a/dom/media/webspeech/synth/moz.build +++ b/dom/media/webspeech/synth/moz.build @@ -37,7 +37,9 @@ if CONFIG['MOZ_WEBSPEECH']: 'test/nsFakeSynthServices.cpp' ] - if CONFIG['MOZ_SYNTH_PICO']: + if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'windows': + DIRS = ['windows'] + elif CONFIG['MOZ_SYNTH_PICO']: DIRS = ['pico'] IPDL_SOURCES += [ diff --git a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl index a3ed0dfd4e..af654f6978 100644 --- a/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl +++ b/dom/media/webspeech/synth/nsISynthVoiceRegistry.idl @@ -7,21 +7,22 @@ interface nsISpeechService; -[scriptable, builtinclass, uuid(53dcc868-4193-4c3c-a1d9-fe5a0a6af2fb)] +[scriptable, builtinclass, uuid(dac09c3a-156e-4025-a4ab-bc88b0ea92e7)] interface nsISynthVoiceRegistry : nsISupports { /** * Register a speech synthesis voice. * - * @param aService the service that provides this voice. - * @param aUri a unique identifier for this voice. - * @param aName human-readable name for this voice. - * @param aLang a BCP 47 language tag. - * @param aLocalService true if service does not require network. + * @param aService the service that provides this voice. + * @param aUri a unique identifier for this voice. + * @param aName human-readable name for this voice. + * @param aLang a BCP 47 language tag. + * @param aLocalService true if service does not require network. + * @param aQueuesUtterances true if voice only speaks one utterance at a time */ void addVoice(in nsISpeechService aService, in DOMString aUri, in DOMString aName, in DOMString aLang, - in boolean aLocalService); + in boolean aLocalService, in boolean aQueuesUtterances); /** * Remove a speech synthesis voice. diff --git a/dom/media/webspeech/synth/nsSpeechTask.cpp b/dom/media/webspeech/synth/nsSpeechTask.cpp index a923b48aae..96f5ad838d 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.cpp +++ b/dom/media/webspeech/synth/nsSpeechTask.cpp @@ -7,6 +7,7 @@ #include "AudioSegment.h" #include "nsSpeechTask.h" #include "SpeechSynthesis.h" +#include "nsSynthVoiceRegistry.h" // GetCurrentTime is defined in winbase.h as zero argument macro forwarding to // GetTickCount() and conflicts with nsSpeechTask::GetCurrentTime(). @@ -33,15 +34,15 @@ public: void DoNotifyStarted() { if (mSpeechTask) { - mSpeechTask->DispatchStartImpl(); + mSpeechTask->DispatchStartInner(); } } void DoNotifyFinished() { if (mSpeechTask) { - mSpeechTask->DispatchEndImpl(mSpeechTask->GetCurrentTime(), - mSpeechTask->GetCurrentCharOffset()); + mSpeechTask->DispatchEndInner(mSpeechTask->GetCurrentTime(), + mSpeechTask->GetCurrentCharOffset()); } } @@ -96,6 +97,9 @@ NS_IMPL_CYCLE_COLLECTING_RELEASE(nsSpeechTask) nsSpeechTask::nsSpeechTask(SpeechSynthesisUtterance* aUtterance) : mUtterance(aUtterance) + , mInited(false) + , mPrePaused(false) + , mPreCanceled(false) , mCallback(nullptr) , mIndirectAudio(false) { @@ -107,6 +111,9 @@ nsSpeechTask::nsSpeechTask(float aVolume, const nsAString& aText) : mUtterance(nullptr) , mVolume(aVolume) , mText(aText) + , mInited(false) + , mPrePaused(false) + , mPreCanceled(false) , mCallback(nullptr) , mIndirectAudio(false) { @@ -130,10 +137,16 @@ nsSpeechTask::~nsSpeechTask() } void -nsSpeechTask::BindStream(ProcessedMediaStream* aStream) +nsSpeechTask::Init(ProcessedMediaStream* aStream) { - mStream = MediaStreamGraph::GetInstance()->CreateSourceStream(nullptr); - mPort = aStream->AllocateInputPort(mStream, 0); + if (aStream) { + mStream = aStream->Graph()->CreateSourceStream(nullptr); + mPort = aStream->AllocateInputPort(mStream, 0); + mIndirectAudio = false; + } else { + mIndirectAudio = true; + } + mInited = true; } void @@ -153,13 +166,14 @@ nsSpeechTask::Setup(nsISpeechTaskCallback* aCallback, mCallback = aCallback; if (mIndirectAudio) { + MOZ_ASSERT(!mStream); if (argc > 0) { NS_WARNING("Audio info arguments in Setup() are ignored for indirect audio services."); } return NS_OK; } - // mStream is set up in BindStream() that should be called before this. + // mStream is set up in Init() that should be called before this. MOZ_ASSERT(mStream); mStream->AddListener(new SynthStreamListener(this)); @@ -294,6 +308,13 @@ nsSpeechTask::DispatchStart() return NS_ERROR_FAILURE; } + return DispatchStartInner(); +} + +nsresult +nsSpeechTask::DispatchStartInner() +{ + nsSynthVoiceRegistry::GetInstance()->SetIsSpeaking(true); return DispatchStartImpl(); } @@ -316,7 +337,7 @@ nsSpeechTask::DispatchStartImpl(const nsAString& aUri) mUtterance->mState = SpeechSynthesisUtterance::STATE_SPEAKING; mUtterance->mChosenVoiceURI = aUri; mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("start"), 0, 0, - NS_LITERAL_STRING("")); + EmptyString()); return NS_OK; } @@ -329,6 +350,16 @@ nsSpeechTask::DispatchEnd(float aElapsedTime, uint32_t aCharIndex) return NS_ERROR_FAILURE; } + return DispatchEndInner(aElapsedTime, aCharIndex); +} + +nsresult +nsSpeechTask::DispatchEndInner(float aElapsedTime, uint32_t aCharIndex) +{ + if (!mPreCanceled) { + nsSynthVoiceRegistry::GetInstance()->SpeakNext(); + } + return DispatchEndImpl(aElapsedTime, aCharIndex); } @@ -389,9 +420,11 @@ nsSpeechTask::DispatchPauseImpl(float aElapsedTime, uint32_t aCharIndex) } mUtterance->mPaused = true; - mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("pause"), - aCharIndex, aElapsedTime, - NS_LITERAL_STRING("")); + if (mUtterance->mState == SpeechSynthesisUtterance::STATE_SPEAKING) { + mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("pause"), + aCharIndex, aElapsedTime, + EmptyString()); + } return NS_OK; } @@ -419,9 +452,12 @@ nsSpeechTask::DispatchResumeImpl(float aElapsedTime, uint32_t aCharIndex) } mUtterance->mPaused = false; - mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("resume"), - aCharIndex, aElapsedTime, - NS_LITERAL_STRING("")); + if (mUtterance->mState == SpeechSynthesisUtterance::STATE_SPEAKING) { + mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("resume"), + aCharIndex, aElapsedTime, + EmptyString()); + } + return NS_OK; } @@ -447,7 +483,7 @@ nsSpeechTask::DispatchErrorImpl(float aElapsedTime, uint32_t aCharIndex) mUtterance->mState = SpeechSynthesisUtterance::STATE_ENDED; mUtterance->DispatchSpeechSynthesisEvent(NS_LITERAL_STRING("error"), aCharIndex, aElapsedTime, - NS_LITERAL_STRING("")); + EmptyString()); return NS_OK; } @@ -517,6 +553,13 @@ nsSpeechTask::Pause() if (mStream) { mStream->ChangeExplicitBlockerCount(1); + } + + if (!mInited) { + mPrePaused = true; + } + + if (!mIndirectAudio) { DispatchPauseImpl(GetCurrentTime(), GetCurrentCharOffset()); } } @@ -533,6 +576,14 @@ nsSpeechTask::Resume() if (mStream) { mStream->ChangeExplicitBlockerCount(-1); + } + + if (mPrePaused) { + mPrePaused = false; + nsSynthVoiceRegistry::GetInstance()->ResumeQueue(); + } + + if (!mIndirectAudio) { DispatchResumeImpl(GetCurrentTime(), GetCurrentCharOffset()); } } @@ -551,7 +602,14 @@ nsSpeechTask::Cancel() if (mStream) { mStream->ChangeExplicitBlockerCount(1); - DispatchEndImpl(GetCurrentTime(), GetCurrentCharOffset()); + } + + if (!mInited) { + mPreCanceled = true; + } + + if (!mIndirectAudio) { + DispatchEndInner(GetCurrentTime(), GetCurrentCharOffset()); } } diff --git a/dom/media/webspeech/synth/nsSpeechTask.h b/dom/media/webspeech/synth/nsSpeechTask.h index 3d2956f5c5..63a1e1aa9b 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.h +++ b/dom/media/webspeech/synth/nsSpeechTask.h @@ -43,12 +43,20 @@ public: void SetSpeechSynthesis(SpeechSynthesis* aSpeechSynthesis); - void SetIndirectAudio(bool aIndirectAudio) { mIndirectAudio = aIndirectAudio; } - - void BindStream(ProcessedMediaStream* aStream); + void Init(ProcessedMediaStream* aStream); void SetChosenVoiceURI(const nsAString& aUri); + bool IsPreCanceled() + { + return mPreCanceled; + }; + + bool IsPrePaused() + { + return mPrePaused; + } + protected: virtual ~nsSpeechTask(); @@ -77,11 +85,21 @@ protected: nsString mText; + bool mInited; + + bool mPrePaused; + + bool mPreCanceled; + private: void End(); void SendAudioImpl(nsRefPtr& aSamples, uint32_t aDataLen); + nsresult DispatchStartInner(); + + nsresult DispatchEndInner(float aElapsedTime, uint32_t aCharIndex); + nsRefPtr mStream; nsRefPtr mPort; diff --git a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp index f6c610b009..6db88d6ba2 100644 --- a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp +++ b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp @@ -17,6 +17,7 @@ #include "mozilla/StaticPtr.h" #include "mozilla/dom/ContentChild.h" #include "mozilla/dom/ContentParent.h" +#include "mozilla/Preferences.h" #include "mozilla/unused.h" #include "SpeechSynthesisChild.h" @@ -72,12 +73,14 @@ private: public: VoiceData(nsISpeechService* aService, const nsAString& aUri, - const nsAString& aName, const nsAString& aLang, bool aIsLocal) + const nsAString& aName, const nsAString& aLang, + bool aIsLocal, bool aQueuesUtterances) : mService(aService) , mUri(aUri) , mName(aName) , mLang(aLang) - , mIsLocal(aIsLocal) {} + , mIsLocal(aIsLocal) + , mIsQueued(aQueuesUtterances) {} NS_INLINE_DECL_REFCOUNTING(VoiceData) @@ -90,16 +93,56 @@ public: nsString mLang; bool mIsLocal; + + bool mIsQueued; +}; + +// GlobalQueueItem + +class GlobalQueueItem final +{ +private: + // Private destructor, to discourage deletion outside of Release(): + ~GlobalQueueItem() {} + +public: + GlobalQueueItem(VoiceData* aVoice, nsSpeechTask* aTask, const nsAString& aText, + const float& aVolume, const float& aRate, const float& aPitch) + : mVoice(aVoice) + , mTask(aTask) + , mText(aText) + , mVolume(aVolume) + , mRate(aRate) + , mPitch(aPitch) {} + + NS_INLINE_DECL_REFCOUNTING(GlobalQueueItem) + + nsRefPtr mVoice; + + nsRefPtr mTask; + + nsString mText; + + float mVolume; + + float mRate; + + float mPitch; + + bool mIsLocal; }; // nsSynthVoiceRegistry static StaticRefPtr gSynthVoiceRegistry; +static bool sForceGlobalQueue = false; NS_IMPL_ISUPPORTS(nsSynthVoiceRegistry, nsISynthVoiceRegistry) nsSynthVoiceRegistry::nsSynthVoiceRegistry() : mSpeechSynthChild(nullptr) + , mUseGlobalQueue(false) + , mIsSpeaking(false) { if (XRE_IsContentProcess()) { @@ -108,19 +151,22 @@ nsSynthVoiceRegistry::nsSynthVoiceRegistry() InfallibleTArray voices; InfallibleTArray defaults; + bool isSpeaking; - mSpeechSynthChild->SendReadVoiceList(&voices, &defaults); + mSpeechSynthChild->SendReadVoicesAndState(&voices, &defaults, &isSpeaking); for (uint32_t i = 0; i < voices.Length(); ++i) { RemoteVoice voice = voices[i]; AddVoiceImpl(nullptr, voice.voiceURI(), voice.name(), voice.lang(), - voice.localService()); + voice.localService(), voice.queued()); } for (uint32_t i = 0; i < defaults.Length(); ++i) { SetDefaultVoice(defaults[i], true); } + + mIsSpeaking = isSpeaking; } } @@ -133,10 +179,10 @@ nsSynthVoiceRegistry::~nsSynthVoiceRegistry() if (mStream) { if (!mStream->IsDestroyed()) { - mStream->Destroy(); - } + mStream->Destroy(); + } - mStream = nullptr; + mStream = nullptr; } mUriVoiceMap.Clear(); @@ -149,6 +195,8 @@ nsSynthVoiceRegistry::GetInstance() if (!gSynthVoiceRegistry) { gSynthVoiceRegistry = new nsSynthVoiceRegistry(); + Preferences::AddBoolVarCache(&sForceGlobalQueue, + "media.webspeech.synth.force_global_queue"); } return gSynthVoiceRegistry; @@ -166,24 +214,27 @@ void nsSynthVoiceRegistry::Shutdown() { LOG(LogLevel::Debug, ("[%s] nsSynthVoiceRegistry::Shutdown()", - (XRE_IsContentProcess()) ? "Content" : "Default")); + (XRE_IsContentProcess()) ? "Content" : "Default")); gSynthVoiceRegistry = nullptr; } void -nsSynthVoiceRegistry::SendVoices(InfallibleTArray* aVoices, - InfallibleTArray* aDefaults) +nsSynthVoiceRegistry::SendVoicesAndState(InfallibleTArray* aVoices, + InfallibleTArray* aDefaults, + bool* aIsSpeaking) { for (uint32_t i=0; i < mVoices.Length(); ++i) { nsRefPtr voice = mVoices[i]; aVoices->AppendElement(RemoteVoice(voice->mUri, voice->mName, voice->mLang, - voice->mIsLocal)); + voice->mIsLocal, voice->mIsQueued)); } for (uint32_t i=0; i < mDefaultVoices.Length(); ++i) { aDefaults->AppendElement(mDefaultVoices[i]->mUri); } + + *aIsSpeaking = IsSpeaking(); } void @@ -209,7 +260,7 @@ nsSynthVoiceRegistry::RecvAddVoice(const RemoteVoice& aVoice) gSynthVoiceRegistry->AddVoiceImpl(nullptr, aVoice.voiceURI(), aVoice.name(), aVoice.lang(), - aVoice.localService()); + aVoice.localService(), aVoice.queued()); } void @@ -224,25 +275,38 @@ nsSynthVoiceRegistry::RecvSetDefaultVoice(const nsAString& aUri, bool aIsDefault gSynthVoiceRegistry->SetDefaultVoice(aUri, aIsDefault); } +void +nsSynthVoiceRegistry::RecvIsSpeakingChanged(bool aIsSpeaking) +{ + // If we dont have a local instance of the registry yet, we will get the + // speaking state on construction. + if(!gSynthVoiceRegistry) { + return; + } + + gSynthVoiceRegistry->mIsSpeaking = aIsSpeaking; +} + NS_IMETHODIMP nsSynthVoiceRegistry::AddVoice(nsISpeechService* aService, const nsAString& aUri, const nsAString& aName, const nsAString& aLang, - bool aLocalService) + bool aLocalService, + bool aQueuesUtterances) { LOG(LogLevel::Debug, - ("nsSynthVoiceRegistry::AddVoice uri='%s' name='%s' lang='%s' local=%s", + ("nsSynthVoiceRegistry::AddVoice uri='%s' name='%s' lang='%s' local=%s queued=%s", NS_ConvertUTF16toUTF8(aUri).get(), NS_ConvertUTF16toUTF8(aName).get(), NS_ConvertUTF16toUTF8(aLang).get(), - aLocalService ? "true" : "false")); + aLocalService ? "true" : "false", + aQueuesUtterances ? "true" : "false")); if(NS_WARN_IF(XRE_IsContentProcess())) { return NS_ERROR_NOT_AVAILABLE; } - return AddVoiceImpl(aService, aUri, aName, aLang, - aLocalService); + return AddVoiceImpl(aService, aUri, aName, aLang, aLocalService, aQueuesUtterances); } NS_IMETHODIMP @@ -268,6 +332,22 @@ nsSynthVoiceRegistry::RemoveVoice(nsISpeechService* aService, mDefaultVoices.RemoveElement(retval); mUriVoiceMap.Remove(aUri); + if (retval->mIsQueued && !sForceGlobalQueue) { + // Check if this is the last queued voice, and disable the global queue if + // it is. + bool queued = false; + for (uint32_t i = 0; i < mVoices.Length(); i++) { + VoiceData* voice = mVoices[i]; + if (voice->mIsQueued) { + queued = true; + break; + } + } + if (!queued) { + mUseGlobalQueue = false; + } + } + nsTArray ssplist; GetAllSpeechSynthActors(ssplist); @@ -395,7 +475,8 @@ nsSynthVoiceRegistry::AddVoiceImpl(nsISpeechService* aService, const nsAString& aUri, const nsAString& aName, const nsAString& aLang, - bool aLocalService) + bool aLocalService, + bool aQueuesUtterances) { bool found = false; mUriVoiceMap.GetWeak(aUri, &found); @@ -404,10 +485,11 @@ nsSynthVoiceRegistry::AddVoiceImpl(nsISpeechService* aService, } nsRefPtr voice = new VoiceData(aService, aUri, aName, aLang, - aLocalService); + aLocalService, aQueuesUtterances); mVoices.AppendElement(voice); mUriVoiceMap.Put(aUri, voice); + mUseGlobalQueue |= aQueuesUtterances; nsTArray ssplist; GetAllSpeechSynthActors(ssplist); @@ -416,7 +498,8 @@ nsSynthVoiceRegistry::AddVoiceImpl(nsISpeechService* aService, mozilla::dom::RemoteVoice ssvoice(nsString(aUri), nsString(aName), nsString(aLang), - aLocalService); + aLocalService, + aQueuesUtterances); for (uint32_t i = 0; i < ssplist.Length(); ++i) { unused << ssplist[i]->SendVoiceAdded(ssvoice); @@ -574,10 +657,7 @@ nsSynthVoiceRegistry::Speak(const nsAString& aText, const float& aPitch, nsSpeechTask* aTask) { - LOG(LogLevel::Debug, - ("nsSynthVoiceRegistry::Speak text='%s' lang='%s' uri='%s' rate=%f pitch=%f", - NS_ConvertUTF16toUTF8(aText).get(), NS_ConvertUTF16toUTF8(aLang).get(), - NS_ConvertUTF16toUTF8(aUri).get(), aRate, aPitch)); + MOZ_ASSERT(XRE_IsParentProcess()); VoiceData* voice = FindBestMatch(aUri, aLang); @@ -589,24 +669,134 @@ nsSynthVoiceRegistry::Speak(const nsAString& aText, aTask->SetChosenVoiceURI(voice->mUri); - LOG(LogLevel::Debug, ("nsSynthVoiceRegistry::Speak - Using voice URI: %s", - NS_ConvertUTF16toUTF8(voice->mUri).get())); + if (mUseGlobalQueue || sForceGlobalQueue) { + LOG(LogLevel::Debug, + ("nsSynthVoiceRegistry::Speak queueing text='%s' lang='%s' uri='%s' rate=%f pitch=%f", + NS_ConvertUTF16toUTF8(aText).get(), NS_ConvertUTF16toUTF8(aLang).get(), + NS_ConvertUTF16toUTF8(aUri).get(), aRate, aPitch)); + nsRefPtr item = new GlobalQueueItem(voice, aTask, aText, + aVolume, aRate, aPitch); + mGlobalQueue.AppendElement(item); + + if (mGlobalQueue.Length() == 1) { + SpeakImpl(item->mVoice, item->mTask, item->mText, item->mVolume, item->mRate, + item->mPitch); + } + } else { + SpeakImpl(voice, aTask, aText, aVolume, aRate, aPitch); + } +} + +void +nsSynthVoiceRegistry::SpeakNext() +{ + MOZ_ASSERT(XRE_IsParentProcess()); + + LOG(LogLevel::Debug, + ("nsSynthVoiceRegistry::SpeakNext %d", mGlobalQueue.IsEmpty())); + + SetIsSpeaking(false); + + if (mGlobalQueue.IsEmpty()) { + return; + } + + mGlobalQueue.RemoveElementAt(0); + + while (!mGlobalQueue.IsEmpty()) { + nsRefPtr item = mGlobalQueue.ElementAt(0); + if (item->mTask->IsPreCanceled()) { + mGlobalQueue.RemoveElementAt(0); + continue; + } + if (!item->mTask->IsPrePaused()) { + SpeakImpl(item->mVoice, item->mTask, item->mText, item->mVolume, + item->mRate, item->mPitch); + } + break; + } +} + +void +nsSynthVoiceRegistry::ResumeQueue() +{ + MOZ_ASSERT(XRE_IsParentProcess()); + LOG(LogLevel::Debug, + ("nsSynthVoiceRegistry::ResumeQueue %d", mGlobalQueue.IsEmpty())); + + if (mGlobalQueue.IsEmpty()) { + return; + } + + nsRefPtr item = mGlobalQueue.ElementAt(0); + if (!item->mTask->IsPrePaused()) { + SpeakImpl(item->mVoice, item->mTask, item->mText, item->mVolume, + item->mRate, item->mPitch); + } +} + +bool +nsSynthVoiceRegistry::IsSpeaking() +{ + return mIsSpeaking; +} + +void +nsSynthVoiceRegistry::SetIsSpeaking(bool aIsSpeaking) +{ + MOZ_ASSERT(XRE_IsParentProcess()); + + // Only set to 'true' if global queue is enabled. + mIsSpeaking = aIsSpeaking && (mUseGlobalQueue || sForceGlobalQueue); + + nsTArray ssplist; + GetAllSpeechSynthActors(ssplist); + for (uint32_t i = 0; i < ssplist.Length(); ++i) { + unused << ssplist[i]->SendIsSpeakingChanged(aIsSpeaking); + } +} + +void +nsSynthVoiceRegistry::DropGlobalQueue() +{ + if (XRE_IsParentProcess()) { + mGlobalQueue.Clear(); + SetIsSpeaking(false); + } else { + mSpeechSynthChild->SendDropGlobalQueue(); + } +} + +void +nsSynthVoiceRegistry::SpeakImpl(VoiceData* aVoice, + nsSpeechTask* aTask, + const nsAString& aText, + const float& aVolume, + const float& aRate, + const float& aPitch) +{ + LOG(LogLevel::Debug, + ("nsSynthVoiceRegistry::SpeakImpl queueing text='%s' uri='%s' rate=%f pitch=%f", + NS_ConvertUTF16toUTF8(aText).get(), NS_ConvertUTF16toUTF8(aVoice->mUri).get(), + aRate, aPitch)); SpeechServiceType serviceType; - DebugOnly rv = voice->mService->GetServiceType(&serviceType); + DebugOnly rv = aVoice->mService->GetServiceType(&serviceType); NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "Failed to get speech service type"); if (serviceType == nsISpeechService::SERVICETYPE_INDIRECT_AUDIO) { - aTask->SetIndirectAudio(true); + aTask->Init(nullptr); } else { if (!mStream) { - mStream = MediaStreamGraph::GetInstance()->CreateTrackUnionStream(nullptr); + mStream = + MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER, + AudioChannel::Normal)->CreateTrackUnionStream(nullptr); } - aTask->BindStream(mStream); + aTask->Init(mStream); } - voice->mService->Speak(aText, voice->mUri, aVolume, aRate, aPitch, aTask); + aVoice->mService->Speak(aText, aVoice->mUri, aVolume, aRate, aPitch, aTask); } } // namespace dom diff --git a/dom/media/webspeech/synth/nsSynthVoiceRegistry.h b/dom/media/webspeech/synth/nsSynthVoiceRegistry.h index a8adb130b1..94569bee18 100644 --- a/dom/media/webspeech/synth/nsSynthVoiceRegistry.h +++ b/dom/media/webspeech/synth/nsSynthVoiceRegistry.h @@ -23,6 +23,7 @@ class SpeechSynthesisUtterance; class SpeechSynthesisChild; class nsSpeechTask; class VoiceData; +class GlobalQueueItem; class nsSynthVoiceRegistry final : public nsISynthVoiceRegistry { @@ -39,8 +40,19 @@ public: const nsAString& aUri, const float& aVolume, const float& aRate, const float& aPitch, nsSpeechTask* aTask); - void SendVoices(InfallibleTArray* aVoices, - InfallibleTArray* aDefaults); + void SendVoicesAndState(InfallibleTArray* aVoices, + InfallibleTArray* aDefaults, + bool* aIsSpeaking); + + void SpeakNext(); + + void ResumeQueue(); + + bool IsSpeaking(); + + void SetIsSpeaking(bool aIsSpeaking); + + void DropGlobalQueue(); static nsSynthVoiceRegistry* GetInstance(); @@ -52,6 +64,8 @@ public: static void RecvSetDefaultVoice(const nsAString& aUri, bool aIsDefault); + static void RecvIsSpeakingChanged(bool aIsSpeaking); + static void Shutdown(); private: @@ -65,17 +79,31 @@ private: const nsAString& aUri, const nsAString& aName, const nsAString& aLang, - bool aLocalService); + bool aLocalService, + bool aQueuesUtterances); - nsTArray > mVoices; + void SpeakImpl(VoiceData* aVoice, + nsSpeechTask* aTask, + const nsAString& aText, + const float& aVolume, + const float& aRate, + const float& aPitch); - nsTArray > mDefaultVoices; + nsTArray> mVoices; + + nsTArray> mDefaultVoices; nsRefPtrHashtable mUriVoiceMap; SpeechSynthesisChild* mSpeechSynthChild; nsRefPtr mStream; + + bool mUseGlobalQueue; + + nsTArray> mGlobalQueue; + + bool mIsSpeaking; }; } // namespace dom diff --git a/dom/media/webspeech/synth/pico/nsPicoService.cpp b/dom/media/webspeech/synth/pico/nsPicoService.cpp index 76612b8a48..c2ae1135bf 100644 --- a/dom/media/webspeech/synth/pico/nsPicoService.cpp +++ b/dom/media/webspeech/synth/pico/nsPicoService.cpp @@ -519,9 +519,11 @@ PicoAddVoiceTraverser(const nsAString& aUri, name.AssignLiteral("Pico "); name.Append(aVoice->mLanguage); + // This service is multi-threaded and can handle more than one utterance at a + // time before previous utterances end. So, aQueuesUtterances == false DebugOnly rv = data->mRegistry->AddVoice( - data->mService, aUri, name, aVoice->mLanguage, true); + data->mService, aUri, name, aVoice->mLanguage, true, false); NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "Failed to add voice"); return PL_DHASH_NEXT; diff --git a/dom/media/webspeech/synth/test/file_speech_cancel.html b/dom/media/webspeech/synth/test/file_speech_cancel.html index 78a4731763..293dc76fb2 100644 --- a/dom/media/webspeech/synth/test/file_speech_cancel.html +++ b/dom/media/webspeech/synth/test/file_speech_cancel.html @@ -71,6 +71,14 @@ utterance3.addEventListener('end', function(e) { SimpleTest.finish(); }); +// Speak/cancel while paused (Bug 1187105) +speechSynthesis.pause(); +speechSynthesis.speak(new SpeechSynthesisUtterance("hello.")); +ok(speechSynthesis.pending, "paused speechSynthesis has an utterance queued."); +speechSynthesis.cancel(); +ok(!speechSynthesis.pending, "paused speechSynthesis has no utterance queued."); +speechSynthesis.resume(); + speechSynthesis.speak(utterance); ok(!speechSynthesis.speaking, "speechSynthesis is not speaking yet."); ok(speechSynthesis.pending, "speechSynthesis has an utterance queued."); diff --git a/dom/media/webspeech/synth/test/nsFakeSynthServices.cpp b/dom/media/webspeech/synth/test/nsFakeSynthServices.cpp index dd28bb47eb..6e6c5fc30b 100644 --- a/dom/media/webspeech/synth/test/nsFakeSynthServices.cpp +++ b/dom/media/webspeech/synth/test/nsFakeSynthServices.cpp @@ -287,7 +287,9 @@ AddVoices(nsISpeechService* aService, const VoiceDetails* aVoices, uint32_t aLen NS_ConvertUTF8toUTF16 name(aVoices[i].name); NS_ConvertUTF8toUTF16 uri(aVoices[i].uri); NS_ConvertUTF8toUTF16 lang(aVoices[i].lang); - registry->AddVoice(aService, uri, name, lang, true); + // These services can handle more than one utterance at a time and have + // several speaking simultaniously. So, aQueuesUtterances == false + registry->AddVoice(aService, uri, name, lang, true, false); if (aVoices[i].defaultVoice) { registry->SetDefaultVoice(uri, true); } diff --git a/dom/media/webspeech/synth/windows/SapiModule.cpp b/dom/media/webspeech/synth/windows/SapiModule.cpp new file mode 100644 index 0000000000..729dabcba3 --- /dev/null +++ b/dom/media/webspeech/synth/windows/SapiModule.cpp @@ -0,0 +1,57 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ModuleUtils.h" +#include "nsIClassInfoImpl.h" + +#include "SapiService.h" + +using namespace mozilla::dom; + +#define SAPISERVICE_CID \ + {0x21b4a45b, 0x9806, 0x4021, {0xa7, 0x06, 0xd7, 0x68, 0xab, 0x05, 0x48, 0xf9}} + +#define SAPISERVICE_CONTRACTID "@mozilla.org/synthsapi;1" + +// Defines SapiServiceConstructor +NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(SapiService, + SapiService::GetInstanceForService) + +// Defines kSAPISERVICE_CID +NS_DEFINE_NAMED_CID(SAPISERVICE_CID); + +static const mozilla::Module::CIDEntry kCIDs[] = { + { &kSAPISERVICE_CID, true, nullptr, SapiServiceConstructor }, + { nullptr } +}; + +static const mozilla::Module::ContractIDEntry kContracts[] = { + { SAPISERVICE_CONTRACTID, &kSAPISERVICE_CID }, + { nullptr } +}; + +static const mozilla::Module::CategoryEntry kCategories[] = { + { "profile-after-change", "Sapi Speech Synth", SAPISERVICE_CONTRACTID }, + { nullptr } +}; + +static void +UnloadSapiModule() +{ + SapiService::Shutdown(); +} + +static const mozilla::Module kModule = { + mozilla::Module::kVersion, + kCIDs, + kContracts, + kCategories, + nullptr, + nullptr, + UnloadSapiModule +}; + +NSMODULE_DEFN(synthsapi) = &kModule; diff --git a/dom/media/webspeech/synth/windows/SapiService.cpp b/dom/media/webspeech/synth/windows/SapiService.cpp new file mode 100644 index 0000000000..cacb10c6f8 --- /dev/null +++ b/dom/media/webspeech/synth/windows/SapiService.cpp @@ -0,0 +1,396 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.h" +#include "SapiService.h" +#include "nsServiceManagerUtils.h" +#include "nsWin32Locale.h" + +#include "mozilla/dom/nsSynthVoiceRegistry.h" +#include "mozilla/dom/nsSpeechTask.h" +#include "mozilla/Preferences.h" + +namespace mozilla { +namespace dom { + +StaticRefPtr SapiService::sSingleton; + +class SapiCallback final : public nsISpeechTaskCallback +{ +public: + SapiCallback(nsISpeechTask* aTask, ISpVoice* aSapiClient, + uint32_t aTextOffset, uint32_t aSpeakTextLen) + : mTask(aTask) + , mSapiClient(aSapiClient) + , mTextOffset(aTextOffset) + , mSpeakTextLen(aSpeakTextLen) + , mCurrentIndex(0) + , mStreamNum(0) + { + mStartingTime = GetTickCount(); + } + + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_CLASS_AMBIGUOUS(SapiCallback, nsISpeechTaskCallback) + + NS_DECL_NSISPEECHTASKCALLBACK + + ULONG GetStreamNum() const { return mStreamNum; } + void SetStreamNum(ULONG aValue) { mStreamNum = aValue; } + + void OnSpeechEvent(const SPEVENT& speechEvent); + +private: + ~SapiCallback() { } + + // This pointer is used to dispatch events + nsCOMPtr mTask; + nsRefPtr mSapiClient; + + uint32_t mTextOffset; + uint32_t mSpeakTextLen; + + // Used for calculating the time taken to speak the utterance + double mStartingTime; + uint32_t mCurrentIndex; + + ULONG mStreamNum; +}; + +NS_IMPL_CYCLE_COLLECTION(SapiCallback, mTask); + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(SapiCallback) + NS_INTERFACE_MAP_ENTRY(nsISpeechTaskCallback) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsISpeechTaskCallback) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(SapiCallback) +NS_IMPL_CYCLE_COLLECTING_RELEASE(SapiCallback) + +NS_IMETHODIMP +SapiCallback::OnPause() +{ + if (FAILED(mSapiClient->Pause())) { + return NS_ERROR_FAILURE; + } + mTask->DispatchPause(GetTickCount() - mStartingTime, mCurrentIndex); + return NS_OK; +} + +NS_IMETHODIMP +SapiCallback::OnResume() +{ + if (FAILED(mSapiClient->Resume())) { + return NS_ERROR_FAILURE; + } + mTask->DispatchResume(GetTickCount() - mStartingTime, mCurrentIndex); + return NS_OK; +} + +NS_IMETHODIMP +SapiCallback::OnCancel() +{ + // After cancel, mCurrentIndex may be updated. + // At cancel case, use mCurrentIndex for DispatchEnd. + mSpeakTextLen = 0; + // Purge all the previous utterances and speak an empty string + if (FAILED(mSapiClient->Speak(nullptr, SPF_PURGEBEFORESPEAK, nullptr))) { + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +void +SapiCallback::OnSpeechEvent(const SPEVENT& speechEvent) +{ + switch (speechEvent.eEventId) { + case SPEI_START_INPUT_STREAM: + mTask->DispatchStart(); + break; + case SPEI_END_INPUT_STREAM: + if (mSpeakTextLen) { + mCurrentIndex = mSpeakTextLen; + } + mTask->DispatchEnd(GetTickCount() - mStartingTime, mCurrentIndex); + mTask = nullptr; + break; + case SPEI_TTS_BOOKMARK: + mCurrentIndex = static_cast(speechEvent.lParam) - mTextOffset; + mTask->DispatchBoundary(NS_LITERAL_STRING("mark"), + GetTickCount() - mStartingTime, mCurrentIndex); + break; + case SPEI_WORD_BOUNDARY: + mCurrentIndex = static_cast(speechEvent.lParam) - mTextOffset; + mTask->DispatchBoundary(NS_LITERAL_STRING("word"), + GetTickCount() - mStartingTime, mCurrentIndex); + break; + case SPEI_SENTENCE_BOUNDARY: + mCurrentIndex = static_cast(speechEvent.lParam) - mTextOffset; + mTask->DispatchBoundary(NS_LITERAL_STRING("sentence"), + GetTickCount() - mStartingTime, mCurrentIndex); + break; + default: + break; + } +} + +// static +void __stdcall +SapiService::SpeechEventCallback(WPARAM aWParam, LPARAM aLParam) +{ + nsRefPtr service = (SapiService*) aWParam; + + SPEVENT speechEvent; + while (service->mSapiClient->GetEvents(1, &speechEvent, nullptr) == S_OK) { + for (size_t i = 0; i < service->mCallbacks.Length(); i++) { + nsRefPtr callback = service->mCallbacks[i]; + if (callback->GetStreamNum() == speechEvent.ulStreamNum) { + callback->OnSpeechEvent(speechEvent); + if (speechEvent.eEventId == SPEI_END_INPUT_STREAM) { + service->mCallbacks.RemoveElementAt(i); + } + break; + } + } + } +} + +NS_INTERFACE_MAP_BEGIN(SapiService) + NS_INTERFACE_MAP_ENTRY(nsISpeechService) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsISpeechService) +NS_INTERFACE_MAP_END + +NS_IMPL_ADDREF(SapiService) +NS_IMPL_RELEASE(SapiService) + +SapiService::SapiService() + : mInitialized(false) +{ +} + +SapiService::~SapiService() +{ +} + +bool +SapiService::Init() +{ + MOZ_ASSERT(!mInitialized); + + if (Preferences::GetBool("media.webspeech.synth.test")) { + // When enabled, we shouldn't add OS backend (Bug 1160844) + return false; + } + + if (FAILED(CoCreateInstance(CLSID_SpVoice, nullptr, CLSCTX_ALL, IID_ISpVoice, + getter_AddRefs(mSapiClient)))) { + return false; + } + + // Set interest for all the events we are interested in + ULONGLONG eventMask = + SPFEI(SPEI_START_INPUT_STREAM) | + SPFEI(SPEI_TTS_BOOKMARK) | + SPFEI(SPEI_WORD_BOUNDARY) | + SPFEI(SPEI_SENTENCE_BOUNDARY) | + SPFEI(SPEI_END_INPUT_STREAM); + + if (FAILED(mSapiClient->SetInterest(eventMask, eventMask))) { + return false; + } + + // Get all the voices from sapi and register in the SynthVoiceRegistry + if (!RegisterVoices()) { + return false; + } + + // Set the callback function for receiving the events + mSapiClient->SetNotifyCallbackFunction( + (SPNOTIFYCALLBACK*) SapiService::SpeechEventCallback, (WPARAM) this, 0); + + mInitialized = true; + return true; +} + +bool +SapiService::RegisterVoices() +{ + nsresult rv; + + nsCOMPtr registry = + do_GetService(NS_SYNTHVOICEREGISTRY_CONTRACTID); + if (!registry) { + return false; + } + + nsRefPtr category; + if (FAILED(CoCreateInstance(CLSID_SpObjectTokenCategory, nullptr, CLSCTX_ALL, + IID_ISpObjectTokenCategory, + getter_AddRefs(category)))) { + return false; + } + if (FAILED(category->SetId(SPCAT_VOICES, FALSE))) { + return false; + } + + nsRefPtr voiceTokens; + if (FAILED(category->EnumTokens(nullptr, nullptr, + getter_AddRefs(voiceTokens)))) { + return false; + } + + while (true) { + nsRefPtr voiceToken; + if (voiceTokens->Next(1, getter_AddRefs(voiceToken), nullptr) != S_OK) { + break; + } + + nsRefPtr attributes; + if (FAILED(voiceToken->OpenKey(L"Attributes", + getter_AddRefs(attributes)))) { + continue; + } + + WCHAR* language = nullptr; + if (FAILED(attributes->GetStringValue(L"Language", &language))) { + continue; + } + + // Language attribute is LCID by hex. So we need convert to locale + // name. + nsAutoString hexLcid; + LCID lcid = wcstol(language, nullptr, 16); + CoTaskMemFree(language); + nsAutoString locale; + nsWin32Locale::GetXPLocale(lcid, locale); + + WCHAR* description = nullptr; + if (FAILED(voiceToken->GetStringValue(nullptr, &description))) { + continue; + } + + nsAutoString uri; + uri.AssignLiteral("urn:moz-tts:sapi:"); + uri.Append(description); + uri.AppendLiteral("?"); + uri.Append(locale); + + // This service can only speak one utterance at a time, se we set + // aQueuesUtterances to true in order to track global state and schedule + // access to this service. + rv = registry->AddVoice(this, uri, nsDependentString(description), locale, + true, true); + CoTaskMemFree(description); + if (NS_FAILED(rv)) { + continue; + } + + mVoices.Put(uri, voiceToken); + } + + return true; +} + +NS_IMETHODIMP +SapiService::Speak(const nsAString& aText, const nsAString& aUri, + float aVolume, float aRate, float aPitch, + nsISpeechTask* aTask) +{ + NS_ENSURE_TRUE(mInitialized, NS_ERROR_NOT_AVAILABLE); + + nsRefPtr voiceToken; + if (!mVoices.Get(aUri, getter_AddRefs(voiceToken))) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (FAILED(mSapiClient->SetVoice(voiceToken))) { + return NS_ERROR_FAILURE; + } + if (FAILED(mSapiClient->SetVolume(static_cast(aVolume * 100)))) { + return NS_ERROR_FAILURE; + } + if (FAILED(mSapiClient->SetRate(static_cast(10 * log10(aRate))))) { + return NS_ERROR_FAILURE; + } + + // Set the pitch using xml + nsAutoString xml; + xml.AssignLiteral(""); + uint32_t textOffset = xml.Length(); + + xml.Append(aText); + xml.AppendLiteral(""); + + nsRefPtr callback = + new SapiCallback(aTask, mSapiClient, textOffset, aText.Length()); + + // The last three parameters doesn't matter for an indirect service + nsresult rv = aTask->Setup(callback, 0, 0, 0); + if (NS_FAILED(rv)) { + return rv; + } + + ULONG streamNum; + if (FAILED(mSapiClient->Speak(xml.get(), SPF_ASYNC, &streamNum))) { + aTask->Setup(nullptr, 0, 0, 0); + return NS_ERROR_FAILURE; + } + + callback->SetStreamNum(streamNum); + // streamNum reassigns same value when last stream is finished even if + // callback for stream end isn't called + // So we cannot use data hashtable and has to add it to vector at last. + mCallbacks.AppendElement(callback); + + return NS_OK; +} + +NS_IMETHODIMP +SapiService::GetServiceType(SpeechServiceType* aServiceType) +{ + *aServiceType = nsISpeechService::SERVICETYPE_INDIRECT_AUDIO; + return NS_OK; +} + +SapiService* +SapiService::GetInstance() +{ + MOZ_ASSERT(NS_IsMainThread()); + if (XRE_GetProcessType() != GeckoProcessType_Default) { + MOZ_ASSERT(false, + "SapiService can only be started on main gecko process"); + return nullptr; + } + + if (!sSingleton) { + nsRefPtr service = new SapiService(); + if (service->Init()) { + sSingleton = service; + } + } + return sSingleton; +} + +already_AddRefed +SapiService::GetInstanceForService() +{ + nsRefPtr sapiService = GetInstance(); + return sapiService.forget(); +} + +void +SapiService::Shutdown() +{ + if (!sSingleton) { + return; + } + sSingleton = nullptr; +} + +} // namespace dom +} // namespace mozilla diff --git a/dom/media/webspeech/synth/windows/SapiService.h b/dom/media/webspeech/synth/windows/SapiService.h new file mode 100644 index 0000000000..9b77adb486 --- /dev/null +++ b/dom/media/webspeech/synth/windows/SapiService.h @@ -0,0 +1,57 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_SapiService_h +#define mozilla_dom_SapiService_h + +#include "nsAutoPtr.h" +#include "nsISpeechService.h" +#include "nsRefPtrHashtable.h" +#include "nsTArray.h" +#include "mozilla/StaticPtr.h" + +#include +#include + +namespace mozilla { +namespace dom { + +class SapiCallback; + +class SapiService final : public nsISpeechService +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSISPEECHSERVICE + + SapiService(); + bool Init(); + + static SapiService* GetInstance(); + static already_AddRefed GetInstanceForService(); + + static void Shutdown(); + + static void __stdcall SpeechEventCallback(WPARAM aWParam, LPARAM aLParam); + +private: + virtual ~SapiService(); + + bool RegisterVoices(); + + nsRefPtrHashtable mVoices; + nsTArray> mCallbacks; + nsRefPtr mSapiClient; + + bool mInitialized; + + static StaticRefPtr sSingleton; +}; + +} // namespace dom +} // namespace mozilla + +#endif diff --git a/dom/media/webspeech/synth/windows/moz.build b/dom/media/webspeech/synth/windows/moz.build new file mode 100644 index 0000000000..d90edc8b0f --- /dev/null +++ b/dom/media/webspeech/synth/windows/moz.build @@ -0,0 +1,13 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + 'SapiModule.cpp', + 'SapiService.cpp' +] +include('/ipc/chromium/chromium-config.mozbuild') + +FINAL_LIBRARY = 'xul' diff --git a/dom/webidl/SpeechSynthesis.webidl b/dom/webidl/SpeechSynthesis.webidl index b48cf3d2f4..f7a0275170 100644 --- a/dom/webidl/SpeechSynthesis.webidl +++ b/dom/webidl/SpeechSynthesis.webidl @@ -23,4 +23,7 @@ interface SpeechSynthesis { [UnsafeInPrerendering] void resume(); sequence getVoices(); + + [ChromeOnly] + void dropGlobalQueue(); }; diff --git a/gfx/layers/Layers.cpp b/gfx/layers/Layers.cpp index ac76d4f683..bba440efeb 100644 --- a/gfx/layers/Layers.cpp +++ b/gfx/layers/Layers.cpp @@ -1188,14 +1188,14 @@ ContainerLayer::SortChildrenBy3DZOrder(nsTArray& aArray) } else { if (toSort.Length() > 0) { SortLayersBy3DZOrder(toSort); - aArray.MoveElementsFrom(toSort); + aArray.AppendElements(Move(toSort)); } aArray.AppendElement(l); } } if (toSort.Length() > 0) { SortLayersBy3DZOrder(toSort); - aArray.MoveElementsFrom(toSort); + aArray.AppendElements(Move(toSort)); } } diff --git a/gfx/layers/composite/LayerManagerComposite.h b/gfx/layers/composite/LayerManagerComposite.h index 062d5afae8..500463d655 100644 --- a/gfx/layers/composite/LayerManagerComposite.h +++ b/gfx/layers/composite/LayerManagerComposite.h @@ -269,7 +269,7 @@ public: } void ExtractImageCompositeNotifications(nsTArray* aNotifications) { - aNotifications->MoveElementsFrom(mImageCompositeNotifications); + aNotifications->AppendElements(Move(mImageCompositeNotifications)); } private: diff --git a/layout/base/nsDisplayList.cpp b/layout/base/nsDisplayList.cpp index c37b08ff67..8166a528f3 100644 --- a/layout/base/nsDisplayList.cpp +++ b/layout/base/nsDisplayList.cpp @@ -1804,7 +1804,7 @@ void FlushFramesArray(nsTArray& aSource, nsTArray* a aSource.Sort(); uint32_t length = aSource.Length(); for (uint32_t i = 0; i < length; i++) { - aDest->MoveElementsFrom(aSource[i].mFrames); + aDest->AppendElements(Move(aSource[i].mFrames)); } aSource.Clear(); } diff --git a/layout/base/nsDisplayList.h b/layout/base/nsDisplayList.h index 94e1484f2c..7e0a508f10 100644 --- a/layout/base/nsDisplayList.h +++ b/layout/base/nsDisplayList.h @@ -2927,7 +2927,7 @@ protected: mBounds.UnionRect(mBounds, aOther->mBounds); mVisibleRect.UnionRect(mVisibleRect, aOther->mVisibleRect); mMergedFrames.AppendElement(aOther->mFrame); - mMergedFrames.MoveElementsFrom(aOther->mMergedFrames); + mMergedFrames.AppendElements(mozilla::Move(aOther->mMergedFrames)); } nsDisplayList mList; diff --git a/layout/base/nsPresContext.h b/layout/base/nsPresContext.h index 478511baaf..ea519c8ca5 100644 --- a/layout/base/nsPresContext.h +++ b/layout/base/nsPresContext.h @@ -110,7 +110,7 @@ public: void TakeFrom(nsInvalidateRequestList* aList) { - mRequests.MoveElementsFrom(aList->mRequests); + mRequests.AppendElements(mozilla::Move(aList->mRequests)); } bool IsEmpty() { return mRequests.IsEmpty(); } diff --git a/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp b/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp index bd5214cadf..3dd6923d5b 100644 --- a/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp +++ b/media/webrtc/signaling/src/peerconnection/PeerConnectionImpl.cpp @@ -87,6 +87,7 @@ #include "AudioStreamTrack.h" #include "VideoStreamTrack.h" #include "nsIScriptGlobalObject.h" +#include "MediaStreamGraph.h" #include "DOMMediaStream.h" #include "rlogringbuffer.h" #include "WebrtcGlobalInformation.h" @@ -439,9 +440,12 @@ PeerConnectionImpl::~PeerConnectionImpl() already_AddRefed PeerConnectionImpl::MakeMediaStream() { - nsRefPtr stream = - DOMMediaStream::CreateSourceStream(GetWindow()); + MediaStreamGraph* graph = + MediaStreamGraph::GetInstance(MediaStreamGraph::AUDIO_THREAD_DRIVER, + AudioChannel::Normal); + nsRefPtr stream = + DOMMediaStream::CreateSourceStream(GetWindow(), graph); #if !defined(MOZILLA_EXTERNAL_LINKAGE) // Make the stream data (audio/video samples) accessible to the receiving page. // We're only certain that privacy hasn't been requested if we're connected. diff --git a/media/webrtc/signaling/test/FakeMediaStreams.h b/media/webrtc/signaling/test/FakeMediaStreams.h index 1c565288f0..33e5f7f24c 100644 --- a/media/webrtc/signaling/test/FakeMediaStreams.h +++ b/media/webrtc/signaling/test/FakeMediaStreams.h @@ -29,11 +29,42 @@ class nsIDOMWindow; namespace mozilla { - class MediaStreamGraph; class MediaStreamGraphImpl; class MediaSegment; }; + +namespace mozilla { + +class MediaStreamGraph; + +static MediaStreamGraph* gGraph; + +struct AudioChannel { + enum { + Normal + }; +}; + +class MediaStreamGraph { +public: + // Keep this in sync with the enum in MediaStreamGraph.h + enum GraphDriverType { + AUDIO_THREAD_DRIVER, + SYSTEM_THREAD_DRIVER, + OFFLINE_THREAD_DRIVER + }; + static MediaStreamGraph* GetInstance(GraphDriverType aDriverType, + uint32_t aType) { + if (gGraph) { + return gGraph; + } + gGraph = new MediaStreamGraph(); + return gGraph; + } +}; +} + class Fake_VideoSink { public: Fake_VideoSink() {} @@ -318,7 +349,7 @@ public: NS_DECL_THREADSAFE_ISUPPORTS static already_AddRefed - CreateSourceStream(nsIDOMWindow* aWindow, uint32_t aHintContents = 0) { + CreateSourceStream(nsIDOMWindow* aWindow, mozilla::MediaStreamGraph* aGraph, uint32_t aHintContents = 0) { Fake_SourceMediaStream *source = new Fake_SourceMediaStream(); nsRefPtr ds = new Fake_DOMMediaStream(source); diff --git a/parser/html/nsHtml5Speculation.cpp b/parser/html/nsHtml5Speculation.cpp index 528dc4cc4b..f9b5fa38fc 100644 --- a/parser/html/nsHtml5Speculation.cpp +++ b/parser/html/nsHtml5Speculation.cpp @@ -4,6 +4,8 @@ #include "nsHtml5Speculation.h" +using namespace mozilla; + nsHtml5Speculation::nsHtml5Speculation(nsHtml5OwningUTF16Buffer* aBuffer, int32_t aStart, int32_t aStartLineNumber, @@ -24,11 +26,7 @@ nsHtml5Speculation::~nsHtml5Speculation() void nsHtml5Speculation::MoveOpsFrom(nsTArray& aOpQueue) { - if (mOpQueue.IsEmpty()) { - mOpQueue.SwapElements(aOpQueue); - return; - } - mOpQueue.MoveElementsFrom(aOpQueue); + mOpQueue.AppendElements(Move(aOpQueue)); } void diff --git a/parser/html/nsHtml5TreeOpExecutor.cpp b/parser/html/nsHtml5TreeOpExecutor.cpp index fbdb13050c..1775bc684f 100644 --- a/parser/html/nsHtml5TreeOpExecutor.cpp +++ b/parser/html/nsHtml5TreeOpExecutor.cpp @@ -790,11 +790,7 @@ void nsHtml5TreeOpExecutor::MoveOpsFrom(nsTArray& aOpQueue) { NS_PRECONDITION(mFlushState == eNotFlushing, "mOpQueue modified during tree op execution."); - if (mOpQueue.IsEmpty()) { - mOpQueue.SwapElements(aOpQueue); - return; - } - mOpQueue.MoveElementsFrom(aOpQueue); + mOpQueue.AppendElements(Move(aOpQueue)); } void diff --git a/parser/html/nsHtml5TreeOpStage.cpp b/parser/html/nsHtml5TreeOpStage.cpp index 97e51e9297..5f7fe7496f 100644 --- a/parser/html/nsHtml5TreeOpStage.cpp +++ b/parser/html/nsHtml5TreeOpStage.cpp @@ -4,6 +4,8 @@ #include "nsHtml5TreeOpStage.h" +using namespace mozilla; + nsHtml5TreeOpStage::nsHtml5TreeOpStage() : mMutex("nsHtml5TreeOpStage mutex") { @@ -17,11 +19,7 @@ void nsHtml5TreeOpStage::MoveOpsFrom(nsTArray& aOpQueue) { mozilla::MutexAutoLock autoLock(mMutex); - if (mOpQueue.IsEmpty()) { - mOpQueue.SwapElements(aOpQueue); - } else { - mOpQueue.MoveElementsFrom(aOpQueue); - } + mOpQueue.AppendElements(Move(aOpQueue)); } void @@ -29,38 +27,22 @@ nsHtml5TreeOpStage::MoveOpsAndSpeculativeLoadsTo(nsTArray& nsTArray& aSpeculativeLoadQueue) { mozilla::MutexAutoLock autoLock(mMutex); - if (aOpQueue.IsEmpty()) { - mOpQueue.SwapElements(aOpQueue); - } else { - aOpQueue.MoveElementsFrom(mOpQueue); - } - if (aSpeculativeLoadQueue.IsEmpty()) { - mSpeculativeLoadQueue.SwapElements(aSpeculativeLoadQueue); - } else { - aSpeculativeLoadQueue.MoveElementsFrom(mSpeculativeLoadQueue); - } + aOpQueue.AppendElements(Move(mOpQueue)); + aSpeculativeLoadQueue.AppendElements(Move(mSpeculativeLoadQueue)); } void nsHtml5TreeOpStage::MoveSpeculativeLoadsFrom(nsTArray& aSpeculativeLoadQueue) { mozilla::MutexAutoLock autoLock(mMutex); - if (mSpeculativeLoadQueue.IsEmpty()) { - mSpeculativeLoadQueue.SwapElements(aSpeculativeLoadQueue); - } else { - mSpeculativeLoadQueue.MoveElementsFrom(aSpeculativeLoadQueue); - } + mSpeculativeLoadQueue.AppendElements(Move(aSpeculativeLoadQueue)); } void nsHtml5TreeOpStage::MoveSpeculativeLoadsTo(nsTArray& aSpeculativeLoadQueue) { mozilla::MutexAutoLock autoLock(mMutex); - if (aSpeculativeLoadQueue.IsEmpty()) { - mSpeculativeLoadQueue.SwapElements(aSpeculativeLoadQueue); - } else { - aSpeculativeLoadQueue.MoveElementsFrom(mSpeculativeLoadQueue); - } + aSpeculativeLoadQueue.AppendElements(Move(mSpeculativeLoadQueue)); } #ifdef DEBUG diff --git a/toolkit/library/moz.build b/toolkit/library/moz.build index 801cc8da28..3b604a5c77 100644 --- a/toolkit/library/moz.build +++ b/toolkit/library/moz.build @@ -352,7 +352,8 @@ if CONFIG['OS_ARCH'] == 'WINNT': 'wbemuuid', 'wintrust', 'wtsapi32', - 'locationapi' + 'locationapi', + 'sapi', ] if CONFIG['ACCESSIBILITY']: OS_LIBS += [ diff --git a/tools/profiler/GeckoTaskTracer.cpp b/tools/profiler/GeckoTaskTracer.cpp index 02927a7138..f366032287 100644 --- a/tools/profiler/GeckoTaskTracer.cpp +++ b/tools/profiler/GeckoTaskTracer.cpp @@ -203,7 +203,7 @@ void TraceInfo::MoveLogsInto(TraceInfoLogsType& aResult) { MutexAutoLock lock(mLogsMutex); - aResult.MoveElementsFrom(mLogs); + aResult.AppendElements(Move(mLogs)); } void diff --git a/xpcom/glue/nsTArray.h b/xpcom/glue/nsTArray.h index 268d47efa4..bb1510d976 100644 --- a/xpcom/glue/nsTArray.h +++ b/xpcom/glue/nsTArray.h @@ -1556,6 +1556,41 @@ public: return AppendElements(aArray); } + // Move all elements from another array to the end of this array. + // @return A pointer to the newly appended elements, or null on OOM. +protected: + template + elem_type* AppendElements(nsTArray_Impl&& aArray) + { + MOZ_ASSERT(&aArray != this, "argument must be different aArray"); + if (Length() == 0) { + SwapElements(aArray); + return Elements(); + } + + index_type len = Length(); + index_type otherLen = aArray.Length(); + if (!Alloc::Successful(this->template EnsureCapacity( + len + otherLen, sizeof(elem_type)))) { + return nullptr; + } + copy_type::CopyElements(Elements() + len, aArray.Elements(), otherLen, + sizeof(elem_type)); + this->IncrementLength(otherLen); + aArray.template ShiftData(0, otherLen, 0, sizeof(elem_type), + MOZ_ALIGNOF(elem_type)); + return Elements() + len; + } +public: + + template + /* MOZ_WARN_UNUSED_RESULT */ + elem_type* AppendElements(nsTArray_Impl&& aArray, + const mozilla::fallible_t&) + { + return AppendElements(mozilla::Move(aArray)); + } + // Append a new element, move constructing if possible. protected: template @@ -1625,32 +1660,6 @@ public: return AppendElement(); } - // Move all elements from another array to the end of this array without - // calling copy constructors or destructors. - // @return A pointer to the newly appended elements, or null on OOM. - template - elem_type* MoveElementsFrom(nsTArray_Impl& aArray) - { - MOZ_ASSERT(&aArray != this, "argument must be different aArray"); - index_type len = Length(); - index_type otherLen = aArray.Length(); - if (!Alloc::Successful(this->template EnsureCapacity( - len + otherLen, sizeof(elem_type)))) { - return nullptr; - } - copy_type::CopyElements(Elements() + len, aArray.Elements(), otherLen, - sizeof(elem_type)); - this->IncrementLength(otherLen); - aArray.template ShiftData(0, otherLen, 0, sizeof(elem_type), - MOZ_ALIGNOF(elem_type)); - return Elements() + len; - } - template - elem_type* MoveElementsFrom(nsTArray_Impl&& aArray) - { - return MoveElementsFrom(aArray); - } - // This method removes a range of elements from this array. // @param aStart The starting index of the elements to remove. // @param aCount The number of elements to remove. @@ -2142,7 +2151,6 @@ public: using base_type::InsertElementAt; using base_type::InsertElementsAt; using base_type::InsertElementSorted; - using base_type::MoveElementsFrom; using base_type::ReplaceElementsAt; using base_type::SetCapacity; using base_type::SetLength; diff --git a/xpcom/tests/gtest/TestTArray.cpp b/xpcom/tests/gtest/TestTArray.cpp index 86364de3e6..fb5a7d1d89 100644 --- a/xpcom/tests/gtest/TestTArray.cpp +++ b/xpcom/tests/gtest/TestTArray.cpp @@ -35,7 +35,25 @@ const nsTArray& FakeHugeArray() } #endif -TEST(TArray, assign) +TEST(TArray, AppendElementsRvalue) +{ + nsTArray array; + + nsTArray temp(DummyArray()); + array.AppendElements(Move(temp)); + ASSERT_EQ(DummyArray(), array); + ASSERT_TRUE(temp.IsEmpty()); + + temp = DummyArray(); + array.AppendElements(Move(temp)); + nsTArray expected; + expected.AppendElements(DummyArray()); + expected.AppendElements(DummyArray()); + ASSERT_EQ(expected, array); + ASSERT_TRUE(temp.IsEmpty()); +} + +TEST(TArray, Assign) { nsTArray array; array.Assign(DummyArray()); @@ -54,4 +72,15 @@ TEST(TArray, assign) ASSERT_EQ(DummyArray(), array2); } +TEST(TArray, AssignmentOperatorSelfAssignment) +{ + nsTArray array; + array = DummyArray(); + + array = array; + ASSERT_EQ(DummyArray(), array); + array = Move(array); + ASSERT_EQ(DummyArray(), array); } + +} // namespace TestTArray