From 22d3be643bda1ec49e87eceaee9ac794a3ff02d0 Mon Sep 17 00:00:00 2001 From: roytam1 Date: Tue, 18 Oct 2022 16:11:45 +0800 Subject: [PATCH] import changes from `dev' branch of rmottola/Arctic-Fox: - fix mac build (11ff07d07d) - Merge branch 'dev' of https://github.com/rmottola/Arctic-Fox into dev (b1d5bda99a) - Bug 1171682 - Disable WebGL in safemode. r=jgilbert (4b388af71d) - Bug 1212724 - Fix compile error in non-unified mode. r=nical (a089c1d600) - bug 1210266 remove unused AudioNode::Callback() r=padenot (7cf3b8ac8d) - bug 1210266 unfriend AudioBufferSourceNode from AudioNode r=padenot (91055f431b) - bug 1210267 use DOMEventTargetHelper::LastRelease instead of custom Release r=baku (f81093ce16) - bug 1179662 call DisconnectFromGraph once only during unlink r=padenot (d95a512d7c) - Bug 1189506. Give AudioContext non-owning pointers to all its AudioNodes. r=karl (f9c2505e25) - bug 1197028 move AllocateAudioBlock to AudioBlock.h r=padenot (59e140347f) - bug 1197028 introduce AudioBlockBuffer r=padenot (bc33ccfc7f) - bug 1199559 write offline buffer in a format suitable for direct use by AudioBuffer r=padenot (69fd69c667) - bug 1199560 finish offline audio context processing even when allocation fails r=padenot (5863c0a56a) - Bug 1140448 - Improving the performances of how AudioEventTimeline calculates values, r=padenot (1a239c48e7) - bug 1191648 don't keep ScriptProcessorNode alive when it has no audioprocess listener r=padenot (8e1e5eb67d) - bug 1191649 determine ScriptProcessor connected status on main thread r=padenot (a919a422c4) - bug 1191648 don't create audioprocess event when there is no listener r=padenot (98ed82f86a) - bug 1199559 write audioprocess input buffer in a format suitable for direct use by AudioBuffer r=padenot (9ffcb9c64d) - bug 1197028 use AudioChunk::ChannelCount() r=padenot (1cf63e9959) - Bug 1148230 - Eliminate the duplicate subexpression (0f1ad073ff) - bug 1197028 use AudioChunk::GetDuration() r=padenot (b9c30d524c) - bug 1197028 introduce AudioBlock to keep track of downstream references to AudioBlockBuffer r=padenot (dfe5d1cb2f) - bug 1199559 add a helper to fallibly allocate ThreadSharedFloatArrayBufferList with buffers r=padenot (4e0c756087) - bug 1197028 use AudioBlock for web audio processing to reuse buffers shared downstream r=padenot (bf12911645) - bug 1201854 handle stop time precisely even when resampling r=padenot (8901626678) - back out part of bug 1197028 (d5d5bfc98a) - Bug 1189506. Put AudioContext::State inline. r=karl (ded4e9a6c0) - bug 916387 keep ScriptProcessorNode alive after input is GCed r=padenot (9430a56c6a) - Bug 1157137 - Fix WebAudio ScriptProcessorNode sometimes gaining high latency. r=padenot (32eadcafaf) - Bug 1189506. Pass AudioContext to AudioNodeStream::Create. r=karlt (8446b0d16d) - bug 1205558 remove unnecessary ScriptProcessorNodeEngine::mSource r=padenot (1e058b4390) - bug 1053011 align "extra" time on AudioContext with processing block size r=padenot (afeb49adbb) - Bug 1188099 - (Part 3) Introduce [ChromeOnly] SpeechSynthesis.forceEnd for tests. r=smaug r=kdavis (53d765144f) - Bug 1189506. Convert ChangeExplicitBlockerCount to MediaStream::Suspend/Resume. r=padenot (cb074a339e) - bug 1205540 don't send more null chunks than necessary to AnalyserNode r=padenot (d382e1f4ae) - bug 1205540 provide querying whether engines need to continue processing even without input r=padenot (36e44cb77b) - bug 1203380 destroy AudioBlocks on AudioNodeStream on graph thread r=padenot (777e68da76) - Bug 1201393. Create an iterator for MediaStreamGraph to iterate over all its streams. r=karlt (3fe295d8c4) - Bug 1195051 - Part 1: Do not unmute the destination node as soon as the AudioContext is constructed; r=padenot (4243840184) - Bug 1195051 - Part 2: Mute the destination node when the AudioContext is suspended, and unmute when resumed; r=padenot (e07d9e3268) - Bug 1201393. Make suspended MediaStreams implicitly always block. r=padenot (d4557acf43) - Bug 1189506. Make AudioContext responsible for tracking all nodes which need to be suspended and resumed. r=padenot (04410070e7) - Bug 1189506. Make suspending/resuming streams more reusable. r=padenot (503052804e) - bug 1201855 rearrange CopyFromBuffer to separate code using numFrames r=padenot (9e2147d19c) - revert blocked to finished (2bed009b25) - bug 1205540 account for active inputs and skip processing when streams are inactive r=padenot (a20049ae19) - bug 864171 move "extra" time accounting for AudioContext with no nodes to destination stream r=padenot (8ef43b8f25) - bug 1208327 make enum AudioContextOperation strongly typed and forward declare instead of including AudioContext.h r=roc (35cc6748c6) - Bug 1201393. Remove usage of FLAG_BLOCK_OUTPUT from MediaRecorder. r=jwwang (d7ddf40ba2) - Bug 1201393. Remove usage of FLAG_BLOCK_INPUT from MediaRecorder. r=jwwang (587979ca8a) - Bug 1189506. Remove usage of FLAG_BLOCK_OUTPUT from MediaManager. r=jesup (d2cb000648) - Bug 1201393. Remove usage of FLAG_BLOCK_* from OutputStreamData::Connect. r=jwwang (e31f1effc4) - Bug 1189506. Don't bother blocking captured media-element MediaStreams while we're not decoding. r=jwwang (c7240f6fc3) - Bug 1201393. Remove usage of FLAG_BLOCK_INPUT from AudioParam/AudioNode. r=padenot (57a3e05283) - bug 1191649 add notification of input node changes r=padenot (752ae93e82) - bug 916387 add a notification of garbage collected input node r=padenot (6336b50f51) - Bug 1201393. Remove usage of FLAG_BLOCK_INPUT from MediaStreamAudioSourceNode. r=jwwang (df4d77f09a) - Bug 1189506. Remove aFlags parameter from AllocateInputPort. r=karlt (b62e152ec3) - bug 1205540 make source stream available during RemoveInput r=padenot (45341fac7f) - Bug 1189506. Remove MediaInputPort::mFlags. r=karlt (61cb5dce71) - Bug 1170958 - Allow MediaInputPort to lock to a specific input track. r=roc (a5ba676c3d) - Bug 1200579 - Stop copying AudioParam timelines. r=karlt (0720f80914) - bug 1209286 remove now unnecessary StreamTimeToDOMTime and DOMTimeToStreamTime r=padenot (ff93dd9d3a) - Bug 1140450 - Lower speex_resampler quality for Web Audio API. r=padenot (2f34f0b90c) - bug 1201855 keep track of buffer position even when there are no channels r=padenot (f3cdcd3bfc) - bug 1201855 send ended event even when the buffer has no channel data r=padenot (5175efcf0a) - backout from from bug 1197028 (ab2235c6b9) - Bug 1163958 - Reduce the allocation in MediaStreamGraph - patch 2, r=padenot (0b00b72341) - Bug 1189506. Fix multi-track MediaStream audio output. r=karlt (b136e91cc5) - Bug 1189506. Simplify PlayAudio based on the fact that track time units == stream time units. r=karlt (e3a164170c) - Bug 1189506. Remove misleading comment. r=karlt (f491ee4f02) - Bug 1189506. Simplify blocking code now that stream blocking decision are always independent of other streams. r=karlt (646ee9a8da) - Bug 1189506. Remove MediaStream::mBlockInThisPhase. r=karlt (4cfc75216f) - Bug 1189506. Remove mExplicitBlockerCount and related code since it's always zero now. r=karlt (1d45b877fc) - Bug 1189506. Remove unused MediaStreamGraph::GetBufferedTicks. r=karlt (5f90c53e87) - Bug 1189506. Simplify blocking calculations based on the observation that once a stream starts blocking in a given processing interval, it must stay blocked. r=karlt (bd1d2a90d0) - Bug 1189506. Replace MediaStream::mBlocked with simpler MediaStream::mStartBlocking. r=karlt (215bacd2fd) - Bug 1189506. Inline RecomputeBlocking. r=karlt (336a6ab1e9) - Bug 1189506. Inline ComputeStreamBlockTime. r=karlt (0b0256bb64) - Bug 1189506. Remove unused NotifyConsumptionChanged. r=karlt (c9f1250b34) - Bug 1189506. Remove unused mFlushSourcesNow/mFlushSourcesOnNextIteration. r=karlt (734d5fff71) - Bug 1189506. Factor out code from OneIteration into helper methods. r=karlt (52b5030073) - Bug 1189506. Move setting of mStateComputedTime to OneIteration so it's near setting mProcessedTime. r=karlt (c3507aaa84) - Bug 1189506. No need to pass aNextCurrentTime to UpdateCurrentTimeForStreams. r=karlt (1c6141e03e) - Bug 1189506. Inline StreamNotifyOutput/StreamNotifyFinished. r=karlt (cdac2c6405) - Bug 1189506. Rename StreamTimeToGraphTime/GraphTimeToStreamTime to ...WithBlocking. r=karlt (e61ffd53a4) - bug 1199559 permit writing to ThreadSharedFloatArrayBufferList when not shared r=padenot (4eaa511691) - bug 1199559 add a factory method to accept generated buffer contents in a format suitable for direct use r=padenot (801f9c6c35) - Bug 1170958 - Add input stream and track as args to NotifyQueuedTrackChanges. r=roc (b3677c801a) - bug 1196111 don't keep AudioContext alive from AudioBuffer r=baku (f1b113c655) - bug 1207003 fetch stream position once instead of three times r=padenot (2cef872dc0) - bug 1207003 remove unused aStream parameter r=padenot (8c3fa1ee88) - bug 1207003 add GraphTime parameter to ProcessBlock() and remove GetCurrentPosition() r=padenot (5f442537a2) - bug 1206362 be careful about double -> int conversion r=padenot (956051f9f1) - Bug 1189506. Remove invalid assertion. r=karlt (4fa8b5a7b1) - Bug 1189506. Create StreamTimeToGraphTime/GraphTimeToStreamTime that don't take account of blocking, and call them from AudioNodeStream. r=karlt (d10f5b1b62) - bug 1205558 use destination stream for audio node engine time r=padenot (a9361c32ac) - bug 1205558 remove unused AudioNodeStream* aSource parameter r=padenot (7667751920) - Bug 1189506. Use mProcessedTime in some places instead of passing aFrm. r=karlt (c5537555e5) - Bug 1189506. Use mStateComputedTime in some places instead of passing aTo. r=karlt (69ed378a74) - Bug 1189506. Use mProcessedTime/mStateComputedTime in ProduceDataForStreamsBlockByBlock. karlt (bc1c49c537) - bug 1214493 restore fractional start time accidentally rounded in 13e85dc6b41b r=padenot (87b7b5de82) - missing bit of 1205558 (a46dbdc0de) - Bug 1185176 - Account for the fact that it is possible for nodes to not have streams. r=karlt (1b5729312b) - Bug 1210266 use parameter index instead of node callback for sending timeline events r=padenot (6501350f34) - remove blocked for finished (89fcb0509c) - bug 1215096 correct off-by-one error in playback position of resampled buffers r=padenot (38c7351675) - bug 1020370 adjust assert to tolerate large skipFracNum r=padenot (f4f5c7f10f) - bug 1020370 use int64_t to avoid overflow in subsample calcs r=padenot (9ec82a2c36) - missing bit of bug 1201855 use unsigned integers for buffer positions (7e83ff4598) - bug 1179662 call UnregisterAudioBufferSourceNode only once r=padenot (40a5d8ab9a) - bug 1205558 introduce SecondsToNearestStreamTime r=padenot (719acd8bc2) - missing bits of Bug 1189506. Make AudioContext responsible (8bd5a1044d) - Bug 1189506. Call GraphTimeToStreamTime in ExtractPendingInput since we know no blocking time has been determined yet. r=karlt (bab799c1d0) --- dom/camera/CameraPreviewMediaStream.cpp | 24 - dom/camera/CameraPreviewMediaStream.h | 3 +- dom/canvas/WebGLContext.cpp | 4 + dom/html/HTMLMediaElement.cpp | 8 +- dom/media/AudioCaptureStream.cpp | 19 +- dom/media/AudioCaptureStream.h | 5 +- dom/media/AudioSegment.h | 37 +- dom/media/DOMMediaStream.cpp | 17 +- dom/media/GraphDriver.cpp | 4 +- dom/media/GraphDriver.h | 1 - dom/media/MediaManager.cpp | 2 +- dom/media/MediaRecorder.cpp | 13 +- dom/media/MediaStreamGraph.cpp | 1088 ++++++----------- dom/media/MediaStreamGraph.h | 216 ++-- dom/media/MediaStreamGraphImpl.h | 191 ++- dom/media/SharedBuffer.h | 10 +- dom/media/StreamBuffer.cpp | 23 +- dom/media/StreamBuffer.h | 40 +- dom/media/TrackUnionStream.cpp | 26 +- dom/media/encoder/MediaEncoder.cpp | 4 +- dom/media/encoder/MediaEncoder.h | 4 +- dom/media/imagecapture/CaptureTask.cpp | 4 +- dom/media/imagecapture/CaptureTask.h | 4 +- dom/media/mediasink/DecodedStream.cpp | 32 +- dom/media/webaudio/AnalyserNode.cpp | 36 +- dom/media/webaudio/AudioBlock.cpp | 162 +++ dom/media/webaudio/AudioBlock.h | 120 ++ dom/media/webaudio/AudioBuffer.cpp | 24 +- dom/media/webaudio/AudioBuffer.h | 23 +- dom/media/webaudio/AudioBufferSourceNode.cpp | 278 +++-- dom/media/webaudio/AudioBufferSourceNode.h | 2 - dom/media/webaudio/AudioContext.cpp | 104 +- dom/media/webaudio/AudioContext.h | 42 +- dom/media/webaudio/AudioDestinationNode.cpp | 146 ++- dom/media/webaudio/AudioDestinationNode.h | 8 +- dom/media/webaudio/AudioEventTimeline.h | 614 ++++++---- dom/media/webaudio/AudioNode.cpp | 58 +- dom/media/webaudio/AudioNode.h | 24 +- dom/media/webaudio/AudioNodeEngine.cpp | 60 +- dom/media/webaudio/AudioNodeEngine.h | 74 +- .../webaudio/AudioNodeExternalInputStream.cpp | 35 +- .../webaudio/AudioNodeExternalInputStream.h | 3 +- dom/media/webaudio/AudioNodeStream.cpp | 279 +++-- dom/media/webaudio/AudioNodeStream.h | 99 +- dom/media/webaudio/AudioParam.cpp | 26 +- dom/media/webaudio/AudioParam.h | 94 +- dom/media/webaudio/AudioParamTimeline.h | 64 +- dom/media/webaudio/BiquadFilterNode.cpp | 90 +- dom/media/webaudio/BiquadFilterNode.h | 6 - dom/media/webaudio/ChannelMergerNode.cpp | 8 +- dom/media/webaudio/ChannelSplitterNode.cpp | 6 +- dom/media/webaudio/ConvolverNode.cpp | 23 +- dom/media/webaudio/DelayBuffer.cpp | 18 +- dom/media/webaudio/DelayBuffer.h | 10 +- dom/media/webaudio/DelayNode.cpp | 54 +- dom/media/webaudio/DelayNode.h | 1 - dom/media/webaudio/DynamicsCompressorNode.cpp | 100 +- dom/media/webaudio/DynamicsCompressorNode.h | 7 - dom/media/webaudio/GainNode.cpp | 51 +- dom/media/webaudio/GainNode.h | 3 - .../MediaStreamAudioDestinationNode.cpp | 4 +- .../webaudio/MediaStreamAudioSourceNode.cpp | 4 +- dom/media/webaudio/OscillatorNode.cpp | 68 +- dom/media/webaudio/OscillatorNode.h | 2 - dom/media/webaudio/PannerNode.cpp | 39 +- dom/media/webaudio/PanningUtils.h | 8 +- dom/media/webaudio/ScriptProcessorNode.cpp | 283 +++-- dom/media/webaudio/ScriptProcessorNode.h | 33 +- dom/media/webaudio/StereoPannerNode.cpp | 57 +- dom/media/webaudio/StereoPannerNode.h | 1 - dom/media/webaudio/WaveShaperNode.cpp | 15 +- dom/media/webaudio/WebAudioUtils.cpp | 30 +- dom/media/webaudio/WebAudioUtils.h | 19 +- .../webaudio/blink/DynamicsCompressor.cpp | 8 +- dom/media/webaudio/blink/DynamicsCompressor.h | 6 +- dom/media/webaudio/blink/HRTFElevation.cpp | 2 +- dom/media/webaudio/blink/HRTFPanner.cpp | 17 +- dom/media/webaudio/blink/HRTFPanner.h | 6 +- dom/media/webaudio/blink/Reverb.cpp | 52 +- dom/media/webaudio/blink/Reverb.h | 6 +- .../compiledtest/TestAudioEventTimeline.cpp | 11 + dom/media/webaudio/moz.build | 3 +- .../recognition/SpeechStreamListener.cpp | 4 +- .../recognition/SpeechStreamListener.h | 4 +- dom/media/webspeech/synth/SpeechSynthesis.cpp | 10 +- dom/media/webspeech/synth/SpeechSynthesis.h | 2 +- .../webspeech/synth/ipc/PSpeechSynthesis.ipdl | 2 - .../synth/ipc/PSpeechSynthesisRequest.ipdl | 2 + .../synth/ipc/SpeechSynthesisChild.cpp | 7 + .../synth/ipc/SpeechSynthesisChild.h | 2 + .../synth/ipc/SpeechSynthesisParent.cpp | 16 +- .../synth/ipc/SpeechSynthesisParent.h | 4 +- dom/media/webspeech/synth/nsSpeechTask.cpp | 24 +- dom/media/webspeech/synth/nsSpeechTask.h | 2 + .../webspeech/synth/nsSynthVoiceRegistry.cpp | 11 - .../webspeech/synth/nsSynthVoiceRegistry.h | 2 - dom/webidl/SpeechSynthesis.webidl | 3 +- gfx/2d/QuartzSupport.mm | 2 - .../client/TextureClientRecycleAllocator.cpp | 1 + .../src/mediapipeline/MediaPipeline.cpp | 4 +- .../src/mediapipeline/MediaPipeline.h | 12 +- .../webrtc/signaling/test/FakeMediaStreams.h | 5 +- .../signaling/test/FakeMediaStreamsImpl.h | 4 +- widget/cocoa/nsMenuItemIconX.mm | 1 - 104 files changed, 2820 insertions(+), 2509 deletions(-) create mode 100644 dom/media/webaudio/AudioBlock.cpp create mode 100644 dom/media/webaudio/AudioBlock.h diff --git a/dom/camera/CameraPreviewMediaStream.cpp b/dom/camera/CameraPreviewMediaStream.cpp index 12182d95b4..dbd3fb6a7a 100644 --- a/dom/camera/CameraPreviewMediaStream.cpp +++ b/dom/camera/CameraPreviewMediaStream.cpp @@ -40,7 +40,6 @@ CameraPreviewMediaStream::CameraPreviewMediaStream(DOMMediaStream* aWrapper) MediaStreamGraph::GetInstance( MediaStreamGraph::SYSTEM_THREAD_DRIVER, AudioChannel::Normal)); mFakeMediaStreamGraph = new FakeMediaStreamGraph(); - mIsConsumed = false; } void @@ -64,15 +63,6 @@ CameraPreviewMediaStream::AddVideoOutput(VideoFrameContainer* aContainer) MutexAutoLock lock(mMutex); RefPtr container = aContainer; AddVideoOutputImpl(container.forget()); - - if (mVideoOutputs.Length() > 1) { - return; - } - mIsConsumed = true; - for (uint32_t j = 0; j < mListeners.Length(); ++j) { - MediaStreamListener* l = mListeners[j]; - l->NotifyConsumptionChanged(mFakeMediaStreamGraph, MediaStreamListener::CONSUMED); - } } void @@ -80,20 +70,6 @@ CameraPreviewMediaStream::RemoveVideoOutput(VideoFrameContainer* aContainer) { MutexAutoLock lock(mMutex); RemoveVideoOutputImpl(aContainer); - - if (!mVideoOutputs.IsEmpty()) { - return; - } - mIsConsumed = false; - for (uint32_t j = 0; j < mListeners.Length(); ++j) { - MediaStreamListener* l = mListeners[j]; - l->NotifyConsumptionChanged(mFakeMediaStreamGraph, MediaStreamListener::NOT_CONSUMED); - } -} - -void -CameraPreviewMediaStream::ChangeExplicitBlockerCount(int32_t aDelta) -{ } void diff --git a/dom/camera/CameraPreviewMediaStream.h b/dom/camera/CameraPreviewMediaStream.h index ded2b30ec4..acb5b5c486 100644 --- a/dom/camera/CameraPreviewMediaStream.h +++ b/dom/camera/CameraPreviewMediaStream.h @@ -48,7 +48,8 @@ public: virtual void RemoveAudioOutput(void* aKey) override; virtual void AddVideoOutput(VideoFrameContainer* aContainer) override; virtual void RemoveVideoOutput(VideoFrameContainer* aContainer) override; - virtual void ChangeExplicitBlockerCount(int32_t aDelta) override; + virtual void Suspend() override {} + virtual void Resume() override {} virtual void AddListener(MediaStreamListener* aListener) override; virtual void RemoveListener(MediaStreamListener* aListener) override; virtual void Destroy() override; diff --git a/dom/canvas/WebGLContext.cpp b/dom/canvas/WebGLContext.cpp index 552599d84d..afdb7f342e 100644 --- a/dom/canvas/WebGLContext.cpp +++ b/dom/canvas/WebGLContext.cpp @@ -909,6 +909,10 @@ WebGLContext::SetDimensions(int32_t signedWidth, int32_t signedHeight) NS_ENSURE_TRUE(Preferences::GetRootBranch(), NS_ERROR_FAILURE); bool disabled = Preferences::GetBool("webgl.disabled", false); + + // TODO: When we have software webgl support we should use that instead. + disabled |= gfxPlatform::InSafeMode(); + if (disabled) { GenerateWarning("WebGL creation is disabled, and so disallowed here."); return NS_ERROR_FAILURE; diff --git a/dom/html/HTMLMediaElement.cpp b/dom/html/HTMLMediaElement.cpp index 02b96e41f1..a22f33a8db 100644 --- a/dom/html/HTMLMediaElement.cpp +++ b/dom/html/HTMLMediaElement.cpp @@ -1871,10 +1871,6 @@ HTMLMediaElement::CaptureStreamInternal(bool aFinishWhenEnded, out->mFinishWhenEnded = aFinishWhenEnded; mAudioCaptured = true; - // Block the output stream initially. - // Decoders are responsible for removing the block while they are playing - // back into the output stream. - out->mStream->GetStream()->ChangeExplicitBlockerCount(1); if (mDecoder) { mDecoder->AddOutputStream(out->mStream->GetStream()->AsProcessedStream(), aFinishWhenEnded); @@ -3037,7 +3033,9 @@ public: virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) override + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override { MutexAutoLock lock(mMutex); if (mInitialSize != gfx::IntSize(0,0) || diff --git a/dom/media/AudioCaptureStream.cpp b/dom/media/AudioCaptureStream.cpp index f220503289..459d1cba81 100644 --- a/dom/media/AudioCaptureStream.cpp +++ b/dom/media/AudioCaptureStream.cpp @@ -29,8 +29,8 @@ namespace mozilla // We are mixing to mono until PeerConnection can accept stereo static const uint32_t MONO = 1; -AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper) - : ProcessedMediaStream(aWrapper), mTrackCreated(false) +AudioCaptureStream::AudioCaptureStream(DOMMediaStream* aWrapper, TrackID aTrackId) + : ProcessedMediaStream(aWrapper), mTrackId(aTrackId), mTrackCreated(false) { MOZ_ASSERT(NS_IsMainThread()); MOZ_COUNT_CTOR(AudioCaptureStream); @@ -48,14 +48,14 @@ AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) { uint32_t inputCount = mInputs.Length(); - StreamBuffer::Track* track = EnsureTrack(AUDIO_TRACK); + StreamBuffer::Track* track = EnsureTrack(mTrackId); // Notify the DOM everything is in order. if (!mTrackCreated) { for (uint32_t i = 0; i < mListeners.Length(); i++) { MediaStreamListener* l = mListeners[i]; AudioSegment tmp; l->NotifyQueuedTrackChanges( - Graph(), AUDIO_TRACK, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp); + Graph(), mTrackId, 0, MediaStreamListener::TRACK_EVENT_CREATED, tmp); l->NotifyFinishedTrackCreation(Graph()); } mTrackCreated = true; @@ -65,8 +65,7 @@ AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, // HTMLMediaElement with a stream as source, or an AudioContext), a cycle // situation occur. This can work if it's an AudioContext with at least one // DelayNode, but the MSG will mute the whole cycle otherwise. - bool blocked = mFinished || mBlocked.GetAt(aFrom); - if (blocked || InMutedCycle() || inputCount == 0) { + if (mFinished || InMutedCycle() || inputCount == 0) { track->Get()->AppendNullData(aTo - aFrom); } else { // We mix down all the tracks of all inputs, to a stereo track. Everything @@ -78,8 +77,8 @@ AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, StreamBuffer::TrackIter tracks(s->GetStreamBuffer(), MediaSegment::AUDIO); while (!tracks.IsEnded()) { AudioSegment* inputSegment = tracks->Get(); - StreamTime inputStart = s->GraphTimeToStreamTime(aFrom); - StreamTime inputEnd = s->GraphTimeToStreamTime(aTo); + StreamTime inputStart = s->GraphTimeToStreamTimeWithBlocking(aFrom); + StreamTime inputEnd = s->GraphTimeToStreamTimeWithBlocking(aTo); AudioSegment toMix; toMix.AppendSlice(*inputSegment, inputStart, inputEnd); // Care for streams blocked in the [aTo, aFrom] range. @@ -95,7 +94,7 @@ AudioCaptureStream::ProcessInput(GraphTime aFrom, GraphTime aTo, } // Regardless of the status of the input tracks, we go foward. - mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime((aTo))); + mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking((aTo))); } void @@ -128,6 +127,6 @@ AudioCaptureStream::MixerCallback(AudioDataValue* aMixedBuffer, } // Now we have mixed data, simply append it to out track. - EnsureTrack(AUDIO_TRACK)->Get()->AppendAndConsumeChunk(&chunk); + EnsureTrack(mTrackId)->Get()->AppendAndConsumeChunk(&chunk); } } diff --git a/dom/media/AudioCaptureStream.h b/dom/media/AudioCaptureStream.h index 322dcd8804..752a3b63f2 100644 --- a/dom/media/AudioCaptureStream.h +++ b/dom/media/AudioCaptureStream.h @@ -8,6 +8,7 @@ #include "MediaStreamGraph.h" #include "AudioMixer.h" +#include "StreamBuffer.h" #include namespace mozilla @@ -22,17 +23,17 @@ class AudioCaptureStream : public ProcessedMediaStream, public MixerCallbackReceiver { public: - explicit AudioCaptureStream(DOMMediaStream* aWrapper); + explicit AudioCaptureStream(DOMMediaStream* aWrapper, TrackID aTrackId); virtual ~AudioCaptureStream(); void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; protected: - enum { AUDIO_TRACK = 1 }; void MixerCallback(AudioDataValue* aMixedBuffer, AudioSampleFormat aFormat, uint32_t aChannels, uint32_t aFrames, uint32_t aSampleRate) override; AudioMixer mMixer; + TrackID mTrackId; bool mTrackCreated; }; } diff --git a/dom/media/AudioSegment.h b/dom/media/AudioSegment.h index f06d744a6e..878a89d830 100644 --- a/dom/media/AudioSegment.h +++ b/dom/media/AudioSegment.h @@ -192,42 +192,7 @@ struct AudioChunk { mBufferFormat = AUDIO_FORMAT_SILENCE; } - bool IsSilentOrSubnormal() const - { - if (!mBuffer) { - return true; - } - - for (uint32_t i = 0, length = mChannelData.Length(); i < length; ++i) { - const float* channel = static_cast(mChannelData[i]); - for (StreamTime frame = 0; frame < mDuration; ++frame) { - if (fabs(channel[frame]) >= FLT_MIN) { - return false; - } - } - } - - return true; - } - - int ChannelCount() const { return mChannelData.Length(); } - - float* ChannelFloatsForWrite(size_t aChannel) - { - MOZ_ASSERT(mBufferFormat == AUDIO_FORMAT_FLOAT32); - MOZ_ASSERT(!mBuffer->IsShared()); - return static_cast(const_cast(mChannelData[aChannel])); - } - - void ReleaseBufferIfShared() - { - if (mBuffer && mBuffer->IsShared()) { - // Remove pointers into the buffer, but keep the array allocation for - // chunk re-use. - mChannelData.ClearAndRetainStorage(); - mBuffer = nullptr; - } - } + size_t ChannelCount() const { return mChannelData.Length(); } bool IsMuted() const { return mVolume == 0.0f; } diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 6180ab77f7..6c40eebd1c 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -41,8 +41,10 @@ public: public: TrackChange(StreamListener* aListener, TrackID aID, StreamTime aTrackOffset, - uint32_t aEvents, MediaSegment::Type aType) + uint32_t aEvents, MediaSegment::Type aType, + MediaStream* aInputStream, TrackID aInputTrackID) : mListener(aListener), mID(aID), mEvents(aEvents), mType(aType) + , mInputStream(aInputStream), mInputTrackID(aInputTrackID) { } @@ -82,6 +84,8 @@ public: TrackID mID; uint32_t mEvents; MediaSegment::Type mType; + RefPtr mInputStream; + TrackID mInputTrackID; }; /** @@ -94,12 +98,14 @@ public: virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) override + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override { if (aTrackEvents & (TRACK_EVENT_CREATED | TRACK_EVENT_ENDED)) { RefPtr runnable = new TrackChange(this, aID, aTrackOffset, aTrackEvents, - aQueuedMedia.GetType()); + aQueuedMedia.GetType(), aInputStream, aInputTrackID); aGraph->DispatchToMainThreadAfterStreamStateUpdate(runnable.forget()); } } @@ -302,7 +308,10 @@ DOMMediaStream::InitAudioCaptureStream(nsIDOMWindow* aWindow, { mWindow = aWindow; - InitStreamCommon(aGraph->CreateAudioCaptureStream(this)); + const TrackID AUDIO_TRACK = 1; + + InitStreamCommon(aGraph->CreateAudioCaptureStream(this, AUDIO_TRACK)); + CreateDOMTrack(AUDIO_TRACK, MediaSegment::AUDIO); } void diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp index 6e03c9ee2b..e65d05874a 100644 --- a/dom/media/GraphDriver.cpp +++ b/dom/media/GraphDriver.cpp @@ -5,6 +5,7 @@ #include #include "CubebUtils.h" +#include "webaudio/AudioContext.h" #ifdef XP_MACOSX #include @@ -299,8 +300,6 @@ ThreadedDriver::RunThread() (long)mIterationStart, (long)mIterationEnd, (long)stateComputedTime, (long)nextStateComputedTime)); - mGraphImpl->mFlushSourcesNow = mGraphImpl->mFlushSourcesOnNextIteration; - mGraphImpl->mFlushSourcesOnNextIteration = false; stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime); if (mNextDriver && stillProcessing) { @@ -967,7 +966,6 @@ AudioCallbackDriver::DeviceChangedCallback() { STREAM_LOG(LogLevel::Error, ("Switching to SystemClockDriver during output switch")); mSelfReference.Take(this); mCallbackReceivedWhileSwitching = 0; - mGraphImpl->mFlushSourcesOnNextIteration = true; mNextDriver = new SystemClockDriver(GraphImpl()); mNextDriver->SetGraphTime(this, mIterationStart, mIterationEnd); mGraphImpl->SetCurrentDriver(mNextDriver); diff --git a/dom/media/GraphDriver.h b/dom/media/GraphDriver.h index e2efb4d623..dd4a0efe48 100644 --- a/dom/media/GraphDriver.h +++ b/dom/media/GraphDriver.h @@ -13,7 +13,6 @@ #include "AudioSegment.h" #include "SelfRef.h" #include "mozilla/Atomics.h" -#include "AudioContext.h" struct cubeb_stream; diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 020d240c56..277fe9751a 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -821,7 +821,7 @@ public: msg); trackunion->GetStream()->AsProcessedStream()->SetAutofinish(true); RefPtr port = trackunion->GetStream()->AsProcessedStream()-> - AllocateInputPort(stream, MediaInputPort::FLAG_BLOCK_OUTPUT); + AllocateInputPort(stream); trackunion->mSourceStream = stream; trackunion->mPort = port.forget(); // Log the relationship between SourceMediaStream and TrackUnion stream diff --git a/dom/media/MediaRecorder.cpp b/dom/media/MediaRecorder.cpp index e066744e7f..0b36d7cacf 100644 --- a/dom/media/MediaRecorder.cpp +++ b/dom/media/MediaRecorder.cpp @@ -413,7 +413,7 @@ public: MOZ_ASSERT(NS_IsMainThread()); NS_ENSURE_TRUE(mTrackUnionStream, NS_ERROR_FAILURE); - mTrackUnionStream->ChangeExplicitBlockerCount(1); + mTrackUnionStream->Suspend(); return NS_OK; } @@ -424,7 +424,7 @@ public: MOZ_ASSERT(NS_IsMainThread()); NS_ENSURE_TRUE(mTrackUnionStream, NS_ERROR_FAILURE); - mTrackUnionStream->ChangeExplicitBlockerCount(-1); + mTrackUnionStream->Resume(); return NS_OK; } @@ -540,8 +540,7 @@ private: mTrackUnionStream->SetAutofinish(true); // Bind this Track Union Stream with Source Media. - mInputPort = mTrackUnionStream->AllocateInputPort(mRecorder->GetSourceMediaStream(), - MediaInputPort::FLAG_BLOCK_OUTPUT); + mInputPort = mTrackUnionStream->AllocateInputPort(mRecorder->GetSourceMediaStream()); DOMMediaStream* domStream = mRecorder->Stream(); if (domStream) { @@ -773,13 +772,11 @@ MediaRecorder::MediaRecorder(AudioNode& aSrcAudioNode, AudioNodeStream::Flags flags = AudioNodeStream::EXTERNAL_OUTPUT | AudioNodeStream::NEED_MAIN_THREAD_FINISHED; - mPipeStream = AudioNodeStream::Create(ctx->Graph(), engine, flags); + mPipeStream = AudioNodeStream::Create(ctx, engine, flags); AudioNodeStream* ns = aSrcAudioNode.GetStream(); if (ns) { mInputPort = mPipeStream->AllocateInputPort(aSrcAudioNode.GetStream(), - MediaInputPort::FLAG_BLOCK_INPUT, - 0, - aSrcOutput); + TRACK_ANY, 0, aSrcOutput); } } mAudioNode = &aSrcAudioNode; diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index a1562e74eb..d18ba3786a 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -102,28 +102,14 @@ MediaStreamGraphImpl::FinishStream(MediaStream* aStream) SetStreamOrderDirty(); } -static const GraphTime START_TIME_DELAYED = -1; - void MediaStreamGraphImpl::AddStreamGraphThread(MediaStream* aStream) { - // Check if we're adding a stream to a suspended context, in which case, we - // add it to mSuspendedStreams, and delay setting mBufferStartTime - bool contextSuspended = false; - if (aStream->AsAudioNodeStream()) { - for (uint32_t i = 0; i < mSuspendedStreams.Length(); i++) { - if (aStream->AudioContextId() == mSuspendedStreams[i]->AudioContextId()) { - contextSuspended = true; - } - } - } - - if (contextSuspended) { - aStream->mBufferStartTime = START_TIME_DELAYED; + aStream->mBufferStartTime = mProcessedTime; + if (aStream->IsSuspended()) { mSuspendedStreams.AppendElement(aStream); STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to the graph, in the suspended stream array", aStream)); } else { - aStream->mBufferStartTime = mProcessedTime; mStreams.AppendElement(aStream); STREAM_LOG(LogLevel::Debug, ("Adding media stream %p to the graph", aStream)); } @@ -149,29 +135,17 @@ MediaStreamGraphImpl::RemoveStreamGraphThread(MediaStream* aStream) // Ensure that mFirstCycleBreaker and mMixer are updated when necessary. SetStreamOrderDirty(); - mStreams.RemoveElement(aStream); - mSuspendedStreams.RemoveElement(aStream); + if (aStream->IsSuspended()) { + mSuspendedStreams.RemoveElement(aStream); + } else { + mStreams.RemoveElement(aStream); + } NS_RELEASE(aStream); // probably destroying it STREAM_LOG(LogLevel::Debug, ("Removing media stream %p from the graph", aStream)); } -void -MediaStreamGraphImpl::UpdateConsumptionState(SourceMediaStream* aStream) -{ - MediaStreamListener::Consumption state = - aStream->mIsConsumed ? MediaStreamListener::CONSUMED - : MediaStreamListener::NOT_CONSUMED; - if (state != aStream->mLastConsumptionState) { - aStream->mLastConsumptionState = state; - for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) { - MediaStreamListener* l = aStream->mListeners[j]; - l->NotifyConsumptionChanged(this, state); - } - } -} - void MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream, GraphTime aDesiredUpToTime, @@ -183,11 +157,8 @@ MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream, if (aStream->mPullEnabled && !aStream->mFinished && !aStream->mListeners.IsEmpty()) { // Compute how much stream time we'll need assuming we don't block - // the stream at all between mBlockingDecisionsMadeUntilTime and - // aDesiredUpToTime. - StreamTime t = - GraphTimeToStreamTime(aStream, mStateComputedTime) + - (aDesiredUpToTime - mStateComputedTime); + // the stream at all. + StreamTime t = aStream->GraphTimeToStreamTime(aDesiredUpToTime); STREAM_LOG(LogLevel::Verbose, ("Calling NotifyPull aStream=%p t=%f current end=%f", aStream, MediaTimeToSeconds(t), MediaTimeToSeconds(aStream->mBuffer.GetEnd()))); @@ -266,79 +237,36 @@ MediaStreamGraphImpl::ExtractPendingInput(SourceMediaStream* aStream, } StreamTime -MediaStreamGraphImpl::GraphTimeToStreamTime(MediaStream* aStream, - GraphTime aTime) +MediaStreamGraphImpl::GraphTimeToStreamTimeWithBlocking(MediaStream* aStream, + GraphTime aTime) { MOZ_ASSERT(aTime <= mStateComputedTime, - "Don't ask about times where we haven't made blocking decisions yet"); - if (aTime <= mProcessedTime) { - return std::max(0, aTime - aStream->mBufferStartTime); - } - GraphTime t = mProcessedTime; - StreamTime s = t - aStream->mBufferStartTime; - while (t < aTime) { - GraphTime end; - if (!aStream->mBlocked.GetAt(t, &end)) { - s += std::min(aTime, end) - t; - } - t = end; - } - return std::max(0, s); -} - -StreamTime -MediaStreamGraphImpl::GraphTimeToStreamTimeOptimistic(MediaStream* aStream, - GraphTime aTime) -{ - GraphTime computedUpToTime = std::min(mStateComputedTime, aTime); - StreamTime s = GraphTimeToStreamTime(aStream, computedUpToTime); - return s + (aTime - computedUpToTime); + "Don't ask about times where we haven't made blocking decisions yet"); + return std::max(0, + std::min(aTime, aStream->mStartBlocking) - aStream->mBufferStartTime); } GraphTime -MediaStreamGraphImpl::StreamTimeToGraphTime(MediaStream* aStream, - StreamTime aTime, uint32_t aFlags) +MediaStreamGraphImpl::StreamTimeToGraphTimeWithBlocking(MediaStream* aStream, + StreamTime aTime, uint32_t aFlags) { + // Avoid overflows if (aTime >= STREAM_TIME_MAX) { return GRAPH_TIME_MAX; } - MediaTime bufferElapsedToCurrentTime = - mProcessedTime - aStream->mBufferStartTime; - if (aTime < bufferElapsedToCurrentTime || - (aTime == bufferElapsedToCurrentTime && !(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL))) { - return aTime + aStream->mBufferStartTime; - } - MediaTime streamAmount = aTime - bufferElapsedToCurrentTime; - NS_ASSERTION(streamAmount >= 0, "Can't answer queries before current time"); - - GraphTime t = mProcessedTime; - while (t < GRAPH_TIME_MAX) { - if (!(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL) && streamAmount == 0) { - return t; - } - bool blocked; - GraphTime end; - if (t < mStateComputedTime) { - blocked = aStream->mBlocked.GetAt(t, &end); - end = std::min(end, mStateComputedTime); - } else { - blocked = false; - end = GRAPH_TIME_MAX; - } - if (blocked) { - t = end; - } else { - if (streamAmount == 0) { - // No more stream time to consume at time t, so we're done. - break; - } - MediaTime consume = std::min(end - t, streamAmount); - streamAmount -= consume; - t += consume; - } + // Assume we're unblocked from 0..mStartBlocking, blocked from mStartBlocking + // to mStateComputedTime, and unblocked from mStateComputedTime..forever + GraphTime timeAssumingNoBlocking = aTime + aStream->mBufferStartTime; + if (timeAssumingNoBlocking < aStream->mStartBlocking || + (timeAssumingNoBlocking == aStream->mStartBlocking && + !(aFlags & INCLUDE_TRAILING_BLOCKED_INTERVAL))) { + return timeAssumingNoBlocking; } - return t; + // XXX we generally shouldn't need to call this for aTime >= mStartBlocking! + // Check callers. + + return timeAssumingNoBlocking + (mStateComputedTime - aStream->mStartBlocking); } GraphTime @@ -348,109 +276,74 @@ MediaStreamGraphImpl::IterationEnd() const } void -MediaStreamGraphImpl::StreamNotifyOutput(MediaStream* aStream) +MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime) { - for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) { - MediaStreamListener* l = aStream->mListeners[j]; - l->NotifyOutput(this, mProcessedTime); - } -} + for (MediaStream* stream : AllStreams()) { + // Calculate blocked time and fire Blocked/Unblocked events + GraphTime blockedTime = mStateComputedTime - stream->mStartBlocking; + NS_ASSERTION(blockedTime >= 0, "Error in blocking time"); -void -MediaStreamGraphImpl::StreamReadyToFinish(MediaStream* aStream) -{ - MOZ_ASSERT(aStream->mFinished); - MOZ_ASSERT(!aStream->mNotifiedFinished); + if (stream->mStartBlocking > aPrevCurrentTime && stream->mNotifiedBlocked) { + for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyBlockingChanged(this, MediaStreamListener::UNBLOCKED); + } + stream->mNotifiedBlocked = false; + } + if (stream->mStartBlocking < mStateComputedTime && !stream->mNotifiedBlocked) { + for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyBlockingChanged(this, MediaStreamListener::BLOCKED); + } + stream->mNotifiedBlocked = true; + } - // The stream is fully finished when all of its track data has been played - // out. - if (mProcessedTime >= - aStream->StreamTimeToGraphTime(aStream->GetStreamBuffer().GetAllTracksEnd())) { - aStream->mNotifiedFinished = true; - SetStreamOrderDirty(); - for (uint32_t j = 0; j < aStream->mListeners.Length(); ++j) { - MediaStreamListener* l = aStream->mListeners[j]; - l->NotifyEvent(this, MediaStreamListener::EVENT_FINISHED); + stream->AdvanceTimeVaryingValuesToCurrentTime(mStateComputedTime, + blockedTime); + + STREAM_LOG(LogLevel::Verbose, + ("MediaStream %p bufferStartTime=%f blockedTime=%f", stream, + MediaTimeToSeconds(stream->mBufferStartTime), + MediaTimeToSeconds(blockedTime))); + + if (stream->mStartBlocking > aPrevCurrentTime) { + NS_ASSERTION(!stream->mNotifiedFinished, + "Shouldn't have already notified of finish *and* have output!"); + for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyOutput(this, mProcessedTime); + } + } + + // The stream is fully finished when all of its track data has been played + // out. + if (stream->mFinished && !stream->mNotifiedFinished && + mProcessedTime >= + stream->StreamTimeToGraphTimeWithBlocking(stream->GetStreamBuffer().GetAllTracksEnd())) { + stream->mNotifiedFinished = true; + SetStreamOrderDirty(); + for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { + MediaStreamListener* l = stream->mListeners[j]; + l->NotifyEvent(this, MediaStreamListener::EVENT_FINISHED); + } } } } -void -MediaStreamGraphImpl::UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, - GraphTime aNextCurrentTime) -{ - nsTArray* runningAndSuspendedPair[2]; - runningAndSuspendedPair[0] = &mStreams; - runningAndSuspendedPair[1] = &mSuspendedStreams; - - for (uint32_t array = 0; array < 2; array++) { - for (uint32_t i = 0; i < runningAndSuspendedPair[array]->Length(); ++i) { - MediaStream* stream = (*runningAndSuspendedPair[array])[i]; - - // Calculate blocked time and fire Blocked/Unblocked events - GraphTime blockedTime = 0; - GraphTime t = aPrevCurrentTime; - // include |nextCurrentTime| to ensure NotifyBlockingChanged() is called - // before NotifyEvent(this, EVENT_FINISHED) when |nextCurrentTime == - // stream end time| - while (t <= aNextCurrentTime) { - GraphTime end; - bool blocked = stream->mBlocked.GetAt(t, &end); - if (blocked) { - blockedTime += std::min(end, aNextCurrentTime) - t; - } - if (blocked != stream->mNotifiedBlocked) { - for (uint32_t j = 0; j < stream->mListeners.Length(); ++j) { - MediaStreamListener* l = stream->mListeners[j]; - l->NotifyBlockingChanged(this, blocked - ? MediaStreamListener::BLOCKED - : MediaStreamListener::UNBLOCKED); - } - stream->mNotifiedBlocked = blocked; - } - t = end; - } - - stream->AdvanceTimeVaryingValuesToCurrentTime(aNextCurrentTime, - blockedTime); - // Advance mBlocked last so that AdvanceTimeVaryingValuesToCurrentTime - // can rely on the value of mBlocked. - stream->mBlocked.AdvanceCurrentTime(aNextCurrentTime); - - if (runningAndSuspendedPair[array] == &mStreams) { - bool streamHasOutput = blockedTime < aNextCurrentTime - aPrevCurrentTime; - NS_ASSERTION(!streamHasOutput || !stream->mNotifiedFinished, - "Shouldn't have already notified of finish *and* have output!"); - - if (streamHasOutput) { - StreamNotifyOutput(stream); - } - - if (stream->mFinished && !stream->mNotifiedFinished) { - StreamReadyToFinish(stream); - } - } - STREAM_LOG(LogLevel::Verbose, - ("MediaStream %p bufferStartTime=%f blockedTime=%f", stream, - MediaTimeToSeconds(stream->mBufferStartTime), - MediaTimeToSeconds(blockedTime))); - } - } -} - -bool -MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime, - GraphTime aEndBlockingDecisions, GraphTime* aEnd) +GraphTime +MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, + GraphTime aEndBlockingDecisions) { // Finished streams can't underrun. ProcessedMediaStreams also can't cause // underrun currently, since we'll always be able to produce data for them // unless they block on some other stream. if (aStream->mFinished || aStream->AsProcessedStream()) { - return false; + return aEndBlockingDecisions; } - GraphTime bufferEnd = - StreamTimeToGraphTime(aStream, aStream->GetBufferEnd(), - INCLUDE_TRAILING_BLOCKED_INTERVAL); + // This stream isn't finished or suspended. We don't need to call + // StreamTimeToGraphTime since an underrun is the only thing that can block + // it. + GraphTime bufferEnd = aStream->GetBufferEnd() + aStream->mBufferStartTime; #ifdef DEBUG if (bufferEnd < mProcessedTime) { STREAM_LOG(LogLevel::Error, ("MediaStream %p underrun, " @@ -461,53 +354,7 @@ MediaStreamGraphImpl::WillUnderrun(MediaStream* aStream, GraphTime aTime, NS_ASSERTION(bufferEnd >= mProcessedTime, "Buffer underran"); } #endif - // We should block after bufferEnd. - if (bufferEnd <= aTime) { - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p will block due to data underrun at %ld, " - "bufferEnd %ld", - aStream, aTime, bufferEnd)); - return true; - } - // We should keep blocking if we're currently blocked and we don't have - // data all the way through to aEndBlockingDecisions. If we don't have - // data all the way through to aEndBlockingDecisions, we'll block soon, - // but we might as well remain unblocked and play the data we've got while - // we can. - if (bufferEnd < aEndBlockingDecisions && aStream->mBlocked.GetBefore(aTime)) { - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p will block due to speculative data underrun, " - "bufferEnd %f (end at %ld)", - aStream, MediaTimeToSeconds(bufferEnd), bufferEnd)); - return true; - } - // Reconsider decisions at bufferEnd - *aEnd = std::min(*aEnd, bufferEnd); - return false; -} - -void -MediaStreamGraphImpl::MarkConsumed(MediaStream* aStream) -{ - if (aStream->mIsConsumed) { - return; - } - aStream->mIsConsumed = true; - - ProcessedMediaStream* ps = aStream->AsProcessedStream(); - if (!ps) { - return; - } - // Mark all the inputs to this stream as consumed - for (uint32_t i = 0; i < ps->mInputs.Length(); ++i) { - MarkConsumed(ps->mInputs[i]->mSource); - } -} - -bool -MediaStreamGraphImpl::StreamSuspended(MediaStream* aStream) -{ - // Only AudioNodeStreams can be suspended, so we can shortcut here. - return aStream->AsAudioNodeStream() && - mSuspendedStreams.IndexOf(aStream) != mSuspendedStreams.NoIndex; + return std::min(bufferEnd, aEndBlockingDecisions); } namespace { @@ -526,8 +373,6 @@ MediaStreamGraphImpl::UpdateStreamOrder() bool audioTrackPresent = false; for (uint32_t i = 0; i < mStreams.Length(); ++i) { MediaStream* stream = mStreams[i]; - stream->mIsConsumed = false; - stream->mInBlockingSet = false; #ifdef MOZ_WEBRTC if (stream->AsSourceStream() && stream->AsSourceStream()->NeedsMixing()) { @@ -597,9 +442,6 @@ MediaStreamGraphImpl::UpdateStreamOrder() for (uint32_t i = 0; i < mStreams.Length(); ++i) { MediaStream* s = mStreams[i]; - if (s->IsIntrinsicallyConsumed()) { - MarkConsumed(s); - } ProcessedMediaStream* ps = s->AsProcessedStream(); if (ps) { // The dfsStack initially contains a list of all processed streams in @@ -640,7 +482,7 @@ MediaStreamGraphImpl::UpdateStreamOrder() // Not-visited input streams should be processed first. // SourceMediaStreams have already been ordered. for (uint32_t i = inputs.Length(); i--; ) { - if (StreamSuspended(inputs[i]->mSource)) { + if (inputs[i]->mSource->IsSuspended()) { continue; } auto input = inputs[i]->mSource->AsProcessedStream(); @@ -666,7 +508,7 @@ MediaStreamGraphImpl::UpdateStreamOrder() // unless it is part of the cycle. uint32_t cycleStackMarker = 0; for (uint32_t i = inputs.Length(); i--; ) { - if (StreamSuspended(inputs[i]->mSource)) { + if (inputs[i]->mSource->IsSuspended()) { continue; } auto input = inputs[i]->mSource->AsProcessedStream(); @@ -757,150 +599,6 @@ MediaStreamGraphImpl::UpdateStreamOrder() MOZ_ASSERT(orderedStreamCount == mFirstCycleBreaker); } -void -MediaStreamGraphImpl::RecomputeBlocking(GraphTime aEndBlockingDecisions) -{ - STREAM_LOG(LogLevel::Verbose, ("Media graph %p computing blocking for time %f", - this, MediaTimeToSeconds(mStateComputedTime))); - nsTArray* runningAndSuspendedPair[2]; - runningAndSuspendedPair[0] = &mStreams; - runningAndSuspendedPair[1] = &mSuspendedStreams; - - for (uint32_t array = 0; array < 2; array++) { - for (uint32_t i = 0; i < (*runningAndSuspendedPair[array]).Length(); ++i) { - MediaStream* stream = (*runningAndSuspendedPair[array])[i]; - if (!stream->mInBlockingSet) { - // Compute a partition of the streams containing 'stream' such that we - // can - // compute the blocking status of each subset independently. - nsAutoTArray streamSet; - AddBlockingRelatedStreamsToSet(&streamSet, stream); - - GraphTime end; - for (GraphTime t = mStateComputedTime; - t < aEndBlockingDecisions; t = end) { - end = GRAPH_TIME_MAX; - RecomputeBlockingAt(streamSet, t, aEndBlockingDecisions, &end); - } - } - } - } - STREAM_LOG(LogLevel::Verbose, ("Media graph %p computed blocking for interval %f to %f", - this, MediaTimeToSeconds(mStateComputedTime), - MediaTimeToSeconds(aEndBlockingDecisions))); - - MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime); - // The next state computed time can be the same as the previous: it - // means the driver would be have been blocking indefinitly, but the graph has - // been woken up right after having been to sleep. - MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime); - mStateComputedTime = aEndBlockingDecisions; -} - -void -MediaStreamGraphImpl::AddBlockingRelatedStreamsToSet(nsTArray* aStreams, - MediaStream* aStream) -{ - if (aStream->mInBlockingSet) - return; - aStream->mInBlockingSet = true; - aStreams->AppendElement(aStream); - for (uint32_t i = 0; i < aStream->mConsumers.Length(); ++i) { - MediaInputPort* port = aStream->mConsumers[i]; - if (port->mFlags & (MediaInputPort::FLAG_BLOCK_INPUT | MediaInputPort::FLAG_BLOCK_OUTPUT)) { - AddBlockingRelatedStreamsToSet(aStreams, port->mDest); - } - } - ProcessedMediaStream* ps = aStream->AsProcessedStream(); - if (ps) { - for (uint32_t i = 0; i < ps->mInputs.Length(); ++i) { - MediaInputPort* port = ps->mInputs[i]; - if (port->mFlags & (MediaInputPort::FLAG_BLOCK_INPUT | MediaInputPort::FLAG_BLOCK_OUTPUT)) { - AddBlockingRelatedStreamsToSet(aStreams, port->mSource); - } - } - } -} - -void -MediaStreamGraphImpl::MarkStreamBlocking(MediaStream* aStream) -{ - if (aStream->mBlockInThisPhase) - return; - aStream->mBlockInThisPhase = true; - for (uint32_t i = 0; i < aStream->mConsumers.Length(); ++i) { - MediaInputPort* port = aStream->mConsumers[i]; - if (port->mFlags & MediaInputPort::FLAG_BLOCK_OUTPUT) { - MarkStreamBlocking(port->mDest); - } - } - ProcessedMediaStream* ps = aStream->AsProcessedStream(); - if (ps) { - for (uint32_t i = 0; i < ps->mInputs.Length(); ++i) { - MediaInputPort* port = ps->mInputs[i]; - if (port->mFlags & MediaInputPort::FLAG_BLOCK_INPUT) { - MarkStreamBlocking(port->mSource); - } - } - } -} - -void -MediaStreamGraphImpl::RecomputeBlockingAt(const nsTArray& aStreams, - GraphTime aTime, - GraphTime aEndBlockingDecisions, - GraphTime* aEnd) -{ - for (uint32_t i = 0; i < aStreams.Length(); ++i) { - MediaStream* stream = aStreams[i]; - stream->mBlockInThisPhase = false; - } - - for (uint32_t i = 0; i < aStreams.Length(); ++i) { - MediaStream* stream = aStreams[i]; - - if (stream->mFinished) { - GraphTime endTime = StreamTimeToGraphTime(stream, - stream->GetStreamBuffer().GetAllTracksEnd()); - if (endTime <= aTime) { - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p is blocked due to being finished", stream)); - // We'll block indefinitely - MarkStreamBlocking(stream); - *aEnd = std::min(*aEnd, aEndBlockingDecisions); - continue; - } else { - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p is finished, but not blocked yet (end at %f, with blocking at %f)", - stream, MediaTimeToSeconds(stream->GetBufferEnd()), - MediaTimeToSeconds(endTime))); - *aEnd = std::min(*aEnd, endTime); - } - } - - GraphTime end; - bool explicitBlock = stream->mExplicitBlockerCount.GetAt(aTime, &end) > 0; - *aEnd = std::min(*aEnd, end); - if (explicitBlock) { - STREAM_LOG(LogLevel::Verbose, ("MediaStream %p is blocked due to explicit blocker", stream)); - MarkStreamBlocking(stream); - continue; - } - - bool underrun = WillUnderrun(stream, aTime, aEndBlockingDecisions, aEnd); - if (underrun) { - // We'll block indefinitely - MarkStreamBlocking(stream); - *aEnd = std::min(*aEnd, aEndBlockingDecisions); - continue; - } - } - NS_ASSERTION(*aEnd > aTime, "Failed to advance!"); - - for (uint32_t i = 0; i < aStreams.Length(); ++i) { - MediaStream* stream = aStreams[i]; - stream->mBlocked.SetAtAndAfter(aTime, stream->mBlockInThisPhase); - } -} - void MediaStreamGraphImpl::NotifyHasCurrentData(MediaStream* aStream) { @@ -914,8 +612,7 @@ MediaStreamGraphImpl::NotifyHasCurrentData(MediaStream* aStream) } void -MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTime, - MediaStream* aStream) +MediaStreamGraphImpl::CreateOrDestroyAudioStreams(MediaStream* aStream) { MOZ_ASSERT(mRealtime, "Should only attempt to create audio streams in real-time mode"); @@ -924,38 +621,40 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTim return; } + if (!aStream->GetStreamBuffer().GetAndResetTracksDirty()) { + return; + } + nsAutoTArray audioOutputStreamsFound; for (uint32_t i = 0; i < aStream->mAudioOutputStreams.Length(); ++i) { audioOutputStreamsFound.AppendElement(false); } - if (!aStream->mAudioOutputs.IsEmpty()) { - for (StreamBuffer::TrackIter tracks(aStream->GetStreamBuffer(), MediaSegment::AUDIO); - !tracks.IsEnded(); tracks.Next()) { - uint32_t i; - for (i = 0; i < audioOutputStreamsFound.Length(); ++i) { - if (aStream->mAudioOutputStreams[i].mTrackID == tracks->GetID()) { - break; - } + for (StreamBuffer::TrackIter tracks(aStream->GetStreamBuffer(), MediaSegment::AUDIO); + !tracks.IsEnded(); tracks.Next()) { + uint32_t i; + for (i = 0; i < audioOutputStreamsFound.Length(); ++i) { + if (aStream->mAudioOutputStreams[i].mTrackID == tracks->GetID()) { + break; } - if (i < audioOutputStreamsFound.Length()) { - audioOutputStreamsFound[i] = true; - } else { - MediaStream::AudioOutputStream* audioOutputStream = - aStream->mAudioOutputStreams.AppendElement(); - audioOutputStream->mAudioPlaybackStartTime = aAudioOutputStartTime; - audioOutputStream->mBlockedAudioTime = 0; - audioOutputStream->mLastTickWritten = 0; - audioOutputStream->mTrackID = tracks->GetID(); + } + if (i < audioOutputStreamsFound.Length()) { + audioOutputStreamsFound[i] = true; + } else { + MediaStream::AudioOutputStream* audioOutputStream = + aStream->mAudioOutputStreams.AppendElement(); + audioOutputStream->mAudioPlaybackStartTime = mProcessedTime; + audioOutputStream->mBlockedAudioTime = 0; + audioOutputStream->mLastTickWritten = 0; + audioOutputStream->mTrackID = tracks->GetID(); - if (!CurrentDriver()->AsAudioCallbackDriver() && - !CurrentDriver()->Switching()) { - MonitorAutoLock mon(mMonitor); - if (mLifecycleState == LIFECYCLE_RUNNING) { - AudioCallbackDriver* driver = new AudioCallbackDriver(this); - mMixer.AddCallback(driver); - CurrentDriver()->SwitchAtNextIteration(driver); - } + if (!CurrentDriver()->AsAudioCallbackDriver() && + !CurrentDriver()->Switching()) { + MonitorAutoLock mon(mMonitor); + if (mLifecycleState == LIFECYCLE_RUNNING) { + AudioCallbackDriver* driver = new AudioCallbackDriver(this); + mMixer.AddCallback(driver); + CurrentDriver()->SwitchAtNextIteration(driver); } } } @@ -969,60 +668,41 @@ MediaStreamGraphImpl::CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTim } StreamTime -MediaStreamGraphImpl::PlayAudio(MediaStream* aStream, - GraphTime aFrom, GraphTime aTo) +MediaStreamGraphImpl::PlayAudio(MediaStream* aStream) { MOZ_ASSERT(mRealtime, "Should only attempt to play audio in realtime mode"); - StreamTime ticksWritten = 0; - // We compute the number of needed ticks by converting a difference of graph - // time rather than by substracting two converted stream time to ensure that - // the rounding between {Graph,Stream}Time and track ticks is not dependant - // on the absolute value of the {Graph,Stream}Time, and so that number of - // ticks to play is the same for each cycle. - StreamTime ticksNeeded = aTo - aFrom; - - if (aStream->mAudioOutputStreams.IsEmpty()) { - return 0; - } - float volume = 0.0f; for (uint32_t i = 0; i < aStream->mAudioOutputs.Length(); ++i) { volume += aStream->mAudioOutputs[i].mVolume; } + StreamTime ticksWritten = 0; + for (uint32_t i = 0; i < aStream->mAudioOutputStreams.Length(); ++i) { + ticksWritten = 0; + MediaStream::AudioOutputStream& audioOutput = aStream->mAudioOutputStreams[i]; StreamBuffer::Track* track = aStream->mBuffer.FindTrack(audioOutput.mTrackID); AudioSegment* audio = track->Get(); AudioSegment output; - // offset and audioOutput.mLastTickWritten can differ by at most one sample, - // because of the rounding issue. We track that to ensure we don't skip a - // sample. One sample may be played twice, but this should not happen - // again during an unblocked sequence of track samples. - StreamTime offset = GraphTimeToStreamTime(aStream, aFrom); + StreamTime offset = GraphTimeToStreamTimeWithBlocking(aStream, mProcessedTime); // We don't update aStream->mBufferStartTime here to account for time spent // blocked. Instead, we'll update it in UpdateCurrentTimeForStreams after // the blocked period has completed. But we do need to make sure we play // from the right offsets in the stream buffer, even if we've already // written silence for some amount of blocked time after the current time. - GraphTime t = aFrom; - while (ticksNeeded) { - GraphTime end; - bool blocked = aStream->mBlocked.GetAt(t, &end); - end = std::min(end, aTo); + GraphTime t = mProcessedTime; + while (t < mStateComputedTime) { + bool blocked = t >= aStream->mStartBlocking; + GraphTime end = blocked ? mStateComputedTime : aStream->mStartBlocking; + NS_ASSERTION(end <= mStateComputedTime, "mStartBlocking is wrong!"); // Check how many ticks of sound we can provide if we are blocked some // time in the middle of this cycle. - StreamTime toWrite = 0; - if (end >= aTo) { - toWrite = ticksNeeded; - } else { - toWrite = end - t; - } - ticksNeeded -= toWrite; + StreamTime toWrite = end - t; if (blocked) { output.InsertNullDataAtStart(toWrite); @@ -1128,8 +808,8 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) RefPtr blackImage; MOZ_ASSERT(mProcessedTime >= aStream->mBufferStartTime, "frame position before buffer?"); - StreamTime frameBufferTime = GraphTimeToStreamTime(aStream, mProcessedTime); - StreamTime bufferEndTime = GraphTimeToStreamTime(aStream, mStateComputedTime); + StreamTime frameBufferTime = GraphTimeToStreamTimeWithBlocking(aStream, mProcessedTime); + StreamTime bufferEndTime = GraphTimeToStreamTimeWithBlocking(aStream, mStateComputedTime); StreamTime start; const VideoChunk* chunk; for ( ; @@ -1168,7 +848,7 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) // its start time. These times only differ in the case of multiple // tracks. GraphTime frameTime = - StreamTimeToGraphTime(aStream, frameBufferTime, + StreamTimeToGraphTimeWithBlocking(aStream, frameBufferTime, INCLUDE_TRAILING_BLOCKED_INTERVAL); TimeStamp targetTime = currentTimeStamp + TimeDuration::FromSeconds(MediaTimeToSeconds(frameTime - IterationEnd())); @@ -1281,16 +961,16 @@ MediaStreamGraphImpl::PrepareUpdatesToMainThreadState(bool aFinalUpdate) // We don't want to frequently update the main thread about timing update // when we are not running in realtime. if (aFinalUpdate || ShouldUpdateMainThread()) { - mStreamUpdates.SetCapacity(mStreamUpdates.Length() + mStreams.Length()); - for (uint32_t i = 0; i < mStreams.Length(); ++i) { - MediaStream* stream = mStreams[i]; + mStreamUpdates.SetCapacity(mStreamUpdates.Length() + mStreams.Length() + + mSuspendedStreams.Length()); + for (MediaStream* stream : AllStreams()) { if (!stream->MainThreadNeedsUpdates()) { continue; } StreamUpdate* update = mStreamUpdates.AppendElement(); update->mStream = stream; update->mNextMainThreadCurrentTime = - GraphTimeToStreamTime(stream, mProcessedTime); + GraphTimeToStreamTimeWithBlocking(stream, mProcessedTime); update->mNextMainThreadFinished = stream->mNotifiedFinished; } if (!mPendingUpdateRunnables.IsEmpty()) { @@ -1319,14 +999,12 @@ MediaStreamGraphImpl::RoundUpToNextAudioBlock(GraphTime aTime) void MediaStreamGraphImpl::ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex, - TrackRate aSampleRate, - GraphTime aFrom, - GraphTime aTo) + TrackRate aSampleRate) { MOZ_ASSERT(aStreamIndex <= mFirstCycleBreaker, "Cycle breaker is not AudioNodeStream?"); - GraphTime t = aFrom; - while (t < aTo) { + GraphTime t = mProcessedTime; + while (t < mStateComputedTime) { GraphTime next = RoundUpToNextAudioBlock(t); for (uint32_t i = mFirstCycleBreaker; i < mStreams.Length(); ++i) { auto ns = static_cast(mStreams[i]); @@ -1336,28 +1014,21 @@ MediaStreamGraphImpl::ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex, for (uint32_t i = aStreamIndex; i < mStreams.Length(); ++i) { ProcessedMediaStream* ps = mStreams[i]->AsProcessedStream(); if (ps) { - ps->ProcessInput(t, next, (next == aTo) ? ProcessedMediaStream::ALLOW_FINISH : 0); - } - } - // Remove references to shared AudioChunk buffers from downstream nodes - // first so that upstream nodes can re-use next iteration. - for (uint32_t i = mStreams.Length(); i--; ) { - AudioNodeStream* ns = mStreams[i]->AsAudioNodeStream(); - if (ns) { - ns->ReleaseSharedBuffers(); + ps->ProcessInput(t, next, + (next == mStateComputedTime) ? ProcessedMediaStream::ALLOW_FINISH : 0); } } t = next; } - NS_ASSERTION(t == aTo, "Something went wrong with rounding to block boundaries"); + NS_ASSERTION(t == mStateComputedTime, + "Something went wrong with rounding to block boundaries"); } bool MediaStreamGraphImpl::AllFinishedStreamsNotified() { - for (uint32_t i = 0; i < mStreams.Length(); ++i) { - MediaStream* s = mStreams[i]; - if (s->mFinished && !s->mNotifiedFinished) { + for (MediaStream* stream : AllStreams()) { + if (stream->mFinished && !stream->mNotifiedFinished) { return false; } } @@ -1365,7 +1036,7 @@ MediaStreamGraphImpl::AllFinishedStreamsNotified() } void -MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecision) +MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecisions) { // Calculate independent action times for each batch of messages (each // batch corresponding to an event loop task). This isolates the performance @@ -1379,17 +1050,45 @@ MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecision) } mFrontMessageQueue.Clear(); + MOZ_ASSERT(aEndBlockingDecisions >= mProcessedTime); + // The next state computed time can be the same as the previous: it + // means the driver would be have been blocking indefinitly, but the graph has + // been woken up right after having been to sleep. + MOZ_ASSERT(aEndBlockingDecisions >= mStateComputedTime); + UpdateStreamOrder(); bool ensureNextIteration = false; - // Grab pending stream input. - for (uint32_t i = 0; i < mStreams.Length(); ++i) { - SourceMediaStream* is = mStreams[i]->AsSourceStream(); - if (is) { - UpdateConsumptionState(is); - ExtractPendingInput(is, aEndBlockingDecision, &ensureNextIteration); + // Grab pending stream input and compute blocking time + for (MediaStream* stream : mStreams) { + if (SourceMediaStream* is = stream->AsSourceStream()) { + ExtractPendingInput(is, aEndBlockingDecisions, &ensureNextIteration); } + + if (stream->mFinished) { + // The stream's not suspended, and since it's finished, underruns won't + // stop it playing out. So there's no blocking other than what we impose + // here. + GraphTime endTime = stream->GetStreamBuffer().GetAllTracksEnd() + + stream->mBufferStartTime; + if (endTime <= mStateComputedTime) { + STREAM_LOG(LogLevel::Verbose, ("MediaStream %p is blocked due to being finished", stream)); + stream->mStartBlocking = mStateComputedTime; + } else { + STREAM_LOG(LogLevel::Verbose, ("MediaStream %p is finished, but not blocked yet (end at %f, with blocking at %f)", + stream, MediaTimeToSeconds(stream->GetBufferEnd()), + MediaTimeToSeconds(endTime))); + // Data can't be added to a finished stream, so underruns are irrelevant. + stream->mStartBlocking = std::min(endTime, aEndBlockingDecisions); + } + } else { + stream->mStartBlocking = WillUnderrun(stream, aEndBlockingDecisions); + } + } + + for (MediaStream* stream : mSuspendedStreams) { + stream->mStartBlocking = mStateComputedTime; } // The loop is woken up so soon that IterationEnd() barely advances and we @@ -1399,16 +1098,13 @@ MediaStreamGraphImpl::UpdateGraph(GraphTime aEndBlockingDecision) // We should ensure next iteration so that pending blocking changes will be // computed in next loop. if (ensureNextIteration || - aEndBlockingDecision == mStateComputedTime) { + aEndBlockingDecisions == mStateComputedTime) { EnsureNextIteration(); } - - // Figure out which streams are blocked and when. - RecomputeBlocking(aEndBlockingDecision); } void -MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) +MediaStreamGraphImpl::Process() { // Play stream contents. bool allBlockedForever = true; @@ -1440,12 +1136,13 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) #endif // Since an AudioNodeStream is present, go ahead and // produce audio block by block for all the rest of the streams. - ProduceDataForStreamsBlockByBlock(i, n->SampleRate(), aFrom, aTo); + ProduceDataForStreamsBlockByBlock(i, n->SampleRate()); doneAllProducing = true; } else { - ps->ProcessInput(aFrom, aTo, ProcessedMediaStream::ALLOW_FINISH); + ps->ProcessInput(mProcessedTime, mStateComputedTime, + ProcessedMediaStream::ALLOW_FINISH); NS_WARN_IF_FALSE(stream->mBuffer.GetEnd() >= - GraphTimeToStreamTime(stream, aTo), + GraphTimeToStreamTimeWithBlocking(stream, mStateComputedTime), "Stream did not produce enough data"); } } @@ -1453,9 +1150,9 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) NotifyHasCurrentData(stream); // Only playback audio and video in real-time mode if (mRealtime) { - CreateOrDestroyAudioStreams(aFrom, stream); + CreateOrDestroyAudioStreams(stream); if (CurrentDriver()->AsAudioCallbackDriver()) { - StreamTime ticksPlayedForThisStream = PlayAudio(stream, aFrom, aTo); + StreamTime ticksPlayedForThisStream = PlayAudio(stream); if (!ticksPlayed) { ticksPlayed = ticksPlayedForThisStream; } else { @@ -1465,8 +1162,7 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) } PlayVideo(stream); } - GraphTime end; - if (!stream->mBlocked.GetAt(aTo, &end) || end < GRAPH_TIME_MAX) { + if (stream->mStartBlocking > mProcessedTime) { allBlockedForever = false; } } @@ -1494,62 +1190,69 @@ MediaStreamGraphImpl::Process(GraphTime aFrom, GraphTime aTo) } } +void +MediaStreamGraphImpl::MaybeProduceMemoryReport() +{ + MonitorAutoLock lock(mMemoryReportMonitor); + if (mNeedsMemoryReport) { + mNeedsMemoryReport = false; + + for (MediaStream* s : AllStreams()) { + AudioNodeStream* stream = s->AsAudioNodeStream(); + if (stream) { + AudioNodeSizes usage; + stream->SizeOfAudioNodesIncludingThis(MallocSizeOf, usage); + mAudioStreamSizes.AppendElement(usage); + } + } + + lock.Notify(); + } +} + +bool +MediaStreamGraphImpl::UpdateMainThreadState() +{ + MonitorAutoLock lock(mMonitor); + bool finalUpdate = mForceShutDown || + (mProcessedTime >= mEndTime && AllFinishedStreamsNotified()) || + (IsEmpty() && mBackMessageQueue.IsEmpty()); + PrepareUpdatesToMainThreadState(finalUpdate); + if (finalUpdate) { + // Enter shutdown mode. The stable-state handler will detect this + // and complete shutdown. Destroy any streams immediately. + STREAM_LOG(LogLevel::Debug, ("MediaStreamGraph %p waiting for main thread cleanup", this)); + // We'll shut down this graph object if it does not get restarted. + mLifecycleState = LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP; + // No need to Destroy streams here. The main-thread owner of each + // stream is responsible for calling Destroy on them. + return false; + } + + CurrentDriver()->WaitForNextIteration(); + + SwapMessageQueues(); + return true; +} + bool MediaStreamGraphImpl::OneIteration(GraphTime aStateEnd) { - { - MonitorAutoLock lock(mMemoryReportMonitor); - if (mNeedsMemoryReport) { - mNeedsMemoryReport = false; + MaybeProduceMemoryReport(); - for (uint32_t i = 0; i < mStreams.Length(); ++i) { - AudioNodeStream* stream = mStreams[i]->AsAudioNodeStream(); - if (stream) { - AudioNodeSizes usage; - stream->SizeOfAudioNodesIncludingThis(MallocSizeOf, usage); - mAudioStreamSizes.AppendElement(usage); - } - } - - lock.Notify(); - } - } - - GraphTime stateFrom = mStateComputedTime; GraphTime stateEnd = std::min(aStateEnd, mEndTime); UpdateGraph(stateEnd); - Process(stateFrom, stateEnd); + mStateComputedTime = stateEnd; + + Process(); + + GraphTime oldProcessedTime = mProcessedTime; mProcessedTime = stateEnd; - UpdateCurrentTimeForStreams(stateFrom, stateEnd); + UpdateCurrentTimeForStreams(oldProcessedTime); - // Send updates to the main thread and wait for the next control loop - // iteration. - { - MonitorAutoLock lock(mMonitor); - bool finalUpdate = mForceShutDown || - (stateEnd >= mEndTime && AllFinishedStreamsNotified()) || - (IsEmpty() && mBackMessageQueue.IsEmpty()); - PrepareUpdatesToMainThreadState(finalUpdate); - if (finalUpdate) { - // Enter shutdown mode. The stable-state handler will detect this - // and complete shutdown. Destroy any streams immediately. - STREAM_LOG(LogLevel::Debug, ("MediaStreamGraph %p waiting for main thread cleanup", this)); - // We'll shut down this graph object if it does not get restarted. - mLifecycleState = LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP; - // No need to Destroy streams here. The main-thread owner of each - // stream is responsible for calling Destroy on them. - return false; - } - - CurrentDriver()->WaitForNextIteration(); - - SwapMessageQueues(); - } - mFlushSourcesNow = false; - - return true; + return UpdateMainThreadState(); } void @@ -1622,8 +1325,8 @@ public: // delete it. NS_ASSERTION(mGraph->mForceShutDown || !mGraph->mRealtime, "Not in forced shutdown?"); - for (uint32_t i = 0; i < mGraph->mStreams.Length(); ++i) { - DOMMediaStream* s = mGraph->mStreams[i]->GetWrapper(); + for (MediaStream* stream : mGraph->AllStreams()) { + DOMMediaStream* s = stream->GetWrapper(); if (s) { s->NotifyMediaStreamGraphShutdown(); } @@ -1923,8 +1626,8 @@ MediaStreamGraphImpl::AppendMessage(ControlMessage* aMessage) MediaStream::MediaStream(DOMMediaStream* aWrapper) : mBufferStartTime(0) - , mExplicitBlockerCount(0) - , mBlocked(false) + , mStartBlocking(GRAPH_TIME_MAX) + , mSuspendedCount(0) , mFinished(false) , mNotifiedFinished(false) , mNotifiedBlocked(false) @@ -1964,11 +1667,9 @@ MediaStream::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const amount += mBuffer.SizeOfExcludingThis(aMallocSizeOf); amount += mAudioOutputs.ShallowSizeOfExcludingThis(aMallocSizeOf); amount += mVideoOutputs.ShallowSizeOfExcludingThis(aMallocSizeOf); - amount += mExplicitBlockerCount.SizeOfExcludingThis(aMallocSizeOf); amount += mListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); amount += mMainThreadListeners.ShallowSizeOfExcludingThis(aMallocSizeOf); amount += mDisabledTrackIDs.ShallowSizeOfExcludingThis(aMallocSizeOf); - amount += mBlocked.SizeOfExcludingThis(aMallocSizeOf); amount += mConsumers.ShallowSizeOfExcludingThis(aMallocSizeOf); return amount; @@ -2011,19 +1712,29 @@ MediaStream::SetGraphImpl(MediaStreamGraph* aGraph) StreamTime MediaStream::GraphTimeToStreamTime(GraphTime aTime) { - return GraphImpl()->GraphTimeToStreamTime(this, aTime); -} - -StreamTime -MediaStream::GraphTimeToStreamTimeOptimistic(GraphTime aTime) -{ - return GraphImpl()->GraphTimeToStreamTimeOptimistic(this, aTime); + NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime, + "Don't call this when there's pending blocking time!"); + return aTime - mBufferStartTime; } GraphTime MediaStream::StreamTimeToGraphTime(StreamTime aTime) { - return GraphImpl()->StreamTimeToGraphTime(this, aTime, 0); + NS_ASSERTION(mStartBlocking == GraphImpl()->mStateComputedTime, + "Don't call this when there's pending blocking time!"); + return aTime + mBufferStartTime; +} + +StreamTime +MediaStream::GraphTimeToStreamTimeWithBlocking(GraphTime aTime) +{ + return GraphImpl()->GraphTimeToStreamTimeWithBlocking(this, aTime); +} + +GraphTime +MediaStream::StreamTimeToGraphTimeWithBlocking(StreamTime aTime) +{ + return GraphImpl()->StreamTimeToGraphTimeWithBlocking(this, aTime, 0); } void @@ -2203,42 +1914,20 @@ MediaStream::RemoveVideoOutput(VideoFrameContainer* aContainer) } void -MediaStream::ChangeExplicitBlockerCount(int32_t aDelta) +MediaStream::Suspend() { class Message : public ControlMessage { public: - Message(MediaStream* aStream, int32_t aDelta) : - ControlMessage(aStream), mDelta(aDelta) {} + explicit Message(MediaStream* aStream) : + ControlMessage(aStream) {} virtual void Run() { - mStream->ChangeExplicitBlockerCountImpl( - mStream->GraphImpl()->mStateComputedTime, mDelta); + mStream->GraphImpl()->IncrementSuspendCount(mStream); } - int32_t mDelta; }; // This can happen if this method has been called asynchronously, and the // stream has been destroyed since then. - if (mMainThreadDestroyed) { - return; - } - GraphImpl()->AppendMessage(new Message(this, aDelta)); -} - -void -MediaStream::BlockStreamIfNeeded() -{ - class Message : public ControlMessage { - public: - explicit Message(MediaStream* aStream) : ControlMessage(aStream) - { } - virtual void Run() - { - mStream->BlockStreamIfNeededImpl( - mStream->GraphImpl()->mStateComputedTime); - } - }; - if (mMainThreadDestroyed) { return; } @@ -2246,19 +1935,20 @@ MediaStream::BlockStreamIfNeeded() } void -MediaStream::UnblockStreamIfNeeded() +MediaStream::Resume() { class Message : public ControlMessage { public: - explicit Message(MediaStream* aStream) : ControlMessage(aStream) - { } + explicit Message(MediaStream* aStream) : + ControlMessage(aStream) {} virtual void Run() { - mStream->UnblockStreamIfNeededImpl( - mStream->GraphImpl()->mStateComputedTime); + mStream->GraphImpl()->DecrementSuspendCount(mStream); } }; + // This can happen if this method has been called asynchronously, and the + // stream has been destroyed since then. if (mMainThreadDestroyed) { return; } @@ -2688,20 +2378,6 @@ SourceMediaStream::EndAllTrackAndFinish() // we will call NotifyEvent() to let GetUserMedia know } -StreamTime -SourceMediaStream::GetBufferedTicks(TrackID aID) -{ - StreamBuffer::Track* track = mBuffer.FindTrack(aID); - if (track) { - MediaSegment* segment = track->GetSegment(); - if (segment) { - return segment->GetDuration() - - GraphTimeToStreamTime(GraphImpl()->mStateComputedTime); - } - } - return 0; -} - void SourceMediaStream::RegisterForAudioMixing() { @@ -2736,8 +2412,8 @@ MediaInputPort::Disconnect() return; mSource->RemoveConsumer(this); - mSource = nullptr; mDest->RemoveInput(this); + mSource = nullptr; mDest = nullptr; GraphImpl()->SetStreamOrderDirty(); @@ -2747,21 +2423,15 @@ MediaInputPort::InputInterval MediaInputPort::GetNextInputInterval(GraphTime aTime) { InputInterval result = { GRAPH_TIME_MAX, GRAPH_TIME_MAX, false }; - GraphTime t = aTime; - GraphTime end; - for (;;) { - if (!mDest->mBlocked.GetAt(t, &end)) { - break; - } - if (end >= GRAPH_TIME_MAX) { - return result; - } - t = end; + if (aTime >= mDest->mStartBlocking) { + return result; + } + result.mStart = aTime; + result.mEnd = mDest->mStartBlocking; + result.mInputIsBlocked = aTime >= mSource->mStartBlocking; + if (!result.mInputIsBlocked) { + result.mEnd = std::min(result.mEnd, mSource->mStartBlocking); } - result.mStart = t; - GraphTime sourceEnd; - result.mInputIsBlocked = mSource->mBlocked.GetAt(t, &sourceEnd); - result.mEnd = std::min(end, sourceEnd); return result; } @@ -2807,8 +2477,39 @@ MediaInputPort::SetGraphImpl(MediaStreamGraphImpl* aGraph) mGraph = aGraph; } +void +MediaInputPort::BlockTrackIdImpl(TrackID aTrackId) +{ + mBlockedTracks.AppendElement(aTrackId); +} + +void +MediaInputPort::BlockTrackId(TrackID aTrackId) +{ + class Message : public ControlMessage { + public: + explicit Message(MediaInputPort* aPort, TrackID aTrackId) + : ControlMessage(aPort->GetDestination()), + mPort(aPort), mTrackId(aTrackId) {} + virtual void Run() + { + mPort->BlockTrackIdImpl(mTrackId); + } + virtual void RunDuringShutdown() + { + Run(); + } + RefPtr mPort; + TrackID mTrackId; + }; + + MOZ_ASSERT(aTrackId != TRACK_NONE && aTrackId != TRACK_INVALID && aTrackId != TRACK_ANY, + "Only explicit TrackID is allowed"); + GraphImpl()->AppendMessage(new Message(this, aTrackId)); +} + already_AddRefed -ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, uint32_t aFlags, +ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, TrackID aTrackID, uint16_t aInputNumber, uint16_t aOutputNumber) { // This method creates two references to the MediaInputPort: one for @@ -2831,8 +2532,10 @@ ProcessedMediaStream::AllocateInputPort(MediaStream* aStream, uint32_t aFlags, } RefPtr mPort; }; - MOZ_ASSERT(aStream->GraphImpl() == GraphImpl()); - RefPtr port = new MediaInputPort(aStream, this, aFlags, + + MOZ_ASSERT(aTrackID != TRACK_NONE && aTrackID != TRACK_INVALID, + "Only TRACK_ANY and explicit ID are allowed"); + RefPtr port = new MediaInputPort(aStream, aTrackID, this, aInputNumber, aOutputNumber); port->SetGraphImpl(GraphImpl()); GraphImpl()->AppendMessage(new Message(port)); @@ -2895,8 +2598,6 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested, , mEndTime(GRAPH_TIME_MAX) , mForceShutDown(false) , mPostedRunInStableStateEvent(false) - , mFlushSourcesNow(false) - , mFlushSourcesOnNextIteration(false) , mDetectedNotRunning(false) , mPostedRunInStableState(false) , mRealtime(aDriverRequested != OFFLINE_THREAD_DRIVER) @@ -3145,19 +2846,23 @@ MediaStreamGraph::CreateTrackUnionStream(DOMMediaStream* aWrapper) } ProcessedMediaStream* -MediaStreamGraph::CreateAudioCaptureStream(DOMMediaStream* aWrapper) +MediaStreamGraph::CreateAudioCaptureStream(DOMMediaStream* aWrapper, + TrackID aTrackId) { - AudioCaptureStream* stream = new AudioCaptureStream(aWrapper); + AudioCaptureStream* stream = new AudioCaptureStream(aWrapper, aTrackId); AddStream(stream); return stream; } void -MediaStreamGraph::AddStream(MediaStream* aStream) +MediaStreamGraph::AddStream(MediaStream* aStream, uint32_t aFlags) { NS_ADDREF(aStream); MediaStreamGraphImpl* graph = static_cast(this); aStream->SetGraphImpl(graph); + if (aFlags & ADD_STREAM_SUSPENDED) { + aStream->IncrementSuspendCount(); + } graph->AppendMessage(new CreateMessage(aStream)); } @@ -3220,75 +2925,46 @@ MediaStreamGraph::NotifyWhenGraphStarted(AudioNodeStream* aStream) } void -MediaStreamGraphImpl::ResetVisitedStreamState() +MediaStreamGraphImpl::IncrementSuspendCount(MediaStream* aStream) { - // Reset the visited/consumed/blocked state of the streams. - nsTArray* runningAndSuspendedPair[2]; - runningAndSuspendedPair[0] = &mStreams; - runningAndSuspendedPair[1] = &mSuspendedStreams; + if (!aStream->IsSuspended()) { + MOZ_ASSERT(mStreams.Contains(aStream)); + mStreams.RemoveElement(aStream); + mSuspendedStreams.AppendElement(aStream); + SetStreamOrderDirty(); + } + aStream->IncrementSuspendCount(); +} - for (uint32_t array = 0; array < 2; array++) { - for (uint32_t i = 0; i < runningAndSuspendedPair[array]->Length(); ++i) { - ProcessedMediaStream* ps = - (*runningAndSuspendedPair[array])[i]->AsProcessedStream(); - if (ps) { - ps->mCycleMarker = NOT_VISITED; - ps->mIsConsumed = false; - ps->mInBlockingSet = false; - } +void +MediaStreamGraphImpl::DecrementSuspendCount(MediaStream* aStream) +{ + bool wasSuspended = aStream->IsSuspended(); + aStream->DecrementSuspendCount(); + if (wasSuspended && !aStream->IsSuspended()) { + MOZ_ASSERT(mSuspendedStreams.Contains(aStream)); + mSuspendedStreams.RemoveElement(aStream); + mStreams.AppendElement(aStream); + ProcessedMediaStream* ps = aStream->AsProcessedStream(); + if (ps) { + ps->mCycleMarker = NOT_VISITED; } + SetStreamOrderDirty(); } } void -MediaStreamGraphImpl::StreamSetForAudioContext(dom::AudioContext::AudioContextId aAudioContextId, - mozilla::LinkedList& aStreamSet) -{ - nsTArray* runningAndSuspendedPair[2]; - runningAndSuspendedPair[0] = &mStreams; - runningAndSuspendedPair[1] = &mSuspendedStreams; - - for (uint32_t array = 0; array < 2; array++) { - for (uint32_t i = 0; i < runningAndSuspendedPair[array]->Length(); ++i) { - MediaStream* stream = (*runningAndSuspendedPair[array])[i]; - if (aAudioContextId == stream->AudioContextId()) { - aStreamSet.insertFront(stream); - } - } - } -} - -void -MediaStreamGraphImpl::MoveStreams(AudioContextOperation aAudioContextOperation, - mozilla::LinkedList& aStreamSet) +MediaStreamGraphImpl::SuspendOrResumeStreams(AudioContextOperation aAudioContextOperation, + const nsTArray& aStreamSet) { // For our purpose, Suspend and Close are equivalent: we want to remove the // streams from the set of streams that are going to be processed. - nsTArray& from = - aAudioContextOperation == AudioContextOperation::Resume ? mSuspendedStreams - : mStreams; - nsTArray& to = - aAudioContextOperation == AudioContextOperation::Resume ? mStreams - : mSuspendedStreams; - - MediaStream* stream; - while ((stream = aStreamSet.getFirst())) { - // It is posible to not find the stream here, if there has been two - // suspend/resume/close calls in a row. - auto i = from.IndexOf(stream); - if (i != from.NoIndex) { - from.RemoveElementAt(i); - to.AppendElement(stream); + for (MediaStream* stream : aStreamSet) { + if (aAudioContextOperation == AudioContextOperation::Resume) { + DecrementSuspendCount(stream); + } else { + IncrementSuspendCount(stream); } - - // If streams got added during a period where an AudioContext was suspended, - // set their buffer start time to the appropriate value now: - if (aAudioContextOperation == AudioContextOperation::Resume && - stream->mBufferStartTime == START_TIME_DELAYED) { - stream->mBufferStartTime = mProcessedTime; - } - - stream->remove(); } STREAM_LOG(LogLevel::Debug, ("Moving streams between suspended and running" "state: mStreams: %d, mSuspendedStreams: %d\n", mStreams.Length(), @@ -3317,9 +2993,15 @@ MediaStreamGraphImpl::AudioContextOperationCompleted(MediaStream* aStream, AudioContextState state; switch (aOperation) { - case Suspend: state = AudioContextState::Suspended; break; - case Resume: state = AudioContextState::Running; break; - case Close: state = AudioContextState::Closed; break; + case AudioContextOperation::Suspend: + state = AudioContextState::Suspended; + break; + case AudioContextOperation::Resume: + state = AudioContextState::Running; + break; + case AudioContextOperation::Close: + state = AudioContextState::Closed; + break; default: MOZ_CRASH("Not handled."); } @@ -3329,22 +3011,13 @@ MediaStreamGraphImpl::AudioContextOperationCompleted(MediaStream* aStream, } void -MediaStreamGraphImpl::ApplyAudioContextOperationImpl(AudioNodeStream* aStream, - AudioContextOperation aOperation, - void* aPromise) +MediaStreamGraphImpl::ApplyAudioContextOperationImpl( + MediaStream* aDestinationStream, const nsTArray& aStreams, + AudioContextOperation aOperation, void* aPromise) { MOZ_ASSERT(CurrentDriver()->OnThread()); - mozilla::LinkedList streamSet; - SetStreamOrderDirty(); - - ResetVisitedStreamState(); - - StreamSetForAudioContext(aStream->AudioContextId(), streamSet); - - MoveStreams(aOperation, streamSet); - MOZ_ASSERT(!streamSet.getFirst(), - "Streams should be removed from the list after having been moved."); + SuspendOrResumeStreams(aOperation, aStreams); // If we have suspended the last AudioContext, and we don't have other // streams that have audio, this graph will automatically switch to a @@ -3363,11 +3036,12 @@ MediaStreamGraphImpl::ApplyAudioContextOperationImpl(AudioNodeStream* aStream, mMixer.AddCallback(driver); CurrentDriver()->SwitchAtNextIteration(driver); } - driver->EnqueueStreamAndPromiseForOperation(aStream, aPromise, aOperation); + driver->EnqueueStreamAndPromiseForOperation(aDestinationStream, + aPromise, aOperation); } else { // We are resuming a context, but we are already using an // AudioCallbackDriver, we can resolve the promise now. - AudioContextOperationCompleted(aStream, aPromise, aOperation); + AudioContextOperationCompleted(aDestinationStream, aPromise, aOperation); } } // Close, suspend: check if we are going to switch to a @@ -3389,7 +3063,8 @@ MediaStreamGraphImpl::ApplyAudioContextOperationImpl(AudioNodeStream* aStream, } if (!audioTrackPresent && CurrentDriver()->AsAudioCallbackDriver()) { CurrentDriver()->AsAudioCallbackDriver()-> - EnqueueStreamAndPromiseForOperation(aStream, aPromise, aOperation); + EnqueueStreamAndPromiseForOperation(aDestinationStream, aPromise, + aOperation); SystemClockDriver* driver; if (CurrentDriver()->NextDriver()) { @@ -3402,31 +3077,34 @@ MediaStreamGraphImpl::ApplyAudioContextOperationImpl(AudioNodeStream* aStream, } else { // We are closing or suspending an AudioContext, but something else is // using the audio stream, we can resolve the promise now. - AudioContextOperationCompleted(aStream, aPromise, aOperation); + AudioContextOperationCompleted(aDestinationStream, aPromise, aOperation); } } } void -MediaStreamGraph::ApplyAudioContextOperation(AudioNodeStream* aNodeStream, +MediaStreamGraph::ApplyAudioContextOperation(MediaStream* aDestinationStream, + const nsTArray& aStreams, AudioContextOperation aOperation, void* aPromise) { class AudioContextOperationControlMessage : public ControlMessage { public: - AudioContextOperationControlMessage(AudioNodeStream* aStream, + AudioContextOperationControlMessage(MediaStream* aDestinationStream, + const nsTArray& aStreams, AudioContextOperation aOperation, void* aPromise) - : ControlMessage(aStream) + : ControlMessage(aDestinationStream) + , mStreams(aStreams) , mAudioContextOperation(aOperation) , mPromise(aPromise) { } virtual void Run() { - mStream->GraphImpl()->ApplyAudioContextOperationImpl( - mStream->AsAudioNodeStream(), mAudioContextOperation, mPromise); + mStream->GraphImpl()->ApplyAudioContextOperationImpl(mStream, + mStreams, mAudioContextOperation, mPromise); } virtual void RunDuringShutdown() { @@ -3434,13 +3112,17 @@ MediaStreamGraph::ApplyAudioContextOperation(AudioNodeStream* aNodeStream, } private: + // We don't need strong references here for the same reason ControlMessage + // doesn't. + nsTArray mStreams; AudioContextOperation mAudioContextOperation; void* mPromise; }; MediaStreamGraphImpl* graphImpl = static_cast(this); graphImpl->AppendMessage( - new AudioContextOperationControlMessage(aNodeStream, aOperation, aPromise)); + new AudioContextOperationControlMessage(aDestinationStream, aStreams, + aOperation, aPromise)); } bool @@ -3532,7 +3214,7 @@ MediaStreamGraphImpl::ConnectToCaptureStream(uint64_t aWindowId, for (uint32_t i = 0; i < mWindowCaptureStreams.Length(); i++) { if (mWindowCaptureStreams[i].mWindowId == aWindowId) { ProcessedMediaStream* sink = mWindowCaptureStreams[i].mCaptureStreamSink; - return sink->AllocateInputPort(aMediaStream, 0); + return sink->AllocateInputPort(aMediaStream); } } return nullptr; diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 39419843a7..c40d251eb5 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -24,7 +24,6 @@ #include "nsAutoRef.h" #include #include "DOMMediaStream.h" -#include "AudioContext.h" class nsIRunnable; @@ -39,6 +38,10 @@ namespace mozilla { extern PRLogModuleInfo* gMediaStreamGraphLog; +namespace dom { + enum class AudioContextOperation; +} + /* * MediaStreamGraph is a framework for synchronized audio/video processing * and playback. It is designed to be used by other browser components such as @@ -102,18 +105,6 @@ protected: public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStreamListener) - enum Consumption { - CONSUMED, - NOT_CONSUMED - }; - - /** - * Notify that the stream is hooked up and we'd like to start or stop receiving - * data on it. Only fires on SourceMediaStreams. - * The initial state is assumed to be NOT_CONSUMED. - */ - virtual void NotifyConsumptionChanged(MediaStreamGraph* aGraph, Consumption aConsuming) {} - /** * When a SourceMediaStream has pulling enabled, and the MediaStreamGraph * control loop is ready to pull, this gets called. A NotifyPull implementation @@ -174,11 +165,16 @@ public: * aTrackEvents can be any combination of TRACK_EVENT_CREATED and * TRACK_EVENT_ENDED. aQueuedMedia is the data being added to the track * at aTrackOffset (relative to the start of the stream). + * aInputStream and aInputTrackID will be set if the changes originated + * from an input stream's track. In practice they will only be used for + * ProcessedMediaStreams. */ virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) {} + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream = nullptr, + TrackID aInputTrackID = TRACK_INVALID) {} /** * Notify that all new tracks this iteration have been created. @@ -319,7 +315,6 @@ public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream) explicit MediaStream(DOMMediaStream* aWrapper); - virtual dom::AudioContext::AudioContextId AudioContextId() const { return 0; } protected: // Protected destructor, to discourage deletion outside of Release(): @@ -363,11 +358,13 @@ public: // Only the first enabled video track is played. virtual void AddVideoOutput(VideoFrameContainer* aContainer); virtual void RemoveVideoOutput(VideoFrameContainer* aContainer); - // Explicitly block. Useful for example if a media element is pausing - // and we need to stop its stream emitting its buffered data. - virtual void ChangeExplicitBlockerCount(int32_t aDelta); - void BlockStreamIfNeeded(); - void UnblockStreamIfNeeded(); + // Explicitly suspend. Useful for example if a media element is pausing + // and we need to stop its stream emitting its buffered data. As soon as the + // Suspend message reaches the graph, the stream stops processing. It + // ignores its inputs and produces silence/no video until Resumed. Its + // current time does not advance. + virtual void Suspend(); + virtual void Resume(); // Events will be dispatched by calling methods of aListener. virtual void AddListener(MediaStreamListener* aListener); virtual void RemoveListener(MediaStreamListener* aListener); @@ -466,40 +463,10 @@ public: aContainer->ClearFutureFrames(); mVideoOutputs.RemoveElement(aContainer); } - void ChangeExplicitBlockerCountImpl(GraphTime aTime, int32_t aDelta) - { - mExplicitBlockerCount.SetAtAndAfter(aTime, mExplicitBlockerCount.GetAt(aTime) + aDelta); - } - void BlockStreamIfNeededImpl(GraphTime aTime) - { - bool blocked = mExplicitBlockerCount.GetAt(aTime) > 0; - if (blocked) { - return; - } - ChangeExplicitBlockerCountImpl(aTime, 1); - } - void UnblockStreamIfNeededImpl(GraphTime aTime) - { - bool blocked = mExplicitBlockerCount.GetAt(aTime) > 0; - if (!blocked) { - return; - } - ChangeExplicitBlockerCountImpl(aTime, -1); - } void AddListenerImpl(already_AddRefed aListener); void RemoveListenerImpl(MediaStreamListener* aListener); void RemoveAllListenersImpl(); virtual void SetTrackEnabledImpl(TrackID aTrackID, bool aEnabled); - /** - * Returns true when this stream requires the contents of its inputs even if - * its own outputs are not being consumed. This is used to signal inputs to - * this stream that they are being consumed; when they're not being consumed, - * we make some optimizations. - */ - virtual bool IsIntrinsicallyConsumed() const - { - return !mAudioOutputs.IsEmpty() || !mVideoOutputs.IsEmpty(); - } void AddConsumer(MediaInputPort* aPort) { @@ -513,7 +480,7 @@ public: { return mConsumers.Length(); } - const StreamBuffer& GetStreamBuffer() { return mBuffer; } + StreamBuffer& GetStreamBuffer() { return mBuffer; } GraphTime GetStreamBufferStartTime() { return mBufferStartTime; } double StreamTimeToSeconds(StreamTime aTime) @@ -526,6 +493,12 @@ public: NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time"); return (aTime*1000000)/mBuffer.GraphRate(); } + StreamTime SecondsToNearestStreamTime(double aSeconds) + { + NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX/TRACK_RATE_MAX, + "Bad seconds"); + return mBuffer.GraphRate() * aSeconds + 0.5; + } StreamTime MicrosecondsToStreamTimeRoundDown(int64_t aMicroseconds) { return (aMicroseconds*mBuffer.GraphRate())/1000000; } @@ -543,19 +516,28 @@ public: * to ensure we know exactly how much time this stream will be blocked during * the interval. */ + StreamTime GraphTimeToStreamTimeWithBlocking(GraphTime aTime); + /** + * Convert graph time to stream time. This assumes there is no blocking time + * to take account of, which is always true except between a stream + * having its blocking time calculated in UpdateGraph and its blocking time + * taken account of in UpdateCurrentTimeForStreams. + */ StreamTime GraphTimeToStreamTime(GraphTime aTime); /** - * Convert graph time to stream time. aTime can be > mStateComputedTime, - * in which case we optimistically assume the stream will not be blocked - * after mStateComputedTime. + * Convert stream time to graph time. This assumes there is no blocking time + * to take account of, which is always true except between a stream + * having its blocking time calculated in UpdateGraph and its blocking time + * taken account of in UpdateCurrentTimeForStreams. */ - StreamTime GraphTimeToStreamTimeOptimistic(GraphTime aTime); + GraphTime StreamTimeToGraphTime(StreamTime aTime); /** * Convert stream time to graph time. The result can be > mStateComputedTime, * in which case we did the conversion optimistically assuming the stream * will not be blocked after mStateComputedTime. */ - GraphTime StreamTimeToGraphTime(StreamTime aTime); + GraphTime StreamTimeToGraphTimeWithBlocking(StreamTime aTime); + bool IsFinishedOnGraphThread() { return mFinished; } void FinishOnGraphThread(); @@ -583,12 +565,18 @@ public: void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; } dom::AudioChannel AudioChannelType() const { return mAudioChannelType; } + bool IsSuspended() { return mSuspendedCount > 0; } + void IncrementSuspendCount() { ++mSuspendedCount; } + void DecrementSuspendCount() + { + NS_ASSERTION(mSuspendedCount > 0, "Suspend count underrun"); + --mSuspendedCount; + } + protected: void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime, GraphTime aBlockedTime) { mBufferStartTime += aBlockedTime; - mExplicitBlockerCount.AdvanceCurrentTime(aCurrentTime); - mBuffer.ForgetUpTo(aCurrentTime - mBufferStartTime); } @@ -636,20 +624,15 @@ protected: // We record the last played video frame to avoid playing the frame again // with a different frame id. VideoFrame mLastPlayedVideoFrame; - // The number of times this stream has been explicitly blocked by the control - // API, minus the number of times it has been explicitly unblocked. - TimeVarying mExplicitBlockerCount; nsTArray > mListeners; nsTArray mMainThreadListeners; nsTArray mDisabledTrackIDs; - // Precomputed blocking status (over GraphTime). - // This is only valid between the graph's mCurrentTime and - // mStateComputedTime. The stream is considered to have - // not been blocked before mCurrentTime (its mBufferStartTime is increased - // as necessary to account for that time instead) --- this avoids us having to - // record the entire history of the stream's blocking-ness in mBlocked. - TimeVarying mBlocked; + // GraphTime at which this stream starts blocking. + // This is only valid up to mStateComputedTime. The stream is considered to + // have not been blocked before mCurrentTime (its mBufferStartTime is increased + // as necessary to account for that time instead). + GraphTime mStartBlocking; // MediaInputPorts to which this is connected nsTArray mConsumers; @@ -670,6 +653,12 @@ protected: }; nsTArray mAudioOutputStreams; + /** + * Number of outstanding suspend operations on this stream. Stream is + * suspended when this is > 0. + */ + int32_t mSuspendedCount; + /** * When true, this means the stream will be finished once all * buffered data has been consumed. @@ -697,16 +686,6 @@ protected: */ bool mNotifiedHasCurrentData; - // True if the stream is being consumed (i.e. has track data being played, - // or is feeding into some stream that is being consumed). - bool mIsConsumed; - // Temporary data for computing blocking status of streams - // True if we've added this stream to the set of streams we're computing - // blocking for. - bool mInBlockingSet; - // True if this stream should be blocked in this phase. - bool mBlockInThisPhase; - // This state is only used on the main thread. DOMMediaStream* mWrapper; // Main-thread views of state @@ -732,7 +711,6 @@ class SourceMediaStream : public MediaStream public: explicit SourceMediaStream(DOMMediaStream* aWrapper) : MediaStream(aWrapper), - mLastConsumptionState(MediaStreamListener::NOT_CONSUMED), mMutex("mozilla::media::SourceMediaStream"), mUpdateKnownTracksTime(0), mPullEnabled(false), @@ -858,17 +836,6 @@ public: */ void EndAllTrackAndFinish(); - /** - * Note: Only call from Media Graph thread (eg NotifyPull) - * - * Returns amount of time (data) that is currently buffered in the track, - * assuming playout via PlayAudio or via a TrackUnion - note that - * NotifyQueuedTrackChanges() on a SourceMediaStream will occur without - * any "extra" buffering, but NotifyQueued TrackChanges() on a TrackUnion - * will be buffered. - */ - StreamTime GetBufferedTicks(TrackID aID); - void RegisterForAudioMixing(); // XXX need a Reset API @@ -940,9 +907,6 @@ protected: void NotifyDirectConsumers(TrackData *aTrack, MediaSegment *aSegment); - // Media stream graph thread only - MediaStreamListener::Consumption mLastConsumptionState; - // This must be acquired *before* MediaStreamGraphImpl's lock, if they are // held together. Mutex mMutex; @@ -962,6 +926,10 @@ protected: * We make these refcounted so that stream-related messages with MediaInputPort* * pointers can be sent to the main thread safely. * + * A port can be locked to a specific track in the source stream, in which case + * only this track will be forwarded to the destination stream. TRACK_ANY + * can used to signal that all tracks shall be forwarded. + * * When a port's source or destination stream dies, the stream's DestroyImpl * calls MediaInputPort::Disconnect to disconnect the port from * the source and destination streams. @@ -977,12 +945,12 @@ class MediaInputPort final { private: // Do not call this constructor directly. Instead call aDest->AllocateInputPort. - MediaInputPort(MediaStream* aSource, ProcessedMediaStream* aDest, - uint32_t aFlags, uint16_t aInputNumber, - uint16_t aOutputNumber) + MediaInputPort(MediaStream* aSource, TrackID& aSourceTrack, + ProcessedMediaStream* aDest, + uint16_t aInputNumber, uint16_t aOutputNumber) : mSource(aSource) + , mSourceTrack(aSourceTrack) , mDest(aDest) - , mFlags(aFlags) , mInputNumber(aInputNumber) , mOutputNumber(aOutputNumber) , mGraph(nullptr) @@ -999,20 +967,6 @@ private: public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort) - /** - * The FLAG_BLOCK_INPUT and FLAG_BLOCK_OUTPUT flags can be used to control - * exactly how the blocking statuses of the input and output streams affect - * each other. - */ - enum { - // When set, blocking on the output stream forces blocking on the input - // stream. - FLAG_BLOCK_INPUT = 0x01, - // When set, blocking on the input stream forces blocking on the output - // stream. - FLAG_BLOCK_OUTPUT = 0x02 - }; - // Called on graph manager thread // Do not call these from outside MediaStreamGraph.cpp! void Init(); @@ -1028,8 +982,22 @@ public: // Any thread MediaStream* GetSource() { return mSource; } + TrackID GetSourceTrackId() { return mSourceTrack; } ProcessedMediaStream* GetDestination() { return mDest; } + // Block aTrackId in the port. Consumers will interpret this track as ended. + void BlockTrackId(TrackID aTrackId); +private: + void BlockTrackIdImpl(TrackID aTrackId); + +public: + // Returns true if aTrackId has not been blocked and this port has not + // been locked to another track. + bool PassTrackThrough(TrackID aTrackId) { + return !mBlockedTracks.Contains(aTrackId) && + (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId); + } + uint16_t InputNumber() const { return mInputNumber; } uint16_t OutputNumber() const { return mOutputNumber; } @@ -1075,12 +1043,13 @@ private: friend class ProcessedMediaStream; // Never modified after Init() MediaStream* mSource; + TrackID mSourceTrack; ProcessedMediaStream* mDest; - uint32_t mFlags; // The input and output numbers are optional, and are currently only used by // Web Audio. const uint16_t mInputNumber; const uint16_t mOutputNumber; + nsTArray mBlockedTracks; // Our media stream graph MediaStreamGraphImpl* mGraph; @@ -1102,9 +1071,13 @@ public: /** * Allocates a new input port attached to source aStream. * This stream can be removed by calling MediaInputPort::Remove(). + * The input port is tied to aTrackID in the source stream. + * aTrackID can be set to TRACK_ANY to automatically forward all tracks from + * aStream. To end a track in the destination stream forwarded with TRACK_ANY, + * it can be blocked in the input port through MediaInputPort::BlockTrackId(). */ already_AddRefed AllocateInputPort(MediaStream* aStream, - uint32_t aFlags = 0, + TrackID aTrackID = TRACK_ANY, uint16_t aInputNumber = 0, uint16_t aOutputNumber = 0); /** @@ -1251,27 +1224,34 @@ public: /** * Create a stream that will mix all its audio input. */ - ProcessedMediaStream* CreateAudioCaptureStream(DOMMediaStream* aWrapper); + ProcessedMediaStream* CreateAudioCaptureStream(DOMMediaStream* aWrapper, + TrackID aTrackId); + enum { + ADD_STREAM_SUSPENDED = 0x01 + }; /** * Add a new stream to the graph. Main thread. */ - void AddStream(MediaStream* aStream); + void AddStream(MediaStream* aStream, uint32_t aFlags = 0); /* From the main thread, ask the MSG to send back an event when the graph * thread is running, and audio is being processed. */ void NotifyWhenGraphStarted(AudioNodeStream* aNodeStream); /* From the main thread, suspend, resume or close an AudioContext. - * aNodeStream is the stream of the DestinationNode of the AudioContext. + * aStreams are the streams of all the AudioNodes of the AudioContext that + * need to be suspended or resumed. This can be empty if this is a second + * consecutive suspend call and all the nodes are already suspended. * * This can possibly pause the graph thread, releasing system resources, if * all streams have been suspended/closed. * * When the operation is complete, aPromise is resolved. */ - void ApplyAudioContextOperation(AudioNodeStream* aNodeStream, + void ApplyAudioContextOperation(MediaStream* aDestinationStream, + const nsTArray& aStreams, dom::AudioContextOperation aState, - void * aPromise); + void* aPromise); bool IsNonRealtime() const; /** diff --git a/dom/media/MediaStreamGraphImpl.h b/dom/media/MediaStreamGraphImpl.h index ffd5cd47a7..1768acd8fd 100644 --- a/dom/media/MediaStreamGraphImpl.h +++ b/dom/media/MediaStreamGraphImpl.h @@ -166,12 +166,17 @@ public: } #endif } - /* - * This does the actual iteration: Message processing, MediaStream ordering, - * blocking computation and processing. - */ - void DoIteration(); + void MaybeProduceMemoryReport(); + + /** + * Returns true if this MediaStreamGraph should keep running + */ + bool UpdateMainThreadState(); + + /** + * Returns true if this MediaStreamGraph should keep running + */ bool OneIteration(GraphTime aStateEnd); bool Running() const @@ -215,10 +220,9 @@ public: bool ShouldUpdateMainThread(); // The following methods are the various stages of RunThread processing. /** - * Advance all stream state to the new current time. + * Advance all stream state to mStateComputedTime. */ - void UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime, - GraphTime aNextCurrentTime); + void UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime); /** * Process graph message for this iteration, update stream processing order, * and recompute stream blocking until aEndBlockingDecisions. @@ -231,9 +235,10 @@ public: mFrontMessageQueue.SwapElements(mBackMessageQueue); } /** - * Do all the processing and play the audio and video, ffrom aFrom to aTo. + * Do all the processing and play the audio and video, from + * mProcessedTime to mStateComputedTime. */ - void Process(GraphTime aFrom, GraphTime aTo); + void Process(); /** * Update the consumption state of aStream to reflect whether its data * is needed or not. @@ -249,17 +254,6 @@ public: * Update "have enough data" flags in aStream. */ void UpdateBufferSufficiencyState(SourceMediaStream* aStream); - /** - * Mark aStream and all its inputs (recursively) as consumed. - */ - static void MarkConsumed(MediaStream* aStream); - - /** - * Given the Id of an AudioContext, return the set of all MediaStreams that - * are part of this context. - */ - void StreamSetForAudioContext(dom::AudioContext::AudioContextId aAudioContextId, - mozilla::LinkedList& aStreamSet); /** * Called when a suspend/resume/close operation has been completed, on the @@ -273,28 +267,28 @@ public: * Apply and AudioContext operation (suspend/resume/closed), on the graph * thread. */ - void ApplyAudioContextOperationImpl(AudioNodeStream* aStream, + void ApplyAudioContextOperationImpl(MediaStream* aDestinationStream, + const nsTArray& aStreams, dom::AudioContextOperation aOperation, void* aPromise); + /** + * Increment suspend count on aStream and move it to mSuspendedStreams if + * necessary. + */ + void IncrementSuspendCount(MediaStream* aStream); + /** + * Increment suspend count on aStream and move it to mStreams if + * necessary. + */ + void DecrementSuspendCount(MediaStream* aStream); + /* * Move streams from the mStreams to mSuspendedStream if suspending/closing an * AudioContext, or the inverse when resuming an AudioContext. */ - void MoveStreams(dom::AudioContextOperation aAudioContextOperation, - mozilla::LinkedList& aStreamSet); - - /* - * Reset some state about the streams before suspending them, or resuming - * them. - */ - void ResetVisitedStreamState(); - - /* - * True if a stream is suspended, that is, is not in mStreams, but in - * mSuspendedStream. - */ - bool StreamSuspended(MediaStream* aStream); + void SuspendOrResumeStreams(dom::AudioContextOperation aAudioContextOperation, + const nsTArray& aStreamSet); /** * Sort mStreams so that every stream not in a cycle is after any streams @@ -302,72 +296,32 @@ public: * Also sets mIsConsumed on every stream. */ void UpdateStreamOrder(); - /** - * Compute the blocking states of streams from mStateComputedTime - * until the desired future time aEndBlockingDecisions. - * Updates mStateComputedTime and sets MediaStream::mBlocked - * for all streams. - */ - void RecomputeBlocking(GraphTime aEndBlockingDecisions); - // The following methods are used to help RecomputeBlocking. - /** - * If aStream isn't already in aStreams, add it and recursively call - * AddBlockingRelatedStreamsToSet on all the streams whose blocking - * status could depend on or affect the state of aStream. - */ - void AddBlockingRelatedStreamsToSet(nsTArray* aStreams, - MediaStream* aStream); - /** - * Mark a stream blocked at time aTime. If this results in decisions that need - * to be revisited at some point in the future, *aEnd will be reduced to the - * first time in the future to recompute those decisions. - */ - void MarkStreamBlocking(MediaStream* aStream); - /** - * Recompute blocking for the streams in aStreams for the interval starting at aTime. - * If this results in decisions that need to be revisited at some point - * in the future, *aEnd will be reduced to the first time in the future to - * recompute those decisions. - */ - void RecomputeBlockingAt(const nsTArray& aStreams, - GraphTime aTime, GraphTime aEndBlockingDecisions, - GraphTime* aEnd); /** * Returns smallest value of t such that t is a multiple of * WEBAUDIO_BLOCK_SIZE and t > aTime. */ GraphTime RoundUpToNextAudioBlock(GraphTime aTime); /** - * Produce data for all streams >= aStreamIndex for the given time interval. + * Produce data for all streams >= aStreamIndex for the current time interval. * Advances block by block, each iteration producing data for all streams * for a single block. * This is called whenever we have an AudioNodeStream in the graph. */ void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex, - TrackRate aSampleRate, - GraphTime aFrom, - GraphTime aTo); + TrackRate aSampleRate); /** - * Returns true if aStream will underrun at aTime for its own playback. - * aEndBlockingDecisions is when we plan to stop making blocking decisions. - * *aEnd will be reduced to the first time in the future to recompute these - * decisions. + * If aStream will underrun between aTime, and aEndBlockingDecisions, returns + * the time at which the underrun will start. Otherwise return + * aEndBlockingDecisions. */ - bool WillUnderrun(MediaStream* aStream, GraphTime aTime, - GraphTime aEndBlockingDecisions, GraphTime* aEnd); + GraphTime WillUnderrun(MediaStream* aStream, GraphTime aEndBlockingDecisions); /** * Given a graph time aTime, convert it to a stream time taking into * account the time during which aStream is scheduled to be blocked. */ - StreamTime GraphTimeToStreamTime(MediaStream* aStream, GraphTime aTime); - /** - * Given a graph time aTime, convert it to a stream time taking into - * account the time during which aStream is scheduled to be blocked, and - * when we don't know whether it's blocked or not, we assume it's not blocked. - */ - StreamTime GraphTimeToStreamTimeOptimistic(MediaStream* aStream, GraphTime aTime); + StreamTime GraphTimeToStreamTimeWithBlocking(MediaStream* aStream, GraphTime aTime); enum { INCLUDE_TRAILING_BLOCKED_INTERVAL = 0x01 @@ -382,7 +336,7 @@ public: * interval is included in the time returned if and only if * aFlags includes INCLUDE_TRAILING_BLOCKED_INTERVAL. */ - GraphTime StreamTimeToGraphTime(MediaStream* aStream, StreamTime aTime, + GraphTime StreamTimeToGraphTimeWithBlocking(MediaStream* aStream, StreamTime aTime, uint32_t aFlags = 0); /** * Call NotifyHaveCurrentData on aStream's listeners. @@ -392,12 +346,12 @@ public: * If aStream needs an audio stream but doesn't have one, create it. * If aStream doesn't need an audio stream but has one, destroy it. */ - void CreateOrDestroyAudioStreams(GraphTime aAudioOutputStartTime, MediaStream* aStream); + void CreateOrDestroyAudioStreams(MediaStream* aStream); /** * Queue audio (mix of stream audio and silence for blocked intervals) * to the audio output stream. Returns the number of frames played. */ - StreamTime PlayAudio(MediaStream* aStream, GraphTime aFrom, GraphTime aTo); + StreamTime PlayAudio(MediaStream* aStream); /** * Set the correct current video frame for stream aStream. */ @@ -527,6 +481,56 @@ public: already_AddRefed ConnectToCaptureStream(uint64_t aWindowId, MediaStream* aMediaStream); + class StreamSet { + public: + class iterator { + public: + explicit iterator(MediaStreamGraphImpl& aGraph) + : mGraph(&aGraph), mArrayNum(-1), mArrayIndex(0) + { + ++(*this); + } + iterator() : mGraph(nullptr), mArrayNum(2), mArrayIndex(0) {} + MediaStream* operator*() + { + return Array()->ElementAt(mArrayIndex); + } + iterator operator++() + { + ++mArrayIndex; + while (mArrayNum < 2 && + (mArrayNum < 0 || mArrayIndex >= Array()->Length())) { + ++mArrayNum; + mArrayIndex = 0; + } + return *this; + } + bool operator==(const iterator& aOther) const + { + return mArrayNum == aOther.mArrayNum && mArrayIndex == aOther.mArrayIndex; + } + bool operator!=(const iterator& aOther) const + { + return !(*this == aOther); + } + private: + nsTArray* Array() + { + return mArrayNum == 0 ? &mGraph->mStreams : &mGraph->mSuspendedStreams; + } + MediaStreamGraphImpl* mGraph; + int mArrayNum; + uint32_t mArrayIndex; + }; + + explicit StreamSet(MediaStreamGraphImpl& aGraph) : mGraph(aGraph) {} + iterator begin() { return iterator(mGraph); } + iterator end() { return iterator(); } + private: + MediaStreamGraphImpl& mGraph; + }; + StreamSet AllStreams() { return StreamSet(*this); } + // Data members // /** @@ -554,6 +558,10 @@ public: * and are therefore not doing any processing. */ nsTArray mSuspendedStreams; + /** + * Suspended AudioContext IDs + */ + nsTHashtable mSuspendedContexts; /** * Streams from mFirstCycleBreaker to the end of mStreams produce output * before they receive input. They correspond to DelayNodes that are in @@ -679,13 +687,6 @@ public: */ bool mPostedRunInStableStateEvent; - /** - * Used to flush any accumulated data when the output streams - * may have stalled (on Mac after an output device change) - */ - bool mFlushSourcesNow; - bool mFlushSourcesOnNextIteration; - // Main thread only /** @@ -735,10 +736,6 @@ public: private: virtual ~MediaStreamGraphImpl(); - void StreamNotifyOutput(MediaStream* aStream); - - void StreamReadyToFinish(MediaStream* aStream); - MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf) /** diff --git a/dom/media/SharedBuffer.h b/dom/media/SharedBuffer.h index 98670906c9..d7a0320fb0 100644 --- a/dom/media/SharedBuffer.h +++ b/dom/media/SharedBuffer.h @@ -13,6 +13,8 @@ namespace mozilla { +class AudioBlockBuffer; + /** * Base class for objects with a thread-safe refcount and a virtual * destructor. @@ -23,6 +25,8 @@ public: bool IsShared() { return mRefCnt.get() > 1; } + virtual AudioBlockBuffer* AsAudioBlockBuffer() { return nullptr; }; + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { return 0; @@ -41,9 +45,9 @@ protected: * Heap-allocated chunk of arbitrary data with threadsafe refcounting. * Typically you would allocate one of these, fill it in, and then treat it as * immutable while it's shared. - * This only guarantees 4-byte alignment of the data. For alignment we - * simply assume that the refcount is at least 4-byte aligned and its size - * is divisible by 4. + * This only guarantees 4-byte alignment of the data. For alignment we simply + * assume that the memory from malloc is at least 4-byte aligned and the + * refcount's size is large enough that SharedBuffer's size is divisible by 4. */ class SharedBuffer : public ThreadSharedObject { public: diff --git a/dom/media/StreamBuffer.cpp b/dom/media/StreamBuffer.cpp index 668463c5ba..3f225c2793 100644 --- a/dom/media/StreamBuffer.cpp +++ b/dom/media/StreamBuffer.cpp @@ -63,14 +63,26 @@ StreamBuffer::GetAllTracksEnd() const StreamBuffer::Track* StreamBuffer::FindTrack(TrackID aID) { - if (aID == TRACK_NONE) + if (aID == TRACK_NONE || mTracks.IsEmpty()) { return nullptr; - for (uint32_t i = 0; i < mTracks.Length(); ++i) { - Track* track = mTracks[i]; - if (track->GetID() == aID) { - return track; + } + + // The tracks are sorted by ID. We can use a binary search. + + uint32_t left = 0, right = mTracks.Length() - 1; + while (left <= right) { + uint32_t middle = (left + right) / 2; + if (mTracks[middle]->GetID() == aID) { + return mTracks[middle]; + } + + if (mTracks[middle]->GetID() > aID) { + right = middle - 1; + } else { + left = middle + 1; } } + return nullptr; } @@ -89,6 +101,7 @@ StreamBuffer::ForgetUpTo(StreamTime aTime) Track* track = mTracks[i]; if (track->IsEnded() && track->GetEnd() <= aTime) { mTracks.RemoveElementAt(i); + mTracksDirty = true; --i; continue; } diff --git a/dom/media/StreamBuffer.h b/dom/media/StreamBuffer.h index 79af493911..e8354f5a0b 100644 --- a/dom/media/StreamBuffer.h +++ b/dom/media/StreamBuffer.h @@ -19,6 +19,7 @@ namespace mozilla { typedef int32_t TrackID; const TrackID TRACK_NONE = 0; const TrackID TRACK_INVALID = -1; +const TrackID TRACK_ANY = -2; inline TrackTicks RateConvertTicksRoundDown(TrackRate aOutRate, TrackRate aInRate, @@ -50,7 +51,8 @@ inline TrackTicks RateConvertTicksRoundUp(TrackRate aOutRate, * the data for each track is a MediaSegment. The set of tracks can vary * over the timeline of the StreamBuffer. */ -class StreamBuffer { +class StreamBuffer +{ public: /** * Every track has a start time --- when it started in the StreamBuffer. @@ -63,7 +65,8 @@ public: * TODO Add TimeVarying mEnabled. * Takes ownership of aSegment. */ - class Track { + class Track final + { Track(TrackID aID, StreamTime aStart, MediaSegment* aSegment) : mStart(aStart), mSegment(aSegment), @@ -75,11 +78,13 @@ public: NS_ASSERTION(aID > TRACK_NONE, "Bad track ID"); NS_ASSERTION(0 <= aStart && aStart <= aSegment->GetDuration(), "Bad start position"); } + public: ~Track() { MOZ_COUNT_DTOR(Track); } + template T* Get() const { if (mSegment->GetType() == T::StaticType()) { @@ -87,6 +92,7 @@ public: } return nullptr; } + MediaSegment* GetSegment() const { return mSegment; } TrackID GetID() const { return mID; } bool IsEnded() const { return mEnded; } @@ -129,7 +135,7 @@ public: return amount; } - protected: + private: friend class StreamBuffer; // Start offset is in ticks at rate mRate @@ -143,7 +149,8 @@ public: bool mEnded; }; - class CompareTracksByID { + class MOZ_STACK_CLASS CompareTracksByID final + { public: bool Equals(Track* aA, Track* aB) const { return aA->GetID() == aB->GetID(); @@ -154,7 +161,9 @@ public: }; StreamBuffer() - : mTracksKnownTime(0), mForgottenTime(0) + : mTracksKnownTime(0) + , mForgottenTime(0) + , mTracksDirty(false) #ifdef DEBUG , mGraphRateIsSet(false) #endif @@ -206,6 +215,7 @@ public: Track* track = new Track(aID, aStart, aSegment); mTracks.InsertElementSorted(track, CompareTracksByID()); + mTracksDirty = true; if (mTracksKnownTime == STREAM_TIME_MAX) { // There exists code like @@ -216,6 +226,7 @@ public: } return *track; } + void AdvanceKnownTracksTime(StreamTime aKnownTime) { NS_ASSERTION(aKnownTime >= mTracksKnownTime, "Can't move tracks-known time earlier"); @@ -241,7 +252,8 @@ public: Track* FindTrack(TrackID aID); - class TrackIter { + class MOZ_STACK_CLASS TrackIter final + { public: /** * Iterate through the tracks of aBuffer in order of ID. @@ -294,14 +306,28 @@ public: return mForgottenTime; } + bool GetAndResetTracksDirty() + { + if (!mTracksDirty) { + return false; + } + + mTracksDirty = false; + return true; + } + protected: TrackRate mGraphRate; // StreamTime per second // Any new tracks added will start at or after this time. In other words, the track // list is complete and correct for all times less than this time. StreamTime mTracksKnownTime; StreamTime mForgottenTime; + +private: // All known tracks for this StreamBuffer - nsTArray > mTracks; + nsTArray> mTracks; + bool mTracksDirty; + #ifdef DEBUG bool mGraphRateIsSet; #endif diff --git a/dom/media/TrackUnionStream.cpp b/dom/media/TrackUnionStream.cpp index a540e857ed..e23a8bcd6a 100644 --- a/dom/media/TrackUnionStream.cpp +++ b/dom/media/TrackUnionStream.cpp @@ -96,18 +96,19 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : if (map->mInputPort == mInputs[i] && map->mInputTrackID == tracks->GetID()) { bool trackFinished; StreamBuffer::Track* outputTrack = mBuffer.FindTrack(map->mOutputTrackID); - if (!outputTrack || outputTrack->IsEnded()) { + found = true; + if (!outputTrack || outputTrack->IsEnded() || + !mInputs[i]->PassTrackThrough(tracks->GetID())) { trackFinished = true; } else { CopyTrackData(tracks.get(), j, aFrom, aTo, &trackFinished); } mappedTracksFinished[j] = trackFinished; mappedTracksWithMatchingInputTracks[j] = true; - found = true; break; } } - if (!found) { + if (!found && mInputs[i]->PassTrackThrough(tracks->GetID())) { bool trackFinished = false; trackAdded = true; uint32_t mapIndex = AddTrack(mInputs[i], tracks.get(), aFrom); @@ -138,7 +139,7 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : // so we're finished now. FinishOnGraphThread(); } else { - mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTime(aTo)); + mBuffer.AdvanceKnownTracksTime(GraphTimeToStreamTimeWithBlocking(aTo)); } if (allHaveCurrentData) { // We can make progress if we're not blocked @@ -183,7 +184,7 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : // Round up the track start time so the track, if anything, starts a // little later than the true time. This means we'll have enough // samples in our input stream to go just beyond the destination time. - StreamTime outputStart = GraphTimeToStreamTime(aFrom); + StreamTime outputStart = GraphTimeToStreamTimeWithBlocking(aFrom); nsAutoPtr segment; segment = aTrack->GetSegment()->CreateEmptyClone(); @@ -191,7 +192,8 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : MediaStreamListener* l = mListeners[j]; l->NotifyQueuedTrackChanges(Graph(), id, outputStart, MediaStreamListener::TRACK_EVENT_CREATED, - *segment); + *segment, + aPort->GetSource(), aTrack->GetID()); } segment->AppendNullData(outputStart); StreamBuffer::Track* track = @@ -223,7 +225,9 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : segment = outputTrack->GetSegment()->CreateEmptyClone(); l->NotifyQueuedTrackChanges(Graph(), outputTrack->GetID(), offset, MediaStreamListener::TRACK_EVENT_ENDED, - *segment); + *segment, + mTrackMap[aIndex].mInputPort->GetSource(), + mTrackMap[aIndex].mInputTrackID); } outputTrack->SetEnded(); } @@ -244,7 +248,7 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : for (GraphTime t = aFrom; t < aTo; t = next) { MediaInputPort::InputInterval interval = map->mInputPort->GetNextInputInterval(t); interval.mEnd = std::min(interval.mEnd, aTo); - StreamTime inputEnd = source->GraphTimeToStreamTime(interval.mEnd); + StreamTime inputEnd = source->GraphTimeToStreamTimeWithBlocking(interval.mEnd); StreamTime inputTrackEndPoint = STREAM_TIME_MAX; if (aInputTrack->IsEnded() && @@ -269,12 +273,12 @@ TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : } else if (InMutedCycle()) { segment->AppendNullData(ticks); } else { - if (GraphImpl()->StreamSuspended(source)) { + if (source->IsSuspended()) { segment->AppendNullData(aTo - aFrom); } else { - MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTime(interval.mStart), + MOZ_ASSERT(outputTrack->GetEnd() == GraphTimeToStreamTimeWithBlocking(interval.mStart), "Samples missing"); - StreamTime inputStart = source->GraphTimeToStreamTime(interval.mStart); + StreamTime inputStart = source->GraphTimeToStreamTimeWithBlocking(interval.mStart); segment->AppendSlice(*aInputTrack->GetSegment(), std::min(inputTrackEndPoint, inputStart), std::min(inputTrackEndPoint, inputEnd)); diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp index 84e2e211ba..7c1183466a 100644 --- a/dom/media/encoder/MediaEncoder.cpp +++ b/dom/media/encoder/MediaEncoder.cpp @@ -41,7 +41,9 @@ MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) { // Process the incoming raw track data from MediaStreamGraph, called on the // thread of MediaStreamGraph. diff --git a/dom/media/encoder/MediaEncoder.h b/dom/media/encoder/MediaEncoder.h index 94a3d6dc62..2922a3ad86 100644 --- a/dom/media/encoder/MediaEncoder.h +++ b/dom/media/encoder/MediaEncoder.h @@ -85,7 +85,9 @@ public : virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) override; + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override; /** * Notified the stream is being removed. diff --git a/dom/media/imagecapture/CaptureTask.cpp b/dom/media/imagecapture/CaptureTask.cpp index fcd786ac17..1cf41f0b95 100644 --- a/dom/media/imagecapture/CaptureTask.cpp +++ b/dom/media/imagecapture/CaptureTask.cpp @@ -86,7 +86,9 @@ void CaptureTask::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) { if (mImageGrabbedOrTrackEnd) { return; diff --git a/dom/media/imagecapture/CaptureTask.h b/dom/media/imagecapture/CaptureTask.h index 349559dc1d..d8957aed83 100644 --- a/dom/media/imagecapture/CaptureTask.h +++ b/dom/media/imagecapture/CaptureTask.h @@ -35,7 +35,9 @@ public: virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) override; + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override; virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamGraphEvent aEvent) override; diff --git a/dom/media/mediasink/DecodedStream.cpp b/dom/media/mediasink/DecodedStream.cpp index 25815972dc..3876be02f2 100644 --- a/dom/media/mediasink/DecodedStream.cpp +++ b/dom/media/mediasink/DecodedStream.cpp @@ -35,7 +35,7 @@ public: { MutexAutoLock lock(mMutex); if (mStream) { - mLastOutputTime = mStream->StreamTimeToMicroseconds(mStream->GraphTimeToStreamTime(aCurrentTime)); + mLastOutputTime = mStream->StreamTimeToMicroseconds(mStream->GraphTimeToStreamTimeWithBlocking(aCurrentTime)); } } @@ -86,14 +86,21 @@ private: }; static void -UpdateStreamBlocking(MediaStream* aStream, bool aBlocking) +UpdateStreamSuspended(MediaStream* aStream, bool aBlocking) { - int32_t delta = aBlocking ? 1 : -1; if (NS_IsMainThread()) { - aStream->ChangeExplicitBlockerCount(delta); + if (aBlocking) { + aStream->Suspend(); + } else { + aStream->Resume(); + } } else { - nsCOMPtr r = NS_NewRunnableMethodWithArg( - aStream, &MediaStream::ChangeExplicitBlockerCount, delta); + nsCOMPtr r; + if (aBlocking) { + r = NS_NewRunnableMethod(aStream, &MediaStream::Suspend); + } else { + r = NS_NewRunnableMethod(aStream, &MediaStream::Resume); + } AbstractThread::MainThread()->Dispatch(r.forget()); } } @@ -189,7 +196,7 @@ DecodedStreamData::SetPlaying(bool aPlaying) { if (mPlaying != aPlaying) { mPlaying = aPlaying; - UpdateStreamBlocking(mStream, !mPlaying); + UpdateStreamSuspended(mStream, !mPlaying); } } @@ -253,13 +260,7 @@ OutputStreamData::Connect(MediaStream* aStream) MOZ_ASSERT(!mPort, "Already connected?"); MOZ_ASSERT(!mStream->IsDestroyed(), "Can't connect a destroyed stream."); - // The output stream must stay in sync with the input stream, so if - // either stream is blocked, we block the other. - mPort = mStream->AllocateInputPort(aStream, - MediaInputPort::FLAG_BLOCK_INPUT | MediaInputPort::FLAG_BLOCK_OUTPUT); - // Unblock the output stream now. The input stream is responsible for - // controlling blocking from now on. - mStream->ChangeExplicitBlockerCount(-1); + mPort = mStream->AllocateInputPort(aStream); } bool @@ -279,9 +280,6 @@ OutputStreamData::Disconnect() mPort->Destroy(); mPort = nullptr; } - // Block the stream again. It will be unlocked when connecting - // to the input stream. - mStream->ChangeExplicitBlockerCount(1); return true; } diff --git a/dom/media/webaudio/AnalyserNode.cpp b/dom/media/webaudio/AnalyserNode.cpp index 755230377a..dc4c59a254 100644 --- a/dom/media/webaudio/AnalyserNode.cpp +++ b/dom/media/webaudio/AnalyserNode.cpp @@ -59,20 +59,46 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { *aOutput = aInput; - RefPtr transfer = new TransferBuffer(aStream, aInput); + if (aInput.IsNull()) { + // If AnalyserNode::mChunks has only null chunks, then there is no need + // to send further null chunks. + if (mChunksToProcess <= 0) { + if (mChunksToProcess != INT32_MIN) { + mChunksToProcess = INT32_MIN; + aStream->CheckForInactive(); + } + return; + } + + --mChunksToProcess; + } else { + // This many null chunks will be required to empty AnalyserNode::mChunks. + mChunksToProcess = CHUNK_COUNT; + } + + RefPtr transfer = + new TransferBuffer(aStream, aInput.AsAudioChunk()); NS_DispatchToMainThread(transfer); } + virtual bool IsActive() const override + { + return mChunksToProcess != INT32_MIN; + } + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } + + int32_t mChunksToProcess = INT32_MIN; }; AnalyserNode::AnalyserNode(AudioContext* aContext) @@ -85,7 +111,7 @@ AnalyserNode::AnalyserNode(AudioContext* aContext) , mMaxDecibels(-30.) , mSmoothingTimeConstant(.8) { - mStream = AudioNodeStream::Create(aContext->Graph(), + mStream = AudioNodeStream::Create(aContext, new AnalyserNodeEngine(this), AudioNodeStream::NO_STREAM_FLAGS); @@ -329,7 +355,7 @@ AnalyserNode::GetTimeDomainData(float* aData, size_t aLength) for (size_t writeIndex = 0; writeIndex < aLength; ) { const AudioChunk& chunk = mChunks[readChunk & (CHUNK_COUNT - 1)]; - const size_t channelCount = chunk.mChannelData.Length(); + const size_t channelCount = chunk.ChannelCount(); size_t copyLength = std::min(aLength - writeIndex, WEBAUDIO_BLOCK_SIZE); float* dataOut = &aData[writeIndex]; diff --git a/dom/media/webaudio/AudioBlock.cpp b/dom/media/webaudio/AudioBlock.cpp new file mode 100644 index 0000000000..064235b092 --- /dev/null +++ b/dom/media/webaudio/AudioBlock.cpp @@ -0,0 +1,162 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AudioBlock.h" + +namespace mozilla { + +/** + * Heap-allocated buffer of channels of 128-sample float arrays, with + * threadsafe refcounting. Typically you would allocate one of these, fill it + * in, and then treat it as immutable while it's shared. + * + * Downstream references are accounted specially so that the creator of the + * buffer can reuse and modify its contents next iteration if other references + * are all downstream temporary references held by AudioBlock. + * + * This only guarantees 4-byte alignment of the data. For alignment we simply + * assume that the memory from malloc is at least 4-byte aligned and that + * AudioBlockBuffer's size is divisible by 4. + */ +class AudioBlockBuffer final : public ThreadSharedObject { +public: + + virtual AudioBlockBuffer* AsAudioBlockBuffer() override { return this; }; + + float* ChannelData(uint32_t aChannel) + { + return reinterpret_cast(this + 1) + aChannel * WEBAUDIO_BLOCK_SIZE; + } + + static already_AddRefed Create(uint32_t aChannelCount) + { + CheckedInt size = WEBAUDIO_BLOCK_SIZE; + size *= aChannelCount; + size *= sizeof(float); + size += sizeof(AudioBlockBuffer); + if (!size.isValid()) { + MOZ_CRASH(); + } + void* m = moz_xmalloc(size.value()); + RefPtr p = new (m) AudioBlockBuffer(); + NS_ASSERTION((reinterpret_cast(p.get() + 1) - reinterpret_cast(p.get())) % 4 == 0, + "AudioBlockBuffers should be at least 4-byte aligned"); + return p.forget(); + } + + // Graph thread only. + void DownstreamRefAdded() { ++mDownstreamRefCount; } + void DownstreamRefRemoved() { + MOZ_ASSERT(mDownstreamRefCount > 0); + --mDownstreamRefCount; + } + // Whether this is shared by any owners that are not downstream. + // Called only from owners with a reference that is not a downstream + // reference. Graph thread only. + bool HasLastingShares() + { + // mRefCnt is atomic and so reading its value is defined even when + // modifications may happen on other threads. mDownstreamRefCount is + // not modified on any other thread. + // + // If all other references are downstream references (managed on this, the + // graph thread), then other threads are not using this buffer and cannot + // add further references. This method can safely return false. The + // buffer contents can be modified. + // + // If there are other references that are not downstream references, then + // this method will return true. The buffer will be assumed to be still + // in use and so will not be reused. + nsrefcnt count = mRefCnt; + // This test is strictly less than because the caller has a reference + // that is not a downstream reference. + MOZ_ASSERT(mDownstreamRefCount < count); + return count != mDownstreamRefCount + 1; + } + + virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override + { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + +private: + AudioBlockBuffer() {} + ~AudioBlockBuffer() override { MOZ_ASSERT(mDownstreamRefCount == 0); } + + nsAutoRefCnt mDownstreamRefCount; +}; + +AudioBlock::~AudioBlock() +{ + ClearDownstreamMark(); +} + +void +AudioBlock::SetBuffer(ThreadSharedObject* aNewBuffer) +{ + if (aNewBuffer == mBuffer) { + return; + } + + ClearDownstreamMark(); + + mBuffer = aNewBuffer; + + if (!aNewBuffer) { + return; + } + + AudioBlockBuffer* buffer = aNewBuffer->AsAudioBlockBuffer(); + if (buffer) { + buffer->DownstreamRefAdded(); + mBufferIsDownstreamRef = true; + } +} + +void +AudioBlock::ClearDownstreamMark() { + if (mBufferIsDownstreamRef) { + mBuffer->AsAudioBlockBuffer()->DownstreamRefRemoved(); + mBufferIsDownstreamRef = false; + } +} + +void +AudioBlock::AssertNoLastingShares() { + MOZ_ASSERT(!mBuffer->AsAudioBlockBuffer()->HasLastingShares()); +} + +void +AudioBlock::AllocateChannels(uint32_t aChannelCount) +{ + MOZ_ASSERT(mDuration == WEBAUDIO_BLOCK_SIZE); + + if (mBufferIsDownstreamRef) { + // This is not our buffer to re-use. + ClearDownstreamMark(); + } else if (mBuffer && ChannelCount() == aChannelCount) { + AudioBlockBuffer* buffer = mBuffer->AsAudioBlockBuffer(); + if (buffer && !buffer->HasLastingShares()) { + MOZ_ASSERT(mBufferFormat == AUDIO_FORMAT_FLOAT32); + // No need to allocate again. + mVolume = 1.0f; + return; + } + } + + // XXX for SIMD purposes we should do something here to make sure the + // channel buffers are 16-byte aligned. + RefPtr buffer = AudioBlockBuffer::Create(aChannelCount); + mChannelData.SetLength(aChannelCount); + for (uint32_t i = 0; i < aChannelCount; ++i) { + mChannelData[i] = buffer->ChannelData(i); + } + mBuffer = buffer.forget(); + mVolume = 1.0f; + mBufferFormat = AUDIO_FORMAT_FLOAT32; +} + +} // namespace mozilla diff --git a/dom/media/webaudio/AudioBlock.h b/dom/media/webaudio/AudioBlock.h new file mode 100644 index 0000000000..5ad2b1703c --- /dev/null +++ b/dom/media/webaudio/AudioBlock.h @@ -0,0 +1,120 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef MOZILLA_AUDIOBLOCK_H_ +#define MOZILLA_AUDIOBLOCK_H_ + +#include "AudioSegment.h" + +namespace mozilla { + +/** + * An AudioChunk whose buffer contents need to be valid only for one + * processing block iteration, after which contents can be overwritten if the + * buffer has not been passed to longer term storage or to another thread, + * which may happen though AsAudioChunk() or AsMutableChunk(). + * + * Use on graph thread only. + */ +class AudioBlock : private AudioChunk +{ +public: + AudioBlock() { + mDuration = WEBAUDIO_BLOCK_SIZE; + } + AudioBlock(const AudioChunk& aChunk) { + mDuration = WEBAUDIO_BLOCK_SIZE; + operator=(aChunk); + } + ~AudioBlock(); + + using AudioChunk::GetDuration; + using AudioChunk::IsNull; + using AudioChunk::ChannelCount; + using AudioChunk::ChannelData; + using AudioChunk::SizeOfExcludingThisIfUnshared; + using AudioChunk::SizeOfExcludingThis; + // mDuration is not exposed. Use GetDuration(). + // mBuffer is not exposed. Use SetBuffer(). + using AudioChunk::mChannelData; + using AudioChunk::mVolume; + using AudioChunk::mBufferFormat; + + const AudioChunk& AsAudioChunk() const { return *this; } + AudioChunk* AsMutableChunk() { + void ClearDownstreamMark(); + return this; + } + + /** + * Allocates, if necessary, aChannelCount buffers of WEBAUDIO_BLOCK_SIZE float + * samples for writing. + */ + void AllocateChannels(uint32_t aChannelCount); + + float* ChannelFloatsForWrite(size_t aChannel) + { + MOZ_ASSERT(mBufferFormat == AUDIO_FORMAT_FLOAT32); +#if DEBUG + AssertNoLastingShares(); +#endif + return static_cast(const_cast(mChannelData[aChannel])); + } + + void SetBuffer(ThreadSharedObject* aNewBuffer); + void SetNull(StreamTime aDuration) { + MOZ_ASSERT(aDuration == WEBAUDIO_BLOCK_SIZE); + SetBuffer(nullptr); + mChannelData.Clear(); + mVolume = 1.0f; + mBufferFormat = AUDIO_FORMAT_SILENCE; + } + + AudioBlock& operator=(const AudioChunk& aChunk) { + MOZ_ASSERT(aChunk.mDuration == WEBAUDIO_BLOCK_SIZE); + SetBuffer(aChunk.mBuffer); + mChannelData = aChunk.mChannelData; + mVolume = aChunk.mVolume; + mBufferFormat = aChunk.mBufferFormat; + return *this; + } + + bool IsMuted() const { return mVolume == 0.0f; } + + bool IsSilentOrSubnormal() const + { + if (!mBuffer) { + return true; + } + + for (uint32_t i = 0, length = mChannelData.Length(); i < length; ++i) { + const float* channel = static_cast(mChannelData[i]); + for (StreamTime frame = 0; frame < mDuration; ++frame) { + if (fabs(channel[frame]) >= FLT_MIN) { + return false; + } + } + } + + return true; + } + +private: + void ClearDownstreamMark(); + void AssertNoLastingShares(); + + // mBufferIsDownstreamRef is set only when mBuffer references an + // AudioBlockBuffer created in a different AudioBlock. That can happen when + // this AudioBlock is on a node downstream from the node which created the + // buffer. When this is set, the AudioBlockBuffer is notified that this + // reference does prevent the upstream node from re-using the buffer next + // iteration and modifying its contents. The AudioBlockBuffer is also + // notified when mBuffer releases this reference. + bool mBufferIsDownstreamRef = false; +}; + +} // namespace mozilla + +#endif // MOZILLA_AUDIOBLOCK_H_ diff --git a/dom/media/webaudio/AudioBuffer.cpp b/dom/media/webaudio/AudioBuffer.cpp index a6401dbfc4..a022319395 100644 --- a/dom/media/webaudio/AudioBuffer.cpp +++ b/dom/media/webaudio/AudioBuffer.cpp @@ -20,14 +20,12 @@ namespace dom { NS_IMPL_CYCLE_COLLECTION_CLASS(AudioBuffer) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(AudioBuffer) - NS_IMPL_CYCLE_COLLECTION_UNLINK(mContext) NS_IMPL_CYCLE_COLLECTION_UNLINK(mJSChannels) NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER tmp->ClearJSChannels(); NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(AudioBuffer) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mContext) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_SCRIPT_OBJECTS NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END @@ -42,12 +40,17 @@ NS_IMPL_CYCLE_COLLECTION_ROOT_NATIVE(AudioBuffer, AddRef) NS_IMPL_CYCLE_COLLECTION_UNROOT_NATIVE(AudioBuffer, Release) AudioBuffer::AudioBuffer(AudioContext* aContext, uint32_t aNumberOfChannels, - uint32_t aLength, float aSampleRate) - : mContext(aContext), + uint32_t aLength, float aSampleRate, + already_AddRefed + aInitialContents) + : mOwnerWindow(do_GetWeakReference(aContext->GetOwner())), + mSharedChannels(aInitialContents), mLength(aLength), mSampleRate(aSampleRate) { - mJSChannels.SetCapacity(aNumberOfChannels); + MOZ_ASSERT(!mSharedChannels || + mSharedChannels->GetChannels() == aNumberOfChannels); + mJSChannels.SetLength(aNumberOfChannels); mozilla::HoldJSObjects(this); } @@ -66,6 +69,8 @@ AudioBuffer::ClearJSChannels() /* static */ already_AddRefed AudioBuffer::Create(AudioContext* aContext, uint32_t aNumberOfChannels, uint32_t aLength, float aSampleRate, + already_AddRefed + aInitialContents, JSContext* aJSContext, ErrorResult& aRv) { // Note that a buffer with zero channels is permitted here for the sake of @@ -80,7 +85,12 @@ AudioBuffer::Create(AudioContext* aContext, uint32_t aNumberOfChannels, } RefPtr buffer = - new AudioBuffer(aContext, aNumberOfChannels, aLength, aSampleRate); + new AudioBuffer(aContext, aNumberOfChannels, aLength, aSampleRate, + Move(aInitialContents)); + + if (buffer->mSharedChannels) { + return buffer.forget(); + } for (uint32_t i = 0; i < aNumberOfChannels; ++i) { JS::Rooted array(aJSContext, @@ -89,7 +99,7 @@ AudioBuffer::Create(AudioContext* aContext, uint32_t aNumberOfChannels, aRv.Throw(NS_ERROR_OUT_OF_MEMORY); return nullptr; } - buffer->mJSChannels.AppendElement(array.get()); + buffer->mJSChannels[i] = array; } return buffer.forget(); diff --git a/dom/media/webaudio/AudioBuffer.h b/dom/media/webaudio/AudioBuffer.h index 84d62b680d..b9bd2637dd 100644 --- a/dom/media/webaudio/AudioBuffer.h +++ b/dom/media/webaudio/AudioBuffer.h @@ -33,19 +33,32 @@ class AudioContext; class AudioBuffer final : public nsWrapperCache { public: + // If non-null, aInitialContents must have number of channels equal to + // aNumberOfChannels and their lengths must be at least aLength. static already_AddRefed Create(AudioContext* aContext, uint32_t aNumberOfChannels, uint32_t aLength, float aSampleRate, + already_AddRefed aInitialContents, JSContext* aJSContext, ErrorResult& aRv); + static already_AddRefed + Create(AudioContext* aContext, uint32_t aNumberOfChannels, + uint32_t aLength, float aSampleRate, + JSContext* aJSContext, ErrorResult& aRv) + { + return Create(aContext, aNumberOfChannels, aLength, aSampleRate, + nullptr, aJSContext, aRv); + } + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(AudioBuffer) NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_NATIVE_CLASS(AudioBuffer) - AudioContext* GetParentObject() const + nsPIDOMWindow* GetParentObject() const { - return mContext; + nsCOMPtr parentObject = do_QueryReferent(mOwnerWindow); + return parentObject; } virtual JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; @@ -98,13 +111,15 @@ public: protected: AudioBuffer(AudioContext* aContext, uint32_t aNumberOfChannels, - uint32_t aLength, float aSampleRate); + uint32_t aLength, float aSampleRate, + already_AddRefed + aInitialContents); ~AudioBuffer(); bool RestoreJSChannelData(JSContext* aJSContext); void ClearJSChannels(); - RefPtr mContext; + nsWeakPtr mOwnerWindow; // Float32Arrays nsAutoTArray, 2> mJSChannels; diff --git a/dom/media/webaudio/AudioBufferSourceNode.cpp b/dom/media/webaudio/AudioBufferSourceNode.cpp index 69f05cfc91..50c0a94c4b 100644 --- a/dom/media/webaudio/AudioBufferSourceNode.cpp +++ b/dom/media/webaudio/AudioBufferSourceNode.cpp @@ -19,22 +19,7 @@ namespace mozilla { namespace dom { -NS_IMPL_CYCLE_COLLECTION_CLASS(AudioBufferSourceNode) - -NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(AudioBufferSourceNode) - NS_IMPL_CYCLE_COLLECTION_UNLINK(mBuffer) - NS_IMPL_CYCLE_COLLECTION_UNLINK(mPlaybackRate) - NS_IMPL_CYCLE_COLLECTION_UNLINK(mDetune) - if (tmp->Context()) { - tmp->DisconnectFromGraph(); - } -NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(AudioNode) - -NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(AudioBufferSourceNode, AudioNode) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mBuffer) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPlaybackRate) - NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mDetune) -NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END +NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioBufferSourceNode, AudioNode, mBuffer, mPlaybackRate, mDetune) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(AudioBufferSourceNode) NS_INTERFACE_MAP_END_INHERITING(AudioNode) @@ -59,7 +44,9 @@ public: mResampler(nullptr), mRemainingResamplerTail(0), mBufferEnd(0), mLoopStart(0), mLoopEnd(0), - mBufferSampleRate(0), mBufferPosition(0), mChannels(0), + mBufferPosition(0), mBufferSampleRate(0), + // mResamplerOutRate is initialized in UpdateResampler(). + mChannels(0), mDopplerShift(1.0f), mDestination(aDestination->Stream()), mPlaybackRateTimeline(1.0f), @@ -79,18 +66,19 @@ public: mSource = aSource; } - virtual void SetTimelineParameter(uint32_t aIndex, - const dom::AudioParamTimeline& aValue, - TrackRate aSampleRate) override + virtual void RecvTimelineEvent(uint32_t aIndex, + dom::AudioTimelineEvent& aEvent) override { + MOZ_ASSERT(mDestination); + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case AudioBufferSourceNode::PLAYBACKRATE: - mPlaybackRateTimeline = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mPlaybackRateTimeline, mSource, mDestination); + mPlaybackRateTimeline.InsertEvent(aEvent); break; case AudioBufferSourceNode::DETUNE: - mDetuneTimeline = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mDetuneTimeline, mSource, mDestination); + mDetuneTimeline.InsertEvent(aEvent); break; default: NS_ERROR("Bad AudioBufferSourceNodeEngine TimelineParameter"); @@ -109,8 +97,7 @@ public: switch (aIndex) { case AudioBufferSourceNode::START: MOZ_ASSERT(!mStart, "Another START?"); - mStart = - mSource->FractionalTicksFromDestinationTime(mDestination, aParam); + mStart = aParam * mDestination->SampleRate(); // Round to nearest mBeginProcessing = mStart + 0.5; break; @@ -124,16 +111,30 @@ public: virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) override { switch (aIndex) { - case AudioBufferSourceNode::SAMPLE_RATE: mBufferSampleRate = aParam; break; + case AudioBufferSourceNode::SAMPLE_RATE: + MOZ_ASSERT(aParam > 0); + mBufferSampleRate = aParam; + mSource->SetActive(); + break; case AudioBufferSourceNode::BUFFERSTART: + MOZ_ASSERT(aParam >= 0); if (mBufferPosition == 0) { mBufferPosition = aParam; } break; - case AudioBufferSourceNode::BUFFEREND: mBufferEnd = aParam; break; + case AudioBufferSourceNode::BUFFEREND: + MOZ_ASSERT(aParam >= 0); + mBufferEnd = aParam; + break; case AudioBufferSourceNode::LOOP: mLoop = !!aParam; break; - case AudioBufferSourceNode::LOOPSTART: mLoopStart = aParam; break; - case AudioBufferSourceNode::LOOPEND: mLoopEnd = aParam; break; + case AudioBufferSourceNode::LOOPSTART: + MOZ_ASSERT(aParam >= 0); + mLoopStart = aParam; + break; + case AudioBufferSourceNode::LOOPEND: + MOZ_ASSERT(aParam >= 0); + mLoopEnd = aParam; + break; default: NS_ERROR("Bad AudioBufferSourceNodeEngine Int32Parameter"); } @@ -163,25 +164,26 @@ public: mBeginProcessing = mStart + 0.5; } - if (aOutRate == mBufferSampleRate && !mResampler) { + if (aChannels == 0 || + (aOutRate == mBufferSampleRate && !mResampler)) { + mResamplerOutRate = aOutRate; return; } if (!mResampler) { mChannels = aChannels; mResampler = speex_resampler_init(mChannels, mBufferSampleRate, aOutRate, - SPEEX_RESAMPLER_QUALITY_DEFAULT, + SPEEX_RESAMPLER_QUALITY_MIN, nullptr); } else { - uint32_t currentOutSampleRate, currentInSampleRate; - speex_resampler_get_rate(mResampler, ¤tInSampleRate, - ¤tOutSampleRate); - if (currentOutSampleRate == static_cast(aOutRate)) { + if (mResamplerOutRate == aOutRate) { return; } - speex_resampler_set_rate(mResampler, currentInSampleRate, aOutRate); + speex_resampler_set_rate(mResampler, mBufferSampleRate, aOutRate); } + mResamplerOutRate = aOutRate; + if (!BegunResampling()) { // Low pass filter effects from the resampler mean that samples before // the start time are influenced by resampling the buffer. The input @@ -202,11 +204,10 @@ public: // Borrow a full buffer of size WEBAUDIO_BLOCK_SIZE from the source buffer // at offset aSourceOffset. This avoids copying memory. - void BorrowFromInputBuffer(AudioChunk* aOutput, + void BorrowFromInputBuffer(AudioBlock* aOutput, uint32_t aChannels) { - aOutput->mDuration = WEBAUDIO_BLOCK_SIZE; - aOutput->mBuffer = mBuffer; + aOutput->SetBuffer(mBuffer); aOutput->mChannelData.SetLength(aChannels); for (uint32_t i = 0; i < aChannels; ++i) { aOutput->mChannelData[i] = mBuffer->GetData(i) + mBufferPosition; @@ -217,7 +218,7 @@ public: // Copy aNumberOfFrames frames from the source buffer at offset aSourceOffset // and put it at offset aBufferOffset in the destination buffer. - void CopyFromInputBuffer(AudioChunk* aOutput, + void CopyFromInputBuffer(AudioBlock* aOutput, uint32_t aChannels, uintptr_t aOffsetWithinBlock, uint32_t aNumberOfFrames) { @@ -234,15 +235,16 @@ public: // The number of frames consumed/produced depends on the amount of space // remaining in both the input and output buffer, and the playback rate (that // is, the ratio between the output samplerate and the input samplerate). - void CopyFromInputBufferWithResampling(AudioNodeStream* aStream, - AudioChunk* aOutput, + void CopyFromInputBufferWithResampling(AudioBlock* aOutput, uint32_t aChannels, uint32_t* aOffsetWithinBlock, + uint32_t aAvailableInOutput, StreamTime* aCurrentPosition, - int32_t aBufferMax) { - // TODO: adjust for mStop (see bug 913854 comment 9). - uint32_t availableInOutputBuffer = - WEBAUDIO_BLOCK_SIZE - *aOffsetWithinBlock; + uint32_t aBufferMax) + { + if (*aOffsetWithinBlock == 0) { + aOutput->AllocateChannels(aChannels); + } SpeexResamplerState* resampler = mResampler; MOZ_ASSERT(aChannels > 0); @@ -254,7 +256,7 @@ public: // format-converted for resampling by estimating how many will be used. // This may be a little small if still filling the resampler with // initial data, but we'll get called again and it will work out. - uint32_t inputLimit = availableInOutputBuffer * ratioNum / ratioDen + 10; + uint32_t inputLimit = aAvailableInOutput * ratioNum / ratioDen + 10; if (!BegunResampling()) { // First time the resampler is used. uint32_t inputLatency = speex_resampler_get_input_latency(resampler); @@ -263,15 +265,18 @@ public: // buffer, but correct for input latency. If starting before mStart, // then align the resampler so that the time corresponding to the // first input sample is mStart. - uint32_t skipFracNum = inputLatency * ratioDen; + int64_t skipFracNum = static_cast(inputLatency) * ratioDen; double leadTicks = mStart - *aCurrentPosition; if (leadTicks > 0.0) { // Round to nearest output subsample supported by the resampler at // these rates. - skipFracNum -= leadTicks * ratioNum + 0.5; - MOZ_ASSERT(skipFracNum < INT32_MAX, "mBeginProcessing is wrong?"); + int64_t leadSubsamples = leadTicks * ratioNum + 0.5; + MOZ_ASSERT(leadSubsamples <= skipFracNum, + "mBeginProcessing is wrong?"); + skipFracNum -= leadSubsamples; } - speex_resampler_set_skip_frac_num(resampler, skipFracNum); + speex_resampler_set_skip_frac_num(resampler, + std::min(skipFracNum, UINT32_MAX)); mBeginProcessing = -STREAM_TIME_MAX; } @@ -281,7 +286,7 @@ public: uint32_t inSamples = inputLimit; const float* inputData = mBuffer->GetData(i) + mBufferPosition; - uint32_t outSamples = availableInOutputBuffer; + uint32_t outSamples = aAvailableInOutput; float* outputData = aOutput->ChannelFloatsForWrite(i) + *aOffsetWithinBlock; @@ -306,7 +311,7 @@ public: } else { for (uint32_t i = 0; true; ) { uint32_t inSamples = mRemainingResamplerTail; - uint32_t outSamples = availableInOutputBuffer; + uint32_t outSamples = aAvailableInOutput; float* outputData = aOutput->ChannelFloatsForWrite(i) + *aOffsetWithinBlock; @@ -316,8 +321,8 @@ public: static_cast(nullptr), &inSamples, outputData, &outSamples); if (++i == aChannels) { + MOZ_ASSERT(inSamples <= mRemainingResamplerTail); mRemainingResamplerTail -= inSamples; - MOZ_ASSERT(mRemainingResamplerTail >= 0); *aOffsetWithinBlock += outSamples; *aCurrentPosition += outSamples; break; @@ -334,7 +339,7 @@ public: * allocate the output buffer, and also optimizes the case where it can avoid * memory allocations. */ - void FillWithZeroes(AudioChunk* aOutput, + void FillWithZeroes(AudioBlock* aOutput, uint32_t aChannels, uint32_t* aOffsetWithinBlock, StreamTime* aCurrentPosition, @@ -344,11 +349,11 @@ public: uint32_t numFrames = std::min(WEBAUDIO_BLOCK_SIZE - *aOffsetWithinBlock, aMaxPos - *aCurrentPosition); - if (numFrames == WEBAUDIO_BLOCK_SIZE) { + if (numFrames == WEBAUDIO_BLOCK_SIZE || !aChannels) { aOutput->SetNull(numFrames); } else { if (*aOffsetWithinBlock == 0) { - AllocateAudioBlock(aChannels, aOutput); + aOutput->AllocateChannels(aChannels); } WriteZeroesToAudioBlock(aOutput, *aOffsetWithinBlock, numFrames); } @@ -365,38 +370,58 @@ public: * This function knows when it needs to allocate the output buffer, and also * optimizes the case where it can avoid memory allocations. */ - void CopyFromBuffer(AudioNodeStream* aStream, - AudioChunk* aOutput, + void CopyFromBuffer(AudioBlock* aOutput, uint32_t aChannels, uint32_t* aOffsetWithinBlock, StreamTime* aCurrentPosition, - int32_t aBufferMax) + uint32_t aBufferMax) { MOZ_ASSERT(*aCurrentPosition < mStop); - uint32_t numFrames = - std::min(std::min(WEBAUDIO_BLOCK_SIZE - *aOffsetWithinBlock, - aBufferMax - mBufferPosition), - mStop - *aCurrentPosition); - if (numFrames == WEBAUDIO_BLOCK_SIZE && !mResampler) { + uint32_t availableInOutput = + std::min(WEBAUDIO_BLOCK_SIZE - *aOffsetWithinBlock, + mStop - *aCurrentPosition); + if (mResampler) { + CopyFromInputBufferWithResampling(aOutput, aChannels, + aOffsetWithinBlock, availableInOutput, + aCurrentPosition, aBufferMax); + return; + } + + if (aChannels == 0) { + aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); + // There is no attempt here to limit advance so that mBufferPosition is + // limited to aBufferMax. The only observable affect of skipping the + // check would be in the precise timing of the ended event if the loop + // attribute is reset after playback has looped. + *aOffsetWithinBlock += availableInOutput; + *aCurrentPosition += availableInOutput; + // Rounding at the start and end of the period means that fractional + // increments essentially accumulate if outRate remains constant. If + // outRate is varying, then accumulation happens on average but not + // precisely. + TrackTicks start = *aCurrentPosition * + mBufferSampleRate / mResamplerOutRate; + TrackTicks end = (*aCurrentPosition + availableInOutput) * + mBufferSampleRate / mResamplerOutRate; + mBufferPosition += end - start; + return; + } + + uint32_t numFrames = std::min(aBufferMax - mBufferPosition, + availableInOutput); + if (numFrames == WEBAUDIO_BLOCK_SIZE) { MOZ_ASSERT(mBufferPosition < aBufferMax); BorrowFromInputBuffer(aOutput, aChannels); - *aOffsetWithinBlock += numFrames; - *aCurrentPosition += numFrames; - mBufferPosition += numFrames; } else { if (*aOffsetWithinBlock == 0) { - AllocateAudioBlock(aChannels, aOutput); - } - if (!mResampler) { - MOZ_ASSERT(mBufferPosition < aBufferMax); - CopyFromInputBuffer(aOutput, aChannels, *aOffsetWithinBlock, numFrames); - *aOffsetWithinBlock += numFrames; - *aCurrentPosition += numFrames; - mBufferPosition += numFrames; - } else { - CopyFromInputBufferWithResampling(aStream, aOutput, aChannels, aOffsetWithinBlock, aCurrentPosition, aBufferMax); + aOutput->AllocateChannels(aChannels); } + MOZ_ASSERT(mBufferPosition < aBufferMax); + CopyFromInputBuffer(aOutput, aChannels, *aOffsetWithinBlock, numFrames); } + *aOffsetWithinBlock += numFrames; + *aCurrentPosition += numFrames; + mBufferPosition += numFrames; } int32_t ComputeFinalOutSampleRate(float aPlaybackRate, float aDetune) @@ -410,7 +435,7 @@ public: return rate ? rate : mBufferSampleRate; } - void UpdateSampleRateIfNeeded(uint32_t aChannels) + void UpdateSampleRateIfNeeded(uint32_t aChannels, StreamTime aStreamPosition) { float playbackRate; float detune; @@ -418,12 +443,12 @@ public: if (mPlaybackRateTimeline.HasSimpleValue()) { playbackRate = mPlaybackRateTimeline.GetValue(); } else { - playbackRate = mPlaybackRateTimeline.GetValueAtTime(mSource->GetCurrentPosition()); + playbackRate = mPlaybackRateTimeline.GetValueAtTime(aStreamPosition); } if (mDetuneTimeline.HasSimpleValue()) { detune = mDetuneTimeline.GetValue(); } else { - detune = mDetuneTimeline.GetValueAtTime(mSource->GetCurrentPosition()); + detune = mDetuneTimeline.GetValueAtTime(aStreamPosition); } if (playbackRate <= 0 || mozilla::IsNaN(playbackRate)) { playbackRate = 1.0f; @@ -436,25 +461,23 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { - if (!mBuffer || !mBufferEnd) { + if (mBufferSampleRate == 0) { + // start() has not yet been called or no buffer has yet been set aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); return; } - uint32_t channels = mBuffer->GetChannels(); - if (!channels) { - aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); - return; - } + StreamTime streamPosition = mDestination->GraphTimeToStreamTime(aFrom); + uint32_t channels = mBuffer ? mBuffer->GetChannels() : 0; - UpdateSampleRateIfNeeded(channels); + UpdateSampleRateIfNeeded(channels, streamPosition); uint32_t written = 0; - StreamTime streamPosition = aStream->GetCurrentPosition(); while (written < WEBAUDIO_BLOCK_SIZE) { if (mStop != STREAM_TIME_MAX && streamPosition >= mStop) { @@ -473,10 +496,10 @@ public: if (mBufferPosition >= mLoopEnd) { mBufferPosition = mLoopStart; } - CopyFromBuffer(aStream, aOutput, channels, &written, &streamPosition, mLoopEnd); + CopyFromBuffer(aOutput, channels, &written, &streamPosition, mLoopEnd); } else { if (mBufferPosition < mBufferEnd || mRemainingResamplerTail) { - CopyFromBuffer(aStream, aOutput, channels, &written, &streamPosition, mBufferEnd); + CopyFromBuffer(aOutput, channels, &written, &streamPosition, mBufferEnd); } else { FillWithZeroes(aOutput, channels, &written, &streamPosition, STREAM_TIME_MAX); } @@ -491,6 +514,12 @@ public: } } + virtual bool IsActive() const override + { + // Whether buffer has been set and start() has been called. + return mBufferSampleRate != 0; + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { // Not owned: @@ -528,12 +557,13 @@ public: SpeexResamplerState* mResampler; // mRemainingResamplerTail, like mBufferPosition, and // mBufferEnd, is measured in input buffer samples. - int mRemainingResamplerTail; - int32_t mBufferEnd; - int32_t mLoopStart; - int32_t mLoopEnd; + uint32_t mRemainingResamplerTail; + uint32_t mBufferEnd; + uint32_t mLoopStart; + uint32_t mLoopEnd; + uint32_t mBufferPosition; int32_t mBufferSampleRate; - int32_t mBufferPosition; + int32_t mResamplerOutRate; uint32_t mChannels; float mDopplerShift; AudioNodeStream* mDestination; @@ -551,13 +581,13 @@ AudioBufferSourceNode::AudioBufferSourceNode(AudioContext* aContext) , mLoopStart(0.0) , mLoopEnd(0.0) // mOffset and mDuration are initialized in Start(). - , mPlaybackRate(new AudioParam(this, SendPlaybackRateToStream, 1.0f, "playbackRate")) - , mDetune(new AudioParam(this, SendDetuneToStream, 0.0f, "detune")) + , mPlaybackRate(new AudioParam(this, PLAYBACKRATE, 1.0f, "playbackRate")) + , mDetune(new AudioParam(this, DETUNE, 0.0f, "detune")) , mLoop(false) , mStartCalled(false) { AudioBufferSourceNodeEngine* engine = new AudioBufferSourceNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NEED_MAIN_THREAD_FINISHED); engine->SetSourceStream(mStream); mStream->AddMainThreadListener(this); @@ -570,11 +600,12 @@ AudioBufferSourceNode::~AudioBufferSourceNode() void AudioBufferSourceNode::DestroyMediaStream() { - if (mStream) { + bool hadStream = mStream; + if (hadStream) { mStream->RemoveMainThreadListener(this); } AudioNode::DestroyMediaStream(); - if (Context()) { + if (hadStream && Context()) { Context()->UnregisterAudioBufferSourceNode(this); } } @@ -638,7 +669,7 @@ AudioBufferSourceNode::Start(double aWhen, double aOffset, // Don't set parameter unnecessarily if (aWhen > 0.0) { - ns->SetDoubleParameter(START, mContext->DOMTimeToStreamTime(aWhen)); + ns->SetDoubleParameter(START, aWhen); } } @@ -651,16 +682,15 @@ AudioBufferSourceNode::SendBufferParameterToStream(JSContext* aCx) } if (mBuffer) { - float rate = mBuffer->SampleRate(); RefPtr data = mBuffer->GetThreadSharedChannelsForRate(aCx); ns->SetBuffer(data.forget()); - ns->SetInt32Parameter(SAMPLE_RATE, rate); if (mStartCalled) { SendOffsetAndDurationParametersToStream(ns); } } else { + ns->SetInt32Parameter(BUFFEREND, 0); ns->SetBuffer(nullptr); MarkInactive(); @@ -674,6 +704,8 @@ AudioBufferSourceNode::SendOffsetAndDurationParametersToStream(AudioNodeStream* "Only call this when we have a buffer and start() has been called"); float rate = mBuffer->SampleRate(); + aStream->SetInt32Parameter(SAMPLE_RATE, rate); + int32_t bufferEnd = mBuffer->Length(); int32_t offsetSamples = std::max(0, NS_lround(mOffset * rate)); @@ -683,8 +715,14 @@ AudioBufferSourceNode::SendOffsetAndDurationParametersToStream(AudioNodeStream* } if (mDuration != std::numeric_limits::min()) { - bufferEnd = std::min(bufferEnd, - offsetSamples + NS_lround(mDuration * rate)); + MOZ_ASSERT(mDuration >= 0.0); // provided by Start() + MOZ_ASSERT(rate >= 0.0f); // provided by AudioBuffer::Create() + static_assert(std::numeric_limits::digits >= + std::numeric_limits::digits, + "bufferEnd should be represented exactly by double"); + // + 0.5 rounds mDuration to nearest sample when assigned to bufferEnd. + bufferEnd = std::min(bufferEnd, + offsetSamples + mDuration * rate + 0.5); } aStream->SetInt32Parameter(BUFFEREND, bufferEnd); @@ -747,26 +785,6 @@ AudioBufferSourceNode::NotifyMainThreadStreamFinished() MarkInactive(); } -void -AudioBufferSourceNode::SendPlaybackRateToStream(AudioNode* aNode) -{ - AudioBufferSourceNode* This = static_cast(aNode); - if (!This->mStream) { - return; - } - SendTimelineParameterToStream(This, PLAYBACKRATE, *This->mPlaybackRate); -} - -void -AudioBufferSourceNode::SendDetuneToStream(AudioNode* aNode) -{ - AudioBufferSourceNode* This = static_cast(aNode); - if (!This->mStream) { - return; - } - SendTimelineParameterToStream(This, DETUNE, *This->mDetune); -} - void AudioBufferSourceNode::SendDopplerShiftToStream(double aDopplerShift) { @@ -805,7 +823,7 @@ AudioBufferSourceNode::SendLoopParametersToStream() // looping impossible. SendInt32ParameterToStream(LOOP, 0); } - } else if (!mLoop) { + } else { SendInt32ParameterToStream(LOOP, 0); } } diff --git a/dom/media/webaudio/AudioBufferSourceNode.h b/dom/media/webaudio/AudioBufferSourceNode.h index f4d9b56194..0362a7072c 100644 --- a/dom/media/webaudio/AudioBufferSourceNode.h +++ b/dom/media/webaudio/AudioBufferSourceNode.h @@ -129,8 +129,6 @@ private: void SendLoopParametersToStream(); void SendBufferParameterToStream(JSContext* aCx); void SendOffsetAndDurationParametersToStream(AudioNodeStream* aStream); - static void SendPlaybackRateToStream(AudioNode* aNode); - static void SendDetuneToStream(AudioNode* aNode); private: double mLoopStart; diff --git a/dom/media/webaudio/AudioContext.cpp b/dom/media/webaudio/AudioContext.cpp index 9dfcac4ae5..7a87cb1cb8 100644 --- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -100,11 +100,11 @@ AudioContext::AudioContext(nsPIDOMWindow* aWindow, , mSampleRate(GetSampleRateForAudioContext(aIsOffline, aSampleRate)) , mAudioContextState(AudioContextState::Suspended) , mNumberOfChannels(aNumberOfChannels) - , mNodeCount(0) , mIsOffline(aIsOffline) , mIsStarted(!aIsOffline) , mIsShutDown(false) , mCloseCalled(false) + , mSuspendCalled(false) { bool mute = aWindow->AddAudioContext(this); @@ -661,8 +661,8 @@ double AudioContext::CurrentTime() const { MediaStream* stream = Destination()->Stream(); - return StreamTimeToDOMTime(stream-> - StreamTimeToSeconds(stream->GetCurrentTime())); + return stream->StreamTimeToSeconds(stream->GetCurrentTime() + + Destination()->ExtraCurrentTime()); } void @@ -686,11 +686,6 @@ AudioContext::Shutdown() } } -AudioContextState AudioContext::State() const -{ - return mAudioContextState; -} - StateChangeTask::StateChangeTask(AudioContext* aAudioContext, void* aPromise, AudioContextState aNewState) @@ -822,6 +817,19 @@ AudioContext::OnStateChanged(void* aPromise, AudioContextState aNewState) mAudioContextState = aNewState; } +nsTArray +AudioContext::GetAllStreams() const +{ + nsTArray streams; + for (auto iter = mAllNodes.ConstIter(); !iter.Done(); iter.Next()) { + MediaStream* s = iter.Get()->GetKey()->GetStream(); + if (s) { + streams.AppendElement(s); + } + } + return streams; +} + already_AddRefed AudioContext::Suspend(ErrorResult& aRv) { @@ -847,17 +855,24 @@ AudioContext::Suspend(ErrorResult& aRv) return promise.forget(); } - Destination()->DestroyAudioChannelAgent(); - - MediaStream* ds = DestinationStream(); - if (ds) { - ds->BlockStreamIfNeeded(); - } + Destination()->Suspend(); mPromiseGripArray.AppendElement(promise); + + nsTArray streams; + // If mSuspendCalled is true then we already suspended all our streams, + // so don't suspend them again (since suspend(); suspend(); resume(); should + // cancel both suspends). But we still need to do ApplyAudioContextOperation + // to ensure our new promise is resolved. + if (!mSuspendCalled) { + streams = GetAllStreams(); + } Graph()->ApplyAudioContextOperation(DestinationStream()->AsAudioNodeStream(), + streams, AudioContextOperation::Suspend, promise); + mSuspendCalled = true; + return promise.forget(); } @@ -887,17 +902,23 @@ AudioContext::Resume(ErrorResult& aRv) return promise.forget(); } - Destination()->CreateAudioChannelAgent(); + Destination()->Resume(); - MediaStream* ds = DestinationStream(); - if (ds) { - ds->UnblockStreamIfNeeded(); + nsTArray streams; + // If mSuspendCalled is false then we already resumed all our streams, + // so don't resume them again (since suspend(); resume(); resume(); should + // be OK). But we still need to do ApplyAudioContextOperation + // to ensure our new promise is resolved. + if (mSuspendCalled) { + streams = GetAllStreams(); } - mPromiseGripArray.AppendElement(promise); Graph()->ApplyAudioContextOperation(DestinationStream()->AsAudioNodeStream(), + streams, AudioContextOperation::Resume, promise); + mSuspendCalled = false; + return promise.forget(); } @@ -921,8 +942,6 @@ AudioContext::Close(ErrorResult& aRv) return promise.forget(); } - mCloseCalled = true; - if (Destination()) { Destination()->DestroyAudioChannelAgent(); } @@ -933,28 +952,43 @@ AudioContext::Close(ErrorResult& aRv) // this point, so we need extra null-checks. MediaStream* ds = DestinationStream(); if (ds) { - Graph()->ApplyAudioContextOperation(ds->AsAudioNodeStream(), - AudioContextOperation::Close, promise); - - if (ds) { - ds->BlockStreamIfNeeded(); + nsTArray streams; + // If mSuspendCalled or mCloseCalled are true then we already suspended + // all our streams, so don't suspend them again. But we still need to do + // ApplyAudioContextOperation to ensure our new promise is resolved. + if (!mSuspendCalled && !mCloseCalled) { + streams = GetAllStreams(); } + Graph()->ApplyAudioContextOperation(ds->AsAudioNodeStream(), streams, + AudioContextOperation::Close, promise); } + mCloseCalled = true; + return promise.forget(); } void -AudioContext::UpdateNodeCount(int32_t aDelta) +AudioContext::RegisterNode(AudioNode* aNode) { - bool firstNode = mNodeCount == 0; - mNodeCount += aDelta; - MOZ_ASSERT(mNodeCount >= 0); + MOZ_ASSERT(!mAllNodes.Contains(aNode)); + mAllNodes.PutEntry(aNode); // mDestinationNode may be null when we're destroying nodes unlinked by CC. // Skipping unnecessary calls after shutdown avoids RunInStableState events // getting stuck in CycleCollectedJSRuntime during final cycle collection // (bug 1200514). - if (!firstNode && mDestination && !mIsShutDown) { - mDestination->SetIsOnlyNodeForContext(mNodeCount == 1); + if (mDestination && !mIsShutDown) { + mDestination->SetIsOnlyNodeForContext(mAllNodes.Count() == 1); + } +} + +void +AudioContext::UnregisterNode(AudioNode* aNode) +{ + MOZ_ASSERT(mAllNodes.Contains(aNode)); + mAllNodes.RemoveEntry(aNode); + // mDestinationNode may be null when we're destroying nodes unlinked by CC + if (mDestination) { + mDestination->SetIsOnlyNodeForContext(mAllNodes.Count() == 1); } } @@ -1052,12 +1086,6 @@ AudioContext::CollectReports(nsIHandleReportCallback* aHandleReport, amount, "Memory used by AudioContext objects (Web Audio)."); } -double -AudioContext::ExtraCurrentTime() const -{ - return mDestination->ExtraCurrentTime(); -} - BasicWaveFormCache* AudioContext::GetBasicWaveFormCache() { diff --git a/dom/media/webaudio/AudioContext.h b/dom/media/webaudio/AudioContext.h index d9cf57b4ad..0331e6bf6f 100644 --- a/dom/media/webaudio/AudioContext.h +++ b/dom/media/webaudio/AudioContext.h @@ -29,7 +29,7 @@ namespace WebCore { class PeriodicWave; -}; +} // namespace WebCore class nsPIDOMWindow; @@ -111,7 +111,7 @@ private: AudioContextState mNewState; }; -enum AudioContextOperation { Suspend, Resume, Close }; +enum class AudioContextOperation { Suspend, Resume, Close }; class AudioContext final : public DOMEventTargetHelper, public nsIMemoryReporter @@ -175,16 +175,14 @@ public: return mSampleRate; } - AudioContextId Id() const - { - return mId; - } + bool ShouldSuspendNewStream() const { return mSuspendCalled; } double CurrentTime() const; AudioListener* Listener(); - AudioContextState State() const; + AudioContextState State() const { return mAudioContextState; } + // Those three methods return a promise to content, that is resolved when an // (possibly long) operation is completed on the MSG (and possibly other) // thread(s). To avoid having to match the calls and asychronous result when @@ -303,17 +301,8 @@ public: AudioChannel TestAudioChannelInAudioNodeStream(); - void UpdateNodeCount(int32_t aDelta); - - double DOMTimeToStreamTime(double aTime) const - { - return aTime - ExtraCurrentTime(); - } - - double StreamTimeToDOMTime(double aTime) const - { - return aTime + ExtraCurrentTime(); - } + void RegisterNode(AudioNode* aNode); + void UnregisterNode(AudioNode* aNode); void OnStateChanged(void* aPromise, AudioContextState aNewState); @@ -323,15 +312,6 @@ public: IMPL_EVENT_HANDLER(mozinterruptend) private: - /** - * Returns the amount of extra time added to the current time of the - * AudioDestinationNode's MediaStream to get this AudioContext's currentTime. - * Must be subtracted from all DOM API parameter times that are on the same - * timeline as AudioContext's currentTime to get times we can pass to the - * MediaStreamGraph. - */ - double ExtraCurrentTime() const; - void RemoveFromDecodeQueue(WebAudioDecodeJob* aDecodeJob); void ShutdownDecoder(); @@ -343,6 +323,8 @@ private: bool CheckClosed(ErrorResult& aRv); + nsTArray GetAllStreams() const; + private: // Each AudioContext has an id, that is passed down the MediaStreams that // back the AudioNodes, so we can easily compute the set of all the @@ -361,6 +343,8 @@ private: // See RegisterActiveNode. These will keep the AudioContext alive while it // is rendering and the window remains alive. nsTHashtable > mActiveNodes; + // Raw (non-owning) references to all AudioNodes for this AudioContext. + nsTHashtable > mAllNodes; // Hashsets containing all the PannerNodes, to compute the doppler shift. // These are weak pointers. nsTHashtable > mPannerNodes; @@ -368,13 +352,13 @@ private: RefPtr mBasicWaveFormCache; // Number of channels passed in the OfflineAudioContext ctor. uint32_t mNumberOfChannels; - // Number of nodes that currently exist for this AudioContext - int32_t mNodeCount; bool mIsOffline; bool mIsStarted; bool mIsShutDown; // Close has been called, reject suspend and resume call. bool mCloseCalled; + // Suspend has been called with no following resume. + bool mSuspendCalled; }; static const dom::AudioContext::AudioContextId NO_AUDIO_CONTEXT = 0; diff --git a/dom/media/webaudio/AudioDestinationNode.cpp b/dom/media/webaudio/AudioDestinationNode.cpp index 410e149441..105a44cc77 100644 --- a/dom/media/webaudio/AudioDestinationNode.cpp +++ b/dom/media/webaudio/AudioDestinationNode.cpp @@ -32,8 +32,6 @@ static uint8_t gWebAudioOutputKey; class OfflineDestinationNodeEngine final : public AudioNodeEngine { public: - typedef AutoFallibleTArray, 2> InputChannels; - OfflineDestinationNodeEngine(AudioDestinationNode* aNode, uint32_t aNumberOfChannels, uint32_t aLength, @@ -48,8 +46,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { // Do this just for the sake of political correctness; this output @@ -61,60 +60,40 @@ public: // These allocations might fail if content provides a huge number of // channels or size, but it's OK since we'll deal with the failure // gracefully. - if (mInputChannels.SetLength(mNumberOfChannels, fallible)) { - for (uint32_t i = 0; i < mNumberOfChannels; ++i) { - mInputChannels[i] = new (fallible) float[mLength]; - if (!mInputChannels[i]) { - mInputChannels.Clear(); - break; - } - } - } + mBuffer = ThreadSharedFloatArrayBufferList:: + Create(mNumberOfChannels, mLength, fallible); mBufferAllocated = true; } // Handle the case of allocation failure in the input buffer - if (mInputChannels.IsEmpty()) { - return; - } - - if (mWriteIndex >= mLength) { - NS_ASSERTION(mWriteIndex == mLength, "Overshot length"); - // Don't record any more. - return; - } + uint32_t outputChannelCount = mBuffer ? mNumberOfChannels : 0; // Record our input buffer MOZ_ASSERT(mWriteIndex < mLength, "How did this happen?"); const uint32_t duration = std::min(WEBAUDIO_BLOCK_SIZE, mLength - mWriteIndex); - const uint32_t commonChannelCount = std::min(mInputChannels.Length(), - aInput.mChannelData.Length()); - // First, copy as many channels in the input as we have - for (uint32_t i = 0; i < commonChannelCount; ++i) { - if (aInput.IsNull()) { - PodZero(mInputChannels[i] + mWriteIndex, duration); + const uint32_t inputChannelCount = aInput.ChannelCount(); + for (uint32_t i = 0; i < outputChannelCount; ++i) { + float* outputData = mBuffer->GetDataForWrite(i) + mWriteIndex; + if (aInput.IsNull() || i >= inputChannelCount) { + PodZero(outputData, duration); } else { const float* inputBuffer = static_cast(aInput.mChannelData[i]); if (duration == WEBAUDIO_BLOCK_SIZE) { // Use the optimized version of the copy with scale operation AudioBlockCopyChannelWithScale(inputBuffer, aInput.mVolume, - mInputChannels[i] + mWriteIndex); + outputData); } else { if (aInput.mVolume == 1.0f) { - PodCopy(mInputChannels[i] + mWriteIndex, inputBuffer, duration); + PodCopy(outputData, inputBuffer, duration); } else { for (uint32_t j = 0; j < duration; ++j) { - mInputChannels[i][mWriteIndex + j] = aInput.mVolume * inputBuffer[j]; + outputData[j] = aInput.mVolume * inputBuffer[j]; } } } } } - // Then, silence all of the remaining channels - for (uint32_t i = commonChannelCount; i < mInputChannels.Length(); ++i) { - PodZero(mInputChannels[i] + mWriteIndex, duration); - } mWriteIndex += duration; if (mWriteIndex >= mLength) { @@ -126,6 +105,14 @@ public: } } + virtual bool IsActive() const override + { + // Keep processing to track stream time, which is used for all timelines + // associated with the same AudioContext. + return true; + } + + class OnCompleteTask final : public nsRunnable { public: @@ -165,14 +152,11 @@ public: // Create the input buffer ErrorResult rv; RefPtr renderedBuffer = - AudioBuffer::Create(context, mInputChannels.Length(), - mLength, mSampleRate, cx, rv); + AudioBuffer::Create(context, mNumberOfChannels, mLength, mSampleRate, + mBuffer.forget(), cx, rv); if (rv.Failed()) { return; } - for (uint32_t i = 0; i < mInputChannels.Length(); ++i) { - renderedBuffer->SetRawChannelContents(i, mInputChannels[i]); - } aNode->ResolvePromise(renderedBuffer); @@ -186,7 +170,7 @@ public: virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); - amount += mInputChannels.ShallowSizeOfExcludingThis(aMallocSizeOf); + amount += mBuffer->SizeOfIncludingThis(aMallocSizeOf); return amount; } @@ -196,11 +180,11 @@ public: } private: - // The input to the destination node is recorded in the mInputChannels buffer. + // The input to the destination node is recorded in mBuffer. // When this buffer fills up with mLength frames, the buffered input is sent // to the main thread in order to dispatch OfflineAudioCompletionEvent. - InputChannels mInputChannels; - // An index representing the next offset in mInputChannels to be written to. + RefPtr mBuffer; + // An index representing the next offset in mBuffer to be written to. uint32_t mWriteIndex; uint32_t mNumberOfChannels; // How many frames the OfflineAudioContext intends to produce. @@ -243,19 +227,25 @@ public: explicit DestinationNodeEngine(AudioDestinationNode* aNode) : AudioNodeEngine(aNode) , mVolume(1.0f) - , mLastInputMuted(false) + , mLastInputMuted(true) + , mSuspended(false) { MOZ_ASSERT(aNode); } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { *aOutput = aInput; aOutput->mVolume *= mVolume; + if (mSuspended) { + return; + } + bool newInputMuted = aInput.IsNull() || aInput.IsMuted(); if (newInputMuted != mLastInputMuted) { mLastInputMuted = newInputMuted; @@ -267,6 +257,16 @@ public: } } + virtual bool IsActive() const override + { + // Keep processing to track stream time, which is used for all timelines + // associated with the same AudioContext. If there are no other engines + // for the AudioContext, then this could return false to suspend the + // stream, but the stream is blocked anyway through + // AudioDestinationNode::SetIsOnlyNodeForContext(). + return true; + } + virtual void SetDoubleParameter(uint32_t aIndex, double aParam) override { if (aIndex == VOLUME) { @@ -274,8 +274,19 @@ public: } } + virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) override + { + if (aIndex == SUSPENDED) { + mSuspended = !!aParam; + if (mSuspended) { + mLastInputMuted = true; + } + } + } + enum Parameters { VOLUME, + SUSPENDED, }; virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override @@ -286,6 +297,7 @@ public: private: float mVolume; bool mLastInputMuted; + bool mSuspended; }; static bool UseAudioChannelService() @@ -320,7 +332,6 @@ AudioDestinationNode::AudioDestinationNode(AudioContext* aContext, , mAudioChannel(AudioChannel::Normal) , mIsOffline(aIsOffline) , mAudioChannelAgentPlaying(false) - , mExtraCurrentTime(0) , mExtraCurrentTimeSinceLastStartedBlocking(0) , mExtraCurrentTimeUpdatedSinceLastStableState(false) , mCaptured(false) @@ -337,7 +348,7 @@ AudioDestinationNode::AudioDestinationNode(AudioContext* aContext, AudioNodeStream::NEED_MAIN_THREAD_CURRENT_TIME | AudioNodeStream::NEED_MAIN_THREAD_FINISHED | AudioNodeStream::EXTERNAL_OUTPUT; - mStream = AudioNodeStream::Create(graph, engine, flags); + mStream = AudioNodeStream::Create(aContext, engine, flags, graph); mStream->AddMainThreadListener(this); mStream->AddAudioOutput(&gWebAudioOutputKey); @@ -454,6 +465,20 @@ AudioDestinationNode::Unmute() SendDoubleParameterToStream(DestinationNodeEngine::VOLUME, 1.0f); } +void +AudioDestinationNode::Suspend() +{ + DestroyAudioChannelAgent(); + SendInt32ParameterToStream(DestinationNodeEngine::SUSPENDED, 1); +} + +void +AudioDestinationNode::Resume() +{ + CreateAudioChannelAgent(); + SendInt32ParameterToStream(DestinationNodeEngine::SUSPENDED, 0); +} + void AudioDestinationNode::OfflineShutdown() { @@ -616,10 +641,6 @@ AudioDestinationNode::CreateAudioChannelAgent() static_cast(mAudioChannel), this); - // The AudioChannelAgent must start playing immediately in order to avoid - // race conditions with mozinterruptbegin/end events. - InputMuted(false); - WindowAudioCaptureChanged(); } @@ -639,17 +660,20 @@ AudioDestinationNode::ScheduleStableStateNotification() nsContentUtils::RunInStableState(event.forget()); } -double +StreamTime AudioDestinationNode::ExtraCurrentTime() { if (!mStartedBlockingDueToBeingOnlyNode.IsNull() && !mExtraCurrentTimeUpdatedSinceLastStableState) { mExtraCurrentTimeUpdatedSinceLastStableState = true; - mExtraCurrentTimeSinceLastStartedBlocking = + // Round to nearest processing block. + double seconds = (TimeStamp::Now() - mStartedBlockingDueToBeingOnlyNode).ToSeconds(); + mExtraCurrentTimeSinceLastStartedBlocking = WEBAUDIO_BLOCK_SIZE * + StreamTime(seconds * Context()->SampleRate() / WEBAUDIO_BLOCK_SIZE + 0.5); ScheduleStableStateNotification(); } - return mExtraCurrentTime + mExtraCurrentTimeSinceLastStartedBlocking; + return mExtraCurrentTimeSinceLastStartedBlocking; } void @@ -674,7 +698,7 @@ AudioDestinationNode::SetIsOnlyNodeForContext(bool aIsOnlyNode) } if (aIsOnlyNode) { - mStream->ChangeExplicitBlockerCount(1); + mStream->Suspend(); mStartedBlockingDueToBeingOnlyNode = TimeStamp::Now(); // Don't do an update of mExtraCurrentTimeSinceLastStartedBlocking until the next stable state. mExtraCurrentTimeUpdatedSinceLastStableState = true; @@ -682,9 +706,8 @@ AudioDestinationNode::SetIsOnlyNodeForContext(bool aIsOnlyNode) } else { // Force update of mExtraCurrentTimeSinceLastStartedBlocking if necessary ExtraCurrentTime(); - mExtraCurrentTime += mExtraCurrentTimeSinceLastStartedBlocking; + mStream->AdvanceAndResume(mExtraCurrentTimeSinceLastStartedBlocking); mExtraCurrentTimeSinceLastStartedBlocking = 0; - mStream->ChangeExplicitBlockerCount(-1); mStartedBlockingDueToBeingOnlyNode = TimeStamp(); } } @@ -695,7 +718,10 @@ AudioDestinationNode::InputMuted(bool aMuted) MOZ_ASSERT(Context() && !Context()->IsOffline()); if (!mAudioChannelAgent) { - return; + if (aMuted) { + return; + } + CreateAudioChannelAgent(); } if (aMuted) { diff --git a/dom/media/webaudio/AudioDestinationNode.h b/dom/media/webaudio/AudioDestinationNode.h index ceec936b1d..dc88cb6587 100644 --- a/dom/media/webaudio/AudioDestinationNode.h +++ b/dom/media/webaudio/AudioDestinationNode.h @@ -53,6 +53,9 @@ public: void Mute(); void Unmute(); + void Suspend(); + void Resume(); + void StartRendering(Promise* aPromise); void OfflineShutdown(); @@ -64,7 +67,7 @@ public: // An amount that should be added to the MediaStream's current time to // get the AudioContext.currentTime. - double ExtraCurrentTime(); + StreamTime ExtraCurrentTime(); // When aIsOnlyNode is true, this is the only node for the AudioContext. void SetIsOnlyNodeForContext(bool aIsOnlyNode); @@ -109,8 +112,7 @@ private: bool mAudioChannelAgentPlaying; TimeStamp mStartedBlockingDueToBeingOnlyNode; - double mExtraCurrentTime; - double mExtraCurrentTimeSinceLastStartedBlocking; + StreamTime mExtraCurrentTimeSinceLastStartedBlocking; bool mExtraCurrentTimeUpdatedSinceLastStableState; bool mCaptured; }; diff --git a/dom/media/webaudio/AudioEventTimeline.h b/dom/media/webaudio/AudioEventTimeline.h index eb8fc75e36..121637471a 100644 --- a/dom/media/webaudio/AudioEventTimeline.h +++ b/dom/media/webaudio/AudioEventTimeline.h @@ -12,28 +12,34 @@ #include "mozilla/FloatingPoint.h" #include "mozilla/PodOperations.h" +#include "MainThreadUtils.h" #include "nsTArray.h" #include "math.h" #include "WebAudioUtils.h" namespace mozilla { +class MediaStream; + namespace dom { -// This is an internal helper class and should not be used outside of this header. struct AudioTimelineEvent final { enum Type : uint32_t { SetValue, + SetValueAtTime, LinearRamp, ExponentialRamp, SetTarget, - SetValueCurve + SetValueCurve, + Stream, + Cancel }; AudioTimelineEvent(Type aType, double aTime, float aValue, double aTimeConstant = 0.0, - float aDuration = 0.0, const float* aCurve = nullptr, uint32_t aCurveLength = 0) + float aDuration = 0.0, const float* aCurve = nullptr, + uint32_t aCurveLength = 0) : mType(aType) , mTimeConstant(aTimeConstant) , mDuration(aDuration) @@ -49,11 +55,23 @@ struct AudioTimelineEvent final } } + explicit AudioTimelineEvent(MediaStream* aStream) + : mType(Stream) + , mStream(aStream) +#ifdef DEBUG + , mTimeIsInTicks(false) +#endif + { + } + AudioTimelineEvent(const AudioTimelineEvent& rhs) { PodCopy(this, &rhs, 1); + if (rhs.mType == AudioTimelineEvent::SetValueCurve) { SetCurveParams(rhs.mCurve, rhs.mCurveLength); + } else if (rhs.mType == AudioTimelineEvent::Stream) { + new (&mStream) decltype(mStream)(rhs.mStream); } } @@ -64,30 +82,6 @@ struct AudioTimelineEvent final } } - bool IsValid() const - { - if (mType == AudioTimelineEvent::SetValueCurve) { - if (!mCurve || !mCurveLength) { - return false; - } - for (uint32_t i = 0; i < mCurveLength; ++i) { - if (!IsValid(mCurve[i])) { - return false; - } - } - } - - if (mType == AudioTimelineEvent::SetTarget && - WebAudioUtils::FuzzyEqual(mTimeConstant, 0.0)) { - return false; - } - - return IsValid(mTime) && - IsValid(mValue) && - IsValid(mTimeConstant) && - IsValid(mDuration); - } - template TimeType Time() const; @@ -114,6 +108,22 @@ struct AudioTimelineEvent final float mValue; uint32_t mCurveLength; }; + // mCurve contains a buffer of SetValueCurve samples. We sample the + // values in the buffer depending on how far along we are in time. + // If we're at time T and the event has started as time T0 and has a + // duration of D, we sample the buffer at floor(mCurveLength*(T-T0)/D) + // if T mStream; + double mTimeConstant; + double mDuration; +#ifdef DEBUG + bool mTimeIsInTicks; +#endif + +private: + // This member is accessed using the `Time` method, for safety. + // // The time for an event can either be in absolute value or in ticks. // Initially the time of the event is always in absolute value. // In order to convert it to ticks, call SetTimeInTicks. Once this @@ -123,23 +133,6 @@ struct AudioTimelineEvent final double mTime; int64_t mTimeInTicks; }; - // mCurve contains a buffer of SetValueCurve samples. We sample the - // values in the buffer depending on how far along we are in time. - // If we're at time T and the event has started as time T0 and has a - // duration of D, we sample the buffer at floor(mCurveLength*(T-T0)/D) - // if T @@ -152,6 +145,7 @@ inline double AudioTimelineEvent::Time() const template <> inline int64_t AudioTimelineEvent::Time() const { + MOZ_ASSERT(!NS_IsMainThread()); MOZ_ASSERT(mTimeIsInTicks); return mTimeInTicks; } @@ -171,7 +165,119 @@ public: : mValue(aDefaultValue), mComputedValue(aDefaultValue), mLastComputedValue(aDefaultValue) + { } + + bool ValidateEvent(AudioTimelineEvent& aEvent, + ErrorResult& aRv) { + MOZ_ASSERT(NS_IsMainThread()); + + // Validate the event itself + if (!WebAudioUtils::IsTimeValid(aEvent.template Time()) || + !WebAudioUtils::IsTimeValid(aEvent.mTimeConstant)) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + + if (aEvent.mType == AudioTimelineEvent::SetValueCurve) { + if (!aEvent.mCurve || !aEvent.mCurveLength) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + for (uint32_t i = 0; i < aEvent.mCurveLength; ++i) { + if (!IsValid(aEvent.mCurve[i])) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + } + } + + if (aEvent.mType == AudioTimelineEvent::SetTarget && + WebAudioUtils::FuzzyEqual(aEvent.mTimeConstant, 0.0)) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + + bool timeAndValueValid = IsValid(aEvent.mValue) && + IsValid(aEvent.mDuration); + if (!timeAndValueValid) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + + // Make sure that non-curve events don't fall within the duration of a + // curve event. + for (unsigned i = 0; i < mEvents.Length(); ++i) { + if (mEvents[i].mType == AudioTimelineEvent::SetValueCurve && + mEvents[i].template Time() <= aEvent.template Time() && + (mEvents[i].template Time() + mEvents[i].mDuration) >= aEvent.template Time()) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + } + + // Make sure that curve events don't fall in a range which includes other + // events. + if (aEvent.mType == AudioTimelineEvent::SetValueCurve) { + for (unsigned i = 0; i < mEvents.Length(); ++i) { + if (mEvents[i].template Time() > aEvent.template Time() && + mEvents[i].template Time() < (aEvent.template Time() + aEvent.mDuration)) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + } + } + + // Make sure that invalid values are not used for exponential curves + if (aEvent.mType == AudioTimelineEvent::ExponentialRamp) { + if (aEvent.mValue <= 0.f) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + const AudioTimelineEvent* previousEvent = GetPreviousEvent(aEvent.template Time()); + if (previousEvent) { + if (previousEvent->mValue <= 0.f) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + } else { + if (mValue <= 0.f) { + aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); + return false; + } + } + } + return true; + } + + template + void InsertEvent(const AudioTimelineEvent& aEvent) + { + for (unsigned i = 0; i < mEvents.Length(); ++i) { + if (aEvent.template Time() == mEvents[i].template Time()) { + if (aEvent.mType == mEvents[i].mType) { + // If times and types are equal, replace the event + mEvents.ReplaceElementAt(i, aEvent); + } else { + // Otherwise, place the element after the last event of another type + do { + ++i; + } while (i < mEvents.Length() && + aEvent.mType != mEvents[i].mType && + aEvent.template Time() == mEvents[i].template Time()); + mEvents.InsertElementAt(i, aEvent); + } + return; + } + // Otherwise, place the event right after the latest existing event + if (aEvent.template Time() < mEvents[i].template Time()) { + mEvents.InsertElementAt(i, aEvent); + return; + } + } + + // If we couldn't find a place for the event, just append it to the list + mEvents.AppendElement(aEvent); } bool HasSimpleValue() const @@ -202,38 +308,58 @@ public: void SetValueAtTime(float aValue, double aStartTime, ErrorResult& aRv) { - InsertEvent(AudioTimelineEvent(AudioTimelineEvent::SetValue, aStartTime, aValue), aRv); + AudioTimelineEvent event(AudioTimelineEvent::SetValueAtTime, aStartTime, aValue); + + if (ValidateEvent(event, aRv)) { + InsertEvent(event); + } } void LinearRampToValueAtTime(float aValue, double aEndTime, ErrorResult& aRv) { - InsertEvent(AudioTimelineEvent(AudioTimelineEvent::LinearRamp, aEndTime, aValue), aRv); + AudioTimelineEvent event(AudioTimelineEvent::LinearRamp, aEndTime, aValue); + + if (ValidateEvent(event, aRv)) { + InsertEvent(event); + } } void ExponentialRampToValueAtTime(float aValue, double aEndTime, ErrorResult& aRv) { - InsertEvent(AudioTimelineEvent(AudioTimelineEvent::ExponentialRamp, aEndTime, aValue), aRv); + AudioTimelineEvent event(AudioTimelineEvent::ExponentialRamp, aEndTime, aValue); + + if (ValidateEvent(event, aRv)) { + InsertEvent(event); + } } void SetTargetAtTime(float aTarget, double aStartTime, double aTimeConstant, ErrorResult& aRv) { - InsertEvent(AudioTimelineEvent(AudioTimelineEvent::SetTarget, aStartTime, aTarget, aTimeConstant), aRv); + AudioTimelineEvent event(AudioTimelineEvent::SetTarget, aStartTime, aTarget, aTimeConstant); + + if (ValidateEvent(event, aRv)) { + InsertEvent(event); + } } void SetValueCurveAtTime(const float* aValues, uint32_t aValuesLength, double aStartTime, double aDuration, ErrorResult& aRv) { - InsertEvent(AudioTimelineEvent(AudioTimelineEvent::SetValueCurve, aStartTime, 0.0f, 0.0f, aDuration, aValues, aValuesLength), aRv); + AudioTimelineEvent event(AudioTimelineEvent::SetValueCurve, aStartTime, 0.0f, 0.0f, aDuration, aValues, aValuesLength); + if (ValidateEvent(event, aRv)) { + InsertEvent(event); + } } - void CancelScheduledValues(double aStartTime) + template + void CancelScheduledValues(TimeType aStartTime) { for (unsigned i = 0; i < mEvents.Length(); ++i) { - if (mEvents[i].mTime >= aStartTime) { + if (mEvents[i].template Time() >= aStartTime) { #ifdef DEBUG // Sanity check: the array should be sorted, so all of the following // events should have a time greater than aStartTime too. for (unsigned j = i + 1; j < mEvents.Length(); ++j) { - MOZ_ASSERT(mEvents[j].mTime >= aStartTime); + MOZ_ASSERT(mEvents[j].template Time() >= aStartTime); } #endif mEvents.TruncateLength(i); @@ -263,143 +389,100 @@ public: template float GetValueAtTime(TimeType aTime) { - mComputedValue = GetValueAtTimeHelper(aTime); + GetValuesAtTimeHelper(aTime, &mComputedValue, 1); return mComputedValue; } + template + void GetValuesAtTime(TimeType aTime, float* aBuffer, const size_t aSize) + { + MOZ_ASSERT(aBuffer); + GetValuesAtTimeHelper(aTime, aBuffer, aSize); + mComputedValue = aBuffer[aSize - 1]; + } + // This method computes the AudioParam value at a given time based on the event timeline template - float GetValueAtTimeHelper(TimeType aTime) + void GetValuesAtTimeHelper(TimeType aTime, float* aBuffer, const size_t aSize) { + MOZ_ASSERT(aBuffer); + MOZ_ASSERT(aSize); + + size_t lastEventId = 0; const AudioTimelineEvent* previous = nullptr; const AudioTimelineEvent* next = nullptr; - bool bailOut = false; - for (unsigned i = 0; !bailOut && i < mEvents.Length(); ++i) { - switch (mEvents[i].mType) { - case AudioTimelineEvent::SetValue: - case AudioTimelineEvent::SetTarget: - case AudioTimelineEvent::LinearRamp: - case AudioTimelineEvent::ExponentialRamp: - case AudioTimelineEvent::SetValueCurve: - if (TimesEqual(aTime, mEvents[i].template Time())) { + + // Let's remove old events except the last one: we need it to calculate some curves. + while (mEvents.Length() > 1 && + aTime > mEvents[1].template Time()) { + mEvents.RemoveElementAt(0); + } + + for (size_t bufferIndex = 0; bufferIndex < aSize; ++bufferIndex, ++aTime) { + for (; !bailOut && lastEventId < mEvents.Length(); ++lastEventId) { + +#ifdef DEBUG + const AudioTimelineEvent* current = &mEvents[lastEventId]; + MOZ_ASSERT(current->mType == AudioTimelineEvent::SetValueAtTime || + current->mType == AudioTimelineEvent::SetTarget || + current->mType == AudioTimelineEvent::LinearRamp || + current->mType == AudioTimelineEvent::ExponentialRamp || + current->mType == AudioTimelineEvent::SetValueCurve); +#endif + + if (TimesEqual(aTime, mEvents[lastEventId].template Time())) { mLastComputedValue = mComputedValue; // Find the last event with the same time - do { - ++i; - } while (i < mEvents.Length() && - aTime == mEvents[i].template Time()); - - // SetTarget nodes can be handled no matter what their next node is (if they have one) - if (mEvents[i - 1].mType == AudioTimelineEvent::SetTarget) { - // Follow the curve, without regard to the next event, starting at - // the last value of the last event. - return ExponentialApproach(mEvents[i - 1].template Time(), - mLastComputedValue, mEvents[i - 1].mValue, - mEvents[i - 1].mTimeConstant, aTime); + while (lastEventId < mEvents.Length() - 1 && + TimesEqual(aTime, mEvents[lastEventId + 1].template Time())) { + ++lastEventId; } - - // SetValueCurve events can be handled no matter what their event node is (if they have one) - if (mEvents[i - 1].mType == AudioTimelineEvent::SetValueCurve) { - return ExtractValueFromCurve(mEvents[i - 1].template Time(), - mEvents[i - 1].mCurve, - mEvents[i - 1].mCurveLength, - mEvents[i - 1].mDuration, aTime); - } - - // For other event types - return mEvents[i - 1].mValue; + break; } + previous = next; - next = &mEvents[i]; - if (aTime < mEvents[i].template Time()) { + next = &mEvents[lastEventId]; + if (aTime < mEvents[lastEventId].template Time()) { bailOut = true; } - break; - default: - MOZ_ASSERT(false, "unreached"); + } + + // The time was found in the list of events. + if (!bailOut && lastEventId < mEvents.Length()) { + MOZ_ASSERT(TimesEqual(aTime, mEvents[lastEventId].template Time())); + + // SetTarget nodes can be handled no matter what their next node is (if they have one) + if (mEvents[lastEventId].mType == AudioTimelineEvent::SetTarget) { + // Follow the curve, without regard to the next event, starting at + // the last value of the last event. + aBuffer[bufferIndex] = ExponentialApproach(mEvents[lastEventId].template Time(), + mLastComputedValue, mEvents[lastEventId].mValue, + mEvents[lastEventId].mTimeConstant, aTime); + continue; + } + + // SetValueCurve events can be handled no matter what their event node is (if they have one) + if (mEvents[lastEventId].mType == AudioTimelineEvent::SetValueCurve) { + aBuffer[bufferIndex] = ExtractValueFromCurve(mEvents[lastEventId].template Time(), + mEvents[lastEventId].mCurve, + mEvents[lastEventId].mCurveLength, + mEvents[lastEventId].mDuration, aTime); + continue; + } + + // For other event types + aBuffer[bufferIndex] = mEvents[lastEventId].mValue; + continue; + } + + // Handle the case where the time is past all of the events + if (!bailOut) { + aBuffer[bufferIndex] = GetValuesAtTimeHelperInternal(aTime, next, nullptr); + } else { + aBuffer[bufferIndex] = GetValuesAtTimeHelperInternal(aTime, previous, next); } } - // Handle the case where the time is past all of the events - if (!bailOut) { - previous = next; - next = nullptr; - } - - // Just return the default value if we did not find anything - if (!previous && !next) { - return mValue; - } - - // If the requested time is before all of the existing events - if (!previous) { - return mValue; - } - - // SetTarget nodes can be handled no matter what their next node is (if they have one) - if (previous->mType == AudioTimelineEvent::SetTarget) { - return ExponentialApproach(previous->template Time(), - mLastComputedValue, previous->mValue, - previous->mTimeConstant, aTime); - } - - // SetValueCurve events can be handled no mattar what their next node is (if they have one) - if (previous->mType == AudioTimelineEvent::SetValueCurve) { - return ExtractValueFromCurve(previous->template Time(), - previous->mCurve, previous->mCurveLength, - previous->mDuration, aTime); - } - - // If the requested time is after all of the existing events - if (!next) { - switch (previous->mType) { - case AudioTimelineEvent::SetValue: - case AudioTimelineEvent::LinearRamp: - case AudioTimelineEvent::ExponentialRamp: - // The value will be constant after the last event - return previous->mValue; - case AudioTimelineEvent::SetValueCurve: - return ExtractValueFromCurve(previous->template Time(), - previous->mCurve, previous->mCurveLength, - previous->mDuration, aTime); - case AudioTimelineEvent::SetTarget: - MOZ_ASSERT(false, "unreached"); - } - MOZ_ASSERT(false, "unreached"); - } - - // Finally, handle the case where we have both a previous and a next event - - // First, handle the case where our range ends up in a ramp event - switch (next->mType) { - case AudioTimelineEvent::LinearRamp: - return LinearInterpolate(previous->template Time(), previous->mValue, next->template Time(), next->mValue, aTime); - case AudioTimelineEvent::ExponentialRamp: - return ExponentialInterpolate(previous->template Time(), previous->mValue, next->template Time(), next->mValue, aTime); - case AudioTimelineEvent::SetValue: - case AudioTimelineEvent::SetTarget: - case AudioTimelineEvent::SetValueCurve: - break; - } - - // Now handle all other cases - switch (previous->mType) { - case AudioTimelineEvent::SetValue: - case AudioTimelineEvent::LinearRamp: - case AudioTimelineEvent::ExponentialRamp: - // If the next event type is neither linear or exponential ramp, the - // value is constant. - return previous->mValue; - case AudioTimelineEvent::SetValueCurve: - return ExtractValueFromCurve(previous->template Time(), - previous->mCurve, previous->mCurveLength, - previous->mDuration, aTime); - case AudioTimelineEvent::SetTarget: - MOZ_ASSERT(false, "unreached"); - } - - MOZ_ASSERT(false, "unreached"); - return 0.0f; } // Return the number of events scheduled @@ -436,17 +519,104 @@ public: return aCurve[uint32_t(aCurveLength * ratio)]; } - void ConvertEventTimesToTicks(int64_t (*aConvertor)(double aTime, void* aClosure), void* aClosure, - int32_t aSampleRate) + template + float GetValuesAtTimeHelperInternal(TimeType aTime, + const AudioTimelineEvent* aPrevious, + const AudioTimelineEvent* aNext) { - for (unsigned i = 0; i < mEvents.Length(); ++i) { - mEvents[i].SetTimeInTicks(aConvertor(mEvents[i].template Time(), aClosure)); - mEvents[i].mTimeConstant *= aSampleRate; - mEvents[i].mDuration *= aSampleRate; + // If the requested time is before all of the existing events + if (!aPrevious) { + return mValue; } + + // SetTarget nodes can be handled no matter what their next node is (if + // they have one) + if (aPrevious->mType == AudioTimelineEvent::SetTarget) { + return ExponentialApproach(aPrevious->template Time(), + mLastComputedValue, aPrevious->mValue, + aPrevious->mTimeConstant, aTime); + } + + // SetValueCurve events can be handled no mattar what their next node is + // (if they have one) + if (aPrevious->mType == AudioTimelineEvent::SetValueCurve) { + return ExtractValueFromCurve(aPrevious->template Time(), + aPrevious->mCurve, aPrevious->mCurveLength, + aPrevious->mDuration, aTime); + } + + // If the requested time is after all of the existing events + if (!aNext) { + switch (aPrevious->mType) { + case AudioTimelineEvent::SetValueAtTime: + case AudioTimelineEvent::LinearRamp: + case AudioTimelineEvent::ExponentialRamp: + // The value will be constant after the last event + return aPrevious->mValue; + case AudioTimelineEvent::SetValueCurve: + return ExtractValueFromCurve(aPrevious->template Time(), + aPrevious->mCurve, aPrevious->mCurveLength, + aPrevious->mDuration, aTime); + case AudioTimelineEvent::SetTarget: + MOZ_ASSERT(false, "unreached"); + case AudioTimelineEvent::SetValue: + case AudioTimelineEvent::Cancel: + case AudioTimelineEvent::Stream: + MOZ_ASSERT(false, "Should have been handled earlier."); + } + MOZ_ASSERT(false, "unreached"); + } + + // Finally, handle the case where we have both a previous and a next event + + // First, handle the case where our range ends up in a ramp event + switch (aNext->mType) { + case AudioTimelineEvent::LinearRamp: + return LinearInterpolate(aPrevious->template Time(), + aPrevious->mValue, + aNext->template Time(), + aNext->mValue, aTime); + + case AudioTimelineEvent::ExponentialRamp: + return ExponentialInterpolate(aPrevious->template Time(), + aPrevious->mValue, + aNext->template Time(), + aNext->mValue, aTime); + + case AudioTimelineEvent::SetValueAtTime: + case AudioTimelineEvent::SetTarget: + case AudioTimelineEvent::SetValueCurve: + break; + case AudioTimelineEvent::SetValue: + case AudioTimelineEvent::Cancel: + case AudioTimelineEvent::Stream: + MOZ_ASSERT(false, "Should have been handled earlier."); + } + + // Now handle all other cases + switch (aPrevious->mType) { + case AudioTimelineEvent::SetValueAtTime: + case AudioTimelineEvent::LinearRamp: + case AudioTimelineEvent::ExponentialRamp: + // If the next event type is neither linear or exponential ramp, the + // value is constant. + return aPrevious->mValue; + case AudioTimelineEvent::SetValueCurve: + return ExtractValueFromCurve(aPrevious->template Time(), + aPrevious->mCurve, aPrevious->mCurveLength, + aPrevious->mDuration, aTime); + case AudioTimelineEvent::SetTarget: + MOZ_ASSERT(false, "unreached"); + case AudioTimelineEvent::SetValue: + case AudioTimelineEvent::Cancel: + case AudioTimelineEvent::Stream: + MOZ_ASSERT(false, "Should have been handled earlier."); + } + + MOZ_ASSERT(false, "unreached"); + return 0.0f; } -private: const AudioTimelineEvent* GetPreviousEvent(double aTime) const { const AudioTimelineEvent* previous = nullptr; @@ -455,22 +625,22 @@ private: bool bailOut = false; for (unsigned i = 0; !bailOut && i < mEvents.Length(); ++i) { switch (mEvents[i].mType) { - case AudioTimelineEvent::SetValue: + case AudioTimelineEvent::SetValueAtTime: case AudioTimelineEvent::SetTarget: case AudioTimelineEvent::LinearRamp: case AudioTimelineEvent::ExponentialRamp: case AudioTimelineEvent::SetValueCurve: - if (aTime == mEvents[i].mTime) { + if (aTime == mEvents[i].template Time()) { // Find the last event with the same time do { ++i; } while (i < mEvents.Length() && - aTime == mEvents[i].mTime); + aTime == mEvents[i].template Time()); return &mEvents[i - 1]; } previous = next; next = &mEvents[i]; - if (aTime < mEvents[i].mTime) { + if (aTime < mEvents[i].template Time()) { bailOut = true; } break; @@ -485,85 +655,12 @@ private: return previous; } - - void InsertEvent(const AudioTimelineEvent& aEvent, ErrorResult& aRv) +private: + static bool IsValid(double value) { - if (!aEvent.IsValid()) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - - // Make sure that non-curve events don't fall within the duration of a - // curve event. - for (unsigned i = 0; i < mEvents.Length(); ++i) { - if (mEvents[i].mType == AudioTimelineEvent::SetValueCurve && - mEvents[i].mTime <= aEvent.mTime && - (mEvents[i].mTime + mEvents[i].mDuration) >= aEvent.mTime) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - } - - // Make sure that curve events don't fall in a range which includes other - // events. - if (aEvent.mType == AudioTimelineEvent::SetValueCurve) { - for (unsigned i = 0; i < mEvents.Length(); ++i) { - if (mEvents[i].mTime > aEvent.mTime && - mEvents[i].mTime < (aEvent.mTime + aEvent.mDuration)) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - } - } - - // Make sure that invalid values are not used for exponential curves - if (aEvent.mType == AudioTimelineEvent::ExponentialRamp) { - if (aEvent.mValue <= 0.f) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - const AudioTimelineEvent* previousEvent = GetPreviousEvent(aEvent.mTime); - if (previousEvent) { - if (previousEvent->mValue <= 0.f) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - } else { - if (mValue <= 0.f) { - aRv.Throw(NS_ERROR_DOM_SYNTAX_ERR); - return; - } - } - } - - for (unsigned i = 0; i < mEvents.Length(); ++i) { - if (aEvent.mTime == mEvents[i].mTime) { - if (aEvent.mType == mEvents[i].mType) { - // If times and types are equal, replace the event - mEvents.ReplaceElementAt(i, aEvent); - } else { - // Otherwise, place the element after the last event of another type - do { - ++i; - } while (i < mEvents.Length() && - aEvent.mType != mEvents[i].mType && - aEvent.mTime == mEvents[i].mTime); - mEvents.InsertElementAt(i, aEvent); - } - return; - } - // Otherwise, place the event right after the latest existing event - if (aEvent.mTime < mEvents[i].mTime) { - mEvents.InsertElementAt(i, aEvent); - return; - } - } - - // If we couldn't find a place for the event, just append it to the list - mEvents.AppendElement(aEvent); + return mozilla::IsFinite(value); } -private: // This is a sorted array of the events in the timeline. Queries of this // data structure should probably be more frequent than modifications to it, // and that is the reason why we're using a simple array as the data structure. @@ -581,4 +678,3 @@ private: } // namespace mozilla #endif - diff --git a/dom/media/webaudio/AudioNode.cpp b/dom/media/webaudio/AudioNode.cpp index 55ff4d87ad..aa2c94902f 100644 --- a/dom/media/webaudio/AudioNode.cpp +++ b/dom/media/webaudio/AudioNode.cpp @@ -23,7 +23,7 @@ NS_IMPL_CYCLE_COLLECTION_CLASS(AudioNode) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(AudioNode, DOMEventTargetHelper) tmp->DisconnectFromGraph(); if (tmp->mContext) { - tmp->mContext->UpdateNodeCount(-1); + tmp->mContext->UnregisterNode(tmp); } NS_IMPL_CYCLE_COLLECTION_UNLINK(mContext) NS_IMPL_CYCLE_COLLECTION_UNLINK(mOutputNodes) @@ -37,19 +37,7 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(AudioNode, NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_ADDREF_INHERITED(AudioNode, DOMEventTargetHelper) - -NS_IMETHODIMP_(MozExternalRefCountType) -AudioNode::Release() -{ - if (mRefCnt.get() == 1) { - // We are about to be deleted, disconnect the object from the graph before - // the derived type is destroyed. - DisconnectFromGraph(); - } - nsrefcnt r = DOMEventTargetHelper::Release(); - NS_LOG_RELEASE(this, r, "AudioNode"); - return r; -} +NS_IMPL_RELEASE_INHERITED(AudioNode, DOMEventTargetHelper) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(AudioNode) NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference) @@ -69,7 +57,7 @@ AudioNode::AudioNode(AudioContext* aContext, { MOZ_ASSERT(aContext); DOMEventTargetHelper::BindToOwner(aContext->GetParentObject()); - aContext->UpdateNodeCount(1); + aContext->RegisterNode(this); } AudioNode::~AudioNode() @@ -80,7 +68,7 @@ AudioNode::~AudioNode() MOZ_ASSERT(!mStream, "The webaudio-node-demise notification must have been sent"); if (mContext) { - mContext->UpdateNodeCount(-1); + mContext->UnregisterNode(this); } } @@ -145,9 +133,8 @@ FindIndexOfNodeWithPorts(const nsTArray& aInputNodes, const AudioNode void AudioNode::DisconnectFromGraph() { - // Addref this temporarily so the refcount bumping below doesn't destroy us - // prematurely - RefPtr kungFuDeathGrip = this; + MOZ_ASSERT(mRefCnt.get() > mInputNodes.Length(), + "Caller should be holding a reference"); // The idea here is that we remove connections one by one, and at each step // the graph is in a valid state. @@ -168,6 +155,8 @@ AudioNode::DisconnectFromGraph() // It doesn't matter which one we remove, since we're going to remove all // entries for this node anyway. output->mInputNodes.RemoveElementAt(inputIndex); + // This effects of this connection will remain. + output->NotifyHasPhantomInput(); } while (!mOutputParams.IsEmpty()) { @@ -218,10 +207,11 @@ AudioNode::Connect(AudioNode& aDestination, uint32_t aOutput, MOZ_ASSERT(aInput <= UINT16_MAX, "Unexpected large input port number"); MOZ_ASSERT(aOutput <= UINT16_MAX, "Unexpected large output port number"); input->mStreamPort = destinationStream-> - AllocateInputPort(mStream, MediaInputPort::FLAG_BLOCK_INPUT, - static_cast(aInput), - static_cast(aOutput)); + AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK, + static_cast(aInput), + static_cast(aOutput)); } + aDestination.NotifyInputsChanged(); // This connection may have connected a panner and a source. Context()->UpdatePannerSource(); @@ -262,7 +252,7 @@ AudioNode::Connect(AudioParam& aDestination, uint32_t aOutput, // Setup our stream as an input to the AudioParam's stream MOZ_ASSERT(aOutput <= UINT16_MAX, "Unexpected large output port number"); input->mStreamPort = - ps->AllocateInputPort(mStream, MediaInputPort::FLAG_BLOCK_INPUT, + ps->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK, 0, static_cast(aOutput)); } } @@ -291,18 +281,10 @@ AudioNode::SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& void AudioNode::SendChannelMixingParametersToStream() { - MOZ_ASSERT(mStream, "How come we don't have a stream here?"); - mStream->SetChannelMixingParameters(mChannelCount, mChannelCountMode, - mChannelInterpretation); -} - -void -AudioNode::SendTimelineParameterToStream(AudioNode* aNode, uint32_t aIndex, - const AudioParamTimeline& aValue) -{ - AudioNodeStream* ns = aNode->mStream; - MOZ_ASSERT(ns, "How come we don't have a stream here?"); - ns->SetTimelineParameter(aIndex, aValue); + if (mStream) { + mStream->SetChannelMixingParameters(mChannelCount, mChannelCountMode, + mChannelInterpretation); + } } void @@ -347,6 +329,7 @@ AudioNode::Disconnect(uint32_t aOutput, ErrorResult& aRv) // could be for different output ports. RefPtr output = mOutputNodes[i].forget(); mOutputNodes.RemoveElementAt(i); + output->NotifyInputsChanged(); if (mStream) { RefPtr runnable = new RunnableRelease(output.forget()); mStream->RunAfterPendingUpdates(runnable.forget()); @@ -416,8 +399,9 @@ AudioNode::SetPassThrough(bool aPassThrough) { MOZ_ASSERT(NumberOfInputs() <= 1 && NumberOfOutputs() == 1); mPassThrough = aPassThrough; - MOZ_ASSERT(mStream, "How come we don't have a stream here?"); - mStream->SetPassThrough(mPassThrough); + if (mStream) { + mStream->SetPassThrough(mPassThrough); + } } } // namespace dom diff --git a/dom/media/webaudio/AudioNode.h b/dom/media/webaudio/AudioNode.h index a343ab4eb8..9c172f8769 100644 --- a/dom/media/webaudio/AudioNode.h +++ b/dom/media/webaudio/AudioNode.h @@ -97,6 +97,15 @@ public: virtual void Disconnect(uint32_t aOutput, ErrorResult& aRv); + // Called after input nodes have been explicitly added or removed through + // the Connect() or Disconnect() methods. + virtual void NotifyInputsChanged() {} + // Indicate that the node should continue indefinitely to behave as if an + // input is connected, even though there is no longer a corresponding entry + // in mInputNodes. Called after an input node has been removed because it + // is being garbage collected. + virtual void NotifyHasPhantomInput() {} + // The following two virtual methods must be implemented by each node type // to provide their number of input and output ports. These numbers are // constant for the lifetime of the node. Both default to 1. @@ -168,7 +177,7 @@ public: }; // Returns the stream, if any. - AudioNodeStream* GetStream() { return mStream; } + AudioNodeStream* GetStream() const { return mStream; } const nsTArray& InputNodes() const { @@ -202,20 +211,21 @@ public: virtual const char* NodeType() const = 0; private: - friend class AudioBufferSourceNode; - // This could possibly delete 'this'. + virtual void LastRelease() override + { + // We are about to be deleted, disconnect the object from the graph before + // the derived type is destroyed. + DisconnectFromGraph(); + } + // Callers must hold a reference to 'this'. void DisconnectFromGraph(); protected: - static void Callback(AudioNode* aNode) { /* not implemented */ } - // Helpers for sending different value types to streams void SendDoubleParameterToStream(uint32_t aIndex, double aValue); void SendInt32ParameterToStream(uint32_t aIndex, int32_t aValue); void SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue); void SendChannelMixingParametersToStream(); - static void SendTimelineParameterToStream(AudioNode* aNode, uint32_t aIndex, - const AudioParamTimeline& aValue); private: RefPtr mContext; diff --git a/dom/media/webaudio/AudioNodeEngine.cpp b/dom/media/webaudio/AudioNodeEngine.cpp index 8fd231094a..e950aafc8d 100644 --- a/dom/media/webaudio/AudioNodeEngine.cpp +++ b/dom/media/webaudio/AudioNodeEngine.cpp @@ -12,38 +12,36 @@ namespace mozilla { -void -AllocateAudioBlock(uint32_t aChannelCount, AudioChunk* aChunk) +already_AddRefed +ThreadSharedFloatArrayBufferList::Create(uint32_t aChannelCount, + size_t aLength, + const mozilla::fallible_t&) { - CheckedInt size = WEBAUDIO_BLOCK_SIZE; - size *= aChannelCount; - size *= sizeof(float); - if (!size.isValid()) { - MOZ_CRASH(); - } - // XXX for SIMD purposes we should do something here to make sure the - // channel buffers are 16-byte aligned. - RefPtr buffer = SharedBuffer::Create(size.value()); - aChunk->mDuration = WEBAUDIO_BLOCK_SIZE; - aChunk->mChannelData.SetLength(aChannelCount); - float* data = static_cast(buffer->Data()); + RefPtr buffer = + new ThreadSharedFloatArrayBufferList(aChannelCount); + for (uint32_t i = 0; i < aChannelCount; ++i) { - aChunk->mChannelData[i] = data + i*WEBAUDIO_BLOCK_SIZE; + float* channelData = js_pod_malloc(aLength); + if (!channelData) { + return nullptr; + } + + buffer->SetData(i, channelData, js_free, channelData); } - aChunk->mBuffer = buffer.forget(); - aChunk->mVolume = 1.0f; - aChunk->mBufferFormat = AUDIO_FORMAT_FLOAT32; + + return buffer.forget(); } void -WriteZeroesToAudioBlock(AudioChunk* aChunk, uint32_t aStart, uint32_t aLength) +WriteZeroesToAudioBlock(AudioBlock* aChunk, + uint32_t aStart, uint32_t aLength) { MOZ_ASSERT(aStart + aLength <= WEBAUDIO_BLOCK_SIZE); MOZ_ASSERT(!aChunk->IsNull(), "You should pass a non-null chunk"); if (aLength == 0) return; - for (uint32_t i = 0; i < aChunk->mChannelData.Length(); ++i) { + for (uint32_t i = 0; i < aChunk->ChannelCount(); ++i) { PodZero(aChunk->ChannelFloatsForWrite(i) + aStart, aLength); } } @@ -273,4 +271,26 @@ AudioBufferSumOfSquares(const float* aInput, uint32_t aLength) return sum; } +void +AudioNodeEngine::ProcessBlock(AudioNodeStream* aStream, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, + bool* aFinished) +{ + MOZ_ASSERT(mInputCount <= 1 && mOutputCount <= 1); + *aOutput = aInput; +} + +void +AudioNodeEngine::ProcessBlocksOnPorts(AudioNodeStream* aStream, + const OutputChunks& aInput, + OutputChunks& aOutput, + bool* aFinished) +{ + MOZ_ASSERT(mInputCount > 1 || mOutputCount > 1); + // Only produce one output port, and drop all other input ports. + aOutput[0] = aInput[0]; +} + } // namespace mozilla diff --git a/dom/media/webaudio/AudioNodeEngine.h b/dom/media/webaudio/AudioNodeEngine.h index 3d2f5fee60..72de8aed9c 100644 --- a/dom/media/webaudio/AudioNodeEngine.h +++ b/dom/media/webaudio/AudioNodeEngine.h @@ -17,8 +17,10 @@ namespace dom { struct ThreeDPoint; class AudioParamTimeline; class DelayNodeEngine; +struct AudioTimelineEvent; } // namespace dom +class AudioBlock; class AudioNodeStream; /** @@ -31,12 +33,19 @@ class ThreadSharedFloatArrayBufferList final : public ThreadSharedObject { public: /** - * Construct with null data. + * Construct with null channel data pointers. */ explicit ThreadSharedFloatArrayBufferList(uint32_t aCount) { mContents.SetLength(aCount); } + /** + * Create with buffers suitable for transfer to + * JS_NewArrayBufferWithContents(). The buffer contents are uninitialized + * and so should be set using GetDataForWrite(). + */ + static already_AddRefed + Create(uint32_t aChannelCount, size_t aLength, const mozilla::fallible_t&); struct Storage final { @@ -58,7 +67,7 @@ public: } void* mDataToFree; void (*mFree)(void*); - const float* mSampleData; + float* mSampleData; }; /** @@ -69,12 +78,21 @@ public: * This can be called on any thread. */ const float* GetData(uint32_t aIndex) const { return mContents[aIndex].mSampleData; } + /** + * This can be called on any thread, but only when the calling thread is the + * only owner. + */ + float* GetDataForWrite(uint32_t aIndex) + { + MOZ_ASSERT(!IsShared()); + return mContents[aIndex].mSampleData; + } /** * Call this only during initialization, before the object is handed to * any other thread. */ - void SetData(uint32_t aIndex, void* aDataToFree, void (*aFreeFunc)(void*), const float* aData) + void SetData(uint32_t aIndex, void* aDataToFree, void (*aFreeFunc)(void*), float* aData) { Storage* s = &mContents[aIndex]; if (s->mFree) { @@ -113,15 +131,11 @@ private: nsAutoTArray mContents; }; -/** - * Allocates an AudioChunk with fresh buffers of WEBAUDIO_BLOCK_SIZE float samples. - */ -void AllocateAudioBlock(uint32_t aChannelCount, AudioChunk* aChunk); - /** * aChunk must have been allocated by AllocateAudioBlock. */ -void WriteZeroesToAudioBlock(AudioChunk* aChunk, uint32_t aStart, uint32_t aLength); +void WriteZeroesToAudioBlock(AudioBlock* aChunk, uint32_t aStart, + uint32_t aLength); /** * Copy with scale. aScale == 1.0f should be optimized. @@ -239,7 +253,7 @@ class AudioNodeEngine { public: // This should be compatible with AudioNodeStream::OutputChunks. - typedef nsAutoTArray OutputChunks; + typedef nsAutoTArray OutputChunks; explicit AudioNodeEngine(dom::AudioNode* aNode) : mNode(aNode) @@ -269,11 +283,10 @@ public: { NS_ERROR("Invalid SetInt32Parameter index"); } - virtual void SetTimelineParameter(uint32_t aIndex, - const dom::AudioParamTimeline& aValue, - TrackRate aSampleRate) + virtual void RecvTimelineEvent(uint32_t aIndex, + dom::AudioTimelineEvent& aValue) { - NS_ERROR("Invalid SetTimelineParameter index"); + NS_ERROR("Invalid RecvTimelineEvent index"); } virtual void SetThreeDPointParameter(uint32_t aIndex, const dom::ThreeDPoint& aValue) @@ -296,23 +309,23 @@ public: * aInput is guaranteed to have float sample format (if it has samples at all) * and to have been resampled to the sampling rate for the stream, and to have * exactly WEBAUDIO_BLOCK_SIZE samples. - * *aFinished is set to false by the caller. If the callee sets it to true, - * we'll finish the stream and not call this again. + * *aFinished is set to false by the caller. The callee must not set this to + * true unless silent output is produced. If set to true, we'll finish the + * stream, consider this input inactive on any downstream nodes, and not + * call this again. */ virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, - bool* aFinished) - { - MOZ_ASSERT(mInputCount <= 1 && mOutputCount <= 1); - *aOutput = aInput; - } + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, + bool* aFinished); /** * Produce the next block of audio samples, before input is provided. * ProcessBlock() will be called later, and it then should not change * aOutput. This is used only for DelayNodeEngine in a feedback loop. */ - virtual void ProduceBlockBeforeInput(AudioChunk* aOutput) + virtual void ProduceBlockBeforeInput(GraphTime aFrom, + AudioBlock* aOutput) { NS_NOTREACHED("ProduceBlockBeforeInput called on wrong engine\n"); } @@ -335,12 +348,13 @@ public: virtual void ProcessBlocksOnPorts(AudioNodeStream* aStream, const OutputChunks& aInput, OutputChunks& aOutput, - bool* aFinished) - { - MOZ_ASSERT(mInputCount > 1 || mOutputCount > 1); - // Only produce one output port, and drop all other input ports. - aOutput[0] = aInput[0]; - } + bool* aFinished); + + // IsActive() returns true if the engine needs to continue processing an + // unfinished stream even when it has silent or no input connections. This + // includes tail-times and when sources have been scheduled to start. If + // returning false, then the stream can be suspended. + virtual bool IsActive() const { return false; } bool HasNode() const { diff --git a/dom/media/webaudio/AudioNodeExternalInputStream.cpp b/dom/media/webaudio/AudioNodeExternalInputStream.cpp index a4927ca857..a9affd72e3 100644 --- a/dom/media/webaudio/AudioNodeExternalInputStream.cpp +++ b/dom/media/webaudio/AudioNodeExternalInputStream.cpp @@ -12,8 +12,8 @@ using namespace mozilla::dom; namespace mozilla { -AudioNodeExternalInputStream::AudioNodeExternalInputStream(AudioNodeEngine* aEngine, TrackRate aSampleRate, uint32_t aContextId) - : AudioNodeStream(aEngine, NO_STREAM_FLAGS, aSampleRate, aContextId) +AudioNodeExternalInputStream::AudioNodeExternalInputStream(AudioNodeEngine* aEngine, TrackRate aSampleRate) + : AudioNodeStream(aEngine, NO_STREAM_FLAGS, aSampleRate) { MOZ_COUNT_CTOR(AudioNodeExternalInputStream); } @@ -27,13 +27,14 @@ AudioNodeExternalInputStream::~AudioNodeExternalInputStream() AudioNodeExternalInputStream::Create(MediaStreamGraph* aGraph, AudioNodeEngine* aEngine) { + AudioContext* ctx = aEngine->NodeMainThread()->Context(); MOZ_ASSERT(NS_IsMainThread()); - MOZ_ASSERT(aGraph->GraphRate() == aEngine->NodeMainThread()->Context()->SampleRate()); + MOZ_ASSERT(aGraph->GraphRate() == ctx->SampleRate()); RefPtr stream = - new AudioNodeExternalInputStream(aEngine, aGraph->GraphRate(), - aEngine->NodeMainThread()->Context()->Id()); - aGraph->AddStream(stream); + new AudioNodeExternalInputStream(aEngine, aGraph->GraphRate()); + aGraph->AddStream(stream, + ctx->ShouldSuspendNewStream() ? MediaStreamGraph::ADD_STREAM_SUSPENDED : 0); return stream.forget(); } @@ -44,7 +45,7 @@ AudioNodeExternalInputStream::Create(MediaStreamGraph* aGraph, */ template static void -CopyChunkToBlock(AudioChunk& aInput, AudioChunk *aBlock, +CopyChunkToBlock(AudioChunk& aInput, AudioBlock *aBlock, uint32_t aOffsetInBlock) { uint32_t blockChannels = aBlock->ChannelCount(); @@ -79,7 +80,7 @@ CopyChunkToBlock(AudioChunk& aInput, AudioChunk *aBlock, * channels in every chunk of aSegment. aBlock must be float format or null. */ static void ConvertSegmentToAudioBlock(AudioSegment* aSegment, - AudioChunk* aBlock, + AudioBlock* aBlock, int32_t aFallbackChannelCount) { NS_ASSERTION(aSegment->GetDuration() == WEBAUDIO_BLOCK_SIZE, "Bad segment duration"); @@ -95,7 +96,7 @@ static void ConvertSegmentToAudioBlock(AudioSegment* aSegment, } } - AllocateAudioBlock(aFallbackChannelCount, aBlock); + aBlock->AllocateChannels(aFallbackChannelCount); uint32_t duration = 0; for (AudioSegment::ChunkIterator ci(*aSegment); !ci.IsEnded(); ci.Next()) { @@ -138,6 +139,10 @@ AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, for (StreamBuffer::TrackIter tracks(source->mBuffer, MediaSegment::AUDIO); !tracks.IsEnded(); tracks.Next()) { const StreamBuffer::Track& inputTrack = *tracks; + if (!mInputs[0]->PassTrackThrough(tracks->GetID())) { + continue; + } + const AudioSegment& inputSegment = *static_cast(inputTrack.GetSegment()); if (inputSegment.IsNull()) { @@ -153,8 +158,8 @@ AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, break; next = interval.mEnd; - StreamTime outputStart = GraphTimeToStreamTime(interval.mStart); - StreamTime outputEnd = GraphTimeToStreamTime(interval.mEnd); + StreamTime outputStart = GraphTimeToStreamTimeWithBlocking(interval.mStart); + StreamTime outputEnd = GraphTimeToStreamTimeWithBlocking(interval.mEnd); StreamTime ticks = outputEnd - outputStart; if (interval.mInputIsBlocked) { @@ -162,10 +167,10 @@ AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, } else { StreamTime inputStart = std::min(inputSegment.GetDuration(), - source->GraphTimeToStreamTime(interval.mStart)); + source->GraphTimeToStreamTimeWithBlocking(interval.mStart)); StreamTime inputEnd = std::min(inputSegment.GetDuration(), - source->GraphTimeToStreamTime(interval.mEnd)); + source->GraphTimeToStreamTimeWithBlocking(interval.mEnd)); segment.AppendSlice(inputSegment, inputStart, inputEnd); // Pad if we're looking past the end of the track @@ -182,11 +187,11 @@ AudioNodeExternalInputStream::ProcessInput(GraphTime aFrom, GraphTime aTo, if (inputChannels) { nsAutoTArray downmixBuffer; for (uint32_t i = 0; i < audioSegments.Length(); ++i) { - AudioChunk tmpChunk; + AudioBlock tmpChunk; ConvertSegmentToAudioBlock(&audioSegments[i], &tmpChunk, inputChannels); if (!tmpChunk.IsNull()) { if (accumulateIndex == 0) { - AllocateAudioBlock(inputChannels, &mLastChunks[0]); + mLastChunks[0].AllocateChannels(inputChannels); } AccumulateInputChunk(accumulateIndex, tmpChunk, &mLastChunks[0], &downmixBuffer); accumulateIndex++; diff --git a/dom/media/webaudio/AudioNodeExternalInputStream.h b/dom/media/webaudio/AudioNodeExternalInputStream.h index 4de2664574..89d39d432e 100644 --- a/dom/media/webaudio/AudioNodeExternalInputStream.h +++ b/dom/media/webaudio/AudioNodeExternalInputStream.h @@ -25,8 +25,7 @@ public: Create(MediaStreamGraph* aGraph, AudioNodeEngine* aEngine); protected: - AudioNodeExternalInputStream(AudioNodeEngine* aEngine, TrackRate aSampleRate, - uint32_t aContextId); + AudioNodeExternalInputStream(AudioNodeEngine* aEngine, TrackRate aSampleRate); ~AudioNodeExternalInputStream(); public: diff --git a/dom/media/webaudio/AudioNodeStream.cpp b/dom/media/webaudio/AudioNodeStream.cpp index 0688a2a276..3d5c87b447 100644 --- a/dom/media/webaudio/AudioNodeStream.cpp +++ b/dom/media/webaudio/AudioNodeStream.cpp @@ -11,6 +11,7 @@ #include "AudioChannelFormat.h" #include "AudioParamTimeline.h" #include "AudioContext.h" +#include "nsMathUtils.h" using namespace mozilla::dom; @@ -27,14 +28,13 @@ namespace mozilla { AudioNodeStream::AudioNodeStream(AudioNodeEngine* aEngine, Flags aFlags, - TrackRate aSampleRate, - AudioContext::AudioContextId aContextId) + TrackRate aSampleRate) : ProcessedMediaStream(nullptr), mEngine(aEngine), mSampleRate(aSampleRate), - mAudioContextId(aContextId), mFlags(aFlags), mNumberOfInputChannels(2), + mIsActive(aEngine->IsActive()), mMarkAsFinishedAfterThisBlock(false), mAudioParamStream(false), mPassThrough(false) @@ -50,30 +50,40 @@ AudioNodeStream::AudioNodeStream(AudioNodeEngine* aEngine, AudioNodeStream::~AudioNodeStream() { + MOZ_ASSERT(mActiveInputCount == 0); MOZ_COUNT_DTOR(AudioNodeStream); } +void +AudioNodeStream::DestroyImpl() +{ + // These are graph thread objects, so clean up on graph thread. + mInputChunks.Clear(); + mLastChunks.Clear(); + + ProcessedMediaStream::DestroyImpl(); +} + /* static */ already_AddRefed -AudioNodeStream::Create(MediaStreamGraph* aGraph, AudioNodeEngine* aEngine, - Flags aFlags) +AudioNodeStream::Create(AudioContext* aCtx, AudioNodeEngine* aEngine, + Flags aFlags, MediaStreamGraph* aGraph) { MOZ_ASSERT(NS_IsMainThread()); // MediaRecorders use an AudioNodeStream, but no AudioNode AudioNode* node = aEngine->NodeMainThread(); - MOZ_ASSERT(!node || aGraph->GraphRate() == node->Context()->SampleRate()); + MediaStreamGraph* graph = aGraph ? aGraph : aCtx->Graph(); + MOZ_ASSERT(graph->GraphRate() == aCtx->SampleRate()); - dom::AudioContext::AudioContextId contextIdForStream = node ? node->Context()->Id() : - NO_AUDIO_CONTEXT; RefPtr stream = - new AudioNodeStream(aEngine, aFlags, aGraph->GraphRate(), - contextIdForStream); - if (aEngine->HasNode()) { - stream->SetChannelMixingParametersImpl(aEngine->NodeMainThread()->ChannelCount(), - aEngine->NodeMainThread()->ChannelCountModeValue(), - aEngine->NodeMainThread()->ChannelInterpretationValue()); + new AudioNodeStream(aEngine, aFlags, graph->GraphRate()); + if (node) { + stream->SetChannelMixingParametersImpl(node->ChannelCount(), + node->ChannelCountModeValue(), + node->ChannelInterpretationValue()); } - aGraph->AddStream(stream); + graph->AddStream(stream, + aCtx->ShouldSuspendNewStream() ? MediaStreamGraph::ADD_STREAM_SUSPENDED : 0); return stream.forget(); } @@ -138,15 +148,15 @@ AudioNodeStream::SetStreamTimeParameter(uint32_t aIndex, AudioContext* aContext, }; GraphImpl()->AppendMessage(new Message(this, aIndex, - aContext->DestinationStream(), - aContext->DOMTimeToStreamTime(aStreamTime))); + aContext->DestinationStream(), + aStreamTime)); } void AudioNodeStream::SetStreamTimeParameterImpl(uint32_t aIndex, MediaStream* aRelativeToStream, double aStreamTime) { - StreamTime ticks = TicksFromDestinationTime(aRelativeToStream, aStreamTime); + StreamTime ticks = aRelativeToStream->SecondsToNearestStreamTime(aStreamTime); mEngine->SetStreamTimeParameter(aIndex, ticks); } @@ -193,29 +203,29 @@ AudioNodeStream::SetInt32Parameter(uint32_t aIndex, int32_t aValue) } void -AudioNodeStream::SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue) +AudioNodeStream::SendTimelineEvent(uint32_t aIndex, + const AudioTimelineEvent& aEvent) { class Message final : public ControlMessage { public: Message(AudioNodeStream* aStream, uint32_t aIndex, - const AudioParamTimeline& aValue) + const AudioTimelineEvent& aEvent) : ControlMessage(aStream), - mValue(aValue), + mEvent(aEvent), mSampleRate(aStream->SampleRate()), mIndex(aIndex) {} virtual void Run() override { static_cast(mStream)->Engine()-> - SetTimelineParameter(mIndex, mValue, mSampleRate); + RecvTimelineEvent(mIndex, mEvent); } - AudioParamTimeline mValue; + AudioTimelineEvent mEvent; TrackRate mSampleRate; uint32_t mIndex; }; - GraphImpl()->AppendMessage(new Message(this, aIndex, aValue)); + GraphImpl()->AppendMessage(new Message(this, aIndex, aEvent)); } void @@ -367,12 +377,38 @@ AudioNodeStream::ComputedNumberOfChannels(uint32_t aInputChannelCount) } } +class AudioNodeStream::AdvanceAndResumeMessage final : public ControlMessage { +public: + AdvanceAndResumeMessage(AudioNodeStream* aStream, StreamTime aAdvance) : + ControlMessage(aStream), mAdvance(aAdvance) {} + virtual void Run() override + { + auto ns = static_cast(mStream); + ns->mBufferStartTime -= mAdvance; + + StreamBuffer::Track* track = ns->EnsureTrack(AUDIO_TRACK); + track->Get()->AppendNullData(mAdvance); + + ns->GraphImpl()->DecrementSuspendCount(mStream); + } +private: + StreamTime mAdvance; +}; + void -AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) +AudioNodeStream::AdvanceAndResume(StreamTime aAdvance) +{ + mMainThreadCurrentTime += aAdvance; + GraphImpl()->AppendMessage(new AdvanceAndResumeMessage(this, aAdvance)); +} + +void +AudioNodeStream::ObtainInputBlock(AudioBlock& aTmpChunk, + uint32_t aPortIndex) { uint32_t inputCount = mInputs.Length(); uint32_t outputChannelCount = 1; - nsAutoTArray inputChunks; + nsAutoTArray inputChunks; for (uint32_t i = 0; i < inputCount; ++i) { if (aPortIndex != mInputs[i]->InputNumber()) { // This input is connected to a different port @@ -385,7 +421,7 @@ AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) continue; } - AudioChunk* chunk = &a->mLastChunks[mInputs[i]->OutputNumber()]; + const AudioBlock* chunk = &a->mLastChunks[mInputs[i]->OutputNumber()]; MOZ_ASSERT(chunk); if (chunk->IsNull() || chunk->mChannelData.IsEmpty()) { continue; @@ -393,20 +429,20 @@ AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) inputChunks.AppendElement(chunk); outputChannelCount = - GetAudioChannelsSuperset(outputChannelCount, chunk->mChannelData.Length()); + GetAudioChannelsSuperset(outputChannelCount, chunk->ChannelCount()); } outputChannelCount = ComputedNumberOfChannels(outputChannelCount); uint32_t inputChunkCount = inputChunks.Length(); if (inputChunkCount == 0 || - (inputChunkCount == 1 && inputChunks[0]->mChannelData.Length() == 0)) { + (inputChunkCount == 1 && inputChunks[0]->ChannelCount() == 0)) { aTmpChunk.SetNull(WEBAUDIO_BLOCK_SIZE); return; } if (inputChunkCount == 1 && - inputChunks[0]->mChannelData.Length() == outputChannelCount) { + inputChunks[0]->ChannelCount() == outputChannelCount) { aTmpChunk = *inputChunks[0]; return; } @@ -416,7 +452,7 @@ AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) return; } - AllocateAudioBlock(outputChannelCount, &aTmpChunk); + aTmpChunk.AllocateChannels(outputChannelCount); // The static storage here should be 1KB, so it's fine nsAutoTArray downmixBuffer; @@ -426,12 +462,13 @@ AudioNodeStream::ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex) } void -AudioNodeStream::AccumulateInputChunk(uint32_t aInputIndex, const AudioChunk& aChunk, - AudioChunk* aBlock, +AudioNodeStream::AccumulateInputChunk(uint32_t aInputIndex, + const AudioBlock& aChunk, + AudioBlock* aBlock, nsTArray* aDownmixBuffer) { nsAutoTArray channels; - UpMixDownMixChunk(&aChunk, aBlock->mChannelData.Length(), channels, *aDownmixBuffer); + UpMixDownMixChunk(&aChunk, aBlock->ChannelCount(), channels, *aDownmixBuffer); for (uint32_t c = 0; c < channels.Length(); ++c) { const float* inputData = static_cast(channels[c]); @@ -451,14 +488,14 @@ AudioNodeStream::AccumulateInputChunk(uint32_t aInputIndex, const AudioChunk& aC } void -AudioNodeStream::UpMixDownMixChunk(const AudioChunk* aChunk, +AudioNodeStream::UpMixDownMixChunk(const AudioBlock* aChunk, uint32_t aOutputChannelCount, nsTArray& aOutputChannels, nsTArray& aDownmixBuffer) { static const float silenceChannel[WEBAUDIO_BLOCK_SIZE] = {0.f}; - for (uint32_t i = 0; i < aChunk->mChannelData.Length(); i++) { + for (uint32_t i = 0; i < aChunk->ChannelCount(); i++) { aOutputChannels.AppendElement(static_cast(aChunk->mChannelData[i])); } if (aOutputChannels.Length() < aOutputChannelCount) { @@ -510,12 +547,14 @@ AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) uint16_t outputCount = mLastChunks.Length(); MOZ_ASSERT(outputCount == std::max(uint16_t(1), mEngine->OutputCount())); - // Consider this stream blocked if it has already finished output. Normally - // mBlocked would reflect this, but due to rounding errors our audio track may - // appear to extend slightly beyond aFrom, so we might not be blocked yet. - bool blocked = mFinished || mBlocked.GetAt(aFrom); - // If the stream has finished at this time, it will be blocked. - if (blocked || InMutedCycle()) { + if (!mIsActive) { + // mLastChunks are already null. +#ifdef DEBUG + for (const auto& chunk : mLastChunks) { + MOZ_ASSERT(chunk.IsNull()); + } +#endif + } else if (InMutedCycle()) { mInputChunks.Clear(); for (uint16_t i = 0; i < outputCount; ++i) { mLastChunks[i].SetNull(WEBAUDIO_BLOCK_SIZE); @@ -533,7 +572,8 @@ AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) mLastChunks[0] = mInputChunks[0]; } else { if (maxInputs <= 1 && outputCount <= 1) { - mEngine->ProcessBlock(this, mInputChunks[0], &mLastChunks[0], &finished); + mEngine->ProcessBlock(this, aFrom, + mInputChunks[0], &mLastChunks[0], &finished); } else { mEngine->ProcessBlocksOnPorts(this, mInputChunks, mLastChunks, &finished); } @@ -544,6 +584,7 @@ AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) } if (finished) { mMarkAsFinishedAfterThisBlock = true; + CheckForInactive(); } if (mDisabledTrackIDs.Contains(static_cast(AUDIO_TRACK))) { @@ -553,8 +594,8 @@ AudioNodeStream::ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) } } - if (!blocked) { - // Don't output anything while blocked + if (!mFinished) { + // Don't output anything while finished AdvanceOutputSegment(); if (mMarkAsFinishedAfterThisBlock && (aFlags & ALLOW_FINISH)) { // This stream was finished the last time that we looked at it, and all @@ -574,15 +615,10 @@ AudioNodeStream::ProduceOutputBeforeInput(GraphTime aFrom) MOZ_ASSERT(!InMutedCycle(), "DelayNodes should break cycles"); MOZ_ASSERT(mLastChunks.Length() == 1); - // Consider this stream blocked if it has already finished output. Normally - // mBlocked would reflect this, but due to rounding errors our audio track may - // appear to extend slightly beyond aFrom, so we might not be blocked yet. - bool blocked = mFinished || mBlocked.GetAt(aFrom); - // If the stream has finished at this time, it will be blocked. - if (blocked) { + if (!mIsActive) { mLastChunks[0].SetNull(WEBAUDIO_BLOCK_SIZE); } else { - mEngine->ProduceBlockBeforeInput(&mLastChunks[0]); + mEngine->ProduceBlockBeforeInput(aFrom, &mLastChunks[0]); NS_ASSERTION(mLastChunks[0].GetDuration() == WEBAUDIO_BLOCK_SIZE, "Invalid WebAudio chunk size"); if (mDisabledTrackIDs.Contains(static_cast(AUDIO_TRACK))) { @@ -598,14 +634,14 @@ AudioNodeStream::AdvanceOutputSegment() AudioSegment* segment = track->Get(); if (mFlags & EXTERNAL_OUTPUT) { - segment->AppendAndConsumeChunk(&mLastChunks[0]); + segment->AppendAndConsumeChunk(mLastChunks[0].AsMutableChunk()); } else { segment->AppendNullData(mLastChunks[0].GetDuration()); } for (uint32_t j = 0; j < mListeners.Length(); ++j) { MediaStreamListener* l = mListeners[j]; - AudioChunk copyChunk = mLastChunks[0]; + AudioChunk copyChunk = mLastChunks[0].AsAudioChunk(); AudioSegment tmpSegment; tmpSegment.AppendAndConsumeChunk(©Chunk); l->NotifyQueuedTrackChanges(Graph(), AUDIO_TRACK, @@ -613,29 +649,6 @@ AudioNodeStream::AdvanceOutputSegment() } } -void -AudioNodeStream::ReleaseSharedBuffers() -{ - // A shared buffer can't be reused, so release the reference now. Keep - // the channel data arrays to save unnecessary free/alloc. - // Release shared output buffers first, as they may be shared with input - // buffers which can be re-used if there are no other references. - for (auto& chunk : mLastChunks) { - chunk.ReleaseBufferIfShared(); - } - for (auto& chunk : mInputChunks) { - chunk.ReleaseBufferIfShared(); - } -} - - -StreamTime -AudioNodeStream::GetCurrentPosition() -{ - NS_ASSERTION(!mFinished, "Don't create another track after finishing"); - return EnsureTrack(AUDIO_TRACK)->Get()->GetDuration(); -} - void AudioNodeStream::FinishOutput() { @@ -656,50 +669,88 @@ AudioNodeStream::FinishOutput() } } -double -AudioNodeStream::FractionalTicksFromDestinationTime(AudioNodeStream* aDestination, - double aSeconds) +void +AudioNodeStream::AddInput(MediaInputPort* aPort) { - MOZ_ASSERT(aDestination->SampleRate() == SampleRate()); - MOZ_ASSERT(SampleRate() == GraphRate()); - - double destinationSeconds = std::max(0.0, aSeconds); - double destinationFractionalTicks = destinationSeconds * SampleRate(); - MOZ_ASSERT(destinationFractionalTicks < STREAM_TIME_MAX); - StreamTime destinationStreamTime = destinationFractionalTicks; // round down - // MediaTime does not have the resolution of double - double offset = destinationFractionalTicks - destinationStreamTime; - - GraphTime graphTime = - aDestination->StreamTimeToGraphTime(destinationStreamTime); - StreamTime thisStreamTime = GraphTimeToStreamTimeOptimistic(graphTime); - double thisFractionalTicks = thisStreamTime + offset; - MOZ_ASSERT(thisFractionalTicks >= 0.0); - return thisFractionalTicks; + ProcessedMediaStream::AddInput(aPort); + AudioNodeStream* ns = aPort->GetSource()->AsAudioNodeStream(); + // Streams that are not AudioNodeStreams are considered active. + if (!ns || (ns->mIsActive && !ns->IsAudioParamStream())) { + IncrementActiveInputCount(); + } +} +void +AudioNodeStream::RemoveInput(MediaInputPort* aPort) +{ + ProcessedMediaStream::RemoveInput(aPort); + AudioNodeStream* ns = aPort->GetSource()->AsAudioNodeStream(); + // Streams that are not AudioNodeStreams are considered active. + if (!ns || (ns->mIsActive && !ns->IsAudioParamStream())) { + DecrementActiveInputCount(); + } } -StreamTime -AudioNodeStream::TicksFromDestinationTime(MediaStream* aDestination, - double aSeconds) +void +AudioNodeStream::SetActive() { - AudioNodeStream* destination = aDestination->AsAudioNodeStream(); - MOZ_ASSERT(destination); + if (mIsActive || mMarkAsFinishedAfterThisBlock) { + return; + } - double thisSeconds = - FractionalTicksFromDestinationTime(destination, aSeconds); - // Round to nearest - StreamTime ticks = thisSeconds + 0.5; - return ticks; + mIsActive = true; + if (IsAudioParamStream()) { + // Consumers merely influence stream order. + // They do not read from the stream. + return; + } + + for (const auto& consumer : mConsumers) { + AudioNodeStream* ns = consumer->GetDestination()->AsAudioNodeStream(); + if (ns) { + ns->IncrementActiveInputCount(); + } + } } -double -AudioNodeStream::DestinationTimeFromTicks(AudioNodeStream* aDestination, - StreamTime aPosition) +void +AudioNodeStream::CheckForInactive() { - MOZ_ASSERT(SampleRate() == aDestination->SampleRate()); - GraphTime graphTime = StreamTimeToGraphTime(aPosition); - StreamTime destinationTime = aDestination->GraphTimeToStreamTimeOptimistic(graphTime); - return StreamTimeToSeconds(destinationTime); + if (((mActiveInputCount > 0 || mEngine->IsActive()) && + !mMarkAsFinishedAfterThisBlock) || + !mIsActive) { + return; + } + + mIsActive = false; + mInputChunks.Clear(); // not required for foreseeable future + for (auto& chunk : mLastChunks) { + chunk.SetNull(WEBAUDIO_BLOCK_SIZE); + } + if (IsAudioParamStream()) { + return; + } + + for (const auto& consumer : mConsumers) { + AudioNodeStream* ns = consumer->GetDestination()->AsAudioNodeStream(); + if (ns) { + ns->DecrementActiveInputCount(); + } + } +} + +void +AudioNodeStream::IncrementActiveInputCount() +{ + ++mActiveInputCount; + SetActive(); +} + +void +AudioNodeStream::DecrementActiveInputCount() +{ + MOZ_ASSERT(mActiveInputCount > 0); + --mActiveInputCount; + CheckForInactive(); } } // namespace mozilla diff --git a/dom/media/webaudio/AudioNodeStream.h b/dom/media/webaudio/AudioNodeStream.h index 8824e1f7b1..f44df3e737 100644 --- a/dom/media/webaudio/AudioNodeStream.h +++ b/dom/media/webaudio/AudioNodeStream.h @@ -8,13 +8,13 @@ #include "MediaStreamGraph.h" #include "mozilla/dom/AudioNodeBinding.h" -#include "AudioSegment.h" +#include "AudioBlock.h" namespace mozilla { namespace dom { struct ThreeDPoint; -class AudioParamTimeline; +struct AudioTimelineEvent; class AudioContext; } // namespace dom @@ -41,7 +41,7 @@ public: enum { AUDIO_TRACK = 1 }; - typedef nsAutoTArray OutputChunks; + typedef nsAutoTArray OutputChunks; // Flags re main thread updates and stream output. typedef unsigned Flags; @@ -57,9 +57,13 @@ public: /** * Create a stream that will process audio for an AudioNode. * Takes ownership of aEngine. + * If aGraph is non-null, use that as the MediaStreamGraph, otherwise use + * aCtx's graph. aGraph is only non-null when called for AudioDestinationNode + * since the context's graph hasn't been set up in that case. */ static already_AddRefed - Create(MediaStreamGraph* aGraph, AudioNodeEngine* aEngine, Flags aKind); + Create(AudioContext* aCtx, AudioNodeEngine* aEngine, Flags aKind, + MediaStreamGraph* aGraph = nullptr); protected: /** @@ -67,8 +71,7 @@ protected: */ AudioNodeStream(AudioNodeEngine* aEngine, Flags aFlags, - TrackRate aSampleRate, - AudioContext::AudioContextId aContextId); + TrackRate aSampleRate); ~AudioNodeStream(); @@ -82,9 +85,10 @@ public: double aStreamTime); void SetDoubleParameter(uint32_t aIndex, double aValue); void SetInt32Parameter(uint32_t aIndex, int32_t aValue); - void SetTimelineParameter(uint32_t aIndex, const dom::AudioParamTimeline& aValue); void SetThreeDPointParameter(uint32_t aIndex, const dom::ThreeDPoint& aValue); void SetBuffer(already_AddRefed&& aBuffer); + // This sends a single event to the timeline on the MSG thread side. + void SendTimelineEvent(uint32_t aIndex, const dom::AudioTimelineEvent& aEvent); // This consumes the contents of aData. aData will be emptied after this returns. void SetRawArrayData(nsTArray& aData); void SetChannelMixingParameters(uint32_t aNumberOfChannels, @@ -102,7 +106,17 @@ public: mAudioParamStream = true; } + /* + * Resume stream after updating its concept of current time by aAdvance. + * Main thread. Used only from AudioDestinationNode when resuming a stream + * suspended to save running the MediaStreamGraph when there are no other + * nodes in the AudioContext. + */ + void AdvanceAndResume(StreamTime aAdvance); + virtual AudioNodeStream* AsAudioNodeStream() override { return this; } + virtual void AddInput(MediaInputPort* aPort) override; + virtual void RemoveInput(MediaInputPort* aPort) override; // Graph thread only void SetStreamTimeParameterImpl(uint32_t aIndex, MediaStream* aRelativeToStream, @@ -117,15 +131,6 @@ public: * the output. This is used only for DelayNodeEngine in a feedback loop. */ void ProduceOutputBeforeInput(GraphTime aFrom); - /** - * Remove references to shared AudioChunk buffers. Called on downstream - * nodes first after an iteration has called ProcessInput() on the entire - * graph, so that upstream nodes can re-use their buffers on the next - * iteration. - */ - void ReleaseSharedBuffers(); - - StreamTime GetCurrentPosition(); bool IsAudioParamStream() const { return mAudioParamStream; @@ -140,34 +145,10 @@ public: return ((mFlags & NEED_MAIN_THREAD_FINISHED) && mFinished) || (mFlags & NEED_MAIN_THREAD_CURRENT_TIME); } - virtual bool IsIntrinsicallyConsumed() const override - { - return true; - } // Any thread AudioNodeEngine* Engine() { return mEngine; } TrackRate SampleRate() const { return mSampleRate; } - AudioContext::AudioContextId AudioContextId() const override { return mAudioContextId; } - - /** - * Convert a time in seconds on the destination stream to ticks - * on this stream, including fractional position between ticks. - */ - double FractionalTicksFromDestinationTime(AudioNodeStream* aDestination, - double aSeconds); - /** - * Convert a time in seconds on the destination stream to StreamTime - * on this stream. - */ - StreamTime TicksFromDestinationTime(MediaStream* aDestination, - double aSeconds); - /** - * Get the destination stream time in seconds corresponding to a position on - * this stream. - */ - double DestinationTimeFromTicks(AudioNodeStream* aDestination, - StreamTime aPosition); size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override; size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override; @@ -175,19 +156,41 @@ public: void SizeOfAudioNodesIncludingThis(MallocSizeOf aMallocSizeOf, AudioNodeSizes& aUsage) const; + /* + * SetActive() is called when either an active input is added or the engine + * for a source node transitions from inactive to active. This is not + * called from engines for processing nodes because they only become active + * when there are active input streams, in which case this stream is already + * active. + */ + void SetActive(); + /* + * CheckForInactive() is called when the engine transitions from active to + * inactive, or an active input is removed, or the stream finishes. If the + * stream is now inactive, then mInputChunks will be cleared and mLastChunks + * will be set to null. ProcessBlock() will not be called on the engine + * again until SetActive() is called. + */ + void CheckForInactive(); protected: + class AdvanceAndResumeMessage; + + virtual void DestroyImpl() override; + void AdvanceOutputSegment(); void FinishOutput(); - void AccumulateInputChunk(uint32_t aInputIndex, const AudioChunk& aChunk, - AudioChunk* aBlock, + void AccumulateInputChunk(uint32_t aInputIndex, const AudioBlock& aChunk, + AudioBlock* aBlock, nsTArray* aDownmixBuffer); - void UpMixDownMixChunk(const AudioChunk* aChunk, uint32_t aOutputChannelCount, + void UpMixDownMixChunk(const AudioBlock* aChunk, uint32_t aOutputChannelCount, nsTArray& aOutputChannels, nsTArray& aDownmixBuffer); uint32_t ComputedNumberOfChannels(uint32_t aInputChannelCount); - void ObtainInputBlock(AudioChunk& aTmpChunk, uint32_t aPortIndex); + void ObtainInputBlock(AudioBlock& aTmpChunk, uint32_t aPortIndex); + void IncrementActiveInputCount(); + void DecrementActiveInputCount(); // The engine that will generate output for this node. nsAutoPtr mEngine; @@ -198,16 +201,18 @@ protected: OutputChunks mLastChunks; // The stream's sampling rate const TrackRate mSampleRate; - // This is necessary to be able to find all the nodes for a given - // AudioContext. It is set on the main thread, in the constructor. - const AudioContext::AudioContextId mAudioContextId; // Whether this is an internal or external stream const Flags mFlags; + // The number of input streams that may provide non-silent input. + uint32_t mActiveInputCount = 0; // The number of input channels that this stream requires. 0 means don't care. uint32_t mNumberOfInputChannels; // The mixing modes ChannelCountMode mChannelCountMode; ChannelInterpretation mChannelInterpretation; + // Streams are considered active if the stream has not finished and either + // the engine is active or there are active input streams. + bool mIsActive; // Whether the stream should be marked as finished as soon // as the current time range has been computed block by block. bool mMarkAsFinishedAfterThisBlock; diff --git a/dom/media/webaudio/AudioParam.cpp b/dom/media/webaudio/AudioParam.cpp index 7fb72164d1..04f833a23e 100644 --- a/dom/media/webaudio/AudioParam.cpp +++ b/dom/media/webaudio/AudioParam.cpp @@ -44,14 +44,14 @@ NS_IMPL_CYCLE_COLLECTION_ROOT_NATIVE(AudioParam, AddRef) NS_IMPL_CYCLE_COLLECTION_UNROOT_NATIVE(AudioParam, Release) AudioParam::AudioParam(AudioNode* aNode, - AudioParam::CallbackType aCallback, + uint32_t aIndex, float aDefaultValue, const char* aName) : AudioParamTimeline(aDefaultValue) , mNode(aNode) - , mCallback(aCallback) - , mDefaultValue(aDefaultValue) , mName(aName) + , mIndex(aIndex) + , mDefaultValue(aDefaultValue) { } @@ -100,7 +100,7 @@ AudioParam::Stream() AudioNodeEngine* engine = new AudioNodeEngine(nullptr); RefPtr stream = - AudioNodeStream::Create(mNode->Context()->Graph(), engine, + AudioNodeStream::Create(mNode->Context(), engine, AudioNodeStream::NO_STREAM_FLAGS); // Force the input to have only one channel, and make it down-mix using @@ -115,15 +115,25 @@ AudioParam::Stream() AudioNodeStream* nodeStream = mNode->GetStream(); if (nodeStream) { mNodeStreamPort = - nodeStream->AllocateInputPort(mStream, MediaInputPort::FLAG_BLOCK_INPUT); + nodeStream->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK); } - // Let the MSG's copy of AudioParamTimeline know about the change in the stream - mCallback(mNode); + // Send the stream to the timeline on the MSG side. + AudioTimelineEvent event(mStream); + SendEventToEngine(event); return mStream; } +void +AudioParam::SendEventToEngine(const AudioTimelineEvent& aEvent) +{ + AudioNodeStream* stream = mNode->GetStream(); + if (stream) { + stream->SendTimelineEvent(mIndex, aEvent); + } +} + float AudioParamTimeline::AudioNodeInputValue(size_t aCounter) const { @@ -133,7 +143,7 @@ AudioParamTimeline::AudioNodeInputValue(size_t aCounter) const // get its value now. We use aCounter to tell us which frame of the last // AudioChunk to look at. float audioNodeInputValue = 0.0f; - const AudioChunk& lastAudioNodeChunk = + const AudioBlock& lastAudioNodeChunk = static_cast(mStream.get())->LastChunks()[0]; if (!lastAudioNodeChunk.IsNull()) { MOZ_ASSERT(lastAudioNodeChunk.GetDuration() == WEBAUDIO_BLOCK_SIZE); diff --git a/dom/media/webaudio/AudioParam.h b/dom/media/webaudio/AudioParam.h index 89f99de0e0..497038b9d8 100644 --- a/dom/media/webaudio/AudioParam.h +++ b/dom/media/webaudio/AudioParam.h @@ -26,10 +26,8 @@ class AudioParam final : public nsWrapperCache, virtual ~AudioParam(); public: - typedef void (*CallbackType)(AudioNode*); - AudioParam(AudioNode* aNode, - CallbackType aCallback, + uint32_t aIndex, float aDefaultValue, const char* aName); @@ -42,11 +40,6 @@ public: return mNode->Context(); } - double DOMTimeToStreamTime(double aTime) const - { - return mNode->Context()->DOMTimeToStreamTime(aTime); - } - virtual JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; // We override SetValueCurveAtTime to convert the Float32Array to the wrapper @@ -58,68 +51,83 @@ public: return; } aValues.ComputeLengthAndData(); - AudioParamTimeline::SetValueCurveAtTime(aValues.Data(), aValues.Length(), - DOMTimeToStreamTime(aStartTime), aDuration, aRv); - mCallback(mNode); + + EventInsertionHelper(aRv, AudioTimelineEvent::SetValueCurve, + aStartTime, 0.0f, 0.0f, aDuration, aValues.Data(), + aValues.Length()); } - // We override the rest of the mutating AudioParamTimeline methods in order to make - // sure that the callback is called every time that this object gets mutated. void SetValue(float aValue) { - // Optimize away setting the same value on an AudioParam - if (HasSimpleValue() && - WebAudioUtils::FuzzyEqual(GetValue(), aValue)) { + AudioTimelineEvent event(AudioTimelineEvent::SetValue, 0.0f, aValue); + + ErrorResult rv; + if (!ValidateEvent(event, rv)) { + MOZ_ASSERT(false, "This should not happen, " + "setting the value should always work"); return; } + AudioParamTimeline::SetValue(aValue); - mCallback(mNode); + + SendEventToEngine(event); } + void SetValueAtTime(float aValue, double aStartTime, ErrorResult& aRv) { if (!WebAudioUtils::IsTimeValid(aStartTime)) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } - AudioParamTimeline::SetValueAtTime(aValue, DOMTimeToStreamTime(aStartTime), aRv); - mCallback(mNode); + EventInsertionHelper(aRv, AudioTimelineEvent::SetValueAtTime, + aStartTime, aValue); } + void LinearRampToValueAtTime(float aValue, double aEndTime, ErrorResult& aRv) { if (!WebAudioUtils::IsTimeValid(aEndTime)) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } - AudioParamTimeline::LinearRampToValueAtTime(aValue, DOMTimeToStreamTime(aEndTime), aRv); - mCallback(mNode); + EventInsertionHelper(aRv, AudioTimelineEvent::LinearRamp, aEndTime, aValue); } + void ExponentialRampToValueAtTime(float aValue, double aEndTime, ErrorResult& aRv) { if (!WebAudioUtils::IsTimeValid(aEndTime)) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } - AudioParamTimeline::ExponentialRampToValueAtTime(aValue, DOMTimeToStreamTime(aEndTime), aRv); - mCallback(mNode); + EventInsertionHelper(aRv, AudioTimelineEvent::ExponentialRamp, + aEndTime, aValue); } - void SetTargetAtTime(float aTarget, double aStartTime, double aTimeConstant, ErrorResult& aRv) + + void SetTargetAtTime(float aTarget, double aStartTime, + double aTimeConstant, ErrorResult& aRv) { if (!WebAudioUtils::IsTimeValid(aStartTime) || !WebAudioUtils::IsTimeValid(aTimeConstant)) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } - AudioParamTimeline::SetTargetAtTime(aTarget, DOMTimeToStreamTime(aStartTime), aTimeConstant, aRv); - mCallback(mNode); + EventInsertionHelper(aRv, AudioTimelineEvent::SetTarget, + aStartTime, aTarget, + aTimeConstant); } + void CancelScheduledValues(double aStartTime, ErrorResult& aRv) { if (!WebAudioUtils::IsTimeValid(aStartTime)) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } - AudioParamTimeline::CancelScheduledValues(DOMTimeToStreamTime(aStartTime)); - mCallback(mNode); + + // Remove some events on the main thread copy. + AudioEventTimeline::CancelScheduledValues(aStartTime); + + AudioTimelineEvent event(AudioTimelineEvent::Cancel, aStartTime, 0.0f); + + SendEventToEngine(event); } uint32_t ParentNodeId() @@ -183,20 +191,40 @@ public: return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } -protected: +private: + void EventInsertionHelper(ErrorResult& aRv, + AudioTimelineEvent::Type aType, + double aTime, float aValue, + double aTimeConstant = 0.0, + float aDuration = 0.0, + const float* aCurve = nullptr, + uint32_t aCurveLength = 0) + { + AudioTimelineEvent event(aType, aTime, aValue, + aTimeConstant, aDuration, aCurve, aCurveLength); + + if (!ValidateEvent(event, aRv)) { + return; + } + + AudioEventTimeline::InsertEvent(event); + + SendEventToEngine(event); + } + + void SendEventToEngine(const AudioTimelineEvent& aEvent); + nsCycleCollectingAutoRefCnt mRefCnt; NS_DECL_OWNINGTHREAD - -private: RefPtr mNode; // For every InputNode, there is a corresponding entry in mOutputParams of the // InputNode's mInputNode. nsTArray mInputNodes; - CallbackType mCallback; - const float mDefaultValue; const char* mName; // The input port used to connect the AudioParam's stream to its node's stream RefPtr mNodeStreamPort; + const uint32_t mIndex; + const float mDefaultValue; }; } // namespace dom diff --git a/dom/media/webaudio/AudioParamTimeline.h b/dom/media/webaudio/AudioParamTimeline.h index d298b7e1db..1070718558 100644 --- a/dom/media/webaudio/AudioParamTimeline.h +++ b/dom/media/webaudio/AudioParamTimeline.h @@ -44,13 +44,44 @@ public: return BaseClass::HasSimpleValue() && !mStream; } + template + float GetValueAtTime(TimeType aTime) + { + return GetValueAtTime(aTime, 0); + } + + template + void InsertEvent(const AudioTimelineEvent& aEvent) + { + if (aEvent.mType == AudioTimelineEvent::Cancel) { + CancelScheduledValues(aEvent.template Time()); + return; + } + if (aEvent.mType == AudioTimelineEvent::Stream) { + mStream = aEvent.mStream; + return; + } + if (aEvent.mType == AudioTimelineEvent::SetValue) { + AudioEventTimeline::SetValue(aEvent.mValue); + return; + } + AudioEventTimeline::InsertEvent(aEvent); + } + // Get the value of the AudioParam at time aTime + aCounter. // aCounter here is an offset to aTime if we try to get the value in ticks, // otherwise it should always be zero. aCounter is meant to be used when + template + float GetValueAtTime(TimeType aTime, size_t aCounter); + + // Get the values of the AudioParam at time aTime + (0 to aSize). + // aBuffer must have the correct aSize. + // aSize here is an offset to aTime if we try to get the value in ticks, + // otherwise it should always be zero. aSize is meant to be used when // getting the value of an a-rate AudioParam for each tick inside an // AudioNodeEngine implementation. template - float GetValueAtTime(TimeType aTime, size_t aCounter = 0); + void GetValuesAtTime(TimeType aTime, float* aBuffer, const size_t aSize); virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { @@ -81,7 +112,6 @@ AudioParamTimeline::GetValueAtTime(double aTime, size_t aCounter) return BaseClass::GetValueAtTime(aTime); } - template<> inline float AudioParamTimeline::GetValueAtTime(int64_t aTime, size_t aCounter) { @@ -93,8 +123,36 @@ AudioParamTimeline::GetValueAtTime(int64_t aTime, size_t aCounter) (mStream ? AudioNodeInputValue(aCounter) : 0.0f); } +template<> inline void +AudioParamTimeline::GetValuesAtTime(double aTime, float* aBuffer, + const size_t aSize) +{ + MOZ_ASSERT(aBuffer); + MOZ_ASSERT(aSize == 1); + + // Getting an AudioParam value on an AudioNode does not consider input from + // other AudioNodes, which is managed only on the graph thread. + *aBuffer = BaseClass::GetValueAtTime(aTime); +} + +template<> inline void +AudioParamTimeline::GetValuesAtTime(int64_t aTime, float* aBuffer, + const size_t aSize) +{ + MOZ_ASSERT(aBuffer); + MOZ_ASSERT(aSize <= WEBAUDIO_BLOCK_SIZE); + MOZ_ASSERT(aSize == 1 || !HasSimpleValue()); + + // Mix the value of the AudioParam itself with that of the AudioNode inputs. + BaseClass::GetValuesAtTime(aTime, aBuffer, aSize); + if (mStream) { + for (size_t i = 0; i < aSize; ++i) { + aBuffer[i] += AudioNodeInputValue(i); + } + } +} + } // namespace dom } // namespace mozilla #endif - diff --git a/dom/media/webaudio/BiquadFilterNode.cpp b/dom/media/webaudio/BiquadFilterNode.cpp index 5526bc36a4..3d69342dd3 100644 --- a/dom/media/webaudio/BiquadFilterNode.cpp +++ b/dom/media/webaudio/BiquadFilterNode.cpp @@ -78,7 +78,6 @@ class BiquadFilterNodeEngine final : public AudioNodeEngine public: BiquadFilterNodeEngine(AudioNode* aNode, AudioDestinationNode* aDestination) : AudioNodeEngine(aNode) - , mSource(nullptr) , mDestination(aDestination->Stream()) // Keep the default values in sync with the default values in // BiquadFilterNode::BiquadFilterNode @@ -90,11 +89,6 @@ public: { } - void SetSourceStream(AudioNodeStream* aSource) - { - mSource = aSource; - } - enum Parameteres { TYPE, FREQUENCY, @@ -110,27 +104,26 @@ public: NS_ERROR("Bad BiquadFilterNode Int32Parameter"); } } - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { - MOZ_ASSERT(mSource && mDestination); + MOZ_ASSERT(mDestination); + + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case FREQUENCY: - mFrequency = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mFrequency, mSource, mDestination); + mFrequency.InsertEvent(aEvent); break; case DETUNE: - mDetune = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mDetune, mSource, mDestination); + mDetune.InsertEvent(aEvent); break; case Q: - mQ = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mQ, mSource, mDestination); + mQ.InsertEvent(aEvent); break; case GAIN: - mGain = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mGain, mSource, mDestination); + mGain.InsertEvent(aEvent); break; default: NS_ERROR("Bad BiquadFilterNodeEngine TimelineParameter"); @@ -138,8 +131,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { float inputBuffer[WEBAUDIO_BLOCK_SIZE]; @@ -155,6 +149,7 @@ public: if (!hasTail) { if (!mBiquads.IsEmpty()) { mBiquads.Clear(); + aStream->CheckForInactive(); RefPtr refchanged = new PlayingRefChangeHandler(aStream, PlayingRefChangeHandler::RELEASE); @@ -168,7 +163,7 @@ public: PodArrayZero(inputBuffer); - } else if(mBiquads.Length() != aInput.mChannelData.Length()){ + } else if(mBiquads.Length() != aInput.ChannelCount()){ if (mBiquads.IsEmpty()) { RefPtr refchanged = new PlayingRefChangeHandler(aStream, PlayingRefChangeHandler::ADDREF); @@ -179,13 +174,13 @@ public: } // Adjust the number of biquads based on the number of channels - mBiquads.SetLength(aInput.mChannelData.Length()); + mBiquads.SetLength(aInput.ChannelCount()); } uint32_t numberOfChannels = mBiquads.Length(); - AllocateAudioBlock(numberOfChannels, aOutput); + aOutput->AllocateChannels(numberOfChannels); - StreamTime pos = aStream->GetCurrentPosition(); + StreamTime pos = mDestination->GraphTimeToStreamTime(aFrom); double freq = mFrequency.GetValueAtTime(pos); double q = mQ.GetValueAtTime(pos); @@ -211,10 +206,14 @@ public: } } + virtual bool IsActive() const override + { + return !mBiquads.IsEmpty(); + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { // Not owned: - // - mSource - probably not owned // - mDestination - probably not owned // - AudioParamTimelines - counted in the AudioNode size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); @@ -228,7 +227,6 @@ public: } private: - AudioNodeStream* mSource; AudioNodeStream* mDestination; BiquadFilterType mType; AudioParamTimeline mFrequency; @@ -244,15 +242,15 @@ BiquadFilterNode::BiquadFilterNode(AudioContext* aContext) ChannelCountMode::Max, ChannelInterpretation::Speakers) , mType(BiquadFilterType::Lowpass) - , mFrequency(new AudioParam(this, SendFrequencyToStream, 350.f, "frequency")) - , mDetune(new AudioParam(this, SendDetuneToStream, 0.f, "detune")) - , mQ(new AudioParam(this, SendQToStream, 1.f, "Q")) - , mGain(new AudioParam(this, SendGainToStream, 0.f, "gain")) + , mFrequency(new AudioParam(this, BiquadFilterNodeEngine::FREQUENCY, + 350.f, "frequency")) + , mDetune(new AudioParam(this, BiquadFilterNodeEngine::DETUNE, 0.f, "detune")) + , mQ(new AudioParam(this, BiquadFilterNodeEngine::Q, 1.f, "Q")) + , mGain(new AudioParam(this, BiquadFilterNodeEngine::GAIN, 0.f, "gain")) { BiquadFilterNodeEngine* engine = new BiquadFilterNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); - engine->SetSourceStream(mStream); } BiquadFilterNode::~BiquadFilterNode() @@ -339,33 +337,5 @@ BiquadFilterNode::GetFrequencyResponse(const Float32Array& aFrequencyHz, biquad.getFrequencyResponse(int(length), frequencies, aMagResponse.Data(), aPhaseResponse.Data()); } -void -BiquadFilterNode::SendFrequencyToStream(AudioNode* aNode) -{ - BiquadFilterNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, BiquadFilterNodeEngine::FREQUENCY, *This->mFrequency); -} - -void -BiquadFilterNode::SendDetuneToStream(AudioNode* aNode) -{ - BiquadFilterNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, BiquadFilterNodeEngine::DETUNE, *This->mDetune); -} - -void -BiquadFilterNode::SendQToStream(AudioNode* aNode) -{ - BiquadFilterNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, BiquadFilterNodeEngine::Q, *This->mQ); -} - -void -BiquadFilterNode::SendGainToStream(AudioNode* aNode) -{ - BiquadFilterNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, BiquadFilterNodeEngine::GAIN, *This->mGain); -} - } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/BiquadFilterNode.h b/dom/media/webaudio/BiquadFilterNode.h index 34e5403441..5a614bf784 100644 --- a/dom/media/webaudio/BiquadFilterNode.h +++ b/dom/media/webaudio/BiquadFilterNode.h @@ -67,12 +67,6 @@ public: protected: virtual ~BiquadFilterNode(); -private: - static void SendFrequencyToStream(AudioNode* aNode); - static void SendDetuneToStream(AudioNode* aNode); - static void SendQToStream(AudioNode* aNode); - static void SendGainToStream(AudioNode* aNode); - private: BiquadFilterType mType; RefPtr mFrequency; diff --git a/dom/media/webaudio/ChannelMergerNode.cpp b/dom/media/webaudio/ChannelMergerNode.cpp index 0de9087523..65e4ab8d11 100644 --- a/dom/media/webaudio/ChannelMergerNode.cpp +++ b/dom/media/webaudio/ChannelMergerNode.cpp @@ -33,20 +33,20 @@ public: // Get the number of output channels, and allocate it size_t channelCount = 0; for (uint16_t i = 0; i < InputCount(); ++i) { - channelCount += aInput[i].mChannelData.Length(); + channelCount += aInput[i].ChannelCount(); } if (channelCount == 0) { aOutput[0].SetNull(WEBAUDIO_BLOCK_SIZE); return; } channelCount = std::min(channelCount, WebAudioUtils::MaxChannelCount); - AllocateAudioBlock(channelCount, &aOutput[0]); + aOutput[0].AllocateChannels(channelCount); // Append each channel in each input to the output size_t channelIndex = 0; for (uint16_t i = 0; true; ++i) { MOZ_ASSERT(i < InputCount()); - for (size_t j = 0; j < aInput[i].mChannelData.Length(); ++j) { + for (size_t j = 0; j < aInput[i].ChannelCount(); ++j) { AudioBlockCopyChannelWithScale( static_cast(aInput[i].mChannelData[j]), aInput[i].mVolume, @@ -73,7 +73,7 @@ ChannelMergerNode::ChannelMergerNode(AudioContext* aContext, ChannelInterpretation::Speakers) , mInputCount(aInputCount) { - mStream = AudioNodeStream::Create(aContext->Graph(), + mStream = AudioNodeStream::Create(aContext, new ChannelMergerNodeEngine(this), AudioNodeStream::NO_STREAM_FLAGS); } diff --git a/dom/media/webaudio/ChannelSplitterNode.cpp b/dom/media/webaudio/ChannelSplitterNode.cpp index 6c8db5e81f..8bf1d3bddc 100644 --- a/dom/media/webaudio/ChannelSplitterNode.cpp +++ b/dom/media/webaudio/ChannelSplitterNode.cpp @@ -32,9 +32,9 @@ public: aOutput.SetLength(OutputCount()); for (uint16_t i = 0; i < OutputCount(); ++i) { - if (i < aInput[0].mChannelData.Length()) { + if (i < aInput[0].ChannelCount()) { // Split out existing channels - AllocateAudioBlock(1, &aOutput[i]); + aOutput[i].AllocateChannels(1); AudioBlockCopyChannelWithScale( static_cast(aInput[0].mChannelData[i]), aInput[0].mVolume, @@ -60,7 +60,7 @@ ChannelSplitterNode::ChannelSplitterNode(AudioContext* aContext, ChannelInterpretation::Speakers) , mOutputCount(aOutputCount) { - mStream = AudioNodeStream::Create(aContext->Graph(), + mStream = AudioNodeStream::Create(aContext, new ChannelSplitterNodeEngine(this), AudioNodeStream::NO_STREAM_FLAGS); } diff --git a/dom/media/webaudio/ConvolverNode.cpp b/dom/media/webaudio/ConvolverNode.cpp index 930414d2c1..3019d2c16c 100644 --- a/dom/media/webaudio/ConvolverNode.cpp +++ b/dom/media/webaudio/ConvolverNode.cpp @@ -102,8 +102,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { if (!mReverb) { @@ -111,15 +112,16 @@ public: return; } - AudioChunk input = aInput; + AudioBlock input = aInput; if (aInput.IsNull()) { if (mLeftOverData > 0) { mLeftOverData -= WEBAUDIO_BLOCK_SIZE; - AllocateAudioBlock(1, &input); + input.AllocateChannels(1); WriteZeroesToAudioBlock(&input, 0, WEBAUDIO_BLOCK_SIZE); } else { if (mLeftOverData != INT32_MIN) { mLeftOverData = INT32_MIN; + aStream->CheckForInactive(); RefPtr refchanged = new PlayingRefChanged(aStream, PlayingRefChanged::RELEASE); aStream->Graph()-> @@ -131,8 +133,8 @@ public: } else { if (aInput.mVolume != 1.0f) { // Pre-multiply the input's volume - uint32_t numChannels = aInput.mChannelData.Length(); - AllocateAudioBlock(numChannels, &input); + uint32_t numChannels = aInput.ChannelCount(); + input.AllocateChannels(numChannels); for (uint32_t i = 0; i < numChannels; ++i) { const float* src = static_cast(aInput.mChannelData[i]); float* dest = input.ChannelFloatsForWrite(i); @@ -149,11 +151,16 @@ public: mLeftOverData = mBufferLength; MOZ_ASSERT(mLeftOverData > 0); } - AllocateAudioBlock(2, aOutput); + aOutput->AllocateChannels(2); mReverb->process(&input, aOutput, WEBAUDIO_BLOCK_SIZE); } + virtual bool IsActive() const override + { + return mLeftOverData != INT32_MIN; + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); @@ -191,7 +198,7 @@ ConvolverNode::ConvolverNode(AudioContext* aContext) , mNormalize(true) { ConvolverNodeEngine* engine = new ConvolverNodeEngine(this, mNormalize); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); } diff --git a/dom/media/webaudio/DelayBuffer.cpp b/dom/media/webaudio/DelayBuffer.cpp index d21e363354..9bc15cf222 100644 --- a/dom/media/webaudio/DelayBuffer.cpp +++ b/dom/media/webaudio/DelayBuffer.cpp @@ -26,10 +26,10 @@ DelayBuffer::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const } void -DelayBuffer::Write(const AudioChunk& aInputChunk) +DelayBuffer::Write(const AudioBlock& aInputChunk) { // We must have a reference to the buffer if there are channels - MOZ_ASSERT(aInputChunk.IsNull() == !aInputChunk.mChannelData.Length()); + MOZ_ASSERT(aInputChunk.IsNull() == !aInputChunk.ChannelCount()); #ifdef DEBUG MOZ_ASSERT(!mHaveWrittenBlock); mHaveWrittenBlock = true; @@ -42,12 +42,12 @@ DelayBuffer::Write(const AudioChunk& aInputChunk) if (mCurrentChunk == mLastReadChunk) { mLastReadChunk = -1; // invalidate cache } - mChunks[mCurrentChunk] = aInputChunk; + mChunks[mCurrentChunk] = aInputChunk.AsAudioChunk(); } void DelayBuffer::Read(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, + AudioBlock* aOutputChunk, ChannelInterpretation aChannelInterpretation) { int chunkCount = mChunks.Length(); @@ -84,7 +84,7 @@ DelayBuffer::Read(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], } if (channelCount) { - AllocateAudioBlock(channelCount, aOutputChunk); + aOutputChunk->AllocateChannels(channelCount); ReadChannels(aPerFrameDelays, aOutputChunk, 0, channelCount, aChannelInterpretation); } else { @@ -97,7 +97,7 @@ DelayBuffer::Read(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], void DelayBuffer::ReadChannel(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, uint32_t aChannel, + AudioBlock* aOutputChunk, uint32_t aChannel, ChannelInterpretation aChannelInterpretation) { if (!mChunks.Length()) { @@ -112,11 +112,11 @@ DelayBuffer::ReadChannel(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], void DelayBuffer::ReadChannels(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, + AudioBlock* aOutputChunk, uint32_t aFirstChannel, uint32_t aNumChannelsToRead, ChannelInterpretation aChannelInterpretation) { - uint32_t totalChannelCount = aOutputChunk->mChannelData.Length(); + uint32_t totalChannelCount = aOutputChunk->ChannelCount(); uint32_t readChannelsEnd = aFirstChannel + aNumChannelsToRead; MOZ_ASSERT(readChannelsEnd <= totalChannelCount); @@ -166,7 +166,7 @@ DelayBuffer::ReadChannels(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], } void -DelayBuffer::Read(double aDelayTicks, AudioChunk* aOutputChunk, +DelayBuffer::Read(double aDelayTicks, AudioBlock* aOutputChunk, ChannelInterpretation aChannelInterpretation) { const bool firstTime = mCurrentDelay < 0.0; diff --git a/dom/media/webaudio/DelayBuffer.h b/dom/media/webaudio/DelayBuffer.h index fef37a81d6..6109e038f2 100644 --- a/dom/media/webaudio/DelayBuffer.h +++ b/dom/media/webaudio/DelayBuffer.h @@ -39,16 +39,16 @@ public: } // Write a WEBAUDIO_BLOCK_SIZE block for aChannelCount channels. - void Write(const AudioChunk& aInputChunk); + void Write(const AudioBlock& aInputChunk); // Read a block with an array of delays, in ticks, for each sample frame. // Each delay should be >= 0 and <= MaxDelayTicks(). void Read(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, + AudioBlock* aOutputChunk, ChannelInterpretation aChannelInterpretation); // Read a block with a constant delay, which will be smoothed with the // previous delay. The delay should be >= 0 and <= MaxDelayTicks(). - void Read(double aDelayTicks, AudioChunk* aOutputChunk, + void Read(double aDelayTicks, AudioBlock* aOutputChunk, ChannelInterpretation aChannelInterpretation); // Read into one of the channels of aOutputChunk, given an array of @@ -56,7 +56,7 @@ public: // channels. aOutputChunk must have already been allocated with at least as // many channels as were in any of the blocks passed to Write(). void ReadChannel(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, uint32_t aChannel, + AudioBlock* aOutputChunk, uint32_t aChannel, ChannelInterpretation aChannelInterpretation); // Advance the buffer pointer @@ -80,7 +80,7 @@ public: private: void ReadChannels(const double aPerFrameDelays[WEBAUDIO_BLOCK_SIZE], - AudioChunk* aOutputChunk, + AudioBlock* aOutputChunk, uint32_t aFirstChannel, uint32_t aNumChannelsToRead, ChannelInterpretation aChannelInterpretation); bool EnsureBuffer(); diff --git a/dom/media/webaudio/DelayNode.cpp b/dom/media/webaudio/DelayNode.cpp index 880b82e65a..f878db6dbe 100644 --- a/dom/media/webaudio/DelayNode.cpp +++ b/dom/media/webaudio/DelayNode.cpp @@ -60,15 +60,16 @@ public: enum Parameters { DELAY, }; - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { + MOZ_ASSERT(mDestination); + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case DELAY: - MOZ_ASSERT(mSource && mDestination); - mDelay = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mDelay, mSource, mDestination); + mDelay.InsertEvent(aEvent); break; default: NS_ERROR("Bad DelayNodeEngine TimelineParameter"); @@ -76,8 +77,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { MOZ_ASSERT(mSource == aStream, "Invalid source stream"); @@ -96,6 +98,8 @@ public: } else { if (mLeftOverData != INT32_MIN) { mLeftOverData = INT32_MIN; + aStream->CheckForInactive(); + // Delete our buffered data now we no longer need it mBuffer.Reset(); @@ -104,7 +108,7 @@ public: aStream->Graph()-> DispatchToMainThreadAfterStreamStateUpdate(refchanged.forget()); } - *aOutput = aInput; + aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); return; } @@ -113,13 +117,13 @@ public: // Skip output update if mLastChunks has already been set by // ProduceBlockBeforeInput() when in a cycle. if (!mHaveProducedBeforeInput) { - UpdateOutputBlock(aOutput, 0.0); + UpdateOutputBlock(aFrom, aOutput, 0.0); } mHaveProducedBeforeInput = false; mBuffer.NextBlock(); } - void UpdateOutputBlock(AudioChunk* aOutput, double minDelay) + void UpdateOutputBlock(GraphTime aFrom, AudioBlock* aOutput, double minDelay) { double maxDelay = mMaxDelay; double sampleRate = mSource->SampleRate(); @@ -136,10 +140,13 @@ public: // Compute the delay values for the duration of the input AudioChunk // If this DelayNode is in a cycle, make sure the delay value is at least // one block. - StreamTime tick = mSource->GetCurrentPosition(); + StreamTime tick = mDestination->GraphTimeToStreamTime(aFrom); + float values[WEBAUDIO_BLOCK_SIZE]; + mDelay.GetValuesAtTime(tick, values,WEBAUDIO_BLOCK_SIZE); + double computedDelay[WEBAUDIO_BLOCK_SIZE]; for (size_t counter = 0; counter < WEBAUDIO_BLOCK_SIZE; ++counter) { - double delayAtTick = mDelay.GetValueAtTime(tick, counter) * sampleRate; + double delayAtTick = values[counter] * sampleRate; double delayAtTickClamped = std::max(minDelay, std::min(delayAtTick, maxDelay)); computedDelay[counter] = delayAtTickClamped; @@ -148,16 +155,22 @@ public: } } - virtual void ProduceBlockBeforeInput(AudioChunk* aOutput) override + virtual void ProduceBlockBeforeInput(GraphTime aFrom, + AudioBlock* aOutput) override { if (mLeftOverData <= 0) { aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); } else { - UpdateOutputBlock(aOutput, WEBAUDIO_BLOCK_SIZE); + UpdateOutputBlock(aFrom, aOutput, WEBAUDIO_BLOCK_SIZE); } mHaveProducedBeforeInput = true; } + virtual bool IsActive() const override + { + return mLeftOverData != INT32_MIN; + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); @@ -190,12 +203,12 @@ DelayNode::DelayNode(AudioContext* aContext, double aMaxDelay) 2, ChannelCountMode::Max, ChannelInterpretation::Speakers) - , mDelay(new AudioParam(this, SendDelayToStream, 0.0f, "delayTime")) + , mDelay(new AudioParam(this, DelayNodeEngine::DELAY, 0.0f, "delayTime")) { DelayNodeEngine* engine = new DelayNodeEngine(this, aContext->Destination(), aContext->SampleRate() * aMaxDelay); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); engine->SetSourceStream(mStream); } @@ -224,12 +237,5 @@ DelayNode::WrapObject(JSContext* aCx, JS::Handle aGivenProto) return DelayNodeBinding::Wrap(aCx, this, aGivenProto); } -void -DelayNode::SendDelayToStream(AudioNode* aNode) -{ - DelayNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DelayNodeEngine::DELAY, *This->mDelay); -} - } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/DelayNode.h b/dom/media/webaudio/DelayNode.h index efa6a3345b..433ad272fe 100644 --- a/dom/media/webaudio/DelayNode.h +++ b/dom/media/webaudio/DelayNode.h @@ -42,7 +42,6 @@ protected: virtual ~DelayNode(); private: - static void SendDelayToStream(AudioNode* aNode); friend class DelayNodeEngine; private: diff --git a/dom/media/webaudio/DynamicsCompressorNode.cpp b/dom/media/webaudio/DynamicsCompressorNode.cpp index 7b762b6b02..d23694a2d7 100644 --- a/dom/media/webaudio/DynamicsCompressorNode.cpp +++ b/dom/media/webaudio/DynamicsCompressorNode.cpp @@ -36,7 +36,6 @@ public: explicit DynamicsCompressorNodeEngine(AudioNode* aNode, AudioDestinationNode* aDestination) : AudioNodeEngine(aNode) - , mSource(nullptr) , mDestination(aDestination->Stream()) // Keep the default value in sync with the default value in // DynamicsCompressorNode::DynamicsCompressorNode. @@ -49,11 +48,6 @@ public: { } - void SetSourceStream(AudioNodeStream* aSource) - { - mSource = aSource; - } - enum Parameters { THRESHOLD, KNEE, @@ -61,31 +55,29 @@ public: ATTACK, RELEASE }; - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { - MOZ_ASSERT(mSource && mDestination); + MOZ_ASSERT(mDestination); + + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case THRESHOLD: - mThreshold = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mThreshold, mSource, mDestination); + mThreshold.InsertEvent(aEvent); break; case KNEE: - mKnee = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mKnee, mSource, mDestination); + mKnee.InsertEvent(aEvent); break; case RATIO: - mRatio = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mRatio, mSource, mDestination); + mRatio.InsertEvent(aEvent); break; case ATTACK: - mAttack = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mAttack, mSource, mDestination); + mAttack.InsertEvent(aEvent); break; case RELEASE: - mRelease = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mRelease, mSource, mDestination); + mRelease.InsertEvent(aEvent); break; default: NS_ERROR("Bad DynamicsCompresssorNodeEngine TimelineParameter"); @@ -93,8 +85,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { if (aInput.IsNull()) { @@ -103,14 +96,14 @@ public: return; } - const uint32_t channelCount = aInput.mChannelData.Length(); + const uint32_t channelCount = aInput.ChannelCount(); if (mCompressor->numberOfChannels() != channelCount) { // Create a new compressor object with a new channel count mCompressor = new WebCore::DynamicsCompressor(aStream->SampleRate(), - aInput.mChannelData.Length()); + aInput.ChannelCount()); } - StreamTime pos = aStream->GetCurrentPosition(); + StreamTime pos = mDestination->GraphTimeToStreamTime(aFrom); mCompressor->setParameterValue(DynamicsCompressor::ParamThreshold, mThreshold.GetValueAtTime(pos)); mCompressor->setParameterValue(DynamicsCompressor::ParamKnee, @@ -122,7 +115,7 @@ public: mCompressor->setParameterValue(DynamicsCompressor::ParamRelease, mRelease.GetValueAtTime(pos)); - AllocateAudioBlock(channelCount, aOutput); + aOutput->AllocateChannels(channelCount); mCompressor->process(&aInput, aOutput, aInput.GetDuration()); SendReductionParamToMainThread(aStream, @@ -132,7 +125,6 @@ public: virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { // Not owned: - // - mSource (probably) // - mDestination (probably) // - Don't count the AudioParamTimelines, their inner refs are owned by the // AudioNode. @@ -180,7 +172,6 @@ private: } private: - AudioNodeStream* mSource; AudioNodeStream* mDestination; AudioParamTimeline mThreshold; AudioParamTimeline mKnee; @@ -195,17 +186,21 @@ DynamicsCompressorNode::DynamicsCompressorNode(AudioContext* aContext) 2, ChannelCountMode::Explicit, ChannelInterpretation::Speakers) - , mThreshold(new AudioParam(this, SendThresholdToStream, -24.f, "threshold")) - , mKnee(new AudioParam(this, SendKneeToStream, 30.f, "knee")) - , mRatio(new AudioParam(this, SendRatioToStream, 12.f, "ratio")) + , mThreshold(new AudioParam(this, DynamicsCompressorNodeEngine::THRESHOLD, + -24.f, "threshold")) + , mKnee(new AudioParam(this, DynamicsCompressorNodeEngine::KNEE, + 30.f, "knee")) + , mRatio(new AudioParam(this, DynamicsCompressorNodeEngine::RATIO, + 12.f, "ratio")) , mReduction(0) - , mAttack(new AudioParam(this, SendAttackToStream, 0.003f, "attack")) - , mRelease(new AudioParam(this, SendReleaseToStream, 0.25f, "release")) + , mAttack(new AudioParam(this, DynamicsCompressorNodeEngine::ATTACK, + 0.003f, "attack")) + , mRelease(new AudioParam(this, DynamicsCompressorNodeEngine::RELEASE, + 0.25f, "release")) { DynamicsCompressorNodeEngine* engine = new DynamicsCompressorNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); - engine->SetSourceStream(mStream); } DynamicsCompressorNode::~DynamicsCompressorNode() @@ -236,40 +231,5 @@ DynamicsCompressorNode::WrapObject(JSContext* aCx, JS::Handle aGivenP return DynamicsCompressorNodeBinding::Wrap(aCx, this, aGivenProto); } -void -DynamicsCompressorNode::SendThresholdToStream(AudioNode* aNode) -{ - DynamicsCompressorNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DynamicsCompressorNodeEngine::THRESHOLD, *This->mThreshold); -} - -void -DynamicsCompressorNode::SendKneeToStream(AudioNode* aNode) -{ - DynamicsCompressorNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DynamicsCompressorNodeEngine::KNEE, *This->mKnee); -} - -void -DynamicsCompressorNode::SendRatioToStream(AudioNode* aNode) -{ - DynamicsCompressorNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DynamicsCompressorNodeEngine::RATIO, *This->mRatio); -} - -void -DynamicsCompressorNode::SendAttackToStream(AudioNode* aNode) -{ - DynamicsCompressorNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DynamicsCompressorNodeEngine::ATTACK, *This->mAttack); -} - -void -DynamicsCompressorNode::SendReleaseToStream(AudioNode* aNode) -{ - DynamicsCompressorNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, DynamicsCompressorNodeEngine::RELEASE, *This->mRelease); -} - } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/DynamicsCompressorNode.h b/dom/media/webaudio/DynamicsCompressorNode.h index 8fe6d2f60d..9e393105f6 100644 --- a/dom/media/webaudio/DynamicsCompressorNode.h +++ b/dom/media/webaudio/DynamicsCompressorNode.h @@ -73,13 +73,6 @@ public: protected: virtual ~DynamicsCompressorNode(); -private: - static void SendThresholdToStream(AudioNode* aNode); - static void SendKneeToStream(AudioNode* aNode); - static void SendRatioToStream(AudioNode* aNode); - static void SendAttackToStream(AudioNode* aNode); - static void SendReleaseToStream(AudioNode* aNode); - private: RefPtr mThreshold; RefPtr mKnee; diff --git a/dom/media/webaudio/GainNode.cpp b/dom/media/webaudio/GainNode.cpp index 56127aa7ab..c376be7003 100644 --- a/dom/media/webaudio/GainNode.cpp +++ b/dom/media/webaudio/GainNode.cpp @@ -28,30 +28,25 @@ class GainNodeEngine final : public AudioNodeEngine public: GainNodeEngine(AudioNode* aNode, AudioDestinationNode* aDestination) : AudioNodeEngine(aNode) - , mSource(nullptr) , mDestination(aDestination->Stream()) // Keep the default value in sync with the default value in GainNode::GainNode. , mGain(1.f) { } - void SetSourceStream(AudioNodeStream* aSource) - { - mSource = aSource; - } - enum Parameters { GAIN }; - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { + MOZ_ASSERT(mDestination); + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case GAIN: - MOZ_ASSERT(mSource && mDestination); - mGain = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mGain, mSource, mDestination); + mGain.InsertEvent(aEvent); break; default: NS_ERROR("Bad GainNodeEngine TimelineParameter"); @@ -59,12 +54,11 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { - MOZ_ASSERT(mSource == aStream, "Invalid source stream"); - if (aInput.IsNull()) { // If input is silent, so is the output aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); @@ -81,18 +75,19 @@ public: // First, compute a vector of gains for each track tick based on the // timeline at hand, and then for each channel, multiply the values // in the buffer with the gain vector. - AllocateAudioBlock(aInput.mChannelData.Length(), aOutput); + aOutput->AllocateChannels(aInput.ChannelCount()); // Compute the gain values for the duration of the input AudioChunk - // XXX we need to add a method to AudioEventTimeline to compute this buffer directly. + StreamTime tick = mDestination->GraphTimeToStreamTime(aFrom); float computedGain[WEBAUDIO_BLOCK_SIZE]; + mGain.GetValuesAtTime(tick, computedGain, WEBAUDIO_BLOCK_SIZE); + for (size_t counter = 0; counter < WEBAUDIO_BLOCK_SIZE; ++counter) { - StreamTime tick = aStream->GetCurrentPosition(); - computedGain[counter] = mGain.GetValueAtTime(tick, counter) * aInput.mVolume; + computedGain[counter] *= aInput.mVolume; } // Apply the gain to the output buffer - for (size_t channel = 0; channel < aOutput->mChannelData.Length(); ++channel) { + for (size_t channel = 0; channel < aOutput->ChannelCount(); ++channel) { const float* inputBuffer = static_cast (aInput.mChannelData[channel]); float* buffer = aOutput->ChannelFloatsForWrite(channel); AudioBlockCopyChannelWithScale(inputBuffer, computedGain, buffer); @@ -103,7 +98,6 @@ public: virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { // Not owned: - // - mSource (probably) // - mDestination (probably) // - mGain - Internal ref owned by AudioNode return AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); @@ -114,7 +108,6 @@ public: return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } - AudioNodeStream* mSource; AudioNodeStream* mDestination; AudioParamTimeline mGain; }; @@ -124,12 +117,11 @@ GainNode::GainNode(AudioContext* aContext) 2, ChannelCountMode::Max, ChannelInterpretation::Speakers) - , mGain(new AudioParam(this, SendGainToStream, 1.0f, "gain")) + , mGain(new AudioParam(this, GainNodeEngine::GAIN, 1.0f, "gain")) { GainNodeEngine* engine = new GainNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); - engine->SetSourceStream(mStream); } GainNode::~GainNode() @@ -156,12 +148,5 @@ GainNode::WrapObject(JSContext* aCx, JS::Handle aGivenProto) return GainNodeBinding::Wrap(aCx, this, aGivenProto); } -void -GainNode::SendGainToStream(AudioNode* aNode) -{ - GainNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, GainNodeEngine::GAIN, *This->mGain); -} - } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/GainNode.h b/dom/media/webaudio/GainNode.h index bf7e5f01da..1baed74662 100644 --- a/dom/media/webaudio/GainNode.h +++ b/dom/media/webaudio/GainNode.h @@ -41,9 +41,6 @@ public: protected: virtual ~GainNode(); -private: - static void SendGainToStream(AudioNode* aNode); - private: RefPtr mGain; }; diff --git a/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp b/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp index 339ab0ba01..69c79e7e55 100644 --- a/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp +++ b/dom/media/webaudio/MediaStreamAudioDestinationNode.cpp @@ -39,9 +39,9 @@ MediaStreamAudioDestinationNode::MediaStreamAudioDestinationNode(AudioContext* a ProcessedMediaStream* outputStream = mDOMStream->GetStream()->AsProcessedStream(); MOZ_ASSERT(!!outputStream); AudioNodeEngine* engine = new AudioNodeEngine(this); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::EXTERNAL_OUTPUT); - mPort = outputStream->AllocateInputPort(mStream); + mPort = outputStream->AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK); nsIDocument* doc = aContext->GetParentObject()->GetExtantDoc(); if (doc) { diff --git a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp index c059a11313..d78007505e 100644 --- a/dom/media/webaudio/MediaStreamAudioSourceNode.cpp +++ b/dom/media/webaudio/MediaStreamAudioSourceNode.cpp @@ -8,6 +8,7 @@ #include "mozilla/dom/MediaStreamAudioSourceNodeBinding.h" #include "AudioNodeEngine.h" #include "AudioNodeExternalInputStream.h" +#include "AudioStreamTrack.h" #include "nsIDocument.h" #include "mozilla/CORSMode.h" @@ -68,8 +69,7 @@ MediaStreamAudioSourceNode::Init(DOMMediaStream* aMediaStream, ErrorResult& aRv) AudioNodeEngine* engine = new MediaStreamAudioSourceNodeEngine(this); mStream = AudioNodeExternalInputStream::Create(graph, engine); ProcessedMediaStream* outputStream = static_cast(mStream.get()); - mInputPort = outputStream->AllocateInputPort(inputStream, - MediaInputPort::FLAG_BLOCK_INPUT); + mInputPort = outputStream->AllocateInputPort(aMediaStream->GetStream()); mInputStream->AddConsumerToKeepAlive(static_cast(this)); PrincipalChanged(mInputStream); // trigger enabling/disabling of the connector diff --git a/dom/media/webaudio/OscillatorNode.cpp b/dom/media/webaudio/OscillatorNode.cpp index 0e3c42444a..2df814f4fc 100644 --- a/dom/media/webaudio/OscillatorNode.cpp +++ b/dom/media/webaudio/OscillatorNode.cpp @@ -57,21 +57,22 @@ public: START, STOP, }; - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { mRecomputeParameters = true; + + MOZ_ASSERT(mDestination); + + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case FREQUENCY: - MOZ_ASSERT(mSource && mDestination); - mFrequency = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mFrequency, mSource, mDestination); + mFrequency.InsertEvent(aEvent); break; case DETUNE: - MOZ_ASSERT(mSource && mDestination); - mDetune = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mDetune, mSource, mDestination); + mDetune.InsertEvent(aEvent); break; default: NS_ERROR("Bad OscillatorNodeEngine TimelineParameter"); @@ -81,7 +82,10 @@ public: virtual void SetStreamTimeParameter(uint32_t aIndex, StreamTime aParam) override { switch (aIndex) { - case START: mStart = aParam; break; + case START: + mStart = aParam; + mSource->SetActive(); + break; case STOP: mStop = aParam; break; default: NS_ERROR("Bad OscillatorNodeEngine StreamTimeParameter"); @@ -277,19 +281,20 @@ public: } } - void ComputeSilence(AudioChunk *aOutput) + void ComputeSilence(AudioBlock *aOutput) { aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { MOZ_ASSERT(mSource == aStream, "Invalid source stream"); - StreamTime ticks = aStream->GetCurrentPosition(); + StreamTime ticks = mDestination->GraphTimeToStreamTime(aFrom); if (mStart == -1) { ComputeSilence(aOutput); return; @@ -307,7 +312,7 @@ public: return; } - AllocateAudioBlock(1, aOutput); + aOutput->AllocateChannels(1); float* output = aOutput->ChannelFloatsForWrite(0); uint32_t start, end; @@ -330,6 +335,12 @@ public: } + virtual bool IsActive() const override + { + // start() has been called. + return mStart != -1; + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); @@ -379,12 +390,13 @@ OscillatorNode::OscillatorNode(AudioContext* aContext) ChannelCountMode::Max, ChannelInterpretation::Speakers) , mType(OscillatorType::Sine) - , mFrequency(new AudioParam(this, SendFrequencyToStream, 440.0f, "frequency")) - , mDetune(new AudioParam(this, SendDetuneToStream, 0.0f, "detune")) + , mFrequency(new AudioParam(this, OscillatorNodeEngine::FREQUENCY, + 440.0f, "frequency")) + , mDetune(new AudioParam(this, OscillatorNodeEngine::DETUNE, 0.0f, "detune")) , mStartCalled(false) { OscillatorNodeEngine* engine = new OscillatorNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NEED_MAIN_THREAD_FINISHED); engine->SetSourceStream(mStream); mStream->AddMainThreadListener(this); @@ -429,26 +441,6 @@ OscillatorNode::DestroyMediaStream() AudioNode::DestroyMediaStream(); } -void -OscillatorNode::SendFrequencyToStream(AudioNode* aNode) -{ - OscillatorNode* This = static_cast(aNode); - if (!This->mStream) { - return; - } - SendTimelineParameterToStream(This, OscillatorNodeEngine::FREQUENCY, *This->mFrequency); -} - -void -OscillatorNode::SendDetuneToStream(AudioNode* aNode) -{ - OscillatorNode* This = static_cast(aNode); - if (!This->mStream) { - return; - } - SendTimelineParameterToStream(This, OscillatorNodeEngine::DETUNE, *This->mDetune); -} - void OscillatorNode::SendTypeToStream() { diff --git a/dom/media/webaudio/OscillatorNode.h b/dom/media/webaudio/OscillatorNode.h index bab5dea697..d8aff9964f 100644 --- a/dom/media/webaudio/OscillatorNode.h +++ b/dom/media/webaudio/OscillatorNode.h @@ -87,8 +87,6 @@ protected: virtual ~OscillatorNode(); private: - static void SendFrequencyToStream(AudioNode* aNode); - static void SendDetuneToStream(AudioNode* aNode); void SendTypeToStream(); void SendPeriodicWaveToStream(); diff --git a/dom/media/webaudio/PannerNode.cpp b/dom/media/webaudio/PannerNode.cpp index 97f37eb233..2f1a4406ff 100644 --- a/dom/media/webaudio/PannerNode.cpp +++ b/dom/media/webaudio/PannerNode.cpp @@ -135,8 +135,9 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool *aFinished) override { if (aInput.IsNull()) { @@ -149,6 +150,7 @@ public: } else { if (mLeftOverData != INT_MIN) { mLeftOverData = INT_MIN; + aStream->CheckForInactive(); mHRTFPanner->reset(); RefPtr refchanged = @@ -156,7 +158,7 @@ public: aStream->Graph()-> DispatchToMainThreadAfterStreamStateUpdate(refchanged.forget()); } - *aOutput = aInput; + aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); return; } } else if (mPanningModelFunction == &PannerNodeEngine::HRTFPanningFunction) { @@ -172,13 +174,18 @@ public: (this->*mPanningModelFunction)(aInput, aOutput); } + virtual bool IsActive() const override + { + return mLeftOverData != INT_MIN; + } + void ComputeAzimuthAndElevation(float& aAzimuth, float& aElevation); float ComputeConeGain(); // Compute how much the distance contributes to the gain reduction. float ComputeDistanceGain(); - void EqualPowerPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput); - void HRTFPanningFunction(const AudioChunk& aInput, AudioChunk* aOutput); + void EqualPowerPanningFunction(const AudioBlock& aInput, AudioBlock* aOutput); + void HRTFPanningFunction(const AudioBlock& aInput, AudioBlock* aOutput); float LinearGainFunction(float aDistance); float InverseGainFunction(float aDistance); @@ -200,7 +207,7 @@ public: } nsAutoPtr mHRTFPanner; - typedef void (PannerNodeEngine::*PanningModelFunction)(const AudioChunk& aInput, AudioChunk* aOutput); + typedef void (PannerNodeEngine::*PanningModelFunction)(const AudioBlock& aInput, AudioBlock* aOutput); PanningModelFunction mPanningModelFunction; typedef float (PannerNodeEngine::*DistanceModelFunction)(float aDistance); DistanceModelFunction mDistanceModelFunction; @@ -240,7 +247,7 @@ PannerNode::PannerNode(AudioContext* aContext) , mConeOuterAngle(360.) , mConeOuterGain(0.) { - mStream = AudioNodeStream::Create(aContext->Graph(), + mStream = AudioNodeStream::Create(aContext, new PannerNodeEngine(this), AudioNodeStream::NO_STREAM_FLAGS); // We should register once we have set up our stream and engine. @@ -302,28 +309,28 @@ PannerNodeEngine::ExponentialGainFunction(float aDistance) } void -PannerNodeEngine::HRTFPanningFunction(const AudioChunk& aInput, - AudioChunk* aOutput) +PannerNodeEngine::HRTFPanningFunction(const AudioBlock& aInput, + AudioBlock* aOutput) { // The output of this node is always stereo, no matter what the inputs are. - AllocateAudioBlock(2, aOutput); + aOutput->AllocateChannels(2); float azimuth, elevation; ComputeAzimuthAndElevation(azimuth, elevation); - AudioChunk input = aInput; - // Gain is applied before the delay and convolution of the HRTF + AudioBlock input = aInput; + // Gain is applied before the delay and convolution of the HRTF. input.mVolume *= ComputeConeGain() * ComputeDistanceGain(); mHRTFPanner->pan(azimuth, elevation, &input, aOutput); } void -PannerNodeEngine::EqualPowerPanningFunction(const AudioChunk& aInput, - AudioChunk* aOutput) +PannerNodeEngine::EqualPowerPanningFunction(const AudioBlock& aInput, + AudioBlock* aOutput) { float azimuth, elevation, gainL, gainR, normalizedAzimuth, distanceGain, coneGain; - int inputChannels = aInput.mChannelData.Length(); + int inputChannels = aInput.ChannelCount(); // If both the listener are in the same spot, and no cone gain is specified, // this node is noop. @@ -335,7 +342,7 @@ PannerNodeEngine::EqualPowerPanningFunction(const AudioChunk& aInput, } // The output of this node is always stereo, no matter what the inputs are. - AllocateAudioBlock(2, aOutput); + aOutput->AllocateChannels(2); ComputeAzimuthAndElevation(azimuth, elevation); coneGain = ComputeConeGain(); diff --git a/dom/media/webaudio/PanningUtils.h b/dom/media/webaudio/PanningUtils.h index dfe7a84afe..a3be3f45e0 100644 --- a/dom/media/webaudio/PanningUtils.h +++ b/dom/media/webaudio/PanningUtils.h @@ -15,7 +15,7 @@ namespace dom { template void -GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, +GainMonoToStereo(const AudioBlock& aInput, AudioBlock* aOutput, T aGainL, T aGainR) { float* outputL = aOutput->ChannelFloatsForWrite(0); @@ -32,7 +32,7 @@ GainMonoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, // depending if the value of the parameters are constant for this block. template void -GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, +GainStereoToStereo(const AudioBlock& aInput, AudioBlock* aOutput, T aGainL, T aGainR, U aOnLeft) { float* outputL = aOutput->ChannelFloatsForWrite(0); @@ -49,10 +49,10 @@ GainStereoToStereo(const AudioChunk& aInput, AudioChunk* aOutput, // T can be float or an array of float, and U can be bool or an array of bool, // depending if the value of the parameters are constant for this block. template -void ApplyStereoPanning(const AudioChunk& aInput, AudioChunk* aOutput, +void ApplyStereoPanning(const AudioBlock& aInput, AudioBlock* aOutput, T aGainL, T aGainR, U aOnLeft) { - if (aInput.mChannelData.Length() == 1) { + if (aInput.ChannelCount() == 1) { GainMonoToStereo(aInput, aOutput, aGainL, aGainR); } else { GainStereoToStereo(aInput, aOutput, aGainL, aGainR, aOnLeft); diff --git a/dom/media/webaudio/ScriptProcessorNode.cpp b/dom/media/webaudio/ScriptProcessorNode.cpp index 95386a244c..45f4d7f43e 100644 --- a/dom/media/webaudio/ScriptProcessorNode.cpp +++ b/dom/media/webaudio/ScriptProcessorNode.cpp @@ -54,8 +54,8 @@ private: size_t ReadyToConsume() const { + // Accessed on both main thread and media graph thread. mMutex.AssertCurrentThreadOwns(); - MOZ_ASSERT(!NS_IsMainThread()); return mBufferList.size(); } @@ -130,34 +130,34 @@ public: if (mLastEventTime.IsNull()) { mLastEventTime = now; } else { - // When the main thread is blocked, and all the event are processed in a - // burst after the main thread unblocks, the |(now - mLastEventTime)| - // interval will be very short. |latency - bufferDuration| will be - // negative, effectively moving back mLatency to a smaller and smaller - // value, until it crosses zero, at which point we stop dropping buffers - // and resume normal operation. This does not work if at the same time, - // the MSG thread was also slowed down, so if the latency on the MSG - // thread is normal, and we are still dropping buffers, and mLatency is - // still more than twice the duration of a buffer, we reset it and stop - // dropping buffers. + // When main thread blocking has built up enough so + // |mLatency > MAX_LATENCY_S|, frame dropping starts. It continues until + // the output buffer is completely empty, at which point the accumulated + // latency is also reset to 0. + // It could happen that the output queue becomes empty before the input + // node has fully caught up. In this case there will be events where + // |(now - mLastEventTime)| is very short, making mLatency negative. + // As this happens and the size of |mLatency| becomes greater than + // MAX_LATENCY_S, frame dropping starts again to maintain an as short + // output queue as possible. float latency = (now - mLastEventTime).ToSeconds(); float bufferDuration = aBufferSize / mSampleRate; mLatency += latency - bufferDuration; mLastEventTime = now; - if (mLatency > MAX_LATENCY_S || - (mDroppingBuffers && mLatency > 0.0 && - fabs(latency - bufferDuration) < bufferDuration)) { + if (fabs(mLatency) > MAX_LATENCY_S) { mDroppingBuffers = true; - return; - } else { - if (mDroppingBuffers) { - mLatency = 0; - } - mDroppingBuffers = false; } } MutexAutoLock lock(mOutputQueue.Lock()); + if (mDroppingBuffers) { + if (mOutputQueue.ReadyToConsume()) { + return; + } + mDroppingBuffers = false; + mLatency = 0; + } + for (uint32_t offset = 0; offset < aBufferSize; offset += WEBAUDIO_BLOCK_SIZE) { AudioChunk& chunk = mOutputQueue.Produce(); if (aBuffer) { @@ -240,58 +240,79 @@ private: class ScriptProcessorNodeEngine final : public AudioNodeEngine { public: - typedef nsAutoTArray, 2> InputChannels; - ScriptProcessorNodeEngine(ScriptProcessorNode* aNode, AudioDestinationNode* aDestination, uint32_t aBufferSize, uint32_t aNumberOfInputChannels) : AudioNodeEngine(aNode) - , mSharedBuffers(aNode->GetSharedBuffers()) - , mSource(nullptr) , mDestination(aDestination->Stream()) + , mSharedBuffers(new SharedBuffers(mDestination->SampleRate())) , mBufferSize(aBufferSize) + , mInputChannelCount(aNumberOfInputChannels) , mInputWriteIndex(0) - , mSeenNonSilenceInput(false) { - mInputChannels.SetLength(aNumberOfInputChannels); - AllocateInputBlock(); } - void SetSourceStream(AudioNodeStream* aSource) + SharedBuffers* GetSharedBuffers() const { - mSource = aSource; + return mSharedBuffers; + } + + enum { + IS_CONNECTED, + }; + + virtual void SetInt32Parameter(uint32_t aIndex, int32_t aParam) override + { + switch (aIndex) { + case IS_CONNECTED: + mIsConnected = aParam; + break; + default: + NS_ERROR("Bad Int32Parameter"); + } // End index switch. } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { // This node is not connected to anything. Per spec, we don't fire the // onaudioprocess event. We also want to clear out the input and output // buffer queue, and output a null buffer. - if (!(aStream->ConsumerCount() || - aStream->AsProcessedStream()->InputPortCount())) { + if (!mIsConnected) { aOutput->SetNull(WEBAUDIO_BLOCK_SIZE); mSharedBuffers->Reset(); - mSeenNonSilenceInput = false; mInputWriteIndex = 0; return; } - // First, record our input buffer - for (uint32_t i = 0; i < mInputChannels.Length(); ++i) { + // The input buffer is allocated lazily when non-null input is received. + if (!aInput.IsNull() && !mInputBuffer) { + mInputBuffer = ThreadSharedFloatArrayBufferList:: + Create(mInputChannelCount, mBufferSize, fallible); + if (mInputBuffer && mInputWriteIndex) { + // Zero leading for null chunks that were skipped. + for (uint32_t i = 0; i < mInputChannelCount; ++i) { + float* channelData = mInputBuffer->GetDataForWrite(i); + PodZero(channelData, mInputWriteIndex); + } + } + } + + // First, record our input buffer, if its allocation succeeded. + uint32_t inputChannelCount = mInputBuffer ? mInputBuffer->GetChannels() : 0; + for (uint32_t i = 0; i < inputChannelCount; ++i) { + float* writeData = mInputBuffer->GetDataForWrite(i) + mInputWriteIndex; if (aInput.IsNull()) { - PodZero(mInputChannels[i] + mInputWriteIndex, - aInput.GetDuration()); + PodZero(writeData, aInput.GetDuration()); } else { - mSeenNonSilenceInput = true; MOZ_ASSERT(aInput.GetDuration() == WEBAUDIO_BLOCK_SIZE, "sanity check"); - MOZ_ASSERT(aInput.mChannelData.Length() == mInputChannels.Length()); + MOZ_ASSERT(aInput.ChannelCount() == inputChannelCount); AudioBlockCopyChannelWithScale(static_cast(aInput.mChannelData[i]), - aInput.mVolume, - mInputChannels[i] + mInputWriteIndex); + aInput.mVolume, writeData); } } mInputWriteIndex += aInput.GetDuration(); @@ -302,24 +323,26 @@ public: *aOutput = mSharedBuffers->GetOutputBuffer(); if (mInputWriteIndex >= mBufferSize) { - SendBuffersToMainThread(aStream); + SendBuffersToMainThread(aStream, aFrom); mInputWriteIndex -= mBufferSize; - mSeenNonSilenceInput = false; - AllocateInputBlock(); } } + virtual bool IsActive() const override + { + // Could return false when !mIsConnected after all output chunks produced + // by main thread events calling + // SharedBuffers::FinishProducingOutputBuffer() have been processed. + return true; + } + virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override { // Not owned: - // - mSharedBuffers - // - mSource (probably) // - mDestination (probably) size_t amount = AudioNodeEngine::SizeOfExcludingThis(aMallocSizeOf); - amount += mInputChannels.ShallowSizeOfExcludingThis(aMallocSizeOf); - for (size_t i = 0; i < mInputChannels.Length(); i++) { - amount += mInputChannels[i].SizeOfExcludingThis(aMallocSizeOf); - } + amount += mSharedBuffers->SizeOfIncludingThis(aMallocSizeOf); + amount += mInputBuffer->SizeOfIncludingThis(aMallocSizeOf); return amount; } @@ -330,80 +353,83 @@ public: } private: - void AllocateInputBlock() - { - for (unsigned i = 0; i < mInputChannels.Length(); ++i) { - if (!mInputChannels[i]) { - mInputChannels[i] = new float[mBufferSize]; - } - } - } - - void SendBuffersToMainThread(AudioNodeStream* aStream) + void SendBuffersToMainThread(AudioNodeStream* aStream, GraphTime aFrom) { MOZ_ASSERT(!NS_IsMainThread()); // we now have a full input buffer ready to be sent to the main thread. - StreamTime playbackTick = mSource->GetCurrentPosition(); + StreamTime playbackTick = mDestination->GraphTimeToStreamTime(aFrom); // Add the duration of the current sample playbackTick += WEBAUDIO_BLOCK_SIZE; // Add the delay caused by the main thread playbackTick += mSharedBuffers->DelaySoFar(); // Compute the playback time in the coordinate system of the destination - double playbackTime = - mSource->DestinationTimeFromTicks(mDestination, playbackTick); + double playbackTime = mDestination->StreamTimeToSeconds(playbackTick); class Command final : public nsRunnable { public: Command(AudioNodeStream* aStream, - InputChannels& aInputChannels, - double aPlaybackTime, - bool aNullInput) + already_AddRefed aInputBuffer, + double aPlaybackTime) : mStream(aStream) + , mInputBuffer(aInputBuffer) , mPlaybackTime(aPlaybackTime) - , mNullInput(aNullInput) { - mInputChannels.SetLength(aInputChannels.Length()); - if (!aNullInput) { - for (uint32_t i = 0; i < mInputChannels.Length(); ++i) { - mInputChannels[i] = aInputChannels[i].forget(); - } - } } NS_IMETHOD Run() override { - RefPtr node = static_cast - (mStream->Engine()->NodeMainThread()); - if (!node) { - return NS_OK; + RefPtr output; + + auto engine = + static_cast(mStream->Engine()); + { + auto node = static_cast + (engine->NodeMainThread()); + if (!node) { + return NS_OK; + } + + if (node->HasListenersFor(nsGkAtoms::onaudioprocess)) { + output = DispatchAudioProcessEvent(node); + } + // The node may have been destroyed during event dispatch. } - AudioContext* context = node->Context(); + + // Append it to our output buffer queue + engine->GetSharedBuffers()-> + FinishProducingOutputBuffer(output, engine->mBufferSize); + + return NS_OK; + } + + // Returns the output buffers if set in event handlers. + ThreadSharedFloatArrayBufferList* + DispatchAudioProcessEvent(ScriptProcessorNode* aNode) + { + AudioContext* context = aNode->Context(); if (!context) { - return NS_OK; + return nullptr; } AutoJSAPI jsapi; - if (NS_WARN_IF(!jsapi.Init(node->GetOwner()))) { - return NS_OK; + if (NS_WARN_IF(!jsapi.Init(aNode->GetOwner()))) { + return nullptr; } JSContext* cx = jsapi.cx(); + uint32_t inputChannelCount = aNode->ChannelCount(); // Create the input buffer RefPtr inputBuffer; - if (!mNullInput) { + if (mInputBuffer) { ErrorResult rv; inputBuffer = - AudioBuffer::Create(context, mInputChannels.Length(), - node->BufferSize(), - context->SampleRate(), cx, rv); + AudioBuffer::Create(context, inputChannelCount, + aNode->BufferSize(), context->SampleRate(), + mInputBuffer.forget(), cx, rv); if (rv.Failed()) { - return NS_OK; - } - // Put the channel data inside it - for (uint32_t i = 0; i < mInputChannels.Length(); ++i) { - inputBuffer->SetRawChannelContents(i, mInputChannels[i]); + return nullptr; } } @@ -412,53 +438,46 @@ private: // avoid creating the input buffer as well. The AudioProcessingEvent class // knows how to lazily create them if needed once the script tries to access // them. Otherwise, we may be able to get away without creating them! - RefPtr event = new AudioProcessingEvent(node, nullptr, nullptr); - event->InitEvent(inputBuffer, - mInputChannels.Length(), - context->StreamTimeToDOMTime(mPlaybackTime)); - node->DispatchTrustedEvent(event); + RefPtr event = + new AudioProcessingEvent(aNode, nullptr, nullptr); + event->InitEvent(inputBuffer, inputChannelCount, mPlaybackTime); + aNode->DispatchTrustedEvent(event); // Steal the output buffers if they have been set. // Don't create a buffer if it hasn't been used to return output; // FinishProducingOutputBuffer() will optimize output = null. // GetThreadSharedChannelsForRate() may also return null after OOM. - RefPtr output; if (event->HasOutputBuffer()) { ErrorResult rv; AudioBuffer* buffer = event->GetOutputBuffer(rv); // HasOutputBuffer() returning true means that GetOutputBuffer() // will not fail. MOZ_ASSERT(!rv.Failed()); - output = buffer->GetThreadSharedChannelsForRate(cx); + return buffer->GetThreadSharedChannelsForRate(cx); } - // Append it to our output buffer queue - node->GetSharedBuffers()->FinishProducingOutputBuffer(output, node->BufferSize()); - - return NS_OK; + return nullptr; } private: RefPtr mStream; - InputChannels mInputChannels; + RefPtr mInputBuffer; double mPlaybackTime; - bool mNullInput; }; - NS_DispatchToMainThread(new Command(aStream, mInputChannels, - playbackTime, - !mSeenNonSilenceInput)); + NS_DispatchToMainThread(new Command(aStream, mInputBuffer.forget(), + playbackTime)); } friend class ScriptProcessorNode; - SharedBuffers* mSharedBuffers; - AudioNodeStream* mSource; AudioNodeStream* mDestination; - InputChannels mInputChannels; + nsAutoPtr mSharedBuffers; + RefPtr mInputBuffer; const uint32_t mBufferSize; + const uint32_t mInputChannelCount; // The write index into the current input buffer uint32_t mInputWriteIndex; - bool mSeenNonSilenceInput; + bool mIsConnected = false; }; ScriptProcessorNode::ScriptProcessorNode(AudioContext* aContext, @@ -469,7 +488,6 @@ ScriptProcessorNode::ScriptProcessorNode(AudioContext* aContext, aNumberOfInputChannels, mozilla::dom::ChannelCountMode::Explicit, mozilla::dom::ChannelInterpretation::Speakers) - , mSharedBuffers(new SharedBuffers(aContext->SampleRate())) , mBufferSize(aBufferSize ? aBufferSize : // respect what the web developer requested 4096) // choose our own buffer size -- 4KB for now @@ -481,9 +499,8 @@ ScriptProcessorNode::ScriptProcessorNode(AudioContext* aContext, aContext->Destination(), BufferSize(), aNumberOfInputChannels); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); - engine->SetSourceStream(mStream); } ScriptProcessorNode::~ScriptProcessorNode() @@ -494,7 +511,6 @@ size_t ScriptProcessorNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf); - amount += mSharedBuffers->SizeOfIncludingThis(aMallocSizeOf); return amount; } @@ -504,12 +520,49 @@ ScriptProcessorNode::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } +void +ScriptProcessorNode::EventListenerAdded(nsIAtom* aType) +{ + AudioNode::EventListenerAdded(aType); + if (aType == nsGkAtoms::onaudioprocess) { + UpdateConnectedStatus(); + } +} + +void +ScriptProcessorNode::EventListenerRemoved(nsIAtom* aType) +{ + AudioNode::EventListenerRemoved(aType); + if (aType == nsGkAtoms::onaudioprocess) { + UpdateConnectedStatus(); + } +} + JSObject* ScriptProcessorNode::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { return ScriptProcessorNodeBinding::Wrap(aCx, this, aGivenProto); } +void +ScriptProcessorNode::UpdateConnectedStatus() +{ + bool isConnected = mHasPhantomInput || + !(OutputNodes().IsEmpty() && OutputParams().IsEmpty() + && InputNodes().IsEmpty()); + + // Events are queued even when there is no listener because a listener + // may be added while events are in the queue. + SendInt32ParameterToStream(ScriptProcessorNodeEngine::IS_CONNECTED, + isConnected); + + if (isConnected && HasListenersFor(nsGkAtoms::onaudioprocess)) { + MarkActive(); + } else { + MarkInactive(); + } +} + } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/ScriptProcessorNode.h b/dom/media/webaudio/ScriptProcessorNode.h index 0bbbca06ae..cc82a01fb1 100644 --- a/dom/media/webaudio/ScriptProcessorNode.h +++ b/dom/media/webaudio/ScriptProcessorNode.h @@ -28,6 +28,9 @@ public: IMPL_EVENT_HANDLER(audioprocess) + virtual void EventListenerAdded(nsIAtom* aType) override; + virtual void EventListenerRemoved(nsIAtom* aType) override; + virtual JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; virtual AudioNode* Connect(AudioNode& aDestination, uint32_t aOutput, @@ -35,7 +38,7 @@ public: { AudioNode* node = AudioNode::Connect(aDestination, aOutput, aInput, aRv); if (!aRv.Failed()) { - MarkActive(); + UpdateConnectedStatus(); } return node; } @@ -45,17 +48,27 @@ public: { AudioNode::Connect(aDestination, aOutput, aRv); if (!aRv.Failed()) { - MarkActive(); + UpdateConnectedStatus(); } } virtual void Disconnect(uint32_t aOutput, ErrorResult& aRv) override { AudioNode::Disconnect(aOutput, aRv); - if (!aRv.Failed() && OutputNodes().IsEmpty() && OutputParams().IsEmpty()) { - MarkInactive(); + if (!aRv.Failed()) { + UpdateConnectedStatus(); } } + virtual void NotifyInputsChanged() override + { + UpdateConnectedStatus(); + } + virtual void NotifyHasPhantomInput() override + { + mHasPhantomInput = true; + // No need to UpdateConnectedStatus() because there was previously an + // input in InputNodes(). + } virtual void SetChannelCount(uint32_t aChannelCount, ErrorResult& aRv) override { @@ -77,11 +90,6 @@ public: return mBufferSize; } - SharedBuffers* GetSharedBuffers() const - { - return mSharedBuffers; - } - uint32_t NumberOfOutputChannels() const { return mNumberOfOutputChannels; @@ -97,13 +105,14 @@ public: virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override; virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override; -protected: +private: virtual ~ScriptProcessorNode(); -private: - nsAutoPtr mSharedBuffers; + void UpdateConnectedStatus(); + const uint32_t mBufferSize; const uint32_t mNumberOfOutputChannels; + bool mHasPhantomInput = false; }; } // namespace dom diff --git a/dom/media/webaudio/StereoPannerNode.cpp b/dom/media/webaudio/StereoPannerNode.cpp index 7fceebbae7..be885a1d16 100644 --- a/dom/media/webaudio/StereoPannerNode.cpp +++ b/dom/media/webaudio/StereoPannerNode.cpp @@ -33,7 +33,6 @@ public: StereoPannerNodeEngine(AudioNode* aNode, AudioDestinationNode* aDestination) : AudioNodeEngine(aNode) - , mSource(nullptr) , mDestination(aDestination->Stream()) // Keep the default value in sync with the default value in // StereoPannerNode::StereoPannerNode. @@ -41,23 +40,19 @@ public: { } - void SetSourceStream(AudioNodeStream* aSource) - { - mSource = aSource; - } - enum Parameters { PAN }; - void SetTimelineParameter(uint32_t aIndex, - const AudioParamTimeline& aValue, - TrackRate aSampleRate) override + void RecvTimelineEvent(uint32_t aIndex, + AudioTimelineEvent& aEvent) override { + MOZ_ASSERT(mDestination); + WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, + mDestination); + switch (aIndex) { case PAN: - MOZ_ASSERT(mSource && mDestination); - mPan = aValue; - WebAudioUtils::ConvertAudioParamToTicks(mPan, mSource, mDestination); + mPan.InsertEvent(aEvent); break; default: NS_ERROR("Bad StereoPannerNode TimelineParameter"); @@ -83,7 +78,7 @@ public: aRightGain = sin(0.5 * M_PI * aPanning); } - void SetToSilentStereoBlock(AudioChunk* aChunk) + void SetToSilentStereoBlock(AudioBlock* aChunk) { for (uint32_t channel = 0; channel < 2; channel++) { float* samples = aChunk->ChannelFloatsForWrite(channel); @@ -93,13 +88,13 @@ public: } } - void UpmixToStereoIfNeeded(const AudioChunk& aInput, AudioChunk* aOutput) + void UpmixToStereoIfNeeded(const AudioBlock& aInput, AudioBlock* aOutput) { if (aInput.ChannelCount() == 2) { *aOutput = aInput; } else { MOZ_ASSERT(aInput.ChannelCount() == 1); - AllocateAudioBlock(2, aOutput); + aOutput->AllocateChannels(2); const float* input = static_cast(aInput.mChannelData[0]); for (uint32_t channel = 0; channel < 2; channel++) { float* output = aOutput->ChannelFloatsForWrite(channel); @@ -109,15 +104,14 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool *aFinished) override { - MOZ_ASSERT(mSource == aStream, "Invalid source stream"); - // The output of this node is always stereo, no matter what the inputs are. MOZ_ASSERT(aInput.ChannelCount() <= 2); - AllocateAudioBlock(2, aOutput); + aOutput->AllocateChannels(2); bool monoToStereo = aInput.ChannelCount() == 1; if (aInput.IsNull()) { @@ -145,15 +139,17 @@ public: float computedGain[2][WEBAUDIO_BLOCK_SIZE]; bool onLeft[WEBAUDIO_BLOCK_SIZE]; + float values[WEBAUDIO_BLOCK_SIZE]; + StreamTime tick = mDestination->GraphTimeToStreamTime(aFrom); + mPan.GetValuesAtTime(tick, values, WEBAUDIO_BLOCK_SIZE); + for (size_t counter = 0; counter < WEBAUDIO_BLOCK_SIZE; ++counter) { - StreamTime tick = aStream->GetCurrentPosition(); float left, right; - float panning = mPan.GetValueAtTime(tick, counter); - GetGainValuesForPanning(panning, monoToStereo, left, right); + GetGainValuesForPanning(values[counter], monoToStereo, left, right); computedGain[0][counter] = left * aInput.mVolume; computedGain[1][counter] = right * aInput.mVolume; - onLeft[counter] = panning <= 0; + onLeft[counter] = values[counter] <= 0; } // Apply the gain to the output buffer @@ -166,7 +162,6 @@ public: return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } - AudioNodeStream* mSource; AudioNodeStream* mDestination; AudioParamTimeline mPan; }; @@ -176,12 +171,11 @@ StereoPannerNode::StereoPannerNode(AudioContext* aContext) 2, ChannelCountMode::Clamped_max, ChannelInterpretation::Speakers) - , mPan(new AudioParam(this, SendPanToStream, 0.f, "pan")) + , mPan(new AudioParam(this, StereoPannerNodeEngine::PAN, 0.f, "pan")) { StereoPannerNodeEngine* engine = new StereoPannerNodeEngine(this, aContext->Destination()); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); - engine->SetSourceStream(mStream); } StereoPannerNode::~StereoPannerNode() @@ -208,12 +202,5 @@ StereoPannerNode::WrapObject(JSContext* aCx, JS::Handle aGivenProto) return StereoPannerNodeBinding::Wrap(aCx, this, aGivenProto); } -void -StereoPannerNode::SendPanToStream(AudioNode* aNode) -{ - StereoPannerNode* This = static_cast(aNode); - SendTimelineParameterToStream(This, StereoPannerNodeEngine::PAN, *This->mPan); -} - } // namespace dom } // namespace mozilla diff --git a/dom/media/webaudio/StereoPannerNode.h b/dom/media/webaudio/StereoPannerNode.h index 6a1719a79e..68204f7571 100644 --- a/dom/media/webaudio/StereoPannerNode.h +++ b/dom/media/webaudio/StereoPannerNode.h @@ -60,7 +60,6 @@ protected: virtual ~StereoPannerNode(); private: - static void SendPanToStream(AudioNode* aNode); RefPtr mPan; }; diff --git a/dom/media/webaudio/WaveShaperNode.cpp b/dom/media/webaudio/WaveShaperNode.cpp index 5dd2d3338d..d571705629 100644 --- a/dom/media/webaudio/WaveShaperNode.cpp +++ b/dom/media/webaudio/WaveShaperNode.cpp @@ -87,12 +87,12 @@ public: mUpSampler = speex_resampler_init(aChannels, aSampleRate, aSampleRate * ValueOf(aType), - SPEEX_RESAMPLER_QUALITY_DEFAULT, + SPEEX_RESAMPLER_QUALITY_MIN, nullptr); mDownSampler = speex_resampler_init(aChannels, aSampleRate * ValueOf(aType), aSampleRate, - SPEEX_RESAMPLER_QUALITY_DEFAULT, + SPEEX_RESAMPLER_QUALITY_MIN, nullptr); mBuffer.SetLength(WEBAUDIO_BLOCK_SIZE*ValueOf(aType)); } @@ -215,11 +215,12 @@ public: } virtual void ProcessBlock(AudioNodeStream* aStream, - const AudioChunk& aInput, - AudioChunk* aOutput, + GraphTime aFrom, + const AudioBlock& aInput, + AudioBlock* aOutput, bool* aFinished) override { - uint32_t channelCount = aInput.mChannelData.Length(); + uint32_t channelCount = aInput.ChannelCount(); if (!mCurve.Length() || !channelCount) { // Optimize the case where we don't have a curve buffer, // or the input is null. @@ -227,7 +228,7 @@ public: return; } - AllocateAudioBlock(channelCount, aOutput); + aOutput->AllocateChannels(channelCount); for (uint32_t i = 0; i < channelCount; ++i) { float* scaledSample = (float *)(aInput.mChannelData[i]); AudioBlockInPlaceScale(scaledSample, aInput.mVolume); @@ -288,7 +289,7 @@ WaveShaperNode::WaveShaperNode(AudioContext* aContext) mozilla::HoldJSObjects(this); WaveShaperNodeEngine* engine = new WaveShaperNodeEngine(this); - mStream = AudioNodeStream::Create(aContext->Graph(), engine, + mStream = AudioNodeStream::Create(aContext, engine, AudioNodeStream::NO_STREAM_FLAGS); } diff --git a/dom/media/webaudio/WebAudioUtils.cpp b/dom/media/webaudio/WebAudioUtils.cpp index 6c11460872..fc1936337d 100644 --- a/dom/media/webaudio/WebAudioUtils.cpp +++ b/dom/media/webaudio/WebAudioUtils.cpp @@ -6,37 +6,19 @@ #include "WebAudioUtils.h" #include "AudioNodeStream.h" -#include "AudioParamTimeline.h" #include "blink/HRTFDatabaseLoader.h" namespace mozilla { namespace dom { -struct ConvertTimeToTickHelper +void WebAudioUtils::ConvertAudioTimelineEventToTicks(AudioTimelineEvent& aEvent, + AudioNodeStream* aDest) { - AudioNodeStream* mSourceStream; - AudioNodeStream* mDestinationStream; - - static int64_t Convert(double aTime, void* aClosure) - { - ConvertTimeToTickHelper* This = static_cast (aClosure); - MOZ_ASSERT(This->mSourceStream->SampleRate() == This->mDestinationStream->SampleRate()); - return This->mSourceStream-> - TicksFromDestinationTime(This->mDestinationStream, aTime); - } -}; - -void -WebAudioUtils::ConvertAudioParamToTicks(AudioParamTimeline& aParam, - AudioNodeStream* aSource, - AudioNodeStream* aDest) -{ - MOZ_ASSERT(!aSource || aSource->SampleRate() == aDest->SampleRate()); - ConvertTimeToTickHelper ctth; - ctth.mSourceStream = aSource; - ctth.mDestinationStream = aDest; - aParam.ConvertEventTimesToTicks(ConvertTimeToTickHelper::Convert, &ctth, aDest->SampleRate()); + aEvent.SetTimeInTicks( + aDest->SecondsToNearestStreamTime(aEvent.Time())); + aEvent.mTimeConstant *= aDest->SampleRate(); + aEvent.mDuration *= aDest->SampleRate(); } void diff --git a/dom/media/webaudio/WebAudioUtils.h b/dom/media/webaudio/WebAudioUtils.h index 060072d940..b15795c20e 100644 --- a/dom/media/webaudio/WebAudioUtils.h +++ b/dom/media/webaudio/WebAudioUtils.h @@ -22,7 +22,7 @@ class AudioNodeStream; namespace dom { -class AudioParamTimeline; +struct AudioTimelineEvent; namespace WebAudioUtils { // 32 is the minimum required by the spec for createBuffer() and @@ -55,17 +55,16 @@ namespace WebAudioUtils { } /** - * Converts AudioParamTimeline floating point time values to tick values - * with respect to a source and a destination AudioNodeStream. + * Converts an AudioTimelineEvent's floating point time values to tick values + * with respect to a destination AudioNodeStream. * - * This needs to be called for each AudioParamTimeline that gets sent to an - * AudioNodeEngine on the engine side where the AudioParamTimeline is - * received. This means that such engines need to be aware of their source - * and destination streams as well. + * This needs to be called for each AudioTimelineEvent that gets sent to an + * AudioNodeEngine, on the engine side where the AudioTimlineEvent is + * received. This means that such engines need to be aware of their + * destination streams as well. */ - void ConvertAudioParamToTicks(AudioParamTimeline& aParam, - AudioNodeStream* aSource, - AudioNodeStream* aDest); + void ConvertAudioTimelineEventToTicks(AudioTimelineEvent& aEvent, + AudioNodeStream* aDest); /** * Converts a linear value to decibels. Returns aMinDecibels if the linear diff --git a/dom/media/webaudio/blink/DynamicsCompressor.cpp b/dom/media/webaudio/blink/DynamicsCompressor.cpp index 746356d1d0..10a285180a 100644 --- a/dom/media/webaudio/blink/DynamicsCompressor.cpp +++ b/dom/media/webaudio/blink/DynamicsCompressor.cpp @@ -27,7 +27,7 @@ */ #include "DynamicsCompressor.h" -#include "AudioSegment.h" +#include "AudioBlock.h" #include #include "AudioNodeEngine.h" @@ -148,14 +148,14 @@ void DynamicsCompressor::setEmphasisParameters(float gain, float anchorFreq, flo setEmphasisStageParameters(3, gain, anchorFreq / (filterStageRatio * filterStageRatio * filterStageRatio)); } -void DynamicsCompressor::process(const AudioChunk* sourceChunk, AudioChunk* destinationChunk, unsigned framesToProcess) +void DynamicsCompressor::process(const AudioBlock* sourceChunk, AudioBlock* destinationChunk, unsigned framesToProcess) { // Though numberOfChannels is retrived from destinationBus, we still name it numberOfChannels instead of numberOfDestinationChannels. // It's because we internally match sourceChannels's size to destinationBus by channel up/down mix. Thus we need numberOfChannels // to do the loop work for both m_sourceChannels and m_destinationChannels. - unsigned numberOfChannels = destinationChunk->mChannelData.Length(); - unsigned numberOfSourceChannels = sourceChunk->mChannelData.Length(); + unsigned numberOfChannels = destinationChunk->ChannelCount(); + unsigned numberOfSourceChannels = sourceChunk->ChannelCount(); MOZ_ASSERT(numberOfChannels == m_numberOfChannels && numberOfSourceChannels); diff --git a/dom/media/webaudio/blink/DynamicsCompressor.h b/dom/media/webaudio/blink/DynamicsCompressor.h index be492bd771..d0eac82f20 100644 --- a/dom/media/webaudio/blink/DynamicsCompressor.h +++ b/dom/media/webaudio/blink/DynamicsCompressor.h @@ -37,12 +37,12 @@ #include "mozilla/MemoryReporting.h" namespace mozilla { -struct AudioChunk; +class AudioBlock; } // namespace mozilla namespace WebCore { -using mozilla::AudioChunk; +using mozilla::AudioBlock; // DynamicsCompressor implements a flexible audio dynamics compression effect such as // is commonly used in musical production and game audio. It lowers the volume @@ -73,7 +73,7 @@ public: DynamicsCompressor(float sampleRate, unsigned numberOfChannels); - void process(const AudioChunk* sourceChunk, AudioChunk* destinationChunk, unsigned framesToProcess); + void process(const AudioBlock* sourceChunk, AudioBlock* destinationChunk, unsigned framesToProcess); void reset(); void setNumberOfChannels(unsigned); unsigned numberOfChannels() const { return m_numberOfChannels; } diff --git a/dom/media/webaudio/blink/HRTFElevation.cpp b/dom/media/webaudio/blink/HRTFElevation.cpp index 2d9dd2984c..f53943d6c5 100644 --- a/dom/media/webaudio/blink/HRTFElevation.cpp +++ b/dom/media/webaudio/blink/HRTFElevation.cpp @@ -232,7 +232,7 @@ nsReturnRef HRTFElevation::createBuiltin(int elevation, float sam SpeexResamplerState* resampler = sampleRate == rawSampleRate ? nullptr : speex_resampler_init(1, rawSampleRate, sampleRate, - SPEEX_RESAMPLER_QUALITY_DEFAULT, nullptr); + SPEEX_RESAMPLER_QUALITY_MIN, nullptr); // Load convolution kernels from HRTF files. int interpolatedIndex = 0; diff --git a/dom/media/webaudio/blink/HRTFPanner.cpp b/dom/media/webaudio/blink/HRTFPanner.cpp index 54f08e21bf..11c2728941 100644 --- a/dom/media/webaudio/blink/HRTFPanner.cpp +++ b/dom/media/webaudio/blink/HRTFPanner.cpp @@ -27,6 +27,7 @@ #include "FFTConvolver.h" #include "HRTFDatabase.h" +#include "AudioBlock.h" using namespace std; using namespace mozilla; @@ -128,28 +129,28 @@ int HRTFPanner::calculateDesiredAzimuthIndexAndBlend(double azimuth, double& azi return desiredAzimuthIndex; } -void HRTFPanner::pan(double desiredAzimuth, double elevation, const AudioChunk* inputBus, AudioChunk* outputBus) +void HRTFPanner::pan(double desiredAzimuth, double elevation, const AudioBlock* inputBus, AudioBlock* outputBus) { #ifdef DEBUG unsigned numInputChannels = - inputBus->IsNull() ? 0 : inputBus->mChannelData.Length(); + inputBus->IsNull() ? 0 : inputBus->ChannelCount(); MOZ_ASSERT(numInputChannels <= 2); - MOZ_ASSERT(inputBus->mDuration == WEBAUDIO_BLOCK_SIZE); + MOZ_ASSERT(inputBus->GetDuration() == WEBAUDIO_BLOCK_SIZE); #endif - bool isOutputGood = outputBus && outputBus->mChannelData.Length() == 2 && outputBus->mDuration == WEBAUDIO_BLOCK_SIZE; + bool isOutputGood = outputBus && outputBus->ChannelCount() == 2 && outputBus->GetDuration() == WEBAUDIO_BLOCK_SIZE; MOZ_ASSERT(isOutputGood); if (!isOutputGood) { if (outputBus) - outputBus->SetNull(outputBus->mDuration); + outputBus->SetNull(outputBus->GetDuration()); return; } HRTFDatabase* database = m_databaseLoader->database(); if (!database) { // not yet loaded - outputBus->SetNull(outputBus->mDuration); + outputBus->SetNull(outputBus->GetDuration()); return; } @@ -159,7 +160,7 @@ void HRTFPanner::pan(double desiredAzimuth, double elevation, const AudioChunk* bool isAzimuthGood = azimuth >= -180.0 && azimuth <= 180.0; MOZ_ASSERT(isAzimuthGood); if (!isAzimuthGood) { - outputBus->SetNull(outputBus->mDuration); + outputBus->SetNull(outputBus->GetDuration()); return; } @@ -223,7 +224,7 @@ void HRTFPanner::pan(double desiredAzimuth, double elevation, const AudioChunk* bool areKernelsGood = kernelL1 && kernelR1 && kernelL2 && kernelR2; MOZ_ASSERT(areKernelsGood); if (!areKernelsGood) { - outputBus->SetNull(outputBus->mDuration); + outputBus->SetNull(outputBus->GetDuration()); return; } diff --git a/dom/media/webaudio/blink/HRTFPanner.h b/dom/media/webaudio/blink/HRTFPanner.h index 12264df36a..f56d0d4232 100644 --- a/dom/media/webaudio/blink/HRTFPanner.h +++ b/dom/media/webaudio/blink/HRTFPanner.h @@ -30,7 +30,7 @@ #include "mozilla/MemoryReporting.h" namespace mozilla { -struct AudioChunk; +class AudioBlock; } // namespace mozilla namespace WebCore { @@ -39,7 +39,7 @@ typedef nsTArray AudioFloatArray; class HRTFDatabaseLoader; -using mozilla::AudioChunk; +using mozilla::AudioBlock; class HRTFPanner { public: @@ -47,7 +47,7 @@ public: ~HRTFPanner(); // chunk durations must be 128 - void pan(double azimuth, double elevation, const AudioChunk* inputBus, AudioChunk* outputBus); + void pan(double azimuth, double elevation, const AudioBlock* inputBus, AudioBlock* outputBus); void reset(); size_t fftSize() const { return m_convolverL1.fftSize(); } diff --git a/dom/media/webaudio/blink/Reverb.cpp b/dom/media/webaudio/blink/Reverb.cpp index 42137595eb..a54f92978b 100644 --- a/dom/media/webaudio/blink/Reverb.cpp +++ b/dom/media/webaudio/blink/Reverb.cpp @@ -144,59 +144,59 @@ void Reverb::initialize(const nsTArray& impulseResponseBuffer, // For "True" stereo processing we allocate a temporary buffer to avoid repeatedly allocating it in the process() method. // It can be bad to allocate memory in a real-time thread. if (numResponseChannels == 4) { - AllocateAudioBlock(2, &m_tempBuffer); + m_tempBuffer.AllocateChannels(2); WriteZeroesToAudioBlock(&m_tempBuffer, 0, WEBAUDIO_BLOCK_SIZE); } } -void Reverb::process(const AudioChunk* sourceBus, AudioChunk* destinationBus, size_t framesToProcess) +void Reverb::process(const AudioBlock* sourceBus, AudioBlock* destinationBus, size_t framesToProcess) { // Do a fairly comprehensive sanity check. // If these conditions are satisfied, all of the source and destination pointers will be valid for the various matrixing cases. - bool isSafeToProcess = sourceBus && destinationBus && sourceBus->mChannelData.Length() > 0 && destinationBus->mChannelData.Length() > 0 - && framesToProcess <= MaxFrameSize && framesToProcess <= size_t(sourceBus->mDuration) && framesToProcess <= size_t(destinationBus->mDuration); + bool isSafeToProcess = sourceBus && destinationBus && sourceBus->ChannelCount() > 0 && destinationBus->mChannelData.Length() > 0 + && framesToProcess <= MaxFrameSize && framesToProcess <= size_t(sourceBus->GetDuration()) && framesToProcess <= size_t(destinationBus->GetDuration()); MOZ_ASSERT(isSafeToProcess); if (!isSafeToProcess) return; // For now only handle mono or stereo output - MOZ_ASSERT(destinationBus->mChannelData.Length() <= 2); + MOZ_ASSERT(destinationBus->ChannelCount() <= 2); float* destinationChannelL = static_cast(const_cast(destinationBus->mChannelData[0])); const float* sourceBusL = static_cast(sourceBus->mChannelData[0]); // Handle input -> output matrixing... - size_t numInputChannels = sourceBus->mChannelData.Length(); - size_t numOutputChannels = destinationBus->mChannelData.Length(); + size_t numInputChannels = sourceBus->ChannelCount(); + size_t numOutputChannels = destinationBus->ChannelCount(); size_t numReverbChannels = m_convolvers.Length(); if (numInputChannels == 2 && numReverbChannels == 2 && numOutputChannels == 2) { // 2 -> 2 -> 2 const float* sourceBusR = static_cast(sourceBus->mChannelData[1]); float* destinationChannelR = static_cast(const_cast(destinationBus->mChannelData[1])); - m_convolvers[0]->process(sourceBusL, sourceBus->mDuration, destinationChannelL, destinationBus->mDuration, framesToProcess); - m_convolvers[1]->process(sourceBusR, sourceBus->mDuration, destinationChannelR, destinationBus->mDuration, framesToProcess); + m_convolvers[0]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelL, destinationBus->GetDuration(), framesToProcess); + m_convolvers[1]->process(sourceBusR, sourceBus->GetDuration(), destinationChannelR, destinationBus->GetDuration(), framesToProcess); } else if (numInputChannels == 1 && numOutputChannels == 2 && numReverbChannels == 2) { // 1 -> 2 -> 2 for (int i = 0; i < 2; ++i) { float* destinationChannel = static_cast(const_cast(destinationBus->mChannelData[i])); - m_convolvers[i]->process(sourceBusL, sourceBus->mDuration, destinationChannel, destinationBus->mDuration, framesToProcess); + m_convolvers[i]->process(sourceBusL, sourceBus->GetDuration(), destinationChannel, destinationBus->GetDuration(), framesToProcess); } } else if (numInputChannels == 1 && numReverbChannels == 1 && numOutputChannels == 2) { // 1 -> 1 -> 2 - m_convolvers[0]->process(sourceBusL, sourceBus->mDuration, destinationChannelL, destinationBus->mDuration, framesToProcess); + m_convolvers[0]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelL, destinationBus->GetDuration(), framesToProcess); // simply copy L -> R float* destinationChannelR = static_cast(const_cast(destinationBus->mChannelData[1])); - bool isCopySafe = destinationChannelL && destinationChannelR && size_t(destinationBus->mDuration) >= framesToProcess && size_t(destinationBus->mDuration) >= framesToProcess; + bool isCopySafe = destinationChannelL && destinationChannelR && size_t(destinationBus->GetDuration()) >= framesToProcess; MOZ_ASSERT(isCopySafe); if (!isCopySafe) return; PodCopy(destinationChannelR, destinationChannelL, framesToProcess); } else if (numInputChannels == 1 && numReverbChannels == 1 && numOutputChannels == 1) { // 1 -> 1 -> 1 - m_convolvers[0]->process(sourceBusL, sourceBus->mDuration, destinationChannelL, destinationBus->mDuration, framesToProcess); + m_convolvers[0]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelL, destinationBus->GetDuration(), framesToProcess); } else if (numInputChannels == 2 && numReverbChannels == 4 && numOutputChannels == 2) { // 2 -> 4 -> 2 ("True" stereo) const float* sourceBusR = static_cast(sourceBus->mChannelData[1]); @@ -206,15 +206,15 @@ void Reverb::process(const AudioChunk* sourceBus, AudioChunk* destinationBus, si float* tempChannelR = static_cast(const_cast(m_tempBuffer.mChannelData[1])); // Process left virtual source - m_convolvers[0]->process(sourceBusL, sourceBus->mDuration, destinationChannelL, destinationBus->mDuration, framesToProcess); - m_convolvers[1]->process(sourceBusL, sourceBus->mDuration, destinationChannelR, destinationBus->mDuration, framesToProcess); + m_convolvers[0]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelL, destinationBus->GetDuration(), framesToProcess); + m_convolvers[1]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelR, destinationBus->GetDuration(), framesToProcess); // Process right virtual source - m_convolvers[2]->process(sourceBusR, sourceBus->mDuration, tempChannelL, m_tempBuffer.mDuration, framesToProcess); - m_convolvers[3]->process(sourceBusR, sourceBus->mDuration, tempChannelR, m_tempBuffer.mDuration, framesToProcess); + m_convolvers[2]->process(sourceBusR, sourceBus->GetDuration(), tempChannelL, m_tempBuffer.GetDuration(), framesToProcess); + m_convolvers[3]->process(sourceBusR, sourceBus->GetDuration(), tempChannelR, m_tempBuffer.GetDuration(), framesToProcess); - AudioBufferAddWithScale(tempChannelL, 1.0f, destinationChannelL, sourceBus->mDuration); - AudioBufferAddWithScale(tempChannelR, 1.0f, destinationChannelR, sourceBus->mDuration); + AudioBufferAddWithScale(tempChannelL, 1.0f, destinationChannelL, sourceBus->GetDuration()); + AudioBufferAddWithScale(tempChannelR, 1.0f, destinationChannelR, sourceBus->GetDuration()); } else if (numInputChannels == 1 && numReverbChannels == 4 && numOutputChannels == 2) { // 1 -> 4 -> 2 (Processing mono with "True" stereo impulse response) // This is an inefficient use of a four-channel impulse response, but we should handle the case. @@ -224,19 +224,19 @@ void Reverb::process(const AudioChunk* sourceBus, AudioChunk* destinationBus, si float* tempChannelR = static_cast(const_cast(m_tempBuffer.mChannelData[1])); // Process left virtual source - m_convolvers[0]->process(sourceBusL, sourceBus->mDuration, destinationChannelL, destinationBus->mDuration, framesToProcess); - m_convolvers[1]->process(sourceBusL, sourceBus->mDuration, destinationChannelR, destinationBus->mDuration, framesToProcess); + m_convolvers[0]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelL, destinationBus->GetDuration(), framesToProcess); + m_convolvers[1]->process(sourceBusL, sourceBus->GetDuration(), destinationChannelR, destinationBus->GetDuration(), framesToProcess); // Process right virtual source - m_convolvers[2]->process(sourceBusL, sourceBus->mDuration, tempChannelL, m_tempBuffer.mDuration, framesToProcess); - m_convolvers[3]->process(sourceBusL, sourceBus->mDuration, tempChannelR, m_tempBuffer.mDuration, framesToProcess); + m_convolvers[2]->process(sourceBusL, sourceBus->GetDuration(), tempChannelL, m_tempBuffer.GetDuration(), framesToProcess); + m_convolvers[3]->process(sourceBusL, sourceBus->GetDuration(), tempChannelR, m_tempBuffer.GetDuration(), framesToProcess); - AudioBufferAddWithScale(tempChannelL, 1.0f, destinationChannelL, sourceBus->mDuration); - AudioBufferAddWithScale(tempChannelR, 1.0f, destinationChannelR, sourceBus->mDuration); + AudioBufferAddWithScale(tempChannelL, 1.0f, destinationChannelL, sourceBus->GetDuration()); + AudioBufferAddWithScale(tempChannelR, 1.0f, destinationChannelR, sourceBus->GetDuration()); } else { // Handle gracefully any unexpected / unsupported matrixing // FIXME: add code for 5.1 support... - destinationBus->SetNull(destinationBus->mDuration); + destinationBus->SetNull(destinationBus->GetDuration()); } } diff --git a/dom/media/webaudio/blink/Reverb.h b/dom/media/webaudio/blink/Reverb.h index 97d01b454d..fd4445ec33 100644 --- a/dom/media/webaudio/blink/Reverb.h +++ b/dom/media/webaudio/blink/Reverb.h @@ -32,7 +32,7 @@ #include "ReverbConvolver.h" #include "nsAutoPtr.h" #include "nsTArray.h" -#include "AudioSegment.h" +#include "AudioBlock.h" #include "mozilla/MemoryReporting.h" namespace mozilla { @@ -50,7 +50,7 @@ public: // renderSliceSize is a rendering hint, so the FFTs can be optimized to not all occur at the same time (very bad when rendering on a real-time thread). Reverb(mozilla::ThreadSharedFloatArrayBufferList* impulseResponseBuffer, size_t impulseResponseBufferLength, size_t renderSliceSize, size_t maxFFTSize, size_t numberOfChannels, bool useBackgroundThreads, bool normalize, float sampleRate); - void process(const mozilla::AudioChunk* sourceBus, mozilla::AudioChunk* destinationBus, size_t framesToProcess); + void process(const mozilla::AudioBlock* sourceBus, mozilla::AudioBlock* destinationBus, size_t framesToProcess); void reset(); size_t impulseResponseLength() const { return m_impulseResponseLength; } @@ -66,7 +66,7 @@ private: nsTArray > m_convolvers; // For "True" stereo processing - mozilla::AudioChunk m_tempBuffer; + mozilla::AudioBlock m_tempBuffer; }; } // namespace WebCore diff --git a/dom/media/webaudio/compiledtest/TestAudioEventTimeline.cpp b/dom/media/webaudio/compiledtest/TestAudioEventTimeline.cpp index dd69075a89..76616c2f3a 100644 --- a/dom/media/webaudio/compiledtest/TestAudioEventTimeline.cpp +++ b/dom/media/webaudio/compiledtest/TestAudioEventTimeline.cpp @@ -9,6 +9,17 @@ #include #include +// Mock the MediaStream class +namespace mozilla { +class MediaStream +{ + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream) +private: + ~MediaStream() { + }; +}; +} + using namespace mozilla; using namespace mozilla::dom; using std::numeric_limits; diff --git a/dom/media/webaudio/moz.build b/dom/media/webaudio/moz.build index ec78cded3b..fed52629e1 100644 --- a/dom/media/webaudio/moz.build +++ b/dom/media/webaudio/moz.build @@ -17,7 +17,7 @@ MOCHITEST_CHROME_MANIFESTS += ['test/chrome.ini'] EXPORTS += [ 'AlignedTArray.h', - 'AudioContext.h', + 'AudioBlock.h', 'AudioEventTimeline.h', 'AudioNodeEngine.h', 'AudioNodeExternalInputStream.h', @@ -64,6 +64,7 @@ EXPORTS.mozilla.dom += [ UNIFIED_SOURCES += [ 'AnalyserNode.cpp', + 'AudioBlock.cpp', 'AudioBuffer.cpp', 'AudioBufferSourceNode.cpp', 'AudioContext.cpp', diff --git a/dom/media/webspeech/recognition/SpeechStreamListener.cpp b/dom/media/webspeech/recognition/SpeechStreamListener.cpp index bcc7d58514..6e7b76b1f6 100644 --- a/dom/media/webspeech/recognition/SpeechStreamListener.cpp +++ b/dom/media/webspeech/recognition/SpeechStreamListener.cpp @@ -33,7 +33,9 @@ SpeechStreamListener::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) { AudioSegment* audio = const_cast( static_cast(&aQueuedMedia)); diff --git a/dom/media/webspeech/recognition/SpeechStreamListener.h b/dom/media/webspeech/recognition/SpeechStreamListener.h index 0634fe1f38..55d51a4e97 100644 --- a/dom/media/webspeech/recognition/SpeechStreamListener.h +++ b/dom/media/webspeech/recognition/SpeechStreamListener.h @@ -27,7 +27,9 @@ public: virtual void NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, TrackID aID, StreamTime aTrackOffset, uint32_t aTrackEvents, - const MediaSegment& aQueuedMedia) override; + const MediaSegment& aQueuedMedia, + MediaStream* aInputStream, + TrackID aInputTrackID) override; virtual void NotifyEvent(MediaStreamGraph* aGraph, MediaStreamListener::MediaStreamGraphEvent event) override; diff --git a/dom/media/webspeech/synth/SpeechSynthesis.cpp b/dom/media/webspeech/synth/SpeechSynthesis.cpp index 45600107e6..f8960d06d6 100644 --- a/dom/media/webspeech/synth/SpeechSynthesis.cpp +++ b/dom/media/webspeech/synth/SpeechSynthesis.cpp @@ -274,12 +274,14 @@ SpeechSynthesis::GetVoices(nsTArray< RefPtr >& aResult) } } -// For testing purposes, allows us to drop anything in the global queue from -// content, and bring the browser to initial state. +// For testing purposes, allows us to cancel the current task that is +// misbehaving, and flush the queue. void -SpeechSynthesis::DropGlobalQueue() +SpeechSynthesis::ForceEnd() { - nsSynthVoiceRegistry::GetInstance()->DropGlobalQueue(); + if (mCurrentTask) { + mCurrentTask->ForceEnd(); + } } } // namespace dom diff --git a/dom/media/webspeech/synth/SpeechSynthesis.h b/dom/media/webspeech/synth/SpeechSynthesis.h index c8ead468b0..fe031e22d4 100644 --- a/dom/media/webspeech/synth/SpeechSynthesis.h +++ b/dom/media/webspeech/synth/SpeechSynthesis.h @@ -54,7 +54,7 @@ public: void GetVoices(nsTArray< RefPtr >& aResult); - void DropGlobalQueue(); + void ForceEnd(); private: virtual ~SpeechSynthesis(); diff --git a/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl b/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl index 7939e1abf9..0b4f506119 100644 --- a/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl +++ b/dom/media/webspeech/synth/ipc/PSpeechSynthesis.ipdl @@ -41,8 +41,6 @@ parent: sync ReadVoicesAndState() returns (RemoteVoice[] aVoices, nsString[] aDefaults, bool aIsSpeaking); - - DropGlobalQueue(); }; } // namespace dom diff --git a/dom/media/webspeech/synth/ipc/PSpeechSynthesisRequest.ipdl b/dom/media/webspeech/synth/ipc/PSpeechSynthesisRequest.ipdl index b7d0f8bd84..9a146145bb 100644 --- a/dom/media/webspeech/synth/ipc/PSpeechSynthesisRequest.ipdl +++ b/dom/media/webspeech/synth/ipc/PSpeechSynthesisRequest.ipdl @@ -21,6 +21,8 @@ async protocol PSpeechSynthesisRequest Cancel(); + ForceEnd(); + child: __delete__(bool aIsError, float aElapsedTime, uint32_t aCharIndex); diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp index 56b8aee1b8..c6f4e60258 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.cpp @@ -184,5 +184,12 @@ SpeechTaskChild::Cancel() mActor->SendCancel(); } +void +SpeechTaskChild::ForceEnd() +{ + MOZ_ASSERT(mActor); + mActor->SendForceEnd(); +} + } // namespace dom } // namespace mozilla diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h index 887bbe2a06..4714e8d1a6 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisChild.h @@ -90,6 +90,8 @@ public: virtual void Cancel() override; + virtual void ForceEnd() override; + private: SpeechSynthesisRequestChild* mActor; }; diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp index 6caa1d1d7d..2584f29f35 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.cpp @@ -34,14 +34,6 @@ SpeechSynthesisParent::RecvReadVoicesAndState(InfallibleTArray* aVo return true; } -bool -SpeechSynthesisParent::RecvDropGlobalQueue() -{ - nsSynthVoiceRegistry::GetInstance()->DropGlobalQueue(); - - return true; -} - PSpeechSynthesisRequestParent* SpeechSynthesisParent::AllocPSpeechSynthesisRequestParent(const nsString& aText, const nsString& aLang, @@ -127,6 +119,14 @@ SpeechSynthesisRequestParent::RecvCancel() return true; } +bool +SpeechSynthesisRequestParent::RecvForceEnd() +{ + MOZ_ASSERT(mTask); + mTask->ForceEnd(); + return true; +} + // SpeechTaskParent nsresult diff --git a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h index b17049012e..3b10c0ee32 100644 --- a/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h +++ b/dom/media/webspeech/synth/ipc/SpeechSynthesisParent.h @@ -28,8 +28,6 @@ public: InfallibleTArray* aDefaults, bool* aIsSpeaking) override; - bool RecvDropGlobalQueue() override; - protected: SpeechSynthesisParent(); virtual ~SpeechSynthesisParent(); @@ -69,6 +67,8 @@ protected: virtual bool RecvResume() override; virtual bool RecvCancel() override; + + virtual bool RecvForceEnd() override; }; class SpeechTaskParent : public nsSpeechTask diff --git a/dom/media/webspeech/synth/nsSpeechTask.cpp b/dom/media/webspeech/synth/nsSpeechTask.cpp index 682110b0d4..c338e60251 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.cpp +++ b/dom/media/webspeech/synth/nsSpeechTask.cpp @@ -19,6 +19,8 @@ extern PRLogModuleInfo* GetSpeechSynthLog(); #define LOG(type, msg) MOZ_LOG(GetSpeechSynthLog(), type, msg) +#define AUDIO_TRACK 1 + namespace mozilla { namespace dom { @@ -186,7 +188,7 @@ nsSpeechTask::Setup(nsISpeechTaskCallback* aCallback, mChannels = aChannels; AudioSegment* segment = new AudioSegment(); - mStream->AddAudioTrack(1, aRate, 0, segment); + mStream->AddAudioTrack(AUDIO_TRACK, aRate, 0, segment); mStream->AddAudioOutput(this); mStream->SetAudioOutputVolume(this, mVolume); @@ -558,7 +560,7 @@ nsSpeechTask::Pause() } if (mStream) { - mStream->ChangeExplicitBlockerCount(1); + mStream->Suspend(); } if (!mInited) { @@ -581,7 +583,7 @@ nsSpeechTask::Resume() } if (mStream) { - mStream->ChangeExplicitBlockerCount(-1); + mStream->Resume(); } if (mPrePaused) { @@ -607,7 +609,7 @@ nsSpeechTask::Cancel() } if (mStream) { - mStream->ChangeExplicitBlockerCount(1); + mStream->Suspend(); } if (!mInited) { @@ -619,6 +621,20 @@ nsSpeechTask::Cancel() } } +void +nsSpeechTask::ForceEnd() +{ + if (mStream) { + mStream->Suspend(); + } + + if (!mInited) { + mPreCanceled = true; + } + + DispatchEndInner(GetCurrentTime(), GetCurrentCharOffset()); +} + float nsSpeechTask::GetCurrentTime() { diff --git a/dom/media/webspeech/synth/nsSpeechTask.h b/dom/media/webspeech/synth/nsSpeechTask.h index aeb6baabd1..ec4acbb5b4 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.h +++ b/dom/media/webspeech/synth/nsSpeechTask.h @@ -37,6 +37,8 @@ public: virtual void Cancel(); + virtual void ForceEnd(); + float GetCurrentTime(); uint32_t GetCurrentCharOffset(); diff --git a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp index 6f1040aa25..73c046c4ed 100644 --- a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp +++ b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp @@ -756,17 +756,6 @@ nsSynthVoiceRegistry::SetIsSpeaking(bool aIsSpeaking) } } -void -nsSynthVoiceRegistry::DropGlobalQueue() -{ - if (XRE_IsParentProcess()) { - mGlobalQueue.Clear(); - SetIsSpeaking(false); - } else { - mSpeechSynthChild->SendDropGlobalQueue(); - } -} - void nsSynthVoiceRegistry::SpeakImpl(VoiceData* aVoice, nsSpeechTask* aTask, diff --git a/dom/media/webspeech/synth/nsSynthVoiceRegistry.h b/dom/media/webspeech/synth/nsSynthVoiceRegistry.h index 58f3e7e72c..6dc4ccf07c 100644 --- a/dom/media/webspeech/synth/nsSynthVoiceRegistry.h +++ b/dom/media/webspeech/synth/nsSynthVoiceRegistry.h @@ -52,8 +52,6 @@ public: void SetIsSpeaking(bool aIsSpeaking); - void DropGlobalQueue(); - static nsSynthVoiceRegistry* GetInstance(); static already_AddRefed GetInstanceForService(); diff --git a/dom/webidl/SpeechSynthesis.webidl b/dom/webidl/SpeechSynthesis.webidl index f7a0275170..8ac63f5a48 100644 --- a/dom/webidl/SpeechSynthesis.webidl +++ b/dom/webidl/SpeechSynthesis.webidl @@ -25,5 +25,6 @@ interface SpeechSynthesis { sequence getVoices(); [ChromeOnly] - void dropGlobalQueue(); + // Force an utterance to end. Circumvents bad speech service implementations. + void forceEnd(); }; diff --git a/gfx/2d/QuartzSupport.mm b/gfx/2d/QuartzSupport.mm index 96827108ce..05966af6d5 100644 --- a/gfx/2d/QuartzSupport.mm +++ b/gfx/2d/QuartzSupport.mm @@ -25,8 +25,6 @@ - (void)setContentsScale:(double)scale; @end -using RefPtr; - CGColorSpaceRef CreateSystemColorSpace() { CGColorSpaceRef cspace = ::CGDisplayCopyColorSpace(::CGMainDisplayID()); if (!cspace) { diff --git a/gfx/layers/client/TextureClientRecycleAllocator.cpp b/gfx/layers/client/TextureClientRecycleAllocator.cpp index 1a136b3f71..d1208bb0df 100644 --- a/gfx/layers/client/TextureClientRecycleAllocator.cpp +++ b/gfx/layers/client/TextureClientRecycleAllocator.cpp @@ -5,6 +5,7 @@ #include "gfxPlatform.h" #include "mozilla/layers/ISurfaceAllocator.h" +#include "mozilla/layers/CompositableForwarder.h" #include "TextureClientRecycleAllocator.h" namespace mozilla { diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 04d2f0c275..7b1a2a47d3 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -865,7 +865,9 @@ void MediaPipelineTransmit::PipelineListener:: NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, StreamTime offset, uint32_t events, - const MediaSegment& queued_media) { + const MediaSegment& queued_media, + MediaStream* aInputStream, + TrackID aInputTrackID) { MOZ_MTLOG(ML_DEBUG, "MediaPipeline::NotifyQueuedTrackChanges()"); // ignore non-direct data if we're also getting direct data diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h index 1a31ac3bcc..02b86521d6 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.h @@ -482,7 +482,9 @@ public: virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, StreamTime offset, uint32_t events, - const MediaSegment& queued_media) override; + const MediaSegment& queued_media, + MediaStream* input_stream, + TrackID input_tid) override; virtual void NotifyPull(MediaStreamGraph* aGraph, StreamTime aDesiredTime) override {} // Implement MediaStreamDirectListener @@ -629,7 +631,9 @@ class MediaPipelineReceiveAudio : public MediaPipelineReceive { virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, StreamTime offset, uint32_t events, - const MediaSegment& queued_media) override {} + const MediaSegment& queued_media, + MediaStream* input_stream, + TrackID input_tid) override {} virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override; private: @@ -725,7 +729,9 @@ class MediaPipelineReceiveVideo : public MediaPipelineReceive { virtual void NotifyQueuedTrackChanges(MediaStreamGraph* graph, TrackID tid, StreamTime offset, uint32_t events, - const MediaSegment& queued_media) override {} + const MediaSegment& queued_media, + MediaStream* input_stream, + TrackID input_tid) override {} virtual void NotifyPull(MediaStreamGraph* graph, StreamTime desired_time) override; // Accessors for external writes from the renderer diff --git a/media/webrtc/signaling/test/FakeMediaStreams.h b/media/webrtc/signaling/test/FakeMediaStreams.h index eec0265bbe..af189e071f 100644 --- a/media/webrtc/signaling/test/FakeMediaStreams.h +++ b/media/webrtc/signaling/test/FakeMediaStreams.h @@ -74,6 +74,7 @@ protected: virtual ~Fake_VideoSink() {} }; +class Fake_MediaStream; class Fake_SourceMediaStream; class Fake_MediaStreamListener @@ -85,7 +86,9 @@ public: virtual void NotifyQueuedTrackChanges(mozilla::MediaStreamGraph* aGraph, mozilla::TrackID aID, mozilla::StreamTime aTrackOffset, uint32_t aTrackEvents, - const mozilla::MediaSegment& aQueuedMedia) = 0; + const mozilla::MediaSegment& aQueuedMedia, + Fake_MediaStream* aInputStream, + mozilla::TrackID aInputTrackID) = 0; virtual void NotifyPull(mozilla::MediaStreamGraph* aGraph, mozilla::StreamTime aDesiredTime) = 0; NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Fake_MediaStreamListener) diff --git a/media/webrtc/signaling/test/FakeMediaStreamsImpl.h b/media/webrtc/signaling/test/FakeMediaStreamsImpl.h index 30ce661564..47bf91465f 100644 --- a/media/webrtc/signaling/test/FakeMediaStreamsImpl.h +++ b/media/webrtc/signaling/test/FakeMediaStreamsImpl.h @@ -116,7 +116,9 @@ void Fake_AudioStreamSource::Periodic() { 0, // TrackID 0, // Offset TODO(ekr@rtfm.com) fix 0, // ??? - segment); + segment, + nullptr, // Input stream + -1); // Input track id } } diff --git a/widget/cocoa/nsMenuItemIconX.mm b/widget/cocoa/nsMenuItemIconX.mm index 3be533599d..d76b29315e 100644 --- a/widget/cocoa/nsMenuItemIconX.mm +++ b/widget/cocoa/nsMenuItemIconX.mm @@ -43,7 +43,6 @@ #include "nsIContentPolicy.h" using mozilla::gfx::SourceSurface; -using RefPtr; static const uint32_t kIconWidth = 16; static const uint32_t kIconHeight = 16;