diff --git a/browser/confvars.sh b/browser/confvars.sh index 162d47d8de..02c67ba5c7 100644 --- a/browser/confvars.sh +++ b/browser/confvars.sh @@ -31,7 +31,6 @@ MOZ_DISABLE_EXPORT_JS=1 MOZ_WEBGL_CONFORMANT=1 MOZ_ACTIVITIES=1 MOZ_JSDOWNLOADS=1 -MOZ_WEBM_ENCODER=1 MOZ_PHOENIX_EXTENSIONS=1 MOZ_BROWSER_STATUSBAR=1 diff --git a/configure.in b/configure.in index 897c9d85cc..2efbb00391 100644 --- a/configure.in +++ b/configure.in @@ -5412,13 +5412,10 @@ fi if test -n "$MOZ_TREMOR"; then AC_DEFINE(MOZ_TREMOR) - # Tremor doesn't have an encoder. - MOZ_WEBM_ENCODER= fi -if test -n "$MOZ_WEBM_ENCODER"; then - AC_DEFINE(MOZ_WEBM_ENCODER) -fi +MOZ_WEBM_ENCODER=1 +AC_DEFINE(MOZ_WEBM_ENCODER) AC_SUBST(MOZ_WEBM_ENCODER) dnl ================================== diff --git a/dom/animation/AnimValuesStyleRule.h b/dom/animation/AnimValuesStyleRule.h index 2d80ddb531..e2ff8cce1b 100644 --- a/dom/animation/AnimValuesStyleRule.h +++ b/dom/animation/AnimValuesStyleRule.h @@ -58,14 +58,6 @@ public: StyleAnimationValue mValue; }; - void AddPropertiesToSet(nsCSSPropertySet& aSet) const - { - for (size_t i = 0, i_end = mPropertyValuePairs.Length(); i < i_end; ++i) { - const PropertyValuePair &cv = mPropertyValuePairs[i]; - aSet.AddProperty(cv.mProperty); - } - } - private: ~AnimValuesStyleRule() {} diff --git a/dom/locales/en-US/chrome/dom/dom.properties b/dom/locales/en-US/chrome/dom/dom.properties index c122178639..fa423e9828 100644 --- a/dom/locales/en-US/chrome/dom/dom.properties +++ b/dom/locales/en-US/chrome/dom/dom.properties @@ -110,11 +110,14 @@ MediaLoadSourceMediaNotMatched=Specified "media" attribute of "%1$S" does not ma MediaLoadUnsupportedMimeType=HTTP "Content-Type" of "%1$S" is not supported. Load of media resource %2$S failed. # LOCALIZATION NOTE: %S is the URL of the media resource which failed to load because of error in decoding. MediaLoadDecodeError=Media resource %S could not be decoded. +MediaWidevineNoWMFNoSilverlight=Trying to play Widevine with no Windows Media Foundation (nor Silverlight fallback), see https://support.mozilla.org/kb/fix-video-audio-problems-firefox-windows # LOCALIZATION NOTE: %S is a comma-separated list of codecs (e.g. 'video/mp4, video/webm') -MediaCannotPlayNoDecoders=Cannot play media. No decoders for requested formats: %S +MediaWMFNeeded=To play video formats %S, you need to install extra Microsoft software, see https://support.mozilla.org/kb/fix-video-audio-problems-firefox-windows # LOCALIZATION NOTE: %S is a comma-separated list of codecs (e.g. 'video/mp4, video/webm') MediaPlatformDecoderNotFound=The video on this page can't be played. Your system may not have the required video codecs for: %S # LOCALIZATION NOTE: %S is a comma-separated list of codecs (e.g. 'video/mp4, video/webm') +MediaCannotPlayNoDecoders=Cannot play media. No decoders for requested formats: %S +# LOCALIZATION NOTE: %S is a comma-separated list of codecs (e.g. 'video/mp4, video/webm') MediaNoDecoders=No decoders for some of the requested formats: %S # LOCALIZATION NOTE: Do not translate "MediaRecorder". MediaRecorderMultiTracksNotSupported=MediaRecorder does not support recording multiple tracks of the same type at this time. diff --git a/dom/media/AudioChannelFormat.h b/dom/media/AudioChannelFormat.h index ec9180a629..203ebaf5b0 100644 --- a/dom/media/AudioChannelFormat.h +++ b/dom/media/AudioChannelFormat.h @@ -65,6 +65,8 @@ GetAudioChannelsSuperset(uint32_t aChannels1, uint32_t aChannels2); * every input channel is multiplied by the same coefficient for every output * channel it contributes to. */ +const float SQRT_ONE_HALF = sqrt(0.5); + struct DownMixMatrix { // Every input channel c is copied to output channel mInputDestination[c] // after multiplying by mInputCoefficient[c]. @@ -83,19 +85,19 @@ gDownMixMatrices[CUSTOM_CHANNEL_LAYOUTS*(CUSTOM_CHANNEL_LAYOUTS - 1)/2] = { { 0, IGNORE, IGNORE }, IGNORE, { 1.0f, IGNORE_F, IGNORE_F } }, { { 0, 0, 0, 0 }, IGNORE, { 0.25f, 0.25f, 0.25f, 0.25f } }, { { 0, IGNORE, IGNORE, IGNORE, IGNORE }, IGNORE, { 1.0f, IGNORE_F, IGNORE_F, IGNORE_F, IGNORE_F } }, - { { 0, 0, 0, IGNORE, 0, 0 }, IGNORE, { 0.7071f, 0.7071f, 1.0f, IGNORE_F, 0.5f, 0.5f } }, + { { 0, 0, 0, IGNORE, 0, 0 }, IGNORE, { SQRT_ONE_HALF, SQRT_ONE_HALF, 1.0f, IGNORE_F, 0.5f, 0.5f } }, // Downmixes to stereo { { 0, 1, IGNORE }, IGNORE, { 1.0f, 1.0f, IGNORE_F } }, { { 0, 1, 0, 1 }, IGNORE, { 0.5f, 0.5f, 0.5f, 0.5f } }, { { 0, 1, IGNORE, IGNORE, IGNORE }, IGNORE, { 1.0f, 1.0f, IGNORE_F, IGNORE_F, IGNORE_F } }, - { { 0, 1, 0, IGNORE, 0, 1 }, 1, { 1.0f, 1.0f, 0.7071f, IGNORE_F, 0.7071f, 0.7071f } }, + { { 0, 1, 0, IGNORE, 0, 1 }, 1, { 1.0f, 1.0f, SQRT_ONE_HALF, IGNORE_F, SQRT_ONE_HALF, SQRT_ONE_HALF } }, // Downmixes to 3-channel { { 0, 1, 2, IGNORE }, IGNORE, { 1.0f, 1.0f, 1.0f, IGNORE_F } }, { { 0, 1, 2, IGNORE, IGNORE }, IGNORE, { 1.0f, 1.0f, 1.0f, IGNORE_F, IGNORE_F } }, { { 0, 1, 2, IGNORE, IGNORE, IGNORE }, IGNORE, { 1.0f, 1.0f, 1.0f, IGNORE_F, IGNORE_F, IGNORE_F } }, // Downmixes to quad { { 0, 1, 2, 3, IGNORE }, IGNORE, { 1.0f, 1.0f, 1.0f, 1.0f, IGNORE_F } }, - { { 0, 1, 0, IGNORE, 2, 3 }, 1, { 1.0f, 1.0f, 0.7071f, IGNORE_F, 1.0f, 1.0f } }, + { { 0, 1, 0, IGNORE, 2, 3 }, 1, { 1.0f, 1.0f, SQRT_ONE_HALF, IGNORE_F, 1.0f, 1.0f } }, // Downmixes to 5-channel { { 0, 1, 2, 3, 4, IGNORE }, IGNORE, { 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, IGNORE_F } } }; diff --git a/dom/media/AudioCompactor.h b/dom/media/AudioCompactor.h index 86957e7be8..827c75e256 100644 --- a/dom/media/AudioCompactor.h +++ b/dom/media/AudioCompactor.h @@ -49,7 +49,7 @@ public: while (aFrames > 0) { uint32_t samples = GetChunkSamples(aFrames, aChannels, maxSlop); - if (aFrames * aChannels > mSamplesPadding) { + if (samples / aChannels > mSamplesPadding / aChannels + 1) { samples -= mSamplesPadding; } AlignedAudioBuffer buffer(samples); diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index a5d72b5955..0308d042a1 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -35,6 +35,12 @@ #undef LOG #endif +// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to +// GetTickCount() and conflicts with MediaStream::GetCurrentTime. +#ifdef GetCurrentTime +#undef GetCurrentTime +#endif + using namespace mozilla; using namespace mozilla::dom; using namespace mozilla::layers; @@ -1405,7 +1411,6 @@ DOMHwMediaStream::Init(MediaStream* stream, OverlayImage* aImage) #endif srcStream->AddTrack(TRACK_VIDEO_PRIMARY, 0, new VideoSegment()); srcStream->AppendToTrack(TRACK_VIDEO_PRIMARY, &segment); - srcStream->FinishAddTracks(); srcStream->AdvanceKnownTracksTime(STREAM_TIME_MAX); } } diff --git a/dom/media/DOMMediaStream.h b/dom/media/DOMMediaStream.h index 434ca6c747..57d0e7a73a 100644 --- a/dom/media/DOMMediaStream.h +++ b/dom/media/DOMMediaStream.h @@ -16,12 +16,6 @@ #include "mozilla/DOMEventTargetHelper.h" #include "PrincipalChangeObserver.h" -// GetCurrentTime is defined in winbase.h as zero argument macro forwarding to -// GetTickCount() and conflicts with NS_DECL_NSIDOMMEDIASTREAM, containing -// currentTime getter. -#ifdef GetCurrentTime -#undef GetCurrentTime -#endif // X11 has a #define for CurrentTime. Unbelievable :-(. // See dom/media/webaudio/AudioContext.h for more fun! #ifdef CurrentTime diff --git a/dom/media/DecoderDoctorDiagnostics.cpp b/dom/media/DecoderDoctorDiagnostics.cpp index edf294ca75..a409c84fea 100644 --- a/dom/media/DecoderDoctorDiagnostics.cpp +++ b/dom/media/DecoderDoctorDiagnostics.cpp @@ -8,11 +8,13 @@ #include "mozilla/dom/DecoderDoctorNotificationBinding.h" #include "mozilla/Logging.h" +#include "mozilla/Preferences.h" #include "nsGkAtoms.h" #include "nsIDocument.h" #include "nsIObserverService.h" #include "nsITimer.h" #include "nsIWeakReference.h" +#include "nsPluginHost.h" static mozilla::LazyLogModule sDecoderDoctorLog("DecoderDoctor"); #define DD_LOG(level, arg, ...) MOZ_LOG(sDecoderDoctorLog, level, (arg, ##__VA_ARGS__)) @@ -265,7 +267,7 @@ void DecoderDoctorDocumentWatcher::ReportAnalysis( dom::DecoderDoctorNotificationType aNotificationType, const char* aReportStringId, - const nsAString& aFormats) + const nsAString& aParams) { MOZ_ASSERT(NS_IsMainThread()); @@ -273,24 +275,82 @@ DecoderDoctorDocumentWatcher::ReportAnalysis( return; } - const char16_t* params[] = { aFormats.Data() }; + // 'params' will only be forwarded for non-empty strings. + const char16_t* params[1] = { aParams.Data() }; DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::ReportAnalysis() ReportToConsole - aMsg='%s' params[0]='%s'", this, mDocument, aReportStringId, - NS_ConvertUTF16toUTF8(params[0]).get()); + aParams.IsEmpty() ? "" : NS_ConvertUTF16toUTF8(params[0]).get()); nsContentUtils::ReportToConsole(nsIScriptError::warningFlag, NS_LITERAL_CSTRING("Media"), mDocument, nsContentUtils::eDOM_PROPERTIES, aReportStringId, - params, - ArrayLength(params)); + aParams.IsEmpty() ? nullptr : params, + aParams.IsEmpty() ? 0 : 1); - // For now, disable all front-end notifications by default. - // TODO: Future bugs will use finer-grained filtering instead. - if (Preferences::GetBool("media.decoderdoctor.enable-notification-bar", false)) { - DispatchNotification( - mDocument->GetInnerWindow(), aNotificationType, aFormats); + // "media.decoder-doctor.notifications-allowed" controls which notifications + // may be dispatched to the front-end. It either contains: + // - '*' -> Allow everything. + // - Comma-separater list of ids -> Allow if aReportStringId (from + // dom.properties) is one of them. + // - Nothing (missing or empty) -> Disable everything. + nsAdoptingCString filter = + Preferences::GetCString("media.decoder-doctor.notifications-allowed"); + filter.StripWhitespace(); + bool allowed = false; + if (!filter || filter.IsEmpty()) { + // Allow nothing. + } else if (filter.EqualsLiteral("*")) { + allowed = true; + } else for (uint32_t start = 0; start < filter.Length(); ) { + int32_t comma = filter.FindChar(',', start); + uint32_t end = (comma >= 0) ? uint32_t(comma) : filter.Length(); + if (strncmp(aReportStringId, filter.Data() + start, end - start) == 0) { + allowed = true; + break; + } + // Skip comma. End of line will be caught in for 'while' clause. + start = end + 1; } + if (allowed) { + DispatchNotification( + mDocument->GetInnerWindow(), aNotificationType, aParams); + } +} + +enum SilverlightPresence { + eNoSilverlight, + eSilverlightDisabled, + eSilverlightEnabled +}; +static SilverlightPresence +CheckSilverlight() +{ + MOZ_ASSERT(NS_IsMainThread()); + RefPtr pluginHost = nsPluginHost::GetInst(); + if (!pluginHost) { + return eNoSilverlight; + } + nsTArray> plugins; + pluginHost->GetPlugins(plugins, /*aIncludeDisabled*/ true); + for (const auto& plugin : plugins) { + for (const auto& mime : plugin->MimeTypes()) { + if (mime.LowerCaseEqualsLiteral("application/x-silverlight") + || mime.LowerCaseEqualsLiteral("application/x-silverlight-2")) { + return plugin->IsEnabled() ? eSilverlightEnabled : eSilverlightDisabled; + } + } + } + + return eNoSilverlight; +} + +static void AppendToStringList(nsAString& list, const nsAString& item) +{ + if (!list.IsEmpty()) { + list += NS_LITERAL_STRING(", "); + } + list += item; } void @@ -299,35 +359,53 @@ DecoderDoctorDocumentWatcher::SynthesizeAnalysis() MOZ_ASSERT(NS_IsMainThread()); bool canPlay = false; +#if defined(XP_WIN) + bool WMFNeeded = false; +#endif #if defined(MOZ_FFMPEG) bool FFMpegNeeded = false; #endif + nsAutoString playableFormats; nsAutoString unplayableFormats; + nsAutoString supportedKeySystems; nsAutoString unsupportedKeySystems; + DecoderDoctorDiagnostics::KeySystemIssue lastKeySystemIssue = + DecoderDoctorDiagnostics::eUnset; for (auto& diag : mDiagnosticsSequence) { switch (diag.mDecoderDoctorDiagnostics.Type()) { case DecoderDoctorDiagnostics::eFormatSupportCheck: if (diag.mDecoderDoctorDiagnostics.CanPlay()) { canPlay = true; + AppendToStringList(playableFormats, + diag.mDecoderDoctorDiagnostics.Format()); } else { +#if defined(XP_WIN) + if (diag.mDecoderDoctorDiagnostics.DidWMFFailToLoad()) { + WMFNeeded = true; + } +#endif #if defined(MOZ_FFMPEG) if (diag.mDecoderDoctorDiagnostics.DidFFmpegFailToLoad()) { FFMpegNeeded = true; } #endif - if (!unplayableFormats.IsEmpty()) { - unplayableFormats += NS_LITERAL_STRING(", "); - } - unplayableFormats += diag.mDecoderDoctorDiagnostics.Format(); + AppendToStringList(unplayableFormats, + diag.mDecoderDoctorDiagnostics.Format()); } break; case DecoderDoctorDiagnostics::eMediaKeySystemAccessRequest: - if (!diag.mDecoderDoctorDiagnostics.IsKeySystemSupported()) { - if (!unsupportedKeySystems.IsEmpty()) { - unsupportedKeySystems += NS_LITERAL_STRING(", "); + if (diag.mDecoderDoctorDiagnostics.IsKeySystemSupported()) { + AppendToStringList(supportedKeySystems, + diag.mDecoderDoctorDiagnostics.KeySystem()); + } else { + AppendToStringList(unsupportedKeySystems, + diag.mDecoderDoctorDiagnostics.KeySystem()); + DecoderDoctorDiagnostics::KeySystemIssue issue = + diag.mDecoderDoctorDiagnostics.GetKeySystemIssue(); + if (issue != DecoderDoctorDiagnostics::eUnset) { + lastKeySystemIssue = issue; } - unsupportedKeySystems += diag.mDecoderDoctorDiagnostics.KeySystem(); } break; default: @@ -339,33 +417,61 @@ DecoderDoctorDocumentWatcher::SynthesizeAnalysis() } } - if (!canPlay) { + // Look at Key System issues first, as they may influence format checks. + if (!unsupportedKeySystems.IsEmpty() && supportedKeySystems.IsEmpty()) { + // No supported key systems! + switch (lastKeySystemIssue) { + case DecoderDoctorDiagnostics::eWidevineWithNoWMF: + if (CheckSilverlight() != eSilverlightEnabled) { + DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - unsupported key systems: %s, widevine without WMF nor Silverlight", + this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); + ReportAnalysis(dom::DecoderDoctorNotificationType::Platform_decoder_not_found, + "MediaWidevineNoWMFNoSilverlight", NS_LITERAL_STRING("")); + return; + } + break; + default: + break; + } + } + + if (!canPlay && !unplayableFormats.IsEmpty()) { +#if defined(XP_WIN) + if (WMFNeeded) { + DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - formats: %s -> Cannot play media because WMF was not found", + this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); + ReportAnalysis(dom::DecoderDoctorNotificationType::Platform_decoder_not_found, + "MediaWMFNeeded", unplayableFormats); + return; + } +#endif #if defined(MOZ_FFMPEG) if (FFMpegNeeded) { DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - unplayable formats: %s -> Cannot play media because platform decoder was not found", this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); ReportAnalysis(dom::DecoderDoctorNotificationType::Platform_decoder_not_found, "MediaPlatformDecoderNotFound", unplayableFormats); - } else -#endif - { - DD_WARN("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - Cannot play media, unplayable formats: %s", - this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); - ReportAnalysis(dom::DecoderDoctorNotificationType::Cannot_play, - "MediaCannotPlayNoDecoders", unplayableFormats); + return; } - } else if (!unplayableFormats.IsEmpty()) { +#endif + DD_WARN("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - Cannot play media, unplayable formats: %s", + this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); + ReportAnalysis(dom::DecoderDoctorNotificationType::Cannot_play, + "MediaCannotPlayNoDecoders", unplayableFormats); + return; + } + if (!unplayableFormats.IsEmpty()) { DD_INFO("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - Can play media, but no decoders for some requested formats: %s", this, mDocument, NS_ConvertUTF16toUTF8(unplayableFormats).get()); - if (Preferences::GetBool("media.decoderdoctor.verbose", false)) { + if (Preferences::GetBool("media.decoder-doctor.verbose", false)) { ReportAnalysis( dom::DecoderDoctorNotificationType::Can_play_but_some_missing_decoders, "MediaNoDecoders", unplayableFormats); } - } else { - DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - Can play media, decoders available for all requested formats", - this, mDocument); + return; } + DD_DEBUG("DecoderDoctorDocumentWatcher[%p, doc=%p]::SynthesizeAnalysis() - Can play media, decoders available for all requested formats", + this, mDocument); } void diff --git a/dom/media/FileBlockCache.cpp b/dom/media/FileBlockCache.cpp index 3a9e929144..8121b5e4d5 100644 --- a/dom/media/FileBlockCache.cpp +++ b/dom/media/FileBlockCache.cpp @@ -87,6 +87,14 @@ void FileBlockCache::Close() } } +template +bool +ContainerContains(const Container& aContainer, const Value& value) +{ + return std::find(aContainer.begin(), aContainer.end(), value) + != aContainer.end(); +} + nsresult FileBlockCache::WriteBlock(uint32_t aBlockIndex, const uint8_t* aData) { MonitorAutoLock mon(mDataMonitor); @@ -99,16 +107,16 @@ nsresult FileBlockCache::WriteBlock(uint32_t aBlockIndex, const uint8_t* aData) bool blockAlreadyHadPendingChange = mBlockChanges[aBlockIndex] != nullptr; mBlockChanges[aBlockIndex] = new BlockChange(aData); - if (!blockAlreadyHadPendingChange || !mChangeIndexList.Contains(aBlockIndex)) { + if (!blockAlreadyHadPendingChange || !ContainerContains(mChangeIndexList, aBlockIndex)) { // We either didn't already have a pending change for this block, or we // did but we didn't have an entry for it in mChangeIndexList (we're in the process // of writing it and have removed the block's index out of mChangeIndexList // in Run() but not finished writing the block to file yet). Add the blocks // index to the end of mChangeIndexList to ensure the block is written as // as soon as possible. - mChangeIndexList.PushBack(aBlockIndex); + mChangeIndexList.push_back(aBlockIndex); } - NS_ASSERTION(mChangeIndexList.Contains(aBlockIndex), "Must have entry for new block"); + NS_ASSERTION(ContainerContains(mChangeIndexList, aBlockIndex), "Must have entry for new block"); EnsureWriteScheduled(); @@ -196,10 +204,10 @@ nsresult FileBlockCache::Run() { MonitorAutoLock mon(mDataMonitor); NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread"); - NS_ASSERTION(!mChangeIndexList.IsEmpty(), "Only dispatch when there's work to do"); + NS_ASSERTION(!mChangeIndexList.empty(), "Only dispatch when there's work to do"); NS_ASSERTION(mIsWriteScheduled, "Should report write running or scheduled."); - while (!mChangeIndexList.IsEmpty()) { + while (!mChangeIndexList.empty()) { if (!mIsOpen) { // We've been closed, abort, discarding unwritten changes. mIsWriteScheduled = false; @@ -217,7 +225,8 @@ nsresult FileBlockCache::Run() // Hold a reference to the change, in case another change // overwrites the mBlockChanges entry for this block while we drop // mDataMonitor to take mFileMonitor. - int32_t blockIndex = mChangeIndexList.PopFront(); + int32_t blockIndex = mChangeIndexList.front(); + mChangeIndexList.pop_front(); RefPtr change = mBlockChanges[blockIndex]; MOZ_ASSERT(change, "Change index list should only contain entries for blocks " @@ -326,14 +335,14 @@ nsresult FileBlockCache::MoveBlock(int32_t aSourceBlockIndex, int32_t aDestBlock } if (mBlockChanges[aDestBlockIndex] == nullptr || - !mChangeIndexList.Contains(aDestBlockIndex)) { + !ContainerContains(mChangeIndexList, aDestBlockIndex)) { // Only add another entry to the change index list if we don't already // have one for this block. We won't have an entry when either there's // no pending change for this block, or if there is a pending change for // this block and we're in the process of writing it (we've popped the // block's index out of mChangeIndexList in Run() but not finished writing // the block to file yet. - mChangeIndexList.PushBack(aDestBlockIndex); + mChangeIndexList.push_back(aDestBlockIndex); } // If the source block hasn't yet been written to file then the dest block @@ -346,7 +355,7 @@ nsresult FileBlockCache::MoveBlock(int32_t aSourceBlockIndex, int32_t aDestBlock EnsureWriteScheduled(); - NS_ASSERTION(mChangeIndexList.Contains(aDestBlockIndex), + NS_ASSERTION(ContainerContains(mChangeIndexList, aDestBlockIndex), "Should have scheduled block for change"); return NS_OK; diff --git a/dom/media/FileBlockCache.h b/dom/media/FileBlockCache.h index c184801d26..e803e40759 100644 --- a/dom/media/FileBlockCache.h +++ b/dom/media/FileBlockCache.h @@ -15,6 +15,7 @@ #include "nsDeque.h" #include "nsThreadUtils.h" #include "mozilla/SharedThreadPool.h" +#include struct PRFileDesc; @@ -125,38 +126,6 @@ public: } }; - class Int32Queue : private nsDeque { - public: - int32_t PopFront() { - int32_t front = ObjectAt(0); - nsDeque::PopFront(); - return front; - } - - void PushBack(int32_t aValue) { - nsDeque::Push(reinterpret_cast(aValue)); - } - - bool Contains(int32_t aValue) { - for (size_t i = 0; i < GetSize(); ++i) { - if (ObjectAt(i) == aValue) { - return true; - } - } - return false; - } - - bool IsEmpty() { - return nsDeque::GetSize() == 0; - } - - private: - int32_t ObjectAt(size_t aIndex) { - void* v = nsDeque::ObjectAt(aIndex); - return reinterpret_cast(v); - } - }; - private: int64_t BlockIndexToOffset(int32_t aBlockIndex) { return static_cast(aBlockIndex) * BLOCK_SIZE; @@ -203,8 +172,7 @@ private: // main thread). nsCOMPtr mThread; // Queue of pending block indexes that need to be written or moved. - //AutoTArray mChangeIndexList; - Int32Queue mChangeIndexList; + std::deque mChangeIndexList; // True if we've dispatched an event to commit all pending block changes // to file on mThread. bool mIsWriteScheduled; diff --git a/dom/media/MP3Demuxer.cpp b/dom/media/MP3Demuxer.cpp index c54c2c441e..a96808ba28 100644 --- a/dom/media/MP3Demuxer.cpp +++ b/dom/media/MP3Demuxer.cpp @@ -148,6 +148,15 @@ MP3TrackDemuxer::Init() { return mSamplesPerSecond && mChannels; } +media::TimeUnit +MP3TrackDemuxer::SeekPosition() const { + TimeUnit pos = Duration(mFrameIndex); + if (Duration() > TimeUnit()) { + pos = std::min(Duration(), pos); + } + return pos; +} + const FrameParser::Frame& MP3TrackDemuxer::LastFrame() const { return mParser.PrevFrame(); @@ -158,15 +167,6 @@ MP3TrackDemuxer::DemuxSample() { return GetNextFrame(FindNextFrame()); } -media::TimeUnit -MP3TrackDemuxer::SeekPosition() const { - TimeUnit pos = Duration(mFrameIndex); - if (Duration() > TimeUnit()) { - pos = std::min(Duration(), pos); - } - return pos; -} - const ID3Parser::ID3Header& MP3TrackDemuxer::ID3Header() const { return mParser.ID3Header(); @@ -203,7 +203,7 @@ MP3TrackDemuxer::FastSeek(const TimeUnit& aTime) { if (!aTime.ToMicroseconds()) { // Quick seek to the beginning of the stream. mFrameIndex = 0; - } else if (vbr.IsTOCPresent() && Duration().ToMicroseconds() > 0) { + } else if (vbr.IsTOCPresent() && Duration().ToMicroseconds() > 0) { // Use TOC for more precise seeking. const float durationFrac = static_cast(aTime.ToMicroseconds()) / Duration().ToMicroseconds(); @@ -1064,7 +1064,7 @@ FrameParser::VBRHeader::ParseXing(ByteReader* aReader) { // Skip across the VBR header ID tag. aReader->ReadU32(); mType = XING; - } + } uint32_t flags = 0; if (aReader->CanRead32()) { flags = aReader->ReadU32(); diff --git a/dom/media/MP3Demuxer.h b/dom/media/MP3Demuxer.h index d32f2cb089..502cc5cadb 100644 --- a/dom/media/MP3Demuxer.h +++ b/dom/media/MP3Demuxer.h @@ -372,9 +372,11 @@ public: // or a 0-duration if unknown. media::TimeUnit Duration(int64_t aNumFrames) const; + // Returns the estimated current seek position time. + media::TimeUnit SeekPosition() const; + const FrameParser::Frame& LastFrame() const; RefPtr DemuxSample(); - media::TimeUnit SeekPosition() const; const ID3Parser::ID3Header& ID3Header() const; const FrameParser::VBRHeader& VBRInfo() const; @@ -422,7 +424,7 @@ private: // Returns the estimated frame index for the given offset. int64_t FrameIndexFromOffset(int64_t aOffset) const; - // Returns the estimated frame index for the given time. + // Returns the estimated frame index for the given time. int64_t FrameIndexFromTime(const media::TimeUnit& aTime) const; // Restricts the read size aSize to prevent blocking reads past stream length. diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 84c0e10e85..cef367f76a 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -733,7 +733,7 @@ MediaDecoder::InitializeStateMachine() MOZ_ASSERT(NS_IsMainThread()); NS_ASSERTION(mDecoderStateMachine, "Cannot initialize null state machine!"); - nsresult rv = mDecoderStateMachine->Init(); + nsresult rv = mDecoderStateMachine->Init(this); NS_ENSURE_SUCCESS(rv, rv); // If some parameters got set before the state machine got created, diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp index ec39942c25..8a07186e4e 100644 --- a/dom/media/MediaDecoderStateMachine.cpp +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -295,11 +295,6 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); - // Dispatch initialization that needs to happen on that task queue. - nsCOMPtr r = NS_NewRunnableMethodWithArg>( - this, &MediaDecoderStateMachine::InitializationTask, aDecoder); - mTaskQueue->Dispatch(r.forget()); - InitVideoQueuePrefs(); mBufferingWait = IsRealTime() ? 0 : 15; @@ -313,22 +308,6 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, // timeEndPeriod() call. timeBeginPeriod(1); #endif - - mAudioQueueListener = AudioQueue().PopEvent().Connect( - mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped); - mVideoQueueListener = VideoQueue().PopEvent().Connect( - mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped); - - mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread()); - - mMediaSink = CreateMediaSink(mAudioCaptured); - -#ifdef MOZ_EME - mCDMProxyPromise.Begin(aDecoder->RequestCDMProxy()->Then( - OwnerThread(), __func__, this, - &MediaDecoderStateMachine::OnCDMProxyReady, - &MediaDecoderStateMachine::OnCDMProxyNotReady)); -#endif } MediaDecoderStateMachine::~MediaDecoderStateMachine() @@ -403,8 +382,7 @@ MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured) RefPtr mediaSink = new VideoSink(mTaskQueue, audioSink, mVideoQueue, - mVideoFrameContainer, mRealTime, - *mFrameStats, + mVideoFrameContainer, *mFrameStats, sVideoQueueSendToCompositorSize); return mediaSink.forget(); } @@ -1065,14 +1043,35 @@ bool MediaDecoderStateMachine::IsPlaying() const return mMediaSink->IsPlaying(); } -nsresult MediaDecoderStateMachine::Init() +nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder) { MOZ_ASSERT(NS_IsMainThread()); + + // Dispatch initialization that needs to happen on that task queue. + nsCOMPtr r = NS_NewRunnableMethodWithArg>( + this, &MediaDecoderStateMachine::InitializationTask, aDecoder); + mTaskQueue->Dispatch(r.forget()); + + mAudioQueueListener = AudioQueue().PopEvent().Connect( + mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped); + mVideoQueueListener = VideoQueue().PopEvent().Connect( + mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped); + + mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread()); + + mMediaSink = CreateMediaSink(mAudioCaptured); + +#ifdef MOZ_EME + mCDMProxyPromise.Begin(aDecoder->RequestCDMProxy()->Then( + OwnerThread(), __func__, this, + &MediaDecoderStateMachine::OnCDMProxyReady, + &MediaDecoderStateMachine::OnCDMProxyNotReady)); +#endif + nsresult rv = mReader->Init(); NS_ENSURE_SUCCESS(rv, rv); - nsCOMPtr r = NS_NewRunnableMethod( - this, &MediaDecoderStateMachine::ReadMetadata); + r = NS_NewRunnableMethod(this, &MediaDecoderStateMachine::ReadMetadata); OwnerThread()->Dispatch(r.forget()); return NS_OK; diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index 734c1639b0..0954219757 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -142,7 +142,7 @@ public: MediaDecoderReader* aReader, bool aRealTime = false); - nsresult Init(); + nsresult Init(MediaDecoder* aDecoder); // Enumeration for the valid decoding states enum State { diff --git a/dom/media/MediaEventSource.h b/dom/media/MediaEventSource.h index cff00190e5..26c4cca4ad 100644 --- a/dom/media/MediaEventSource.h +++ b/dom/media/MediaEventSource.h @@ -9,6 +9,7 @@ #include "mozilla/AbstractThread.h" #include "mozilla/Atomics.h" +#include "mozilla/IndexSequence.h" #include "mozilla/Mutex.h" #include "mozilla/Tuple.h" #include "mozilla/TypeTraits.h" diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp index 2a0ff44476..effc95f6f5 100644 --- a/dom/media/encoder/MediaEncoder.cpp +++ b/dom/media/encoder/MediaEncoder.cpp @@ -15,11 +15,7 @@ #include "OggWriter.h" #include "OpusTrackEncoder.h" -#ifdef MOZ_VORBIS -#include "VorbisTrackEncoder.h" -#endif #ifdef MOZ_WEBM_ENCODER -#include "VorbisTrackEncoder.h" #include "VP8TrackEncoder.h" #include "WebMWriter.h" #endif @@ -78,6 +74,16 @@ MediaEncoder::NotifyQueuedTrackChanges(MediaStreamGraph* aGraph, if (!mDirectConnected) { NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, aQueuedMedia); } else { + if (aTrackEvents != 0) { + // forward events (TRACK_EVENT_ENDED) but not the media + if (aQueuedMedia.GetType() == MediaSegment::VIDEO) { + VideoSegment segment; + NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment); + } else { + AudioSegment segment; + NotifyRealtimeData(aGraph, aID, aTrackOffset, aTrackEvents, segment); + } + } if (mSuspended == RECORD_RESUMED) { if (mVideoEncoder) { if (aQueuedMedia.GetType() == MediaSegment::VIDEO) { @@ -135,8 +141,9 @@ MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate, else if (MediaEncoder::IsWebMEncoderEnabled() && (aMIMEType.EqualsLiteral(VIDEO_WEBM) || (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) { - if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK) { - audioEncoder = new VorbisTrackEncoder(); + if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK + && MediaDecoder::IsOpusEnabled()) { + audioEncoder = new OpusTrackEncoder(); NS_ENSURE_TRUE(audioEncoder, nullptr); } videoEncoder = new VP8TrackEncoder(); diff --git a/dom/media/encoder/OmxTrackEncoder.cpp b/dom/media/encoder/OmxTrackEncoder.cpp index 8016e4b1ef..4678fb73bd 100644 --- a/dom/media/encoder/OmxTrackEncoder.cpp +++ b/dom/media/encoder/OmxTrackEncoder.cpp @@ -113,16 +113,12 @@ OmxVideoTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) while (!iter.IsEnded()) { VideoChunk chunk = *iter; - // Send only the unique video frames to OMXCodecWrapper. - if (mLastFrame != chunk.mFrame) { - uint64_t totalDurationUs = mTotalFrameDuration * USECS_PER_S / mTrackRate; - layers::Image* img = (chunk.IsNull() || chunk.mFrame.GetForceBlack()) ? - nullptr : chunk.mFrame.GetImage(); - rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs); - NS_ENSURE_SUCCESS(rv, rv); - } + uint64_t totalDurationUs = mTotalFrameDuration * USECS_PER_S / mTrackRate; + layers::Image* img = (chunk.IsNull() || chunk.mFrame.GetForceBlack()) ? + nullptr : chunk.mFrame.GetImage(); + rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs); + NS_ENSURE_SUCCESS(rv, rv); - mLastFrame.TakeFrom(&chunk.mFrame); mTotalFrameDuration += chunk.GetDuration(); iter.Next(); diff --git a/dom/media/encoder/OpusTrackEncoder.cpp b/dom/media/encoder/OpusTrackEncoder.cpp index 3785d7f06d..6de0adacd4 100644 --- a/dom/media/encoder/OpusTrackEncoder.cpp +++ b/dom/media/encoder/OpusTrackEncoder.cpp @@ -129,6 +129,7 @@ OpusTrackEncoder::OpusTrackEncoder() , mEncoder(nullptr) , mLookahead(0) , mResampler(nullptr) + , mOutputTimeStamp(0) { } @@ -229,6 +230,8 @@ OpusTrackEncoder::GetMetadata() } RefPtr meta = new OpusMetadata(); + meta->mChannels = mChannels; + meta->mSamplingFrequency = mSamplingRate; mLookahead = 0; int error = opus_encoder_ctl(mEncoder, OPUS_GET_LOOKAHEAD(&mLookahead)); @@ -437,6 +440,9 @@ OpusTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) } audiodata->SwapInFrameData(frameData); + mOutputTimeStamp += FramesToUsecs(GetPacketDuration(), kOpusSamplingRate).value(); + audiodata->SetTimeStamp(mOutputTimeStamp); + LOG("[Opus] mOutputTimeStamp %lld.",mOutputTimeStamp); aData.AppendEncodedFrame(audiodata); return result >= 0 ? NS_OK : NS_ERROR_FAILURE; } diff --git a/dom/media/encoder/OpusTrackEncoder.h b/dom/media/encoder/OpusTrackEncoder.h index 557065b454..8fd21d49be 100644 --- a/dom/media/encoder/OpusTrackEncoder.h +++ b/dom/media/encoder/OpusTrackEncoder.h @@ -22,7 +22,8 @@ public: nsTArray mIdHeader; // The Comment Header of OggOpus. nsTArray mCommentHeader; - + int32_t mChannels; + float mSamplingFrequency; MetadataKind GetKind() const override { return METADATA_OPUS; } }; @@ -80,6 +81,9 @@ private: * They will be prepended to the resampled frames next encoding cycle. */ nsTArray mResampledLeftover; + + // TimeStamp in microseconds. + uint64_t mOutputTimeStamp; }; } // namespace mozilla diff --git a/dom/media/encoder/VP8TrackEncoder.cpp b/dom/media/encoder/VP8TrackEncoder.cpp index 48a33a4d19..7cb677bfef 100644 --- a/dom/media/encoder/VP8TrackEncoder.cpp +++ b/dom/media/encoder/VP8TrackEncoder.cpp @@ -112,7 +112,9 @@ VP8TrackEncoder::Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth, config.rc_dropframe_thresh = 0; config.rc_end_usage = VPX_CBR; config.g_pass = VPX_RC_ONE_PASS; - config.rc_resize_allowed = 1; + // ffmpeg doesn't currently support streams that use resize. + // Therefore, for safety, we should turn it off until it does. + config.rc_resize_allowed = 0; config.rc_undershoot_pct = 100; config.rc_overshoot_pct = 15; config.rc_buf_initial_sz = 500; @@ -167,7 +169,7 @@ VP8TrackEncoder::GetMetadata() return meta.forget(); } -nsresult +bool VP8TrackEncoder::GetEncodedPartitions(EncodedFrameContainer& aData) { vpx_codec_iter_t iter = nullptr; @@ -195,21 +197,18 @@ VP8TrackEncoder::GetEncodedPartitions(EncodedFrameContainer& aData) } } - if (!frameData.IsEmpty() && - (pkt->data.frame.pts == mEncodedTimestamp)) { + if (!frameData.IsEmpty()) { // Copy the encoded data to aData. EncodedFrame* videoData = new EncodedFrame(); videoData->SetFrameType(frameType); // Convert the timestamp and duration to Usecs. - CheckedInt64 timestamp = FramesToUsecs(mEncodedTimestamp, mTrackRate); + CheckedInt64 timestamp = FramesToUsecs(pkt->data.frame.pts, mTrackRate); if (timestamp.isValid()) { - videoData->SetTimeStamp( - (uint64_t)FramesToUsecs(mEncodedTimestamp, mTrackRate).value()); + videoData->SetTimeStamp((uint64_t)timestamp.value()); } CheckedInt64 duration = FramesToUsecs(pkt->data.frame.duration, mTrackRate); if (duration.isValid()) { - videoData->SetDuration( - (uint64_t)FramesToUsecs(pkt->data.frame.duration, mTrackRate).value()); + videoData->SetDuration((uint64_t)duration.value()); } videoData->SwapInFrameData(frameData); VP8LOG("GetEncodedPartitions TimeStamp %lld Duration %lld\n", @@ -218,7 +217,7 @@ VP8TrackEncoder::GetEncodedPartitions(EncodedFrameContainer& aData) aData.AppendEncodedFrame(videoData); } - return NS_OK; + return !!pkt; } static bool isYUV420(const PlanarYCbCrImage::Data *aData) @@ -363,7 +362,7 @@ nsresult VP8TrackEncoder::PrepareRawFrame(VideoChunk &aChunk) return NS_ERROR_FAILURE; } - VP8LOG("Converted an %s frame to I420\n"); + VP8LOG("Converted an %s frame to I420\n", yuvFormat.c_str()); } else { // Not YCbCr at all. Try to get access to the raw data and convert. @@ -629,11 +628,15 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) if (EOS) { VP8LOG("mEndOfStream is true\n"); mEncodingComplete = true; - if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp, - mEncodedFrameDuration, 0, VPX_DL_REALTIME)) { - return NS_ERROR_FAILURE; - } - GetEncodedPartitions(aData); + // Bug 1243611, keep calling vpx_codec_encode and vpx_codec_get_cx_data + // until vpx_codec_get_cx_data return null. + + do { + if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp, + mEncodedFrameDuration, 0, VPX_DL_REALTIME)) { + return NS_ERROR_FAILURE; + } + } while(GetEncodedPartitions(aData)); } return NS_OK ; diff --git a/dom/media/encoder/VP8TrackEncoder.h b/dom/media/encoder/VP8TrackEncoder.h index 3642f5858e..2953cd6d70 100644 --- a/dom/media/encoder/VP8TrackEncoder.h +++ b/dom/media/encoder/VP8TrackEncoder.h @@ -54,7 +54,9 @@ private: StreamTime aProcessedDuration); // Get the encoded data from encoder to aData. - nsresult GetEncodedPartitions(EncodedFrameContainer& aData); + // Return value: false if the vpx_codec_get_cx_data returns null + // for EOS detection. + bool GetEncodedPartitions(EncodedFrameContainer& aData); // Prepare the input data to the mVPXImageWrapper for encoding. nsresult PrepareRawFrame(VideoChunk &aChunk); diff --git a/dom/media/encoder/VorbisTrackEncoder.cpp b/dom/media/encoder/VorbisTrackEncoder.cpp deleted file mode 100644 index bf4a6b84b2..0000000000 --- a/dom/media/encoder/VorbisTrackEncoder.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "VorbisTrackEncoder.h" -#include -#include -#include "WebMWriter.h" -#include "GeckoProfiler.h" - -// One actually used: Encoding using a VBR quality mode. The usable range is -.1 -// (lowest quality, smallest file) to 1. (highest quality, largest file). -// Example quality mode .4: 44kHz stereo coupled, roughly 128kbps VBR -// ret = vorbis_encode_init_vbr(&vi,2,44100,.4); -static const float BASE_QUALITY = 0.4f; - -namespace mozilla { - -#undef LOG -LazyLogModule gVorbisTrackEncoderLog("VorbisTrackEncoder"); -#define VORBISLOG(msg, ...) MOZ_LOG(gVorbisTrackEncoderLog, mozilla::LogLevel::Debug, \ - (msg, ##__VA_ARGS__)) - -VorbisTrackEncoder::VorbisTrackEncoder() - : AudioTrackEncoder() -{ - MOZ_COUNT_CTOR(VorbisTrackEncoder); -} - -VorbisTrackEncoder::~VorbisTrackEncoder() -{ - MOZ_COUNT_DTOR(VorbisTrackEncoder); - if (mInitialized) { - vorbis_block_clear(&mVorbisBlock); - vorbis_dsp_clear(&mVorbisDsp); - vorbis_info_clear(&mVorbisInfo); - } -} - -nsresult -VorbisTrackEncoder::Init(int aChannels, int aSamplingRate) -{ - NS_ENSURE_TRUE(aChannels > 0, NS_ERROR_INVALID_ARG); - NS_ENSURE_TRUE(aChannels <= 8, NS_ERROR_INVALID_ARG); - NS_ENSURE_TRUE(aSamplingRate >= 8000, NS_ERROR_INVALID_ARG); - NS_ENSURE_TRUE(aSamplingRate <= 192000, NS_ERROR_INVALID_ARG); - - // This monitor is used to wake up other methods that are waiting for encoder - // to be completely initialized. - ReentrantMonitorAutoEnter mon(mReentrantMonitor); - mChannels = aChannels; - mSamplingRate = aSamplingRate; - - int ret = 0; - vorbis_info_init(&mVorbisInfo); - double quality = mAudioBitrate ? (double)mAudioBitrate/aSamplingRate : - BASE_QUALITY; - - VORBISLOG("quality %f", quality); - ret = vorbis_encode_init_vbr(&mVorbisInfo, mChannels, mSamplingRate, - quality); - - mInitialized = (ret == 0); - - if (mInitialized) { - // Set up the analysis state and auxiliary encoding storage - vorbis_analysis_init(&mVorbisDsp, &mVorbisInfo); - vorbis_block_init(&mVorbisDsp, &mVorbisBlock); - } - - mon.NotifyAll(); - - return ret == 0 ? NS_OK : NS_ERROR_FAILURE; -} - -void VorbisTrackEncoder::WriteLacing(nsTArray *aOutput, int32_t aLacing) -{ - while (aLacing >= 255) { - aLacing -= 255; - aOutput->AppendElement(255); - } - aOutput->AppendElement((uint8_t)aLacing); -} - -already_AddRefed -VorbisTrackEncoder::GetMetadata() -{ - PROFILER_LABEL("VorbisTrackEncoder", "GetMetadata", - js::ProfileEntry::Category::OTHER); - { - // Wait if encoder is not initialized. - ReentrantMonitorAutoEnter mon(mReentrantMonitor); - while (!mCanceled && !mInitialized) { - mon.Wait(); - } - } - - if (mCanceled || mEncodingComplete) { - return nullptr; - } - - // Vorbis codec specific data - // http://matroska.org/technical/specs/codecid/index.html - RefPtr meta = new VorbisMetadata(); - meta->mBitDepth = 32; // float for desktop - meta->mChannels = mChannels; - meta->mSamplingFrequency = mSamplingRate; - ogg_packet header; - ogg_packet header_comm; - ogg_packet header_code; - // Add comment - vorbis_comment vorbisComment; - vorbis_comment_init(&vorbisComment); - vorbis_comment_add_tag(&vorbisComment, "ENCODER", - NS_LITERAL_CSTRING("Mozilla VorbisTrackEncoder " MOZ_APP_UA_VERSION).get()); - vorbis_analysis_headerout(&mVorbisDsp, &vorbisComment, - &header,&header_comm, &header_code); - vorbis_comment_clear(&vorbisComment); - // number of distinct packets - 1 - meta->mData.AppendElement(2); - // Xiph-style lacing header.bytes, header_comm.bytes - WriteLacing(&(meta->mData), header.bytes); - WriteLacing(&(meta->mData), header_comm.bytes); - - // Append the three packets - meta->mData.AppendElements(header.packet, header.bytes); - meta->mData.AppendElements(header_comm.packet, header_comm.bytes); - meta->mData.AppendElements(header_code.packet, header_code.bytes); - - return meta.forget(); -} - -void -VorbisTrackEncoder::GetEncodedFrames(EncodedFrameContainer& aData) -{ - // vorbis does some data preanalysis, then divvies up blocks for - // more involved (potentially parallel) processing. Get a single - // block for encoding now. - while (vorbis_analysis_blockout(&mVorbisDsp, &mVorbisBlock) == 1) { - ogg_packet oggPacket; - if (vorbis_analysis(&mVorbisBlock, &oggPacket) == 0) { - VORBISLOG("vorbis_analysis_blockout block size %d", oggPacket.bytes); - EncodedFrame* audiodata = new EncodedFrame(); - audiodata->SetFrameType(EncodedFrame::VORBIS_AUDIO_FRAME); - audiodata->SetTimeStamp(oggPacket.granulepos * PR_USEC_PER_SEC - / mSamplingRate); - nsTArray frameData; - frameData.AppendElements(oggPacket.packet, oggPacket.bytes); - audiodata->SwapInFrameData(frameData); - aData.AppendEncodedFrame(audiodata); - } - } -} - -nsresult -VorbisTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData) -{ - if (mEosSetInEncoder) { - return NS_OK; - } - - PROFILER_LABEL("VorbisTrackEncoder", "GetEncodedTrack", - js::ProfileEntry::Category::OTHER); - - nsAutoPtr sourceSegment; - sourceSegment = new AudioSegment(); - { - // Move all the samples from mRawSegment to sourceSegment. We only hold - // the monitor in this block. - ReentrantMonitorAutoEnter mon(mReentrantMonitor); - - // Wait if mEncoder is not initialized, or when not enough raw data, but is - // not the end of stream nor is being canceled. - while (!mCanceled && mRawSegment.GetDuration() < GetPacketDuration() && - !mEndOfStream) { - mon.Wait(); - } - VORBISLOG("GetEncodedTrack passes wait, duration is %lld\n", - mRawSegment.GetDuration()); - if (mCanceled || mEncodingComplete) { - return NS_ERROR_FAILURE; - } - - sourceSegment->AppendFrom(&mRawSegment); - } - - if (mEndOfStream && (sourceSegment->GetDuration() == 0) - && !mEosSetInEncoder) { - mEncodingComplete = true; - mEosSetInEncoder = true; - VORBISLOG("[Vorbis] Done encoding."); - vorbis_analysis_wrote(&mVorbisDsp, 0); - GetEncodedFrames(aData); - - return NS_OK; - } - - // Start encoding data. - AudioSegment::ChunkIterator iter(*sourceSegment); - - AudioDataValue **vorbisBuffer = - vorbis_analysis_buffer(&mVorbisDsp, (int)sourceSegment->GetDuration()); - - int framesCopied = 0; - AutoTArray interleavedPcm; - AutoTArray nonInterleavedPcm; - interleavedPcm.SetLength(sourceSegment->GetDuration() * mChannels); - nonInterleavedPcm.SetLength(sourceSegment->GetDuration() * mChannels); - while (!iter.IsEnded()) { - AudioChunk chunk = *iter; - int frameToCopy = chunk.GetDuration(); - if (!chunk.IsNull()) { - InterleaveTrackData(chunk, frameToCopy, mChannels, - interleavedPcm.Elements() + framesCopied * mChannels); - } else { // empty data - memset(interleavedPcm.Elements() + framesCopied * mChannels, 0, - frameToCopy * mChannels * sizeof(AudioDataValue)); - } - framesCopied += frameToCopy; - iter.Next(); - } - // De-interleave the interleavedPcm. - DeInterleaveTrackData(interleavedPcm.Elements(), framesCopied, mChannels, - nonInterleavedPcm.Elements()); - // Copy the nonInterleavedPcm to vorbis buffer. - for(uint8_t i = 0; i < mChannels; ++i) { - memcpy(vorbisBuffer[i], nonInterleavedPcm.Elements() + framesCopied * i, - framesCopied * sizeof(AudioDataValue)); - } - - // Now the vorbisBuffer contain the all data in non-interleaved. - // Tell the library how much we actually submitted. - vorbis_analysis_wrote(&mVorbisDsp, framesCopied); - VORBISLOG("vorbis_analysis_wrote framesCopied %d\n", framesCopied); - GetEncodedFrames(aData); - - return NS_OK; -} - -} // namespace mozilla diff --git a/dom/media/encoder/VorbisTrackEncoder.h b/dom/media/encoder/VorbisTrackEncoder.h deleted file mode 100644 index c15355edc0..0000000000 --- a/dom/media/encoder/VorbisTrackEncoder.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef VorbisTrackEncoder_h_ -#define VorbisTrackEncoder_h_ - -#include "TrackEncoder.h" -#include "nsCOMPtr.h" -#include - -namespace mozilla { - -class VorbisTrackEncoder : public AudioTrackEncoder -{ -public: - VorbisTrackEncoder(); - virtual ~VorbisTrackEncoder(); - - already_AddRefed GetMetadata() final override; - - nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override; - -protected: - /** - * http://xiph.org/vorbis/doc/libvorbis/vorbis_analysis_buffer.html - * We use 1024 samples for the write buffer; libvorbis will construct packets - * with the appropriate duration for the encoding mode internally. - */ - int GetPacketDuration() final override { - return 1024; - } - - nsresult Init(int aChannels, int aSamplingRate) final override; - -private: - // Write Xiph-style lacing to aOutput. - void WriteLacing(nsTArray *aOutput, int32_t aLacing); - - // Get the encoded data from vorbis encoder and append into aData. - void GetEncodedFrames(EncodedFrameContainer& aData); - - // vorbis codec members - // Struct that stores all the static vorbis bitstream settings. - vorbis_info mVorbisInfo; - // Central working state for the PCM->packet encoder. - vorbis_dsp_state mVorbisDsp; - // Local working space for PCM->packet encode. - vorbis_block mVorbisBlock; -}; - -} // namespace mozilla - -#endif diff --git a/dom/media/encoder/moz.build b/dom/media/encoder/moz.build index 853b0d6d3b..e504161a71 100644 --- a/dom/media/encoder/moz.build +++ b/dom/media/encoder/moz.build @@ -30,11 +30,9 @@ if CONFIG['MOZ_OMX_ENCODER']: UNIFIED_SOURCES += ['OmxTrackEncoder.cpp'] if CONFIG['MOZ_WEBM_ENCODER']: - EXPORTS += ['VorbisTrackEncoder.h', - 'VP8TrackEncoder.h', + EXPORTS += ['VP8TrackEncoder.h', ] - UNIFIED_SOURCES += ['VorbisTrackEncoder.cpp', - 'VP8TrackEncoder.cpp', + UNIFIED_SOURCES += ['VP8TrackEncoder.cpp', ] LOCAL_INCLUDES += ['/media/libyuv/include'] diff --git a/dom/media/fmp4/MP4Demuxer.cpp b/dom/media/fmp4/MP4Demuxer.cpp index 14a7ff1e03..7173634ec6 100644 --- a/dom/media/fmp4/MP4Demuxer.cpp +++ b/dom/media/fmp4/MP4Demuxer.cpp @@ -302,6 +302,9 @@ MP4TrackDemuxer::GetSamples(int32_t aNumSamples) } RefPtr sample; while (aNumSamples && (sample = mIterator->GetNext())) { + if (!sample->Size()) { + continue; + } samples->mSamples.AppendElement(sample); aNumSamples--; } diff --git a/dom/media/gmp-plugin/gmp-test-output-protection.h b/dom/media/gmp-plugin/gmp-test-output-protection.h index 4a56f6bd71..57855ca71f 100644 --- a/dom/media/gmp-plugin/gmp-test-output-protection.h +++ b/dom/media/gmp-plugin/gmp-test-output-protection.h @@ -35,13 +35,6 @@ static BOOL CALLBACK EnumDisplayMonitorsCallback(HMONITOR hMonitor, HDC hdc, failureMsgs->push_back("FAIL GetMonitorInfoA call failed"); } - DISPLAY_DEVICEA dd; - ZeroMemory(&dd, sizeof(dd)); - dd.cb = sizeof(dd); - if (!EnumDisplayDevicesA(miex.szDevice, 0, &dd, 1)) { - failureMsgs->push_back("FAIL EnumDisplayDevicesA call failed"); - } - ULONG numVideoOutputs = 0; IOPMVideoOutput** opmVideoOutputArray = nullptr; HRESULT hr = sOPMGetVideoOutputsFromHMONITORProc(hMonitor, @@ -49,7 +42,7 @@ static BOOL CALLBACK EnumDisplayMonitorsCallback(HMONITOR hMonitor, HDC hdc, &numVideoOutputs, &opmVideoOutputArray); if (S_OK != hr) { - if (0x8007001f != hr && 0x80070032 != hr) { + if (0x8007001f != hr && 0x80070032 != hr && 0xc02625e5 != hr) { char msg[100]; sprintf(msg, "FAIL OPMGetVideoOutputsFromHMONITOR call failed: HRESULT=0x%08x", hr); failureMsgs->push_back(msg); @@ -57,6 +50,13 @@ static BOOL CALLBACK EnumDisplayMonitorsCallback(HMONITOR hMonitor, HDC hdc, return true; } + DISPLAY_DEVICEA dd; + ZeroMemory(&dd, sizeof(dd)); + dd.cb = sizeof(dd); + if (!EnumDisplayDevicesA(miex.szDevice, 0, &dd, 1)) { + failureMsgs->push_back("FAIL EnumDisplayDevicesA call failed"); + } + for (ULONG i = 0; i < numVideoOutputs; ++i) { OPM_RANDOM_NUMBER opmRandomNumber; BYTE* certificate = nullptr; diff --git a/dom/media/gmp/GMPServiceParent.cpp b/dom/media/gmp/GMPServiceParent.cpp index 5a4020c3a6..183e394214 100644 --- a/dom/media/gmp/GMPServiceParent.cpp +++ b/dom/media/gmp/GMPServiceParent.cpp @@ -696,30 +696,34 @@ GeckoMediaPluginServiceParent::UnloadPlugins() NS_LITERAL_CSTRING("Starting to unload plugins")); #endif + nsTArray> plugins; { MutexAutoLock lock(mMutex); - LOGD(("%s::%s plugins:%u including async:%u", __CLASS__, __FUNCTION__, - mPlugins.Length(), mAsyncShutdownPlugins.Length())); + // Move all plugins references to a local array. This way mMutex won't be + // locked when calling CloseActive (to avoid inter-locking). + plugins = Move(mPlugins); + } + + LOGD(("%s::%s plugins:%u including async:%u", __CLASS__, __FUNCTION__, + plugins.Length(), mAsyncShutdownPlugins.Length())); #ifdef DEBUG - for (const auto& plugin : mPlugins) { - LOGD(("%s::%s plugin: '%s'", __CLASS__, __FUNCTION__, - plugin->GetDisplayName().get())); - } - for (const auto& plugin : mAsyncShutdownPlugins) { - LOGD(("%s::%s async plugin: '%s'", __CLASS__, __FUNCTION__, - plugin->GetDisplayName().get())); - } + for (const auto& plugin : plugins) { + LOGD(("%s::%s plugin: '%s'", __CLASS__, __FUNCTION__, + plugin->GetDisplayName().get())); + } + for (const auto& plugin : mAsyncShutdownPlugins) { + LOGD(("%s::%s async plugin: '%s'", __CLASS__, __FUNCTION__, + plugin->GetDisplayName().get())); + } #endif - // Note: CloseActive may be async; it could actually finish - // shutting down when all the plugins have unloaded. - for (size_t i = 0; i < mPlugins.Length(); i++) { + // Note: CloseActive may be async; it could actually finish + // shutting down when all the plugins have unloaded. + for (const auto& plugin : plugins) { #ifdef MOZ_CRASHREPORTER - SetAsyncShutdownPluginState(mPlugins[i], 'S', - NS_LITERAL_CSTRING("CloseActive")); + SetAsyncShutdownPluginState(plugin, 'S', + NS_LITERAL_CSTRING("CloseActive")); #endif - mPlugins[i]->CloseActive(true); - } - mPlugins.Clear(); + plugin->CloseActive(true); } #ifdef MOZ_CRASHREPORTER @@ -1033,8 +1037,6 @@ CreateGMPParent() return new GMPParent(); } - - GMPParent* GeckoMediaPluginServiceParent::ClonePlugin(const GMPParent* aOriginal) { @@ -1145,6 +1147,13 @@ GeckoMediaPluginServiceParent::RemoveOnGMPThread(const nsAString& aDirectory, } if (aDeleteFromDisk && !inUse) { + // Ensure the GMP dir and all files in it are writable, so we have + // permission to delete them. + directory->SetPermissions(0700); + DirectoryEnumerator iter(directory, DirectoryEnumerator::FilesAndDirs); + for (nsCOMPtr dirEntry; (dirEntry = iter.Next()) != nullptr;) { + dirEntry->SetPermissions(0700); + } if (NS_SUCCEEDED(directory->Remove(true))) { mPluginsWaitingForDeletion.RemoveElement(aDirectory); NS_DispatchToMainThread(new NotifyObserversTask("gmp-directory-deleted", diff --git a/dom/media/gmp/GMPStorageParent.cpp b/dom/media/gmp/GMPStorageParent.cpp index 58ac58953e..5a83b5adb8 100644 --- a/dom/media/gmp/GMPStorageParent.cpp +++ b/dom/media/gmp/GMPStorageParent.cpp @@ -568,7 +568,7 @@ GMPStorageParent::GMPStorageParent(const nsCString& aNodeId, GMPParent* aPlugin) : mNodeId(aNodeId) , mPlugin(aPlugin) - , mShutdown(false) + , mShutdown(true) { } @@ -602,6 +602,7 @@ GMPStorageParent::Init() mStorage = MakeUnique(); } + mShutdown = false; return NS_OK; } diff --git a/dom/media/gmp/GMPStorageParent.h b/dom/media/gmp/GMPStorageParent.h index 8789f68fb5..60cbea9d8c 100644 --- a/dom/media/gmp/GMPStorageParent.h +++ b/dom/media/gmp/GMPStorageParent.h @@ -54,6 +54,7 @@ private: const nsCString mNodeId; RefPtr mPlugin; + // True after Shutdown(), or if Init() has not completed successfully. bool mShutdown; }; diff --git a/dom/media/gtest/MockMediaResource.cpp b/dom/media/gtest/MockMediaResource.cpp index 096efcc3bd..4e75f8d9d3 100644 --- a/dom/media/gtest/MockMediaResource.cpp +++ b/dom/media/gtest/MockMediaResource.cpp @@ -64,7 +64,6 @@ MockMediaResource::GetLength() if (mFileHandle == nullptr) { return -1; } - fseek(mFileHandle, 0, SEEK_END); return ftell(mFileHandle); } diff --git a/dom/media/gtest/TestGMPCrossOrigin.cpp b/dom/media/gtest/TestGMPCrossOrigin.cpp index 8fc36728eb..326232523e 100644 --- a/dom/media/gtest/TestGMPCrossOrigin.cpp +++ b/dom/media/gtest/TestGMPCrossOrigin.cpp @@ -541,8 +541,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback EXPECT_TRUE(IsGMPStorageIsEmpty()); - const nsString origin1 = NS_LITERAL_STRING("example1.com"); - const nsString origin2 = NS_LITERAL_STRING("example2.org"); + const nsString origin1 = NS_LITERAL_STRING("http://example1.com"); + const nsString origin2 = NS_LITERAL_STRING("http://example2.org"); nsCString PBnodeId1 = GetNodeId(origin1, origin2, true); nsCString PBnodeId2 = GetNodeId(origin1, origin2, true); @@ -578,8 +578,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback // Once we clear storage, the node ids generated for the same origin-pair // should be different. - const nsString origin1 = NS_LITERAL_STRING("example1.com"); - const nsString origin2 = NS_LITERAL_STRING("example2.org"); + const nsString origin1 = NS_LITERAL_STRING("http://example1.com"); + const nsString origin2 = NS_LITERAL_STRING("http://example2.org"); nsCString nodeId3 = GetNodeId(origin1, origin2, false); EXPECT_TRUE(!aNodeId1.Equals(nodeId3)); @@ -683,8 +683,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(NS_LITERAL_CSTRING("test-storage complete"), NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("example1.com"), - NS_LITERAL_STRING("example2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example1.com"), + NS_LITERAL_STRING("http://example2.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -704,8 +704,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback this, &GMPStorageTest::TestForgetThisSite_AnotherSite); Expect(NS_LITERAL_CSTRING("test-storage complete"), r); - CreateDecryptor(NS_LITERAL_STRING("example1.com"), - NS_LITERAL_STRING("example2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example1.com"), + NS_LITERAL_STRING("http://example2.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -718,8 +718,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback this, &GMPStorageTest::TestForgetThisSite_CollectSiteInfo); Expect(NS_LITERAL_CSTRING("test-storage complete"), r); - CreateDecryptor(NS_LITERAL_STRING("example3.com"), - NS_LITERAL_STRING("example4.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example3.com"), + NS_LITERAL_STRING("http://example4.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -747,7 +747,7 @@ class GMPStorageTest : public GMPDecryptorProxyCallback void TestForgetThisSite_CollectSiteInfo() { nsAutoPtr siteInfo( - new NodeInfo(NS_LITERAL_CSTRING("example1.com"))); + new NodeInfo(NS_LITERAL_CSTRING("http://example1.com"))); // Collect nodeIds that are expected to remain for later comparison. EnumerateGMPStorageDir(NS_LITERAL_CSTRING("id"), NodeIdCollector(siteInfo)); // Invoke "Forget this site" on the main thread. @@ -837,8 +837,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback this, &GMPStorageTest::TestClearRecentHistory1_Clear); Expect(NS_LITERAL_CSTRING("test-storage complete"), r); - CreateDecryptor(NS_LITERAL_STRING("example1.com"), - NS_LITERAL_STRING("example2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example1.com"), + NS_LITERAL_STRING("http://example2.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -859,8 +859,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback this, &GMPStorageTest::TestClearRecentHistory2_Clear); Expect(NS_LITERAL_CSTRING("test-storage complete"), r); - CreateDecryptor(NS_LITERAL_STRING("example1.com"), - NS_LITERAL_STRING("example2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example1.com"), + NS_LITERAL_STRING("http://example2.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -881,8 +881,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback this, &GMPStorageTest::TestClearRecentHistory3_Clear); Expect(NS_LITERAL_CSTRING("test-storage complete"), r); - CreateDecryptor(NS_LITERAL_STRING("example1.com"), - NS_LITERAL_STRING("example2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example1.com"), + NS_LITERAL_STRING("http://example2.com"), false, NS_LITERAL_CSTRING("test-storage")); } @@ -995,8 +995,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback // Open decryptor on one, origin, write a record, and test that that // record can't be read on another origin. - CreateDecryptor(NS_LITERAL_STRING("example3.com"), - NS_LITERAL_STRING("example4.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example3.com"), + NS_LITERAL_STRING("http://example4.com"), false, update); } @@ -1009,8 +1009,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(NS_LITERAL_CSTRING("retrieve crossOriginTestRecordId succeeded (length 0 bytes)"), NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("example5.com"), - NS_LITERAL_STRING("example6.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example5.com"), + NS_LITERAL_STRING("http://example6.com"), false, NS_LITERAL_CSTRING("retrieve crossOriginTestRecordId")); } @@ -1026,8 +1026,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback // open another, and test that record can be read, close decryptor, // then send pb-last-context-closed notification, then open decryptor // and check that it can't read that data; it should have been purged. - CreateDecryptor(NS_LITERAL_STRING("pb1.com"), - NS_LITERAL_STRING("pb2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://pb1.com"), + NS_LITERAL_STRING("http://pb2.com"), true, NS_LITERAL_CSTRING("store pbdata test-pb-data")); } @@ -1039,8 +1039,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback NS_NewRunnableMethod(this, &GMPStorageTest::TestPBStorage_RecordRetrievedContinuation)); - CreateDecryptor(NS_LITERAL_STRING("pb1.com"), - NS_LITERAL_STRING("pb2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://pb1.com"), + NS_LITERAL_STRING("http://pb2.com"), true, NS_LITERAL_CSTRING("retrieve pbdata")); } @@ -1053,8 +1053,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("pb1.com"), - NS_LITERAL_STRING("pb2.com"), + CreateDecryptor(NS_LITERAL_STRING("http://pb1.com"), + NS_LITERAL_STRING("http://pb2.com"), true, NS_LITERAL_CSTRING("retrieve pbdata")); } @@ -1084,20 +1084,20 @@ class GMPStorageTest : public GMPDecryptorProxyCallback void TestAsyncShutdownTimeout() { // Create decryptors that timeout in their async shutdown. // If the gtest hangs on shutdown, test fails! - CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("example7.com"), - NS_LITERAL_STRING("example8.com"), + CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("http://example7.com"), + NS_LITERAL_STRING("http://example8.com"), &GMPStorageTest::TestAsyncShutdownTimeout2); }; void TestAsyncShutdownTimeout2() { - CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("example9.com"), - NS_LITERAL_STRING("example10.com"), + CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("http://example9.com"), + NS_LITERAL_STRING("http://example10.com"), &GMPStorageTest::TestAsyncShutdownTimeout3); }; void TestAsyncShutdownTimeout3() { - CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("example11.com"), - NS_LITERAL_STRING("example12.com"), + CreateAsyncShutdownTimeoutGMP(NS_LITERAL_STRING("http://example11.com"), + NS_LITERAL_STRING("http://example12.com"), &GMPStorageTest::SetFinished); }; @@ -1120,8 +1120,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback // Test that a GMP can write to storage during shutdown, and retrieve // that written data in a subsequent session. - CreateDecryptor(NS_LITERAL_STRING("example13.com"), - NS_LITERAL_STRING("example14.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example13.com"), + NS_LITERAL_STRING("http://example14.com"), false, update); } @@ -1139,8 +1139,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(response, NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("example13.com"), - NS_LITERAL_STRING("example14.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example13.com"), + NS_LITERAL_STRING("http://example14.com"), false, NS_LITERAL_CSTRING("retrieve-shutdown-token")); } @@ -1152,8 +1152,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(NS_LITERAL_CSTRING("OP tests completed"), NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("example15.com"), - NS_LITERAL_STRING("example16.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example15.com"), + NS_LITERAL_STRING("http://example16.com"), false, NS_LITERAL_CSTRING("test-op-apis")); } @@ -1163,8 +1163,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(NS_LITERAL_CSTRING("retrieved plugin-voucher: gmp-fake placeholder voucher"), NS_NewRunnableMethod(this, &GMPStorageTest::SetFinished)); - CreateDecryptor(NS_LITERAL_STRING("example17.com"), - NS_LITERAL_STRING("example18.com"), + CreateDecryptor(NS_LITERAL_STRING("http://example17.com"), + NS_LITERAL_STRING("http://example18.com"), false, NS_LITERAL_CSTRING("retrieve-plugin-voucher")); } @@ -1213,8 +1213,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback Expect(response, continuation); } - CreateDecryptor(NS_LITERAL_STRING("foo.com"), - NS_LITERAL_STRING("bar.com"), + CreateDecryptor(NS_LITERAL_STRING("http://foo.com"), + NS_LITERAL_STRING("http://bar.com"), aPrivateBrowsing, Move(updates)); } @@ -1266,8 +1266,8 @@ class GMPStorageTest : public GMPDecryptorProxyCallback update.Append(longRecordName); update.AppendLiteral(" "); update.Append(data); - CreateDecryptor(NS_LITERAL_STRING("fuz.com"), - NS_LITERAL_STRING("baz.com"), + CreateDecryptor(NS_LITERAL_STRING("http://fuz.com"), + NS_LITERAL_STRING("http://baz.com"), false, update); } diff --git a/dom/media/gtest/TestMP4Demuxer.cpp b/dom/media/gtest/TestMP4Demuxer.cpp index 193c6d4d8d..9d65f253e0 100644 --- a/dom/media/gtest/TestMP4Demuxer.cpp +++ b/dom/media/gtest/TestMP4Demuxer.cpp @@ -19,7 +19,7 @@ using namespace mp4_demuxer; class AutoTaskQueue; -#define DO_FAIL []()->void { EXPECT_TRUE(false); } +#define DO_FAIL [binding]()->void { EXPECT_TRUE(false); binding->mTaskQueue->BeginShutdown(); } class MP4DemuxerBinding { @@ -50,6 +50,7 @@ public: void RunTestAndWait(const Function& aFunction) { Function func(aFunction); + RefPtr binding = this; mDemuxer->Init()->Then(mTaskQueue, __func__, Move(func), DO_FAIL); mTaskQueue->AwaitShutdownAndIdle(); } @@ -60,7 +61,7 @@ public: MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); RefPtr track = aTrackDemuxer; - RefPtr self = this; + RefPtr binding = this; int64_t time = -1; while (mIndex < mSamples.Length()) { @@ -80,13 +81,13 @@ public: DispatchTask( - [track, time, self] () { - track->Seek(media::TimeUnit::FromMicroseconds(time))->Then(self->mTaskQueue, __func__, - [track, time, self] () { - track->GetSamples()->Then(self->mTaskQueue, __func__, - [track, time, self] (RefPtr aSamples) { + [track, time, binding] () { + track->Seek(media::TimeUnit::FromMicroseconds(time))->Then(binding->mTaskQueue, __func__, + [track, time, binding] () { + track->GetSamples()->Then(binding->mTaskQueue, __func__, + [track, time, binding] (RefPtr aSamples) { EXPECT_EQ(time, aSamples->mSamples[0]->mTime); - self->CheckTrackKeyFrame(track); + binding->CheckTrackKeyFrame(track); }, DO_FAIL ); @@ -105,32 +106,32 @@ public: MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); RefPtr track = aTrackDemuxer; - RefPtr self = this; + RefPtr binding = this; RefPtr p = mCheckTrackSamples.Ensure(__func__); DispatchTask( - [track, self] () { - track->GetSamples()->Then(self->mTaskQueue, __func__, - [track, self] (RefPtr aSamples) { + [track, binding] () { + track->GetSamples()->Then(binding->mTaskQueue, __func__, + [track, binding] (RefPtr aSamples) { if (aSamples->mSamples.Length()) { - self->mSamples.AppendElements(aSamples->mSamples); - self->CheckTrackSamples(track); + binding->mSamples.AppendElements(aSamples->mSamples); + binding->CheckTrackSamples(track); } }, - [self] (DemuxerFailureReason aReason) { + [binding] (DemuxerFailureReason aReason) { if (aReason == DemuxerFailureReason::DEMUXER_ERROR) { EXPECT_TRUE(false); - self->mCheckTrackSamples.Reject(NS_ERROR_FAILURE, __func__); + binding->mCheckTrackSamples.Reject(NS_ERROR_FAILURE, __func__); } else if (aReason == DemuxerFailureReason::END_OF_STREAM) { - EXPECT_TRUE(self->mSamples.Length() > 1); - for (uint32_t i = 0; i < (self->mSamples.Length() - 1); i++) { - EXPECT_LT(self->mSamples[i]->mTimecode, self->mSamples[i + 1]->mTimecode); - if (self->mSamples[i]->mKeyframe) { - self->mKeyFrameTimecodes.AppendElement(self->mSamples[i]->mTimecode); + EXPECT_TRUE(binding->mSamples.Length() > 1); + for (uint32_t i = 0; i < (binding->mSamples.Length() - 1); i++) { + EXPECT_LT(binding->mSamples[i]->mTimecode, binding->mSamples[i + 1]->mTimecode); + if (binding->mSamples[i]->mKeyframe) { + binding->mKeyFrameTimecodes.AppendElement(binding->mSamples[i]->mTimecode); } } - self->mCheckTrackSamples.Resolve(true, __func__); + binding->mCheckTrackSamples.Resolve(true, __func__); } } ); @@ -199,7 +200,7 @@ ToCryptoString(const CryptoSample& aCrypto) #ifndef XP_WIN // VC2013 doesn't support C++11 array initialization. -TEST(MP4Demuxer, CENCFrag) +TEST(MP4Demuxer, CENCFragVideo) { const char* video[] = { "1 16 7e571d037e571d037e571d037e571d03 00000000000000000000000000000000 5,684 5,16980", @@ -280,7 +281,10 @@ TEST(MP4Demuxer, CENCFrag) binding->mTaskQueue->BeginShutdown(); }, DO_FAIL); }); +} +TEST(MP4Demuxer, CENCFragAudio) +{ const char* audio[] = { "1 16 7e571d047e571d047e571d047e571d04 00000000000000000000000000000000 0,281", "1 16 7e571d047e571d047e571d047e571d04 00000000000000000000000000000012 0,257", @@ -377,21 +381,22 @@ TEST(MP4Demuxer, CENCFrag) "1 16 7e571d047e571d047e571d047e571d04 000000000000000000000000000008cd 0,433", "1 16 7e571d047e571d047e571d047e571d04 000000000000000000000000000008e9 0,481", }; - RefPtr audiobinding = new MP4DemuxerBinding("gizmo-frag.mp4"); - audiobinding->RunTestAndWait([audiobinding, audio] () { + RefPtr binding = new MP4DemuxerBinding("gizmo-frag.mp4"); + + binding->RunTestAndWait([binding, audio] () { // grab all audio samples. - audiobinding->mAudioTrack = audiobinding->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); - audiobinding->CheckTrackSamples(audiobinding->mAudioTrack) - ->Then(audiobinding->mTaskQueue, __func__, - [audiobinding, audio] () { - EXPECT_TRUE(audiobinding->mSamples.Length() > 1); - for (uint32_t i = 0; i < audiobinding->mSamples.Length(); i++) { - nsCString text = ToCryptoString(audiobinding->mSamples[i]->mCrypto); + binding->mAudioTrack = binding->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); + binding->CheckTrackSamples(binding->mAudioTrack) + ->Then(binding->mTaskQueue, __func__, + [binding, audio] () { + EXPECT_TRUE(binding->mSamples.Length() > 1); + for (uint32_t i = 0; i < binding->mSamples.Length(); i++) { + nsCString text = ToCryptoString(binding->mSamples[i]->mCrypto); EXPECT_STREQ(audio[i++], text.get()); } - EXPECT_EQ(ArrayLength(audio), audiobinding->mSamples.Length()); - audiobinding->mTaskQueue->BeginShutdown(); + EXPECT_EQ(ArrayLength(audio), binding->mSamples.Length()); + binding->mTaskQueue->BeginShutdown(); }, DO_FAIL); }); } @@ -426,3 +431,24 @@ TEST(MP4Demuxer, GetNextKeyframe) ); }); } + +TEST(MP4Demuxer, ZeroInLastMoov) +{ + RefPtr binding = new MP4DemuxerBinding("short-zero-in-moov.mp4"); + binding->RunTestAndWait([binding] () { + // It demuxes without error. That is sufficient. + binding->mTaskQueue->BeginShutdown(); + }); +} + + +TEST(MP4Demuxer, ZeroInMoovQuickTime) +{ + RefPtr binding = new MP4DemuxerBinding("short-zero-inband.mov"); + binding->RunTestAndWait([binding] () { + // It demuxes without error. That is sufficient. + binding->mTaskQueue->BeginShutdown(); + }); +} + +#undef DO_FAIL diff --git a/dom/media/gtest/TestVPXDecoding.cpp b/dom/media/gtest/TestVPXDecoding.cpp new file mode 100644 index 0000000000..d14c4d24e9 --- /dev/null +++ b/dom/media/gtest/TestVPXDecoding.cpp @@ -0,0 +1,103 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public +* License, v. 2.0. If a copy of the MPL was not distributed with this +* file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/ArrayUtils.h" +#include "nsTArray.h" +#include "VPXDecoder.h" + +#include + +using namespace mozilla; + +static void +ReadVPXFile(const char* aPath, nsTArray& aBuffer) +{ + FILE* f = fopen(aPath, "rb"); + ASSERT_NE(f, (FILE *) nullptr); + + int r = fseek(f, 0, SEEK_END); + ASSERT_EQ(r, 0); + + long size = ftell(f); + ASSERT_NE(size, -1); + aBuffer.SetLength(size); + + r = fseek(f, 0, SEEK_SET); + ASSERT_EQ(r, 0); + + size_t got = fread(aBuffer.Elements(), 1, size, f); + ASSERT_EQ(got, size_t(size)); + + r = fclose(f); + ASSERT_EQ(r, 0); +} + +static +vpx_codec_iface_t* +ParseIVFConfig(nsTArray& data, vpx_codec_dec_cfg_t& config) +{ + if (data.Length() < 32 + 12) { + // Not enough data for file & first frame headers. + return nullptr; + } + if (data[0] != 'D' || data[1] != 'K' || data[2] != 'I' || data[3] != 'F') { + // Expect 'DKIP' + return nullptr; + } + if (data[4] != 0 || data[5] != 0) { + // Expect version==0. + return nullptr; + } + if (data[8] != 'V' || data[9] != 'P' + || (data[10] != '8' && data[10] != '9') + || data[11] != '0') { + // Expect 'VP80' or 'VP90'. + return nullptr; + } + config.w = uint32_t(data[12]) || (uint32_t(data[13]) << 8); + config.h = uint32_t(data[14]) || (uint32_t(data[15]) << 8); + vpx_codec_iface_t* codec = (data[10] == '8') + ? vpx_codec_vp8_dx() + : vpx_codec_vp9_dx(); + // Remove headers, to just leave raw VPx data to be decoded. + data.RemoveElementsAt(0, 32 + 12); + return codec; +} + +struct TestFileData { + const char* mFilename; + vpx_codec_err_t mDecodeResult; +}; +static const TestFileData testFiles[] = { + { "test_case_1224361.vp8.ivf", VPX_CODEC_OK }, + { "test_case_1224363.vp8.ivf", VPX_CODEC_CORRUPT_FRAME }, + { "test_case_1224369.vp8.ivf", VPX_CODEC_CORRUPT_FRAME } +}; + +TEST(libvpx, test_cases) +{ + for (size_t test = 0; test < ArrayLength(testFiles); ++test) { + nsTArray data; + ReadVPXFile(testFiles[test].mFilename, data); + ASSERT_GT(data.Length(), 0u); + + vpx_codec_dec_cfg_t config; + vpx_codec_iface_t* dx = ParseIVFConfig(data, config); + ASSERT_TRUE(dx); + config.threads = 2; + + vpx_codec_ctx_t ctx; + PodZero(&ctx); + vpx_codec_err_t r = vpx_codec_dec_init(&ctx, dx, &config, 0); + ASSERT_EQ(VPX_CODEC_OK, r); + + r = vpx_codec_decode(&ctx, data.Elements(), data.Length(), nullptr, 0); + // This test case is known to be corrupt. + EXPECT_EQ(testFiles[test].mDecodeResult, r); + + r = vpx_codec_destroy(&ctx); + EXPECT_EQ(VPX_CODEC_OK, r); + } +} diff --git a/dom/media/gtest/TestVorbisTrackEncoder.cpp b/dom/media/gtest/TestVorbisTrackEncoder.cpp deleted file mode 100644 index 9166afcf5b..0000000000 --- a/dom/media/gtest/TestVorbisTrackEncoder.cpp +++ /dev/null @@ -1,228 +0,0 @@ -/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "gtest/gtest.h" -#include "VorbisTrackEncoder.h" -#include "WebMWriter.h" -#include "MediaStreamGraph.h" - -using namespace mozilla; - -class TestVorbisTrackEncoder : public VorbisTrackEncoder -{ -public: - // Return true if it has successfully initialized the vorbis encoder. - bool TestVorbisCreation(int aChannels, int aSamplingRate) - { - if (Init(aChannels, aSamplingRate) == NS_OK) { - return true; - } - return false; - } -}; - -static bool -TestVorbisInit(int aChannels, int aSamplingRate) -{ - TestVorbisTrackEncoder encoder; - return encoder.TestVorbisCreation(aChannels, aSamplingRate); -} - -static int -ReadLacing(const uint8_t* aInput, uint32_t aInputLength, uint32_t& aReadBytes) -{ - aReadBytes = 0; - - int packetSize = 0; - while (aReadBytes < aInputLength) { - if (aInput[aReadBytes] == 255) { - packetSize += 255; - aReadBytes++; - } else { // the last byte - packetSize += aInput[aReadBytes]; - aReadBytes++; - break; - } - } - - return packetSize; -} - -static bool -parseVorbisMetadata(nsTArray& aData, int aChannels, int aRate) -{ - uint32_t offset = 0; - // the first byte should be 2. - if (aData.ElementAt(0) != 2) { - return false; - } - offset = 1; - - // Read the length of header and header_comm - uint32_t readbytes; - ogg_packet header; - ogg_packet header_comm; - ogg_packet header_code; - memset(&header, 0, sizeof(ogg_packet)); - memset(&header_comm, 0, sizeof(ogg_packet)); - memset(&header_code, 0, sizeof(ogg_packet)); - - int header_length; - int header_comm_length; - int header_code_length; - EXPECT_TRUE(offset < aData.Length()); - header_length = ReadLacing(aData.Elements()+offset, aData.Length()-offset, - readbytes); - offset += readbytes; - EXPECT_TRUE(offset < aData.Length()); - header_comm_length = ReadLacing(aData.Elements()+offset, - aData.Length()-offset, readbytes); - offset += readbytes; - EXPECT_TRUE(offset < aData.Length()); - // The rest length is header_code. - header_code_length = aData.Length() - offset - header_length - - header_comm_length; - EXPECT_TRUE(header_code_length >= 32); - - // Verify the three header packets by vorbis_synthesis_headerin. - // Raise the b_o_s (begin of stream) flag. - header.b_o_s = true; - header.packet = aData.Elements() + offset; - header.bytes = header_length; - offset += header_length; - header_comm.packet = aData.Elements() + offset; - header_comm.bytes = header_comm_length; - offset += header_comm_length; - header_code.packet = aData.Elements() + offset; - header_code.bytes = header_code_length; - - vorbis_info vi; - vorbis_comment vc; - vorbis_info_init(&vi); - vorbis_comment_init(&vc); - - EXPECT_TRUE(0 == vorbis_synthesis_headerin(&vi, &vc, &header)); - - EXPECT_TRUE(0 == vorbis_synthesis_headerin(&vi, &vc, &header_comm)); - - EXPECT_TRUE(0 == vorbis_synthesis_headerin(&vi, &vc, &header_code)); - - EXPECT_TRUE(vi.channels == aChannels); - EXPECT_TRUE(vi.rate == aRate); - - vorbis_info_clear(&vi); - vorbis_comment_clear(&vc); - - return true; -} - -// Test init function -TEST(VorbisTrackEncoder, Init) -{ - // Channel number range test - // Expect false with 0 or negative channels of input signal. - EXPECT_FALSE(TestVorbisInit(0, 16000)); - EXPECT_FALSE(TestVorbisInit(-1, 16000)); - EXPECT_FALSE(TestVorbisInit(8 + 1, 16000)); - - // Sample rate and channel range test. - for (int i = 1; i <= 8; i++) { - EXPECT_FALSE(TestVorbisInit(i, -1)); - EXPECT_FALSE(TestVorbisInit(i, 2000)); - EXPECT_FALSE(TestVorbisInit(i, 4000)); - EXPECT_FALSE(TestVorbisInit(i, 7999)); - EXPECT_TRUE(TestVorbisInit(i, 8000)); - EXPECT_TRUE(TestVorbisInit(i, 11000)); - EXPECT_TRUE(TestVorbisInit(i, 16000)); - EXPECT_TRUE(TestVorbisInit(i, 22050)); - EXPECT_TRUE(TestVorbisInit(i, 32000)); - EXPECT_TRUE(TestVorbisInit(i, 44100)); - EXPECT_TRUE(TestVorbisInit(i, 48000)); - EXPECT_TRUE(TestVorbisInit(i, 96000)); - EXPECT_TRUE(TestVorbisInit(i, 192000)); - EXPECT_FALSE(TestVorbisInit(i, 192001)); - EXPECT_FALSE(TestVorbisInit(i, 200000 + 1)); - } -} - -// Test metadata -TEST(VorbisTrackEncoder, Metadata) -{ - // Initiate vorbis encoder. - TestVorbisTrackEncoder encoder; - int channels = 1; - int rate = 44100; - encoder.TestVorbisCreation(channels, rate); - - RefPtr meta = encoder.GetMetadata(); - RefPtr vorbisMetadata(static_cast(meta.get())); - - // According to initialization parameters, verify the correctness - // of vorbisMetadata. - EXPECT_TRUE(vorbisMetadata->mChannels == channels); - EXPECT_TRUE(vorbisMetadata->mSamplingFrequency == rate); - EXPECT_TRUE(parseVorbisMetadata(vorbisMetadata->mData, channels, rate)); -} - -// Test encode function -TEST(VorbisTrackEncoder, EncodedFrame) -{ - // Initiate vorbis encoder - TestVorbisTrackEncoder encoder; - int channels = 1; - int rate = 44100; - encoder.TestVorbisCreation(channels, rate); - - // Generate 1 second samples. - // Reference PeerConnectionMedia.h::Fake_AudioGenerator - RefPtr samples = - mozilla::SharedBuffer::Create(rate * sizeof(AudioDataValue)); - AudioDataValue* data = static_cast(samples->Data()); - for (int i = 0; i < rate; i++) { - data[i] = ((i%8)*4000) - (7*4000)/2; - } - AutoTArray channelData; - channelData.AppendElement(data); - AudioSegment segment; - segment.AppendFrames(samples.forget(), channelData, 44100); - - // Track change notification. - encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, 0, segment); - - // Pull Encoded data back from encoder and verify encoded samples. - EncodedFrameContainer container; - EXPECT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container))); - // Should have some encoded data. - EXPECT_TRUE(container.GetEncodedFrames().Length() > 0); - EXPECT_TRUE(container.GetEncodedFrames().ElementAt(0)->GetFrameData().Length() - > 0); - EXPECT_TRUE(container.GetEncodedFrames().ElementAt(0)->GetFrameType() == - EncodedFrame::FrameType::VORBIS_AUDIO_FRAME); - // Encoded data doesn't have duration and timestamp. - EXPECT_TRUE(container.GetEncodedFrames().ElementAt(0)->GetDuration() == 0); - EXPECT_TRUE(container.GetEncodedFrames().ElementAt(0)->GetTimeStamp() == 0); -} - -// EOS test -TEST(VorbisTrackEncoder, EncodeComplete) -{ - // Initiate vorbis encoder - TestVorbisTrackEncoder encoder; - int channels = 1; - int rate = 44100; - encoder.TestVorbisCreation(channels, rate); - - // Track end notification. - AudioSegment segment; - encoder.NotifyQueuedTrackChanges(nullptr, 0, 0, - MediaStreamListener::TRACK_EVENT_ENDED, - segment); - - // Pull Encoded Data back from encoder. Since we had send out - // EOS to encoder, encoder.GetEncodedTrack should return - // NS_OK immidiately. - EncodedFrameContainer container; - EXPECT_TRUE(NS_SUCCEEDED(encoder.GetEncodedTrack(container))); -} diff --git a/dom/media/gtest/TestWebMWriter.cpp b/dom/media/gtest/TestWebMWriter.cpp index 7b938b0ffe..3460c75d50 100644 --- a/dom/media/gtest/TestWebMWriter.cpp +++ b/dom/media/gtest/TestWebMWriter.cpp @@ -7,16 +7,16 @@ #include "mozilla/CheckedInt.h" #include "mozilla/MathAlgorithms.h" #include "nestegg/nestegg.h" -#include "VorbisTrackEncoder.h" +#include "OpusTrackEncoder.h" #include "VP8TrackEncoder.h" #include "WebMWriter.h" using namespace mozilla; -class WebMVorbisTrackEncoder : public VorbisTrackEncoder +class WebMOpusTrackEncoder : public OpusTrackEncoder { public: - bool TestVorbisCreation(int aChannels, int aSamplingRate) + bool TestOpusCreation(int aChannels, int aSamplingRate) { if (NS_SUCCEEDED(Init(aChannels, aSamplingRate))) { return true; @@ -50,11 +50,11 @@ public: mTimestamp(0) {} - void SetVorbisMetadata(int aChannels, int aSampleRate) { - WebMVorbisTrackEncoder vorbisEncoder; - EXPECT_TRUE(vorbisEncoder.TestVorbisCreation(aChannels, aSampleRate)); - RefPtr vorbisMeta = vorbisEncoder.GetMetadata(); - SetMetadata(vorbisMeta); + void SetOpusMetadata(int aChannels, int aSampleRate) { + WebMOpusTrackEncoder opusEncoder; + EXPECT_TRUE(opusEncoder.TestOpusCreation(aChannels, aSampleRate)); + RefPtr opusMeta = opusEncoder.GetMetadata(); + SetMetadata(opusMeta); } void SetVP8Metadata(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth, int32_t aDisplayHeight,TrackRate aTrackRate) { @@ -108,10 +108,10 @@ TEST(WebMWriter, Metadata) writer.GetContainerData(&encodedBuf, ContainerWriter::FLUSH_NEEDED); EXPECT_TRUE(encodedBuf.Length() == 0); - // Set vorbis metadata. + // Set opus metadata. int channel = 1; int sampleRate = 44100; - writer.SetVorbisMetadata(channel, sampleRate); + writer.SetOpusMetadata(channel, sampleRate); // No output data since we didn't set both audio/video // metadata in writer. @@ -137,10 +137,10 @@ TEST(WebMWriter, Cluster) { TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK | ContainerWriter::CREATE_VIDEO_TRACK); - // Set vorbis metadata. + // Set opus metadata. int channel = 1; int sampleRate = 48000; - writer.SetVorbisMetadata(channel, sampleRate); + writer.SetOpusMetadata(channel, sampleRate); // Set vp8 metadata int32_t width = 320; int32_t height = 240; @@ -180,10 +180,10 @@ TEST(WebMWriter, FLUSH_NEEDED) { TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK | ContainerWriter::CREATE_VIDEO_TRACK); - // Set vorbis metadata. + // Set opus metadata. int channel = 2; int sampleRate = 44100; - writer.SetVorbisMetadata(channel, sampleRate); + writer.SetOpusMetadata(channel, sampleRate); // Set vp8 metadata int32_t width = 176; int32_t height = 352; @@ -305,10 +305,10 @@ TEST(WebMWriter, bug970774_aspect_ratio) { TestWebMWriter writer(ContainerWriter::CREATE_AUDIO_TRACK | ContainerWriter::CREATE_VIDEO_TRACK); - // Set vorbis metadata. + // Set opus metadata. int channel = 1; int sampleRate = 44100; - writer.SetVorbisMetadata(channel, sampleRate); + writer.SetOpusMetadata(channel, sampleRate); // Set vp8 metadata int32_t width = 640; int32_t height = 480; diff --git a/dom/media/gtest/moz.build b/dom/media/gtest/moz.build index 9487ef7ca1..d54c3e6e4b 100644 --- a/dom/media/gtest/moz.build +++ b/dom/media/gtest/moz.build @@ -26,7 +26,6 @@ UNIFIED_SOURCES += [ if CONFIG['MOZ_WEBM_ENCODER']: UNIFIED_SOURCES += [ 'TestVideoTrackEncoder.cpp', - 'TestVorbisTrackEncoder.cpp', 'TestWebMWriter.cpp', ] @@ -39,9 +38,14 @@ TEST_HARNESS_FILES.gtest += [ 'mediasource_test.mp4', 'noise.mp3', 'noise_vbr.mp3', + 'short-zero-in-moov.mp4', + 'short-zero-inband.mov', 'small-shot-false-positive.mp3', 'small-shot.mp3', 'test.webm', + 'test_case_1224361.vp8.ivf', + 'test_case_1224363.vp8.ivf', + 'test_case_1224369.vp8.ivf', ] include('/ipc/chromium/chromium-config.mozbuild') diff --git a/dom/media/gtest/short-zero-in-moov.mp4 b/dom/media/gtest/short-zero-in-moov.mp4 new file mode 100644 index 0000000000..577318c8fa Binary files /dev/null and b/dom/media/gtest/short-zero-in-moov.mp4 differ diff --git a/dom/media/gtest/short-zero-inband.mov b/dom/media/gtest/short-zero-inband.mov new file mode 100644 index 0000000000..9c18642865 Binary files /dev/null and b/dom/media/gtest/short-zero-inband.mov differ diff --git a/dom/media/gtest/test_case_1224361.vp8.ivf b/dom/media/gtest/test_case_1224361.vp8.ivf new file mode 100644 index 0000000000..e2fe942f0e Binary files /dev/null and b/dom/media/gtest/test_case_1224361.vp8.ivf differ diff --git a/dom/media/gtest/test_case_1224363.vp8.ivf b/dom/media/gtest/test_case_1224363.vp8.ivf new file mode 100644 index 0000000000..6d2e4e0206 Binary files /dev/null and b/dom/media/gtest/test_case_1224363.vp8.ivf differ diff --git a/dom/media/gtest/test_case_1224369.vp8.ivf b/dom/media/gtest/test_case_1224369.vp8.ivf new file mode 100644 index 0000000000..2f8deb1148 Binary files /dev/null and b/dom/media/gtest/test_case_1224369.vp8.ivf differ diff --git a/dom/media/mediasink/VideoSink.cpp b/dom/media/mediasink/VideoSink.cpp index 7bd97cb5da..158daf3cc8 100644 --- a/dom/media/mediasink/VideoSink.cpp +++ b/dom/media/mediasink/VideoSink.cpp @@ -4,6 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "MediaQueue.h" #include "VideoSink.h" namespace mozilla { @@ -24,7 +25,6 @@ VideoSink::VideoSink(AbstractThread* aThread, MediaSink* aAudioSink, MediaQueue& aVideoQueue, VideoFrameContainer* aContainer, - bool aRealTime, FrameStatistics& aFrameStats, uint32_t aVQueueSentToCompositerSize) : mOwnerThread(aThread) @@ -32,10 +32,8 @@ VideoSink::VideoSink(AbstractThread* aThread, , mVideoQueue(aVideoQueue) , mContainer(aContainer) , mProducerID(ImageContainer::AllocateProducerID()) - , mRealTime(aRealTime) , mFrameStats(aFrameStats) , mVideoFrameEndTime(-1) - , mOldDroppedCount(0) , mHasVideo(false) , mUpdateScheduler(aThread) , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize) @@ -365,10 +363,6 @@ VideoSink::RenderVideoFrames(int32_t aMaxFrames, frame->mTime, frame->mFrameID, VideoQueue().GetSize()); } mContainer->SetCurrentFrames(frames[0]->As()->mDisplay, images); - - uint32_t dropped = mContainer->GetDroppedImageCount(); - mFrameStats.NotifyDecodedFrames(0, 0, dropped - mOldDroppedCount); - mOldDroppedCount = dropped; } void @@ -390,7 +384,7 @@ VideoSink::UpdateRenderedVideoFrames() int32_t framesRemoved = 0; while (VideoQueue().GetSize() > 0) { MediaData* nextFrame = VideoQueue().PeekFront(); - if (!mRealTime && nextFrame->mTime > clockTime) { + if (nextFrame->mTime > clockTime) { remainingTime = nextFrame->mTime - clockTime; break; } diff --git a/dom/media/mediasink/VideoSink.h b/dom/media/mediasink/VideoSink.h index e2f534fdce..da19a98ff9 100644 --- a/dom/media/mediasink/VideoSink.h +++ b/dom/media/mediasink/VideoSink.h @@ -33,7 +33,6 @@ public: MediaSink* aAudioSink, MediaQueue& aVideoQueue, VideoFrameContainer* aContainer, - bool aRealTime, FrameStatistics& aFrameStats, uint32_t aVQueueSentToCompositerSize); @@ -116,9 +115,6 @@ private: // FrameIDs. A unique and immutable value per VideoSink. const ProducerID mProducerID; - // True if we are decoding a real-time stream. - const bool mRealTime; - // Used to notify MediaDecoder's frame statistics FrameStatistics& mFrameStats; @@ -130,8 +126,6 @@ private: // in microseconds. int64_t mVideoFrameEndTime; - uint32_t mOldDroppedCount; - // Event listeners for VideoQueue MediaEventListener mPushListener; MediaEventListener mFinishListener; diff --git a/dom/media/mediasource/AutoTaskQueue.h b/dom/media/mediasource/AutoTaskQueue.h new file mode 100644 index 0000000000..54412e7d63 --- /dev/null +++ b/dom/media/mediasource/AutoTaskQueue.h @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_AUTOTASKQUEUE_H_ +#define MOZILLA_AUTOTASKQUEUE_H_ + +#include "mozilla/RefPtr.h" +#include "mozilla/TaskQueue.h" + +namespace mozilla { + +// A convenience TaskQueue not requiring explicit shutdown. +class AutoTaskQueue : public AbstractThread +{ +public: + explicit AutoTaskQueue(already_AddRefed aPool, bool aSupportsTailDispatch = false) + : AbstractThread(aSupportsTailDispatch) + , mTaskQueue(new TaskQueue(Move(aPool), aSupportsTailDispatch)) + {} + + TaskDispatcher& TailDispatcher() override + { + return mTaskQueue->TailDispatcher(); + } + + void Dispatch(already_AddRefed aRunnable, + DispatchFailureHandling aFailureHandling = AssertDispatchSuccess, + DispatchReason aReason = NormalDispatch) override + { + mTaskQueue->Dispatch(Move(aRunnable), aFailureHandling, aReason); + } + + // Blocks until all tasks finish executing. + void AwaitIdle() { mTaskQueue->AwaitIdle(); } + + bool IsEmpty() { return mTaskQueue->IsEmpty(); } + + // Returns true if the current thread is currently running a Runnable in + // the task queue. + bool IsCurrentThreadIn() override { return mTaskQueue->IsCurrentThreadIn(); } + +private: + ~AutoTaskQueue() + { + RefPtr taskqueue = mTaskQueue; + nsCOMPtr task = + NS_NewRunnableFunction([taskqueue]() { taskqueue->BeginShutdown(); }); + AbstractThread::MainThread()->Dispatch(task.forget()); + } + RefPtr mTaskQueue; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/mediasource/MediaSource.cpp b/dom/media/mediasource/MediaSource.cpp index a2dae0513a..b14d05c1b6 100644 --- a/dom/media/mediasource/MediaSource.cpp +++ b/dom/media/mediasource/MediaSource.cpp @@ -535,16 +535,6 @@ MediaSource::DurationChange(double aOldDuration, double aNewDuration) // TODO: If partial audio frames/text cues exist, clamp duration based on mSourceBuffers. } -void -MediaSource::NotifyEvicted(double aStart, double aEnd) -{ - MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG("NotifyEvicted(aStart=%f, aEnd=%f)", aStart, aEnd); - // Cycle through all SourceBuffers and tell them to evict data in - // the given range. - mSourceBuffers->Evict(aStart, aEnd); -} - void MediaSource::GetMozDebugReaderData(nsAString& aString) { diff --git a/dom/media/mediasource/MediaSource.h b/dom/media/mediasource/MediaSource.h index 7824aebb60..bb13e148fc 100644 --- a/dom/media/mediasource/MediaSource.h +++ b/dom/media/mediasource/MediaSource.h @@ -105,11 +105,6 @@ public: return mPrincipal; } - // Called by SourceBuffers to notify this MediaSource that data has - // been evicted from the buffered data. The start and end times - // that were evicted are provided. - void NotifyEvicted(double aStart, double aEnd); - // Returns a string describing the state of the MediaSource internal // buffered data. Used for debugging purposes. void GetMozDebugReaderData(nsAString& aString); diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp index 881dc939d4..54b04f4e03 100644 --- a/dom/media/mediasource/MediaSourceDecoder.cpp +++ b/dom/media/mediasource/MediaSourceDecoder.cpp @@ -61,7 +61,7 @@ MediaSourceDecoder::Load(nsIStreamListener**) return NS_ERROR_FAILURE; } - nsresult rv = GetStateMachine()->Init(); + nsresult rv = GetStateMachine()->Init(this); NS_ENSURE_SUCCESS(rv, rv); SetStateMachineParameters(); @@ -292,6 +292,11 @@ bool MediaSourceDecoder::CanPlayThrough() { MOZ_ASSERT(NS_IsMainThread()); + + if (NextFrameBufferedStatus() == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE) { + return false; + } + if (IsNaN(mMediaSource->Duration())) { // Don't have any data yet. return false; diff --git a/dom/media/mediasource/MediaSourceDemuxer.cpp b/dom/media/mediasource/MediaSourceDemuxer.cpp index 348198d73f..715dc9858e 100644 --- a/dom/media/mediasource/MediaSourceDemuxer.cpp +++ b/dom/media/mediasource/MediaSourceDemuxer.cpp @@ -12,6 +12,7 @@ #include "MediaSourceUtils.h" #include "SourceBufferList.h" #include "nsPrintfCString.h" +#include "OpusDecoder.h" namespace mozilla { @@ -20,8 +21,8 @@ using media::TimeUnit; using media::TimeIntervals; MediaSourceDemuxer::MediaSourceDemuxer() - : mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK), - /* aSupportsTailDispatch = */ false)) + : mTaskQueue(new AutoTaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK), + /* aSupportsTailDispatch = */ false)) , mMonitor("MediaSourceDemuxer") { MOZ_ASSERT(NS_IsMainThread()); @@ -249,8 +250,6 @@ MediaSourceDemuxer::GetManager(TrackType aTrack) MediaSourceDemuxer::~MediaSourceDemuxer() { mInitPromise.RejectIfExists(DemuxerFailureReason::SHUTDOWN, __func__); - mTaskQueue->BeginShutdown(); - mTaskQueue = nullptr; } void @@ -296,6 +295,10 @@ MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent, , mType(aType) , mMonitor("MediaSourceTrackDemuxer") , mReset(true) + , mPreRoll( + TimeUnit::FromMicroseconds( + OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) + ? 80000 : 0)) { } @@ -379,14 +382,24 @@ MediaSourceTrackDemuxer::DoSeek(media::TimeUnit aTime) { TimeIntervals buffered = mManager->Buffered(mType); buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ); + TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::FromMicroseconds(0)); - if (!buffered.Contains(aTime)) { - // We don't have the data to seek to. - return SeekPromise::CreateAndReject(DemuxerFailureReason::WAITING_FOR_DATA, - __func__); + if (!buffered.Contains(seekTime)) { + if (!buffered.Contains(aTime)) { + // We don't have the data to seek to. + return SeekPromise::CreateAndReject(DemuxerFailureReason::WAITING_FOR_DATA, + __func__); + } + // Theorically we should reject the promise with WAITING_FOR_DATA, + // however, to avoid unwanted regressions we assume that if at this time + // we don't have the wanted data it won't come later. + // Instead of using the pre-rolled time, use the earliest time available in + // the interval. + TimeIntervals::IndexType index = buffered.Find(aTime); + MOZ_ASSERT(index != TimeIntervals::NoIndex); + seekTime = buffered[index].mStart; } - TimeUnit seekTime = - mManager->Seek(mType, aTime, MediaSourceDemuxer::EOS_FUZZ); + seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ); bool error; RefPtr sample = mManager->GetSample(mType, diff --git a/dom/media/mediasource/MediaSourceDemuxer.h b/dom/media/mediasource/MediaSourceDemuxer.h index 8be870cbf6..529a64c8e6 100644 --- a/dom/media/mediasource/MediaSourceDemuxer.h +++ b/dom/media/mediasource/MediaSourceDemuxer.h @@ -10,7 +10,7 @@ #include "mozilla/Atomics.h" #include "mozilla/Maybe.h" #include "mozilla/Monitor.h" -#include "mozilla/TaskQueue.h" +#include "AutoTaskQueue.h" #include "MediaDataDemuxer.h" #include "MediaDecoderReader.h" @@ -47,7 +47,7 @@ public: /* interface for TrackBuffersManager */ void AttachSourceBuffer(TrackBuffersManager* aSourceBuffer); void DetachSourceBuffer(TrackBuffersManager* aSourceBuffer); - TaskQueue* GetTaskQueue() { return mTaskQueue; } + AutoTaskQueue* GetTaskQueue() { return mTaskQueue; } // Returns a string describing the state of the MediaSource internal // buffered data. Used for debugging purposes. @@ -73,7 +73,7 @@ private: return !GetTaskQueue() || GetTaskQueue()->IsCurrentThreadIn(); } - RefPtr mTaskQueue; + RefPtr mTaskQueue; nsTArray> mDemuxers; nsTArray> mSourceBuffers; @@ -133,6 +133,10 @@ private: // Set to true following a reset. Ensure that the next sample demuxed // is available at position 0. bool mReset; + + // Amount of pre-roll time when seeking. + // Set to 80ms if track is Opus. + const media::TimeUnit mPreRoll; }; } // namespace mozilla diff --git a/dom/media/mediasource/SourceBuffer.cpp b/dom/media/mediasource/SourceBuffer.cpp index 22d4ec8cf2..16ea9d870f 100644 --- a/dom/media/mediasource/SourceBuffer.cpp +++ b/dom/media/mediasource/SourceBuffer.cpp @@ -33,24 +33,29 @@ extern mozilla::LogModule* GetMediaSourceAPILog(); #define MSE_DEBUGV(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, ("SourceBuffer(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) #define MSE_API(arg, ...) MOZ_LOG(GetMediaSourceAPILog(), mozilla::LogLevel::Debug, ("SourceBuffer(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) +#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) +#define MP4_READER_DORMANT_HEURISTIC +#else +#undef MP4_READER_DORMANT_HEURISTIC +#endif + namespace mozilla { using media::TimeUnit; +typedef SourceBufferAttributes::AppendState AppendState; namespace dom { void SourceBuffer::SetMode(SourceBufferAppendMode aMode, ErrorResult& aRv) { - typedef mozilla::SourceBufferContentManager::AppendState AppendState; - MOZ_ASSERT(NS_IsMainThread()); MSE_API("SetMode(aMode=%d)", aMode); if (!IsAttached() || mUpdating) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return; } - if (mAttributes->mGenerateTimestamps && + if (mCurrentAttributes.mGenerateTimestamps && aMode == SourceBufferAppendMode::Segments) { aRv.Throw(NS_ERROR_DOM_TYPE_ERR); return; @@ -59,24 +64,22 @@ SourceBuffer::SetMode(SourceBufferAppendMode aMode, ErrorResult& aRv) if (mMediaSource->ReadyState() == MediaSourceReadyState::Ended) { mMediaSource->SetReadyState(MediaSourceReadyState::Open); } - if (mContentManager->GetAppendState() == AppendState::PARSING_MEDIA_SEGMENT){ + if (mCurrentAttributes.GetAppendState() == AppendState::PARSING_MEDIA_SEGMENT){ aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return; } if (aMode == SourceBufferAppendMode::Sequence) { // Will set GroupStartTimestamp to GroupEndTimestamp. - mContentManager->RestartGroupStartTimestamp(); + mCurrentAttributes.RestartGroupStartTimestamp(); } - mAttributes->SetAppendMode(aMode); + mCurrentAttributes.SetAppendMode(aMode); } void SourceBuffer::SetTimestampOffset(double aTimestampOffset, ErrorResult& aRv) { - typedef mozilla::SourceBufferContentManager::AppendState AppendState; - MOZ_ASSERT(NS_IsMainThread()); MSE_API("SetTimestampOffset(aTimestampOffset=%f)", aTimestampOffset); if (!IsAttached() || mUpdating) { @@ -87,13 +90,13 @@ SourceBuffer::SetTimestampOffset(double aTimestampOffset, ErrorResult& aRv) if (mMediaSource->ReadyState() == MediaSourceReadyState::Ended) { mMediaSource->SetReadyState(MediaSourceReadyState::Open); } - if (mContentManager->GetAppendState() == AppendState::PARSING_MEDIA_SEGMENT){ + if (mCurrentAttributes.GetAppendState() == AppendState::PARSING_MEDIA_SEGMENT){ aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); return; } - mAttributes->SetApparentTimestampOffset(aTimestampOffset); - if (mAttributes->GetAppendMode() == SourceBufferAppendMode::Sequence) { - mContentManager->SetGroupStartTimestamp(mAttributes->GetTimestampOffset()); + mCurrentAttributes.SetApparentTimestampOffset(aTimestampOffset); + if (mCurrentAttributes.GetAppendMode() == SourceBufferAppendMode::Sequence) { + mCurrentAttributes.SetGroupStartTimestamp(mCurrentAttributes.GetTimestampOffset()); } } @@ -108,7 +111,7 @@ SourceBuffer::GetBuffered(ErrorResult& aRv) return nullptr; } bool rangeChanged = true; - media::TimeIntervals intersection = mContentManager->Buffered(); + media::TimeIntervals intersection = mTrackBuffersManager->Buffered(); MSE_DEBUGV("intersection=%s", DumpTimeRanges(intersection).get()); if (mBuffered) { media::TimeIntervals currentValue(mBuffered); @@ -127,7 +130,7 @@ SourceBuffer::GetBuffered(ErrorResult& aRv) media::TimeIntervals SourceBuffer::GetTimeIntervals() { - return mContentManager->Buffered(); + return mTrackBuffersManager->Buffered(); } void @@ -140,11 +143,11 @@ SourceBuffer::SetAppendWindowStart(double aAppendWindowStart, ErrorResult& aRv) return; } if (aAppendWindowStart < 0 || - aAppendWindowStart >= mAttributes->GetAppendWindowEnd()) { + aAppendWindowStart >= mCurrentAttributes.GetAppendWindowEnd()) { aRv.Throw(NS_ERROR_DOM_TYPE_ERR); return; } - mAttributes->SetAppendWindowStart(aAppendWindowStart); + mCurrentAttributes.SetAppendWindowStart(aAppendWindowStart); } void @@ -157,11 +160,11 @@ SourceBuffer::SetAppendWindowEnd(double aAppendWindowEnd, ErrorResult& aRv) return; } if (IsNaN(aAppendWindowEnd) || - aAppendWindowEnd <= mAttributes->GetAppendWindowStart()) { + aAppendWindowEnd <= mCurrentAttributes.GetAppendWindowStart()) { aRv.Throw(NS_ERROR_DOM_TYPE_ERR); return; } - mAttributes->SetAppendWindowEnd(aAppendWindowEnd); + mCurrentAttributes.SetAppendWindowEnd(aAppendWindowEnd); } void @@ -196,9 +199,9 @@ SourceBuffer::Abort(ErrorResult& aRv) return; } AbortBufferAppend(); - mContentManager->ResetParserState(); - mAttributes->SetAppendWindowStart(0); - mAttributes->SetAppendWindowEnd(PositiveInfinity()); + ResetParserState(); + mCurrentAttributes.SetAppendWindowStart(0); + mCurrentAttributes.SetAppendWindowEnd(PositiveInfinity()); } void @@ -207,15 +210,18 @@ SourceBuffer::AbortBufferAppend() if (mUpdating) { if (mPendingAppend.Exists()) { mPendingAppend.Disconnect(); - mContentManager->AbortAppendData(); - // Some data may have been added by the Segment Parser Loop. - // Check if we need to update the duration. - CheckEndTime(); + mTrackBuffersManager->AbortAppendData(); } AbortUpdating(); } } +void +SourceBuffer::ResetParserState() +{ + mTrackBuffersManager->ResetParserState(mCurrentAttributes); +} + void SourceBuffer::Remove(double aStart, double aEnd, ErrorResult& aRv) { @@ -246,9 +252,10 @@ void SourceBuffer::RangeRemoval(double aStart, double aEnd) { StartUpdating(); + RefPtr self = this; - mContentManager->RangeRemoval(TimeUnit::FromSeconds(aStart), - TimeUnit::FromSeconds(aEnd)) + mTrackBuffersManager->RangeRemoval(TimeUnit::FromSeconds(aStart), + TimeUnit::FromSeconds(aEnd)) ->Then(AbstractThread::MainThread(), __func__, [self] (bool) { self->StopUpdating(); }, []() { MOZ_ASSERT(false); }); @@ -264,12 +271,12 @@ SourceBuffer::Detach() return; } AbortBufferAppend(); - if (mContentManager) { - mContentManager->Detach(); - mMediaSource->GetDecoder()->GetDemuxer()->DetachSourceBuffer( - static_cast(mContentManager.get())); + if (mTrackBuffersManager) { + mTrackBuffersManager->Detach(); + mMediaSource->GetDecoder()->GetDemuxer()->DetachSourceBuffer( + mTrackBuffersManager.get()); } - mContentManager = nullptr; + mTrackBuffersManager = nullptr; mMediaSource = nullptr; } @@ -279,7 +286,7 @@ SourceBuffer::Ended() MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(IsAttached()); MSE_DEBUG("Ended"); - mContentManager->Ended(); + mTrackBuffersManager->Ended(); // We want the MediaSourceReader to refresh its buffered range as it may // have been modified (end lined up). mMediaSource->GetDecoder()->NotifyDataArrived(); @@ -288,36 +295,34 @@ SourceBuffer::Ended() SourceBuffer::SourceBuffer(MediaSource* aMediaSource, const nsACString& aType) : DOMEventTargetHelper(aMediaSource->GetParentObject()) , mMediaSource(aMediaSource) + , mCurrentAttributes(aType.LowerCaseEqualsLiteral("audio/mpeg") || + aType.LowerCaseEqualsLiteral("audio/aac")) , mUpdating(false) , mActive(false) , mType(aType) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aMediaSource); - mEvictionThreshold = Preferences::GetUint("media.mediasource.eviction_threshold", - 100 * (1 << 20)); - bool generateTimestamps = false; - if (aType.LowerCaseEqualsLiteral("audio/mpeg") || - aType.LowerCaseEqualsLiteral("audio/aac")) { - generateTimestamps = true; - } - mAttributes = new SourceBufferAttributes(generateTimestamps); - mContentManager = - SourceBufferContentManager::CreateManager(mAttributes, - aMediaSource->GetDecoder(), - aType); - MSE_DEBUG("Create mContentManager=%p", - mContentManager.get()); + mTrackBuffersManager = + new TrackBuffersManager(aMediaSource->GetDecoder(), aType); + + // Now that we know what type we're dealing with, enable dormant as needed. +#if defined(MP4_READER_DORMANT_HEURISTIC) + aMediaSource->GetDecoder()->NotifyDormantSupported(Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false)); +#endif + + MSE_DEBUG("Create mTrackBuffersManager=%p", + mTrackBuffersManager.get()); ErrorResult dummy; - if (mAttributes->mGenerateTimestamps) { + if (mCurrentAttributes.mGenerateTimestamps) { SetMode(SourceBufferAppendMode::Sequence, dummy); } else { SetMode(SourceBufferAppendMode::Segments, dummy); } mMediaSource->GetDecoder()->GetDemuxer()->AttachSourceBuffer( - static_cast(mContentManager.get())); + mTrackBuffersManager.get()); } SourceBuffer::~SourceBuffer() @@ -392,7 +397,7 @@ SourceBuffer::CheckEndTime() { MOZ_ASSERT(NS_IsMainThread()); // Check if we need to update mMediaSource duration - double endTime = mContentManager->GroupEndTimestamp().ToSeconds(); + double endTime = mCurrentAttributes.GetGroupEndTimestamp().ToSeconds(); double duration = mMediaSource->Duration(); if (endTime > duration) { mMediaSource->SetDuration(endTime, MSRangeRemovalAction::SKIP); @@ -408,33 +413,21 @@ SourceBuffer::AppendData(const uint8_t* aData, uint32_t aLength, ErrorResult& aR if (!data) { return; } - mContentManager->AppendData(data, mAttributes->GetTimestampOffset()); - StartUpdating(); - BufferAppend(); -} - -void -SourceBuffer::BufferAppend() -{ - MOZ_ASSERT(mUpdating); - MOZ_ASSERT(mMediaSource); - MOZ_ASSERT(!mPendingAppend.Exists()); - - mPendingAppend.Begin(mContentManager->BufferAppend() + mPendingAppend.Begin(mTrackBuffersManager->AppendData(data, mCurrentAttributes) ->Then(AbstractThread::MainThread(), __func__, this, &SourceBuffer::AppendDataCompletedWithSuccess, &SourceBuffer::AppendDataErrored)); } void -SourceBuffer::AppendDataCompletedWithSuccess(bool aHasActiveTracks) +SourceBuffer::AppendDataCompletedWithSuccess(SourceBufferTask::AppendBufferResult aResult) { MOZ_ASSERT(mUpdating); mPendingAppend.Complete(); - if (aHasActiveTracks) { + if (aResult.first()) { if (!mActive) { mActive = true; mMediaSource->SourceBufferIsActive(this); @@ -447,6 +440,8 @@ SourceBuffer::AppendDataCompletedWithSuccess(bool aHasActiveTracks) mMediaSource->GetDecoder()->NotifyBytesDownloaded(); } + mCurrentAttributes = aResult.second(); + CheckEndTime(); StopUpdating(); @@ -474,7 +469,7 @@ SourceBuffer::AppendError(bool aDecoderError) { MOZ_ASSERT(NS_IsMainThread()); - mContentManager->ResetParserState(); + ResetParserState(); mUpdating = false; @@ -492,7 +487,7 @@ SourceBuffer::AppendError(bool aDecoderError) already_AddRefed SourceBuffer::PrepareAppend(const uint8_t* aData, uint32_t aLength, ErrorResult& aRv) { - typedef SourceBufferContentManager::EvictDataResult Result; + typedef TrackBuffersManager::EvictDataResult Result; if (!IsAttached() || mUpdating) { aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); @@ -513,34 +508,16 @@ SourceBuffer::PrepareAppend(const uint8_t* aData, uint32_t aLength, ErrorResult& } // Eviction uses a byte threshold. If the buffer is greater than the - // number of bytes then data is evicted. The time range for this - // eviction is reported back to the media source. It will then - // evict data before that range across all SourceBuffers it knows - // about. - // TODO: Make the eviction threshold smaller for audio-only streams. + // number of bytes then data is evicted. // TODO: Drive evictions off memory pressure notifications. // TODO: Consider a global eviction threshold rather than per TrackBuffer. - TimeUnit newBufferStartTime; - // Attempt to evict the amount of data we are about to add by lowering the - // threshold. - uint32_t toEvict = - (mEvictionThreshold > aLength) ? mEvictionThreshold - aLength : aLength; + // Give a chance to the TrackBuffersManager to evict some data if needed. Result evicted = - mContentManager->EvictData(TimeUnit::FromSeconds(mMediaSource->GetDecoder()->GetCurrentTime()), - toEvict, &newBufferStartTime); - if (evicted == Result::DATA_EVICTED) { - MSE_DEBUG("AppendData Evict; current buffered start=%f", - GetBufferedStart()); - - // We notify that we've evicted from the time range 0 through to - // the current start point. - mMediaSource->NotifyEvicted(0.0, newBufferStartTime.ToSeconds()); - } + mTrackBuffersManager->EvictData(TimeUnit::FromSeconds(mMediaSource->GetDecoder()->GetCurrentTime()), + aLength); // See if we have enough free space to append our new data. - // As we can only evict once we have playable data, we must give a chance - // to the DASH player to provide a complete media segment. - if (aLength > mEvictionThreshold || evicted == Result::BUFFER_FULL) { + if (evicted == Result::BUFFER_FULL) { aRv.Throw(NS_ERROR_DOM_QUOTA_EXCEEDED_ERR); return nullptr; } @@ -571,25 +548,11 @@ SourceBuffer::GetBufferedEnd() return ranges->Length() > 0 ? ranges->GetEndTime() : 0; } -void -SourceBuffer::Evict(double aStart, double aEnd) -{ - MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG("Evict(aStart=%f, aEnd=%f)", aStart, aEnd); - double currentTime = mMediaSource->GetDecoder()->GetCurrentTime(); - double evictTime = aEnd; - const double safety_threshold = 5; - if (currentTime + safety_threshold >= evictTime) { - evictTime -= safety_threshold; - } - mContentManager->EvictBefore(TimeUnit::FromSeconds(evictTime)); -} - NS_IMPL_CYCLE_COLLECTION_CLASS(SourceBuffer) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(SourceBuffer) // Tell the TrackBuffer to end its current SourceBufferResource. - SourceBufferContentManager* manager = tmp->mContentManager; + TrackBuffersManager* manager = tmp->mTrackBuffersManager; if (manager) { manager->Detach(); } diff --git a/dom/media/mediasource/SourceBuffer.h b/dom/media/mediasource/SourceBuffer.h index 1cd8c23a61..6595fbef20 100644 --- a/dom/media/mediasource/SourceBuffer.h +++ b/dom/media/mediasource/SourceBuffer.h @@ -14,6 +14,7 @@ #include "mozilla/Atomics.h" #include "mozilla/Attributes.h" #include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/UniquePtr.h" #include "mozilla/dom/SourceBufferBinding.h" #include "mozilla/dom/TypedArray.h" #include "mozilla/mozalloc.h" @@ -24,8 +25,8 @@ #include "nsISupports.h" #include "nsString.h" #include "nscore.h" -#include "SourceBufferContentManager.h" -#include "mozilla/Monitor.h" +#include "TrackBuffersManager.h" +#include "SourceBufferTask.h" class JSObject; struct JSContext; @@ -35,109 +36,18 @@ namespace mozilla { class ErrorResult; class MediaByteBuffer; template class AsyncEventRunner; -class TrackBuffersManager; namespace dom { class TimeRanges; -class SourceBufferAttributes { -public: - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SourceBufferAttributes); - explicit SourceBufferAttributes(bool aGenerateTimestamp) - : mGenerateTimestamps(aGenerateTimestamp) - , mMonitor("SourceBufferAttributes") - , mAppendWindowStart(0) - , mAppendWindowEnd(PositiveInfinity()) - , mAppendMode(SourceBufferAppendMode::Segments) - , mApparentTimestampOffset(0) - {} - - double GetAppendWindowStart() - { - MonitorAutoLock mon(mMonitor); - return mAppendWindowStart; - } - - double GetAppendWindowEnd() - { - MonitorAutoLock mon(mMonitor); - return mAppendWindowEnd; - } - - void SetAppendWindowStart(double aWindowStart) - { - MonitorAutoLock mon(mMonitor); - mAppendWindowStart = aWindowStart; - } - - void SetAppendWindowEnd(double aWindowEnd) - { - MonitorAutoLock mon(mMonitor); - mAppendWindowEnd = aWindowEnd; - } - - double GetApparentTimestampOffset() - { - MonitorAutoLock mon(mMonitor); - return mApparentTimestampOffset; - } - - void SetApparentTimestampOffset(double aTimestampOffset) - { - MonitorAutoLock mon(mMonitor); - mApparentTimestampOffset = aTimestampOffset; - mTimestampOffset = media::TimeUnit::FromSeconds(aTimestampOffset); - } - - media::TimeUnit GetTimestampOffset() - { - MonitorAutoLock mon(mMonitor); - return mTimestampOffset; - } - - void SetTimestampOffset(media::TimeUnit& aTimestampOffset) - { - MonitorAutoLock mon(mMonitor); - mTimestampOffset = aTimestampOffset; - mApparentTimestampOffset = aTimestampOffset.ToSeconds(); - } - - SourceBufferAppendMode GetAppendMode() - { - MonitorAutoLock mon(mMonitor); - return mAppendMode; - } - - void SetAppendMode(SourceBufferAppendMode aAppendMode) - { - MonitorAutoLock mon(mMonitor); - mAppendMode = aAppendMode; - } - - // mGenerateTimestamp isn't mutable once the source buffer has been constructed - // We don't need a monitor to protect it across threads. - const bool mGenerateTimestamps; - -private: - ~SourceBufferAttributes() {}; - - // Monitor protecting all members below. - Monitor mMonitor; - double mAppendWindowStart; - double mAppendWindowEnd; - SourceBufferAppendMode mAppendMode; - double mApparentTimestampOffset; - media::TimeUnit mTimestampOffset; -}; - class SourceBuffer final : public DOMEventTargetHelper { public: /** WebIDL Methods. */ SourceBufferAppendMode Mode() const { - return mAttributes->GetAppendMode(); + return mCurrentAttributes.GetAppendMode(); } void SetMode(SourceBufferAppendMode aMode, ErrorResult& aRv); @@ -152,21 +62,21 @@ public: double TimestampOffset() const { - return mAttributes->GetApparentTimestampOffset(); + return mCurrentAttributes.GetApparentTimestampOffset(); } void SetTimestampOffset(double aTimestampOffset, ErrorResult& aRv); double AppendWindowStart() const { - return mAttributes->GetAppendWindowStart(); + return mCurrentAttributes.GetAppendWindowStart(); } void SetAppendWindowStart(double aAppendWindowStart, ErrorResult& aRv); double AppendWindowEnd() const { - return mAttributes->GetAppendWindowEnd(); + return mCurrentAttributes.GetAppendWindowEnd(); } void SetAppendWindowEnd(double aAppendWindowEnd, ErrorResult& aRv); @@ -206,9 +116,6 @@ public: void Ended(); - // Evict data in the source buffer in the given time range. - void Evict(double aStart, double aEnd); - double GetBufferedStart(); double GetBufferedEnd(); @@ -233,6 +140,7 @@ private: void StartUpdating(); void StopUpdating(); void AbortUpdating(); + void ResetParserState(); // If the media segment contains data beyond the current duration, // then run the duration change algorithm with new duration set to the @@ -241,7 +149,6 @@ private: // Shared implementation of AppendBuffer overloads. void AppendData(const uint8_t* aData, uint32_t aLength, ErrorResult& aRv); - void BufferAppend(); // Implement the "Append Error Algorithm". // Will call endOfStream() with "decode" error if aDecodeError is true. @@ -255,21 +162,19 @@ private: uint32_t aLength, ErrorResult& aRv); - void AppendDataCompletedWithSuccess(bool aHasActiveTracks); + void AppendDataCompletedWithSuccess(SourceBufferTask::AppendBufferResult aResult); void AppendDataErrored(nsresult aError); RefPtr mMediaSource; - uint32_t mEvictionThreshold; - - RefPtr mContentManager; - RefPtr mAttributes; + RefPtr mTrackBuffersManager; + SourceBufferAttributes mCurrentAttributes; bool mUpdating; mozilla::Atomic mActive; - MozPromiseRequestHolder mPendingAppend; + MozPromiseRequestHolder mPendingAppend; const nsCString mType; RefPtr mBuffered; diff --git a/dom/media/mediasource/SourceBufferAttributes.h b/dom/media/mediasource/SourceBufferAttributes.h new file mode 100644 index 0000000000..0af80a4fe4 --- /dev/null +++ b/dom/media/mediasource/SourceBufferAttributes.h @@ -0,0 +1,157 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_SourceBufferAttributes_h_ +#define mozilla_SourceBufferAttributes_h_ + +#include "TimeUnits.h" +#include "mozilla/dom/SourceBufferBinding.h" +#include "mozilla/Maybe.h" + +namespace mozilla { + +class SourceBufferAttributes { +public: + + // Current state as per Segment Parser Loop Algorithm + // http://w3c.github.io/media-source/index.html#sourcebuffer-segment-parser-loop + enum class AppendState + { + WAITING_FOR_SEGMENT, + PARSING_INIT_SEGMENT, + PARSING_MEDIA_SEGMENT, + }; + + explicit SourceBufferAttributes(bool aGenerateTimestamp) + : mGenerateTimestamps(aGenerateTimestamp) + , mAppendWindowStart(0) + , mAppendWindowEnd(PositiveInfinity()) + , mAppendMode(dom::SourceBufferAppendMode::Segments) + , mApparentTimestampOffset(0) + , mAppendState(AppendState::WAITING_FOR_SEGMENT) + {} + + SourceBufferAttributes(const SourceBufferAttributes& aOther) = default; + + double GetAppendWindowStart() const + { + return mAppendWindowStart; + } + + double GetAppendWindowEnd() const + { + return mAppendWindowEnd; + } + + void SetAppendWindowStart(double aWindowStart) + { + mAppendWindowStart = aWindowStart; + } + + void SetAppendWindowEnd(double aWindowEnd) + { + mAppendWindowEnd = aWindowEnd; + } + + double GetApparentTimestampOffset() const + { + return mApparentTimestampOffset; + } + + void SetApparentTimestampOffset(double aTimestampOffset) + { + mApparentTimestampOffset = aTimestampOffset; + mTimestampOffset = media::TimeUnit::FromSeconds(aTimestampOffset); + } + + media::TimeUnit GetTimestampOffset() const + { + return mTimestampOffset; + } + + void SetTimestampOffset(const media::TimeUnit& aTimestampOffset) + { + mTimestampOffset = aTimestampOffset; + mApparentTimestampOffset = aTimestampOffset.ToSeconds(); + } + + dom::SourceBufferAppendMode GetAppendMode() const + { + return mAppendMode; + } + + void SetAppendMode(dom::SourceBufferAppendMode aAppendMode) + { + mAppendMode = aAppendMode; + } + + void SetGroupStartTimestamp(const media::TimeUnit& aGroupStartTimestamp) + { + mGroupStartTimestamp = Some(aGroupStartTimestamp); + } + + media::TimeUnit GetGroupStartTimestamp() const + { + return mGroupStartTimestamp.ref(); + } + + bool HaveGroupStartTimestamp() const + { + return mGroupStartTimestamp.isSome(); + } + + void ResetGroupStartTimestamp() + { + mGroupStartTimestamp.reset(); + } + + void RestartGroupStartTimestamp() + { + mGroupStartTimestamp = Some(mGroupEndTimestamp); + } + + media::TimeUnit GetGroupEndTimestamp() const + { + return mGroupEndTimestamp; + } + + void SetGroupEndTimestamp(const media::TimeUnit& aGroupEndTimestamp) + { + mGroupEndTimestamp = aGroupEndTimestamp; + } + + AppendState GetAppendState() const + { + return mAppendState; + } + + void SetAppendState(AppendState aState) + { + mAppendState = aState; + } + + // mGenerateTimestamp isn't mutable once the source buffer has been constructed + bool mGenerateTimestamps; + + SourceBufferAttributes& operator=(const SourceBufferAttributes& aOther) = default; + +private: + SourceBufferAttributes() = delete; + + double mAppendWindowStart; + double mAppendWindowEnd; + dom::SourceBufferAppendMode mAppendMode; + double mApparentTimestampOffset; + media::TimeUnit mTimestampOffset; + Maybe mGroupStartTimestamp; + media::TimeUnit mGroupEndTimestamp; + // The current append state as per https://w3c.github.io/media-source/#sourcebuffer-append-state + AppendState mAppendState; +}; + +} // end namespace mozilla + +#endif /* mozilla_SourceBufferAttributes_h_ */ diff --git a/dom/media/mediasource/SourceBufferContentManager.cpp b/dom/media/mediasource/SourceBufferContentManager.cpp deleted file mode 100644 index 965dca9a8e..0000000000 --- a/dom/media/mediasource/SourceBufferContentManager.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#include "SourceBufferContentManager.h" -#include "mozilla/Preferences.h" -#include "TrackBuffersManager.h" - -namespace mozilla { - -#if defined(MOZ_GONK_MEDIACODEC) || defined(XP_WIN) || defined(MOZ_APPLEMEDIA) || defined(MOZ_FFMPEG) -#define MP4_READER_DORMANT_HEURISTIC -#else -#undef MP4_READER_DORMANT_HEURISTIC -#endif - -already_AddRefed -SourceBufferContentManager::CreateManager(dom::SourceBufferAttributes* aAttributes, - MediaSourceDecoder* aParentDecoder, - const nsACString &aType) -{ - RefPtr manager; - manager = new TrackBuffersManager(aAttributes, aParentDecoder, aType); - - // Now that we know what type we're dealing with, enable dormant as needed. -#if defined(MP4_READER_DORMANT_HEURISTIC) - aParentDecoder->NotifyDormantSupported(Preferences::GetBool("media.decoder.heuristic.dormant.enabled", false)); -#endif - - return manager.forget(); -} - -} // namespace mozilla diff --git a/dom/media/mediasource/SourceBufferContentManager.h b/dom/media/mediasource/SourceBufferContentManager.h deleted file mode 100644 index 563b1e1d8a..0000000000 --- a/dom/media/mediasource/SourceBufferContentManager.h +++ /dev/null @@ -1,120 +0,0 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=8 sts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MOZILLA_SOURCEBUFFERCONTENTMANAGER_H_ -#define MOZILLA_SOURCEBUFFERCONTENTMANAGER_H_ - -#include "mozilla/MozPromise.h" - -#include "MediaData.h" -#include "MediaSourceDecoder.h" -#include "TimeUnits.h" -#include "nsString.h" - -namespace mozilla { - -namespace dom { - class SourceBufferAttributes; -} - -namespace dom { -class SourceBuffer; -} - -class SourceBufferContentManager { -public: - NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SourceBufferContentManager); - - typedef MozPromise AppendPromise; - typedef AppendPromise RangeRemovalPromise; - - static already_AddRefed - CreateManager(dom::SourceBufferAttributes* aAttributes, - MediaSourceDecoder* aParentDecoder, - const nsACString& aType); - - // Add data to the end of the input buffer. - // Returns false if the append failed. - virtual bool - AppendData(MediaByteBuffer* aData, media::TimeUnit aTimestampOffset) = 0; - - // Run MSE Buffer Append Algorithm - // 3.5.5 Buffer Append Algorithm. - // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append - virtual RefPtr BufferAppend() = 0; - - // Abort any pending AppendData. - virtual void AbortAppendData() = 0; - - // Run MSE Reset Parser State Algorithm. - // 3.5.2 Reset Parser State - // http://w3c.github.io/media-source/#sourcebuffer-reset-parser-state - virtual void ResetParserState() = 0; - - // Runs MSE range removal algorithm. - // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal - virtual RefPtr RangeRemoval(media::TimeUnit aStart, - media::TimeUnit aEnd) = 0; - - enum class EvictDataResult : int8_t - { - NO_DATA_EVICTED, - DATA_EVICTED, - CANT_EVICT, - BUFFER_FULL, - }; - - // Evicts data up to aPlaybackTime. aThreshold is used to - // bound the data being evicted. It will not evict more than aThreshold - // bytes. aBufferStartTime contains the new start time of the data after the - // eviction. - virtual EvictDataResult - EvictData(media::TimeUnit aPlaybackTime, - uint32_t aThreshold, - media::TimeUnit* aBufferStartTime) = 0; - - // Evicts data up to aTime. - virtual void EvictBefore(media::TimeUnit aTime) = 0; - - // Returns the buffered range currently managed. - // This may be called on any thread. - // Buffered must conform to http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered - virtual media::TimeIntervals Buffered() = 0; - - // Return the size of the data managed by this SourceBufferContentManager. - virtual int64_t GetSize() = 0; - - // Indicate that the MediaSource parent object got into "ended" state. - virtual void Ended() = 0; - - // The parent SourceBuffer is about to be destroyed. - virtual void Detach() = 0; - - // Current state as per Segment Parser Loop Algorithm - // http://w3c.github.io/media-source/index.html#sourcebuffer-segment-parser-loop - enum class AppendState : int32_t - { - WAITING_FOR_SEGMENT, - PARSING_INIT_SEGMENT, - PARSING_MEDIA_SEGMENT, - }; - - virtual AppendState GetAppendState() - { - return AppendState::WAITING_FOR_SEGMENT; - } - - virtual void SetGroupStartTimestamp(const media::TimeUnit& aGroupStartTimestamp) {} - virtual void RestartGroupStartTimestamp() {} - virtual media::TimeUnit GroupEndTimestamp() = 0; - -protected: - virtual ~SourceBufferContentManager() { } -}; - -} // namespace mozilla - -#endif /* MOZILLA_SOURCEBUFFERCONTENTMANAGER_H_ */ diff --git a/dom/media/mediasource/SourceBufferList.cpp b/dom/media/mediasource/SourceBufferList.cpp index 58591843d5..7a2bccf71a 100644 --- a/dom/media/mediasource/SourceBufferList.cpp +++ b/dom/media/mediasource/SourceBufferList.cpp @@ -130,16 +130,6 @@ SourceBufferList::RangeRemoval(double aStart, double aEnd) } } -void -SourceBufferList::Evict(double aStart, double aEnd) -{ - MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG("Evict(aStart=%f, aEnd=%f)", aStart, aEnd); - for (uint32_t i = 0; i < mSourceBuffers.Length(); ++i) { - mSourceBuffers[i]->Evict(aStart, aEnd); - } -} - void SourceBufferList::Ended() { diff --git a/dom/media/mediasource/SourceBufferList.h b/dom/media/mediasource/SourceBufferList.h index 1f70f3fd97..d144fc7573 100644 --- a/dom/media/mediasource/SourceBufferList.h +++ b/dom/media/mediasource/SourceBufferList.h @@ -75,9 +75,6 @@ public: // Mark all SourceBuffers input buffers as ended. void Ended(); - // Evicts data for the given time range from each SourceBuffer in the list. - void Evict(double aStart, double aEnd); - // Returns the highest end time of any of the Sourcebuffers. double GetHighestBufferedEndTime(); diff --git a/dom/media/mediasource/SourceBufferResource.cpp b/dom/media/mediasource/SourceBufferResource.cpp index 32c4230e66..6992049ba1 100644 --- a/dom/media/mediasource/SourceBufferResource.cpp +++ b/dom/media/mediasource/SourceBufferResource.cpp @@ -134,7 +134,7 @@ SourceBufferResource::ReadFromCache(char* aBuffer, int64_t aOffset, uint32_t aCo } uint32_t -SourceBufferResource::EvictData(uint64_t aPlaybackOffset, uint32_t aThreshold, +SourceBufferResource::EvictData(uint64_t aPlaybackOffset, int64_t aThreshold, ErrorResult& aRv) { SBR_DEBUG("EvictData(aPlaybackOffset=%llu," diff --git a/dom/media/mediasource/SourceBufferResource.h b/dom/media/mediasource/SourceBufferResource.h index 32253c1405..ddf20b33e5 100644 --- a/dom/media/mediasource/SourceBufferResource.h +++ b/dom/media/mediasource/SourceBufferResource.h @@ -111,9 +111,9 @@ public: ReentrantMonitorAutoEnter mon(mMonitor); return mEnded; } - // Remove data from resource if it holds more than the threshold - // number of bytes. Returns amount evicted. - uint32_t EvictData(uint64_t aPlaybackOffset, uint32_t aThreshold, + // Remove data from resource if it holds more than the threshold reduced by + // the given number of bytes. Returns amount evicted. + uint32_t EvictData(uint64_t aPlaybackOffset, int64_t aThresholdReduct, ErrorResult& aRv); // Remove data from resource before the given offset. diff --git a/dom/media/mediasource/SourceBufferTask.h b/dom/media/mediasource/SourceBufferTask.h new file mode 100644 index 0000000000..db42774a0d --- /dev/null +++ b/dom/media/mediasource/SourceBufferTask.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_SOURCEBUFFERTASK_H_ +#define MOZILLA_SOURCEBUFFERTASK_H_ + +#include "mozilla/MozPromise.h" +#include "mozilla/Pair.h" +#include "mozilla/RefPtr.h" +#include "SourceBufferAttributes.h" +#include "TimeUnits.h" + +namespace mozilla { + +class SourceBufferTask { +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SourceBufferTask); + enum class Type { + AppendBuffer, + Abort, + Reset, + RangeRemoval, + EvictData + }; + + typedef Pair AppendBufferResult; + typedef MozPromise AppendPromise; + typedef MozPromise RangeRemovalPromise; + + virtual Type GetType() const = 0; + + template + ReturnType* As() + { + MOZ_ASSERT(this->GetType() == ReturnType::sType); + return static_cast(this); + } + +protected: + virtual ~SourceBufferTask() {} +}; + +class AppendBufferTask : public SourceBufferTask { +public: + AppendBufferTask(MediaByteBuffer* aData, + SourceBufferAttributes aAttributes) + : mBuffer(aData) + , mAttributes(aAttributes) + {} + + static const Type sType = Type::AppendBuffer; + Type GetType() const override { return Type::AppendBuffer; } + + RefPtr mBuffer; + SourceBufferAttributes mAttributes; + MozPromiseHolder mPromise; +}; + +class AbortTask : public SourceBufferTask { +public: + static const Type sType = Type::Abort; + Type GetType() const override { return Type::Abort; } +}; + +class ResetTask : public SourceBufferTask { +public: + static const Type sType = Type::Reset; + Type GetType() const override { return Type::Reset; } +}; + +class RangeRemovalTask : public SourceBufferTask { +public: + explicit RangeRemovalTask(const media::TimeInterval& aRange) + : mRange(aRange) + {} + + static const Type sType = Type::RangeRemoval; + Type GetType() const override { return Type::RangeRemoval; } + + media::TimeInterval mRange; + MozPromiseHolder mPromise; +}; + +class EvictDataTask : public SourceBufferTask { +public: + EvictDataTask(const media::TimeUnit& aPlaybackTime, int64_t aSizetoEvict) + : mPlaybackTime(aPlaybackTime) + , mSizeToEvict(aSizetoEvict) + {} + + static const Type sType = Type::EvictData; + Type GetType() const override { return Type::EvictData; } + + media::TimeUnit mPlaybackTime; + int64_t mSizeToEvict; +}; + +} // end mozilla namespace + +#endif \ No newline at end of file diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index 47092b5522..aef55fc0fe 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -13,6 +13,7 @@ #include "SourceBufferResource.h" #include "SourceBuffer.h" #include "WebMDemuxer.h" +#include "SourceBufferTask.h" #ifdef MOZ_FMP4 #include "MP4Demuxer.h" @@ -38,16 +39,17 @@ using dom::SourceBufferAppendMode; using media::TimeUnit; using media::TimeInterval; using media::TimeIntervals; +typedef SourceBufferTask::AppendBufferResult AppendBufferResult; static const char* -AppendStateToStr(TrackBuffersManager::AppendState aState) +AppendStateToStr(SourceBufferAttributes::AppendState aState) { switch (aState) { - case TrackBuffersManager::AppendState::WAITING_FOR_SEGMENT: + case SourceBufferAttributes::AppendState::WAITING_FOR_SEGMENT: return "WAITING_FOR_SEGMENT"; - case TrackBuffersManager::AppendState::PARSING_INIT_SEGMENT: + case SourceBufferAttributes::AppendState::PARSING_INIT_SEGMENT: return "PARSING_INIT_SEGMENT"; - case TrackBuffersManager::AppendState::PARSING_MEDIA_SEGMENT: + case SourceBufferAttributes::AppendState::PARSING_MEDIA_SEGMENT: return "PARSING_MEDIA_SEGMENT"; default: return "IMPOSSIBLE"; @@ -56,11 +58,9 @@ AppendStateToStr(TrackBuffersManager::AppendState aState) static Atomic sStreamSourceID(0u); -TrackBuffersManager::TrackBuffersManager(dom::SourceBufferAttributes* aAttributes, - MediaSourceDecoder* aParentDecoder, +TrackBuffersManager::TrackBuffersManager(MediaSourceDecoder* aParentDecoder, const nsACString& aType) : mInputBuffer(new MediaByteBuffer) - , mAppendState(AppendState::WAITING_FOR_SEGMENT) , mBufferFull(false) , mFirstInitializationSegmentReceived(false) , mNewMediaSegmentStarted(false) @@ -69,96 +69,198 @@ TrackBuffersManager::TrackBuffersManager(dom::SourceBufferAttributes* aAttribute , mParser(ContainerParser::CreateForMIMEType(aType)) , mProcessedInput(0) , mTaskQueue(aParentDecoder->GetDemuxer()->GetTaskQueue()) - , mSourceBufferAttributes(aAttributes) , mParentDecoder(new nsMainThreadPtrHolder(aParentDecoder, false /* strict */)) - , mEvictionThreshold(Preferences::GetUint("media.mediasource.eviction_threshold", - 100 * (1 << 20))) + , mEnded(false) + , mDetached(false) + , mVideoEvictionThreshold(Preferences::GetUint("media.mediasource.eviction_threshold.video", + 100 * 1024 * 1024)) + , mAudioEvictionThreshold(Preferences::GetUint("media.mediasource.eviction_threshold.audio", + 30 * 1024 * 1024)) , mEvictionOccurred(false) , mMonitor("TrackBuffersManager") - , mAppendRunning(false) { MOZ_ASSERT(NS_IsMainThread(), "Must be instantiated on the main thread"); } TrackBuffersManager::~TrackBuffersManager() { + CancelAllTasks(); ShutdownDemuxers(); } -bool +RefPtr TrackBuffersManager::AppendData(MediaByteBuffer* aData, - TimeUnit aTimestampOffset) + const SourceBufferAttributes& aAttributes) { MOZ_ASSERT(NS_IsMainThread()); MSE_DEBUG("Appending %lld bytes", aData->Length()); mEnded = false; - nsCOMPtr task = - NS_NewRunnableMethodWithArg( - this, &TrackBuffersManager::AppendIncomingBuffer, - IncomingBuffer(aData, aTimestampOffset)); - GetTaskQueue()->Dispatch(task.forget()); - return true; -} -void -TrackBuffersManager::AppendIncomingBuffer(IncomingBuffer aData) -{ - MOZ_ASSERT(OnTaskQueue()); - mIncomingBuffers.AppendElement(aData); + RefPtr buffer = aData; + + return InvokeAsync(GetTaskQueue(), this, + __func__, &TrackBuffersManager::DoAppendData, + buffer, aAttributes); } RefPtr -TrackBuffersManager::BufferAppend() +TrackBuffersManager::DoAppendData(RefPtr aData, + SourceBufferAttributes aAttributes) { - MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG(""); + RefPtr task = new AppendBufferTask(aData, aAttributes); + RefPtr p = task->mPromise.Ensure(__func__); + mQueue.Push(task); - mAppendRunning = true; - return InvokeAsync(GetTaskQueue(), this, - __func__, &TrackBuffersManager::InitSegmentParserLoop); + ProcessTasks(); + + return p; +} + +void +TrackBuffersManager::ProcessTasks() +{ + typedef SourceBufferTask::Type Type; + + if (mDetached) { + return; + } + + if (OnTaskQueue()) { + if (mCurrentTask) { + // Already have a task pending. ProcessTask will be scheduled once the + // current task complete. + return; + } + RefPtr task = mQueue.Pop(); + if (!task) { + // nothing to do. + return; + } + switch (task->GetType()) { + case Type::AppendBuffer: + mCurrentTask = task; + if (!mInputBuffer) { + mInputBuffer = task->As()->mBuffer; + } else if (!mInputBuffer->AppendElements(*task->As()->mBuffer, fallible)) { + RejectAppend(NS_ERROR_OUT_OF_MEMORY, __func__); + return; + } + mSourceBufferAttributes = + MakeUnique(task->As()->mAttributes); + mAppendWindow = + TimeInterval(TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowStart()), + TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowEnd())); + ScheduleSegmentParserLoop(); + break; + case Type::RangeRemoval: + { + bool rv = CodedFrameRemoval(task->As()->mRange); + task->As()->mPromise.Resolve(rv, __func__); + break; + } + case Type::EvictData: + DoEvictData(task->As()->mPlaybackTime, + task->As()->mSizeToEvict); + break; + case Type::Abort: + // not handled yet, and probably never. + break; + case Type::Reset: + CompleteResetParserState(); + break; + default: + NS_WARNING("Invalid Task"); + } + } + nsCOMPtr task = + NS_NewRunnableMethod(this, &TrackBuffersManager::ProcessTasks); + GetTaskQueue()->Dispatch(task.forget()); +} + +// A PromiseHolder will assert upon destruction if it has a pending promise +// that hasn't been completed. It is possible that a task didn't get processed +// due to the owning SourceBuffer having shutdown. +// We resolve/reject all pending promises and remove all pending tasks from the +// queue. +void +TrackBuffersManager::CancelAllTasks() +{ + typedef SourceBufferTask::Type Type; + MOZ_DIAGNOSTIC_ASSERT(mDetached); + + if (mCurrentTask) { + mQueue.Push(mCurrentTask); + mCurrentTask = nullptr; + } + + RefPtr task; + while ((task = mQueue.Pop())) { + switch (task->GetType()) { + case Type::AppendBuffer: + task->As()->mPromise.RejectIfExists(NS_ERROR_ABORT, __func__); + break; + case Type::RangeRemoval: + task->As()->mPromise.ResolveIfExists(false, __func__); + break; + case Type::EvictData: + break; + case Type::Abort: + // not handled yet, and probably never. + break; + case Type::Reset: + break; + default: + NS_WARNING("Invalid Task"); + } + } } // The MSE spec requires that we abort the current SegmentParserLoop // which is then followed by a call to ResetParserState. -// However due to our asynchronous design this causes inherent difficulities. -// As the spec behaviour is non deterministic anyway, we instead wait until the -// current AppendData has completed its run. +// However due to our asynchronous design this causes inherent difficulties. +// As the spec behaviour is non deterministic anyway, we instead process all +// pending frames found in the input buffer. void TrackBuffersManager::AbortAppendData() { MOZ_ASSERT(NS_IsMainThread()); MSE_DEBUG(""); - MonitorAutoLock mon(mMonitor); - while (mAppendRunning) { - mon.Wait(); - } + RefPtr task = new AbortTask(); + mQueue.Push(task); + ProcessTasks(); } void -TrackBuffersManager::ResetParserState() +TrackBuffersManager::ResetParserState(SourceBufferAttributes& aAttributes) { MOZ_ASSERT(NS_IsMainThread()); - MOZ_RELEASE_ASSERT(!mAppendRunning, "Append is running, abort must have been called"); MSE_DEBUG(""); + // Spec states: // 1. If the append state equals PARSING_MEDIA_SEGMENT and the input buffer contains some complete coded frames, then run the coded frame processing algorithm until all of these complete coded frames have been processed. - // SourceBuffer.abort() has ensured that all complete coded frames have been - // processed. As such, we don't need to check for the value of mAppendState. - nsCOMPtr task = - NS_NewRunnableMethod(this, &TrackBuffersManager::CompleteResetParserState); - GetTaskQueue()->Dispatch(task.forget()); + // However, we will wait until all coded frames have been processed regardless + // of the value of append state. + RefPtr task = new ResetTask(); + mQueue.Push(task); + ProcessTasks(); - // 7. Set append state to WAITING_FOR_SEGMENT. - SetAppendState(AppendState::WAITING_FOR_SEGMENT); + // ResetParserState has some synchronous steps that much be performed now. + // The remaining steps will be performed once the ResetTask gets executed. + + // 6. If the mode attribute equals "sequence", then set the group start timestamp to the group end timestamp + if (aAttributes.GetAppendMode() == SourceBufferAppendMode::Sequence) { + aAttributes.SetGroupStartTimestamp(aAttributes.GetGroupEndTimestamp()); + } + // 8. Set append state to WAITING_FOR_SEGMENT. + aAttributes.SetAppendState(AppendState::WAITING_FOR_SEGMENT); } RefPtr TrackBuffersManager::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) { MOZ_ASSERT(NS_IsMainThread()); - MOZ_RELEASE_ASSERT(!mAppendRunning, "Append is running"); MSE_DEBUG("From %.2f to %.2f", aStart.ToSeconds(), aEnd.ToSeconds()); mEnded = false; @@ -169,14 +271,19 @@ TrackBuffersManager::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) } TrackBuffersManager::EvictDataResult -TrackBuffersManager::EvictData(TimeUnit aPlaybackTime, - uint32_t aThreshold, - TimeUnit* aBufferStartTime) +TrackBuffersManager::EvictData(const TimeUnit& aPlaybackTime, int64_t aSize) { MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG(""); - int64_t toEvict = GetSize() - aThreshold; + if (aSize > EvictionThreshold()) { + // We're adding more data than we can hold. + return EvictDataResult::BUFFER_FULL; + } + const int64_t toEvict = GetSize() + aSize - EvictionThreshold(); + + MSE_DEBUG("buffered=%lldkb, eviction threshold=%ukb, evict=%lldkb", + GetSize() / 1024, EvictionThreshold() / 1024, toEvict / 1024); + if (toEvict <= 0) { return EvictDataResult::NO_DATA_EVICTED; } @@ -191,28 +298,13 @@ TrackBuffersManager::EvictData(TimeUnit aPlaybackTime, MSE_DEBUG("Reaching our size limit, schedule eviction of %lld bytes", toEvict); - nsCOMPtr task = - NS_NewRunnableMethodWithArgs( - this, &TrackBuffersManager::DoEvictData, - aPlaybackTime, toEvict); - GetTaskQueue()->Dispatch(task.forget()); + RefPtr task = new EvictDataTask(aPlaybackTime, toEvict); + mQueue.Push(task); + ProcessTasks(); return EvictDataResult::NO_DATA_EVICTED; } -void -TrackBuffersManager::EvictBefore(TimeUnit aTime) -{ - MOZ_ASSERT(NS_IsMainThread()); - MSE_DEBUG(""); - - nsCOMPtr task = - NS_NewRunnableMethodWithArg( - this, &TrackBuffersManager::CodedFrameRemoval, - TimeInterval(TimeUnit::FromSeconds(0), aTime)); - GetTaskQueue()->Dispatch(task.forget()); -} - TimeIntervals TrackBuffersManager::Buffered() { @@ -253,7 +345,7 @@ TrackBuffersManager::Buffered() } int64_t -TrackBuffersManager::GetSize() +TrackBuffersManager::GetSize() const { return mSizeSourceBuffer; } @@ -269,6 +361,7 @@ TrackBuffersManager::Detach() { MOZ_ASSERT(NS_IsMainThread()); MSE_DEBUG(""); + mDetached = true; } void @@ -277,6 +370,10 @@ TrackBuffersManager::CompleteResetParserState() MOZ_ASSERT(OnTaskQueue()); MSE_DEBUG(""); + // We shouldn't change mInputDemuxer while a demuxer init/reset request is + // being processed. See bug 1239983. + MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists(), "Previous AppendBuffer didn't complete"); + for (auto& track : GetTracksList()) { // 2. Unset the last decode timestamp on all track buffers. // 3. Unset the last frame duration on all track buffers. @@ -289,13 +386,7 @@ TrackBuffersManager::CompleteResetParserState() track->mQueuedSamples.Clear(); } - // 6. If the mode attribute equals "sequence", then set the group start timestamp to the group end timestamp - if (mSourceBufferAttributes->GetAppendMode() == SourceBufferAppendMode::Sequence) { - mGroupStartTimestamp = Some(mGroupEndTimestamp); - } - // 7. Remove all bytes from the input buffer. - mIncomingBuffers.Clear(); mInputBuffer = nullptr; if (mCurrentInputBuffer) { mCurrentInputBuffer->EvictAll(); @@ -318,17 +409,20 @@ TrackBuffersManager::CompleteResetParserState() mInputBuffer->AppendElements(*mInitData); } RecreateParser(true); +} - // 8. Set append state to WAITING_FOR_SEGMENT. - SetAppendState(AppendState::WAITING_FOR_SEGMENT); - - // Reject our promise immediately. - mAppendPromise.RejectIfExists(NS_ERROR_ABORT, __func__); +int64_t +TrackBuffersManager::EvictionThreshold() const +{ + if (HasVideo()) { + return mVideoEvictionThreshold; + } + return mAudioEvictionThreshold; } void TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime, - uint32_t aSizeToEvict) + int64_t aSizeToEvict) { MOZ_ASSERT(OnTaskQueue()); @@ -340,7 +434,7 @@ TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime, TimeUnit lowerLimit = std::min(track.mNextSampleTime, aPlaybackTime); uint32_t lastKeyFrameIndex = 0; int64_t toEvict = aSizeToEvict; - uint32_t partialEvict = 0; + int64_t partialEvict = 0; for (uint32_t i = 0; i < buffer.Length(); i++) { const auto& frame = buffer[i]; if (frame->mKeyframe) { @@ -357,10 +451,10 @@ TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime, partialEvict += frame->ComputedSizeOfIncludingThis(); } - int64_t finalSize = mSizeSourceBuffer - aSizeToEvict; + const int64_t finalSize = mSizeSourceBuffer - aSizeToEvict; if (lastKeyFrameIndex > 0) { - MSE_DEBUG("Step1. Evicting %u bytes prior currentTime", + MSE_DEBUG("Step1. Evicting %lld bytes prior currentTime", aSizeToEvict - toEvict); CodedFrameRemoval( TimeInterval(TimeUnit::FromMicroseconds(0), @@ -390,8 +484,8 @@ TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime, toEvict -= frame->ComputedSizeOfIncludingThis(); } if (evictedFramesStartIndex < buffer.Length()) { - MSE_DEBUG("Step2. Evicting %u bytes from trailing data", - mSizeSourceBuffer - finalSize); + MSE_DEBUG("Step2. Evicting %lld bytes from trailing data", + mSizeSourceBuffer - finalSize - toEvict); CodedFrameRemoval( TimeInterval(TimeUnit::FromMicroseconds(buffer[evictedFramesStartIndex]->mTime), TimeUnit::FromInfinity())); @@ -402,8 +496,12 @@ RefPtr TrackBuffersManager::CodedFrameRemovalWithPromise(TimeInterval aInterval) { MOZ_ASSERT(OnTaskQueue()); - bool rv = CodedFrameRemoval(aInterval); - return RangeRemovalPromise::CreateAndResolve(rv, __func__); + + RefPtr task = new RangeRemovalTask(aInterval); + RefPtr p = task->mPromise.Ensure(__func__); + mQueue.Push(task); + ProcessTasks(); + return p; } bool @@ -477,7 +575,7 @@ TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval) mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer; // 4. If buffer full flag equals true and this object is ready to accept more bytes, then set the buffer full flag to false. - if (mBufferFull && mSizeSourceBuffer < mEvictionThreshold) { + if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) { mBufferFull = false; } mEvictionOccurred = true; @@ -503,44 +601,6 @@ TrackBuffersManager::UpdateBufferedRanges() DumpTimeRanges(mAudioTracks.mBufferedRanges).get()); } #endif - - mOfficialGroupEndTimestamp = mGroupEndTimestamp; -} - -RefPtr -TrackBuffersManager::InitSegmentParserLoop() -{ - MOZ_ASSERT(OnTaskQueue()); - MOZ_RELEASE_ASSERT(mAppendPromise.IsEmpty()); - MSE_DEBUG(""); - - RefPtr p = mAppendPromise.Ensure(__func__); - - AppendIncomingBuffers(); - SegmentParserLoop(); - - return p; -} - -void -TrackBuffersManager::AppendIncomingBuffers() -{ - MOZ_ASSERT(OnTaskQueue()); - MonitorAutoLock mon(mMonitor); - for (auto& incomingBuffer : mIncomingBuffers) { - if (!mInputBuffer) { - mInputBuffer = incomingBuffer.first(); - } else if (!mInputBuffer->AppendElements(*incomingBuffer.first(), fallible)) { - RejectAppend(NS_ERROR_OUT_OF_MEMORY, __func__); - } - mTimestampOffset = incomingBuffer.second(); - mLastTimestampOffset = mTimestampOffset; - } - mIncomingBuffers.Clear(); - - mAppendWindow = - TimeInterval(TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowStart()), - TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowEnd())); } void @@ -567,7 +627,7 @@ TrackBuffersManager::SegmentParserLoop() // 4. If the append state equals WAITING_FOR_SEGMENT, then run the following // steps: - if (mAppendState == AppendState::WAITING_FOR_SEGMENT) { + if (mSourceBufferAttributes->GetAppendState() == AppendState::WAITING_FOR_SEGMENT) { if (mParser->IsInitSegmentPresent(mInputBuffer)) { SetAppendState(AppendState::PARSING_INIT_SEGMENT); if (mFirstInitializationSegmentReceived) { @@ -594,7 +654,7 @@ TrackBuffersManager::SegmentParserLoop() // 5. If the append state equals PARSING_INIT_SEGMENT, then run the // following steps: - if (mAppendState == AppendState::PARSING_INIT_SEGMENT) { + if (mSourceBufferAttributes->GetAppendState() == AppendState::PARSING_INIT_SEGMENT) { if (mParser->InitSegmentRange().IsEmpty()) { mInputBuffer = nullptr; NeedMoreData(); @@ -603,7 +663,7 @@ TrackBuffersManager::SegmentParserLoop() InitializationSegmentReceived(); return; } - if (mAppendState == AppendState::PARSING_MEDIA_SEGMENT) { + if (mSourceBufferAttributes->GetAppendState() == AppendState::PARSING_MEDIA_SEGMENT) { // 1. If the first initialization segment received flag is false, then run the append error algorithm with the decode error parameter set to true and abort this algorithm. if (!mFirstInitializationSegmentReceived) { RejectAppend(NS_ERROR_FAILURE, __func__); @@ -630,9 +690,6 @@ TrackBuffersManager::SegmentParserLoop() mPendingInputBuffer = nullptr; } mNewMediaSegmentStarted = false; - if (newData) { - mLastParsedEndTime = Some(TimeUnit::FromMicroseconds(end)); - } } else { // We don't have any data to demux yet, stash aside the data. // This also handles the case: @@ -673,32 +730,44 @@ void TrackBuffersManager::NeedMoreData() { MSE_DEBUG(""); - RestoreCachedVariables(); - mAppendRunning = false; - { - // Wake-up any pending Abort() - MonitorAutoLock mon(mMonitor); - mon.NotifyAll(); + if (mDetached) { + // We've been detached. + return; } - mAppendPromise.ResolveIfExists(mActiveTrack, __func__); + MOZ_DIAGNOSTIC_ASSERT(mCurrentTask && mCurrentTask->GetType() == SourceBufferTask::Type::AppendBuffer); + MOZ_DIAGNOSTIC_ASSERT(mSourceBufferAttributes); + + mCurrentTask->As()->mPromise.Resolve( + SourceBufferTask::AppendBufferResult(mActiveTrack, + *mSourceBufferAttributes), + __func__); + mSourceBufferAttributes = nullptr; + mCurrentTask = nullptr; + ProcessTasks(); } void TrackBuffersManager::RejectAppend(nsresult aRejectValue, const char* aName) { MSE_DEBUG("rv=%d", aRejectValue); - mAppendRunning = false; - { - // Wake-up any pending Abort() - MonitorAutoLock mon(mMonitor); - mon.NotifyAll(); + if (mDetached) { + // We've been detached. + return; } - mAppendPromise.RejectIfExists(aRejectValue, aName); + MOZ_DIAGNOSTIC_ASSERT(mCurrentTask && mCurrentTask->GetType() == SourceBufferTask::Type::AppendBuffer); + + mCurrentTask->As()->mPromise.Reject(aRejectValue, __func__); + mSourceBufferAttributes = nullptr; + mCurrentTask = nullptr; + ProcessTasks(); } void TrackBuffersManager::ScheduleSegmentParserLoop() { + if (mDetached) { + return; + } nsCOMPtr task = NS_NewRunnableMethod(this, &TrackBuffersManager::SegmentParserLoop); GetTaskQueue()->Dispatch(task.forget()); @@ -715,6 +784,9 @@ TrackBuffersManager::ShutdownDemuxers() mAudioTracks.mDemuxer->BreakCycles(); mAudioTracks.mDemuxer = nullptr; } + // We shouldn't change mInputDemuxer while a demuxer init/reset request is + // being processed. See bug 1239983. + MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists()); mInputDemuxer = nullptr; mLastParsedEndTime.reset(); } @@ -724,13 +796,15 @@ TrackBuffersManager::CreateDemuxerforMIMEType() { ShutdownDemuxers(); - if (mType.LowerCaseEqualsLiteral("video/webm") || mType.LowerCaseEqualsLiteral("audio/webm")) { + if (mType.LowerCaseEqualsLiteral("video/webm") || + mType.LowerCaseEqualsLiteral("audio/webm")) { mInputDemuxer = new WebMDemuxer(mCurrentInputBuffer, true /* IsMediaSource*/ ); return; } #ifdef MOZ_FMP4 - if (mType.LowerCaseEqualsLiteral("video/mp4") || mType.LowerCaseEqualsLiteral("audio/mp4")) { + if (mType.LowerCaseEqualsLiteral("video/mp4") || + mType.LowerCaseEqualsLiteral("audio/mp4")) { mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer); return; } @@ -767,19 +841,24 @@ TrackBuffersManager::OnDemuxerResetDone(nsresult) { MOZ_ASSERT(OnTaskQueue()); mDemuxerInitRequest.Complete(); + // mInputDemuxer shouldn't have been destroyed while a demuxer init/reset + // request was being processed. See bug 1239983. + MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer); // Recreate track demuxers. uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack); if (numVideos) { // We currently only handle the first video track. - mVideoTracks.mDemuxer = mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); + mVideoTracks.mDemuxer = + mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); MOZ_ASSERT(mVideoTracks.mDemuxer); } uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack); if (numAudios) { // We currently only handle the first audio track. - mAudioTracks.mDemuxer = mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); + mAudioTracks.mDemuxer = + mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); MOZ_ASSERT(mAudioTracks.mDemuxer); } @@ -791,8 +870,6 @@ TrackBuffersManager::OnDemuxerResetDone(nsresult) mProcessedInput += mPendingInputBuffer->Length(); } - mLastParsedEndTime.reset(); - SegmentParserLoop(); } @@ -847,6 +924,8 @@ void TrackBuffersManager::OnDemuxerInitDone(nsresult) { MOZ_ASSERT(OnTaskQueue()); + MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer, "mInputDemuxer has been destroyed"); + mDemuxerInitRequest.Complete(); MediaInfo info; @@ -854,7 +933,8 @@ TrackBuffersManager::OnDemuxerInitDone(nsresult) uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack); if (numVideos) { // We currently only handle the first video track. - mVideoTracks.mDemuxer = mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); + mVideoTracks.mDemuxer = + mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0); MOZ_ASSERT(mVideoTracks.mDemuxer); info.mVideo = *mVideoTracks.mDemuxer->GetInfo()->GetAsVideoInfo(); info.mVideo.mTrackId = 2; @@ -863,7 +943,8 @@ TrackBuffersManager::OnDemuxerInitDone(nsresult) uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack); if (numAudios) { // We currently only handle the first audio track. - mAudioTracks.mDemuxer = mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); + mAudioTracks.mDemuxer = + mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0); MOZ_ASSERT(mAudioTracks.mDemuxer); info.mAudio = *mAudioTracks.mDemuxer->GetInfo()->GetAsAudioInfo(); info.mAudio.mTrackId = 1; @@ -1004,6 +1085,7 @@ TrackBuffersManager::OnDemuxerInitDone(nsresult) (info.mAudio.mChannels != mAudioTracks.mInfo->GetAsAudioInfo()->mChannels || info.mAudio.mRate != mAudioTracks.mInfo->GetAsAudioInfo()->mRate)) { RejectAppend(NS_ERROR_FAILURE, __func__); + return; } mAudioTracks.mLastInfo = new SharedTrackInfo(info.mAudio, streamID); mVideoTracks.mLastInfo = new SharedTrackInfo(info.mVideo, streamID); @@ -1211,7 +1293,7 @@ TrackBuffersManager::CompleteCodedFrameProcessing() // Return to step 6.4 of Segment Parser Loop algorithm // 4. If this SourceBuffer is full and cannot accept more media data, then set the buffer full flag to true. - if (mSizeSourceBuffer >= mEvictionThreshold) { + if (mSizeSourceBuffer >= EvictionThreshold()) { mBufferFull = true; mEvictionOccurred = false; } @@ -1222,6 +1304,9 @@ TrackBuffersManager::CompleteCodedFrameProcessing() return; } + mLastParsedEndTime = Some(std::max(mAudioTracks.mLastParsedEndTime, + mVideoTracks.mLastParsedEndTime)); + // 6. Remove the media segment bytes from the beginning of the input buffer. // Clear our demuxer from any already processed data. // As we have handled a complete media segment, it is safe to evict all data @@ -1253,12 +1338,14 @@ void TrackBuffersManager::CheckSequenceDiscontinuity(const TimeUnit& aPresentationTime) { if (mSourceBufferAttributes->GetAppendMode() == SourceBufferAppendMode::Sequence && - mGroupStartTimestamp.isSome()) { - mTimestampOffset = mGroupStartTimestamp.ref() - aPresentationTime; - mGroupEndTimestamp = mGroupStartTimestamp.ref(); + mSourceBufferAttributes->HaveGroupStartTimestamp()) { + mSourceBufferAttributes->SetTimestampOffset( + mSourceBufferAttributes->GetGroupStartTimestamp() - aPresentationTime); + mSourceBufferAttributes->SetGroupEndTimestamp( + mSourceBufferAttributes->GetGroupStartTimestamp()); mVideoTracks.mNeedRandomAccessPoint = true; mAudioTracks.mNeedRandomAccessPoint = true; - mGroupStartTimestamp.reset(); + mSourceBufferAttributes->ResetGroupStartTimestamp(); } } @@ -1302,6 +1389,10 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) // a frame be ignored due to the target window. bool needDiscontinuityCheck = true; + if (aSamples.Length()) { + aTrackData.mLastParsedEndTime = TimeUnit(); + } + for (auto& sample : aSamples) { SAMPLE_DEBUG("Processing %s frame(pts:%lld end:%lld, dts:%lld, duration:%lld, " "kf:%d)", @@ -1312,6 +1403,12 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) sample->mDuration, sample->mKeyframe); + const TimeUnit sampleEndTime = + TimeUnit::FromMicroseconds(sample->GetEndTime()); + if (sampleEndTime > aTrackData.mLastParsedEndTime) { + aTrackData.mLastParsedEndTime = sampleEndTime; + } + // We perform step 10 right away as we can't do anything should a keyframe // be needed until we have one. @@ -1339,14 +1436,14 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) TimeInterval sampleInterval = mSourceBufferAttributes->mGenerateTimestamps - ? TimeInterval(mTimestampOffset, - mTimestampOffset + TimeUnit::FromMicroseconds(sample->mDuration)) - : TimeInterval(TimeUnit::FromMicroseconds(sample->mTime) + mTimestampOffset, - TimeUnit::FromMicroseconds(sample->GetEndTime()) + mTimestampOffset); + ? TimeInterval(mSourceBufferAttributes->GetTimestampOffset(), + mSourceBufferAttributes->GetTimestampOffset() + TimeUnit::FromMicroseconds(sample->mDuration)) + : TimeInterval(TimeUnit::FromMicroseconds(sample->mTime) + mSourceBufferAttributes->GetTimestampOffset(), + TimeUnit::FromMicroseconds(sample->GetEndTime()) + mSourceBufferAttributes->GetTimestampOffset()); TimeUnit decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps - ? mTimestampOffset - : TimeUnit::FromMicroseconds(sample->mTimecode) + mTimestampOffset; + ? mSourceBufferAttributes->GetTimestampOffset() + : TimeUnit::FromMicroseconds(sample->mTimecode) + mSourceBufferAttributes->GetTimestampOffset(); // 6. If last decode timestamp for track buffer is set and decode timestamp is less than last decode timestamp: // OR @@ -1361,12 +1458,13 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) // 1a. If mode equals "segments": if (appendMode == SourceBufferAppendMode::Segments) { // Set group end timestamp to presentation timestamp. - mGroupEndTimestamp = sampleInterval.mStart; + mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mStart); } // 1b. If mode equals "sequence": if (appendMode == SourceBufferAppendMode::Sequence) { // Set group start timestamp equal to the group end timestamp. - mGroupStartTimestamp = Some(mGroupEndTimestamp); + mSourceBufferAttributes->SetGroupStartTimestamp( + mSourceBufferAttributes->GetGroupEndTimestamp()); } for (auto& track : GetTracksList()) { // 2. Unset the last decode timestamp on all track buffers. @@ -1387,18 +1485,18 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) continue; } if (appendMode == SourceBufferAppendMode::Sequence) { - // mTimestampOffset was modified during CheckSequenceDiscontinuity. + // mSourceBufferAttributes->GetTimestampOffset() was modified during CheckSequenceDiscontinuity. // We need to update our variables. sampleInterval = mSourceBufferAttributes->mGenerateTimestamps - ? TimeInterval(mTimestampOffset, - mTimestampOffset + TimeUnit::FromMicroseconds(sample->mDuration)) - : TimeInterval(TimeUnit::FromMicroseconds(sample->mTime) + mTimestampOffset, - TimeUnit::FromMicroseconds(sample->GetEndTime()) + mTimestampOffset); + ? TimeInterval(mSourceBufferAttributes->GetTimestampOffset(), + mSourceBufferAttributes->GetTimestampOffset() + TimeUnit::FromMicroseconds(sample->mDuration)) + : TimeInterval(TimeUnit::FromMicroseconds(sample->mTime) + mSourceBufferAttributes->GetTimestampOffset(), + TimeUnit::FromMicroseconds(sample->GetEndTime()) + mSourceBufferAttributes->GetTimestampOffset()); decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps - ? mTimestampOffset - : TimeUnit::FromMicroseconds(sample->mTimecode) + mTimestampOffset; + ? mSourceBufferAttributes->GetTimestampOffset() + : TimeUnit::FromMicroseconds(sample->mTimecode) + mSourceBufferAttributes->GetTimestampOffset(); } trackBuffer.mNeedRandomAccessPoint = false; needDiscontinuityCheck = false; @@ -1452,12 +1550,12 @@ TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) trackBuffer.mHighestEndTimestamp = Some(sampleInterval.mEnd); } // 20. If frame end timestamp is greater than group end timestamp, then set group end timestamp equal to frame end timestamp. - if (sampleInterval.mEnd > mGroupEndTimestamp) { - mGroupEndTimestamp = sampleInterval.mEnd; + if (sampleInterval.mEnd > mSourceBufferAttributes->GetGroupEndTimestamp()) { + mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mEnd); } // 21. If generate timestamps flag equals true, then set timestampOffset equal to frame end timestamp. if (mSourceBufferAttributes->mGenerateTimestamps) { - mTimestampOffset = sampleInterval.mEnd; + mSourceBufferAttributes->SetTimestampOffset(sampleInterval.mEnd); } } @@ -1589,7 +1687,7 @@ TrackBuffersManager::RemoveFrames(const TimeIntervals& aIntervals, { TrackBuffer& data = aTrackData.mBuffers.LastElement(); Maybe firstRemovedIndex; - uint32_t lastRemovedIndex; + uint32_t lastRemovedIndex = 0; // We loop from aStartIndex to avoid removing frames that we inserted earlier // and part of the current coded frame group. This is allows to handle step @@ -1713,56 +1811,11 @@ TrackBuffersManager::GetTracksList() } void -TrackBuffersManager::RestoreCachedVariables() -{ - MOZ_ASSERT(OnTaskQueue()); - if (mTimestampOffset != mLastTimestampOffset) { - mSourceBufferAttributes->SetTimestampOffset(mTimestampOffset); - } -} - -void -TrackBuffersManager::SetAppendState(TrackBuffersManager::AppendState aAppendState) +TrackBuffersManager::SetAppendState(SourceBufferAttributes::AppendState aAppendState) { MSE_DEBUG("AppendState changed from %s to %s", - AppendStateToStr(mAppendState), AppendStateToStr(aAppendState)); - mAppendState = aAppendState; -} - -void -TrackBuffersManager::SetGroupStartTimestamp(const TimeUnit& aGroupStartTimestamp) -{ - if (NS_IsMainThread()) { - nsCOMPtr task = - NS_NewRunnableMethodWithArg( - this, - &TrackBuffersManager::SetGroupStartTimestamp, - aGroupStartTimestamp); - GetTaskQueue()->Dispatch(task.forget()); - return; - } - MOZ_ASSERT(OnTaskQueue()); - mGroupStartTimestamp = Some(aGroupStartTimestamp); -} - -void -TrackBuffersManager::RestartGroupStartTimestamp() -{ - if (NS_IsMainThread()) { - nsCOMPtr task = - NS_NewRunnableMethod(this, &TrackBuffersManager::RestartGroupStartTimestamp); - GetTaskQueue()->Dispatch(task.forget()); - return; - } - MOZ_ASSERT(OnTaskQueue()); - mGroupStartTimestamp = Some(mGroupEndTimestamp); -} - -TimeUnit -TrackBuffersManager::GroupEndTimestamp() -{ - MonitorAutoLock mon(mMonitor); - return mOfficialGroupEndTimestamp; + AppendStateToStr(mSourceBufferAttributes->GetAppendState()), AppendStateToStr(aAppendState)); + mSourceBufferAttributes->SetAppendState(aAppendState); } MediaInfo diff --git a/dom/media/mediasource/TrackBuffersManager.h b/dom/media/mediasource/TrackBuffersManager.h index 34d0203f1e..f82f21158a 100644 --- a/dom/media/mediasource/TrackBuffersManager.h +++ b/dom/media/mediasource/TrackBuffersManager.h @@ -10,13 +10,16 @@ #include "mozilla/Atomics.h" #include "mozilla/Maybe.h" #include "mozilla/Monitor.h" -#include "mozilla/Pair.h" +#include "AutoTaskQueue.h" #include "mozilla/dom/SourceBufferBinding.h" -#include "SourceBufferContentManager.h" +#include "MediaData.h" #include "MediaDataDemuxer.h" #include "MediaSourceDecoder.h" +#include "SourceBufferTask.h" +#include "TimeUnits.h" #include "nsProxyRelease.h" +#include "nsString.h" #include "nsTArray.h" namespace mozilla { @@ -27,56 +30,102 @@ class MediaRawData; class MediaSourceDemuxer; class SourceBufferResource; -namespace dom { - class SourceBufferAttributes; -} - -class TrackBuffersManager : public SourceBufferContentManager { +class SourceBufferTaskQueue { public: - typedef MozPromise CodedFrameProcessingPromise; + SourceBufferTaskQueue() + : mMonitor("SourceBufferTaskQueue") + {} + + void Push(SourceBufferTask* aTask) + { + MonitorAutoLock mon(mMonitor); + mQueue.AppendElement(aTask); + } + + already_AddRefed Pop() + { + MonitorAutoLock mon(mMonitor); + if (!mQueue.Length()) { + return nullptr; + } + RefPtr task = Move(mQueue[0]); + mQueue.RemoveElementAt(0); + return task.forget(); + } + + nsTArray::size_type Length() const + { + MonitorAutoLock mon(mMonitor); + return mQueue.Length(); + } + +private: + mutable Monitor mMonitor; + nsTArray> mQueue; +}; + +class TrackBuffersManager { +public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager); + + enum class EvictDataResult : int8_t + { + NO_DATA_EVICTED, + CANT_EVICT, + BUFFER_FULL, + }; + typedef TrackInfo::TrackType TrackType; typedef MediaData::Type MediaType; typedef nsTArray> TrackBuffer; + typedef SourceBufferTask::AppendPromise AppendPromise; + typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise; - TrackBuffersManager(dom::SourceBufferAttributes* aAttributes, - MediaSourceDecoder* aParentDecoder, + // Interface for SourceBuffer + TrackBuffersManager(MediaSourceDecoder* aParentDecoder, const nsACString& aType); - bool AppendData(MediaByteBuffer* aData, - media::TimeUnit aTimestampOffset) override; + // Queue a task to add data to the end of the input buffer and run the MSE + // Buffer Append Algorithm + // 3.5.5 Buffer Append Algorithm. + // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append + RefPtr AppendData(MediaByteBuffer* aData, + const SourceBufferAttributes& aAttributes); - RefPtr BufferAppend() override; + // Queue a task to abort any pending AppendData. + // Does nothing at this stage. + void AbortAppendData(); - void AbortAppendData() override; - - void ResetParserState() override; + // Queue a task to run MSE Reset Parser State Algorithm. + // 3.5.2 Reset Parser State + void ResetParserState(SourceBufferAttributes& aAttributes); + // Queue a task to run the MSE range removal algorithm. + // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal RefPtr RangeRemoval(media::TimeUnit aStart, - media::TimeUnit aEnd) override; + media::TimeUnit aEnd); - EvictDataResult - EvictData(media::TimeUnit aPlaybackTime, - uint32_t aThreshold, - media::TimeUnit* aBufferStartTime) override; + // Schedule data eviction if necessary as the next call to AppendData will + // add aSize bytes. + // Eviction is done in two steps, first remove data up to aPlaybackTime + // and if still more space is needed remove from the end. + EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, int64_t aSize); - void EvictBefore(media::TimeUnit aTime) override; + // Returns the buffered range currently managed. + // This may be called on any thread. + // Buffered must conform to http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered + media::TimeIntervals Buffered(); - media::TimeIntervals Buffered() override; + // Return the size of the data managed by this SourceBufferContentManager. + int64_t GetSize() const; - int64_t GetSize() override; + // Indicate that the MediaSource parent object got into "ended" state. + void Ended(); - void Ended() override; + // The parent SourceBuffer is about to be destroyed. + void Detach(); - void Detach() override; - - AppendState GetAppendState() override - { - return mAppendState; - } - - void SetGroupStartTimestamp(const media::TimeUnit& aGroupStartTimestamp) override; - void RestartGroupStartTimestamp() override; - media::TimeUnit GroupEndTimestamp() override; + int64_t EvictionThreshold() const; // Interface for MediaSourceDemuxer MediaInfo GetMetadata(); @@ -101,14 +150,16 @@ public: void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes); private: + typedef MozPromise CodedFrameProcessingPromise; + // for MediaSourceDemuxer::GetMozDebugReaderData friend class MediaSourceDemuxer; virtual ~TrackBuffersManager(); // All following functions run on the taskqueue. - RefPtr InitSegmentParserLoop(); + RefPtr DoAppendData(RefPtr aData, + SourceBufferAttributes aAttributes); void ScheduleSegmentParserLoop(); void SegmentParserLoop(); - void AppendIncomingBuffers(); void InitializationSegmentReceived(); void ShutdownDemuxers(); void CreateDemuxerforMIMEType(); @@ -124,7 +175,7 @@ private: RefPtr CodedFrameRemovalWithPromise(media::TimeInterval aInterval); bool CodedFrameRemoval(media::TimeInterval aInterval); - void SetAppendState(AppendState aAppendState); + void SetAppendState(SourceBufferAttributes::AppendState aAppendState); bool HasVideo() const { @@ -135,25 +186,15 @@ private: return mAudioTracks.mNumTracks > 0; } - typedef Pair, media::TimeUnit> IncomingBuffer; - void AppendIncomingBuffer(IncomingBuffer aData); - nsTArray mIncomingBuffers; - // The input buffer as per http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer RefPtr mInputBuffer; - // The current append state as per https://w3c.github.io/media-source/#sourcebuffer-append-state - // Accessed on both the main thread and the task queue. - Atomic mAppendState; // Buffer full flag as per https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag. // Accessed on both the main thread and the task queue. - // TODO: Unused for now. Atomic mBufferFull; bool mFirstInitializationSegmentReceived; // Set to true once a new segment is started. bool mNewMediaSegmentStarted; bool mActiveTrack; - Maybe mGroupStartTimestamp; - media::TimeUnit mGroupEndTimestamp; nsCString mType; // ContainerParser objects and methods. @@ -200,7 +241,7 @@ private: OnDemuxFailed(TrackType::kAudioTrack, aFailure); } - void DoEvictData(const media::TimeUnit& aPlaybackTime, uint32_t aThreshold); + void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict); struct TrackData { TrackData() @@ -237,6 +278,9 @@ private: bool mNeedRandomAccessPoint; RefPtr mDemuxer; MozPromiseRequestHolder mDemuxRequest; + // Highest end timestamp of the last media segment demuxed. + media::TimeUnit mLastParsedEndTime; + // If set, position where the next contiguous frame will be inserted. // If a discontinuity is detected, it will be unset and recalculated upon // the next insertion. @@ -300,8 +344,6 @@ private: MozPromiseRequestHolder mProcessingRequest; MozPromiseHolder mProcessingPromise; - MozPromiseHolder mAppendPromise; - // Trackbuffers definition. nsTArray GetTracksList(); TrackData& GetTracksData(TrackType aTrack) @@ -325,34 +367,43 @@ private: { return !GetTaskQueue() || GetTaskQueue()->IsCurrentThreadIn(); } - RefPtr mTaskQueue; + RefPtr mTaskQueue; + // SourceBuffer Queues and running context. + SourceBufferTaskQueue mQueue; + void ProcessTasks(); + void CancelAllTasks(); + // Set if the TrackBuffersManager is currently processing a task. + // At this stage, this task is always a AppendBufferTask. + RefPtr mCurrentTask; + // Current SourceBuffer state for ongoing task. + // Its content is returned to the SourceBuffer once the AppendBufferTask has + // completed. + UniquePtr mSourceBufferAttributes; + // The current sourcebuffer append window. It's content is equivalent to + // mSourceBufferAttributes.mAppendWindowStart/End media::TimeInterval mAppendWindow; - media::TimeUnit mTimestampOffset; - media::TimeUnit mLastTimestampOffset; - void RestoreCachedVariables(); // Strong references to external objects. - RefPtr mSourceBufferAttributes; nsMainThreadPtrHandle mParentDecoder; // Set to true if mediasource state changed to ended. Atomic mEnded; + // Set to true if the parent SourceBuffer has shutdown. + // We will not reschedule or process new task once mDetached is set. + Atomic mDetached; // Global size of this source buffer content. Atomic mSizeSourceBuffer; - uint32_t mEvictionThreshold; + const int64_t mVideoEvictionThreshold; + const int64_t mAudioEvictionThreshold; Atomic mEvictionOccurred; // Monitor to protect following objects accessed across multiple threads. - // mMonitor is also notified if the value of mAppendRunning becomes false. mutable Monitor mMonitor; - // Set to true while a BufferAppend is running or is pending. - Atomic mAppendRunning; // Stable audio and video track time ranges. media::TimeIntervals mVideoBufferedRanges; media::TimeIntervals mAudioBufferedRanges; - media::TimeUnit mOfficialGroupEndTimestamp; // MediaInfo of the first init segment read. MediaInfo mInfo; }; diff --git a/dom/media/mediasource/moz.build b/dom/media/mediasource/moz.build index 2f32aef185..2872580060 100644 --- a/dom/media/mediasource/moz.build +++ b/dom/media/mediasource/moz.build @@ -7,9 +7,12 @@ MOCHITEST_MANIFESTS += ['test/mochitest.ini'] EXPORTS += [ 'AsyncEventRunner.h', + 'AutoTaskQueue.h', 'MediaSourceDecoder.h', 'MediaSourceDemuxer.h', - 'SourceBufferContentManager.h', + 'SourceBufferAttributes.h', + 'SourceBufferTask.h', + 'TrackBuffersManager.h', ] EXPORTS.mozilla.dom += [ @@ -26,7 +29,6 @@ UNIFIED_SOURCES += [ 'MediaSourceUtils.cpp', 'ResourceQueue.cpp', 'SourceBuffer.cpp', - 'SourceBufferContentManager.cpp', 'SourceBufferList.cpp', 'SourceBufferResource.cpp', 'TrackBuffersManager.cpp', diff --git a/dom/media/webm/EbmlComposer.cpp b/dom/media/webm/EbmlComposer.cpp index a0a0ddf877..3e9defc2da 100644 --- a/dom/media/webm/EbmlComposer.cpp +++ b/dom/media/webm/EbmlComposer.cpp @@ -5,6 +5,7 @@ #include "EbmlComposer.h" #include "mozilla/UniquePtr.h" +#include "mozilla/Endian.h" #include "libmkv/EbmlIDs.h" #include "libmkv/EbmlWriter.h" #include "libmkv/WebMElement.h" @@ -48,8 +49,17 @@ void EbmlComposer::GenerateHeader() } // Audio if (mCodecPrivateData.Length() > 0) { - writeAudioTrack(&ebml, 0x2, 0x0, "A_VORBIS", mSampleFreq, - mChannels, mCodecPrivateData.Elements(), + // Extract the pre-skip from mCodecPrivateData + // then convert it to nanoseconds. + // Details in OpusTrackEncoder.cpp. + mCodecDelay = + (uint64_t)LittleEndian::readUint16(mCodecPrivateData.Elements() + 10) + * PR_NSEC_PER_SEC / 48000; + // Fixed 80ms, convert into nanoseconds. + uint64_t seekPreRoll = 80 * PR_NSEC_PER_MSEC; + writeAudioTrack(&ebml, 0x2, 0x0, "A_OPUS", mSampleFreq, + mChannels, mCodecDelay, seekPreRoll, + mCodecPrivateData.Elements(), mCodecPrivateData.Length()); } } @@ -135,9 +145,12 @@ EbmlComposer::WriteSimpleBlock(EncodedFrame* aFrame) mFlushState |= FLUSH_CLUSTER; } - bool isVorbis = (frameType == EncodedFrame::FrameType::VORBIS_AUDIO_FRAME); + bool isOpus = (frameType == EncodedFrame::FrameType::OPUS_AUDIO_FRAME); short timeCode = aFrame->GetTimeStamp() / PR_USEC_PER_MSEC - mClusterTimecode; - writeSimpleBlock(&ebml, isVorbis ? 0x2 : 0x1, timeCode, isVP8IFrame, + if (isOpus) { + timeCode += mCodecDelay / PR_NSEC_PER_MSEC; + } + writeSimpleBlock(&ebml, isOpus ? 0x2 : 0x1, timeCode, isVP8IFrame, 0, 0, (unsigned char*)aFrame->GetFrameData().Elements(), aFrame->GetFrameData().Length()); MOZ_ASSERT(ebml.offset <= DEFAULT_HEADER_SIZE + @@ -164,14 +177,11 @@ EbmlComposer::SetVideoConfig(uint32_t aWidth, uint32_t aHeight, } void -EbmlComposer::SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels, - uint32_t aBitDepth) +EbmlComposer::SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels) { MOZ_ASSERT(aSampleFreq > 0, "SampleFreq should > 0"); - MOZ_ASSERT(aBitDepth > 0, "BitDepth should > 0"); MOZ_ASSERT(aChannels > 0, "Channels should > 0"); mSampleFreq = aSampleFreq; - mBitDepth = aBitDepth; mChannels = aChannels; } @@ -199,12 +209,12 @@ EbmlComposer::EbmlComposer() : mFlushState(FLUSH_NONE) , mClusterHeaderIndex(0) , mClusterLengthLoc(0) + , mCodecDelay(0) , mClusterTimecode(0) , mWidth(0) , mHeight(0) , mFrameRate(0) , mSampleFreq(0) - , mBitDepth(0) , mChannels(0) {} diff --git a/dom/media/webm/EbmlComposer.h b/dom/media/webm/EbmlComposer.h index 86d84d518e..389d335286 100644 --- a/dom/media/webm/EbmlComposer.h +++ b/dom/media/webm/EbmlComposer.h @@ -22,8 +22,7 @@ public: void SetVideoConfig(uint32_t aWidth, uint32_t aHeight, uint32_t aDisplayWidth, uint32_t aDisplayHeight, float aFrameRate); - void SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels, - uint32_t bitDepth); + void SetAudioConfig(uint32_t aSampleFreq, uint32_t aChannels); /* * Set the CodecPrivateData for writing in header. */ @@ -68,6 +67,8 @@ private: uint64_t mClusterLengthLoc; // Audio codec specific header data. nsTArray mCodecPrivateData; + // Codec delay in nanoseconds. + uint64_t mCodecDelay; // The timecode of the cluster. uint64_t mClusterTimecode; @@ -80,7 +81,6 @@ private: float mFrameRate; // Audio configuration float mSampleFreq; - int mBitDepth; int mChannels; }; diff --git a/dom/media/webm/NesteggPacketHolder.h b/dom/media/webm/NesteggPacketHolder.h index 6cc3ffcb52..f786e66a17 100644 --- a/dom/media/webm/NesteggPacketHolder.h +++ b/dom/media/webm/NesteggPacketHolder.h @@ -3,7 +3,6 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - #if !defined(NesteggPacketHolder_h_) #define NesteggPacketHolder_h_ @@ -105,6 +104,8 @@ private: std::deque> mQueue; }; + } // namespace mozilla #endif + diff --git a/dom/media/webm/WebMBufferedParser.cpp b/dom/media/webm/WebMBufferedParser.cpp index 85a35459a9..767b1a6eda 100644 --- a/dom/media/webm/WebMBufferedParser.cpp +++ b/dom/media/webm/WebMBufferedParser.cpp @@ -328,16 +328,22 @@ bool WebMBufferedState::GetOffsetForTime(uint64_t aTime, int64_t* aOffset) { ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if(mTimeMapping.IsEmpty()) { + return false; + } + uint64_t time = aTime; if (time > 0) { time = time - 1; } uint32_t idx = mTimeMapping.IndexOfFirstElementGt(time, TimeComparator()); if (idx == mTimeMapping.Length()) { - return false; + // Clamp to end + *aOffset = mTimeMapping[mTimeMapping.Length() - 1].mSyncOffset; + } else { + // Idx is within array or has been clamped to start + *aOffset = mTimeMapping[idx].mSyncOffset; } - - *aOffset = mTimeMapping[idx].mSyncOffset; return true; } diff --git a/dom/media/webm/WebMBufferedParser.h b/dom/media/webm/WebMBufferedParser.h index da422cba03..bcc5ffca32 100644 --- a/dom/media/webm/WebMBufferedParser.h +++ b/dom/media/webm/WebMBufferedParser.h @@ -279,9 +279,11 @@ public: bool CalculateBufferedForRange(int64_t aStartOffset, int64_t aEndOffset, uint64_t* aStartTime, uint64_t* aEndTime); - // Returns true if aTime is is present in mTimeMapping and sets aOffset to + // Returns true if mTimeMapping is not empty and sets aOffset to // the latest offset for which decoding can resume without data - // dependencies to arrive at aTime. + // dependencies to arrive at aTime. aTime will be clamped to the start + // of mTimeMapping if it is earlier than the first element, and to the end + // if later than the last bool GetOffsetForTime(uint64_t aTime, int64_t* aOffset); // Returns end offset of init segment or -1 if none found. diff --git a/dom/media/webm/WebMWriter.cpp b/dom/media/webm/WebMWriter.cpp index cdafa74f97..a98a14f4e2 100644 --- a/dom/media/webm/WebMWriter.cpp +++ b/dom/media/webm/WebMWriter.cpp @@ -6,6 +6,7 @@ #include "WebMWriter.h" #include "EbmlComposer.h" #include "GeckoProfiler.h" +#include "OpusTrackEncoder.h" namespace mozilla { @@ -64,11 +65,19 @@ WebMWriter::SetMetadata(TrackMetadataBase* aMetadata) if (aMetadata->GetKind() == TrackMetadataBase::METADATA_VORBIS) { VorbisMetadata* meta = static_cast(aMetadata); MOZ_ASSERT(meta, "Cannot find vorbis encoder metadata"); - mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels, meta->mBitDepth); + mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels); mEbmlComposer->SetAudioCodecPrivateData(meta->mData); mMetadataRequiredFlag = mMetadataRequiredFlag & ~ContainerWriter::CREATE_AUDIO_TRACK; } + if (aMetadata->GetKind() == TrackMetadataBase::METADATA_OPUS) { + OpusMetadata* meta = static_cast(aMetadata); + MOZ_ASSERT(meta, "Cannot find Opus encoder metadata"); + mEbmlComposer->SetAudioConfig(meta->mSamplingFrequency, meta->mChannels); + mEbmlComposer->SetAudioCodecPrivateData(meta->mIdHeader); + mMetadataRequiredFlag = mMetadataRequiredFlag & ~ContainerWriter::CREATE_AUDIO_TRACK; + } + if (!mMetadataRequiredFlag) { mEbmlComposer->GenerateHeader(); } diff --git a/dom/media/webm/WebMWriter.h b/dom/media/webm/WebMWriter.h index 89ef0006cd..c4584562f9 100644 --- a/dom/media/webm/WebMWriter.h +++ b/dom/media/webm/WebMWriter.h @@ -18,7 +18,6 @@ class VorbisMetadata : public TrackMetadataBase public: nsTArray mData; int32_t mChannels; - int32_t mBitDepth; float mSamplingFrequency; MetadataKind GetKind() const override { return METADATA_VORBIS; } }; diff --git a/media/libmkv/EbmlIDs.h b/media/libmkv/EbmlIDs.h index 3b5da19ad7..06b74031a9 100644 --- a/media/libmkv/EbmlIDs.h +++ b/media/libmkv/EbmlIDs.h @@ -106,6 +106,8 @@ enum mkv { CodecID = 0x86, CodecPrivate = 0x63A2, CodecName = 0x258688, + CodecDelay = 0x56AA, + SeekPreRoll = 0x56BB, /* AttachmentLink = 0x7446, */ /* CodecSettings = 0x3A9697, */ /* CodecInfoURL = 0x3B4040, */ diff --git a/media/libmkv/WebMElement.c b/media/libmkv/WebMElement.c index 87d9322f89..4e2a3667f4 100644 --- a/media/libmkv/WebMElement.c +++ b/media/libmkv/WebMElement.c @@ -87,6 +87,7 @@ void writeVideoTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing, } void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing, const char *codecId, double samplingFrequency, unsigned int channels, + uint64_t codecDelay, uint64_t seekPreRoll, unsigned char *private, unsigned long privateSize) { EbmlLoc start; UInt64 trackID; @@ -95,6 +96,8 @@ void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing, trackID = generateTrackID(trackNumber); Ebml_SerializeUnsigned(glob, TrackUID, trackID); Ebml_SerializeUnsigned(glob, TrackType, 2); // audio is always 2 + Ebml_SerializeUnsigned(glob, CodecDelay, codecDelay); + Ebml_SerializeUnsigned(glob, SeekPreRoll, seekPreRoll); // I am using defaults for thesed required fields /* Ebml_SerializeUnsigned(glob, FlagEnabled, 1); Ebml_SerializeUnsigned(glob, FlagDefault, 1); @@ -103,7 +106,7 @@ void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing, Ebml_SerializeString(glob, CodecID, codecId); Ebml_SerializeData(glob, CodecPrivate, private, privateSize); - Ebml_SerializeString(glob, CodecName, "VORBIS"); // fixed for now + Ebml_SerializeString(glob, CodecName, "OPUS"); // fixed for now { EbmlLoc AudioStart; Ebml_StartSubElement(glob, &AudioStart, Audio); diff --git a/media/libmkv/WebMElement.h b/media/libmkv/WebMElement.h index 5f1f26dcc9..500ac7913d 100644 --- a/media/libmkv/WebMElement.h +++ b/media/libmkv/WebMElement.h @@ -25,6 +25,7 @@ void writeVideoTrack(EbmlGlobal *ebml, unsigned int trackNumber, int flagLacing, double frameRate); void writeAudioTrack(EbmlGlobal *glob, unsigned int trackNumber, int flagLacing, const char *codecId, double samplingFrequency, unsigned int channels, + uint64_t codecDelay, uint64_t seekPreRoll, unsigned char *private_, unsigned long privateSize); void writeSimpleBlock(EbmlGlobal *ebml, unsigned char trackNumber, short timeCode, diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 6ce2e785d0..5ae3b749a2 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -382,6 +382,13 @@ pref("media.mp3.enabled", true); pref("media.apple.mp3.enabled", true); pref("media.apple.mp4.enabled", true); #endif + +// Filter what triggers user notifications. +// See DecoderDoctorDocumentWatcher::ReportAnalysis for details. +pref("media.decoder-doctor.notifications-allowed", "MediaWidevineNoWMFNoSilverlight"); +// Whether we report partial failures. +pref("media.decoder-doctor.verbose", false); + #ifdef MOZ_WEBRTC pref("media.navigator.enabled", true); pref("media.navigator.video.enabled", true);