diff --git a/b2g/app/b2g.js b/b2g/app/b2g.js index 3097b6bf2d..5d0d637572 100644 --- a/b2g/app/b2g.js +++ b/b2g/app/b2g.js @@ -945,7 +945,7 @@ pref("devtools.debugger.unix-domain-socket", "/data/local/debugger-socket"); // falling back to Skia/software for smaller canvases #ifdef MOZ_WIDGET_GONK pref("gfx.canvas.azure.backends", "skia"); -pref("gfx.canvas.azure.accelerated", false); +pref("gfx.canvas.azure.accelerated", true); #endif // Turn on dynamic cache size for Skia @@ -980,7 +980,6 @@ pref("media.webspeech.synth.enabled", true); // Enable Web Speech recognition API pref("media.webspeech.recognition.enable", true); -pref("media.webspeech.service.default", "pocketsphinx"); // Downloads API pref("dom.mozDownloads.enabled", true); diff --git a/dom/camera/GonkCameraControl.cpp b/dom/camera/GonkCameraControl.cpp index 84030ebe70..b58c8a5644 100644 --- a/dom/camera/GonkCameraControl.cpp +++ b/dom/camera/GonkCameraControl.cpp @@ -2388,15 +2388,11 @@ void nsGonkCameraControl::OnNewPreviewFrame(layers::TextureClient* aBuffer) { #ifdef MOZ_WIDGET_GONK - RefPtr frame = mImageContainer->CreateImage(ImageFormat::GRALLOC_PLANAR_YCBCR); + RefPtr frame = new GrallocImage(); - GrallocImage* videoImage = static_cast(frame.get()); - - GrallocImage::GrallocData data; - data.mGraphicBuffer = aBuffer; - data.mPicSize = IntSize(mCurrentConfiguration.mPreviewSize.width, - mCurrentConfiguration.mPreviewSize.height); - videoImage->SetData(data); + IntSize picSize(mCurrentConfiguration.mPreviewSize.width, + mCurrentConfiguration.mPreviewSize.height); + frame->SetData(aBuffer, picSize); if (mCapturePoster.exchange(false)) { CreatePoster(frame, diff --git a/dom/canvas/CanvasRenderingContext2D.cpp b/dom/canvas/CanvasRenderingContext2D.cpp index 5aa593b3ec..80232125dd 100644 --- a/dom/canvas/CanvasRenderingContext2D.cpp +++ b/dom/canvas/CanvasRenderingContext2D.cpp @@ -5590,6 +5590,8 @@ CanvasRenderingContext2D::GetCanvasLayer(nsDisplayListBuilder* aBuilder, data.mGLContext = glue->GetGLContext(); data.mFrontbufferGLTex = skiaGLTex; + PersistentBufferProvider *provider = GetBufferProvider(aManager); + data.mBufferProvider = provider; } else { PersistentBufferProvider *provider = GetBufferProvider(aManager); data.mBufferProvider = provider; @@ -5640,6 +5642,8 @@ CanvasRenderingContext2D::GetCanvasLayer(nsDisplayListBuilder* aBuilder, data.mGLContext = glue->GetGLContext(); data.mFrontbufferGLTex = skiaGLTex; + PersistentBufferProvider *provider = GetBufferProvider(aManager); + data.mBufferProvider = provider; } else { PersistentBufferProvider *provider = GetBufferProvider(aManager); data.mBufferProvider = provider; diff --git a/dom/canvas/ImageBitmap.cpp b/dom/canvas/ImageBitmap.cpp index c55e4fa85c..4673827af7 100644 --- a/dom/canvas/ImageBitmap.cpp +++ b/dom/canvas/ImageBitmap.cpp @@ -86,15 +86,7 @@ static already_AddRefed CreateImageFromSurface(SourceSurface* aSurface, ErrorResult& aRv) { MOZ_ASSERT(aSurface); - - layers::CairoImage::Data cairoData; - cairoData.mSize = aSurface->GetSize(); - cairoData.mSourceSurface = aSurface; - - RefPtr image = new layers::CairoImage(); - - image->SetData(cairoData); - + RefPtr image = new layers::CairoImage(aSurface->GetSize(), aSurface); return image.forget(); } diff --git a/dom/html/HTMLCanvasElement.cpp b/dom/html/HTMLCanvasElement.cpp index d1133df058..0e8a449f8a 100644 --- a/dom/html/HTMLCanvasElement.cpp +++ b/dom/html/HTMLCanvasElement.cpp @@ -1173,13 +1173,7 @@ void HTMLCanvasElement::SetFrameCapture(already_AddRefed aSurface) { RefPtr surface = aSurface; - - CairoImage::Data imageData; - imageData.mSize = surface->GetSize(); - imageData.mSourceSurface = surface; - - RefPtr image = new CairoImage(); - image->SetData(imageData); + RefPtr image = new CairoImage(surface->GetSize(), surface); // Loop backwards to allow removing elements in the loop. for (int i = mRequestedFrameListeners.Length() - 1; i >= 0; --i) { diff --git a/dom/media/AudioStream.cpp b/dom/media/AudioStream.cpp index ee09af1b5f..e766d482fe 100644 --- a/dom/media/AudioStream.cpp +++ b/dom/media/AudioStream.cpp @@ -24,7 +24,7 @@ namespace mozilla { #undef LOG #endif -PRLogModuleInfo* gAudioStreamLog = nullptr; +LazyLogModule gAudioStreamLog("AudioStream"); // For simple logs #define LOG(x) MOZ_LOG(gAudioStreamLog, mozilla::LogLevel::Debug, x) diff --git a/dom/media/CubebUtils.cpp b/dom/media/CubebUtils.cpp index 59e03ac102..d9cfcc19dc 100644 --- a/dom/media/CubebUtils.cpp +++ b/dom/media/CubebUtils.cpp @@ -40,7 +40,7 @@ uint32_t sPreferredSampleRate; } // namespace -extern PRLogModuleInfo* gAudioStreamLog; +extern LazyLogModule gAudioStreamLog; static const uint32_t CUBEB_NORMAL_LATENCY_MS = 100; @@ -126,7 +126,6 @@ bool CubebLatencyPrefSet() void InitLibrary() { - gAudioStreamLog = PR_NewLogModule("AudioStream"); PrefChanged(PREF_VOLUME_SCALE, nullptr); Preferences::RegisterCallback(PrefChanged, PREF_VOLUME_SCALE); PrefChanged(PREF_CUBEB_LATENCY, nullptr); diff --git a/dom/media/DOMMediaStream.cpp b/dom/media/DOMMediaStream.cpp index 08a700927c..2100f6b543 100644 --- a/dom/media/DOMMediaStream.cpp +++ b/dom/media/DOMMediaStream.cpp @@ -27,13 +27,13 @@ #undef LOG #endif -static PRLogModuleInfo* gMediaStreamLog; -#define LOG(type, msg) MOZ_LOG(gMediaStreamLog, type, msg) - using namespace mozilla; using namespace mozilla::dom; using namespace mozilla::layers; +static LazyLogModule gMediaStreamLog("MediaStream"); +#define LOG(type, msg) MOZ_LOG(gMediaStreamLog, type, msg) + const TrackID TRACK_VIDEO_PRIMARY = 1; @@ -318,10 +318,6 @@ DOMMediaStream::DOMMediaStream() nsCOMPtr uuidgen = do_GetService("@mozilla.org/uuid-generator;1", &rv); - if (!gMediaStreamLog) { - gMediaStreamLog = PR_NewLogModule("MediaStream"); - } - if (NS_SUCCEEDED(rv) && uuidgen) { nsID uuid; memset(&uuid, 0, sizeof(uuid)); @@ -1048,10 +1044,9 @@ DOMHwMediaStream::DOMHwMediaStream() { #ifdef MOZ_WIDGET_GONK mImageContainer = LayerManager::CreateImageContainer(ImageContainer::ASYNCHRONOUS_OVERLAY); - RefPtr img = mImageContainer->CreateImage(ImageFormat::OVERLAY_IMAGE); - mOverlayImage = static_cast(img.get()); + mOverlayImage = mImageContainer->CreateOverlayImage(); nsAutoTArray images; - images.AppendElement(ImageContainer::NonOwningImage(img)); + images.AppendElement(ImageContainer::NonOwningImage(mOverlayImage)); mImageContainer->SetCurrentImages(images); #endif } diff --git a/dom/media/GraphDriver.cpp b/dom/media/GraphDriver.cpp index fa8cb62ad0..1ab4ca059a 100644 --- a/dom/media/GraphDriver.cpp +++ b/dom/media/GraphDriver.cpp @@ -11,7 +11,7 @@ #include #endif -extern PRLogModuleInfo* gMediaStreamGraphLog; +extern mozilla::LazyLogModule gMediaStreamGraphLog; #define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg) // We don't use NSPR log here because we want this interleaved with adb logcat diff --git a/dom/media/Latency.cpp b/dom/media/Latency.cpp index 870bc99c98..d3400777ad 100644 --- a/dom/media/Latency.cpp +++ b/dom/media/Latency.cpp @@ -37,17 +37,13 @@ const char* LatencyLogIndex2Strings[] = { static StaticRefPtr gAsyncLogger; -PRLogModuleInfo* +LogModule* GetLatencyLog() { - static PRLogModuleInfo* sLog; - if (!sLog) { - sLog = PR_NewLogModule("MediaLatency"); - } + static LazyLogModule sLog("MediaLatency"); return sLog; } - class LogEvent : public nsRunnable { public: @@ -111,6 +107,8 @@ void LogLatency(uint32_t aIndex, uint64_t aID, int64_t aValue) void AsyncLatencyLogger::InitializeStatics() { NS_ASSERTION(NS_IsMainThread(), "Main thread only"); + + //Make sure that the underlying logger is allocated. GetLatencyLog(); gAsyncLogger = new AsyncLatencyLogger(); } diff --git a/dom/media/Latency.h b/dom/media/Latency.h index e7a83d78b8..b86ee7b54b 100644 --- a/dom/media/Latency.h +++ b/dom/media/Latency.h @@ -17,7 +17,7 @@ class AsyncLatencyLogger; -PRLogModuleInfo* GetLatencyLog(); +mozilla::LogModule* GetLatencyLog(); // This class is a singleton. It is refcounted. class AsyncLatencyLogger : public nsIObserver diff --git a/dom/media/MP3Demuxer.cpp b/dom/media/MP3Demuxer.cpp index 3fd60e709e..a3b2152775 100644 --- a/dom/media/MP3Demuxer.cpp +++ b/dom/media/MP3Demuxer.cpp @@ -16,7 +16,7 @@ #include "prenv.h" #ifdef PR_LOGGING -PRLogModuleInfo* gMP3DemuxerLog; +mozilla::LazyLogModule gMP3DemuxerLog("MP3Demuxer"); #define MP3LOG(msg, ...) \ MOZ_LOG(gMP3DemuxerLog, LogLevel::Debug, ("MP3Demuxer " msg, ##__VA_ARGS__)) #define MP3LOGV(msg, ...) \ @@ -112,12 +112,6 @@ MP3TrackDemuxer::MP3TrackDemuxer(MediaResource* aSource) , mChannels(0) { Reset(); - -#ifdef PR_LOGGING - if (!gMP3DemuxerLog) { - gMP3DemuxerLog = PR_NewLogModule("MP3Demuxer"); - } -#endif } bool diff --git a/dom/media/MediaCache.cpp b/dom/media/MediaCache.cpp index eccbce85bf..108ff0c91e 100644 --- a/dom/media/MediaCache.cpp +++ b/dom/media/MediaCache.cpp @@ -24,7 +24,7 @@ namespace mozilla { -PRLogModuleInfo* gMediaCacheLog; +LazyLogModule gMediaCacheLog("MediaCache"); #define CACHE_LOG(type, msg) MOZ_LOG(gMediaCacheLog, type, msg) // Readahead blocks for non-seekable streams will be limited to this @@ -104,9 +104,7 @@ class MediaCache { public: friend class MediaCacheStream::BlockList; typedef MediaCacheStream::BlockList BlockList; - enum { - BLOCK_SIZE = MediaCacheStream::BLOCK_SIZE - }; + static const int64_t BLOCK_SIZE = MediaCacheStream::BLOCK_SIZE; MediaCache() : mNextResourceID(1), mReentrantMonitor("MediaCache.mReentrantMonitor"), @@ -582,10 +580,6 @@ MediaCache::Init() rv = mFileCache->Open(fileDesc); NS_ENSURE_SUCCESS(rv,rv); - if (!gMediaCacheLog) { - gMediaCacheLog = PR_NewLogModule("MediaCache"); - } - MediaCacheFlusher::Init(); return NS_OK; @@ -1834,12 +1828,11 @@ MediaCacheStream::NotifyDataReceived(int64_t aSize, const char* aData, } void -MediaCacheStream::FlushPartialBlockInternal(bool aNotifyAll) +MediaCacheStream::FlushPartialBlockInternal(bool aNotifyAll, + ReentrantMonitorAutoEnter& aReentrantMonitor) { NS_ASSERTION(NS_IsMainThread(), "Only call on main thread"); - ReentrantMonitorAutoEnter mon(gMediaCache->GetReentrantMonitor()); - int32_t blockOffset = int32_t(mChannelOffset%BLOCK_SIZE); if (blockOffset > 0) { CACHE_LOG(LogLevel::Debug, @@ -1861,7 +1854,7 @@ MediaCacheStream::FlushPartialBlockInternal(bool aNotifyAll) // that will never come. if ((blockOffset > 0 || mChannelOffset == 0) && aNotifyAll) { // Wake up readers who may be waiting for this data - mon.NotifyAll(); + aReentrantMonitor.NotifyAll(); } } @@ -1876,7 +1869,7 @@ MediaCacheStream::FlushPartialBlock() // Note: This writes a full block, so if data is not at the end of the // stream, the decoder must subsequently choose correct start and end offsets // for reading/seeking. - FlushPartialBlockInternal(false); + FlushPartialBlockInternal(false, mon); gMediaCache->QueueUpdate(); } @@ -1897,7 +1890,7 @@ MediaCacheStream::NotifyDataEnded(nsresult aStatus) // It is prudent to update channel/cache status before calling // CacheClientNotifyDataEnded() which will read |mChannelEnded|. - FlushPartialBlockInternal(true); + FlushPartialBlockInternal(true, mon); mChannelEnded = true; gMediaCache->QueueUpdate(); @@ -2224,7 +2217,7 @@ MediaCacheStream::Read(char* aBuffer, uint32_t aCount, uint32_t* aBytes) uint32_t streamBlock = uint32_t(mStreamOffset/BLOCK_SIZE); uint32_t offsetInStreamBlock = uint32_t(mStreamOffset - streamBlock*BLOCK_SIZE); - int64_t size = std::min(aCount - count, BLOCK_SIZE - offsetInStreamBlock); + int64_t size = std::min(aCount - count, BLOCK_SIZE - offsetInStreamBlock); if (mStreamLength >= 0) { // Don't try to read beyond the end of the stream @@ -2461,13 +2454,13 @@ nsresult MediaCacheStream::GetCachedRanges(nsTArray& aRanges) // shrink while we're trying to loop over them. NS_ASSERTION(mPinCount > 0, "Must be pinned"); - int64_t startOffset = GetNextCachedData(0); + int64_t startOffset = GetNextCachedDataInternal(0); while (startOffset >= 0) { - int64_t endOffset = GetCachedDataEnd(startOffset); + int64_t endOffset = GetCachedDataEndInternal(startOffset); NS_ASSERTION(startOffset < endOffset, "Buffered range must end after its start"); // Bytes [startOffset..endOffset] are cached. aRanges.AppendElement(MediaByteRange(startOffset, endOffset)); - startOffset = GetNextCachedData(endOffset); + startOffset = GetNextCachedDataInternal(endOffset); NS_ASSERTION(startOffset == -1 || startOffset > endOffset, "Must have advanced to start of next range, or hit end of stream"); } diff --git a/dom/media/MediaCache.h b/dom/media/MediaCache.h index fb0337bee2..36b96a8ceb 100644 --- a/dom/media/MediaCache.h +++ b/dom/media/MediaCache.h @@ -183,10 +183,9 @@ class MediaCache; */ class MediaCacheStream { public: - enum { - // This needs to be a power of two - BLOCK_SIZE = 32768 - }; + // This needs to be a power of two + static const int64_t BLOCK_SIZE = 32768; + enum ReadMode { MODE_METADATA, MODE_PLAYBACK @@ -425,7 +424,7 @@ private: // Used by |NotifyDataEnded| and |FlushPartialBlock|. // If |aNotifyAll| is true, this function will wake up readers who may be // waiting on the media cache monitor. Called on the main thread only. - void FlushPartialBlockInternal(bool aNotify); + void FlushPartialBlockInternal(bool aNotify, ReentrantMonitorAutoEnter& aReentrantMonitor); // A helper function to do the work of closing the stream. Assumes // that the cache monitor is held. Main thread only. // aReentrantMonitor is the nsAutoReentrantMonitor wrapper holding the cache monitor. diff --git a/dom/media/MediaData.cpp b/dom/media/MediaData.cpp index 667c033e78..208054c49e 100644 --- a/dom/media/MediaData.cpp +++ b/dom/media/MediaData.cpp @@ -312,11 +312,11 @@ VideoData::Create(const VideoInfo& aInfo, // format. #ifdef MOZ_WIDGET_GONK if (IsYV12Format(Y, Cb, Cr) && !IsInEmulator()) { - v->mImage = aContainer->CreateImage(ImageFormat::GRALLOC_PLANAR_YCBCR); + v->mImage = new layers::GrallocImage(); } #endif if (!v->mImage) { - v->mImage = aContainer->CreateImage(ImageFormat::PLANAR_YCBCR); + v->mImage = aContainer->CreatePlanarYCbCrImage(); } } else { v->mImage = aImage; @@ -328,7 +328,8 @@ VideoData::Create(const VideoInfo& aInfo, NS_ASSERTION(v->mImage->GetFormat() == ImageFormat::PLANAR_YCBCR || v->mImage->GetFormat() == ImageFormat::GRALLOC_PLANAR_YCBCR, "Wrong format?"); - PlanarYCbCrImage* videoImage = static_cast(v->mImage.get()); + PlanarYCbCrImage* videoImage = v->mImage->AsPlanarYCbCrImage(); + MOZ_ASSERT(videoImage); bool shouldCopyData = (aImage == nullptr); if (!VideoData::SetVideoDataToImage(videoImage, aInfo, aBuffer, aPicture, @@ -339,11 +340,11 @@ VideoData::Create(const VideoInfo& aInfo, #ifdef MOZ_WIDGET_GONK if (!videoImage->IsValid() && !aImage && IsYV12Format(Y, Cb, Cr)) { // Failed to allocate gralloc. Try fallback. - v->mImage = aContainer->CreateImage(ImageFormat::PLANAR_YCBCR); + v->mImage = aContainer->CreatePlanarYCbCrImage(); if (!v->mImage) { return nullptr; } - videoImage = static_cast(v->mImage.get()); + videoImage = v->mImage->AsPlanarYCbCrImage(); if(!VideoData::SetVideoDataToImage(videoImage, aInfo, aBuffer, aPicture, true /* aCopyData */)) { return nullptr; @@ -460,22 +461,9 @@ VideoData::Create(const VideoInfo& aInfo, aInfo.mDisplay, 0)); - v->mImage = aContainer->CreateImage(ImageFormat::GRALLOC_PLANAR_YCBCR); - if (!v->mImage) { - return nullptr; - } - NS_ASSERTION(v->mImage->GetFormat() == ImageFormat::GRALLOC_PLANAR_YCBCR, - "Wrong format?"); - typedef mozilla::layers::GrallocImage GrallocImage; - GrallocImage* videoImage = static_cast(v->mImage.get()); - GrallocImage::GrallocData data; - - data.mPicSize = aPicture.Size(); - data.mGraphicBuffer = aBuffer; - - if (!videoImage->SetData(data)) { - return nullptr; - } + RefPtr image = new layers::GrallocImage(); + image->SetData(aBuffer, aPicture.Size()); + v->mImage = image; return v.forget(); } diff --git a/dom/media/MediaDecoder.cpp b/dom/media/MediaDecoder.cpp index 72e23b10d4..f7dcdc91cf 100644 --- a/dom/media/MediaDecoder.cpp +++ b/dom/media/MediaDecoder.cpp @@ -46,7 +46,7 @@ static const uint64_t ESTIMATED_DURATION_FUZZ_FACTOR_USECS = USECS_PER_S / 2; // avoid redefined macro in unified build #undef DECODER_LOG -PRLogModuleInfo* gMediaDecoderLog; +LazyLogModule gMediaDecoderLog("MediaDecoder"); #define DECODER_LOG(x, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, ("Decoder=%p " x, this, ##__VA_ARGS__)) @@ -113,21 +113,14 @@ public: StaticRefPtr MediaMemoryTracker::sUniqueInstance; #if defined(PR_LOGGING) -PRLogModuleInfo* gMediaTimerLog; -PRLogModuleInfo* gMediaSampleLog; +LazyLogModule gMediaTimerLog("MediaTimer"); +LazyLogModule gMediaSampleLog("MediaSample"); #endif void MediaDecoder::InitStatics() { MOZ_ASSERT(NS_IsMainThread()); - -#if defined(PR_LOGGING) - // Log modules. - gMediaDecoderLog = PR_NewLogModule("MediaDecoder"); - gMediaTimerLog = PR_NewLogModule("MediaTimer"); - gMediaSampleLog = PR_NewLogModule("MediaSample"); -#endif } NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter) diff --git a/dom/media/MediaDecoderReader.cpp b/dom/media/MediaDecoderReader.cpp index 66a1e61d68..50cc6493c2 100644 --- a/dom/media/MediaDecoderReader.cpp +++ b/dom/media/MediaDecoderReader.cpp @@ -22,7 +22,7 @@ namespace mozilla { // Un-comment to enable logging of seek bisections. //#define SEEK_LOGGING -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define DECODER_LOG(x, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, ("Decoder=%p " x, mDecoder, ##__VA_ARGS__)) diff --git a/dom/media/MediaDecoderStateMachine.h b/dom/media/MediaDecoderStateMachine.h index 67c04a58b7..11d97c029a 100644 --- a/dom/media/MediaDecoderStateMachine.h +++ b/dom/media/MediaDecoderStateMachine.h @@ -107,8 +107,8 @@ class AudioSegment; class DecodedStream; class TaskQueue; -extern PRLogModuleInfo* gMediaDecoderLog; -extern PRLogModuleInfo* gMediaSampleLog; +extern LazyLogModule gMediaDecoderLog; +extern LazyLogModule gMediaSampleLog; /* The state machine class. This manages the decoding and seeking in the diff --git a/dom/media/MediaFormatReader.cpp b/dom/media/MediaFormatReader.cpp index f19c4b5fd2..904eb3df6d 100644 --- a/dom/media/MediaFormatReader.cpp +++ b/dom/media/MediaFormatReader.cpp @@ -24,20 +24,10 @@ using mozilla::layers::Image; using mozilla::layers::LayerManager; using mozilla::layers::LayersBackend; -#ifdef PR_LOGGING -PRLogModuleInfo* GetFormatDecoderLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("MediaFormatReader"); - } - return log; -} -#define LOG(arg, ...) MOZ_LOG(GetFormatDecoderLog(), mozilla::LogLevel::Debug, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) -#define LOGV(arg, ...) MOZ_LOG(GetFormatDecoderLog(), mozilla::LogLevel::Verbose, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) -#else -#define LOG(...) -#define LOGV(...) -#endif +static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader"); + +#define LOG(arg, ...) MOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) +#define LOGV(arg, ...) MOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, ("MediaFormatReader(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/MediaManager.cpp b/dom/media/MediaManager.cpp index 7a5280affd..504e7d8cfe 100644 --- a/dom/media/MediaManager.cpp +++ b/dom/media/MediaManager.cpp @@ -104,12 +104,10 @@ namespace mozilla { #undef LOG #endif -PRLogModuleInfo* +LogModule* GetMediaManagerLog() { - static PRLogModuleInfo *sLog; - if (!sLog) - sLog = PR_NewLogModule("MediaManager"); + static LazyLogModule sLog("MediaManager"); return sLog; } #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) diff --git a/dom/media/MediaManager.h b/dom/media/MediaManager.h index 9168a9766f..fc1c7bc411 100644 --- a/dom/media/MediaManager.h +++ b/dom/media/MediaManager.h @@ -51,7 +51,7 @@ struct MediaTrackConstraints; struct MediaTrackConstraintSet; } // namespace dom -extern PRLogModuleInfo* GetMediaManagerLog(); +extern LogModule* GetMediaManagerLog(); #define MM_LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) class MediaDevice : public nsIMediaDevice diff --git a/dom/media/MediaRecorder.cpp b/dom/media/MediaRecorder.cpp index 02189140b7..83da4b65ae 100644 --- a/dom/media/MediaRecorder.cpp +++ b/dom/media/MediaRecorder.cpp @@ -31,7 +31,7 @@ #undef LOG #endif -PRLogModuleInfo* gMediaRecorderLog; +mozilla::LazyLogModule gMediaRecorderLog("MediaRecorder"); #define LOG(type, msg) MOZ_LOG(gMediaRecorderLog, type, msg) namespace mozilla { @@ -747,9 +747,7 @@ MediaRecorder::MediaRecorder(DOMMediaStream& aSourceMediaStream, MOZ_ASSERT(aOwnerWindow); MOZ_ASSERT(aOwnerWindow->IsInnerWindow()); mDOMStream = &aSourceMediaStream; - if (!gMediaRecorderLog) { - gMediaRecorderLog = PR_NewLogModule("MediaRecorder"); - } + RegisterActivityObserver(); } @@ -780,9 +778,7 @@ MediaRecorder::MediaRecorder(AudioNode& aSrcAudioNode, } } mAudioNode = &aSrcAudioNode; - if (!gMediaRecorderLog) { - gMediaRecorderLog = PR_NewLogModule("MediaRecorder"); - } + RegisterActivityObserver(); } diff --git a/dom/media/MediaResource.cpp b/dom/media/MediaResource.cpp index 4dda7e935d..2caaae1f68 100644 --- a/dom/media/MediaResource.cpp +++ b/dom/media/MediaResource.cpp @@ -35,7 +35,7 @@ using mozilla::media::TimeUnit; -PRLogModuleInfo* gMediaResourceLog; +mozilla::LazyLogModule gMediaResourceLog("MediaResource"); #define RESOURCE_LOG(msg, ...) MOZ_LOG(gMediaResourceLog, mozilla::LogLevel::Debug, \ (msg, ##__VA_ARGS__)) // Debug logging macro with object pointer and class name. @@ -78,9 +78,6 @@ ChannelMediaResource::ChannelMediaResource(MediaResourceCallback* aCallback, mIsTransportSeekable(true), mSuspendAgent(mChannel) { - if (!gMediaResourceLog) { - gMediaResourceLog = PR_NewLogModule("MediaResource"); - } } ChannelMediaResource::~ChannelMediaResource() diff --git a/dom/media/MediaShutdownManager.cpp b/dom/media/MediaShutdownManager.cpp index 46952f2227..b55954b664 100644 --- a/dom/media/MediaShutdownManager.cpp +++ b/dom/media/MediaShutdownManager.cpp @@ -12,7 +12,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define DECODER_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) NS_IMPL_ISUPPORTS(MediaShutdownManager, nsIObserver) diff --git a/dom/media/MediaStreamGraph.cpp b/dom/media/MediaStreamGraph.cpp index 456bbd9ef2..a3d3431904 100644 --- a/dom/media/MediaStreamGraph.cpp +++ b/dom/media/MediaStreamGraph.cpp @@ -39,7 +39,7 @@ using namespace mozilla::gfx; namespace mozilla { -PRLogModuleInfo* gMediaStreamGraphLog; +LazyLogModule gMediaStreamGraphLog("MediaStreamGraph"); #define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg) // #define ENABLE_LIFECYCLE_LOG @@ -847,13 +847,11 @@ MediaStreamGraphImpl::PlayVideo(MediaStream* aStream) if (frame->GetForceBlack()) { if (!blackImage) { - blackImage = aStream->mVideoOutputs[0]-> - GetImageContainer()->CreateImage(ImageFormat::PLANAR_YCBCR); + blackImage = aStream->mVideoOutputs[0]->GetImageContainer()->CreatePlanarYCbCrImage(); if (blackImage) { // Sets the image to a single black pixel, which will be scaled to // fill the rendered size. - SetImageToBlackPixel(static_cast - (blackImage.get())); + SetImageToBlackPixel(blackImage->AsPlanarYCbCrImage()); } } if (blackImage) { @@ -2662,10 +2660,6 @@ MediaStreamGraphImpl::MediaStreamGraphImpl(GraphDriverType aDriverRequested, #endif , mAudioChannel(aChannel) { - if (!gMediaStreamGraphLog) { - gMediaStreamGraphLog = PR_NewLogModule("MediaStreamGraph"); - } - if (mRealtime) { if (aDriverRequested == AUDIO_THREAD_DRIVER) { AudioCallbackDriver* driver = new AudioCallbackDriver(this); diff --git a/dom/media/MediaStreamGraph.h b/dom/media/MediaStreamGraph.h index 05eb8b46c6..724ff0c24b 100644 --- a/dom/media/MediaStreamGraph.h +++ b/dom/media/MediaStreamGraph.h @@ -35,7 +35,7 @@ class nsAutoRefTraits : public nsPointerRefTraitsAsyncOpen(mListener); - if (!gRtspMediaResourceLog) { - gRtspMediaResourceLog = PR_NewLogModule("RtspMediaResource"); - } #endif } diff --git a/dom/media/StreamBuffer.cpp b/dom/media/StreamBuffer.cpp index 5af63e5b01..516982ca70 100644 --- a/dom/media/StreamBuffer.cpp +++ b/dom/media/StreamBuffer.cpp @@ -9,7 +9,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaStreamGraphLog; +extern LazyLogModule gMediaStreamGraphLog; #define STREAM_LOG(type, msg) MOZ_LOG(gMediaStreamGraphLog, type, msg) #ifdef DEBUG diff --git a/dom/media/TrackUnionStream.cpp b/dom/media/TrackUnionStream.cpp index e4e844a438..9325641970 100644 --- a/dom/media/TrackUnionStream.cpp +++ b/dom/media/TrackUnionStream.cpp @@ -42,15 +42,12 @@ namespace mozilla { #undef STREAM_LOG #endif -PRLogModuleInfo* gTrackUnionStreamLog; +LazyLogModule gTrackUnionStreamLog("TrackUnionStream"); #define STREAM_LOG(type, msg) MOZ_LOG(gTrackUnionStreamLog, type, msg) TrackUnionStream::TrackUnionStream(DOMMediaStream* aWrapper) : ProcessedMediaStream(aWrapper), mNextAvailableTrackID(1) { - if (!gTrackUnionStreamLog) { - gTrackUnionStreamLog = PR_NewLogModule("TrackUnionStream"); - } } void TrackUnionStream::RemoveInput(MediaInputPort* aPort) diff --git a/dom/media/VideoSegment.cpp b/dom/media/VideoSegment.cpp index 7ac47b5c90..1d97cefa75 100644 --- a/dom/media/VideoSegment.cpp +++ b/dom/media/VideoSegment.cpp @@ -43,17 +43,14 @@ VideoFrame::TakeFrom(VideoFrame* aFrame) /* static */ already_AddRefed VideoFrame::CreateBlackImage(const gfx::IntSize& aSize) { - RefPtr container; - RefPtr image; - container = LayerManager::CreateImageContainer(); - image = container->CreateImage(ImageFormat::PLANAR_YCBCR); + RefPtr container = LayerManager::CreateImageContainer(); + RefPtr image = container->CreatePlanarYCbCrImage(); if (!image) { MOZ_ASSERT(false); return nullptr; } int len = ((aSize.width * aSize.height) * 3 / 2); - PlanarYCbCrImage* planar = static_cast(image.get()); // Generate a black image. ScopedDeletePtr frame(new uint8_t[len]); @@ -80,7 +77,7 @@ VideoFrame::CreateBlackImage(const gfx::IntSize& aSize) data.mStereoMode = StereoMode::MONO; // SetData copies data, so we can free data. - if (!planar->SetData(data)) { + if (!image->SetData(data)) { MOZ_ASSERT(false); return nullptr; } diff --git a/dom/media/WebVTTListener.cpp b/dom/media/WebVTTListener.cpp index 5727e8c718..a93217ad5e 100644 --- a/dom/media/WebVTTListener.cpp +++ b/dom/media/WebVTTListener.cpp @@ -28,16 +28,13 @@ NS_INTERFACE_MAP_END NS_IMPL_CYCLE_COLLECTING_ADDREF(WebVTTListener) NS_IMPL_CYCLE_COLLECTING_RELEASE(WebVTTListener) -PRLogModuleInfo* gTextTrackLog; +LazyLogModule gTextTrackLog("TextTrack"); # define VTT_LOG(...) MOZ_LOG(gTextTrackLog, LogLevel::Debug, (__VA_ARGS__)) WebVTTListener::WebVTTListener(HTMLTrackElement* aElement) : mElement(aElement) { MOZ_ASSERT(mElement, "Must pass an element to the callback"); - if (!gTextTrackLog) { - gTextTrackLog = PR_NewLogModule("TextTrack"); - } VTT_LOG("WebVTTListener created."); } diff --git a/dom/media/android/AndroidMediaReader.cpp b/dom/media/android/AndroidMediaReader.cpp index 42bd96cbea..cb2c5329aa 100644 --- a/dom/media/android/AndroidMediaReader.cpp +++ b/dom/media/android/AndroidMediaReader.cpp @@ -388,8 +388,8 @@ uint8_t * AndroidMediaReader::ImageBufferCallback::CreateI420Image(size_t aWidth, size_t aHeight) { - mImage = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); - PlanarYCbCrImage *yuvImage = static_cast(mImage.get()); + RefPtr yuvImage = mImageContainer->CreatePlanarYCbCrImage(); + mImage = yuvImage; if (!yuvImage) { NS_WARNING("Could not create I420 image"); diff --git a/dom/media/apple/AppleMP3Reader.cpp b/dom/media/apple/AppleMP3Reader.cpp index ad479e7684..8e6e910bbc 100644 --- a/dom/media/apple/AppleMP3Reader.cpp +++ b/dom/media/apple/AppleMP3Reader.cpp @@ -23,7 +23,7 @@ using namespace mozilla::media; namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOGE(...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Error, (__VA_ARGS__)) #define LOGW(...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Warning, (__VA_ARGS__)) #define LOGD(...) MOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) @@ -431,9 +431,10 @@ AppleMP3Reader::SetupDecoder() // Set output format #if defined(MOZ_SAMPLE_TYPE_FLOAT32) outputFormat.mBitsPerChannel = 32; - outputFormat.mFormatFlags = - kLinearPCMFormatFlagIsFloat | - 0; + outputFormat.mFormatFlags = kLinearPCMFormatFlagIsFloat; +#elif defined(MOZ_SAMPLE_TYPE_S16) + outputFormat.mBitsPerChannel = 32; + outputFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger; #else #error Unknown audio sample type #endif diff --git a/dom/media/directshow/AudioSinkFilter.cpp b/dom/media/directshow/AudioSinkFilter.cpp index 3d2b93b46b..5682af1a58 100644 --- a/dom/media/directshow/AudioSinkFilter.cpp +++ b/dom/media/directshow/AudioSinkFilter.cpp @@ -23,7 +23,7 @@ using namespace mozilla::media; namespace mozilla { -PRLogModuleInfo* GetDirectShowLog(); +extern LogModule* GetDirectShowLog(); #define LOG(...) MOZ_LOG(GetDirectShowLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) AudioSinkFilter::AudioSinkFilter(const wchar_t* aObjectName, HRESULT* aOutResult) diff --git a/dom/media/directshow/AudioSinkInputPin.cpp b/dom/media/directshow/AudioSinkInputPin.cpp index d3187c58f2..285dc3ca87 100644 --- a/dom/media/directshow/AudioSinkInputPin.cpp +++ b/dom/media/directshow/AudioSinkInputPin.cpp @@ -15,7 +15,7 @@ using namespace mozilla::media; namespace mozilla { -PRLogModuleInfo* GetDirectShowLog(); +extern LogModule* GetDirectShowLog(); #define LOG(...) MOZ_LOG(GetDirectShowLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) AudioSinkInputPin::AudioSinkInputPin(wchar_t* aObjectName, diff --git a/dom/media/directshow/DirectShowReader.cpp b/dom/media/directshow/DirectShowReader.cpp index b2a52850bd..ed6390a79e 100644 --- a/dom/media/directshow/DirectShowReader.cpp +++ b/dom/media/directshow/DirectShowReader.cpp @@ -18,13 +18,9 @@ using namespace mozilla::media; namespace mozilla { - -PRLogModuleInfo* +LogModule* GetDirectShowLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("DirectShowDecoder"); - } + static LazyLogModule log("DirectShowDecoder"); return log; } diff --git a/dom/media/directshow/SampleSink.cpp b/dom/media/directshow/SampleSink.cpp index 54b62508d2..1102d32f5a 100644 --- a/dom/media/directshow/SampleSink.cpp +++ b/dom/media/directshow/SampleSink.cpp @@ -14,7 +14,7 @@ using namespace mozilla::media; namespace mozilla { -PRLogModuleInfo* GetDirectShowLog(); +extern LogModule* GetDirectShowLog(); #define LOG(...) MOZ_LOG(GetDirectShowLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) SampleSink::SampleSink() diff --git a/dom/media/directshow/SourceFilter.cpp b/dom/media/directshow/SourceFilter.cpp index 3eb11ea4b8..e563b49cf6 100644 --- a/dom/media/directshow/SourceFilter.cpp +++ b/dom/media/directshow/SourceFilter.cpp @@ -20,7 +20,7 @@ namespace mozilla { //#define DEBUG_SOURCE_TRACE 1 #if defined (DEBUG_SOURCE_TRACE) -PRLogModuleInfo* GetDirectShowLog(); +extern LogModule* GetDirectShowLog(); #define DIRECTSHOW_LOG(...) MOZ_LOG(GetDirectShowLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) #else #define DIRECTSHOW_LOG(...) diff --git a/dom/media/encoder/MediaEncoder.cpp b/dom/media/encoder/MediaEncoder.cpp index 7c1183466a..332c9c978d 100644 --- a/dom/media/encoder/MediaEncoder.cpp +++ b/dom/media/encoder/MediaEncoder.cpp @@ -31,7 +31,7 @@ #undef LOG #endif -PRLogModuleInfo* gMediaEncoderLog; +mozilla::LazyLogModule gMediaEncoderLog("MediaEncoder"); #define LOG(type, msg) MOZ_LOG(gMediaEncoderLog, type, msg) namespace mozilla { @@ -79,9 +79,6 @@ MediaEncoder::CreateEncoder(const nsAString& aMIMEType, uint32_t aAudioBitrate, uint32_t aVideoBitrate, uint32_t aBitrate, uint8_t aTrackTypes) { - if (!gMediaEncoderLog) { - gMediaEncoderLog = PR_NewLogModule("MediaEncoder"); - } PROFILER_LABEL("MediaEncoder", "CreateEncoder", js::ProfileEntry::Category::OTHER); diff --git a/dom/media/encoder/TrackEncoder.cpp b/dom/media/encoder/TrackEncoder.cpp index c617980a14..7e2161a0c5 100644 --- a/dom/media/encoder/TrackEncoder.cpp +++ b/dom/media/encoder/TrackEncoder.cpp @@ -18,7 +18,7 @@ namespace mozilla { -PRLogModuleInfo* gTrackEncoderLog; +LazyLogModule gTrackEncoderLog("TrackEncoder"); #define TRACK_LOG(type, msg) MOZ_LOG(gTrackEncoderLog, type, msg) static const int DEFAULT_CHANNELS = 1; @@ -37,9 +37,6 @@ TrackEncoder::TrackEncoder() , mAudioInitCounter(0) , mVideoInitCounter(0) { - if (!gTrackEncoderLog) { - gTrackEncoderLog = PR_NewLogModule("TrackEncoder"); - } } void diff --git a/dom/media/encoder/VP8TrackEncoder.cpp b/dom/media/encoder/VP8TrackEncoder.cpp index 96aeeaeec6..c21a449626 100644 --- a/dom/media/encoder/VP8TrackEncoder.cpp +++ b/dom/media/encoder/VP8TrackEncoder.cpp @@ -17,7 +17,7 @@ namespace mozilla { -PRLogModuleInfo* gVP8TrackEncoderLog; +LazyLogModule gVP8TrackEncoderLog("VP8TrackEncoder"); #define VP8LOG(msg, ...) MOZ_LOG(gVP8TrackEncoderLog, mozilla::LogLevel::Debug, \ (msg, ##__VA_ARGS__)) // Debug logging macro with object pointer and class name. @@ -37,9 +37,6 @@ VP8TrackEncoder::VP8TrackEncoder() , mVPXImageWrapper(new vpx_image_t()) { MOZ_COUNT_CTOR(VP8TrackEncoder); - if (!gVP8TrackEncoderLog) { - gVP8TrackEncoderLog = PR_NewLogModule("VP8TrackEncoder"); - } } VP8TrackEncoder::~VP8TrackEncoder() diff --git a/dom/media/encoder/VorbisTrackEncoder.cpp b/dom/media/encoder/VorbisTrackEncoder.cpp index c22b3e0560..3a1c153bd5 100644 --- a/dom/media/encoder/VorbisTrackEncoder.cpp +++ b/dom/media/encoder/VorbisTrackEncoder.cpp @@ -17,7 +17,7 @@ static const float BASE_QUALITY = 0.4f; namespace mozilla { #undef LOG -PRLogModuleInfo* gVorbisTrackEncoderLog; +LazyLogModule gVorbisTrackEncoderLog("VorbisTrackEncoder"); #define VORBISLOG(msg, ...) MOZ_LOG(gVorbisTrackEncoderLog, mozilla::LogLevel::Debug, \ (msg, ##__VA_ARGS__)) @@ -25,9 +25,6 @@ VorbisTrackEncoder::VorbisTrackEncoder() : AudioTrackEncoder() { MOZ_COUNT_CTOR(VorbisTrackEncoder); - if (!gVorbisTrackEncoderLog) { - gVorbisTrackEncoderLog = PR_NewLogModule("VorbisTrackEncoder"); - } } VorbisTrackEncoder::~VorbisTrackEncoder() diff --git a/dom/media/fmp4/MP4Demuxer.cpp b/dom/media/fmp4/MP4Demuxer.cpp index d281594181..6c11eca7bb 100644 --- a/dom/media/fmp4/MP4Demuxer.cpp +++ b/dom/media/fmp4/MP4Demuxer.cpp @@ -20,11 +20,8 @@ #include "mp4_demuxer/AnnexB.h" #include "mp4_demuxer/H264.h" -PRLogModuleInfo* GetDemuxerLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("MP4Demuxer"); - } +mozilla::LogModule* GetDemuxerLog() { + static mozilla::LazyLogModule log("MP4Demuxer"); return log; } diff --git a/dom/media/gmp/GMPAudioDecoderParent.cpp b/dom/media/gmp/GMPAudioDecoderParent.cpp index b9a8cba8d5..b73c0d973f 100644 --- a/dom/media/gmp/GMPAudioDecoderParent.cpp +++ b/dom/media/gmp/GMPAudioDecoderParent.cpp @@ -17,7 +17,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGV(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Verbose, msg) #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) diff --git a/dom/media/gmp/GMPChild.cpp b/dom/media/gmp/GMPChild.cpp index 4c5bd03dca..6de89b1488 100644 --- a/dom/media/gmp/GMPChild.cpp +++ b/dom/media/gmp/GMPChild.cpp @@ -47,7 +47,7 @@ namespace mozilla { #undef LOG #undef LOGD -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOG(level, x, ...) MOZ_LOG(GetGMPLog(), (level), (x, ##__VA_ARGS__)) #define LOGD(x, ...) LOG(mozilla::LogLevel::Debug, "GMPChild[pid=%d] " x, (int)base::GetCurrentProcId(), ##__VA_ARGS__) diff --git a/dom/media/gmp/GMPContentParent.cpp b/dom/media/gmp/GMPContentParent.cpp index b238445f89..4b89dc7c53 100644 --- a/dom/media/gmp/GMPContentParent.cpp +++ b/dom/media/gmp/GMPContentParent.cpp @@ -20,7 +20,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg) diff --git a/dom/media/gmp/GMPDecryptorParent.cpp b/dom/media/gmp/GMPDecryptorParent.cpp index 49a2807a1c..b58ce17410 100644 --- a/dom/media/gmp/GMPDecryptorParent.cpp +++ b/dom/media/gmp/GMPDecryptorParent.cpp @@ -14,7 +14,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGV(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Verbose, msg) #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) diff --git a/dom/media/gmp/GMPParent.cpp b/dom/media/gmp/GMPParent.cpp index 305bd9e830..f0be727538 100644 --- a/dom/media/gmp/GMPParent.cpp +++ b/dom/media/gmp/GMPParent.cpp @@ -43,7 +43,7 @@ namespace mozilla { #undef LOG #undef LOGD -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOG(level, x, ...) MOZ_LOG(GetGMPLog(), (level), (x, ##__VA_ARGS__)) #define LOGD(x, ...) LOG(mozilla::LogLevel::Debug, "GMPParent[%p|childPid=%d] " x, this, mChildPid, ##__VA_ARGS__) diff --git a/dom/media/gmp/GMPService.cpp b/dom/media/gmp/GMPService.cpp index 5ab8279605..7b06968fa5 100644 --- a/dom/media/gmp/GMPService.cpp +++ b/dom/media/gmp/GMPService.cpp @@ -43,12 +43,10 @@ namespace mozilla { #undef LOG #endif -PRLogModuleInfo* +LogModule* GetGMPLog() { - static PRLogModuleInfo *sLog; - if (!sLog) - sLog = PR_NewLogModule("GMP"); + static LazyLogModule sLog("GMP"); return sLog; } diff --git a/dom/media/gmp/GMPService.h b/dom/media/gmp/GMPService.h index e62993cfde..fd28899585 100644 --- a/dom/media/gmp/GMPService.h +++ b/dom/media/gmp/GMPService.h @@ -21,7 +21,7 @@ template struct already_AddRefed; namespace mozilla { -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); namespace gmp { diff --git a/dom/media/gmp/GMPStorageParent.cpp b/dom/media/gmp/GMPStorageParent.cpp index 2ec2210772..c048f87695 100644 --- a/dom/media/gmp/GMPStorageParent.cpp +++ b/dom/media/gmp/GMPStorageParent.cpp @@ -25,7 +25,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg) diff --git a/dom/media/gmp/GMPTimerParent.cpp b/dom/media/gmp/GMPTimerParent.cpp index 8e58490481..260433d015 100644 --- a/dom/media/gmp/GMPTimerParent.cpp +++ b/dom/media/gmp/GMPTimerParent.cpp @@ -13,7 +13,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg) diff --git a/dom/media/gmp/GMPVideoDecoderParent.cpp b/dom/media/gmp/GMPVideoDecoderParent.cpp index 2b58bb2e6d..ffb4333349 100644 --- a/dom/media/gmp/GMPVideoDecoderParent.cpp +++ b/dom/media/gmp/GMPVideoDecoderParent.cpp @@ -21,7 +21,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGV(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Verbose, msg) #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) diff --git a/dom/media/gmp/GMPVideoEncoderParent.cpp b/dom/media/gmp/GMPVideoEncoderParent.cpp index 311ecef99c..b142e7fabf 100644 --- a/dom/media/gmp/GMPVideoEncoderParent.cpp +++ b/dom/media/gmp/GMPVideoEncoderParent.cpp @@ -23,7 +23,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetGMPLog(); +extern LogModule* GetGMPLog(); #define LOGD(msg) MOZ_LOG(GetGMPLog(), mozilla::LogLevel::Debug, msg) #define LOG(level, msg) MOZ_LOG(GetGMPLog(), (level), msg) diff --git a/dom/media/gstreamer/GStreamerFormatHelper.cpp b/dom/media/gstreamer/GStreamerFormatHelper.cpp index d7649b806a..fd545ac7ea 100644 --- a/dom/media/gstreamer/GStreamerFormatHelper.cpp +++ b/dom/media/gstreamer/GStreamerFormatHelper.cpp @@ -16,7 +16,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOG(msg, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, ("GStreamerFormatHelper " msg, ##__VA_ARGS__)) diff --git a/dom/media/gstreamer/GStreamerReader-0.10.cpp b/dom/media/gstreamer/GStreamerReader-0.10.cpp index 8ecf871c18..6b9683d475 100644 --- a/dom/media/gstreamer/GStreamerReader-0.10.cpp +++ b/dom/media/gstreamer/GStreamerReader-0.10.cpp @@ -45,8 +45,7 @@ GstFlowReturn GStreamerReader::AllocateVideoBufferFull(GstPad* aPad, if (container == nullptr) { return GST_FLOW_ERROR; } - RefPtr image = - container->CreateImage(ImageFormat::PLANAR_YCBCR).downcast(); + RefPtr image = container->CreatePlanarYCbCrImage(); /* prepare a GstBuffer pointing to the underlying PlanarYCbCrImage buffer */ GstBuffer* buf = GST_BUFFER(gst_moz_video_buffer_new()); diff --git a/dom/media/gstreamer/GStreamerReader.cpp b/dom/media/gstreamer/GStreamerReader.cpp index 58bd4dcaa3..aff07e0858 100644 --- a/dom/media/gstreamer/GStreamerReader.cpp +++ b/dom/media/gstreamer/GStreamerReader.cpp @@ -29,7 +29,7 @@ using namespace media; // Un-comment to enable logging of seek bisections. //#define SEEK_LOGGING -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOG(type, msg, ...) \ MOZ_LOG(gMediaDecoderLog, type, ("GStreamerReader(%p) " msg, this, ##__VA_ARGS__)) diff --git a/dom/media/imagecapture/ImageCapture.cpp b/dom/media/imagecapture/ImageCapture.cpp index f016647211..a169d9c380 100644 --- a/dom/media/imagecapture/ImageCapture.cpp +++ b/dom/media/imagecapture/ImageCapture.cpp @@ -18,12 +18,9 @@ namespace mozilla { -PRLogModuleInfo* GetICLog() +LogModule* GetICLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("ImageCapture"); - } + static LazyLogModule log("ImageCapture"); return log; } diff --git a/dom/media/imagecapture/ImageCapture.h b/dom/media/imagecapture/ImageCapture.h index 7a47af45e7..fc8f5385ac 100644 --- a/dom/media/imagecapture/ImageCapture.h +++ b/dom/media/imagecapture/ImageCapture.h @@ -14,7 +14,7 @@ namespace mozilla { #ifndef IC_LOG -PRLogModuleInfo* GetICLog(); +LogModule* GetICLog(); #define IC_LOG(...) MOZ_LOG(GetICLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) #endif diff --git a/dom/media/mediasink/DecodedAudioDataSink.cpp b/dom/media/mediasink/DecodedAudioDataSink.cpp index 526dd019b9..e974648d9d 100644 --- a/dom/media/mediasink/DecodedAudioDataSink.cpp +++ b/dom/media/mediasink/DecodedAudioDataSink.cpp @@ -14,7 +14,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define SINK_LOG(msg, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \ ("DecodedAudioDataSink=%p " msg, this, ##__VA_ARGS__)) diff --git a/dom/media/mediasink/VideoSink.cpp b/dom/media/mediasink/VideoSink.cpp index 6e67139d23..2b673dd670 100644 --- a/dom/media/mediasink/VideoSink.cpp +++ b/dom/media/mediasink/VideoSink.cpp @@ -8,7 +8,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define VSINK_LOG(msg, ...) \ MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \ ("VideoSink=%p " msg, this, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/ContainerParser.cpp b/dom/media/mediasource/ContainerParser.cpp index f2be166346..349723ef04 100644 --- a/dom/media/mediasource/ContainerParser.cpp +++ b/dom/media/mediasource/ContainerParser.cpp @@ -20,7 +20,7 @@ #endif #include "SourceBufferResource.h" -extern PRLogModuleInfo* GetMediaSourceSamplesLog(); +extern mozilla::LogModule* GetMediaSourceSamplesLog(); #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) diff --git a/dom/media/mediasource/MediaSource.cpp b/dom/media/mediasource/MediaSource.cpp index a35d87f13f..bce3d1ca06 100644 --- a/dom/media/mediasource/MediaSource.cpp +++ b/dom/media/mediasource/MediaSource.cpp @@ -36,21 +36,15 @@ struct JSContext; class JSObject; -PRLogModuleInfo* GetMediaSourceLog() +mozilla::LogModule* GetMediaSourceLog() { - static PRLogModuleInfo* sLogModule = nullptr; - if (!sLogModule) { - sLogModule = PR_NewLogModule("MediaSource"); - } + static mozilla::LazyLogModule sLogModule("MediaSource"); return sLogModule; } -PRLogModuleInfo* GetMediaSourceAPILog() +mozilla::LogModule* GetMediaSourceAPILog() { - static PRLogModuleInfo* sLogModule = nullptr; - if (!sLogModule) { - sLogModule = PR_NewLogModule("MediaSource"); - } + static mozilla::LazyLogModule sLogModule("MediaSource"); return sLogModule; } diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp index 80f95fc005..27fe65a9af 100644 --- a/dom/media/mediasource/MediaSourceDecoder.cpp +++ b/dom/media/mediasource/MediaSourceDecoder.cpp @@ -18,7 +18,7 @@ #include "SourceBufferList.h" #include -extern PRLogModuleInfo* GetMediaSourceLog(); +extern mozilla::LogModule* GetMediaSourceLog(); #define MSE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, ("MediaSourceDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) #define MSE_DEBUGV(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, ("MediaSourceDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/MediaSourceResource.h b/dom/media/mediasource/MediaSourceResource.h index 38b2c347fb..467232117d 100644 --- a/dom/media/mediasource/MediaSourceResource.h +++ b/dom/media/mediasource/MediaSourceResource.h @@ -11,7 +11,7 @@ #include "mozilla/Monitor.h" #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetMediaSourceLog(); +extern mozilla::LogModule* GetMediaSourceLog(); #define MSE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, ("MediaSourceResource(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/ResourceQueue.cpp b/dom/media/mediasource/ResourceQueue.cpp index 0197640629..58cc0a2fb5 100644 --- a/dom/media/mediasource/ResourceQueue.cpp +++ b/dom/media/mediasource/ResourceQueue.cpp @@ -10,7 +10,7 @@ #include "mozilla/ErrorResult.h" #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetSourceBufferResourceLog(); +extern mozilla::LogModule* GetSourceBufferResourceLog(); #define SBR_DEBUG(arg, ...) MOZ_LOG(GetSourceBufferResourceLog(), mozilla::LogLevel::Debug, ("ResourceQueue(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) #define SBR_DEBUGV(arg, ...) MOZ_LOG(GetSourceBufferResourceLog(), mozilla::LogLevel::Verbose, ("ResourceQueue(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/SourceBuffer.cpp b/dom/media/mediasource/SourceBuffer.cpp index a78ad159a2..65a26324d0 100644 --- a/dom/media/mediasource/SourceBuffer.cpp +++ b/dom/media/mediasource/SourceBuffer.cpp @@ -26,8 +26,8 @@ struct JSContext; class JSObject; -extern PRLogModuleInfo* GetMediaSourceLog(); -extern PRLogModuleInfo* GetMediaSourceAPILog(); +extern mozilla::LogModule* GetMediaSourceLog(); +extern mozilla::LogModule* GetMediaSourceAPILog(); #define MSE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, ("SourceBuffer(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) #define MSE_DEBUGV(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, ("SourceBuffer(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/SourceBufferList.cpp b/dom/media/mediasource/SourceBufferList.cpp index f84436dfea..58591843d5 100644 --- a/dom/media/mediasource/SourceBufferList.cpp +++ b/dom/media/mediasource/SourceBufferList.cpp @@ -16,8 +16,8 @@ #include "nsThreadUtils.h" #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetMediaSourceLog(); -extern PRLogModuleInfo* GetMediaSourceAPILog(); +extern mozilla::LogModule* GetMediaSourceLog(); +extern mozilla::LogModule* GetMediaSourceAPILog(); #define MSE_API(arg, ...) MOZ_LOG(GetMediaSourceAPILog(), mozilla::LogLevel::Debug, ("SourceBufferList(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) #define MSE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, ("SourceBufferList(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) diff --git a/dom/media/mediasource/SourceBufferResource.cpp b/dom/media/mediasource/SourceBufferResource.cpp index 9611a60f27..32c4230e66 100644 --- a/dom/media/mediasource/SourceBufferResource.cpp +++ b/dom/media/mediasource/SourceBufferResource.cpp @@ -13,12 +13,9 @@ #include "mozilla/Logging.h" #include "MediaData.h" -PRLogModuleInfo* GetSourceBufferResourceLog() +mozilla::LogModule* GetSourceBufferResourceLog() { - static PRLogModuleInfo* sLogModule = nullptr; - if (!sLogModule) { - sLogModule = PR_NewLogModule("SourceBufferResource"); - } + static mozilla::LazyLogModule sLogModule("SourceBufferResource"); return sLogModule; } diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index 2439899b18..e8ccefd006 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -23,22 +23,14 @@ #include -#ifdef PR_LOGGING -extern PRLogModuleInfo* GetMediaSourceLog(); +extern mozilla::LogModule* GetMediaSourceLog(); #define MSE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, ("TrackBuffersManager(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) #define MSE_DEBUGV(arg, ...) MOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, ("TrackBuffersManager(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) -#else -#define MSE_DEBUG(...) -#define MSE_DEBUGV(...) -#endif -PRLogModuleInfo* GetMediaSourceSamplesLog() +mozilla::LogModule* GetMediaSourceSamplesLog() { - static PRLogModuleInfo* sLogModule = nullptr; - if (!sLogModule) { - sLogModule = PR_NewLogModule("MediaSourceSamples"); - } + static mozilla::LazyLogModule sLogModule("MediaSourceSamples"); return sLogModule; } #define SAMPLE_DEBUG(arg, ...) MOZ_LOG(GetMediaSourceSamplesLog(), mozilla::LogLevel::Debug, ("TrackBuffersManager(%p:%s)::%s: " arg, this, mType.get(), __func__, ##__VA_ARGS__)) diff --git a/dom/media/ogg/OggCodecState.cpp b/dom/media/ogg/OggCodecState.cpp index e0761d0fb7..8184053ec7 100644 --- a/dom/media/ogg/OggCodecState.cpp +++ b/dom/media/ogg/OggCodecState.cpp @@ -32,7 +32,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) /** Decoder base class for Ogg-encapsulated streams. */ diff --git a/dom/media/ogg/OggReader.cpp b/dom/media/ogg/OggReader.cpp index 2ff98a899e..690f879a43 100644 --- a/dom/media/ogg/OggReader.cpp +++ b/dom/media/ogg/OggReader.cpp @@ -37,7 +37,7 @@ namespace mozilla { // Un-comment to enable logging of seek bisections. //#define SEEK_LOGGING -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) #ifdef SEEK_LOGGING #define SEEK_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) diff --git a/dom/media/ogg/OpusParser.cpp b/dom/media/ogg/OpusParser.cpp index 8bdaf7f550..c450df9a22 100644 --- a/dom/media/ogg/OpusParser.cpp +++ b/dom/media/ogg/OpusParser.cpp @@ -24,7 +24,7 @@ extern "C" { namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define OPUS_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) OpusParser::OpusParser(): diff --git a/dom/media/omx/AudioOffloadPlayer.cpp b/dom/media/omx/AudioOffloadPlayer.cpp index 9aef66d081..52ff832bf7 100644 --- a/dom/media/omx/AudioOffloadPlayer.cpp +++ b/dom/media/omx/AudioOffloadPlayer.cpp @@ -42,7 +42,7 @@ using namespace android; namespace mozilla { -PRLogModuleInfo* gAudioOffloadPlayerLog; +LazyLogModule gAudioOffloadPlayerLog("AudioOffloadPlayer"); #define AUDIO_OFFLOAD_LOG(type, msg) \ MOZ_LOG(gAudioOffloadPlayerLog, type, msg) @@ -63,10 +63,6 @@ AudioOffloadPlayer::AudioOffloadPlayer(MediaOmxCommonDecoder* aObserver) : { MOZ_ASSERT(NS_IsMainThread()); - if (!gAudioOffloadPlayerLog) { - gAudioOffloadPlayerLog = PR_NewLogModule("AudioOffloadPlayer"); - } - CHECK(aObserver); #if ANDROID_VERSION >= 21 mSessionId = AudioSystem::newAudioUniqueId(); diff --git a/dom/media/omx/AudioOutput.cpp b/dom/media/omx/AudioOutput.cpp index 9754da32b3..132d09d77c 100644 --- a/dom/media/omx/AudioOutput.cpp +++ b/dom/media/omx/AudioOutput.cpp @@ -24,7 +24,7 @@ namespace mozilla { -extern PRLogModuleInfo* gAudioOffloadPlayerLog; +extern LazyLogModule gAudioOffloadPlayerLog; #define AUDIO_OFFLOAD_LOG(type, msg) \ MOZ_LOG(gAudioOffloadPlayerLog, type, msg) @@ -37,9 +37,6 @@ AudioOutput::AudioOutput(int aSessionId, int aUid) : mUid(aUid), mSessionId(aSessionId) { - if (!gAudioOffloadPlayerLog) { - gAudioOffloadPlayerLog = PR_NewLogModule("AudioOffloadPlayer"); - } } AudioOutput::~AudioOutput() diff --git a/dom/media/omx/I420ColorConverterHelper.cpp b/dom/media/omx/I420ColorConverterHelper.cpp index 8afcbc4cb0..f1e248499d 100644 --- a/dom/media/omx/I420ColorConverterHelper.cpp +++ b/dom/media/omx/I420ColorConverterHelper.cpp @@ -10,7 +10,7 @@ #include "mozilla/Logging.h" -PRLogModuleInfo *gI420ColorConverterHelperLog; +mozilla::LazyLogModule gI420ColorConverterHelperLog("I420ColorConverterHelper"); #define LOG(msg...) MOZ_LOG(gI420ColorConverterHelperLog, mozilla::LogLevel::Warning, (msg)) namespace android { @@ -19,9 +19,6 @@ I420ColorConverterHelper::I420ColorConverterHelper() : mHandle(nullptr) , mConverter({nullptr, nullptr, nullptr, nullptr, nullptr}) { - if (!gI420ColorConverterHelperLog) { - gI420ColorConverterHelperLog = PR_NewLogModule("I420ColorConverterHelper"); - } } I420ColorConverterHelper::~I420ColorConverterHelper() diff --git a/dom/media/omx/MediaCodecProxy.cpp b/dom/media/omx/MediaCodecProxy.cpp index 5e53c258a9..d94e97b1c5 100644 --- a/dom/media/omx/MediaCodecProxy.cpp +++ b/dom/media/omx/MediaCodecProxy.cpp @@ -14,7 +14,6 @@ #include #define MCP_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "MediaCodecProxy", __VA_ARGS__) -#define TIMEOUT_DEQUEUE_INPUTBUFFER_MS 1000000ll namespace android { @@ -77,24 +76,20 @@ struct MediaCodecInterfaceWrapper sp MediaCodecProxy::CreateByType(sp aLooper, const char *aMime, - bool aEncoder, - wp aListener) + bool aEncoder) { sp codec = new MediaCodecProxy(aLooper, aMime, - aEncoder, - aListener); + aEncoder); return codec; } MediaCodecProxy::MediaCodecProxy(sp aLooper, const char *aMime, - bool aEncoder, - wp aListener) + bool aEncoder) : mCodecLooper(aLooper) , mCodecMime(aMime) , mCodecEncoder(aEncoder) - , mListener(aListener) , mMediaCodecLock("MediaCodecProxy::mMediaCodecLock") , mPendingRequestMediaResource(false) { @@ -107,50 +102,29 @@ MediaCodecProxy::~MediaCodecProxy() } bool -MediaCodecProxy::AskMediaCodecAndWait() +MediaCodecProxy::AllocateAudioMediaCodec() { if (mResourceClient || mCodec.get()) { return false; } - if (strncasecmp(mCodecMime.get(), "video/", 6) == 0) { - mozilla::MediaSystemResourceType type = - mCodecEncoder ? mozilla::MediaSystemResourceType::VIDEO_ENCODER : - mozilla::MediaSystemResourceType::VIDEO_DECODER; - mResourceClient = new mozilla::MediaSystemResourceClient(type); - mResourceClient->SetListener(this); - } else if (strncasecmp(mCodecMime.get(), "audio/", 6) == 0) { + if (strncasecmp(mCodecMime.get(), "audio/", 6) == 0) { if (allocateCodec()) { return true; } } - - if (!mResourceClient) { - return false; - } - - mozilla::MonitorAutoLock mon(mMediaCodecLock); - mPendingRequestMediaResource = true; - // request video codec - mResourceClient->Acquire(); - - while (mPendingRequestMediaResource) { - mMediaCodecLock.Wait(); - } - MCP_LOG("AskMediaCodecAndWait complete"); - - return true; + return false; } -bool -MediaCodecProxy::AsyncAskMediaCodec() +RefPtr +MediaCodecProxy::AsyncAllocateVideoMediaCodec() { if (mResourceClient || mCodec.get()) { - return false; + return CodecPromise::CreateAndReject(true, __func__); } if (strncasecmp(mCodecMime.get(), "video/", 6) != 0) { - return false; + return CodecPromise::CreateAndReject(true, __func__); } // request video codec mozilla::MediaSystemResourceType type = @@ -160,12 +134,14 @@ MediaCodecProxy::AsyncAskMediaCodec() mResourceClient->SetListener(this); mResourceClient->Acquire(); - return true; + RefPtr p = mCodecPromise.Ensure(__func__); + return p.forget(); } void MediaCodecProxy::ReleaseMediaCodec() { + mCodecPromise.RejectIfExists(true, __func__); releaseCodec(); if (!mResourceClient) { @@ -500,11 +476,7 @@ MediaCodecProxy::ResourceReserved() // Create MediaCodec if (!allocateCodec()) { ReleaseMediaCodec(); - // Notification - sp listener = mListener.promote(); - if (listener != nullptr) { - listener->codecCanceled(); - } + mCodecPromise.RejectIfExists(true, __func__); return; } @@ -513,11 +485,7 @@ MediaCodecProxy::ResourceReserved() mPendingRequestMediaResource = false; mon.NotifyAll(); - // Notification - sp listener = mListener.promote(); - if (listener != nullptr) { - listener->codecReserved(); - } + mCodecPromise.ResolveIfExists(true, __func__); } // Called on ImageBridge thread @@ -525,11 +493,7 @@ void MediaCodecProxy::ResourceReserveFailed() { ReleaseMediaCodec(); - // Notification - sp listener = mListener.promote(); - if (listener != nullptr) { - listener->codecCanceled(); - } + mCodecPromise.RejectIfExists(true, __func__); } bool MediaCodecProxy::Prepare() @@ -571,7 +535,8 @@ bool MediaCodecProxy::UpdateOutputBuffers() } status_t MediaCodecProxy::Input(const uint8_t* aData, uint32_t aDataSize, - int64_t aTimestampUsecs, uint64_t aflags) + int64_t aTimestampUsecs, uint64_t aflags, + int64_t aTimeoutUs) { // Read Lock for mCodec { @@ -583,9 +548,11 @@ status_t MediaCodecProxy::Input(const uint8_t* aData, uint32_t aDataSize, } size_t index; - status_t err = dequeueInputBuffer(&index, TIMEOUT_DEQUEUE_INPUTBUFFER_MS); + status_t err = dequeueInputBuffer(&index, aTimeoutUs); if (err != OK) { - MCP_LOG("dequeueInputBuffer returned %d", err); + if (err != -EAGAIN) { + MCP_LOG("dequeueInputBuffer returned %d", err); + } return err; } diff --git a/dom/media/omx/MediaCodecProxy.h b/dom/media/omx/MediaCodecProxy.h index 9f3d1e34ca..48af32920c 100644 --- a/dom/media/omx/MediaCodecProxy.h +++ b/dom/media/omx/MediaCodecProxy.h @@ -14,7 +14,8 @@ #include "mozilla/media/MediaSystemResourceClient.h" #include "mozilla/Monitor.h" -#include "RefPtr.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" namespace android { // This class is intended to be a proxy for MediaCodec with codec resource @@ -27,20 +28,7 @@ class MediaCodecProxy : public RefBase , public mozilla::MediaSystemResourceReservationListener { public: - /* Codec resource notification listener. - * All functions are called on the Binder thread. - */ - struct CodecResourceListener : public virtual RefBase { - /* The codec resource is reserved and can be granted. - * The client can allocate the requested resource. - */ - virtual void codecReserved() = 0; - /* The codec resource is not reserved any more. - * The client should release the resource as soon as possible if the - * resource is still being held. - */ - virtual void codecCanceled() = 0; - }; + typedef mozilla::MozPromise CodecPromise; enum Capability { kEmptyCapability = 0x00000000, @@ -58,8 +46,7 @@ public: // Only support MediaCodec::CreateByType() static sp CreateByType(sp aLooper, const char *aMime, - bool aEncoder, - wp aListener=nullptr); + bool aEncoder); // MediaCodec methods status_t configure(const sp &aFormat, @@ -129,7 +116,7 @@ public: // If aData is null, will notify decoder input EOS status_t Input(const uint8_t* aData, uint32_t aDataSize, - int64_t aTimestampUsecs, uint64_t flags); + int64_t aTimestampUsecs, uint64_t flags, int64_t aTimeoutUs = 0); status_t Output(MediaBuffer** aBuffer, int64_t aTimeoutUs); bool Prepare(); void ReleaseMediaResources(); @@ -138,14 +125,11 @@ public: void ReleaseMediaBuffer(MediaBuffer* abuffer); - // It asks for the OMX codec and blocked until the resource is grant to be - // allocated. - // Audio codec allocation should use this. - bool AskMediaCodecAndWait(); + // It allocates audio MediaCodec synchronously. + bool AllocateAudioMediaCodec(); - // It asks for the OMX codec asynchronously. - // Only video codec is supported. - bool AsyncAskMediaCodec(); + // It allocates video MediaCodec asynchronously. + RefPtr AsyncAllocateVideoMediaCodec(); // Free the OMX codec so others can allocate it. void ReleaseMediaCodec(); @@ -166,8 +150,7 @@ private: // Constructor for MediaCodecProxy::CreateByType MediaCodecProxy(sp aLooper, const char *aMime, - bool aEncoder, - wp aListener); + bool aEncoder); // Allocate Codec Resource bool allocateCodec(); @@ -179,8 +162,7 @@ private: nsCString mCodecMime; bool mCodecEncoder; - // Codec Resource Notification Listener - wp mListener; + mozilla::MozPromiseHolder mCodecPromise; // Media Resource Management RefPtr mResourceClient; diff --git a/dom/media/omx/MediaCodecReader.cpp b/dom/media/omx/MediaCodecReader.cpp index 698e4f4a63..fcf1bd6d9c 100644 --- a/dom/media/omx/MediaCodecReader.cpp +++ b/dom/media/omx/MediaCodecReader.cpp @@ -68,33 +68,6 @@ IsValidTimestampUs(int64_t aTimestamp) return aTimestamp >= INT64_C(0); } -MediaCodecReader::VideoResourceListener::VideoResourceListener( - MediaCodecReader* aReader) - : mReader(aReader) -{ -} - -MediaCodecReader::VideoResourceListener::~VideoResourceListener() -{ - mReader = nullptr; -} - -void -MediaCodecReader::VideoResourceListener::codecReserved() -{ - if (mReader) { - mReader->VideoCodecReserved(); - } -} - -void -MediaCodecReader::VideoResourceListener::codecCanceled() -{ - if (mReader) { - mReader->VideoCodecCanceled(); - } -} - MediaCodecReader::TrackInputCopier::~TrackInputCopier() { } @@ -276,7 +249,6 @@ MediaCodecReader::MediaCodecReader(AbstractMediaDecoder* aDecoder) , mNextParserPosition(INT64_C(0)) , mParsedDataLength(INT64_C(0)) { - mVideoListener = new VideoResourceListener(this); } MediaCodecReader::~MediaCodecReader() @@ -1111,6 +1083,7 @@ MediaCodecReader::ReleaseCriticalResources() mMediaResourceRequest.DisconnectIfExists(); mMediaResourcePromise.RejectIfExists(true, __func__); mMetadataPromise.RejectIfExists(ReadMetadataFailureReason::METADATA_ERROR, __func__); + mVideoCodecRequest.DisconnectIfExists(); ResetDecode(); // Before freeing a video codec, all video buffers needed to be released @@ -1313,12 +1286,12 @@ MediaCodecReader::CreateMediaCodecs() bool isWaiting = false; RefPtr p = mMediaResourcePromise.Ensure(__func__); - if (!CreateMediaCodec(mLooper, mAudioTrack, false, isWaiting, nullptr)) { + if (!CreateMediaCodec(mLooper, mAudioTrack, isWaiting)) { mMediaResourcePromise.Reject(true, __func__); return p; } - if (!CreateMediaCodec(mLooper, mVideoTrack, true, isWaiting, mVideoListener)) { + if (!CreateMediaCodec(mLooper, mVideoTrack, isWaiting)) { mMediaResourcePromise.Reject(true, __func__); return p; } @@ -1334,16 +1307,14 @@ MediaCodecReader::CreateMediaCodecs() bool MediaCodecReader::CreateMediaCodec(sp& aLooper, Track& aTrack, - bool aAsync, - bool& aIsWaiting, - wp aListener) + bool& aIsWaiting) { if (aTrack.mSource != nullptr && aTrack.mCodec == nullptr) { sp sourceFormat = aTrack.mSource->getFormat(); const char* mime; if (sourceFormat->findCString(kKeyMIMEType, &mime)) { - aTrack.mCodec = MediaCodecProxy::CreateByType(aLooper, mime, false, aListener); + aTrack.mCodec = MediaCodecProxy::CreateByType(aLooper, mime, false); } if (aTrack.mCodec == nullptr) { @@ -1372,7 +1343,7 @@ MediaCodecReader::CreateMediaCodec(sp& aLooper, #endif } - if (!aAsync && aTrack.mCodec->AskMediaCodecAndWait()) { + if (aTrack.mType == Track::kAudio && aTrack.mCodec->AllocateAudioMediaCodec()) { // Pending configure() and start() to codecReserved() if the creation // should be asynchronous. if (!aTrack.mCodec->allocated() || !ConfigureMediaCodec(aTrack)){ @@ -1380,14 +1351,18 @@ MediaCodecReader::CreateMediaCodec(sp& aLooper, DestroyMediaCodec(aTrack); return false; } - } else if (aAsync) { - if (aTrack.mCodec->AsyncAskMediaCodec()) { - aIsWaiting = true; - } else { - NS_WARNING("Couldn't request MediaCodec asynchronously"); - DestroyMediaCodec(aTrack); - return false; - } + } else if (aTrack.mType == Track::kVideo) { + aIsWaiting = true; + RefPtr self = this; + mVideoCodecRequest.Begin(aTrack.mCodec->AsyncAllocateVideoMediaCodec() + ->Then(OwnerThread(), __func__, + [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->mMediaResourcePromise.ResolveIfExists(true, __func__); + }, [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->mMediaResourcePromise.RejectIfExists(true, __func__); + })); } } @@ -1949,18 +1924,5 @@ MediaCodecReader::ClearColorConverterBuffer() mColorConverterBufferSize = 0; } -// Called on Binder thread. -void -MediaCodecReader::VideoCodecReserved() -{ - mMediaResourcePromise.ResolveIfExists(true, __func__); -} - -// Called on Binder thread. -void -MediaCodecReader::VideoCodecCanceled() -{ - mMediaResourcePromise.RejectIfExists(true, __func__); -} } // namespace mozilla diff --git a/dom/media/omx/MediaCodecReader.h b/dom/media/omx/MediaCodecReader.h index 92725470ed..e6b80d4c79 100644 --- a/dom/media/omx/MediaCodecReader.h +++ b/dom/media/omx/MediaCodecReader.h @@ -161,11 +161,6 @@ protected: // Called on MediaCodecReader::mLooper thread. void onMessageReceived(const android::sp& aMessage); - // Receive a notify from ResourceListener. - // Called on Binder thread. - virtual void VideoCodecReserved(); - virtual void VideoCodecCanceled(); - virtual bool CreateExtractor(); virtual void HandleResourceAllocated(); @@ -177,31 +172,12 @@ protected: MozPromiseRequestHolder mMediaResourceRequest; MozPromiseHolder mMediaResourcePromise; + MozPromiseRequestHolder mVideoCodecRequest; + private: virtual bool HasAudio() override; virtual bool HasVideo() override; - // An intermediary class that can be managed by android::sp. - // Redirect codecReserved() and codecCanceled() to MediaCodecReader. - class VideoResourceListener : public android::MediaCodecProxy::CodecResourceListener - { - public: - VideoResourceListener(MediaCodecReader* aReader); - ~VideoResourceListener(); - - virtual void codecReserved(); - virtual void codecCanceled(); - - private: - // Forbidden - VideoResourceListener() = delete; - VideoResourceListener(const VideoResourceListener& rhs) = delete; - const VideoResourceListener& operator=(const VideoResourceListener& rhs) = delete; - - MediaCodecReader* mReader; - }; - friend class VideoResourceListener; - class VorbisInputCopier : public TrackInputCopier { virtual bool Copy(android::MediaBuffer* aSourceBuffer, @@ -338,11 +314,9 @@ private: void DestroyMediaSources(); RefPtr CreateMediaCodecs(); - static bool CreateMediaCodec(android::sp& aLooper, - Track& aTrack, - bool aAsync, - bool& aIsWaiting, - android::wp aListener); + bool CreateMediaCodec(android::sp& aLooper, + Track& aTrack, + bool& aIsWaiting); static bool ConfigureMediaCodec(Track& aTrack); void DestroyMediaCodecs(); static void DestroyMediaCodec(Track& aTrack); @@ -397,8 +371,6 @@ private: void ReleaseAllTextureClients(); - android::sp mVideoListener; - android::sp mLooper; android::sp mMetaData; diff --git a/dom/media/omx/MediaOmxCommonDecoder.cpp b/dom/media/omx/MediaOmxCommonDecoder.cpp index f79d79c729..6d44dbf4cc 100644 --- a/dom/media/omx/MediaOmxCommonDecoder.cpp +++ b/dom/media/omx/MediaOmxCommonDecoder.cpp @@ -20,7 +20,7 @@ using namespace android; namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define DECODER_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) MediaOmxCommonDecoder::MediaOmxCommonDecoder(MediaDecoderOwner* aOwner) diff --git a/dom/media/omx/MediaOmxCommonReader.cpp b/dom/media/omx/MediaOmxCommonReader.cpp index d8c5d20805..a05cd5829a 100644 --- a/dom/media/omx/MediaOmxCommonReader.cpp +++ b/dom/media/omx/MediaOmxCommonReader.cpp @@ -23,7 +23,7 @@ using namespace android; namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define DECODER_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) MediaOmxCommonReader::MediaOmxCommonReader(AbstractMediaDecoder *aDecoder) diff --git a/dom/media/omx/MediaOmxReader.cpp b/dom/media/omx/MediaOmxReader.cpp index d1028cbad2..4298abfce6 100644 --- a/dom/media/omx/MediaOmxReader.cpp +++ b/dom/media/omx/MediaOmxReader.cpp @@ -28,7 +28,7 @@ using namespace android; namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define DECODER_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) class MediaOmxReader::ProcessCachedDataTask : public Task @@ -204,7 +204,7 @@ nsresult MediaOmxReader::InitOmxDecoder() if (!mExtractor.get()) { return NS_ERROR_FAILURE; } - mOmxDecoder = new OmxDecoder(mDecoder); + mOmxDecoder = new OmxDecoder(mDecoder, OwnerThread()); if (!mOmxDecoder->Init(mExtractor)) { return NS_ERROR_FAILURE; } diff --git a/dom/media/omx/OMXCodecProxy.cpp b/dom/media/omx/OMXCodecProxy.cpp index dd250a89ae..6fe150d46e 100644 --- a/dom/media/omx/OMXCodecProxy.cpp +++ b/dom/media/omx/OMXCodecProxy.cpp @@ -65,6 +65,7 @@ OMXCodecProxy::OMXCodecProxy( OMXCodecProxy::~OMXCodecProxy() { mState = ResourceState::END; + mCodecPromise.RejectIfExists(true, __func__); if (mOMXCodec.get()) { wp tmp = mOMXCodec; @@ -87,34 +88,13 @@ OMXCodecProxy::~OMXCodecProxy() mComponentName = nullptr; } -void OMXCodecProxy::setListener(const wp& listener) -{ - Mutex::Autolock autoLock(mLock); - mListener = listener; -} - -void OMXCodecProxy::notifyResourceReserved() -{ - sp listener = mListener.promote(); - if (listener != nullptr) { - listener->codecReserved(); - } -} - -void OMXCodecProxy::notifyResourceCanceled() -{ - sp listener = mListener.promote(); - if (listener != nullptr) { - listener->codecCanceled(); - } -} - -void OMXCodecProxy::requestResource() +RefPtr +OMXCodecProxy::requestResource() { Mutex::Autolock autoLock(mLock); if (mResourceClient) { - return; + return CodecPromise::CreateAndReject(true, __func__); } mState = ResourceState::WAITING; @@ -123,6 +103,9 @@ void OMXCodecProxy::requestResource() mResourceClient = new mozilla::MediaSystemResourceClient(type); mResourceClient->SetListener(this); mResourceClient->Acquire(); + + RefPtr p = mCodecPromise.Ensure(__func__); + return p.forget(); } // Called on ImageBridge thread @@ -132,13 +115,14 @@ OMXCodecProxy::ResourceReserved() Mutex::Autolock autoLock(mLock); if (mState != ResourceState::WAITING) { + mCodecPromise.RejectIfExists(true, __func__); return; } const char *mime; if (!mSrcMeta->findCString(kKeyMIMEType, &mime)) { mState = ResourceState::END; - notifyResourceCanceled(); + mCodecPromise.RejectIfExists(true, __func__); return; } @@ -147,7 +131,7 @@ OMXCodecProxy::ResourceReserved() mOMXCodec = OMXCodec::Create(mOMX, mSrcMeta, mIsEncoder, mSource, mComponentName, mFlags, mNativeWindow); if (mOMXCodec == nullptr) { mState = ResourceState::END; - notifyResourceCanceled(); + mCodecPromise.RejectIfExists(true, __func__); return; } // Check if this video is sized such that we're comfortable @@ -168,7 +152,7 @@ OMXCodecProxy::ResourceReserved() width, height, maxWidth, maxHeight); mOMXCodec.clear(); mState = ResourceState::END; - notifyResourceCanceled(); + mCodecPromise.RejectIfExists(true, __func__); return; } @@ -176,13 +160,13 @@ OMXCodecProxy::ResourceReserved() NS_WARNING("Couldn't start OMX video source"); mOMXCodec.clear(); mState = ResourceState::END; - notifyResourceCanceled(); + mCodecPromise.RejectIfExists(true, __func__); return; } } mState = ResourceState::ACQUIRED; - notifyResourceReserved(); + mCodecPromise.ResolveIfExists(true, __func__); } // Called on ImageBridge thread @@ -191,9 +175,11 @@ OMXCodecProxy::ResourceReserveFailed() { Mutex::Autolock autoLock(mLock); mState = ResourceState::NOT_ACQUIRED; + mCodecPromise.RejectIfExists(true, __func__); } -status_t OMXCodecProxy::start(MetaData *params) +status_t +OMXCodecProxy::start(MetaData *params) { Mutex::Autolock autoLock(mLock); @@ -204,7 +190,8 @@ status_t OMXCodecProxy::start(MetaData *params) return mOMXCodec->start(); } -status_t OMXCodecProxy::stop() +status_t +OMXCodecProxy::stop() { Mutex::Autolock autoLock(mLock); @@ -215,7 +202,8 @@ status_t OMXCodecProxy::stop() return mOMXCodec->stop(); } -sp OMXCodecProxy::getFormat() +sp +OMXCodecProxy::getFormat() { Mutex::Autolock autoLock(mLock); @@ -227,7 +215,8 @@ sp OMXCodecProxy::getFormat() return mOMXCodec->getFormat(); } -status_t OMXCodecProxy::read(MediaBuffer **buffer, const ReadOptions *options) +status_t +OMXCodecProxy::read(MediaBuffer **buffer, const ReadOptions *options) { Mutex::Autolock autoLock(mLock); @@ -238,7 +227,8 @@ status_t OMXCodecProxy::read(MediaBuffer **buffer, const ReadOptions *options) return mOMXCodec->read(buffer, options); } -status_t OMXCodecProxy::pause() +status_t +OMXCodecProxy::pause() { Mutex::Autolock autoLock(mLock); diff --git a/dom/media/omx/OMXCodecProxy.h b/dom/media/omx/OMXCodecProxy.h index 8701b73835..6568db5b07 100644 --- a/dom/media/omx/OMXCodecProxy.h +++ b/dom/media/omx/OMXCodecProxy.h @@ -14,7 +14,8 @@ #include #include "mozilla/media/MediaSystemResourceClient.h" -#include "RefPtr.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" namespace android { @@ -24,20 +25,7 @@ class OMXCodecProxy : public MediaSource , public mozilla::MediaSystemResourceReservationListener { public: - /* Codec resource notification listener. - * All functions are called on the Binder thread. - */ - struct CodecResourceListener : public virtual RefBase { - /* The codec resource is reserved and can be granted. - * The client can allocate the requested resource. - */ - virtual void codecReserved() = 0; - /* The codec resource is not reserved any more. - * The client should release the resource as soon as possible if the - * resource is still being held. - */ - virtual void codecCanceled() = 0; - }; + typedef mozilla::MozPromise CodecPromise; // Enumeration for the valid resource allcoation states enum class ResourceState : int8_t { @@ -56,9 +44,7 @@ public: uint32_t flags = 0, const sp &nativeWindow = nullptr); - void setListener(const wp& listener); - - void requestResource(); + RefPtr requestResource(); // MediaSystemResourceReservationListener void ResourceReserved() override; @@ -87,11 +73,6 @@ protected: virtual ~OMXCodecProxy(); - void notifyResourceReserved(); - void notifyResourceCanceled(); - - void notifyStatusChangedLocked(); - private: OMXCodecProxy(const OMXCodecProxy &); OMXCodecProxy &operator=(const OMXCodecProxy &); @@ -112,9 +93,7 @@ private: RefPtr mResourceClient; ResourceState mState; - - // Codec Resource Notification Listener - wp mListener; + mozilla::MozPromiseHolder mCodecPromise; }; } // namespace android diff --git a/dom/media/omx/OmxDecoder.cpp b/dom/media/omx/OmxDecoder.cpp index 1bc2e137e3..531256ecad 100644 --- a/dom/media/omx/OmxDecoder.cpp +++ b/dom/media/omx/OmxDecoder.cpp @@ -37,7 +37,7 @@ #define OD_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "OmxDecoder", __VA_ARGS__) #undef LOG -PRLogModuleInfo *gOmxDecoderLog; +mozilla::LazyLogModule gOmxDecoderLog("OmxDecoder"); #define LOG(type, msg...) MOZ_LOG(gOmxDecoderLog, type, (msg)) using namespace MPAPI; @@ -46,7 +46,8 @@ using namespace mozilla::gfx; using namespace mozilla::layers; using namespace android; -OmxDecoder::OmxDecoder(AbstractMediaDecoder *aDecoder) : +OmxDecoder::OmxDecoder(AbstractMediaDecoder *aDecoder, + mozilla::TaskQueue* aTaskQueue) : mDecoder(aDecoder), mDisplayWidth(0), mDisplayHeight(0), @@ -59,11 +60,12 @@ OmxDecoder::OmxDecoder(AbstractMediaDecoder *aDecoder) : mAudioChannels(-1), mAudioSampleRate(-1), mDurationUs(-1), - mVideoLastFrameTime(-1), + mLastSeekTime(-1), mVideoBuffer(nullptr), mAudioBuffer(nullptr), mIsVideoSeeking(false), mAudioMetadataRead(false), + mTaskQueue(aTaskQueue), mAudioPaused(false), mVideoPaused(false) { @@ -89,15 +91,6 @@ OmxDecoder::~OmxDecoder() mLooper->stop(); } -void OmxDecoder::codecReserved() -{ - mMediaResourcePromise.ResolveIfExists(true, __func__); -} -void OmxDecoder::codecCanceled() -{ - mMediaResourcePromise.RejectIfExists(true, __func__); -} - static sp sOMX = nullptr; static sp GetOMX() { @@ -107,11 +100,8 @@ static sp GetOMX() return sOMX; } -bool OmxDecoder::Init(sp& extractor) { - if (!gOmxDecoderLog) { - gOmxDecoderLog = PR_NewLogModule("OmxDecoder"); - } - +bool +OmxDecoder::Init(sp& extractor) { sp meta = extractor->getMetaData(); ssize_t audioTrackIndex = -1; @@ -157,7 +147,8 @@ bool OmxDecoder::Init(sp& extractor) { return true; } -bool OmxDecoder::EnsureMetadata() { +bool +OmxDecoder::EnsureMetadata() { // calculate duration int64_t totalDurationUs = 0; int64_t durationUs = 0; @@ -217,7 +208,8 @@ static bool isInEmulator() return !strncmp(propQemu, "1", 1); } -RefPtr OmxDecoder::AllocateMediaResources() +RefPtr +OmxDecoder::AllocateMediaResources() { RefPtr p = mMediaResourcePromise.Ensure(__func__); @@ -274,9 +266,16 @@ RefPtr OmxDecoder::Allocate mMediaResourcePromise.Reject(true, __func__); return p; } else { - sp listener = this; - mVideoSource->setListener(listener); - mVideoSource->requestResource(); + sp self = this; + mVideoCodecRequest.Begin(mVideoSource->requestResource() + ->Then(OwnerThread(), __func__, + [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->mMediaResourcePromise.ResolveIfExists(true, __func__); + }, [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->mMediaResourcePromise.RejectIfExists(true, __func__); + })); } } @@ -337,7 +336,9 @@ RefPtr OmxDecoder::Allocate } -void OmxDecoder::ReleaseMediaResources() { +void +OmxDecoder::ReleaseMediaResources() { + mVideoCodecRequest.DisconnectIfExists(); mMediaResourcePromise.RejectIfExists(true, __func__); ReleaseVideoBuffer(); @@ -381,9 +382,14 @@ void OmxDecoder::ReleaseMediaResources() { mNativeWindowClient.clear(); mNativeWindow.clear(); + + // Reset this variable to make the first seek go to the previous keyframe + // when resuming + mLastSeekTime = -1; } -bool OmxDecoder::SetVideoFormat() { +bool +OmxDecoder::SetVideoFormat() { const char *componentName; if (!mVideoSource->getFormat()->findInt32(kKeyWidth, &mVideoWidth) || @@ -439,7 +445,8 @@ bool OmxDecoder::SetVideoFormat() { return true; } -bool OmxDecoder::SetAudioFormat() { +bool +OmxDecoder::SetAudioFormat() { // If the format changed, update our cached info. if (!mAudioSource->getFormat()->findInt32(kKeyChannelCount, &mAudioChannels) || !mAudioSource->getFormat()->findInt32(kKeySampleRate, &mAudioSampleRate)) { @@ -452,26 +459,30 @@ bool OmxDecoder::SetAudioFormat() { return true; } -void OmxDecoder::ReleaseDecoder() +void +OmxDecoder::ReleaseDecoder() { mDecoder = nullptr; } -void OmxDecoder::ReleaseVideoBuffer() { +void +OmxDecoder::ReleaseVideoBuffer() { if (mVideoBuffer) { mVideoBuffer->release(); mVideoBuffer = nullptr; } } -void OmxDecoder::ReleaseAudioBuffer() { +void +OmxDecoder::ReleaseAudioBuffer() { if (mAudioBuffer) { mAudioBuffer->release(); mAudioBuffer = nullptr; } } -void OmxDecoder::PlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { +void +OmxDecoder::PlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { void *y = aData; void *u = static_cast(y) + mVideoStride * mVideoSliceHeight; void *v = static_cast(u) + mVideoStride/2 * mVideoSliceHeight/2; @@ -483,7 +494,8 @@ void OmxDecoder::PlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aD v, mVideoStride/2, mVideoWidth/2, mVideoHeight/2, 0, 0); } -void OmxDecoder::CbYCrYFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { +void +OmxDecoder::CbYCrYFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { aFrame->Set(aTimeUs, aKeyFrame, aData, aSize, mVideoStride, mVideoSliceHeight, mVideoRotation, aData, mVideoStride, mVideoWidth, mVideoHeight, 1, 1, @@ -491,7 +503,8 @@ void OmxDecoder::CbYCrYFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, s aData, mVideoStride, mVideoWidth/2, mVideoHeight/2, 2, 3); } -void OmxDecoder::SemiPlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { +void +OmxDecoder::SemiPlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { void *y = aData; void *uv = static_cast(y) + (mVideoStride * mVideoSliceHeight); @@ -502,13 +515,15 @@ void OmxDecoder::SemiPlanarYUV420Frame(VideoFrame *aFrame, int64_t aTimeUs, void uv, mVideoStride, mVideoWidth/2, mVideoHeight/2, 1, 1); } -void OmxDecoder::SemiPlanarYVU420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { +void +OmxDecoder::SemiPlanarYVU420Frame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { SemiPlanarYUV420Frame(aFrame, aTimeUs, aData, aSize, aKeyFrame); aFrame->Cb.mOffset = 1; aFrame->Cr.mOffset = 0; } -bool OmxDecoder::ToVideoFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { +bool +OmxDecoder::ToVideoFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, size_t aSize, bool aKeyFrame) { const int OMX_QCOM_COLOR_FormatYVU420SemiPlanar = 0x7FA30C00; aFrame->mGraphicBuffer = nullptr; @@ -533,13 +548,15 @@ bool OmxDecoder::ToVideoFrame(VideoFrame *aFrame, int64_t aTimeUs, void *aData, return true; } -bool OmxDecoder::ToAudioFrame(AudioFrame *aFrame, int64_t aTimeUs, void *aData, size_t aDataOffset, size_t aSize, int32_t aAudioChannels, int32_t aAudioSampleRate) +bool +OmxDecoder::ToAudioFrame(AudioFrame *aFrame, int64_t aTimeUs, void *aData, size_t aDataOffset, size_t aSize, int32_t aAudioChannels, int32_t aAudioSampleRate) { aFrame->Set(aTimeUs, static_cast(aData) + aDataOffset, aSize, aAudioChannels, aAudioSampleRate); return true; } -bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, +bool +OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, bool aKeyframeSkip, bool aDoSeek) { if (!mVideoSource.get()) @@ -559,12 +576,13 @@ bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, MediaSource::ReadOptions::SeekMode seekMode; // If the last timestamp of decoded frame is smaller than seekTime, // seek to next key frame. Otherwise seek to the previos one. - if (mVideoLastFrameTime > aTimeUs || mVideoLastFrameTime == -1) { + OD_LOG("SeekTime: %lld, mLastSeekTime:%lld", aTimeUs, mLastSeekTime); + if (mLastSeekTime == -1 || mLastSeekTime > aTimeUs) { seekMode = MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC; } else { seekMode = MediaSource::ReadOptions::SEEK_NEXT_SYNC; } - + mLastSeekTime = aTimeUs; bool findNextBuffer = true; while (findNextBuffer) { options.setSeekTo(aTimeUs, seekMode); @@ -576,7 +594,7 @@ bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, PostReleaseVideoBuffer(nullptr, FenceHandle()); } else { - err = mVideoSource->read(&mVideoBuffer); + err = mVideoSource->read(&mVideoBuffer); } // If there is no next Keyframe, jump to the previous key frame. @@ -670,7 +688,6 @@ bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, if ((aKeyframeSkip && timeUs < aTimeUs) || length == 0) { aFrame->mShouldSkip = true; } - mVideoLastFrameTime = timeUs; } else if (err == INFO_FORMAT_CHANGED) { // If the format changed, update our cached info. @@ -697,7 +714,8 @@ bool OmxDecoder::ReadVideo(VideoFrame *aFrame, int64_t aTimeUs, return true; } -bool OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs) +bool +OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs) { status_t err; @@ -756,7 +774,8 @@ bool OmxDecoder::ReadAudio(AudioFrame *aFrame, int64_t aSeekTimeUs) return true; } -nsresult OmxDecoder::Play() +nsresult +OmxDecoder::Play() { if (!mVideoPaused && !mAudioPaused) { return NS_OK; @@ -783,7 +802,8 @@ nsresult OmxDecoder::Play() // implement the OMXCodec::Pause() and need a following OMXCodec::Read() with // seek option (define in MediaSource.h) then it is still not supported here. // We need to fix it until it is really happened. -void OmxDecoder::Pause() +void +OmxDecoder::Pause() { /* The implementation of OMXCodec::pause is flawed. * OMXCodec::start will not restore from the paused state and result in @@ -809,7 +829,8 @@ void OmxDecoder::Pause() } // Called on ALooper thread. -void OmxDecoder::onMessageReceived(const sp &msg) +void +OmxDecoder::onMessageReceived(const sp &msg) { switch (msg->what()) { case kNotifyPostReleaseVideoBuffer: @@ -828,7 +849,8 @@ void OmxDecoder::onMessageReceived(const sp &msg) } } -void OmxDecoder::PostReleaseVideoBuffer(MediaBuffer *aBuffer, const FenceHandle& aReleaseFenceHandle) +void +OmxDecoder::PostReleaseVideoBuffer(MediaBuffer *aBuffer, const FenceHandle& aReleaseFenceHandle) { { Mutex::Autolock autoLock(mPendingVideoBuffersLock); @@ -843,7 +865,8 @@ void OmxDecoder::PostReleaseVideoBuffer(MediaBuffer *aBuffer, const FenceHandle& notify->post(); } -void OmxDecoder::ReleaseAllPendingVideoBuffersLocked() +void +OmxDecoder::ReleaseAllPendingVideoBuffersLocked() { Vector releasingVideoBuffers; { @@ -883,7 +906,8 @@ void OmxDecoder::ReleaseAllPendingVideoBuffersLocked() releasingVideoBuffers.clear(); } -void OmxDecoder::RecycleCallbackImp(TextureClient* aClient) +void +OmxDecoder::RecycleCallbackImp(TextureClient* aClient) { aClient->ClearRecycleCallback(); { diff --git a/dom/media/omx/OmxDecoder.h b/dom/media/omx/OmxDecoder.h index 8212cd4246..66e4e9285b 100644 --- a/dom/media/omx/OmxDecoder.h +++ b/dom/media/omx/OmxDecoder.h @@ -20,7 +20,7 @@ class OmxDecoder; namespace android { -class OmxDecoder : public OMXCodecProxy::CodecResourceListener { +class OmxDecoder : public RefBase { typedef MPAPI::AudioFrame AudioFrame; typedef MPAPI::VideoFrame VideoFrame; typedef mozilla::MP3FrameParser MP3FrameParser; @@ -59,7 +59,7 @@ class OmxDecoder : public OMXCodecProxy::CodecResourceListener { int32_t mAudioChannels; int32_t mAudioSampleRate; int64_t mDurationUs; - int64_t mVideoLastFrameTime; + int64_t mLastSeekTime; VideoFrame mVideoFrame; AudioFrame mAudioFrame; @@ -119,6 +119,9 @@ class OmxDecoder : public OMXCodecProxy::CodecResourceListener { // 'true' if a read from the audio stream was done while reading the metadata bool mAudioMetadataRead; + RefPtr mTaskQueue; + + mozilla::MozPromiseRequestHolder mVideoCodecRequest; mozilla::MozPromiseHolder mMediaResourcePromise; void ReleaseVideoBuffer(); @@ -138,13 +141,14 @@ class OmxDecoder : public OMXCodecProxy::CodecResourceListener { bool mAudioPaused; bool mVideoPaused; -public: - explicit OmxDecoder(AbstractMediaDecoder *aDecoder); - ~OmxDecoder(); + mozilla::TaskQueue* OwnerThread() const + { + return mTaskQueue; + } - // OMXCodecProxy::CodecResourceListener - virtual void codecReserved(); - virtual void codecCanceled(); +public: + explicit OmxDecoder(AbstractMediaDecoder *aDecoder, mozilla::TaskQueue* aTaskQueue); + ~OmxDecoder(); // The MediaExtractor provides essential information for creating OMXCodec // instance. Such as video/audio codec, we can retrieve them through the diff --git a/dom/media/omx/RtspOmxReader.cpp b/dom/media/omx/RtspOmxReader.cpp index 145d8c49f4..07e6d8ed40 100644 --- a/dom/media/omx/RtspOmxReader.cpp +++ b/dom/media/omx/RtspOmxReader.cpp @@ -24,7 +24,7 @@ nsresult RtspOmxReader::InitOmxDecoder() NS_ASSERTION(mDecoder->GetResource(), "RtspOmxReader mDecoder->GetResource() is null."); mExtractor = new RtspExtractor(mRtspResource); - mOmxDecoder = new OmxDecoder(mDecoder); + mOmxDecoder = new OmxDecoder(mDecoder, OwnerThread()); if (!mOmxDecoder->Init(mExtractor)) { return NS_ERROR_FAILURE; } diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp index e3f9446e52..cf6bddcec0 100644 --- a/dom/media/platforms/PDMFactory.cpp +++ b/dom/media/platforms/PDMFactory.cpp @@ -227,7 +227,7 @@ PDMFactory::CreateDecoderWithPDM(PlatformDecoderModule* aPDM, } bool -PDMFactory::SupportsMimeType(const nsACString& aMimeType) +PDMFactory::SupportsMimeType(const nsACString& aMimeType) const { if (mEMEPDM) { return mEMEPDM->SupportsMimeType(aMimeType); @@ -305,7 +305,7 @@ PDMFactory::StartupPDM(PlatformDecoderModule* aPDM) } already_AddRefed -PDMFactory::GetDecoder(const nsACString& aMimeType) +PDMFactory::GetDecoder(const nsACString& aMimeType) const { RefPtr pdm; for (auto& current : mCurrentPDMs) { diff --git a/dom/media/platforms/PDMFactory.h b/dom/media/platforms/PDMFactory.h index 76b60958ec..f85bd2ca96 100644 --- a/dom/media/platforms/PDMFactory.h +++ b/dom/media/platforms/PDMFactory.h @@ -35,7 +35,7 @@ public: layers::LayersBackend aLayersBackend = layers::LayersBackend::LAYERS_NONE, layers::ImageContainer* aImageContainer = nullptr); - bool SupportsMimeType(const nsACString& aMimeType); + bool SupportsMimeType(const nsACString& aMimeType) const; #ifdef MOZ_EME // Creates a PlatformDecoderModule that uses a CDMProxy to decrypt or @@ -52,7 +52,9 @@ private: // Startup the provided PDM and add it to our list if successful. bool StartupPDM(PlatformDecoderModule* aPDM); // Returns the first PDM in our list supporting the mimetype. - already_AddRefed GetDecoder(const nsACString& aMimeType); + already_AddRefed + GetDecoder(const nsACString& aMimeType) const; + already_AddRefed CreateDecoderWithPDM(PlatformDecoderModule* aPDM, const TrackInfo& aConfig, diff --git a/dom/media/platforms/PlatformDecoderModule.cpp b/dom/media/platforms/PlatformDecoderModule.cpp index bb02119700..a2b979c374 100644 --- a/dom/media/platforms/PlatformDecoderModule.cpp +++ b/dom/media/platforms/PlatformDecoderModule.cpp @@ -6,10 +6,7 @@ #include "PlatformDecoderModule.h" -PRLogModuleInfo* GetPDMLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("PlatformDecoderModule"); - } +mozilla::LogModule* GetPDMLog() { + static mozilla::LazyLogModule log("PlatformDecoderModule"); return log; } diff --git a/dom/media/platforms/PlatformDecoderModule.h b/dom/media/platforms/PlatformDecoderModule.h index 9ce212a4f4..0f68a9125b 100644 --- a/dom/media/platforms/PlatformDecoderModule.h +++ b/dom/media/platforms/PlatformDecoderModule.h @@ -53,7 +53,7 @@ public: virtual nsresult Startup() { return NS_OK; }; // Indicates if the PlatformDecoderModule supports decoding of aMimeType. - virtual bool SupportsMimeType(const nsACString& aMimeType) = 0; + virtual bool SupportsMimeType(const nsACString& aMimeType) const = 0; enum ConversionRequired { kNeedNone, diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp index 39876f8989..64adc7e96e 100644 --- a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp +++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp @@ -12,7 +12,7 @@ namespace mozilla { bool -AgnosticDecoderModule::SupportsMimeType(const nsACString& aMimeType) +AgnosticDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { return VPXDecoder::IsVPX(aMimeType) || OpusDataDecoder::IsOpus(aMimeType) || diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.h b/dom/media/platforms/agnostic/AgnosticDecoderModule.h index 97cc23e799..ff0d0583c7 100644 --- a/dom/media/platforms/agnostic/AgnosticDecoderModule.h +++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.h @@ -10,7 +10,7 @@ public: AgnosticDecoderModule() = default; virtual ~AgnosticDecoderModule() = default; - bool SupportsMimeType(const nsACString& aMimeType) override; + bool SupportsMimeType(const nsACString& aMimeType) const override; ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.cpp b/dom/media/platforms/agnostic/BlankDecoderModule.cpp index 10e12c9d88..8e93c27e32 100644 --- a/dom/media/platforms/agnostic/BlankDecoderModule.cpp +++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp @@ -246,7 +246,7 @@ public: } bool - SupportsMimeType(const nsACString& aMimeType) override + SupportsMimeType(const nsACString& aMimeType) const override { return true; } diff --git a/dom/media/platforms/agnostic/OpusDecoder.cpp b/dom/media/platforms/agnostic/OpusDecoder.cpp index 89e0207979..2522385486 100644 --- a/dom/media/platforms/agnostic/OpusDecoder.cpp +++ b/dom/media/platforms/agnostic/OpusDecoder.cpp @@ -12,7 +12,7 @@ #include #include // For PRId64 -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define OPUS_DEBUG(arg, ...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, \ ("OpusDataDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp index be1ee2f705..2fb87f6508 100644 --- a/dom/media/platforms/agnostic/VPXDecoder.cpp +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -9,11 +9,12 @@ #include "nsError.h" #include "TimeUnits.h" #include "mozilla/PodOperations.h" +#include "prsystem.h" #include #undef LOG -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(arg, ...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, ("VPXDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) namespace mozilla { @@ -56,13 +57,26 @@ VPXDecoder::Shutdown() RefPtr VPXDecoder::Init() { + int decode_threads = 2; + vpx_codec_iface_t* dx = nullptr; if (mCodec == Codec::VP8) { dx = vpx_codec_vp8_dx(); } else if (mCodec == Codec::VP9) { dx = vpx_codec_vp9_dx(); + if (mInfo.mDisplay.width >= 2048) { + decode_threads = 8; + } else if (mInfo.mDisplay.width >= 1024) { + decode_threads = 4; + } } - if (!dx || vpx_codec_dec_init(&mVPX, dx, nullptr, 0)) { + decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors()); + + vpx_codec_dec_cfg_t config; + config.threads = decode_threads; + config.w = config.h = 0; // set after decode + + if (!dx || vpx_codec_dec_init(&mVPX, dx, &config, 0)) { return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); diff --git a/dom/media/platforms/agnostic/VorbisDecoder.cpp b/dom/media/platforms/agnostic/VorbisDecoder.cpp index dce4c8a97e..8f37a8fbc3 100644 --- a/dom/media/platforms/agnostic/VorbisDecoder.cpp +++ b/dom/media/platforms/agnostic/VorbisDecoder.cpp @@ -12,7 +12,7 @@ #include "nsAutoPtr.h" #undef LOG -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(type, msg) MOZ_LOG(GetPDMLog(), type, msg) namespace mozilla { diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp index 0f5b46dd05..aa7da132e9 100644 --- a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp @@ -228,7 +228,7 @@ GMPDecoderModule::SupportsMimeType(const nsACString& aMimeType, } bool -GMPDecoderModule::SupportsMimeType(const nsACString& aMimeType) +GMPDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { return SupportsMimeType(aMimeType, PreferredGMP(aMimeType)); } diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h index 2444c7a57f..ef5a0f5532 100644 --- a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h @@ -36,7 +36,7 @@ public: DecoderNeedsConversion(const TrackInfo& aConfig) const override; bool - SupportsMimeType(const nsACString& aMimeType) override; + SupportsMimeType(const nsACString& aMimeType) const override; // Main thread only. static void Init(); diff --git a/dom/media/platforms/android/AndroidDecoderModule.cpp b/dom/media/platforms/android/AndroidDecoderModule.cpp index 8f76f95abc..780fc9cb28 100644 --- a/dom/media/platforms/android/AndroidDecoderModule.cpp +++ b/dom/media/platforms/android/AndroidDecoderModule.cpp @@ -12,18 +12,34 @@ #include "GLLibraryEGL.h" #include "MediaData.h" - #include "MediaInfo.h" #include "nsThreadUtils.h" #include "nsAutoPtr.h" #include "nsPromiseFlatString.h" +#include "prlog.h" + #include +static PRLogModuleInfo* AndroidDecoderModuleLog() +{ + static PRLogModuleInfo* sLogModule = nullptr; + if (!sLogModule) { + sLogModule = PR_NewLogModule("AndroidDecoderModule"); + } + return sLogModule; +} + +#undef LOG +#define LOG(arg, ...) MOZ_LOG(AndroidDecoderModuleLog(), \ + mozilla::LogLevel::Debug, ("AndroidDecoderModule(%p)::%s: " arg, \ + this, __func__, ##__VA_ARGS__)) + using namespace mozilla; using namespace mozilla::gl; using namespace mozilla::widget::sdk; +using media::TimeUnit; namespace mozilla { @@ -31,10 +47,11 @@ namespace mozilla { if (mCallback) { \ mCallback->Func(__VA_ARGS__); \ } else { \ - NS_WARNING("callback not set"); \ + NS_WARNING("Callback not set"); \ } -static const char* TranslateMimeType(const nsACString& aMimeType) +static const char* +TranslateMimeType(const nsACString& aMimeType) { if (aMimeType.EqualsLiteral("video/webm; codecs=vp8")) { return "video/x-vnd.on2.vp8"; @@ -44,7 +61,8 @@ static const char* TranslateMimeType(const nsACString& aMimeType) return PromiseFlatCString(aMimeType).get(); } -static MediaCodec::LocalRef CreateDecoder(const nsACString& aMimeType) +static MediaCodec::LocalRef +CreateDecoder(const nsACString& aMimeType) { MediaCodec::LocalRef codec; NS_ENSURE_SUCCESS(MediaCodec::CreateDecoderByType(TranslateMimeType(aMimeType), @@ -52,19 +70,23 @@ static MediaCodec::LocalRef CreateDecoder(const nsACString& aMimeType) return codec; } -class VideoDataDecoder : public MediaCodecDataDecoder { +class VideoDataDecoder : public MediaCodecDataDecoder +{ public: VideoDataDecoder(const VideoInfo& aConfig, - MediaFormat::Param aFormat, MediaDataDecoderCallback* aCallback, + MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback, layers::ImageContainer* aImageContainer) - : MediaCodecDataDecoder(MediaData::Type::VIDEO_DATA, aConfig.mMimeType, aFormat, aCallback) + : MediaCodecDataDecoder(MediaData::Type::VIDEO_DATA, aConfig.mMimeType, + aFormat, aCallback) , mImageContainer(aImageContainer) , mConfig(aConfig) { } - RefPtr Init() override { + RefPtr Init() override + { mSurfaceTexture = AndroidSurfaceTexture::Create(); if (!mSurfaceTexture) { NS_WARNING("Failed to create SurfaceTexture for video decode\n"); @@ -78,26 +100,36 @@ public: return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); } - void Cleanup() override { + void Cleanup() override + { mGLContext = nullptr; } - nsresult Input(MediaRawData* aSample) override { + nsresult Input(MediaRawData* aSample) override + { return MediaCodecDataDecoder::Input(aSample); } - bool WantCopy() { - // Allocating a texture is incredibly slow on PowerVR - return mGLContext->Vendor() != GLVendor::Imagination; + bool WantCopy() const + { + // Allocating a texture is incredibly slow on PowerVR and may fail on + // emulators, see bug 1190379. + return mGLContext->Vendor() != GLVendor::Imagination && + mGLContext->Renderer() != GLRenderer::AndroidEmulator; } - EGLImage CopySurface(layers::Image* img) { + EGLImage CopySurface(layers::Image* img) + { mGLContext->MakeCurrent(); - GLuint tex = CreateTextureForOffscreen(mGLContext, mGLContext->GetGLFormats(), img->GetSize()); + GLuint tex = CreateTextureForOffscreen(mGLContext, mGLContext->GetGLFormats(), + img->GetSize()); - GLBlitHelper helper(mGLContext); - if (!helper.BlitImageToTexture(img, img->GetSize(), tex, LOCAL_GL_TEXTURE_2D)) { + auto helper = mGLContext->BlitHelper(); + const gl::OriginPos destOrigin = gl::OriginPos::TopLeft; + if (!helper->BlitImageToTexture(img, img->GetSize(), tex, + LOCAL_GL_TEXTURE_2D, destOrigin)) + { mGLContext->fDeleteTextures(1, &tex); return nullptr; } @@ -108,28 +140,24 @@ public: }; EGLContext eglContext = static_cast(mGLContext.get())->GetEGLContext(); - EGLImage eglImage = sEGLLibrary.fCreateImage(EGL_DISPLAY(), eglContext, - LOCAL_EGL_GL_TEXTURE_2D_KHR, - (EGLClientBuffer)tex, attribs); + EGLImage eglImage = sEGLLibrary.fCreateImage( + EGL_DISPLAY(), eglContext, LOCAL_EGL_GL_TEXTURE_2D_KHR, + reinterpret_cast(tex), attribs); mGLContext->fDeleteTextures(1, &tex); return eglImage; } nsresult PostOutput(BufferInfo::Param aInfo, MediaFormat::Param aFormat, - const media::TimeUnit& aDuration) override { + const TimeUnit& aDuration) override + { if (!EnsureGLContext()) { return NS_ERROR_FAILURE; } - RefPtr img = mImageContainer->CreateImage(ImageFormat::SURFACE_TEXTURE); - layers::SurfaceTextureImage::Data data; - data.mSurfTex = mSurfaceTexture.get(); - data.mSize = mConfig.mDisplay; - data.mOriginPos = gl::OriginPos::BottomLeft; - - layers::SurfaceTextureImage* stImg = static_cast(img.get()); - stImg->SetData(data); + RefPtr img = + new SurfaceTextureImage(mSurfaceTexture.get(), mConfig.mDisplay, + gl::OriginPos::BottomLeft); if (WantCopy()) { EGLImage eglImage = CopySurface(img); @@ -151,16 +179,8 @@ public: NS_WARNING("No EGL fence support detected, rendering artifacts may occur!"); } - img = mImageContainer->CreateImage(ImageFormat::EGLIMAGE); - layers::EGLImageImage::Data data; - data.mImage = eglImage; - data.mSync = eglSync; - data.mOwns = true; - data.mSize = mConfig.mDisplay; - data.mOriginPos = gl::OriginPos::BottomLeft; - - layers::EGLImageImage* typedImg = static_cast(img.get()); - typedImg->SetData(data); + img = new layers::EGLImageImage(eglImage, eglSync, mConfig.mDisplay, + gl::OriginPos::TopLeft, true /* owns */); } nsresult rv; @@ -192,7 +212,8 @@ public: } protected: - bool EnsureGLContext() { + bool EnsureGLContext() + { if (mGLContext) { return true; } @@ -207,33 +228,33 @@ protected: RefPtr mGLContext; }; -class AudioDataDecoder : public MediaCodecDataDecoder { -private: - uint8_t csd0[2]; - +class AudioDataDecoder : public MediaCodecDataDecoder +{ public: - AudioDataDecoder(const AudioInfo& aConfig, MediaFormat::Param aFormat, MediaDataDecoderCallback* aCallback) - : MediaCodecDataDecoder(MediaData::Type::AUDIO_DATA, aConfig.mMimeType, aFormat, aCallback) + AudioDataDecoder(const AudioInfo& aConfig, MediaFormat::Param aFormat, + MediaDataDecoderCallback* aCallback) + : MediaCodecDataDecoder(MediaData::Type::AUDIO_DATA, aConfig.mMimeType, + aFormat, aCallback) { - JNIEnv* env = GetJNIForThread(); + JNIEnv* const env = jni::GetEnvForThread(); jni::Object::LocalRef buffer(env); - NS_ENSURE_SUCCESS_VOID(aFormat->GetByteBuffer(NS_LITERAL_STRING("csd-0"), &buffer)); + NS_ENSURE_SUCCESS_VOID(aFormat->GetByteBuffer(NS_LITERAL_STRING("csd-0"), + &buffer)); if (!buffer && aConfig.mCodecSpecificConfig->Length() >= 2) { - csd0[0] = (*aConfig.mCodecSpecificConfig)[0]; - csd0[1] = (*aConfig.mCodecSpecificConfig)[1]; - - buffer = jni::Object::LocalRef::Adopt(env, env->NewDirectByteBuffer(csd0, 2)); - NS_ENSURE_SUCCESS_VOID(aFormat->SetByteBuffer(NS_LITERAL_STRING("csd-0"), buffer)); + buffer = jni::Object::LocalRef::Adopt( + env, env->NewDirectByteBuffer(aConfig.mCodecSpecificConfig->Elements(), + aConfig.mCodecSpecificConfig->Length())); + NS_ENSURE_SUCCESS_VOID(aFormat->SetByteBuffer(NS_LITERAL_STRING("csd-0"), + buffer)); } } nsresult Output(BufferInfo::Param aInfo, void* aBuffer, - MediaFormat::Param aFormat, - const media::TimeUnit& aDuration) { + MediaFormat::Param aFormat, const TimeUnit& aDuration) + { // The output on Android is always 16-bit signed - nsresult rv; int32_t numChannels; NS_ENSURE_SUCCESS(rv = @@ -250,7 +271,7 @@ public: NS_ENSURE_SUCCESS(rv = aInfo->Offset(&offset), rv); #ifdef MOZ_SAMPLE_TYPE_S16 - int32_t numSamples = size / 2; + const int32_t numSamples = size / 2; #else #error We only support 16-bit integer PCM #endif @@ -258,8 +279,9 @@ public: const int32_t numFrames = numSamples / numChannels; auto audio = MakeUnique(numSamples); - uint8_t* bufferStart = static_cast(aBuffer) + offset; - PodCopy(audio.get(), reinterpret_cast(bufferStart), numSamples); + const uint8_t* bufferStart = static_cast(aBuffer) + offset; + PodCopy(audio.get(), reinterpret_cast(bufferStart), + numSamples); int64_t presentationTimeUs; NS_ENSURE_SUCCESS(rv = aInfo->PresentationTimeUs(&presentationTimeUs), rv); @@ -275,26 +297,32 @@ public: } }; - -bool AndroidDecoderModule::SupportsMimeType(const nsACString& aMimeType) +bool +AndroidDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { - if (!AndroidBridge::Bridge() || (AndroidBridge::Bridge()->GetAPIVersion() < 16)) { + if (!AndroidBridge::Bridge() || + (AndroidBridge::Bridge()->GetAPIVersion() < 16)) { return false; } + if (aMimeType.EqualsLiteral("video/mp4") || aMimeType.EqualsLiteral("video/avc")) { return true; } - return static_cast(mozilla::CreateDecoder(aMimeType)); + + MediaCodec::LocalRef ref = mozilla::CreateDecoder(aMimeType); + if (!ref) { + return false; + } + ref->Release(); + return true; } already_AddRefed AndroidDecoderModule::CreateVideoDecoder( - const VideoInfo& aConfig, - layers::LayersBackend aLayersBackend, - layers::ImageContainer* aImageContainer, - FlushableTaskQueue* aVideoTaskQueue, - MediaDataDecoderCallback* aCallback) + const VideoInfo& aConfig, layers::LayersBackend aLayersBackend, + layers::ImageContainer* aImageContainer, FlushableTaskQueue* aVideoTaskQueue, + MediaDataDecoderCallback* aCallback) { MediaFormat::LocalRef format; @@ -311,9 +339,9 @@ AndroidDecoderModule::CreateVideoDecoder( } already_AddRefed -AndroidDecoderModule::CreateAudioDecoder(const AudioInfo& aConfig, - FlushableTaskQueue* aAudioTaskQueue, - MediaDataDecoderCallback* aCallback) +AndroidDecoderModule::CreateAudioDecoder( + const AudioInfo& aConfig, FlushableTaskQueue* aAudioTaskQueue, + MediaDataDecoderCallback* aCallback) { MOZ_ASSERT(aConfig.mBitDepth == 16, "We only handle 16-bit audio!"); @@ -329,7 +357,6 @@ AndroidDecoderModule::CreateAudioDecoder(const AudioInfo& aConfig, new AudioDataDecoder(aConfig, format, aCallback); return decoder.forget(); - } PlatformDecoderModule::ConversionRequired @@ -337,9 +364,8 @@ AndroidDecoderModule::DecoderNeedsConversion(const TrackInfo& aConfig) const { if (aConfig.IsVideo()) { return kNeedAnnexB; - } else { - return kNeedNone; } + return kNeedNone; } MediaCodecDataDecoder::MediaCodecDataDecoder(MediaData::Type aType, @@ -353,9 +379,7 @@ MediaCodecDataDecoder::MediaCodecDataDecoder(MediaData::Type aType, , mInputBuffers(nullptr) , mOutputBuffers(nullptr) , mMonitor("MediaCodecDataDecoder::mMonitor") - , mFlushing(false) - , mDraining(false) - , mStopping(false) + , mState(kDecoding) { } @@ -365,7 +389,8 @@ MediaCodecDataDecoder::~MediaCodecDataDecoder() Shutdown(); } -RefPtr MediaCodecDataDecoder::Init() +RefPtr +MediaCodecDataDecoder::Init() { nsresult rv = InitDecoder(nullptr); @@ -375,10 +400,12 @@ RefPtr MediaCodecDataDecoder::Init() return NS_SUCCEEDED(rv) ? InitPromise::CreateAndResolve(type, __func__) : - InitPromise::CreateAndReject(MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); + InitPromise::CreateAndReject( + MediaDataDecoder::DecoderFailureReason::INIT_ERROR, __func__); } -nsresult MediaCodecDataDecoder::InitDecoder(Surface::Param aSurface) +nsresult +MediaCodecDataDecoder::InitDecoder(Surface::Param aSurface) { mDecoder = CreateDecoder(mMimeType); if (!mDecoder) { @@ -399,224 +426,312 @@ nsresult MediaCodecDataDecoder::InitDecoder(Surface::Param aSurface) return NS_OK; } -// This is in usec, so that's 10ms -#define DECODER_TIMEOUT 10000 +// This is in usec, so that's 10ms. +static const int64_t kDecoderTimeout = 10000; -#define HANDLE_DECODER_ERROR() \ +#define BREAK_ON_DECODER_ERROR() \ if (NS_FAILED(res)) { \ - NS_WARNING("exiting decoder loop due to exception"); \ - if (mDraining) { \ + NS_WARNING("Exiting decoder loop due to exception"); \ + if (State() == kDrainDecoder) { \ INVOKE_CALLBACK(DrainComplete); \ - mDraining = false; \ + State(kDecoding); \ } \ INVOKE_CALLBACK(Error); \ break; \ } -nsresult MediaCodecDataDecoder::GetInputBuffer(JNIEnv* env, int index, jni::Object::LocalRef* buffer) +nsresult +MediaCodecDataDecoder::GetInputBuffer( + JNIEnv* aEnv, int aIndex, jni::Object::LocalRef* aBuffer) { - bool retried = false; - while (!*buffer) { - *buffer = jni::Object::LocalRef::Adopt(env->GetObjectArrayElement(mInputBuffers.Get(), index)); - if (!*buffer) { - if (!retried) { - // Reset the input buffers and then try again - nsresult res = ResetInputBuffers(); - if (NS_FAILED(res)) { - return res; - } - retried = true; - } else { - // We already tried resetting the input buffers, return an error - return NS_ERROR_FAILURE; - } + MOZ_ASSERT(aEnv); + MOZ_ASSERT(!*aBuffer); + + int numTries = 2; + + while (numTries--) { + *aBuffer = jni::Object::LocalRef::Adopt( + aEnv->GetObjectArrayElement(mInputBuffers.Get(), aIndex)); + if (*aBuffer) { + return NS_OK; + } + nsresult res = ResetInputBuffers(); + if (NS_FAILED(res)) { + return res; } } + return NS_ERROR_FAILURE; +} + +bool +MediaCodecDataDecoder::WaitForInput() +{ + MonitorAutoLock lock(mMonitor); + + while (State() == kDecoding && mQueue.empty()) { + // Signal that we require more input. + INVOKE_CALLBACK(InputExhausted); + lock.Wait(); + } + + return State() != kStopping; +} + + +MediaRawData* +MediaCodecDataDecoder::PeekNextSample() +{ + MonitorAutoLock lock(mMonitor); + + if (State() == kFlushing) { + mDecoder->Flush(); + ClearQueue(); + State(kDecoding); + lock.Notify(); + return nullptr; + } + + if (mQueue.empty()) { + if (State() == kDrainQueue) { + State(kDrainDecoder); + } + return nullptr; + } + + // We're not stopping or flushing, so try to get a sample. + return mQueue.front(); +} + +nsresult +MediaCodecDataDecoder::QueueSample(const MediaRawData* aSample) +{ + MOZ_ASSERT(aSample); + AutoLocalJNIFrame frame(jni::GetEnvForThread(), 1); + + // We have a sample, try to feed it to the decoder. + int32_t inputIndex = -1; + nsresult res = mDecoder->DequeueInputBuffer(kDecoderTimeout, &inputIndex); + if (NS_FAILED(res)) { + return res; + } + + if (inputIndex < 0) { + // There is no valid input buffer available. + return NS_ERROR_FAILURE; + } + + jni::Object::LocalRef buffer(frame.GetEnv()); + res = GetInputBuffer(frame.GetEnv(), inputIndex, &buffer); + if (NS_FAILED(res)) { + return res; + } + + void* directBuffer = frame.GetEnv()->GetDirectBufferAddress(buffer.Get()); + + MOZ_ASSERT(frame.GetEnv()->GetDirectBufferCapacity(buffer.Get()) >= + aSample->Size(), + "Decoder buffer is not large enough for sample"); + + PodCopy(static_cast(directBuffer), aSample->Data(), aSample->Size()); + + res = mDecoder->QueueInputBuffer(inputIndex, 0, aSample->Size(), + aSample->mTime, 0); + if (NS_FAILED(res)) { + return res; + } + + mDurations.push(TimeUnit::FromMicroseconds(aSample->mDuration)); + return NS_OK; +} + +nsresult +MediaCodecDataDecoder::QueueEOS() +{ + mMonitor.AssertCurrentThreadOwns(); + + nsresult res = NS_OK; + int32_t inputIndex = -1; + res = mDecoder->DequeueInputBuffer(kDecoderTimeout, &inputIndex); + if (NS_FAILED(res) || inputIndex < 0) { + return res; + } + + res = mDecoder->QueueInputBuffer(inputIndex, 0, 0, 0, + MediaCodec::BUFFER_FLAG_END_OF_STREAM); + if (NS_SUCCEEDED(res)) { + State(kDrainWaitEOS); + mMonitor.Notify(); + } + return res; +} + +void +MediaCodecDataDecoder::HandleEOS(int32_t aOutputStatus) +{ + MonitorAutoLock lock(mMonitor); + + if (State() == kDrainWaitEOS) { + State(kDecoding); + mMonitor.Notify(); + + INVOKE_CALLBACK(DrainComplete); + } + + mDecoder->ReleaseOutputBuffer(aOutputStatus, false); +} + +TimeUnit +MediaCodecDataDecoder::GetOutputDuration() +{ + MOZ_ASSERT(!mDurations.empty(), "Should have had a duration queued"); + const TimeUnit duration = mDurations.front(); + mDurations.pop(); + return duration; +} + +nsresult +MediaCodecDataDecoder::ProcessOutput( + BufferInfo::Param aInfo, MediaFormat::Param aFormat, int32_t aStatus) +{ + AutoLocalJNIFrame frame(jni::GetEnvForThread(), 1); + + const TimeUnit duration = GetOutputDuration(); + const auto buffer = jni::Object::LocalRef::Adopt( + frame.GetEnv()->GetObjectArrayElement(mOutputBuffers.Get(), aStatus)); + + if (buffer) { + // The buffer will be null on Android L if we are decoding to a Surface. + void* directBuffer = frame.GetEnv()->GetDirectBufferAddress(buffer.Get()); + Output(aInfo, directBuffer, aFormat, duration); + } + + // The Surface will be updated at this point (for video). + mDecoder->ReleaseOutputBuffer(aStatus, true); + PostOutput(aInfo, aFormat, duration); return NS_OK; } -void MediaCodecDataDecoder::DecoderLoop() +void +MediaCodecDataDecoder::DecoderLoop() { - bool outputDone = false; - - bool draining = false; - bool waitingEOF = false; - - AutoLocalJNIFrame frame(GetJNIForThread(), 1); + bool isOutputDone = false; + AutoLocalJNIFrame frame(jni::GetEnvForThread(), 1); RefPtr sample; - MediaFormat::LocalRef outputFormat(frame.GetEnv()); - nsresult res; + nsresult res = NS_OK; + + while (WaitForInput()) { + sample = PeekNextSample(); - for (;;) { { MonitorAutoLock lock(mMonitor); - while (!mStopping && !mDraining && !mFlushing && mQueue.empty()) { - if (mQueue.empty()) { - // We could be waiting here forever if we don't signal that we need more input - INVOKE_CALLBACK(InputExhausted); - } - lock.Wait(); - } - - if (mStopping) { - // Get out of the loop. This is the only exit point. - break; - } - - if (mFlushing) { - mDecoder->Flush(); - ClearQueue(); - mFlushing = false; - lock.Notify(); - continue; - } - - // We're not stopping or draining, so try to get a sample - if (!mQueue.empty()) { - sample = mQueue.front(); - } - - if (mDraining && !sample && !waitingEOF) { - draining = true; - } - } - - if (draining && !waitingEOF) { - MOZ_ASSERT(!sample, "Shouldn't have a sample when pushing EOF frame"); - - int32_t inputIndex; - res = mDecoder->DequeueInputBuffer(DECODER_TIMEOUT, &inputIndex); - HANDLE_DECODER_ERROR(); - - if (inputIndex >= 0) { - res = mDecoder->QueueInputBuffer(inputIndex, 0, 0, 0, MediaCodec::BUFFER_FLAG_END_OF_STREAM); - HANDLE_DECODER_ERROR(); - - waitingEOF = true; + if (State() == kDrainDecoder) { + MOZ_ASSERT(!sample, "Shouldn't have a sample when pushing EOF frame"); + res = QueueEOS(); + BREAK_ON_DECODER_ERROR(); } } if (sample) { - // We have a sample, try to feed it to the decoder - int inputIndex; - res = mDecoder->DequeueInputBuffer(DECODER_TIMEOUT, &inputIndex); - HANDLE_DECODER_ERROR(); - - if (inputIndex >= 0) { - jni::Object::LocalRef buffer(frame.GetEnv()); - res = GetInputBuffer(frame.GetEnv(), inputIndex, &buffer); - HANDLE_DECODER_ERROR(); - - void* directBuffer = frame.GetEnv()->GetDirectBufferAddress(buffer.Get()); - - MOZ_ASSERT(frame.GetEnv()->GetDirectBufferCapacity(buffer.Get()) >= sample->Size(), - "Decoder buffer is not large enough for sample"); - - { - // We're feeding this to the decoder, so remove it from the queue - MonitorAutoLock lock(mMonitor); - mQueue.pop(); - } - - PodCopy((uint8_t*)directBuffer, sample->Data(), sample->Size()); - - res = mDecoder->QueueInputBuffer(inputIndex, 0, sample->Size(), - sample->mTime, 0); - HANDLE_DECODER_ERROR(); - - mDurations.push(media::TimeUnit::FromMicroseconds(sample->mDuration)); - sample = nullptr; - outputDone = false; + res = QueueSample(sample); + if (NS_SUCCEEDED(res)) { + // We've fed this into the decoder, so remove it from the queue. + MonitorAutoLock lock(mMonitor); + mQueue.pop(); + isOutputDone = false; } } - if (!outputDone) { - BufferInfo::LocalRef bufferInfo; - res = BufferInfo::New(&bufferInfo); - HANDLE_DECODER_ERROR(); + if (isOutputDone) { + continue; + } - int32_t outputStatus; - res = mDecoder->DequeueOutputBuffer(bufferInfo, DECODER_TIMEOUT, &outputStatus); - HANDLE_DECODER_ERROR(); + BufferInfo::LocalRef bufferInfo; + nsresult res = BufferInfo::New(&bufferInfo); + BREAK_ON_DECODER_ERROR(); - if (outputStatus == MediaCodec::INFO_TRY_AGAIN_LATER) { - // We might want to call mCallback->InputExhausted() here, but there seems to be - // some possible bad interactions here with the threading - } else if (outputStatus == MediaCodec::INFO_OUTPUT_BUFFERS_CHANGED) { - res = ResetOutputBuffers(); - HANDLE_DECODER_ERROR(); - } else if (outputStatus == MediaCodec::INFO_OUTPUT_FORMAT_CHANGED) { - res = mDecoder->GetOutputFormat(ReturnTo(&outputFormat)); - HANDLE_DECODER_ERROR(); - } else if (outputStatus < 0) { - NS_WARNING("unknown error from decoder!"); - INVOKE_CALLBACK(Error); + int32_t outputStatus = -1; + res = mDecoder->DequeueOutputBuffer(bufferInfo, kDecoderTimeout, + &outputStatus); + BREAK_ON_DECODER_ERROR(); - // Don't break here just in case it's recoverable. If it's not, others stuff will fail later and - // we'll bail out. - } else { - int32_t flags; - res = bufferInfo->Flags(&flags); - HANDLE_DECODER_ERROR(); + if (outputStatus == MediaCodec::INFO_TRY_AGAIN_LATER) { + // We might want to call mCallback->InputExhausted() here, but there seems + // to be some possible bad interactions here with the threading. + } else if (outputStatus == MediaCodec::INFO_OUTPUT_BUFFERS_CHANGED) { + res = ResetOutputBuffers(); + BREAK_ON_DECODER_ERROR(); + } else if (outputStatus == MediaCodec::INFO_OUTPUT_FORMAT_CHANGED) { + res = mDecoder->GetOutputFormat(ReturnTo(&outputFormat)); + BREAK_ON_DECODER_ERROR(); + } else if (outputStatus < 0) { + NS_WARNING("Unknown error from decoder!"); + INVOKE_CALLBACK(Error); + // Don't break here just in case it's recoverable. If it's not, other + // stuff will fail later and we'll bail out. + } else { + // We have a valid buffer index >= 0 here. + int32_t flags; + nsresult res = bufferInfo->Flags(&flags); + BREAK_ON_DECODER_ERROR(); - // We have a valid buffer index >= 0 here - if (flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) { - if (draining) { - draining = false; - waitingEOF = false; - - mMonitor.Lock(); - mDraining = false; - mMonitor.Notify(); - mMonitor.Unlock(); - - INVOKE_CALLBACK(DrainComplete); - } - - mDecoder->ReleaseOutputBuffer(outputStatus, false); - outputDone = true; - - // We only queue empty EOF frames, so we're done for now - continue; - } - - MOZ_ASSERT(!mDurations.empty(), "Should have had a duration queued"); - - media::TimeUnit duration; - if (!mDurations.empty()) { - duration = mDurations.front(); - mDurations.pop(); - } - - auto buffer = jni::Object::LocalRef::Adopt( - frame.GetEnv()->GetObjectArrayElement(mOutputBuffers.Get(), outputStatus)); - if (buffer) { - // The buffer will be null on Android L if we are decoding to a Surface - void* directBuffer = frame.GetEnv()->GetDirectBufferAddress(buffer.Get()); - Output(bufferInfo, directBuffer, outputFormat, duration); - } - - // The Surface will be updated at this point (for video) - mDecoder->ReleaseOutputBuffer(outputStatus, true); - - PostOutput(bufferInfo, outputFormat, duration); + if (flags & MediaCodec::BUFFER_FLAG_END_OF_STREAM) { + HandleEOS(outputStatus); + isOutputDone = true; + // We only queue empty EOF frames, so we're done for now. + continue; } + + res = ProcessOutput(bufferInfo, outputFormat, outputStatus); + BREAK_ON_DECODER_ERROR(); } } Cleanup(); - // We're done + // We're done. MonitorAutoLock lock(mMonitor); - mStopping = false; + State(kShutdown); mMonitor.Notify(); } -void MediaCodecDataDecoder::ClearQueue() +const char* +MediaCodecDataDecoder::ModuleStateStr(ModuleState aState) { + static const char* kStr[] = { + "Decoding", "Flushing", "DrainQueue", "DrainDecoder", "DrainWaitEOS", + "Stopping", "Shutdown" + }; + + MOZ_ASSERT(aState < sizeof(kStr) / sizeof(kStr[0])); + return kStr[aState]; +} + +MediaCodecDataDecoder::ModuleState +MediaCodecDataDecoder::State() const +{ + return mState; +} + +void +MediaCodecDataDecoder::State(ModuleState aState) +{ + LOG("%s -> %s", ModuleStateStr(mState), ModuleStateStr(aState)); + + if (aState == kDrainDecoder) { + MOZ_ASSERT(mState == kDrainQueue); + } else if (aState == kDrainWaitEOS) { + MOZ_ASSERT(mState == kDrainDecoder); + } + + mState = aState; +} + +void +MediaCodecDataDecoder::ClearQueue() { mMonitor.AssertCurrentThreadOwns(); + while (!mQueue.empty()) { mQueue.pop(); } @@ -625,7 +740,9 @@ void MediaCodecDataDecoder::ClearQueue() } } -nsresult MediaCodecDataDecoder::Input(MediaRawData* aSample) { +nsresult +MediaCodecDataDecoder::Input(MediaRawData* aSample) +{ MonitorAutoLock lock(mMonitor); mQueue.push(aSample); lock.NotifyAll(); @@ -633,53 +750,61 @@ nsresult MediaCodecDataDecoder::Input(MediaRawData* aSample) { return NS_OK; } -nsresult MediaCodecDataDecoder::ResetInputBuffers() +nsresult +MediaCodecDataDecoder::ResetInputBuffers() { return mDecoder->GetInputBuffers(ReturnTo(&mInputBuffers)); } -nsresult MediaCodecDataDecoder::ResetOutputBuffers() +nsresult +MediaCodecDataDecoder::ResetOutputBuffers() { return mDecoder->GetOutputBuffers(ReturnTo(&mOutputBuffers)); } -nsresult MediaCodecDataDecoder::Flush() { +nsresult +MediaCodecDataDecoder::Flush() +{ MonitorAutoLock lock(mMonitor); - mFlushing = true; + State(kFlushing); lock.Notify(); - while (mFlushing) { + while (State() == kFlushing) { lock.Wait(); } return NS_OK; } -nsresult MediaCodecDataDecoder::Drain() { +nsresult +MediaCodecDataDecoder::Drain() +{ MonitorAutoLock lock(mMonitor); - if (mDraining) { + if (State() == kDrainDecoder || State() == kDrainQueue) { return NS_OK; } - mDraining = true; + State(kDrainQueue); lock.Notify(); return NS_OK; } -nsresult MediaCodecDataDecoder::Shutdown() { +nsresult +MediaCodecDataDecoder::Shutdown() +{ MonitorAutoLock lock(mMonitor); - if (!mThread || mStopping) { + if (!mThread || State() == kStopping) { // Already shutdown or in the process of doing so return NS_OK; } - mStopping = true; + State(kStopping); lock.Notify(); - while (mStopping) { + while (State() == kStopping) { lock.Wait(); } diff --git a/dom/media/platforms/android/AndroidDecoderModule.h b/dom/media/platforms/android/AndroidDecoderModule.h index 28ff19639e..d3f216d172 100644 --- a/dom/media/platforms/android/AndroidDecoderModule.h +++ b/dom/media/platforms/android/AndroidDecoderModule.h @@ -36,7 +36,7 @@ public: AndroidDecoderModule() {} virtual ~AndroidDecoderModule() {} - bool SupportsMimeType(const nsACString& aMimeType) override; + bool SupportsMimeType(const nsACString& aMimeType) const override; ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override; @@ -59,8 +59,55 @@ public: nsresult Input(MediaRawData* aSample) override; protected: + enum ModuleState { + kDecoding = 0, + kFlushing, + kDrainQueue, + kDrainDecoder, + kDrainWaitEOS, + kStopping, + kShutdown + }; + friend class AndroidDecoderModule; + static const char* ModuleStateStr(ModuleState aState); + + virtual nsresult InitDecoder(widget::sdk::Surface::Param aSurface); + + virtual nsresult Output(widget::sdk::BufferInfo::Param aInfo, void* aBuffer, + widget::sdk::MediaFormat::Param aFormat, const media::TimeUnit& aDuration) + { + return NS_OK; + } + + virtual nsresult PostOutput(widget::sdk::BufferInfo::Param aInfo, + widget::sdk::MediaFormat::Param aFormat, const media::TimeUnit& aDuration) + { + return NS_OK; + } + + virtual void Cleanup() {}; + + nsresult ResetInputBuffers(); + nsresult ResetOutputBuffers(); + + nsresult GetInputBuffer(JNIEnv* env, int index, jni::Object::LocalRef* buffer); + bool WaitForInput(); + MediaRawData* PeekNextSample(); + nsresult QueueSample(const MediaRawData* aSample); + nsresult QueueEOS(); + void HandleEOS(int32_t aOutputStatus); + media::TimeUnit GetOutputDuration(); + nsresult ProcessOutput(widget::sdk::BufferInfo::Param aInfo, + widget::sdk::MediaFormat::Param aFormat, + int32_t aStatus); + ModuleState State() const; + void State(ModuleState aState); + void DecoderLoop(); + + virtual void ClearQueue(); + MediaData::Type mType; nsAutoCString mMimeType; @@ -77,28 +124,14 @@ protected: // Only these members are protected by mMonitor. Monitor mMonitor; - bool mFlushing; - bool mDraining; - bool mStopping; + + ModuleState mState; SampleQueue mQueue; // Durations are stored in microseconds. std::queue mDurations; - - virtual nsresult InitDecoder(widget::sdk::Surface::Param aSurface); - - virtual nsresult Output(widget::sdk::BufferInfo::Param aInfo, void* aBuffer, widget::sdk::MediaFormat::Param aFormat, const media::TimeUnit& aDuration) { return NS_OK; } - virtual nsresult PostOutput(widget::sdk::BufferInfo::Param aInfo, widget::sdk::MediaFormat::Param aFormat, const media::TimeUnit& aDuration) { return NS_OK; } - virtual void Cleanup() {}; - - nsresult ResetInputBuffers(); - nsresult ResetOutputBuffers(); - - void DecoderLoop(); - nsresult GetInputBuffer(JNIEnv* env, int index, jni::Object::LocalRef* buffer); - virtual void ClearQueue(); }; -} // namwspace mozilla +} // namespace mozilla #endif diff --git a/dom/media/platforms/apple/AppleATDecoder.cpp b/dom/media/platforms/apple/AppleATDecoder.cpp index 6a2ad3cfea..fd8078208d 100644 --- a/dom/media/platforms/apple/AppleATDecoder.cpp +++ b/dom/media/platforms/apple/AppleATDecoder.cpp @@ -11,7 +11,7 @@ #include "AppleATDecoder.h" #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) #define FourCC2Str(n) ((char[5]){(char)(n >> 24), (char)(n >> 16), (char)(n >> 8), (char)(n), 0}) @@ -397,6 +397,9 @@ AppleATDecoder::SetupDecoder(MediaRawData* aSample) mOutputFormat.mFormatFlags = kLinearPCMFormatFlagIsFloat | 0; +#elif defined(MOZ_SAMPLE_TYPE_S16) + mOutputFormat.mBitsPerChannel = 16; + mOutputFormat.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | 0; #else # error Unknown audio sample type #endif diff --git a/dom/media/platforms/apple/AppleCMLinker.cpp b/dom/media/platforms/apple/AppleCMLinker.cpp index ce97e7fd8b..a61cb0e9f2 100644 --- a/dom/media/platforms/apple/AppleCMLinker.cpp +++ b/dom/media/platforms/apple/AppleCMLinker.cpp @@ -9,10 +9,13 @@ #include "AppleCMLinker.h" #include "MainThreadUtils.h" #include "mozilla/ArrayUtils.h" -#include "nsCocoaFeatures.h" #include "nsDebug.h" -extern PRLogModuleInfo* GetPDMLog(); +#ifndef MOZ_WIDGET_UIKIT +#include "nsCocoaFeatures.h" +#endif + +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { @@ -57,7 +60,11 @@ AppleCMLinker::Link() goto fail; } +#ifdef MOZ_WIDGET_UIKIT + if (true) { +#else if (nsCocoaFeatures::OnLionOrLater()) { +#endif #define LINK_FUNC2(func) \ func = (typeof(func))dlsym(sLink, #func); \ if (!func) { \ diff --git a/dom/media/platforms/apple/AppleDecoderModule.cpp b/dom/media/platforms/apple/AppleDecoderModule.cpp index 4132116acc..a1980c3fc9 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.cpp +++ b/dom/media/platforms/apple/AppleDecoderModule.cpp @@ -11,6 +11,7 @@ #include "AppleVDALinker.h" #include "AppleVTDecoder.h" #include "AppleVTLinker.h" +#include "MacIOSurfaceImage.h" #include "mozilla/Preferences.h" #include "mozilla/DebugOnly.h" #include "mozilla/Logging.h" @@ -18,10 +19,12 @@ namespace mozilla { bool AppleDecoderModule::sInitialized = false; +bool AppleDecoderModule::sIsCoreMediaAvailable = false; bool AppleDecoderModule::sIsVTAvailable = false; bool AppleDecoderModule::sIsVTHWAvailable = false; bool AppleDecoderModule::sIsVDAAvailable = false; bool AppleDecoderModule::sForceVDA = false; +bool AppleDecoderModule::sCanUseHardwareVideoDecoder = true; AppleDecoderModule::AppleDecoderModule() { @@ -37,25 +40,32 @@ AppleDecoderModule::Init() { MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); - sForceVDA = Preferences::GetBool("media.apple.forcevda", false); - if (sInitialized) { return; } + Preferences::AddBoolVarCache(&sForceVDA, "media.apple.forcevda", false); + + // Ensure IOSurface framework is loaded. + MacIOSurfaceLib::LoadLibrary(); + const bool loaded = MacIOSurfaceLib::isInit(); + // dlopen VideoDecodeAcceleration.framework if it's available. - sIsVDAAvailable = AppleVDALinker::Link(); + sIsVDAAvailable = loaded && AppleVDALinker::Link(); // dlopen CoreMedia.framework if it's available. - bool haveCoreMedia = AppleCMLinker::Link(); + sIsCoreMediaAvailable = AppleCMLinker::Link(); // dlopen VideoToolbox.framework if it's available. // We must link both CM and VideoToolbox framework to allow for proper // paired Link/Unlink calls - bool haveVideoToolbox = AppleVTLinker::Link(); - sIsVTAvailable = haveCoreMedia && haveVideoToolbox; + bool haveVideoToolbox = loaded && AppleVTLinker::Link(); + sIsVTAvailable = sIsCoreMediaAvailable && haveVideoToolbox; sIsVTHWAvailable = AppleVTLinker::skPropEnableHWAccel != nullptr; + sCanUseHardwareVideoDecoder = loaded && + gfxPlatform::GetPlatform()->CanUseHardwareVideoDecoding(); + sInitialized = true; } @@ -65,7 +75,6 @@ AppleDecoderModule::Startup() if (!sInitialized || (!sIsVDAAvailable && !sIsVTAvailable)) { return NS_ERROR_FAILURE; } - return NS_OK; } @@ -108,12 +117,14 @@ AppleDecoderModule::CreateAudioDecoder(const AudioInfo& aConfig, } bool -AppleDecoderModule::SupportsMimeType(const nsACString& aMimeType) +AppleDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { - return aMimeType.EqualsLiteral("audio/mpeg") || - aMimeType.EqualsLiteral("audio/mp4a-latm") || - aMimeType.EqualsLiteral("video/mp4") || - aMimeType.EqualsLiteral("video/avc"); + return (sIsCoreMediaAvailable && + (aMimeType.EqualsLiteral("audio/mpeg") || + aMimeType.EqualsLiteral("audio/mp4a-latm"))) || + ((sIsVTAvailable || sIsVDAAvailable) && + (aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/avc"))); } PlatformDecoderModule::ConversionRequired diff --git a/dom/media/platforms/apple/AppleDecoderModule.h b/dom/media/platforms/apple/AppleDecoderModule.h index c0b95e99a6..56d61b841f 100644 --- a/dom/media/platforms/apple/AppleDecoderModule.h +++ b/dom/media/platforms/apple/AppleDecoderModule.h @@ -32,15 +32,18 @@ public: FlushableTaskQueue* aAudioTaskQueue, MediaDataDecoderCallback* aCallback) override; - bool SupportsMimeType(const nsACString& aMimeType) override; + bool SupportsMimeType(const nsACString& aMimeType) const override; ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override; static void Init(); + static bool sCanUseHardwareVideoDecoder; + private: static bool sInitialized; + static bool sIsCoreMediaAvailable; static bool sIsVTAvailable; static bool sIsVTHWAvailable; static bool sIsVDAAvailable; diff --git a/dom/media/platforms/apple/AppleVDADecoder.cpp b/dom/media/platforms/apple/AppleVDADecoder.cpp index 6c001801d2..7f4a9923dc 100644 --- a/dom/media/platforms/apple/AppleVDADecoder.cpp +++ b/dom/media/platforms/apple/AppleVDADecoder.cpp @@ -6,6 +6,7 @@ #include +#include "AppleDecoderModule.h" #include "AppleUtils.h" #include "AppleVDADecoder.h" #include "AppleVDALinker.h" @@ -13,17 +14,20 @@ #include "mp4_demuxer/H264.h" #include "MP4Decoder.h" #include "MediaData.h" -#include "MacIOSurfaceImage.h" #include "mozilla/ArrayUtils.h" #include "nsAutoPtr.h" -#include "nsCocoaFeatures.h" #include "nsThreadUtils.h" #include "mozilla/Logging.h" #include "VideoUtils.h" #include #include "gfxPlatform.h" -extern PRLogModuleInfo* GetPDMLog(); +#ifndef MOZ_WIDGET_UIKIT +#include "nsCocoaFeatures.h" +#include "MacIOSurfaceImage.h" +#endif + +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) //#define LOG_MEDIA_SHA1 @@ -42,8 +46,14 @@ AppleVDADecoder::AppleVDADecoder(const VideoInfo& aConfig, , mDisplayHeight(aConfig.mDisplay.height) , mInputIncoming(0) , mIsShutDown(false) +#ifdef MOZ_WIDGET_UIKIT + , mUseSoftwareImages(true) + , mIs106(false) +#else , mUseSoftwareImages(false) , mIs106(!nsCocoaFeatures::OnLionOrLater()) + #endif + , mQueuedSamples(0) , mMonitor("AppleVideoDecoder") , mIsFlushing(false) , mDecoder(nullptr) @@ -213,15 +223,13 @@ PlatformCallback(void* decompressionOutputRefCon, // FIXME: Distinguish between errors and empty flushed frames. if (status != noErr || !image) { NS_WARNING("AppleVDADecoder decoder returned no data"); - return; - } - MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(), - "AppleVDADecoder returned an unexpected image type"); - - if (infoFlags & kVDADecodeInfo_FrameDropped) - { + image = nullptr; + } else if (infoFlags & kVDADecodeInfo_FrameDropped) { NS_WARNING(" ...frame dropped..."); - return; + image = nullptr; + } else { + MOZ_ASSERT(image || CFGetTypeID(image) == CVPixelBufferGetTypeID(), + "AppleVDADecoder returned an unexpected image type"); } AppleVDADecoder* decoder = @@ -257,12 +265,7 @@ PlatformCallback(void* decompressionOutputRefCon, byte_offset, is_sync_point == 1); - // Forward the data back to an object method which can access - // the correct reader's callback. - nsCOMPtr task = - NS_NewRunnableMethodWithArgs, AppleVDADecoder::AppleFrameRef>( - decoder, &AppleVDADecoder::OutputFrame, image, frameRef); - decoder->DispatchOutputTask(task.forget()); + decoder->OutputFrame(image, frameRef); } AppleVDADecoder::AppleFrameRef* @@ -275,28 +278,30 @@ AppleVDADecoder::CreateAppleFrameRef(const MediaRawData* aSample) void AppleVDADecoder::DrainReorderedFrames() { + MonitorAutoLock mon(mMonitor); while (!mReorderQueue.IsEmpty()) { mCallback->Output(mReorderQueue.Pop().get()); } + mQueuedSamples = 0; } void AppleVDADecoder::ClearReorderedFrames() { + MonitorAutoLock mon(mMonitor); while (!mReorderQueue.IsEmpty()) { mReorderQueue.Pop(); } + mQueuedSamples = 0; } // Copy and return a decoded frame. nsresult -AppleVDADecoder::OutputFrame(CFRefPtr aImage, +AppleVDADecoder::OutputFrame(CVPixelBufferRef aImage, AppleVDADecoder::AppleFrameRef aFrameRef) { - MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); - - if (mIsFlushing) { - // We are in the process of flushing; ignore frame. + if (mIsShutDown || mIsFlushing) { + // We are in the process of flushing or shutting down; ignore frame. return NS_OK; } @@ -308,6 +313,19 @@ AppleVDADecoder::OutputFrame(CFRefPtr aImage, aFrameRef.is_sync_point ? " keyframe" : "" ); + if (mQueuedSamples > mMaxRefFrames) { + // We had stopped requesting more input because we had received too much at + // the time. We can ask for more once again. + mCallback->InputExhausted(); + } + MOZ_ASSERT(mQueuedSamples); + mQueuedSamples--; + + if (!aImage) { + // Image was dropped by decoder. + return NS_OK; + } + // Where our resulting image will end up. RefPtr data; // Bounds. @@ -373,16 +391,13 @@ AppleVDADecoder::OutputFrame(CFRefPtr aImage, // Unlock the returned image data. CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); } else { +#ifndef MOZ_WIDGET_UIKIT IOSurfacePtr surface = MacIOSurfaceLib::CVPixelBufferGetIOSurface(aImage); MOZ_ASSERT(surface, "Decoder didn't return an IOSurface backed buffer"); RefPtr macSurface = new MacIOSurface(surface); - RefPtr image = - mImageContainer->CreateImage(ImageFormat::MAC_IOSURFACE); - layers::MacIOSurfaceImage* videoImage = - static_cast(image.get()); - videoImage->SetSurface(macSurface); + RefPtr image = new MacIOSurfaceImage(macSurface); data = VideoData::CreateFromImage(info, @@ -394,6 +409,9 @@ AppleVDADecoder::OutputFrame(CFRefPtr aImage, aFrameRef.is_sync_point, aFrameRef.decode_timestamp.ToMicroseconds(), visible); +#else + MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); +#endif } if (!data) { @@ -404,6 +422,7 @@ AppleVDADecoder::OutputFrame(CFRefPtr aImage, // Frames come out in DTS order but we need to output them // in composition order. + MonitorAutoLock mon(mMonitor); mReorderQueue.Push(data); while (mReorderQueue.Length() > mMaxRefFrames) { mCallback->Output(mReorderQueue.Pop().get()); @@ -471,6 +490,8 @@ AppleVDADecoder::SubmitFrame(MediaRawData* aSample) &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + mQueuedSamples++; + OSStatus rv = VDADecoderDecode(mDecoder, 0, block, @@ -478,6 +499,7 @@ AppleVDADecoder::SubmitFrame(MediaRawData* aSample) if (rv != noErr) { NS_WARNING("AppleVDADecoder: Couldn't pass frame to decoder"); + mCallback->Error(); return NS_ERROR_FAILURE; } @@ -493,7 +515,7 @@ AppleVDADecoder::SubmitFrame(MediaRawData* aSample) } // Ask for more data. - if (!mInputIncoming) { + if (!mInputIncoming && mQueuedSamples <= mMaxRefFrames) { LOG("AppleVDADecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } @@ -520,7 +542,7 @@ AppleVDADecoder::InitializeSession() &mDecoder); if (rv != noErr) { - NS_ERROR("AppleVDADecoder: Couldn't create decoder!"); + NS_WARNING("AppleVDADecoder: Couldn't create hardware VDA decoder"); return NS_ERROR_FAILURE; } @@ -594,6 +616,7 @@ AppleVDADecoder::CreateOutputConfiguration() &kCFTypeDictionaryValueCallBacks); } +#ifndef MOZ_WIDGET_UIKIT // Construct IOSurface Properties const void* IOSurfaceKeys[] = { MacIOSurfaceLib::kPropIsGlobal }; const void* IOSurfaceValues[] = { kCFBooleanTrue }; @@ -624,6 +647,9 @@ AppleVDADecoder::CreateOutputConfiguration() ArrayLength(outputKeys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); +#else + MOZ_ASSERT_UNREACHABLE("No MacIOSurface on iOS"); +#endif } /* static */ @@ -634,7 +660,7 @@ AppleVDADecoder::CreateVDADecoder( MediaDataDecoderCallback* aCallback, layers::ImageContainer* aImageContainer) { - if (!gfxPlatform::GetPlatform()->CanUseHardwareVideoDecoding()) { + if (!AppleDecoderModule::sCanUseHardwareVideoDecoder) { // This GPU is blacklisted for hardware decoding. return nullptr; } diff --git a/dom/media/platforms/apple/AppleVDADecoder.h b/dom/media/platforms/apple/AppleVDADecoder.h index 84113ff834..d40850f9a2 100644 --- a/dom/media/platforms/apple/AppleVDADecoder.h +++ b/dom/media/platforms/apple/AppleVDADecoder.h @@ -81,16 +81,9 @@ public: return true; } - void DispatchOutputTask(already_AddRefed aTask) - { - nsCOMPtr task = aTask; - if (mIsShutDown || mIsFlushing) { - return; - } - mTaskQueue->Dispatch(task.forget()); - } - - nsresult OutputFrame(CFRefPtr aImage, + // Access from the taskqueue and the decoder's thread. + // OutputFrame is thread-safe. + nsresult OutputFrame(CVPixelBufferRef aImage, AppleFrameRef aFrameRef); protected: @@ -108,26 +101,33 @@ protected: RefPtr mTaskQueue; MediaDataDecoderCallback* mCallback; RefPtr mImageContainer; - ReorderQueue mReorderQueue; uint32_t mPictureWidth; uint32_t mPictureHeight; uint32_t mDisplayWidth; uint32_t mDisplayHeight; + // Accessed on multiple threads, but only set in constructor. uint32_t mMaxRefFrames; // Increased when Input is called, and decreased when ProcessFrame runs. // Reaching 0 indicates that there's no pending Input. Atomic mInputIncoming; Atomic mIsShutDown; - bool mUseSoftwareImages; - bool mIs106; + const bool mUseSoftwareImages; + const bool mIs106; + + // Number of times a sample was queued via Input(). Will be decreased upon + // the decoder's callback being invoked. + // This is used to calculate how many frames has been buffered by the decoder. + Atomic mQueuedSamples; // For wait on mIsFlushing during Shutdown() process. + // Protects mReorderQueue. Monitor mMonitor; // Set on reader/decode thread calling Flush() to indicate that output is // not required and so input samples on mTaskQueue need not be processed. // Cleared on mTaskQueue in ProcessDrain(). Atomic mIsFlushing; + ReorderQueue mReorderQueue; private: VDADecoder mDecoder; diff --git a/dom/media/platforms/apple/AppleVDALinker.cpp b/dom/media/platforms/apple/AppleVDALinker.cpp index 79f5c96d74..045b1e24d3 100644 --- a/dom/media/platforms/apple/AppleVDALinker.cpp +++ b/dom/media/platforms/apple/AppleVDALinker.cpp @@ -10,7 +10,7 @@ #include "MainThreadUtils.h" #include "nsDebug.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/platforms/apple/AppleVTDecoder.cpp b/dom/media/platforms/apple/AppleVTDecoder.cpp index 069ec5ca98..000f95c638 100644 --- a/dom/media/platforms/apple/AppleVTDecoder.cpp +++ b/dom/media/platforms/apple/AppleVTDecoder.cpp @@ -7,6 +7,7 @@ #include #include "AppleCMLinker.h" +#include "AppleDecoderModule.h" #include "AppleUtils.h" #include "AppleVTDecoder.h" #include "AppleVTLinker.h" @@ -19,7 +20,7 @@ #include "VideoUtils.h" #include "gfxPlatform.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) //#define LOG_MEDIA_SHA1 @@ -167,18 +168,14 @@ PlatformCallback(void* decompressionOutputRefCon, // Validate our arguments. if (status != noErr || !image) { NS_WARNING("VideoToolbox decoder returned no data"); - return; + image = nullptr; + } else if (flags & kVTDecodeInfo_FrameDropped) { + NS_WARNING(" ...frame tagged as dropped..."); + } else { + MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(), + "VideoToolbox returned an unexpected image type"); } - if (flags & kVTDecodeInfo_FrameDropped) { - NS_WARNING(" ...frame dropped..."); - } - MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(), - "VideoToolbox returned an unexpected image type"); - - nsCOMPtr task = - NS_NewRunnableMethodWithArgs, AppleVTDecoder::AppleFrameRef>( - decoder, &AppleVTDecoder::OutputFrame, image, *frameRef); - decoder->DispatchOutputTask(task.forget()); + decoder->OutputFrame(image, *frameRef); } nsresult @@ -215,7 +212,7 @@ AppleVTDecoder::SubmitFrame(MediaRawData* aSample) // For some reason this gives me a double-free error with stagefright. AutoCFRelease block = nullptr; AutoCFRelease sample = nullptr; - VTDecodeInfoFlags flags; + VTDecodeInfoFlags infoFlags; OSStatus rv; // FIXME: This copies the sample data. I think we can provide @@ -242,20 +239,24 @@ AppleVTDecoder::SubmitFrame(MediaRawData* aSample) return NS_ERROR_FAILURE; } + mQueuedSamples++; + VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; rv = VTDecompressionSessionDecodeFrame(mSession, sample, decodeFlags, CreateAppleFrameRef(aSample), - &flags); - if (rv != noErr) { + &infoFlags); + if (rv != noErr && !(infoFlags & kVTDecodeInfo_FrameDropped)) { + LOG("AppleVTDecoder: Error %d VTDecompressionSessionDecodeFrame", rv); NS_WARNING("Couldn't pass frame to decoder"); + mCallback->Error(); return NS_ERROR_FAILURE; } // Ask for more data. - if (!mInputIncoming) { + if (!mInputIncoming && mQueuedSamples <= mMaxRefFrames) { LOG("AppleVTDecoder task queue empty; requesting more data"); mCallback->InputExhausted(); } @@ -382,7 +383,7 @@ AppleVTDecoder::CreateDecoderSpecification() const void* specKeys[] = { AppleVTLinker::skPropEnableHWAccel }; const void* specValues[1]; - if (gfxPlatform::GetPlatform()->CanUseHardwareVideoDecoding()) { + if (AppleDecoderModule::sCanUseHardwareVideoDecoder) { specValues[0] = kCFBooleanTrue; } else { // This GPU is blacklisted for hardware decoding. diff --git a/dom/media/platforms/apple/AppleVTLinker.cpp b/dom/media/platforms/apple/AppleVTLinker.cpp index a4f7e26604..8adda27225 100644 --- a/dom/media/platforms/apple/AppleVTLinker.cpp +++ b/dom/media/platforms/apple/AppleVTLinker.cpp @@ -11,7 +11,7 @@ #include "mozilla/ArrayUtils.h" #include "nsDebug.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h index a823cc4a26..f81f150e56 100644 --- a/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h +++ b/dom/media/platforms/ffmpeg/FFmpegDecoderModule.h @@ -52,7 +52,7 @@ public: return decoder.forget(); } - bool SupportsMimeType(const nsACString& aMimeType) override + bool SupportsMimeType(const nsACString& aMimeType) const override { AVCodecID audioCodec = FFmpegAudioDecoder::GetCodecId(aMimeType); AVCodecID videoCodec = FFmpegH264Decoder::GetCodecId(aMimeType); diff --git a/dom/media/platforms/ffmpeg/FFmpegLog.h b/dom/media/platforms/ffmpeg/FFmpegLog.h index 19b950bf46..eb1d22d99c 100644 --- a/dom/media/platforms/ffmpeg/FFmpegLog.h +++ b/dom/media/platforms/ffmpeg/FFmpegLog.h @@ -7,7 +7,7 @@ #ifndef __FFmpegLog_h__ #define __FFmpegLog_h__ -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define FFMPEG_LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) #endif // __FFmpegLog_h__ diff --git a/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp b/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp index 201ba39b1f..47227eb33e 100644 --- a/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp +++ b/dom/media/platforms/gonk/GonkAudioDecoderManager.cpp @@ -21,12 +21,14 @@ #include "MediaData.h" #include "MediaInfo.h" +#define CODECCONFIG_TIMEOUT_US 10000LL +#define READ_OUTPUT_BUFFER_TIMEOUT_US 0LL + #include #define GADM_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkAudioDecoderManager", __VA_ARGS__) -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) -#define READ_OUTPUT_BUFFER_TIMEOUT_US 3000 using namespace android; typedef android::MediaCodecProxy MediaCodecProxy; @@ -34,17 +36,15 @@ typedef android::MediaCodecProxy MediaCodecProxy; namespace mozilla { GonkAudioDecoderManager::GonkAudioDecoderManager(const AudioInfo& aConfig) - : mLastDecodedTime(0) - , mAudioChannels(aConfig.mChannels) + : mAudioChannels(aConfig.mChannels) , mAudioRate(aConfig.mRate) , mAudioProfile(aConfig.mProfile) - , mAudioBuffer(nullptr) - , mMonitor("GonkAudioDecoderManager") + , mAudioCompactor(mAudioQueue) { MOZ_COUNT_CTOR(GonkAudioDecoderManager); MOZ_ASSERT(mAudioChannels); - mUserData.AppendElements(aConfig.mCodecSpecificConfig->Elements(), - aConfig.mCodecSpecificConfig->Length()); + mCodecSpecificData = aConfig.mCodecSpecificConfig; + mMimeType = aConfig.mMimeType; } GonkAudioDecoderManager::~GonkAudioDecoderManager() @@ -53,9 +53,9 @@ GonkAudioDecoderManager::~GonkAudioDecoderManager() } RefPtr -GonkAudioDecoderManager::Init(MediaDataDecoderCallback* aCallback) +GonkAudioDecoderManager::Init() { - if (InitMediaCodecProxy(aCallback)) { + if (InitMediaCodecProxy()) { return InitPromise::CreateAndResolve(TrackType::kAudioTrack, __func__); } else { return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); @@ -63,21 +63,18 @@ GonkAudioDecoderManager::Init(MediaDataDecoderCallback* aCallback) } bool -GonkAudioDecoderManager::InitMediaCodecProxy(MediaDataDecoderCallback* aCallback) +GonkAudioDecoderManager::InitMediaCodecProxy() { - if (mLooper != nullptr) { + status_t rv = OK; + if (!InitLoopers(MediaData::AUDIO_DATA)) { return false; } - // Create ALooper - mLooper = new ALooper; - mLooper->setName("GonkAudioDecoderManager"); - mLooper->start(); - mDecoder = MediaCodecProxy::CreateByType(mLooper, "audio/mp4a-latm", false, nullptr); + mDecoder = MediaCodecProxy::CreateByType(mDecodeLooper, mMimeType.get(), false); if (!mDecoder.get()) { return false; } - if (!mDecoder->AskMediaCodecAndWait()) + if (!mDecoder->AllocateAudioMediaCodec()) { mDecoder = nullptr; return false; @@ -86,7 +83,7 @@ GonkAudioDecoderManager::InitMediaCodecProxy(MediaDataDecoderCallback* aCallback // Fixed values GADM_LOG("Configure audio mime type:%s, chan no:%d, sample-rate:%d, profile:%d", mMimeType.get(), mAudioChannels, mAudioRate, mAudioProfile); - format->setString("mime", "audio/mp4a-latm"); + format->setString("mime", mMimeType.get()); format->setInt32("channel-count", mAudioChannels); format->setInt32("sample-rate", mAudioRate); format->setInt32("aac-profile", mAudioProfile); @@ -94,8 +91,12 @@ GonkAudioDecoderManager::InitMediaCodecProxy(MediaDataDecoderCallback* aCallback if (err != OK || !mDecoder->Prepare()) { return false; } - status_t rv = mDecoder->Input(mUserData.Elements(), mUserData.Length(), 0, - android::MediaCodec::BUFFER_FLAG_CODECCONFIG); + + if (mMimeType.EqualsLiteral("audio/mp4a-latm")) { + rv = mDecoder->Input(mCodecSpecificData->Elements(), mCodecSpecificData->Length(), 0, + android::MediaCodec::BUFFER_FLAG_CODECCONFIG, + CODECCONFIG_TIMEOUT_US); + } if (rv == OK) { return true; @@ -105,117 +106,52 @@ GonkAudioDecoderManager::InitMediaCodecProxy(MediaDataDecoderCallback* aCallback } } -bool -GonkAudioDecoderManager::HasQueuedSample() -{ - MonitorAutoLock mon(mMonitor); - return mQueueSample.Length(); -} - nsresult -GonkAudioDecoderManager::Input(MediaRawData* aSample) +GonkAudioDecoderManager::CreateAudioData(MediaBuffer* aBuffer, int64_t aStreamOffset) { - MonitorAutoLock mon(mMonitor); - RefPtr sample; - - if (aSample) { - sample = aSample; - } else { - // It means EOS with empty sample. - sample = new MediaRawData(); - } - - mQueueSample.AppendElement(sample); - - status_t rv; - while (mQueueSample.Length()) { - RefPtr data = mQueueSample.ElementAt(0); - { - MonitorAutoUnlock mon_exit(mMonitor); - rv = mDecoder->Input(reinterpret_cast(data->mData), - data->mSize, - data->mTime, - 0); - } - if (rv == OK) { - mQueueSample.RemoveElementAt(0); - } else if (rv == -EAGAIN || rv == -ETIMEDOUT) { - // In most cases, EAGAIN or ETIMEOUT are safe because OMX can't fill - // buffer on time. - return NS_OK; - } else { - return NS_ERROR_UNEXPECTED; - } - } - - return NS_OK; -} - -nsresult -GonkAudioDecoderManager::CreateAudioData(int64_t aStreamOffset, AudioData **v) { - if (!(mAudioBuffer != nullptr && mAudioBuffer->data() != nullptr)) { + if (!(aBuffer != nullptr && aBuffer->data() != nullptr)) { GADM_LOG("Audio Buffer is not valid!"); return NS_ERROR_UNEXPECTED; } int64_t timeUs; - if (!mAudioBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { + if (!aBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { return NS_ERROR_UNEXPECTED; } - if (mAudioBuffer->range_length() == 0) { + if (aBuffer->range_length() == 0) { // Some decoders may return spurious empty buffers that we just want to ignore // quoted from Android's AwesomePlayer.cpp - ReleaseAudioBuffer(); return NS_ERROR_NOT_AVAILABLE; } - if (mLastDecodedTime > timeUs) { - ReleaseAudioBuffer(); + if (mLastTime > timeUs) { GADM_LOG("Output decoded sample time is revert. time=%lld", timeUs); MOZ_ASSERT(false); return NS_ERROR_NOT_AVAILABLE; } - mLastDecodedTime = timeUs; + mLastTime = timeUs; - const uint8_t *data = static_cast(mAudioBuffer->data()); - size_t dataOffset = mAudioBuffer->range_offset(); - size_t size = mAudioBuffer->range_length(); + const uint8_t *data = static_cast(aBuffer->data()); + size_t dataOffset = aBuffer->range_offset(); + size_t size = aBuffer->range_length(); - nsAutoArrayPtr buffer(new AudioDataValue[size/2]); - memcpy(buffer.get(), data+dataOffset, size); uint32_t frames = size / (2 * mAudioChannels); CheckedInt64 duration = FramesToUsecs(frames, mAudioRate); if (!duration.isValid()) { return NS_ERROR_UNEXPECTED; } - RefPtr audioData = new AudioData(aStreamOffset, - timeUs, - duration.value(), - frames, - buffer.forget(), - mAudioChannels, - mAudioRate); - ReleaseAudioBuffer(); - audioData.forget(v); - return NS_OK; -} - -nsresult -GonkAudioDecoderManager::Flush() -{ - { - MonitorAutoLock mon(mMonitor); - mQueueSample.Clear(); - } - - mLastDecodedTime = 0; - - if (mDecoder->flush() != OK) { - return NS_ERROR_FAILURE; - } + typedef AudioCompactor::NativeCopy OmxCopy; + mAudioCompactor.Push(aStreamOffset, + timeUs, + mAudioRate, + frames, + mAudioChannels, + OmxCopy(data+dataOffset, + size, + mAudioChannels)); return NS_OK; } @@ -224,22 +160,22 @@ GonkAudioDecoderManager::Output(int64_t aStreamOffset, RefPtr& aOutData) { aOutData = nullptr; + if (mAudioQueue.GetSize() > 0) { + aOutData = mAudioQueue.PopFront(); + return mAudioQueue.AtEndOfStream() ? NS_ERROR_ABORT : NS_OK; + } + status_t err; - err = mDecoder->Output(&mAudioBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); + MediaBuffer* audioBuffer = nullptr; + err = mDecoder->Output(&audioBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); + AutoReleaseMediaBuffer a(audioBuffer, mDecoder.get()); switch (err) { case OK: { - RefPtr data; - nsresult rv = CreateAudioData(aStreamOffset, getter_AddRefs(data)); - if (rv == NS_ERROR_NOT_AVAILABLE) { - // Decoder outputs an empty video buffer, try again - return NS_ERROR_NOT_AVAILABLE; - } else if (rv != NS_OK || data == nullptr) { - return NS_ERROR_UNEXPECTED; - } - aOutData = data; - return NS_OK; + nsresult rv = CreateAudioData(audioBuffer, aStreamOffset); + NS_ENSURE_SUCCESS(rv, rv); + break; } case android::INFO_FORMAT_CHANGED: { @@ -281,17 +217,11 @@ GonkAudioDecoderManager::Output(int64_t aStreamOffset, case android::ERROR_END_OF_STREAM: { GADM_LOG("Got EOS frame!"); - RefPtr data; - nsresult rv = CreateAudioData(aStreamOffset, getter_AddRefs(data)); - if (rv == NS_ERROR_NOT_AVAILABLE) { - // For EOS, no need to do any thing. - return NS_ERROR_ABORT; - } else if (rv != NS_OK || data == nullptr) { - GADM_LOG("Failed to create audio data!"); - return NS_ERROR_UNEXPECTED; - } - aOutData = data; - return NS_ERROR_ABORT; + nsresult rv = CreateAudioData(audioBuffer, aStreamOffset); + NS_ENSURE_SUCCESS(rv, NS_ERROR_ABORT); + MOZ_ASSERT(mAudioQueue.GetSize() > 0); + mAudioQueue.Finish(); + break; } case -ETIMEDOUT: { @@ -305,14 +235,22 @@ GonkAudioDecoderManager::Output(int64_t aStreamOffset, } } - return NS_OK; + if (mAudioQueue.GetSize() > 0) { + aOutData = mAudioQueue.PopFront(); + // Return NS_ERROR_ABORT at the last sample. + return mAudioQueue.AtEndOfStream() ? NS_ERROR_ABORT : NS_OK; + } + + return NS_ERROR_NOT_AVAILABLE; } -void GonkAudioDecoderManager::ReleaseAudioBuffer() { - if (mAudioBuffer) { - mDecoder->ReleaseMediaBuffer(mAudioBuffer); - mAudioBuffer = nullptr; - } +nsresult +GonkAudioDecoderManager::Flush() +{ + GADM_LOG("FLUSH<<<"); + mAudioQueue.Reset(); + GADM_LOG(">>>FLUSH"); + return GonkDecoderManager::Flush(); } } // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkAudioDecoderManager.h b/dom/media/platforms/gonk/GonkAudioDecoderManager.h index 4744c24caa..fc23cf133d 100644 --- a/dom/media/platforms/gonk/GonkAudioDecoderManager.h +++ b/dom/media/platforms/gonk/GonkAudioDecoderManager.h @@ -7,13 +7,13 @@ #if !defined(GonkAudioDecoderManager_h_) #define GonkAudioDecoderManager_h_ +#include "AudioCompactor.h" #include "mozilla/RefPtr.h" #include "GonkMediaDataDecoder.h" using namespace android; namespace android { -struct MOZ_EXPORT ALooper; class MOZ_EXPORT MediaBuffer; } // namespace android @@ -24,45 +24,28 @@ typedef android::MediaCodecProxy MediaCodecProxy; public: GonkAudioDecoderManager(const AudioInfo& aConfig); - virtual ~GonkAudioDecoderManager() override; + virtual ~GonkAudioDecoderManager(); - RefPtr Init(MediaDataDecoderCallback* aCallback) override; - - nsresult Input(MediaRawData* aSample) override; + RefPtr Init() override; nsresult Output(int64_t aStreamOffset, RefPtr& aOutput) override; - nsresult Flush() override; - - bool HasQueuedSample() override; + virtual nsresult Flush() override; private: - bool InitMediaCodecProxy(MediaDataDecoderCallback* aCallback); + bool InitMediaCodecProxy(); - nsresult CreateAudioData(int64_t aStreamOffset, - AudioData** aOutData); - - void ReleaseAudioBuffer(); - - int64_t mLastDecodedTime; + nsresult CreateAudioData(MediaBuffer* aBuffer, int64_t aStreamOffset); uint32_t mAudioChannels; uint32_t mAudioRate; const uint32_t mAudioProfile; - nsTArray mUserData; - MediaDataDecoderCallback* mReaderCallback; - android::MediaBuffer* mAudioBuffer; - android::sp mLooper; + MediaQueue mAudioQueue; - // This monitor protects mQueueSample. - Monitor mMonitor; + AudioCompactor mAudioCompactor; - // An queue with the MP4 samples which are waiting to be sent into OMX. - // If an element is an empty MP4Sample, that menas EOS. There should not - // any sample be queued after EOS. - nsTArray> mQueueSample; }; } // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkDecoderModule.cpp b/dom/media/platforms/gonk/GonkDecoderModule.cpp index 2ec8152b8c..3856898301 100644 --- a/dom/media/platforms/gonk/GonkDecoderModule.cpp +++ b/dom/media/platforms/gonk/GonkDecoderModule.cpp @@ -61,12 +61,13 @@ GonkDecoderModule::DecoderNeedsConversion(const TrackInfo& aConfig) const } bool -GonkDecoderModule::SupportsMimeType(const nsACString& aMimeType) +GonkDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { return aMimeType.EqualsLiteral("audio/mp4a-latm") || - aMimeType.EqualsLiteral("video/mp4") || - aMimeType.EqualsLiteral("video/mp4v-es") || - aMimeType.EqualsLiteral("video/avc"); + aMimeType.EqualsLiteral("audio/3gpp") || + aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/mp4v-es") || + aMimeType.EqualsLiteral("video/avc"); } } // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkDecoderModule.h b/dom/media/platforms/gonk/GonkDecoderModule.h index efb80572bc..5034f3ca2b 100644 --- a/dom/media/platforms/gonk/GonkDecoderModule.h +++ b/dom/media/platforms/gonk/GonkDecoderModule.h @@ -35,7 +35,7 @@ public: ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override; - bool SupportsMimeType(const nsACString& aMimeType) override; + bool SupportsMimeType(const nsACString& aMimeType) const override; }; diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp index 659229d701..ec1eec4d57 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.cpp @@ -8,17 +8,184 @@ #include "nsTArray.h" #include "MediaCodecProxy.h" +#include + #include "mozilla/Logging.h" #include #define GMDD_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkMediaDataDecoder", __VA_ARGS__) +#define INPUT_TIMEOUT_US 0LL // Don't wait for buffer if none is available. +#define MIN_QUEUED_SAMPLES 2 -extern PRLogModuleInfo* GetPDMLog(); +#ifdef DEBUG +#include +#endif + +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) using namespace android; namespace mozilla { +GonkDecoderManager::GonkDecoderManager(MediaTaskQueue* aTaskQueue) + : mMonitor("GonkDecoderManager") + , mTaskQueue(aTaskQueue) +{ +} + +bool +GonkDecoderManager::InitLoopers(MediaData::Type aType) +{ + MOZ_ASSERT(mDecodeLooper.get() == nullptr && mTaskLooper.get() == nullptr); + MOZ_ASSERT(aType == MediaData::VIDEO_DATA || aType == MediaData::AUDIO_DATA); + + const char* suffix = (aType == MediaData::VIDEO_DATA ? "video" : "audio"); + mDecodeLooper = new ALooper; + android::AString name("MediaCodecProxy/"); + name.append(suffix); + mDecodeLooper->setName(name.c_str()); + + mTaskLooper = new ALooper; + name.setTo("GonkDecoderManager/"); + name.append(suffix); + mTaskLooper->setName(name.c_str()); + mTaskLooper->registerHandler(this); + +#ifdef DEBUG + sp findThreadId(new AMessage(kNotifyFindLooperId, id())); + findThreadId->post(); +#endif + + return mDecodeLooper->start() == OK && mTaskLooper->start() == OK; +} + +nsresult +GonkDecoderManager::Input(MediaRawData* aSample) +{ + RefPtr sample; + + if (aSample) { + sample = aSample; + } else { + // It means EOS with empty sample. + sample = new MediaRawData(); + } + { + MutexAutoLock lock(mMutex); + mQueuedSamples.AppendElement(sample); + } + + sp input = new AMessage(kNotifyProcessInput, id()); + if (!aSample) { + input->setInt32("input-eos", 1); + } + input->post(); + return NS_OK; +} + +int32_t +GonkDecoderManager::ProcessQueuedSamples() +{ + MOZ_ASSERT(OnTaskLooper()); + + MutexAutoLock lock(mMutex); + status_t rv; + while (mQueuedSamples.Length()) { + RefPtr data = mQueuedSamples.ElementAt(0); + { + rv = mDecoder->Input(reinterpret_cast(data->Data()), + data->Size(), + data->mTime, + 0, + INPUT_TIMEOUT_US); + } + if (rv == OK) { + mQueuedSamples.RemoveElementAt(0); + } else if (rv == -EAGAIN || rv == -ETIMEDOUT) { + // In most cases, EAGAIN or ETIMEOUT are safe because OMX can't fill + // buffer on time. + break; + } else { + return rv; + } + } + return mQueuedSamples.Length(); +} + +nsresult +GonkDecoderManager::Flush() +{ + if (mDecoder == nullptr) { + GMDD_LOG("Decoder is not initialized"); + return NS_ERROR_UNEXPECTED; + } + { + MutexAutoLock lock(mMutex); + mQueuedSamples.Clear(); + } + + mLastTime = 0; + + MonitorAutoLock lock(mFlushMonitor); + mIsFlushing = true; + sp flush = new AMessage(kNotifyProcessFlush, id()); + flush->post(); + while (mIsFlushing) { + lock.Wait(); + } + return NS_OK; +} + +nsresult +GonkDecoderManager::Input(MediaRawData* aSample) +{ + ReentrantMonitorAutoEnter mon(mMonitor); + nsRefPtr sample; + + if (!aSample) { + // It means EOS with empty sample. + sample = new MediaRawData(); + } else { + sample = aSample; + if (!PerformFormatSpecificProcess(sample)) { + return NS_ERROR_FAILURE; + } + } + + mQueueSample.AppendElement(sample); + + status_t rv; + while (mQueueSample.Length()) { + nsRefPtr data = mQueueSample.ElementAt(0); + { + ReentrantMonitorAutoExit mon_exit(mMonitor); + rv = SendSampleToOMX(data); + } + if (rv == OK) { + mQueueSample.RemoveElementAt(0); + } else if (rv == -EAGAIN || rv == -ETIMEDOUT) { + // In most cases, EAGAIN or ETIMEOUT are safe because OMX can't fill + // buffer on time. + return NS_OK; + } else { + return NS_ERROR_UNEXPECTED; + } + } + + return NS_OK; +} + +nsresult +GonkMediaDataDecoder::Init() +{ + sp decoder; + decoder = mManager->Init(mCallback); + mDecoder = decoder; + mDrainComplete = false; + + return NS_OK; +} + nsresult GonkDecoderManager::Shutdown() { @@ -33,16 +200,179 @@ GonkDecoderManager::Shutdown() return NS_OK; } +bool +GonkDecoderManager::HasQueuedSample() +{ + MutexAutoLock lock(mMutex); + return mQueuedSamples.Length(); +} + +void +GonkDecoderManager::ProcessInput(bool aEndOfStream) +{ + MOZ_ASSERT(OnTaskLooper()); + + status_t rv = ProcessQueuedSamples(); + if (rv >= 0) { + if (!aEndOfStream && rv <= MIN_QUEUED_SAMPLES) { + mDecodeCallback->InputExhausted(); + } + + if (mToDo.get() == nullptr) { + mToDo = new AMessage(kNotifyDecoderActivity, id()); + if (aEndOfStream) { + mToDo->setInt32("input-eos", 1); + } + mDecoder->requestActivityNotification(mToDo); + } + } else { + GMDD_LOG("input processed: error#%d", rv); + mDecodeCallback->Error(); + } +} + +void +GonkDecoderManager::ProcessFlush() +{ + MOZ_ASSERT(OnTaskLooper()); + + mLastTime = INT64_MIN; + MonitorAutoLock lock(mFlushMonitor); + mWaitOutput.Clear(); + if (mDecoder->flush() != OK) { + GMDD_LOG("flush error"); + mDecodeCallback->Error(); + } + mIsFlushing = false; + lock.NotifyAll(); +} + +// Use output timestamp to determine which output buffer is already returned +// and remove corresponding info, except for EOS, from the waiting list. +// This method handles the cases that audio decoder sends multiple output +// buffers for one input. +void +GonkDecoderManager::UpdateWaitingList(int64_t aForgetUpTo) +{ + MOZ_ASSERT(OnTaskLooper()); + + size_t i; + for (i = 0; i < mWaitOutput.Length(); i++) { + const auto& item = mWaitOutput.ElementAt(i); + if (item.mEOS || item.mTimestamp > aForgetUpTo) { + break; + } + } + if (i > 0) { + mWaitOutput.RemoveElementsAt(0, i); + } +} + +void +GonkDecoderManager::ProcessToDo(bool aEndOfStream) +{ + MOZ_ASSERT(OnTaskLooper()); + + MOZ_ASSERT(mToDo.get() != nullptr); + mToDo.clear(); + + if (HasQueuedSample()) { + status_t pendingInput = ProcessQueuedSamples(); + if (pendingInput < 0) { + mDecodeCallback->Error(); + return; + } + if (!aEndOfStream && pendingInput <= MIN_QUEUED_SAMPLES) { + mDecodeCallback->InputExhausted(); + } + } + + nsresult rv = NS_OK; + while (mWaitOutput.Length() > 0) { + nsRefPtr output; + int64_t offset = mWaitOutput.ElementAt(0); + rv = Output(offset, output); + if (rv == NS_OK) { + mWaitOutput.RemoveElementAt(0); + mDecodeCallback->Output(output); + } else if (rv == NS_ERROR_ABORT) { + GMDD_LOG("eos output"); + mWaitOutput.RemoveElementAt(0); + MOZ_ASSERT(mQueuedSamples.IsEmpty()); + MOZ_ASSERT(mWaitOutput.IsEmpty()); + // EOS + if (output) { + mDecodeCallback->Output(output); + } + mDecodeCallback->DrainComplete(); + return; + } else if (rv == NS_ERROR_NOT_AVAILABLE) { + break; + } else { + mDecodeCallback->Error(); + return; + } + } + + if (HasQueuedSample() || mWaitOutput.Length() > 0) { + mToDo = new AMessage(kNotifyDecoderActivity, id()); + mDecoder->requestActivityNotification(mToDo); + } +} + +void +GonkDecoderManager::onMessageReceived(const sp &aMessage) +{ + switch (aMessage->what()) { + case kNotifyProcessInput: + { + int32_t eos = 0; + ProcessInput(aMessage->findInt32("input-eos", &eos) && eos); + break; + } + case kNotifyProcessFlush: + { + ProcessFlush(); + break; + } + case kNotifyDecoderActivity: + { + int32_t eos = 0; + ProcessToDo(aMessage->findInt32("input-eos", &eos) && eos); + break; + } +#ifdef DEBUG + case kNotifyFindLooperId: + { + mTaskLooperId = androidGetThreadId(); + MOZ_ASSERT(mTaskLooperId); + break; + } +#endif + default: + { + TRESPASS(); + break; + } + } +} + +#ifdef DEBUG +bool +GonkDecoderManager::OnTaskLooper() +{ + return androidGetThreadId() == mTaskLooperId; +} +#endif + GonkMediaDataDecoder::GonkMediaDataDecoder(GonkDecoderManager* aManager, FlushableTaskQueue* aTaskQueue, MediaDataDecoderCallback* aCallback) : mTaskQueue(aTaskQueue) - , mCallback(aCallback) , mManager(aManager) - , mSignaledEOS(false) - , mDrainComplete(false) { MOZ_COUNT_CTOR(GonkMediaDataDecoder); + mManager->SetDecodeCallback(aCallback); } GonkMediaDataDecoder::~GonkMediaDataDecoder() @@ -53,95 +383,28 @@ GonkMediaDataDecoder::~GonkMediaDataDecoder() RefPtr GonkMediaDataDecoder::Init() { - mDrainComplete = false; - - return mManager->Init(mCallback); + return mManager->Init(); } nsresult GonkMediaDataDecoder::Shutdown() { - return mManager->Shutdown(); + nsresult rv = mManager->Shutdown(); + + // Because codec allocated runnable and init promise is at reader TaskQueue, + // so manager needs to be destroyed at reader TaskQueue to prevent racing. + mManager = nullptr; + return rv; } // Inserts data into the decoder's pipeline. nsresult GonkMediaDataDecoder::Input(MediaRawData* aSample) { - nsCOMPtr runnable( - NS_NewRunnableMethodWithArg>( - this, - &GonkMediaDataDecoder::ProcessDecode, - RefPtr(aSample))); - mTaskQueue->Dispatch(runnable.forget()); + mManager->Input(aSample); return NS_OK; } -void -GonkMediaDataDecoder::ProcessDecode(MediaRawData* aSample) -{ - nsresult rv = mManager->Input(aSample); - if (rv != NS_OK) { - NS_WARNING("GonkMediaDataDecoder failed to input data"); - GMDD_LOG("Failed to input data err: %d",int(rv)); - mCallback->Error(); - return; - } - if (aSample) { - mLastStreamOffset = aSample->mOffset; - } - ProcessOutput(); -} - -void -GonkMediaDataDecoder::ProcessOutput() -{ - RefPtr output; - nsresult rv = NS_ERROR_ABORT; - - while (!mDrainComplete) { - // There are samples in queue, try to send them into decoder when EOS. - if (mSignaledEOS && mManager->HasQueuedSample()) { - GMDD_LOG("ProcessOutput: drain all input samples"); - rv = mManager->Input(nullptr); - } - rv = mManager->Output(mLastStreamOffset, output); - if (rv == NS_OK) { - mCallback->Output(output); - continue; - } else if (rv == NS_ERROR_NOT_AVAILABLE && mSignaledEOS) { - // Try to get more frames before getting EOS frame - continue; - } - else { - break; - } - } - - MOZ_ASSERT_IF(mSignaledEOS, !mManager->HasQueuedSample()); - - if (rv == NS_ERROR_NOT_AVAILABLE && !mSignaledEOS) { - mCallback->InputExhausted(); - return; - } - if (rv != NS_OK) { - NS_WARNING("GonkMediaDataDecoder failed to output data"); - GMDD_LOG("Failed to output data"); - // GonkDecoderManangers report NS_ERROR_ABORT when EOS is reached. - if (rv == NS_ERROR_ABORT) { - if (output) { - mCallback->Output(output); - } - mCallback->DrainComplete(); - mSignaledEOS = false; - mDrainComplete = true; - return; - } - GMDD_LOG("Callback error!"); - mCallback->Error(); - } -} - nsresult GonkMediaDataDecoder::Flush() { @@ -150,25 +413,13 @@ GonkMediaDataDecoder::Flush() // it's executing at all. Note the MP4Reader ignores all output while // flushing. mTaskQueue->Flush(); - mDrainComplete = false; return mManager->Flush(); } -void -GonkMediaDataDecoder::ProcessDrain() -{ - // Notify decoder input EOS by sending a null data. - ProcessDecode(nullptr); - mSignaledEOS = true; - ProcessOutput(); -} - nsresult GonkMediaDataDecoder::Drain() { - nsCOMPtr runnable = - NS_NewRunnableMethod(this, &GonkMediaDataDecoder::ProcessDrain); - mTaskQueue->Dispatch(runnable.forget()); + mManager->Input(nullptr); return NS_OK; } diff --git a/dom/media/platforms/gonk/GonkMediaDataDecoder.h b/dom/media/platforms/gonk/GonkMediaDataDecoder.h index ad3ffc7404..5a699784e9 100644 --- a/dom/media/platforms/gonk/GonkMediaDataDecoder.h +++ b/dom/media/platforms/gonk/GonkMediaDataDecoder.h @@ -7,8 +7,11 @@ #if !defined(GonkMediaDataDecoder_h_) #define GonkMediaDataDecoder_h_ #include "PlatformDecoderModule.h" +#include namespace android { +struct ALooper; +class MediaBuffer; class MediaCodecProxy; } // namespace android @@ -16,7 +19,7 @@ namespace mozilla { class MediaRawData; // Manage the data flow from inputting encoded data and outputting decode data. -class GonkDecoderManager { +class GonkDecoderManager : public android::AHandler { public: typedef TrackInfo::TrackType TrackType; typedef MediaDataDecoder::InitPromise InitPromise; @@ -24,39 +27,142 @@ public: virtual ~GonkDecoderManager() {} - virtual RefPtr Init(MediaDataDecoderCallback* aCallback) = 0; + virtual RefPtr Init() = 0; - // Add samples into OMX decoder or queue them if decoder is out of input buffer. - virtual nsresult Input(MediaRawData* aSample) = 0; + // Asynchronously send sample into mDecoder. If out of input buffer, aSample + // will be queued for later re-send. + nsresult Input(MediaRawData* aSample); - // Produces decoded output, it blocks until output can be produced or a timeout - // is expired or until EOS. Returns NS_OK on success, or NS_ERROR_NOT_AVAILABLE - // if there's not enough data to produce more output. If this returns a failure - // code other than NS_ERROR_NOT_AVAILABLE, an error will be reported to the - // MP4Reader. - // The overrided class should follow the same behaviour. - virtual nsresult Output(int64_t aStreamOffset, - RefPtr& aOutput) = 0; - - // Flush the queued sample. - virtual nsresult Flush() = 0; + // Flush the queued samples and signal decoder to throw all pending input/output away. + nsresult Flush(); // Shutdown decoder and rejects the init promise. - nsresult Shutdown(); + virtual nsresult Shutdown(); - void ClearQueuedSample() { - MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); - mQueueSample.Clear(); + // True if sample is queued. + bool HasQueuedSample(); + + // Set callback for decoder events, such as requesting more input, + // returning output, or reporting error. + void SetDecodeCallback(MediaDataDecoderCallback* aCallback) + { + mDecodeCallback = aCallback; } protected: + GonkDecoderManager() + : mMutex("GonkDecoderManager") + , mLastTime(0) + , mFlushMonitor("GonkDecoderManager::Flush") + , mIsFlushing(false) + , mDecodeCallback(nullptr) + {} + bool InitLoopers(MediaData::Type aType); + + void onMessageReceived(const android::sp &aMessage) override; + + // Produces decoded output. It returns NS_OK on success, or NS_ERROR_NOT_AVAILABLE + // when output is not produced yet. + // If this returns a failure code other than NS_ERROR_NOT_AVAILABLE, an error + // will be reported through mDecodeCallback. + virtual nsresult Output(int64_t aStreamOffset, + RefPtr& aOutput) = 0; + + // Send queued samples to OMX. It returns how many samples are still in + // queue after processing, or negative error code if failed. + int32_t ProcessQueuedSamples(); + + void ProcessInput(bool aEndOfStream); + void ProcessFlush(); + void ProcessToDo(bool aEndOfStream); + + RefPtr mCodecSpecificData; + + nsAutoCString mMimeType; // MediaCodedc's wrapper that performs the decoding. android::sp mDecoder; + // Looper for mDecoder to run on. + android::sp mDecodeLooper; + // Looper to run decode tasks such as processing input, output, flush, and + // recycling output buffers. + android::sp mTaskLooper; + // Message codes for tasks running on mTaskLooper. + enum { + // Decoder will send this to indicate internal state change such as input or + // output buffers availability. Used to run pending input & output tasks. + kNotifyDecoderActivity = 'nda ', + // Signal the decoder to flush. + kNotifyProcessFlush = 'npf ', + // Used to process queued samples when there is new input. + kNotifyProcessInput = 'npi ', +#ifdef DEBUG + kNotifyFindLooperId = 'nfli', +#endif + }; MozPromiseHolder mInitPromise; + Mutex mMutex; // Protects mQueuedSamples. + // A queue that stores the samples waiting to be sent to mDecoder. + // Empty element means EOS and there shouldn't be any sample be queued after it. + // Samples are queued in caller's thread and dequeued in mTaskLooper. + nsTArray> mQueuedSamples; + + // The last decoded frame presentation time. Only accessed on mTaskLooper. + int64_t mLastTime; + + Monitor mFlushMonitor; // Waits for flushing to complete. + bool mIsFlushing; // Protected by mFlushMonitor. + + // Remembers the notification that is currently waiting for the decoder event + // to avoid requesting more than one notification at the time, which is + // forbidden by mDecoder. + android::sp mToDo; + + // Stores the offset of every output that needs to be read from mDecoder. + nsTArray mWaitOutput; + + MediaDataDecoderCallback* mDecodeCallback; // Reports decoder output or error. + +private: + void UpdateWaitingList(int64_t aForgetUpTo); + +#ifdef DEBUG + typedef void* LooperId; + + bool OnTaskLooper(); + LooperId mTaskLooperId; +#endif +}; + +class AutoReleaseMediaBuffer +{ +public: + AutoReleaseMediaBuffer(android::MediaBuffer* aBuffer, android::MediaCodecProxy* aCodec) + : mBuffer(aBuffer) + , mCodec(aCodec) + {} + + ~AutoReleaseMediaBuffer() + { + MOZ_ASSERT(mCodec.get()); + if (mBuffer) { + mCodec->ReleaseMediaBuffer(mBuffer); + } + } + + android::MediaBuffer* forget() + { + android::MediaBuffer* tmp = mBuffer; + mBuffer = nullptr; + return tmp; + } + +private: + android::MediaBuffer* mBuffer; + android::sp mCodec; }; // Samples are decoded using the GonkDecoder (MediaCodec) @@ -83,33 +189,9 @@ public: nsresult Shutdown() override; private: - - // Called on the task queue. Inserts the sample into the decoder, and - // extracts output if available, if aSample is null, it means there is - // no data from source, it will notify the decoder EOS and flush all the - // decoded frames. - void ProcessDecode(MediaRawData* aSample); - - // Called on the task queue. Extracts output if available, and delivers - // it to the reader. Called after ProcessDecode() and ProcessDrain(). - void ProcessOutput(); - - // Called on the task queue. Orders the Gonk to drain, and then extracts - // all available output. - void ProcessDrain(); - RefPtr mTaskQueue; - MediaDataDecoderCallback* mCallback; - nsAutoPtr mManager; - - // The last offset into the media resource that was passed into Input(). - // This is used to approximate the decoder's position in the media resource. - int64_t mLastStreamOffset; - // Set it ture when there is no input data - bool mSignaledEOS; - // Set if there is no more output data from decoder - bool mDrainComplete; + android::sp mManager; }; } // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp b/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp index 2cd468e9dd..1a1cc95cb0 100644 --- a/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp +++ b/dom/media/platforms/gonk/GonkVideoDecoderManager.cpp @@ -14,24 +14,25 @@ #include "nsThreadUtils.h" #include "Layers.h" #include "mozilla/Logging.h" -#include "stagefright/MediaBuffer.h" -#include "stagefright/MetaData.h" -#include "stagefright/MediaErrors.h" -#include -#include +#include +#include +#include #include -#include #include "GonkNativeWindow.h" #include "GonkNativeWindowClient.h" #include "mozilla/layers/GrallocTextureClient.h" +#include "mozilla/layers/ImageBridgeChild.h" #include "mozilla/layers/TextureClient.h" +#include "mozilla/layers/TextureClientRecycleAllocator.h" +#include -#define READ_OUTPUT_BUFFER_TIMEOUT_US 3000 +#define CODECCONFIG_TIMEOUT_US 10000LL +#define READ_OUTPUT_BUFFER_TIMEOUT_US 0LL #include #define GVDM_LOG(...) __android_log_print(ANDROID_LOG_DEBUG, "GonkVideoDecoderManager", __VA_ARGS__) -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) using namespace mozilla::layers; using namespace android; @@ -43,11 +44,10 @@ GonkVideoDecoderManager::GonkVideoDecoderManager( mozilla::layers::ImageContainer* aImageContainer, const VideoInfo& aConfig) : mImageContainer(aImageContainer) - , mReaderCallback(nullptr) , mColorConverterBufferSize(0) , mNativeWindow(nullptr) - , mPendingVideoBuffersLock("GonkVideoDecoderManager::mPendingVideoBuffersLock") - , mMonitor("GonkVideoDecoderManager") + , mPendingReleaseItemsLock("GonkVideoDecoderManager::mPendingReleaseItemsLock") + , mNeedsCopyBuffer(false) { MOZ_COUNT_CTOR(GonkVideoDecoderManager); mMimeType = aConfig.mMimeType; @@ -62,9 +62,6 @@ GonkVideoDecoderManager::GonkVideoDecoderManager( nsIntSize frameSize(mVideoWidth, mVideoHeight); mPicture = pictureRect; mInitialFrame = frameSize; - mHandler = new MessageHandler(this); - mVideoListener = new VideoResourceListener(this); - } GonkVideoDecoderManager::~GonkVideoDecoderManager() @@ -72,9 +69,18 @@ GonkVideoDecoderManager::~GonkVideoDecoderManager() MOZ_COUNT_DTOR(GonkVideoDecoderManager); } -RefPtr -GonkVideoDecoderManager::Init(MediaDataDecoderCallback* aCallback) +nsresult +GonkVideoDecoderManager::Shutdown() { + mVideoCodecRequest.DisconnectIfExists(); + return GonkDecoderManager::Shutdown(); +} + +RefPtr +GonkVideoDecoderManager::Init() +{ + mNeedsCopyBuffer = false; + nsIntSize displaySize(mDisplayWidth, mDisplayHeight); nsIntRect pictureRect(0, 0, mVideoWidth, mVideoHeight); // Validate the container-reported frame and pictureRect sizes. This ensures @@ -85,27 +91,20 @@ GonkVideoDecoderManager::Init(MediaDataDecoderCallback* aCallback) return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - mReaderCallback = aCallback; - mReaderTaskQueue = AbstractThread::GetCurrent()->AsTaskQueue(); - MOZ_ASSERT(!mReaderTaskQueue); + MOZ_ASSERT(mReaderTaskQueue); - if (mLooper.get() != nullptr) { + if (mDecodeLooper.get() != nullptr) { return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } - // Create ALooper - mLooper = new ALooper; - mManagerLooper = new ALooper; - mManagerLooper->setName("GonkVideoDecoderManager"); - // Register AMessage handler to ALooper. - mManagerLooper->registerHandler(mHandler); - // Start ALooper thread. - if (mLooper->start() != OK || mManagerLooper->start() != OK ) { + + if (!InitLoopers(MediaData::VIDEO_DATA)) { return InitPromise::CreateAndReject(DecoderFailureReason::INIT_ERROR, __func__); } + RefPtr p = mInitPromise.Ensure(__func__); - mDecoder = MediaCodecProxy::CreateByType(mLooper, mMimeType.get(), false, mVideoListener); - mDecoder->AsyncAskMediaCodec(); + android::sp self = this; + mDecoder = MediaCodecProxy::CreateByType(mDecodeLooper, mMimeType.get(), false); uint32_t capability = MediaCodecProxy::kEmptyCapability; if (mDecoder->getCapability(&capability) == OK && (capability & @@ -113,130 +112,54 @@ GonkVideoDecoderManager::Init(MediaDataDecoderCallback* aCallback) mNativeWindow = new GonkNativeWindow(); } + mVideoCodecRequest.Begin(mDecoder->AsyncAllocateVideoMediaCodec() + ->Then(mReaderTaskQueue, __func__, + [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->codecReserved(); + }, [self] (bool) -> void { + self->mVideoCodecRequest.Complete(); + self->codecCanceled(); + })); + return p; } -void -GonkVideoDecoderManager::QueueFrameTimeIn(int64_t aPTS, int64_t aDuration) -{ - MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); - - FrameTimeInfo timeInfo = {aPTS, aDuration}; - mFrameTimeInfo.AppendElement(timeInfo); -} - nsresult -GonkVideoDecoderManager::QueueFrameTimeOut(int64_t aPTS, int64_t& aDuration) -{ - MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); - - // Set default to 1 here. - // During seeking, frames could still in MediaCodec and the mFrameTimeInfo could - // be cleared before these frames are out from MediaCodec. This is ok because - // these frames are old frame before seeking. - aDuration = 1; - for (uint32_t i = 0; i < mFrameTimeInfo.Length(); i++) { - const FrameTimeInfo& entry = mFrameTimeInfo.ElementAt(i); - if (i == 0) { - if (entry.pts > aPTS) { - // Codec sent a frame with rollbacked PTS time. It could - // be codec's problem. - ReleaseVideoBuffer(); - return NS_ERROR_NOT_AVAILABLE; - } - } - - // Ideally, the first entry in mFrameTimeInfo should be the one we are looking - // for. However, MediaCodec could dropped frame and the first entry doesn't - // match current decoded frame's PTS. - if (entry.pts == aPTS) { - aDuration = entry.duration; - if (i > 0) { - LOG("Frame could be dropped by MediaCodec, %d dropped frames.", i); - } - mFrameTimeInfo.RemoveElementsAt(0, i+1); - break; - } - } - return NS_OK; -} - -nsresult -GonkVideoDecoderManager::Input(MediaRawData* aSample) -{ - MonitorAutoLock mon(mMonitor); - RefPtr sample; - - if (!aSample) { - // It means EOS with empty sample. - sample = new MediaRawData(); - } else { - sample = aSample; - } - - mQueueSample.AppendElement(sample); - - status_t rv; - while (mQueueSample.Length()) { - RefPtr data = mQueueSample.ElementAt(0); - { - MonitorAutoUnlock mon_unlock(mMonitor); - rv = mDecoder->Input(reinterpret_cast(data->mData), - data->mSize, - data->mTime, - 0); - } - if (rv == OK) { - mQueueSample.RemoveElementAt(0); - } else if (rv == -EAGAIN || rv == -ETIMEDOUT) { - // In most cases, EAGAIN or ETIMEOUT are safe because OMX can't fill - // buffer on time. - return NS_OK; - } else { - return NS_ERROR_UNEXPECTED; - } - } - - return NS_OK; -} - -bool -GonkVideoDecoderManager::HasQueuedSample() -{ - MonitorAutoLock mon(mMonitor); - return mQueueSample.Length(); -} - -nsresult -GonkVideoDecoderManager::CreateVideoData(int64_t aStreamOffset, VideoData **v) +GonkVideoDecoderManager::CreateVideoData(MediaBuffer* aBuffer, + int64_t aStreamOffset, + VideoData **v) { *v = nullptr; RefPtr data; int64_t timeUs; int32_t keyFrame; - if (mVideoBuffer == nullptr) { + if (aBuffer == nullptr) { GVDM_LOG("Video Buffer is not valid!"); return NS_ERROR_UNEXPECTED; } - if (!mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { + AutoReleaseMediaBuffer autoRelease(aBuffer, mDecoder.get()); + + if (!aBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) { GVDM_LOG("Decoder did not return frame time"); return NS_ERROR_UNEXPECTED; } - int64_t duration; - nsresult rv = QueueFrameTimeOut(timeUs, duration); - NS_ENSURE_SUCCESS(rv, rv); + if (mLastTime > timeUs) { + GVDM_LOG("Output decoded sample time is revert. time=%lld", timeUs); + return NS_ERROR_NOT_AVAILABLE; + } + mLastTime = timeUs; - if (mVideoBuffer->range_length() == 0) { + if (aBuffer->range_length() == 0) { // Some decoders may return spurious empty buffers that we just want to ignore // quoted from Android's AwesomePlayer.cpp - ReleaseVideoBuffer(); return NS_ERROR_NOT_AVAILABLE; } - if (!mVideoBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) { + if (!aBuffer->meta_data()->findInt32(kKeyIsSyncFrame, &keyFrame)) { keyFrame = 0; } @@ -253,101 +176,277 @@ GonkVideoDecoderManager::CreateVideoData(int64_t aStreamOffset, VideoData **v) picture.height = (mFrameInfo.mHeight * mPicture.height) / mInitialFrame.height; } - RefPtr textureClient; - - if ((mVideoBuffer->graphicBuffer().get())) { - textureClient = mNativeWindow->getTextureClientFromBuffer(mVideoBuffer->graphicBuffer().get()); - } - - if (textureClient) { - GrallocTextureClientOGL* grallocClient = static_cast(textureClient.get()); - grallocClient->SetMediaBuffer(mVideoBuffer); - textureClient->SetRecycleCallback(GonkVideoDecoderManager::RecycleCallback, this); - - data = VideoData::Create(mInfo.mVideo, - mImageContainer, - aStreamOffset, - timeUs, - duration, - textureClient, - keyFrame, - -1, - picture); + if (aBuffer->graphicBuffer().get()) { + data = CreateVideoDataFromGraphicBuffer(aBuffer, picture); + if (data && !mNeedsCopyBuffer) { + // RecycleCallback() will be responsible for release the buffer. + autoRelease.forget(); + } + mNeedsCopyBuffer = false; } else { - if (!mVideoBuffer->data()) { - GVDM_LOG("No data in Video Buffer!"); - return NS_ERROR_UNEXPECTED; - } - uint8_t *yuv420p_buffer = (uint8_t *)mVideoBuffer->data(); - int32_t stride = mFrameInfo.mStride; - int32_t slice_height = mFrameInfo.mSliceHeight; - - // Converts to OMX_COLOR_FormatYUV420Planar - if (mFrameInfo.mColorFormat != OMX_COLOR_FormatYUV420Planar) { - ARect crop; - crop.top = 0; - crop.bottom = mFrameInfo.mHeight; - crop.left = 0; - crop.right = mFrameInfo.mWidth; - yuv420p_buffer = GetColorConverterBuffer(mFrameInfo.mWidth, mFrameInfo.mHeight); - if (mColorConverter.convertDecoderOutputToI420(mVideoBuffer->data(), - mFrameInfo.mWidth, mFrameInfo.mHeight, crop, yuv420p_buffer) != OK) { - ReleaseVideoBuffer(); - GVDM_LOG("Color conversion failed!"); - return NS_ERROR_UNEXPECTED; - } - stride = mFrameInfo.mWidth; - slice_height = mFrameInfo.mHeight; - } - - size_t yuv420p_y_size = stride * slice_height; - size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); - uint8_t *yuv420p_y = yuv420p_buffer; - uint8_t *yuv420p_u = yuv420p_y + yuv420p_y_size; - uint8_t *yuv420p_v = yuv420p_u + yuv420p_u_size; - - // This is the approximate byte position in the stream. - int64_t pos = aStreamOffset; - - VideoData::YCbCrBuffer b; - b.mPlanes[0].mData = yuv420p_y; - b.mPlanes[0].mWidth = mFrameInfo.mWidth; - b.mPlanes[0].mHeight = mFrameInfo.mHeight; - b.mPlanes[0].mStride = stride; - b.mPlanes[0].mOffset = 0; - b.mPlanes[0].mSkip = 0; - - b.mPlanes[1].mData = yuv420p_u; - b.mPlanes[1].mWidth = (mFrameInfo.mWidth + 1) / 2; - b.mPlanes[1].mHeight = (mFrameInfo.mHeight + 1) / 2; - b.mPlanes[1].mStride = (stride + 1) / 2; - b.mPlanes[1].mOffset = 0; - b.mPlanes[1].mSkip = 0; - - b.mPlanes[2].mData = yuv420p_v; - b.mPlanes[2].mWidth =(mFrameInfo.mWidth + 1) / 2; - b.mPlanes[2].mHeight = (mFrameInfo.mHeight + 1) / 2; - b.mPlanes[2].mStride = (stride + 1) / 2; - b.mPlanes[2].mOffset = 0; - b.mPlanes[2].mSkip = 0; - - data = VideoData::Create( - mInfo.mVideo, - mImageContainer, - pos, - timeUs, - 1, // We don't know the duration. - b, - keyFrame, - -1, - picture); - ReleaseVideoBuffer(); + data = CreateVideoDataFromDataBuffer(aBuffer, picture); } + if (!data) { + return NS_ERROR_UNEXPECTED; + } + // Fill necessary info. + data->mOffset = aStreamOffset; + data->mTime = timeUs; + data->mKeyframe = keyFrame; + data.forget(v); return NS_OK; } +// Copy pixels from one planar YUV to another. +static void +CopyYUV(PlanarYCbCrData& aSource, PlanarYCbCrData& aDestination) +{ + // Fill Y plane. + uint8_t* srcY = aSource.mYChannel; + gfx::IntSize ySize = aSource.mYSize; + uint8_t* destY = aDestination.mYChannel; + // Y plane. + for (int i = 0; i < ySize.height; i++) { + memcpy(destY, srcY, ySize.width); + srcY += aSource.mYStride; + destY += aDestination.mYStride; + } + + // Fill UV plane. + // Line start + uint8_t* srcU = aSource.mCbChannel; + uint8_t* srcV = aSource.mCrChannel; + uint8_t* destU = aDestination.mCbChannel; + uint8_t* destV = aDestination.mCrChannel; + + gfx::IntSize uvSize = aSource.mCbCrSize; + for (int i = 0; i < uvSize.height; i++) { + uint8_t* su = srcU; + uint8_t* sv = srcV; + uint8_t* du = destU; + uint8_t* dv =destV; + for (int j = 0; j < uvSize.width; j++) { + *du++ = *su++; + *dv++ = *sv++; + // Move to next pixel. + su += aSource.mCbSkip; + sv += aSource.mCrSkip; + du += aDestination.mCbSkip; + dv += aDestination.mCrSkip; + } + // Move to next line. + srcU += aSource.mCbCrStride; + srcV += aSource.mCbCrStride; + destU += aDestination.mCbCrStride; + destV += aDestination.mCbCrStride; + } +} + +inline static int +Align(int aX, int aAlign) +{ + return (aX + aAlign - 1) & ~(aAlign - 1); +} + +static void +CopyGraphicBuffer(sp& aSource, sp& aDestination, gfx::IntRect& aPicture) +{ + void* srcPtr = nullptr; + aSource->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &srcPtr); + void* destPtr = nullptr; + aDestination->lock(GraphicBuffer::USAGE_SW_WRITE_OFTEN, &destPtr); + MOZ_ASSERT(srcPtr && destPtr); + + // Build PlanarYCbCrData for source buffer. + PlanarYCbCrData srcData; + switch (aSource->getPixelFormat()) { + case HAL_PIXEL_FORMAT_YV12: + // Android YV12 format is defined in system/core/include/system/graphics.h + srcData.mYChannel = static_cast(srcPtr); + srcData.mYSkip = 0; + srcData.mYSize.width = aSource->getWidth(); + srcData.mYSize.height = aSource->getHeight(); + srcData.mYStride = aSource->getStride(); + // 4:2:0. + srcData.mCbCrSize.width = srcData.mYSize.width / 2; + srcData.mCbCrSize.height = srcData.mYSize.height / 2; + srcData.mCrChannel = srcData.mYChannel + (srcData.mYStride * srcData.mYSize.height); + // Aligned to 16 bytes boundary. + srcData.mCbCrStride = Align(srcData.mYStride / 2, 16); + srcData.mCrSkip = 0; + srcData.mCbChannel = srcData.mCrChannel + (srcData.mCbCrStride * srcData.mCbCrSize.height); + srcData.mCbSkip = 0; + break; + case GrallocImage::HAL_PIXEL_FORMAT_YCbCr_420_SP_VENUS: + // Venus formats are doucmented in kernel/include/media/msm_media_info.h: + srcData.mYChannel = static_cast(srcPtr); + srcData.mYSkip = 0; + srcData.mYSize.width = aSource->getWidth(); + srcData.mYSize.height = aSource->getHeight(); + // - Y & UV Width aligned to 128 + srcData.mYStride = aSource->getStride(); + srcData.mCbCrSize.width = (srcData.mYSize.width + 1) / 2; + srcData.mCbCrSize.height = (srcData.mYSize.height + 1) / 2; + // - Y height aligned to 32 + srcData.mCbChannel = srcData.mYChannel + (srcData.mYStride * Align(srcData.mYSize.height, 32)); + // Interleaved VU plane. + srcData.mCbSkip = 1; + srcData.mCrChannel = srcData.mCbChannel + 1; + srcData.mCrSkip = 1; + srcData.mCbCrStride = srcData.mYStride; + break; + default: + NS_ERROR("Unsupported input gralloc image type. Should never be here."); + } + // Build PlanarYCbCrData for destination buffer. + PlanarYCbCrData destData; + destData.mYChannel = static_cast(destPtr); + destData.mYSkip = 0; + destData.mYSize.width = aDestination->getWidth(); + destData.mYSize.height = aDestination->getHeight(); + destData.mYStride = aDestination->getStride(); + // 4:2:0. + destData.mCbCrSize.width = destData.mYSize.width / 2; + destData.mCbCrSize.height = destData.mYSize.height / 2; + destData.mCrChannel = destData.mYChannel + (destData.mYStride * destData.mYSize.height); + // Aligned to 16 bytes boundary. + destData.mCbCrStride = Align(destData.mYStride / 2, 16); + destData.mCrSkip = 0; + destData.mCbChannel = destData.mCrChannel + (destData.mCbCrStride * destData.mCbCrSize.height); + destData.mCbSkip = 0; + + CopyYUV(srcData, destData); + + aSource->unlock(); + aDestination->unlock(); +} + +already_AddRefed +GonkVideoDecoderManager::CreateVideoDataFromGraphicBuffer(MediaBuffer* aSource, + gfx::IntRect& aPicture) +{ + sp srcBuffer(aSource->graphicBuffer()); + RefPtr textureClient; + + if (mNeedsCopyBuffer) { + // Copy buffer contents for bug 1199809. + if (!mCopyAllocator) { + mCopyAllocator = new TextureClientRecycleAllocator(ImageBridgeChild::GetSingleton()); + } + if (!mCopyAllocator) { + GVDM_LOG("Create buffer allocator failed!"); + return nullptr; + } + + gfx::IntSize size(Align(srcBuffer->getWidth(), 2) , Align(srcBuffer->getHeight(), 2)); + textureClient = + mCopyAllocator->CreateOrRecycle(gfx::SurfaceFormat::YUV, size, + BackendSelector::Content, + TextureFlags::DEFAULT, + ALLOC_DISALLOW_BUFFERTEXTURECLIENT); + if (!textureClient) { + GVDM_LOG("Copy buffer allocation failed!"); + return nullptr; + } + // Update size to match buffer's. + aPicture.width = size.width; + aPicture.height = size.height; + + sp destBuffer = + static_cast(textureClient.get())->GetGraphicBuffer(); + + CopyGraphicBuffer(srcBuffer, destBuffer, aPicture); + } else { + textureClient = mNativeWindow->getTextureClientFromBuffer(srcBuffer.get()); + textureClient->SetRecycleCallback(GonkVideoDecoderManager::RecycleCallback, this); + GrallocTextureClientOGL* grallocClient = static_cast(textureClient.get()); + grallocClient->SetMediaBuffer(aSource); + } + + RefPtr data = VideoData::Create(mInfo.mVideo, + mImageContainer, + 0, // Filled later by caller. + 0, // Filled later by caller. + 1, // No way to pass sample duration from muxer to + // OMX codec, so we hardcode the duration here. + textureClient, + false, // Filled later by caller. + -1, + aPicture); + return data.forget(); +} + +already_AddRefed +GonkVideoDecoderManager::CreateVideoDataFromDataBuffer(MediaBuffer* aSource, gfx::IntRect& aPicture) +{ + if (!aSource->data()) { + GVDM_LOG("No data in Video Buffer!"); + return nullptr; + } + uint8_t *yuv420p_buffer = (uint8_t *)aSource->data(); + int32_t stride = mFrameInfo.mStride; + int32_t slice_height = mFrameInfo.mSliceHeight; + + // Converts to OMX_COLOR_FormatYUV420Planar + if (mFrameInfo.mColorFormat != OMX_COLOR_FormatYUV420Planar) { + ARect crop; + crop.top = 0; + crop.bottom = mFrameInfo.mHeight; + crop.left = 0; + crop.right = mFrameInfo.mWidth; + yuv420p_buffer = GetColorConverterBuffer(mFrameInfo.mWidth, mFrameInfo.mHeight); + if (mColorConverter.convertDecoderOutputToI420(aSource->data(), + mFrameInfo.mWidth, mFrameInfo.mHeight, crop, yuv420p_buffer) != OK) { + GVDM_LOG("Color conversion failed!"); + return nullptr; + } + stride = mFrameInfo.mWidth; + slice_height = mFrameInfo.mHeight; + } + + size_t yuv420p_y_size = stride * slice_height; + size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); + uint8_t *yuv420p_y = yuv420p_buffer; + uint8_t *yuv420p_u = yuv420p_y + yuv420p_y_size; + uint8_t *yuv420p_v = yuv420p_u + yuv420p_u_size; + + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = yuv420p_y; + b.mPlanes[0].mWidth = mFrameInfo.mWidth; + b.mPlanes[0].mHeight = mFrameInfo.mHeight; + b.mPlanes[0].mStride = stride; + b.mPlanes[0].mOffset = 0; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = yuv420p_u; + b.mPlanes[1].mWidth = (mFrameInfo.mWidth + 1) / 2; + b.mPlanes[1].mHeight = (mFrameInfo.mHeight + 1) / 2; + b.mPlanes[1].mStride = (stride + 1) / 2; + b.mPlanes[1].mOffset = 0; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = yuv420p_v; + b.mPlanes[2].mWidth =(mFrameInfo.mWidth + 1) / 2; + b.mPlanes[2].mHeight = (mFrameInfo.mHeight + 1) / 2; + b.mPlanes[2].mStride = (stride + 1) / 2; + b.mPlanes[2].mOffset = 0; + b.mPlanes[2].mSkip = 0; + + RefPtr data = VideoData::Create(mInfo.mVideo, + mImageContainer, + 0, // Filled later by caller. + 0, // Filled later by caller. + 1, // We don't know the duration. + b, + 0, // Filled later by caller. + -1, + aPicture); + + return data.forget(); +} + bool GonkVideoDecoderManager::SetVideoFormat() { @@ -391,23 +490,6 @@ GonkVideoDecoderManager::SetVideoFormat() return false; } -nsresult -GonkVideoDecoderManager::Flush() -{ - { - MonitorAutoLock mon(mMonitor); - mQueueSample.Clear(); - } - - mLastDecodedTime = 0; - - if (mDecoder->flush() != OK) { - return NS_ERROR_FAILURE; - } - - return NS_OK; -} - // Blocks until decoded sample is produced by the deoder. nsresult GonkVideoDecoderManager::Output(int64_t aStreamOffset, @@ -419,13 +501,14 @@ GonkVideoDecoderManager::Output(int64_t aStreamOffset, GVDM_LOG("Decoder is not inited"); return NS_ERROR_UNEXPECTED; } - err = mDecoder->Output(&mVideoBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); + MediaBuffer* outputBuffer = nullptr; + err = mDecoder->Output(&outputBuffer, READ_OUTPUT_BUFFER_TIMEOUT_US); switch (err) { case OK: { RefPtr data; - nsresult rv = CreateVideoData(aStreamOffset, getter_AddRefs(data)); + nsresult rv = CreateVideoData(outputBuffer, aStreamOffset, getter_AddRefs(data)); if (rv == NS_ERROR_NOT_AVAILABLE) { // Decoder outputs a empty video buffer, try again return NS_ERROR_NOT_AVAILABLE; @@ -462,7 +545,7 @@ GonkVideoDecoderManager::Output(int64_t aStreamOffset, { GVDM_LOG("Got the EOS frame!"); RefPtr data; - nsresult rv = CreateVideoData(aStreamOffset, getter_AddRefs(data)); + nsresult rv = CreateVideoData(outputBuffer, aStreamOffset, getter_AddRefs(data)); if (rv == NS_ERROR_NOT_AVAILABLE) { // For EOS, no need to do any thing. return NS_ERROR_ABORT; @@ -489,29 +572,18 @@ GonkVideoDecoderManager::Output(int64_t aStreamOffset, return NS_OK; } -void GonkVideoDecoderManager::ReleaseVideoBuffer() { - if (mVideoBuffer) { - mDecoder->ReleaseMediaBuffer(mVideoBuffer); - mVideoBuffer = nullptr; - } -} - -void -GonkVideoDecoderManager::ClearQueueFrameTime() -{ - MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); - mFrameTimeInfo.Clear(); -} - void GonkVideoDecoderManager::codecReserved() { + if (mInitPromise.IsEmpty()) { + return; + } GVDM_LOG("codecReserved"); sp format = new AMessage; sp surface; // Fixed values - GVDM_LOG("Configure mime type: %s, widht:%d, height:%d", mMimeType.get(), mVideoWidth, mVideoHeight); + GVDM_LOG("Configure video mime type: %s, widht:%d, height:%d", mMimeType.get(), mVideoWidth, mVideoHeight); format->setString("mime", mMimeType.get()); format->setInt32("width", mVideoWidth); format->setInt32("height", mVideoHeight); @@ -520,15 +592,21 @@ GonkVideoDecoderManager::codecReserved() } mDecoder->configure(format, surface, nullptr, 0); mDecoder->Prepare(); - status_t rv = mDecoder->Input(mCodecSpecificData->Elements(), mCodecSpecificData->Length(), 0, - android::MediaCodec::BUFFER_FLAG_CODECCONFIG); + + if (mMimeType.EqualsLiteral("video/mp4v-es")) { + rv = mDecoder->Input(mCodecSpecificData->Elements(), + mCodecSpecificData->Length(), 0, + android::MediaCodec::BUFFER_FLAG_CODECCONFIG, + CODECCONFIG_TIMEOUT_US); + } + if (rv != OK) { GVDM_LOG("Failed to configure codec!!!!"); mInitPromise.Reject(DecoderFailureReason::INIT_ERROR, __func__); return; } - mInitPromise.ResolveIfExists(TrackType::kVideoTrack, __func__); + mInitPromise.Resolve(TrackType::kVideoTrack, __func__); } void @@ -538,7 +616,7 @@ GonkVideoDecoderManager::codecCanceled() mInitPromise.RejectIfExists(DecoderFailureReason::CANCELED, __func__); } -// Called on GonkVideoDecoderManager::mManagerLooper thread. +// Called on GonkDecoderManager::mTaskLooper thread. void GonkVideoDecoderManager::onMessageReceived(const sp &aMessage) { @@ -550,56 +628,10 @@ GonkVideoDecoderManager::onMessageReceived(const sp &aMessage) } default: - TRESPASS(); + { + GonkDecoderManager::onMessageReceived(aMessage); break; - } -} - -GonkVideoDecoderManager::MessageHandler::MessageHandler(GonkVideoDecoderManager *aManager) - : mManager(aManager) -{ -} - -GonkVideoDecoderManager::MessageHandler::~MessageHandler() -{ - mManager = nullptr; -} - -void -GonkVideoDecoderManager::MessageHandler::onMessageReceived(const android::sp &aMessage) -{ - if (mManager != nullptr) { - mManager->onMessageReceived(aMessage); - } -} - -GonkVideoDecoderManager::VideoResourceListener::VideoResourceListener(GonkVideoDecoderManager *aManager) - : mManager(aManager) -{ -} - -GonkVideoDecoderManager::VideoResourceListener::~VideoResourceListener() -{ - mManager = nullptr; -} - -void -GonkVideoDecoderManager::VideoResourceListener::codecReserved() -{ - if (mManager) { - nsCOMPtr r = - NS_NewNonOwningRunnableMethod(mManager, &GonkVideoDecoderManager::codecReserved); - mManager->mReaderTaskQueue->Dispatch(r.forget()); - } -} - -void -GonkVideoDecoderManager::VideoResourceListener::codecCanceled() -{ - if (mManager) { - nsCOMPtr r = - NS_NewNonOwningRunnableMethod(mManager, &GonkVideoDecoderManager::codecCanceled); - mManager->mReaderTaskQueue->Dispatch(r.forget()); + } } } @@ -627,44 +659,44 @@ GonkVideoDecoderManager::RecycleCallback(TextureClient* aClient, void* aClosure) GonkVideoDecoderManager* videoManager = static_cast(aClosure); GrallocTextureClientOGL* client = static_cast(aClient); aClient->ClearRecycleCallback(); - videoManager->PostReleaseVideoBuffer(client->GetMediaBuffer()); + FenceHandle handle = aClient->GetAndResetReleaseFenceHandle(); + videoManager->PostReleaseVideoBuffer(client->GetMediaBuffer(), handle); } void GonkVideoDecoderManager::PostReleaseVideoBuffer( - android::MediaBuffer *aBuffer) + android::MediaBuffer *aBuffer, + FenceHandle aReleaseFence) { { - MutexAutoLock autoLock(mPendingVideoBuffersLock); + MutexAutoLock autoLock(mPendingReleaseItemsLock); if (aBuffer) { - mPendingVideoBuffers.append(aBuffer); + mPendingReleaseItems.AppendElement(ReleaseItem(aBuffer, aReleaseFence)); } } sp notify = - new AMessage(kNotifyPostReleaseBuffer, mHandler->id()); + new AMessage(kNotifyPostReleaseBuffer, id()); notify->post(); } void GonkVideoDecoderManager::ReleaseAllPendingVideoBuffers() { - Vector releasingVideoBuffers; + nsTArray releasingItems; { - MutexAutoLock autoLock(mPendingVideoBuffersLock); - int size = mPendingVideoBuffers.length(); - for (int i = 0; i < size; i++) { - releasingVideoBuffers.append(mPendingVideoBuffers[i]); - } - mPendingVideoBuffers.clear(); + MutexAutoLock autoLock(mPendingReleaseItemsLock); + releasingItems.AppendElements(mPendingReleaseItems); + mPendingReleaseItems.Clear(); } - // Free all pending video buffers without holding mPendingVideoBuffersLock. - int size = releasingVideoBuffers.length(); - for (int i = 0; i < size; i++) { - android::MediaBuffer *buffer; - buffer = releasingVideoBuffers[i]; - mDecoder->ReleaseMediaBuffer(buffer); - buffer = nullptr; + + // Free all pending video buffers without holding mPendingReleaseItemsLock. + size_t size = releasingItems.Length(); + for (size_t i = 0; i < size; i++) { + nsRefPtr fdObj = releasingItems[i].mReleaseFence.GetAndResetFdObj(); + sp fence = new Fence(fdObj->GetAndResetFd()); + fence->waitForever("GonkVideoDecoderManager"); + mDecoder->ReleaseMediaBuffer(releasingItems[i].mBuffer); } - releasingVideoBuffers.clear(); + releasingItems.Clear(); } } // namespace mozilla diff --git a/dom/media/platforms/gonk/GonkVideoDecoderManager.h b/dom/media/platforms/gonk/GonkVideoDecoderManager.h index caa3044112..b1232666e3 100644 --- a/dom/media/platforms/gonk/GonkVideoDecoderManager.h +++ b/dom/media/platforms/gonk/GonkVideoDecoderManager.h @@ -7,20 +7,19 @@ #if !defined(GonkVideoDecoderManager_h_) #define GonkVideoDecoderManager_h_ -#include #include "nsRect.h" #include "GonkMediaDataDecoder.h" #include "mozilla/RefPtr.h" #include "I420ColorConverterHelper.h" #include "MediaCodecProxy.h" -#include #include "GonkNativeWindow.h" #include "GonkNativeWindowClient.h" +#include "mozilla/layers/FenceUtils.h" +#include using namespace android; namespace android { -struct ALooper; class MediaBuffer; struct MOZ_EXPORT AString; class GonkNativeWindow; @@ -30,6 +29,7 @@ namespace mozilla { namespace layers { class TextureClient; +class TextureClientRecycleAllocator; } // namespace mozilla::layers class GonkVideoDecoderManager : public GonkDecoderManager { @@ -38,20 +38,26 @@ typedef mozilla::layers::TextureClient TextureClient; public: GonkVideoDecoderManager(mozilla::layers::ImageContainer* aImageContainer, - const VideoInfo& aConfig); + const VideoInfo& aConfig); - virtual ~GonkVideoDecoderManager() override; + virtual ~GonkVideoDecoderManager(); - RefPtr Init(MediaDataDecoderCallback* aCallback) override; - - nsresult Input(MediaRawData* aSample) override; + RefPtr Init() override; nsresult Output(int64_t aStreamOffset, RefPtr& aOutput) override; - nsresult Flush() override; + nsresult Shutdown() override; - bool HasQueuedSample() override; + // Bug 1199809: workaround to avoid sending the graphic buffer by making a + // copy of output buffer after calling flush(). Bug 1203859 was created to + // reimplementing Gonk PDM on top of OpenMax IL directly. Its buffer + // management will work better with Gecko and solve problems like this. + nsresult Flush() override + { + mNeedsCopyBuffer = true; + return GonkDecoderManager::Flush(); + } static void RecycleCallback(TextureClient* aClient, void* aClosure); @@ -68,69 +74,26 @@ private: int32_t mCropRight = 0; int32_t mCropBottom = 0; }; - class MessageHandler : public android::AHandler - { - public: - MessageHandler(GonkVideoDecoderManager *aManager); - ~MessageHandler(); - void onMessageReceived(const android::sp &aMessage) override; - - private: - // Forbidden - MessageHandler() = delete; - MessageHandler(const MessageHandler &rhs) = delete; - const MessageHandler &operator=(const MessageHandler &rhs) = delete; - - GonkVideoDecoderManager *mManager; - }; - friend class MessageHandler; - - class VideoResourceListener : public android::MediaCodecProxy::CodecResourceListener - { - public: - VideoResourceListener(GonkVideoDecoderManager *aManager); - ~VideoResourceListener(); - - void codecReserved() override; - void codecCanceled() override; - - private: - // Forbidden - VideoResourceListener() = delete; - VideoResourceListener(const VideoResourceListener &rhs) = delete; - const VideoResourceListener &operator=(const VideoResourceListener &rhs) = delete; - - GonkVideoDecoderManager *mManager; - }; - friend class VideoResourceListener; - - // FrameTimeInfo keeps the presentation time stamp (pts) and its duration. - // On MediaDecoderStateMachine, it needs pts and duration to display decoded - // frame correctly. But OMX can carry one field of time info (kKeyTime) so - // we use FrameTimeInfo to keep pts and duration. - struct FrameTimeInfo { - int64_t pts; // presentation time stamp of this frame. - int64_t duration; // the playback duration. - }; + void onMessageReceived(const android::sp &aMessage) override; bool SetVideoFormat(); - nsresult CreateVideoData(int64_t aStreamOffset, VideoData** aOutData); - void ReleaseVideoBuffer(); + nsresult CreateVideoData(MediaBuffer* aBuffer, int64_t aStreamOffset, VideoData** aOutData); + already_AddRefed CreateVideoDataFromGraphicBuffer(android::MediaBuffer* aSource, + gfx::IntRect& aPicture); + already_AddRefed CreateVideoDataFromDataBuffer(android::MediaBuffer* aSource, + gfx::IntRect& aPicture); + uint8_t* GetColorConverterBuffer(int32_t aWidth, int32_t aHeight); // For codec resource management void codecReserved(); void codecCanceled(); - void onMessageReceived(const sp &aMessage); void ReleaseAllPendingVideoBuffers(); - void PostReleaseVideoBuffer(android::MediaBuffer *aBuffer); - - void QueueFrameTimeIn(int64_t aPTS, int64_t aDuration); - nsresult QueueFrameTimeOut(int64_t aPTS, int64_t& aDuration); - void ClearQueueFrameTime(); + void PostReleaseVideoBuffer(android::MediaBuffer *aBuffer, + layers::FenceHandle mReleaseFence); uint32_t mVideoWidth; uint32_t mVideoHeight; @@ -140,24 +103,12 @@ private: nsIntSize mInitialFrame; RefPtr mImageContainer; + RefPtr mCopyAllocator; - android::MediaBuffer* mVideoBuffer; - - RefPtr mCodecSpecificData; - MediaDataDecoderCallback* mReaderCallback; MediaInfo mInfo; - android::sp mVideoListener; - android::sp mHandler; - android::sp mLooper; - android::sp mManagerLooper; + MozPromiseRequestHolder mVideoCodecRequest; FrameInfo mFrameInfo; - // Array of FrameTimeInfo whose corresponding frames are sent to OMX. - // Ideally, it is a FIFO. Input() adds the entry to the end element and - // CreateVideoData() takes the first entry. However, there are exceptions - // due to MediaCodec error or seeking. - nsTArray mFrameTimeInfo; - // color converter android::I420ColorConverterHelper mColorConverter; nsAutoArrayPtr mColorConverterBuffer; @@ -168,25 +119,25 @@ private: kNotifyPostReleaseBuffer = 'nprb', }; - // Hold video's MediaBuffers that are released. - // The holded MediaBuffers are released soon after flush. - Vector mPendingVideoBuffers; - // The lock protects mPendingVideoBuffers. - Mutex mPendingVideoBuffersLock; + struct ReleaseItem { + ReleaseItem(android::MediaBuffer* aBuffer, layers::FenceHandle& aFence) + : mBuffer(aBuffer) + , mReleaseFence(aFence) {} + android::MediaBuffer* mBuffer; + layers::FenceHandle mReleaseFence; + }; + nsTArray mPendingReleaseItems; - nsAutoCString mMimeType; + // The lock protects mPendingReleaseItems. + Mutex mPendingReleaseItemsLock; - // This monitor protects mQueueSample. - Monitor mMonitor; - - // This TaskQueue should be the same one in mReaderCallback->OnReaderTaskQueue(). + // This TaskQueue should be the same one in mDecodeCallback->OnReaderTaskQueue(). // It is for codec resource mangement, decoding task should not dispatch to it. RefPtr mReaderTaskQueue; - // An queue with the MP4 samples which are waiting to be sent into OMX. - // If an element is an empty MP4Sample, that menas EOS. There should not - // any sample be queued after EOS. - nsTArray> mQueueSample; + // Bug 1199809: do we need to make a copy of output buffer? Used only when + // the decoder outputs graphic buffers. + bool mNeedsCopyBuffer; }; } // namespace mozilla diff --git a/dom/media/platforms/wmf/DXVA2Manager.cpp b/dom/media/platforms/wmf/DXVA2Manager.cpp index 3764d3eece..d83c9005dc 100644 --- a/dom/media/platforms/wmf/DXVA2Manager.cpp +++ b/dom/media/platforms/wmf/DXVA2Manager.cpp @@ -72,6 +72,7 @@ private: RefPtr mTextureClientAllocator; RefPtr mDecoderService; UINT32 mResetToken; + bool mFirstFrame; }; void GetDXVA2ExtendedFormatFromMFMediaType(IMFMediaType *pType, @@ -179,6 +180,7 @@ D3D9DXVA2Manager::SupportsConfig(IMFMediaType* aType) D3D9DXVA2Manager::D3D9DXVA2Manager() : mResetToken(0) + , mFirstFrame(true) { MOZ_COUNT_CTOR(D3D9DXVA2Manager); MOZ_ASSERT(NS_IsMainThread()); @@ -345,16 +347,13 @@ D3D9DXVA2Manager::CopyToImage(IMFSample* aSample, getter_AddRefs(surface)); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); - RefPtr image = aImageContainer->CreateImage(ImageFormat::D3D9_RGB32_TEXTURE); - NS_ENSURE_TRUE(image, E_FAIL); - NS_ASSERTION(image->GetFormat() == ImageFormat::D3D9_RGB32_TEXTURE, - "Wrong format?"); + RefPtr image = new D3D9SurfaceImage(mFirstFrame); + hr = image->AllocateAndCopy(mTextureClientAllocator, surface, aRegion); + NS_ENSURE_TRUE(SUCCEEDED(hr), hr); - D3D9SurfaceImage* videoImage = static_cast(image.get()); - hr = videoImage->SetData(D3D9SurfaceImage::Data(surface, aRegion, mTextureClientAllocator)); + mFirstFrame = false; image.forget(aOutImage); - return S_OK; } @@ -523,23 +522,16 @@ D3D11DXVA2Manager::CopyToImage(IMFSample* aVideoSample, // to create a copy of that frame as a sharable resource, save its share // handle, and put that handle into the rendering pipeline. - ImageFormat format = ImageFormat::D3D11_SHARE_HANDLE_TEXTURE; - RefPtr image(aContainer->CreateImage(format)); - NS_ENSURE_TRUE(image, E_FAIL); - NS_ASSERTION(image->GetFormat() == ImageFormat::D3D11_SHARE_HANDLE_TEXTURE, - "Wrong format?"); + RefPtr image = + new D3D11ShareHandleImage(gfx::IntSize(mWidth, mHeight), aRegion); + bool ok = image->AllocateTexture(mTextureClientAllocator); + NS_ENSURE_TRUE(ok, E_FAIL); - D3D11ShareHandleImage* videoImage = static_cast(image.get()); - HRESULT hr = videoImage->SetData(D3D11ShareHandleImage::Data(mTextureClientAllocator, - gfx::IntSize(mWidth, mHeight), - aRegion)); - NS_ENSURE_TRUE(SUCCEEDED(hr), hr); - - hr = mTransform->Input(aVideoSample); + HRESULT hr = mTransform->Input(aVideoSample); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); RefPtr sample; - RefPtr texture = videoImage->GetTexture(); + RefPtr texture = image->GetTexture(); hr = CreateOutputSample(sample, texture); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); diff --git a/dom/media/platforms/wmf/MFTDecoder.cpp b/dom/media/platforms/wmf/MFTDecoder.cpp index 83ff91ba00..5558d6d7b2 100644 --- a/dom/media/platforms/wmf/MFTDecoder.cpp +++ b/dom/media/platforms/wmf/MFTDecoder.cpp @@ -9,7 +9,7 @@ #include "WMFUtils.h" #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp index 1b8d361f1f..a2b5b518ed 100644 --- a/dom/media/platforms/wmf/WMFAudioMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFAudioMFTManager.cpp @@ -13,7 +13,7 @@ #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/platforms/wmf/WMFDecoderModule.cpp b/dom/media/platforms/wmf/WMFDecoderModule.cpp index 6fb1ab7bb6..875e206162 100644 --- a/dom/media/platforms/wmf/WMFDecoderModule.cpp +++ b/dom/media/platforms/wmf/WMFDecoderModule.cpp @@ -164,7 +164,7 @@ CanCreateWMFDecoder() } bool -WMFDecoderModule::SupportsMimeType(const nsACString& aMimeType) +WMFDecoderModule::SupportsMimeType(const nsACString& aMimeType) const { if ((aMimeType.EqualsLiteral("audio/mp4a-latm") || aMimeType.EqualsLiteral("audio/mp4")) && diff --git a/dom/media/platforms/wmf/WMFDecoderModule.h b/dom/media/platforms/wmf/WMFDecoderModule.h index 6ebb83f650..a959e26091 100644 --- a/dom/media/platforms/wmf/WMFDecoderModule.h +++ b/dom/media/platforms/wmf/WMFDecoderModule.h @@ -31,7 +31,7 @@ public: FlushableTaskQueue* aAudioTaskQueue, MediaDataDecoderCallback* aCallback) override; - bool SupportsMimeType(const nsACString& aMimeType) override; + bool SupportsMimeType(const nsACString& aMimeType) const override; ConversionRequired DecoderNeedsConversion(const TrackInfo& aConfig) const override; diff --git a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp index b2b0501237..cdbbf31d45 100644 --- a/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp +++ b/dom/media/platforms/wmf/WMFMediaDataDecoder.cpp @@ -11,7 +11,7 @@ #include "mozilla/Logging.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) namespace mozilla { diff --git a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp index a9d4de38b0..e3642954cc 100644 --- a/dom/media/platforms/wmf/WMFVideoMFTManager.cpp +++ b/dom/media/platforms/wmf/WMFVideoMFTManager.cpp @@ -23,7 +23,7 @@ #include "mozilla/Preferences.h" #include "nsPrintfCString.h" -extern PRLogModuleInfo* GetPDMLog(); +extern mozilla::LogModule* GetPDMLog(); #define LOG(...) MOZ_LOG(GetPDMLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) using mozilla::layers::Image; diff --git a/dom/media/platforms/wrappers/FuzzingWrapper.cpp b/dom/media/platforms/wrappers/FuzzingWrapper.cpp index 2e58facf3e..6b87a8fb5a 100644 --- a/dom/media/platforms/wrappers/FuzzingWrapper.cpp +++ b/dom/media/platforms/wrappers/FuzzingWrapper.cpp @@ -6,11 +6,8 @@ #include "FuzzingWrapper.h" -PRLogModuleInfo* GetFuzzingWrapperLog() { - static PRLogModuleInfo* log = nullptr; - if (!log) { - log = PR_NewLogModule("MediaFuzzingWrapper"); - } +mozilla::LogModule* GetFuzzingWrapperLog() { + static mozilla::LazyLogModule log("MediaFuzzingWrapper"); return log; } #define DFW_LOGD(arg, ...) MOZ_LOG(GetFuzzingWrapperLog(), mozilla::LogLevel::Debug, ("DecoderFuzzingWrapper(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) diff --git a/dom/media/systemservices/CamerasChild.cpp b/dom/media/systemservices/CamerasChild.cpp index 44ad63cdad..d4adf6f024 100644 --- a/dom/media/systemservices/CamerasChild.cpp +++ b/dom/media/systemservices/CamerasChild.cpp @@ -22,7 +22,7 @@ #undef LOG #undef LOG_ENABLED -PRLogModuleInfo *gCamerasChildLog; +mozilla::LazyLogModule gCamerasChildLog("CamerasChild"); #define LOG(args) MOZ_LOG(gCamerasChildLog, mozilla::LogLevel::Debug, args) #define LOG_ENABLED() MOZ_LOG_TEST(gCamerasChildLog, mozilla::LogLevel::Debug) @@ -56,9 +56,6 @@ public: : mCamerasMutex("CamerasSingleton::mCamerasMutex"), mCameras(nullptr), mCamerasChildThread(nullptr) { - if (!gCamerasChildLog) { - gCamerasChildLog = PR_NewLogModule("CamerasChild"); - } LOG(("CamerasSingleton: %p", this)); } @@ -144,10 +141,6 @@ Cameras() { OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); if (!CamerasSingleton::Child()) { MOZ_ASSERT(!NS_IsMainThread(), "Should not be on the main Thread"); - if (!gCamerasChildLog) { - gCamerasChildLog = PR_NewLogModule("CamerasChild"); - } - MOZ_ASSERT(!CamerasSingleton::Thread()); LOG(("No sCameras, setting up IPC Thread")); nsresult rv = NS_NewNamedThread("Cameras IPC", @@ -706,10 +699,6 @@ CamerasChild::CamerasChild() mRequestMutex("mozilla::cameras::CamerasChild::mRequestMutex"), mReplyMonitor("mozilla::cameras::CamerasChild::mReplyMonitor") { - if (!gCamerasChildLog) { - gCamerasChildLog = PR_NewLogModule("CamerasChild"); - } - LOG(("CamerasChild: %p", this)); MOZ_COUNT_CTOR(CamerasChild); diff --git a/dom/media/systemservices/CamerasParent.cpp b/dom/media/systemservices/CamerasParent.cpp index 091ad659de..6459ccab98 100644 --- a/dom/media/systemservices/CamerasParent.cpp +++ b/dom/media/systemservices/CamerasParent.cpp @@ -18,7 +18,7 @@ #undef LOG #undef LOG_ENABLED -PRLogModuleInfo *gCamerasParentLog; +mozilla::LazyLogModule gCamerasParentLog("CamerasParent"); #define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args) #define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) @@ -851,9 +851,6 @@ CamerasParent::CamerasParent() mDestroyed(false), mWebRTCAlive(true) { - if (!gCamerasParentLog) { - gCamerasParentLog = PR_NewLogModule("CamerasParent"); - } LOG(("CamerasParent: %p", this)); mPBackgroundThread = NS_GetCurrentThread(); diff --git a/dom/media/systemservices/LoadManager.cpp b/dom/media/systemservices/LoadManager.cpp index a15052e343..55506bab8d 100644 --- a/dom/media/systemservices/LoadManager.cpp +++ b/dom/media/systemservices/LoadManager.cpp @@ -15,9 +15,11 @@ #include "nsThreadUtils.h" #include "nsReadableUtils.h" #include "nsIObserverService.h" +#include "mozilla/Telemetry.h" +#include "mozilla/ArrayUtils.h" // NSPR_LOG_MODULES=LoadManager:5 -PRLogModuleInfo *gLoadManagerLog = nullptr; +mozilla::LazyLogModule gLoadManagerLog("LoadManager"); #undef LOG #undef LOG_ENABLED #define LOG(args) MOZ_LOG(gLoadManagerLog, mozilla::LogLevel::Debug, args) @@ -44,8 +46,6 @@ LoadManagerSingleton::LoadManagerSingleton(int aLoadMeasurementInterval, mHighLoadThreshold(aHighLoadThreshold), mLowLoadThreshold(aLowLoadThreshold) { - if (!gLoadManagerLog) - gLoadManagerLog = PR_NewLogModule("LoadManager"); LOG(("LoadManager - Initializing (%dms x %d, %f, %f)", mLoadMeasurementInterval, mAveragingMeasurements, mHighLoadThreshold, mLowLoadThreshold)); @@ -53,6 +53,11 @@ LoadManagerSingleton::LoadManagerSingleton(int aLoadMeasurementInterval, mLoadMonitor = new LoadMonitor(mLoadMeasurementInterval); mLoadMonitor->Init(mLoadMonitor); mLoadMonitor->SetLoadChangeCallback(this); + + mLastStateChange = TimeStamp::Now(); + for (auto &in_state : mTimeInState) { + in_state = 0; + } } LoadManagerSingleton::~LoadManagerSingleton() @@ -100,21 +105,22 @@ LoadManagerSingleton::LoadChanged(float aSystemLoad, float aProcesLoad) if (mLoadSumMeasurements >= mAveragingMeasurements) { double averagedLoad = mLoadSum / (float)mLoadSumMeasurements; - webrtc::CPULoadState oldState = mCurrentState; + webrtc::CPULoadState newState = mCurrentState; if (mOveruseActive || averagedLoad > mHighLoadThreshold) { LOG(("LoadManager - LoadStressed")); - mCurrentState = webrtc::kLoadStressed; + newState = webrtc::kLoadStressed; } else if (averagedLoad < mLowLoadThreshold) { LOG(("LoadManager - LoadRelaxed")); - mCurrentState = webrtc::kLoadRelaxed; + newState = webrtc::kLoadRelaxed; } else { LOG(("LoadManager - LoadNormal")); - mCurrentState = webrtc::kLoadNormal; + newState = webrtc::kLoadNormal; } - if (oldState != mCurrentState) - LoadHasChanged(); + if (newState != mCurrentState) { + LoadHasChanged(newState); + } mLoadSum = 0; mLoadSumMeasurements = 0; @@ -128,8 +134,7 @@ LoadManagerSingleton::OveruseDetected() MutexAutoLock lock(mLock); mOveruseActive = true; if (mCurrentState != webrtc::kLoadStressed) { - mCurrentState = webrtc::kLoadStressed; - LoadHasChanged(); + LoadHasChanged(webrtc::kLoadStressed); } } @@ -142,10 +147,18 @@ LoadManagerSingleton::NormalUsage() } void -LoadManagerSingleton::LoadHasChanged() +LoadManagerSingleton::LoadHasChanged(webrtc::CPULoadState aNewState) { mLock.AssertCurrentThreadOwns(); - LOG(("LoadManager - Signaling LoadHasChanged to %d listeners", mObservers.Length())); + LOG(("LoadManager - Signaling LoadHasChanged from %d to %d to %d listeners", + mCurrentState, aNewState, mObservers.Length())); + + // Record how long we spent in this state for later Telemetry or display + TimeStamp now = TimeStamp::Now(); + mTimeInState[mCurrentState] += (now - mLastStateChange).ToMilliseconds(); + mLastStateChange = now; + + mCurrentState = aNewState; for (size_t i = 0; i < mObservers.Length(); i++) { mObservers.ElementAt(i)->onLoadStateChanged(mCurrentState); } @@ -162,6 +175,7 @@ LoadManagerSingleton::AddObserver(webrtc::CPULoadStateObserver * aObserver) mLoadMonitor = new LoadMonitor(mLoadMeasurementInterval); mLoadMonitor->Init(mLoadMonitor); mLoadMonitor->SetLoadChangeCallback(this); + mLastStateChange = TimeStamp::Now(); } } } @@ -176,6 +190,35 @@ LoadManagerSingleton::RemoveObserver(webrtc::CPULoadStateObserver * aObserver) } if (mObservers.Length() == 0) { if (mLoadMonitor) { + // Record how long we spent in the final state for later Telemetry or display + TimeStamp now = TimeStamp::Now(); + mTimeInState[mCurrentState] += (now - mLastStateChange).ToMilliseconds(); + + float total = 0; + for (size_t i = 0; i < MOZ_ARRAY_LENGTH(mTimeInState); i++) { + total += mTimeInState[i]; + } + // Don't include short calls; we don't have reasonable load data, and + // such short calls rarely reach a stable state. Keep relatively + // short calls separate from longer ones + bool log = total > 5*PR_MSEC_PER_SEC; + bool small = log && total < 30*PR_MSEC_PER_SEC; + if (log) { + // Note: We don't care about rounding here; thus total may be < 100 + Telemetry::Accumulate(small ? Telemetry::WEBRTC_LOAD_STATE_RELAXED_SHORT : + Telemetry::WEBRTC_LOAD_STATE_RELAXED, + (uint32_t) (mTimeInState[webrtc::CPULoadState::kLoadRelaxed]/total * 100)); + Telemetry::Accumulate(small ? Telemetry::WEBRTC_LOAD_STATE_NORMAL_SHORT : + Telemetry::WEBRTC_LOAD_STATE_NORMAL, + (uint32_t) (mTimeInState[webrtc::CPULoadState::kLoadNormal]/total * 100)); + Telemetry::Accumulate(small ? Telemetry::WEBRTC_LOAD_STATE_STRESSED_SHORT : + Telemetry::WEBRTC_LOAD_STATE_STRESSED, + (uint32_t) (mTimeInState[webrtc::CPULoadState::kLoadStressed]/total * 100)); + } + for (auto &in_state : mTimeInState) { + in_state = 0; + } + // Dance to avoid deadlock on mLock! RefPtr loadMonitor = mLoadMonitor.forget(); MutexAutoUnlock unlock(mLock); diff --git a/dom/media/systemservices/LoadManager.h b/dom/media/systemservices/LoadManager.h index 8ebbf5786e..2ede35dbb2 100644 --- a/dom/media/systemservices/LoadManager.h +++ b/dom/media/systemservices/LoadManager.h @@ -16,7 +16,7 @@ #include "webrtc/common_types.h" #include "webrtc/video_engine/include/vie_base.h" -extern PRLogModuleInfo *gLoadManagerLog; +extern mozilla::LazyLogModule gLoadManagerLog; namespace mozilla { @@ -50,7 +50,7 @@ private: float aLowLoadThreshold); ~LoadManagerSingleton(); - void LoadHasChanged(); + void LoadHasChanged(webrtc::CPULoadState aNewState); RefPtr mLoadMonitor; @@ -59,6 +59,9 @@ private: Mutex mLock; nsTArray mObservers; webrtc::CPULoadState mCurrentState; + TimeStamp mLastStateChange; + float mTimeInState[static_cast(webrtc::kLoadLast)]; + // Set when overuse was signaled to us, and hasn't been un-signaled yet. bool mOveruseActive; float mLoadSum; diff --git a/dom/media/systemservices/MediaChild.cpp b/dom/media/systemservices/MediaChild.cpp index c99b99413a..2af732c335 100644 --- a/dom/media/systemservices/MediaChild.cpp +++ b/dom/media/systemservices/MediaChild.cpp @@ -13,7 +13,7 @@ #include "nsQueryObject.h" #undef LOG -PRLogModuleInfo *gMediaChildLog; +mozilla::LazyLogModule gMediaChildLog("MediaChild"); #define LOG(args) MOZ_LOG(gMediaChildLog, mozilla::LogLevel::Debug, args) namespace mozilla { @@ -68,9 +68,6 @@ Child* Child::Get() Child::Child() : mActorDestroyed(false) { - if (!gMediaChildLog) { - gMediaChildLog = PR_NewLogModule("MediaChild"); - } LOG(("media::Child: %p", this)); MOZ_COUNT_CTOR(Child); } diff --git a/dom/media/systemservices/MediaParent.cpp b/dom/media/systemservices/MediaParent.cpp index 84a32d41d5..4704c0ef5c 100644 --- a/dom/media/systemservices/MediaParent.cpp +++ b/dom/media/systemservices/MediaParent.cpp @@ -24,7 +24,7 @@ #include "mozilla/Logging.h" #undef LOG -PRLogModuleInfo *gMediaParentLog; +mozilla::LazyLogModule gMediaParentLog("MediaParent"); #define LOG(args) MOZ_LOG(gMediaParentLog, mozilla::LogLevel::Debug, args) // A file in the profile dir is used to persist mOriginKeys used to anonymize @@ -511,8 +511,6 @@ Parent::Parent(bool aSameProcess) , mDestroyed(false) , mSameProcess(aSameProcess) { - if (!gMediaParentLog) - gMediaParentLog = PR_NewLogModule("MediaParent"); LOG(("media::Parent: %p", this)); } diff --git a/dom/media/systemservices/OpenSLESProvider.cpp b/dom/media/systemservices/OpenSLESProvider.cpp index c15d71a22f..8b417884e8 100644 --- a/dom/media/systemservices/OpenSLESProvider.cpp +++ b/dom/media/systemservices/OpenSLESProvider.cpp @@ -14,7 +14,7 @@ // NSPR_LOG_MODULES=OpenSLESProvider:5 #undef LOG #undef LOG_ENABLED -PRLogModuleInfo *gOpenSLESProviderLog; +mozilla::LazyLogModule gOpenSLESProviderLog("OpenSLESProvider"); #define LOG(args) MOZ_LOG(gOpenSLESProviderLog, mozilla::LogLevel::Debug, args) #define LOG_ENABLED() MOZ_LOG_TEST(gOpenSLESProviderLog, mozilla::LogLevel::Debug) @@ -27,8 +27,6 @@ OpenSLESProvider::OpenSLESProvider() mIsRealized(false), mOpenSLESLib(nullptr) { - if (!gOpenSLESProviderLog) - gOpenSLESProviderLog = PR_NewLogModule("OpenSLESProvider"); LOG(("OpenSLESProvider being initialized")); } diff --git a/dom/media/systemservices/OpenSLESProvider.h b/dom/media/systemservices/OpenSLESProvider.h index 2316dd8621..6253e9519e 100644 --- a/dom/media/systemservices/OpenSLESProvider.h +++ b/dom/media/systemservices/OpenSLESProvider.h @@ -28,7 +28,7 @@ SLresult mozilla_realize_sles_engine(SLObjectItf aObjectm); #ifdef __cplusplus #include "mozilla/Mutex.h" -extern PRLogModuleInfo *gOpenSLESProviderLog; +extern mozilla::LazyLogModule gOpenSLESProviderLog; namespace mozilla { diff --git a/dom/media/systemservices/ShmemPool.cpp b/dom/media/systemservices/ShmemPool.cpp index 648842ddb4..c83c41ebf0 100644 --- a/dom/media/systemservices/ShmemPool.cpp +++ b/dom/media/systemservices/ShmemPool.cpp @@ -11,7 +11,7 @@ #undef LOG #undef LOG_ENABLED -extern PRLogModuleInfo *gCamerasParentLog; +extern mozilla::LazyLogModule gCamerasParentLog; #define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args) #define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) diff --git a/dom/media/wave/WaveReader.cpp b/dom/media/wave/WaveReader.cpp index 6f32185f01..38f4083e83 100644 --- a/dom/media/wave/WaveReader.cpp +++ b/dom/media/wave/WaveReader.cpp @@ -23,7 +23,7 @@ namespace mozilla { // Un-comment to enable logging of seek bisections. //#define SEEK_LOGGING -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; #define LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) #ifdef SEEK_LOGGING #define SEEK_LOG(type, msg) MOZ_LOG(gMediaDecoderLog, type, msg) diff --git a/dom/media/webaudio/BufferDecoder.cpp b/dom/media/webaudio/BufferDecoder.cpp index 1bc79dfa32..317c8f11ba 100644 --- a/dom/media/webaudio/BufferDecoder.cpp +++ b/dom/media/webaudio/BufferDecoder.cpp @@ -11,7 +11,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; NS_IMPL_ISUPPORTS0(BufferDecoder) @@ -20,9 +20,6 @@ BufferDecoder::BufferDecoder(MediaResource* aResource) { MOZ_ASSERT(NS_IsMainThread()); MOZ_COUNT_CTOR(BufferDecoder); - if (!gMediaDecoderLog) { - gMediaDecoderLog = PR_NewLogModule("MediaDecoder"); - } } BufferDecoder::~BufferDecoder() diff --git a/dom/media/webm/AudioDecoder.cpp b/dom/media/webm/AudioDecoder.cpp index 1e3cb3a839..d859e85c94 100644 --- a/dom/media/webm/AudioDecoder.cpp +++ b/dom/media/webm/AudioDecoder.cpp @@ -28,7 +28,7 @@ namespace mozilla { -extern PRLogModuleInfo* gMediaDecoderLog; +extern LazyLogModule gMediaDecoderLog; ogg_packet InitOggPacket(const unsigned char* aData, size_t aLength, bool aBOS, bool aEOS, diff --git a/dom/media/webm/WebMDemuxer.cpp b/dom/media/webm/WebMDemuxer.cpp index d8687e094c..80e75eb1d4 100644 --- a/dom/media/webm/WebMDemuxer.cpp +++ b/dom/media/webm/WebMDemuxer.cpp @@ -32,8 +32,8 @@ namespace mozilla { using namespace gfx; -PRLogModuleInfo* gWebMDemuxerLog = nullptr; -extern PRLogModuleInfo* gNesteggLog; +LazyLogModule gWebMDemuxerLog("WebMDemuxer"); +extern LazyLogModule gNesteggLog; // How far ahead will we look when searching future keyframe. In microseconds. // This value is based on what appears to be a reasonable value as most webm @@ -145,12 +145,6 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) , mLastWebMBlockOffset(-1) , mIsMediaSource(aIsMediaSource) { - if (!gNesteggLog) { - gNesteggLog = PR_NewLogModule("Nestegg"); - } - if (!gWebMDemuxerLog) { - gWebMDemuxerLog = PR_NewLogModule("WebMDemuxer"); - } } WebMDemuxer::~WebMDemuxer() diff --git a/dom/media/webm/WebMReader.cpp b/dom/media/webm/WebMReader.cpp index 9095b5d1c3..4083d49483 100644 --- a/dom/media/webm/WebMReader.cpp +++ b/dom/media/webm/WebMReader.cpp @@ -39,8 +39,8 @@ using namespace gfx; using namespace layers; using namespace media; -extern PRLogModuleInfo* gMediaDecoderLog; -PRLogModuleInfo* gNesteggLog; +extern LazyLogModule gMediaDecoderLog; +LazyLogModule gNesteggLog("Nestegg"); // Functions for reading and seeking using MediaResource required for // nestegg_io. The 'user data' passed to these functions is the @@ -137,9 +137,6 @@ WebMReader::WebMReader(AbstractMediaDecoder* aDecoder) , mResource(aDecoder->GetResource()) { MOZ_COUNT_CTOR(WebMReader); - if (!gNesteggLog) { - gNesteggLog = PR_NewLogModule("Nestegg"); - } } WebMReader::~WebMReader() diff --git a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp index 789e67a9fd..40c38f5699 100644 --- a/dom/media/webrtc/MediaEngineCameraVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineCameraVideoSource.cpp @@ -11,7 +11,7 @@ namespace mozilla { using namespace mozilla::gfx; using namespace mozilla::dom; -extern PRLogModuleInfo* GetMediaManagerLog(); +extern LogModule* GetMediaManagerLog(); #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) #define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) diff --git a/dom/media/webrtc/MediaEngineDefault.cpp b/dom/media/webrtc/MediaEngineDefault.cpp index a850350d7d..c1403ecdfb 100644 --- a/dom/media/webrtc/MediaEngineDefault.cpp +++ b/dom/media/webrtc/MediaEngineDefault.cpp @@ -238,9 +238,7 @@ MediaEngineDefaultVideoSource::Notify(nsITimer* aTimer) } // Allocate a single solid color image - RefPtr image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); - RefPtr ycbcr_image = - static_cast(image.get()); + RefPtr ycbcr_image = mImageContainer->CreatePlanarYCbCrImage(); layers::PlanarYCbCrData data; AllocateSolidColorFrame(data, mOpts.mWidth, mOpts.mHeight, 0x80, mCb, mCr); diff --git a/dom/media/webrtc/MediaEngineGonkVideoSource.cpp b/dom/media/webrtc/MediaEngineGonkVideoSource.cpp index e9971485c0..6f3eabda2a 100644 --- a/dom/media/webrtc/MediaEngineGonkVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineGonkVideoSource.cpp @@ -25,7 +25,7 @@ using namespace mozilla::gfx; using namespace android; #undef LOG -extern PRLogModuleInfo* GetMediaManagerLog(); +extern LogModule* GetMediaManagerLog(); #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) #define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) @@ -734,9 +734,9 @@ MediaEngineGonkVideoSource::RotateImage(layers::Image* aImage, uint32_t aWidth, graphicBuffer->lock(GraphicBuffer::USAGE_SW_READ_MASK, &pMem); uint8_t* srcPtr = static_cast(pMem); + // Create a video frame and append it to the track. - ImageFormat format = ImageFormat::GONK_CAMERA_IMAGE; - RefPtr image = mImageContainer->CreateImage(format); + RefPtr image = new GonkCameraImage(); uint32_t dstWidth; uint32_t dstHeight; @@ -751,7 +751,6 @@ MediaEngineGonkVideoSource::RotateImage(layers::Image* aImage, uint32_t aWidth, uint32_t half_width = dstWidth / 2; - layers::GrallocImage* videoImage = static_cast(image.get()); MOZ_ASSERT(mTextureClientAllocator); RefPtr textureClient = mTextureClientAllocator->CreateOrRecycle(gfx::SurfaceFormat::YUV, @@ -760,8 +759,7 @@ MediaEngineGonkVideoSource::RotateImage(layers::Image* aImage, uint32_t aWidth, layers::TextureFlags::DEFAULT, layers::ALLOC_DISALLOW_BUFFERTEXTURECLIENT); if (textureClient) { - RefPtr grallocTextureClient = - static_cast(textureClient.get()); + RefPtr grallocTextureClient = textureClient->AsGrallocTextureClientOGL(); android::sp destBuffer = grallocTextureClient->GetGraphicBuffer(); @@ -784,16 +782,11 @@ MediaEngineGonkVideoSource::RotateImage(layers::Image* aImage, uint32_t aWidth, libyuv::FOURCC_NV21); destBuffer->unlock(); - layers::GrallocImage::GrallocData data; - - data.mPicSize = gfx::IntSize(dstWidth, dstHeight); - data.mGraphicBuffer = textureClient; - videoImage->SetData(data); + image->AsGrallocImage()->SetData(textureClient, gfx::IntSize(dstWidth, dstHeight)); } else { // Handle out of gralloc case. - image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); - layers::PlanarYCbCrImage* videoImage = static_cast(image.get()); - uint8_t* dstPtr = videoImage->AllocateAndGetNewBuffer(size); + image = mImageContainer->CreatePlanarYCbCrImage(); + uint8_t* dstPtr = image->AsPlanarYCbCrImage()->AllocateAndGetNewBuffer(size); libyuv::ConvertToI420(srcPtr, size, dstPtr, dstWidth, @@ -821,7 +814,7 @@ MediaEngineGonkVideoSource::RotateImage(layers::Image* aImage, uint32_t aWidth, data.mPicSize = IntSize(dstWidth, dstHeight); data.mStereoMode = StereoMode::MONO; - videoImage->SetDataNoCopy(data); + image->AsPlanarYCbCrImage()->SetDataNoCopy(data); } graphicBuffer->unlock(); diff --git a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp index 96bc097e58..ebb2bbb2ce 100644 --- a/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineRemoteVideoSource.cpp @@ -11,7 +11,7 @@ #include "MediaTrackConstraints.h" #include "CamerasChild.h" -extern PRLogModuleInfo* GetMediaManagerLog(); +extern mozilla::LogModule* GetMediaManagerLog(); #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) #define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) @@ -289,8 +289,7 @@ MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer, } // Create a video frame and append it to the track. - RefPtr image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR); - layers::PlanarYCbCrImage* videoImage = static_cast(image.get()); + RefPtr image = mImageContainer->CreatePlanarYCbCrImage(); uint8_t* frame = static_cast (buffer); const uint8_t lumaBpp = 8; @@ -310,7 +309,7 @@ MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer, data.mPicSize = IntSize(mWidth, mHeight); data.mStereoMode = StereoMode::MONO; - if (!videoImage->SetData(data)) { + if (!image->SetData(data)) { MOZ_ASSERT(false); return 0; } diff --git a/dom/media/webrtc/MediaEngineTabVideoSource.cpp b/dom/media/webrtc/MediaEngineTabVideoSource.cpp index fce924b996..2b06d8f15b 100644 --- a/dom/media/webrtc/MediaEngineTabVideoSource.cpp +++ b/dom/media/webrtc/MediaEngineTabVideoSource.cpp @@ -298,13 +298,7 @@ MediaEngineTabVideoSource::Draw() { return; } - layers::CairoImage::Data cairoData; - cairoData.mSize = size; - cairoData.mSourceSurface = surface; - - RefPtr image = new layers::CairoImage(); - - image->SetData(cairoData); + RefPtr image = new layers::CairoImage(size, surface); MonitorAutoLock mon(mMonitor); mImage = image; diff --git a/dom/media/webrtc/MediaEngineWebRTC.cpp b/dom/media/webrtc/MediaEngineWebRTC.cpp index 50af662fac..18496e234d 100644 --- a/dom/media/webrtc/MediaEngineWebRTC.cpp +++ b/dom/media/webrtc/MediaEngineWebRTC.cpp @@ -13,12 +13,10 @@ #include "mozilla/Logging.h" -static PRLogModuleInfo* +static mozilla::LogModule* GetUserMediaLog() { - static PRLogModuleInfo *sLog; - if (!sLog) - sLog = PR_NewLogModule("GetUserMedia"); + static mozilla::LazyLogModule sLog("GetUserMedia"); return sLog; } diff --git a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp index 963b82ba1c..cacc0b3590 100644 --- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp +++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp @@ -35,7 +35,7 @@ namespace mozilla { #undef LOG #endif -extern PRLogModuleInfo* GetMediaManagerLog(); +extern LogModule* GetMediaManagerLog(); #define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg) #define LOG_FRAMES(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg) diff --git a/dom/media/webspeech/recognition/Makefile.in b/dom/media/webspeech/recognition/Makefile.in index d7220b7686..4a8058bbdd 100644 --- a/dom/media/webspeech/recognition/Makefile.in +++ b/dom/media/webspeech/recognition/Makefile.in @@ -1,16 +1,16 @@ ifdef MOZ_WEBSPEECH_MODELS MODELSPS_KEEP_PATH := 1 -MODELSPS_FILES := models/dict/cmu07a.dic \ - models/en-us-semi/mixture_weights \ - models/en-us-semi/feat.params \ - models/en-us-semi/mdef \ - models/en-us-semi/means \ - models/en-us-semi/noisedict \ - models/en-us-semi/sendump \ - models/en-us-semi/transition_matrices \ - models/en-us-semi/variances +MODELSPS_FILES := models/dict/en-US.dic \ + models/en-US/mixture_weights \ + models/en-US/feat.params \ + models/en-US/mdef \ + models/en-US/means \ + models/en-US/noisedict \ + models/en-US/sendump \ + models/en-US/transition_matrices \ + models/en-US/variances -MODELSPS_DEST := $(DIST)/bin/models +MODELSPS_DEST := $(DIST)/bin INSTALL_TARGETS += MODELSPS endif diff --git a/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp b/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp index bbf1f29a19..b2c7af8e07 100644 --- a/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp +++ b/dom/media/webspeech/recognition/PocketSphinxSpeechRecognitionService.cpp @@ -54,13 +54,15 @@ public: SpeechRecognitionResultList* resultList = new SpeechRecognitionResultList(mRecognition); SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); - SpeechRecognitionAlternative* alternative = - new SpeechRecognitionAlternative(mRecognition); + if (0 < mRecognition->MaxAlternatives()) { + SpeechRecognitionAlternative* alternative = + new SpeechRecognitionAlternative(mRecognition); - alternative->mTranscript = mResult; - alternative->mConfidence = 100; + alternative->mTranscript = mResult; + alternative->mConfidence = 100; - result->mItems.AppendElement(alternative); + result->mItems.AppendElement(alternative); + } resultList->mItems.AppendElement(result); event->mRecognitionResultList = resultList; @@ -103,9 +105,7 @@ public: rv = ps_end_utt(mPs); if (rv >= 0) { hyp = ps_get_hyp(mPs, &score); - if (hyp == nullptr) { - hypoValue.Assign("ERROR"); - } else { + if (hyp) { hypoValue.Assign(hyp); } } @@ -139,7 +139,7 @@ PocketSphinxSpeechRecognitionService::PocketSphinxSpeechRecognitionService() tmpFile->AppendRelativePath(NS_LITERAL_STRING("..")); #endif tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); - tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-us-semi")); + tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US")); tmpFile->GetPath(aStringAMPath); NS_GetSpecialDirectory(NS_GRE_DIR, getter_AddRefs(tmpFile)); @@ -149,7 +149,7 @@ PocketSphinxSpeechRecognitionService::PocketSphinxSpeechRecognitionService() #endif tmpFile->AppendRelativePath(NS_LITERAL_STRING("models")); // tmpFile->AppendRelativePath(NS_LITERAL_STRING("dict")); // - tmpFile->AppendRelativePath(NS_LITERAL_STRING("cmu07a.dic")); // + tmpFile->AppendRelativePath(NS_LITERAL_STRING("en-US.dic")); // tmpFile->GetPath(aStringDictPath); // FOR B2G PATHS HARDCODED (APPEND /DATA ON THE BEGINING, FOR DESKTOP, ONLY @@ -330,13 +330,15 @@ PocketSphinxSpeechRecognitionService::BuildMockResultList() SpeechRecognitionResultList* resultList = new SpeechRecognitionResultList(mRecognition); SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); - SpeechRecognitionAlternative* alternative = - new SpeechRecognitionAlternative(mRecognition); + if (0 < mRecognition->MaxAlternatives()) { + SpeechRecognitionAlternative* alternative = + new SpeechRecognitionAlternative(mRecognition); - alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); - alternative->mConfidence = 0.0f; + alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); + alternative->mConfidence = 0.0f; - result->mItems.AppendElement(alternative); + result->mItems.AppendElement(alternative); + } resultList->mItems.AppendElement(result); return resultList; diff --git a/dom/media/webspeech/recognition/SpeechGrammarList.cpp b/dom/media/webspeech/recognition/SpeechGrammarList.cpp index 3ad363dc65..d4883378fe 100644 --- a/dom/media/webspeech/recognition/SpeechGrammarList.cpp +++ b/dom/media/webspeech/recognition/SpeechGrammarList.cpp @@ -23,10 +23,9 @@ NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(SpeechGrammarList) NS_INTERFACE_MAP_ENTRY(nsISupports) NS_INTERFACE_MAP_END -SpeechGrammarList::SpeechGrammarList(nsISupports* aParent, nsISpeechRecognitionService* aRecognitionService) +SpeechGrammarList::SpeechGrammarList(nsISupports* aParent) : mParent(aParent) { - this->mRecognitionService = aRecognitionService; } SpeechGrammarList::~SpeechGrammarList() @@ -37,16 +36,9 @@ already_AddRefed SpeechGrammarList::Constructor(const GlobalObject& aGlobal, ErrorResult& aRv) { - nsCOMPtr recognitionService; - recognitionService = GetSpeechRecognitionService(); - if (!recognitionService) { - aRv.Throw(NS_ERROR_NOT_AVAILABLE); - return nullptr; - } else { - RefPtr speechGrammarList = - new SpeechGrammarList(aGlobal.GetAsSupports(), recognitionService); - return speechGrammarList.forget(); - } + RefPtr speechGrammarList = + new SpeechGrammarList(aGlobal.GetAsSupports()); + return speechGrammarList.forget(); } JSObject* @@ -91,7 +83,6 @@ SpeechGrammarList::AddFromString(const nsAString& aString, SpeechGrammar* speechGrammar = new SpeechGrammar(mParent); speechGrammar->SetSrc(aString, aRv); mItems.AppendElement(speechGrammar); - mRecognitionService->ValidateAndSetGrammarList(speechGrammar, nullptr); return; } diff --git a/dom/media/webspeech/recognition/SpeechGrammarList.h b/dom/media/webspeech/recognition/SpeechGrammarList.h index 5f23d9c162..208f8d775e 100644 --- a/dom/media/webspeech/recognition/SpeechGrammarList.h +++ b/dom/media/webspeech/recognition/SpeechGrammarList.h @@ -11,7 +11,6 @@ #include "nsCOMPtr.h" #include "nsCycleCollectionParticipant.h" #include "nsWrapperCache.h" -#include "nsISpeechRecognitionService.h" struct JSContext; @@ -29,7 +28,7 @@ class SpeechGrammarList final : public nsISupports, public nsWrapperCache { public: - explicit SpeechGrammarList(nsISupports* aParent, nsISpeechRecognitionService* aRecognitionService); + explicit SpeechGrammarList(nsISupports* aParent); NS_DECL_CYCLE_COLLECTING_ISUPPORTS NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_CLASS(SpeechGrammarList) @@ -50,8 +49,6 @@ public: already_AddRefed IndexedGetter(uint32_t aIndex, bool& aPresent, ErrorResult& aRv); - nsCOMPtr mRecognitionService; - private: ~SpeechGrammarList(); diff --git a/dom/media/webspeech/recognition/SpeechRecognition.cpp b/dom/media/webspeech/recognition/SpeechRecognition.cpp index 3a9e3ec833..50384741b2 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.cpp +++ b/dom/media/webspeech/recognition/SpeechRecognition.cpp @@ -9,6 +9,8 @@ #include "nsCOMPtr.h" #include "nsCycleCollectionParticipant.h" +#include "mozilla/dom/BindingUtils.h" +#include "mozilla/dom/Element.h" #include "mozilla/dom/SpeechRecognitionBinding.h" #include "mozilla/dom/MediaStreamTrackBinding.h" #include "mozilla/dom/MediaStreamError.h" @@ -19,7 +21,9 @@ #include "endpointer.h" #include "mozilla/dom/SpeechRecognitionEvent.h" +#include "nsIDocument.h" #include "nsIObserverService.h" +#include "nsPIDOMWindow.h" #include "nsServiceManagerUtils.h" #include "nsQueryObject.h" @@ -34,7 +38,8 @@ namespace mozilla { namespace dom { #define PREFERENCE_DEFAULT_RECOGNITION_SERVICE "media.webspeech.service.default" -#define DEFAULT_RECOGNITION_SERVICE "pocketsphinx" +#define DEFAULT_RECOGNITION_SERVICE_PREFIX "pocketsphinx-" +#define DEFAULT_RECOGNITION_SERVICE "pocketsphinx-en-US" #define PREFERENCE_ENDPOINTER_SILENCE_LENGTH "media.webspeech.silence_length" #define PREFERENCE_ENDPOINTER_LONG_SILENCE_LENGTH "media.webspeech.long_silence_length" @@ -48,20 +53,16 @@ static const uint32_t kSPEECH_DETECTION_TIMEOUT_MS = 10000; // kSAMPLE_RATE frames = 1s, kESTIMATION_FRAMES frames = 300ms static const uint32_t kESTIMATION_SAMPLES = 300 * kSAMPLE_RATE / 1000; -PRLogModuleInfo* +LogModule* GetSpeechRecognitionLog() { - static PRLogModuleInfo* sLog; - if (!sLog) { - sLog = PR_NewLogModule("SpeechRecognition"); - } - + static LazyLogModule sLog("SpeechRecognition"); return sLog; } #define SR_LOG(...) MOZ_LOG(GetSpeechRecognitionLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) already_AddRefed -GetSpeechRecognitionService() +GetSpeechRecognitionService(const nsAString& aLang) { nsAutoCString speechRecognitionServiceCID; @@ -69,19 +70,23 @@ GetSpeechRecognitionService() Preferences::GetCString(PREFERENCE_DEFAULT_RECOGNITION_SERVICE); nsAutoCString speechRecognitionService; - if (!prefValue.get() || prefValue.IsEmpty()) { - speechRecognitionService = DEFAULT_RECOGNITION_SERVICE; - } else { + if (!aLang.IsEmpty()) { + speechRecognitionService = + NS_LITERAL_CSTRING(DEFAULT_RECOGNITION_SERVICE_PREFIX) + + NS_ConvertUTF16toUTF8(aLang); + } else if (!prefValue.IsEmpty()) { speechRecognitionService = prefValue; + } else { + speechRecognitionService = DEFAULT_RECOGNITION_SERVICE; } - if (!SpeechRecognition::mTestConfig.mFakeRecognitionService){ + if (SpeechRecognition::mTestConfig.mFakeRecognitionService) { + speechRecognitionServiceCID = + NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "fake"; + } else { speechRecognitionServiceCID = NS_LITERAL_CSTRING(NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX) + speechRecognitionService; - } else { - speechRecognitionServiceCID = - NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "fake"; } nsresult rv; @@ -90,7 +95,9 @@ GetSpeechRecognitionService() return recognitionService.forget(); } -NS_INTERFACE_MAP_BEGIN(SpeechRecognition) +NS_IMPL_CYCLE_COLLECTION_INHERITED(SpeechRecognition, DOMEventTargetHelper, mDOMStream, mSpeechGrammarList) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION_INHERITED(SpeechRecognition) NS_INTERFACE_MAP_ENTRY(nsIObserver) NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) @@ -104,6 +111,9 @@ SpeechRecognition::SpeechRecognition(nsPIDOMWindow* aOwnerWindow) , mEndpointer(kSAMPLE_RATE) , mAudioSamplesPerChunk(mEndpointer.FrameSize()) , mSpeechDetectionTimer(do_CreateInstance(NS_TIMER_CONTRACTID)) + , mSpeechGrammarList(new SpeechGrammarList(GetParentObject())) + , mInterimResults(false) + , mMaxAlternatives(1) { SR_LOG("created SpeechRecognition"); @@ -143,6 +153,16 @@ SpeechRecognition::WrapObject(JSContext* aCx, JS::Handle aGivenProto) return SpeechRecognitionBinding::Wrap(aCx, this, aGivenProto); } +bool +SpeechRecognition::IsAuthorized(JSContext* aCx, JSObject* aGlobal) +{ + bool inCertifiedApp = IsInCertifiedApp(aCx, aGlobal); + bool enableTests = Preferences::GetBool(TEST_PREFERENCE_ENABLE); + bool enableRecognitionEnable = Preferences::GetBool(TEST_PREFERENCE_RECOGNITION_ENABLE); + bool enableRecognitionForceEnable = Preferences::GetBool(TEST_PREFERENCE_RECOGNITION_FORCE_ENABLE); + return (inCertifiedApp || enableRecognitionForceEnable || enableTests) && enableRecognitionEnable; +} + already_AddRefed SpeechRecognition::Constructor(const GlobalObject& aGlobal, ErrorResult& aRv) @@ -608,31 +628,28 @@ SpeechRecognition::ProcessTestEventRequest(nsISupports* aSubject, const nsAStrin } already_AddRefed -SpeechRecognition::GetGrammars(ErrorResult& aRv) const +SpeechRecognition::Grammars() const { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return nullptr; + RefPtr speechGrammarList = mSpeechGrammarList; + return speechGrammarList.forget(); } void -SpeechRecognition::SetGrammars(SpeechGrammarList& aArg, ErrorResult& aRv) +SpeechRecognition::SetGrammars(SpeechGrammarList& aArg) { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return; + mSpeechGrammarList = &aArg; } void -SpeechRecognition::GetLang(nsString& aRetVal, ErrorResult& aRv) const +SpeechRecognition::GetLang(nsString& aRetVal) const { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return; + aRetVal = mLang; } void -SpeechRecognition::SetLang(const nsAString& aArg, ErrorResult& aRv) +SpeechRecognition::SetLang(const nsAString& aArg) { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return; + mLang = aArg; } bool @@ -650,30 +667,28 @@ SpeechRecognition::SetContinuous(bool aArg, ErrorResult& aRv) } bool -SpeechRecognition::GetInterimResults(ErrorResult& aRv) const +SpeechRecognition::InterimResults() const { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return false; + return mInterimResults; } void -SpeechRecognition::SetInterimResults(bool aArg, ErrorResult& aRv) +SpeechRecognition::SetInterimResults(bool aArg) { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); + mInterimResults = aArg; return; } uint32_t -SpeechRecognition::GetMaxAlternatives(ErrorResult& aRv) const +SpeechRecognition::MaxAlternatives() const { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); - return 0; + return mMaxAlternatives; } void -SpeechRecognition::SetMaxAlternatives(uint32_t aArg, ErrorResult& aRv) +SpeechRecognition::SetMaxAlternatives(uint32_t aArg) { - aRv.Throw(NS_ERROR_NOT_IMPLEMENTED); + mMaxAlternatives = aArg; return; } @@ -699,8 +714,11 @@ SpeechRecognition::Start(const Optional>& aStream, Error return; } - mRecognitionService = GetSpeechRecognitionService(); - if (NS_WARN_IF(!mRecognitionService)) { + if (!SetRecognitionService(aRv)) { + return; + } + + if (!ValidateAndSetGrammarList(aRv)) { return; } @@ -728,6 +746,77 @@ SpeechRecognition::Start(const Optional>& aStream, Error NS_DispatchToMainThread(event); } +bool +SpeechRecognition::SetRecognitionService(ErrorResult& aRv) +{ + // See: https://dvcs.w3.org/hg/speech-api/raw-file/tip/webspeechapi.html#dfn-lang + if (!mLang.IsEmpty()) { + mRecognitionService = GetSpeechRecognitionService(mLang); + + if (!mRecognitionService) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + + return true; + } + + nsCOMPtr window = GetOwner(); + if(!window) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + nsCOMPtr document = window->GetExtantDoc(); + if(!document) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + nsCOMPtr element = document->GetRootElement(); + if(!element) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + + nsAutoString lang; + element->GetLang(lang); + mRecognitionService = GetSpeechRecognitionService(lang); + + if (!mRecognitionService) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + + return true; +} + +bool +SpeechRecognition::ValidateAndSetGrammarList(ErrorResult& aRv) +{ + if (!mSpeechGrammarList) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + + uint32_t grammarListLength = mSpeechGrammarList->Length(); + if (0 == grammarListLength) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + + for (uint32_t count = 0; count < grammarListLength; ++count) { + RefPtr speechGrammar = mSpeechGrammarList->Item(count, aRv); + if (aRv.Failed()) { + return false; + } + if (NS_FAILED(mRecognitionService->ValidateAndSetGrammarList(speechGrammar.get(), nullptr))) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + return false; + } + } + + return true; +} + void SpeechRecognition::Stop() { diff --git a/dom/media/webspeech/recognition/SpeechRecognition.h b/dom/media/webspeech/recognition/SpeechRecognition.h index a0d2da84f1..6fe4dc3bd9 100644 --- a/dom/media/webspeech/recognition/SpeechRecognition.h +++ b/dom/media/webspeech/recognition/SpeechRecognition.h @@ -38,17 +38,17 @@ namespace dom { #define TEST_PREFERENCE_ENABLE "media.webspeech.test.enable" #define TEST_PREFERENCE_FAKE_FSM_EVENTS "media.webspeech.test.fake_fsm_events" #define TEST_PREFERENCE_FAKE_RECOGNITION_SERVICE "media.webspeech.test.fake_recognition_service" +#define TEST_PREFERENCE_RECOGNITION_ENABLE "media.webspeech.recognition.enable" +#define TEST_PREFERENCE_RECOGNITION_FORCE_ENABLE "media.webspeech.recognition.force_enable" #define SPEECH_RECOGNITION_TEST_EVENT_REQUEST_TOPIC "SpeechRecognitionTest:RequestEvent" #define SPEECH_RECOGNITION_TEST_END_TOPIC "SpeechRecognitionTest:End" class GlobalObject; class SpeechEvent; -PRLogModuleInfo* GetSpeechRecognitionLog(); +LogModule* GetSpeechRecognitionLog(); #define SR_LOG(...) MOZ_LOG(GetSpeechRecognitionLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) -already_AddRefed GetSpeechRecognitionService(); - class SpeechRecognition final : public DOMEventTargetHelper, public nsIObserver, public SupportsWeakPtr @@ -58,6 +58,7 @@ public: explicit SpeechRecognition(nsPIDOMWindow* aOwnerWindow); NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(SpeechRecognition, DOMEventTargetHelper) NS_DECL_NSIOBSERVER @@ -65,28 +66,30 @@ public: virtual JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) override; + static bool IsAuthorized(JSContext* aCx, JSObject* aGlobal); + static already_AddRefed Constructor(const GlobalObject& aGlobal, ErrorResult& aRv); - already_AddRefed GetGrammars(ErrorResult& aRv) const; + already_AddRefed Grammars() const; - void SetGrammars(mozilla::dom::SpeechGrammarList& aArg, ErrorResult& aRv); + void SetGrammars(mozilla::dom::SpeechGrammarList& aArg); - void GetLang(nsString& aRetVal, ErrorResult& aRv) const; + void GetLang(nsString& aRetVal) const; - void SetLang(const nsAString& aArg, ErrorResult& aRv); + void SetLang(const nsAString& aArg); bool GetContinuous(ErrorResult& aRv) const; void SetContinuous(bool aArg, ErrorResult& aRv); - bool GetInterimResults(ErrorResult& aRv) const; + bool InterimResults() const; - void SetInterimResults(bool aArg, ErrorResult& aRv); + void SetInterimResults(bool aArg); - uint32_t GetMaxAlternatives(ErrorResult& aRv) const; + uint32_t MaxAlternatives() const; - void SetMaxAlternatives(uint32_t aArg, ErrorResult& aRv); + void SetMaxAlternatives(uint32_t aArg); void GetServiceURI(nsString& aRetVal, ErrorResult& aRv) const; @@ -172,6 +175,9 @@ private: void SetState(FSMState state); bool StateBetween(FSMState begin, FSMState end); + bool SetRecognitionService(ErrorResult& aRv); + bool ValidateAndSetGrammarList(ErrorResult& aRv); + class GetUserMediaSuccessCallback : public nsIDOMGetUserMediaSuccessCallback { public: @@ -245,6 +251,32 @@ private: nsCOMPtr mSpeechDetectionTimer; bool mAborted; + nsString mLang; + + RefPtr mSpeechGrammarList; + + // WebSpeechAPI (http://bit.ly/1gIl7DC) states: + // + // 1. Default value MUST be false + // 2. If true, interim results SHOULD be returned + // 3. If false, interim results MUST NOT be returned + // + // Pocketsphinx does not return interm results; so, defaulting + // mInterimResults to false, then ignoring its subsequent value + // is a conforming implementation. + bool mInterimResults; + + // WebSpeechAPI (http://bit.ly/1JAiqeo) states: + // + // 1. Default value is 1 + // 2. Subsequent value is the "maximum number of SpeechRecognitionAlternatives per result" + // + // Pocketsphinx can only return at maximum a single SpeechRecognitionAlternative + // per SpeechRecognitionResult. So defaulting mMaxAlternatives to 1, for all non + // zero values ignoring mMaxAlternatives while for a 0 value returning no + // SpeechRecognitionAlternative per result is a conforming implementation. + uint32_t mMaxAlternatives; + void ProcessTestEventRequest(nsISupports* aSubject, const nsAString& aEventName); const char* GetName(FSMState aId); diff --git a/dom/media/webspeech/recognition/models/dict/cmu07a.dic b/dom/media/webspeech/recognition/models/dict/en-US.dic similarity index 100% rename from dom/media/webspeech/recognition/models/dict/cmu07a.dic rename to dom/media/webspeech/recognition/models/dict/en-US.dic diff --git a/dom/media/webspeech/recognition/models/en-us-semi/README b/dom/media/webspeech/recognition/models/en-US/README similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/README rename to dom/media/webspeech/recognition/models/en-US/README diff --git a/dom/media/webspeech/recognition/models/en-us-semi/feat.params b/dom/media/webspeech/recognition/models/en-US/feat.params similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/feat.params rename to dom/media/webspeech/recognition/models/en-US/feat.params diff --git a/dom/media/webspeech/recognition/models/en-us-semi/mdef b/dom/media/webspeech/recognition/models/en-US/mdef similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/mdef rename to dom/media/webspeech/recognition/models/en-US/mdef diff --git a/dom/media/webspeech/recognition/models/en-us-semi/means b/dom/media/webspeech/recognition/models/en-US/means similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/means rename to dom/media/webspeech/recognition/models/en-US/means diff --git a/dom/media/webspeech/recognition/models/en-us-semi/mixture_weights b/dom/media/webspeech/recognition/models/en-US/mixture_weights similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/mixture_weights rename to dom/media/webspeech/recognition/models/en-US/mixture_weights diff --git a/dom/media/webspeech/recognition/models/en-us-semi/noisedict b/dom/media/webspeech/recognition/models/en-US/noisedict similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/noisedict rename to dom/media/webspeech/recognition/models/en-US/noisedict diff --git a/dom/media/webspeech/recognition/models/en-us-semi/sendump b/dom/media/webspeech/recognition/models/en-US/sendump similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/sendump rename to dom/media/webspeech/recognition/models/en-US/sendump diff --git a/dom/media/webspeech/recognition/models/en-us-semi/transition_matrices b/dom/media/webspeech/recognition/models/en-US/transition_matrices similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/transition_matrices rename to dom/media/webspeech/recognition/models/en-US/transition_matrices diff --git a/dom/media/webspeech/recognition/models/en-us-semi/variances b/dom/media/webspeech/recognition/models/en-US/variances similarity index 100% rename from dom/media/webspeech/recognition/models/en-us-semi/variances rename to dom/media/webspeech/recognition/models/en-US/variances diff --git a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp b/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp index 440337434a..97dc9ceef9 100644 --- a/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp +++ b/dom/media/webspeech/recognition/test/FakeSpeechRecognitionService.cpp @@ -102,12 +102,14 @@ FakeSpeechRecognitionService::BuildMockResultList() { SpeechRecognitionResultList* resultList = new SpeechRecognitionResultList(mRecognition); SpeechRecognitionResult* result = new SpeechRecognitionResult(mRecognition); - SpeechRecognitionAlternative* alternative = new SpeechRecognitionAlternative(mRecognition); + if (0 < mRecognition->MaxAlternatives()) { + SpeechRecognitionAlternative* alternative = new SpeechRecognitionAlternative(mRecognition); - alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); - alternative->mConfidence = 0.0f; + alternative->mTranscript = NS_LITERAL_STRING("Mock final result"); + alternative->mConfidence = 0.0f; - result->mItems.AppendElement(alternative); + result->mItems.AppendElement(alternative); + } resultList->mItems.AppendElement(result); return resultList; diff --git a/dom/media/webspeech/recognition/test/head.js b/dom/media/webspeech/recognition/test/head.js index 8ff7d1569e..b5aa2d6129 100644 --- a/dom/media/webspeech/recognition/test/head.js +++ b/dom/media/webspeech/recognition/test/head.js @@ -44,6 +44,11 @@ function EventManager(sr) { var isDone = false; + // set up grammar + var sgl = new SpeechGrammarList(); + sgl.addFromString("#JSGF V1.0; grammar test; public = hello ;", 1); + sr.grammars = sgl; + // AUDIO_DATA events are asynchronous, // so we queue events requested while they are being // issued to make them seem synchronous diff --git a/dom/media/webspeech/recognition/test/test_audio_capture_error.html b/dom/media/webspeech/recognition/test/test_audio_capture_error.html index 96dda07437..6869f4c9b0 100644 --- a/dom/media/webspeech/recognition/test/test_audio_capture_error.html +++ b/dom/media/webspeech/recognition/test/test_audio_capture_error.html @@ -23,11 +23,16 @@ https://bugzilla.mozilla.org/show_bug.cgi?id=650295 performTest({ eventsToRequest: ['EVENT_AUDIO_ERROR'], expectedEvents: { + 'start': null, + 'audiostart': null, + 'speechstart': null, + 'speechend': null, + 'audioend': null, 'error': buildErrorCallback(errorCodes.AUDIO_CAPTURE), 'end': null }, doneFunc: SimpleTest.finish, - prefs: [["media.webspeech.test.fake_fsm_events", true]] + prefs: [["media.webspeech.test.fake_fsm_events", true], ["media.webspeech.test.fake_recognition_service", true]] }); diff --git a/dom/media/webspeech/synth/SpeechSynthesis.cpp b/dom/media/webspeech/synth/SpeechSynthesis.cpp index 1d98888411..f0942b25be 100644 --- a/dom/media/webspeech/synth/SpeechSynthesis.cpp +++ b/dom/media/webspeech/synth/SpeechSynthesis.cpp @@ -16,14 +16,10 @@ #include "nsIDocument.h" #undef LOG -PRLogModuleInfo* +mozilla::LogModule* GetSpeechSynthLog() { - static PRLogModuleInfo* sLog = nullptr; - - if (!sLog) { - sLog = PR_NewLogModule("SpeechSynthesis"); - } + static mozilla::LazyLogModule sLog("SpeechSynthesis"); return sLog; } diff --git a/dom/media/webspeech/synth/nsSpeechTask.cpp b/dom/media/webspeech/synth/nsSpeechTask.cpp index a418b0b0f8..12290111b3 100644 --- a/dom/media/webspeech/synth/nsSpeechTask.cpp +++ b/dom/media/webspeech/synth/nsSpeechTask.cpp @@ -18,7 +18,7 @@ #endif #undef LOG -extern PRLogModuleInfo* GetSpeechSynthLog(); +extern mozilla::LogModule* GetSpeechSynthLog(); #define LOG(type, msg) MOZ_LOG(GetSpeechSynthLog(), type, msg) #define AUDIO_TRACK 1 diff --git a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp index 580ec00483..290d1c05da 100644 --- a/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp +++ b/dom/media/webspeech/synth/nsSynthVoiceRegistry.cpp @@ -24,7 +24,7 @@ #include "SpeechSynthesisParent.h" #undef LOG -extern PRLogModuleInfo* GetSpeechSynthLog(); +extern mozilla::LogModule* GetSpeechSynthLog(); #define LOG(type, msg) MOZ_LOG(GetSpeechSynthLog(), type, msg) namespace { diff --git a/dom/plugins/base/nsPluginInstanceOwner.cpp b/dom/plugins/base/nsPluginInstanceOwner.cpp index 22fde0dcba..f4839e1ffa 100644 --- a/dom/plugins/base/nsPluginInstanceOwner.cpp +++ b/dom/plugins/base/nsPluginInstanceOwner.cpp @@ -178,16 +178,10 @@ AttachToContainerAsEGLImage(ImageContainer* container, return; } - RefPtr img = container->CreateImage(ImageFormat::EGLIMAGE); - - EGLImageImage::Data data; - data.mImage = image; - data.mSize = gfx::IntSize(rect.width, rect.height); - data.mOriginPos = instance->OriginPos(); - - EGLImageImage* typedImg = static_cast(img.get()); - typedImg->SetData(data); - + RefPtr img = new EGLImageImage( + image, nullptr, + gfx::IntSize(rect.width, rect.height), instance->OriginPos(), + false /* owns */); *out_image = img; } @@ -205,16 +199,10 @@ AttachToContainerAsSurfaceTexture(ImageContainer* container, return; } - RefPtr img = container->CreateImage(ImageFormat::SURFACE_TEXTURE); - - SurfaceTextureImage::Data data; - data.mSurfTex = surfTex; - data.mSize = gfx::IntSize(rect.width, rect.height); - data.mOriginPos = instance->OriginPos(); - - SurfaceTextureImage* typedImg = static_cast(img.get()); - typedImg->SetData(data); - + RefPtr img = new SurfaceTextureImage( + surfTex, + gfx::IntSize(rect.width, rect.height), + instance->OriginPos()); *out_image = img; } #endif @@ -1419,16 +1407,10 @@ nsPluginInstanceOwner::GetImageContainerForVideo(nsNPAPIPluginInstance::VideoInf { RefPtr container = LayerManager::CreateImageContainer(); - RefPtr img = container->CreateImage(ImageFormat::SURFACE_TEXTURE); - - SurfaceTextureImage::Data data; - data.mSurfTex = aVideoInfo->mSurfaceTexture; - data.mOriginPos = gl::OriginPos::BottomLeft; - data.mSize = gfx::IntSize(aVideoInfo->mDimensions.width, aVideoInfo->mDimensions.height); - - SurfaceTextureImage* typedImg = static_cast(img.get()); - typedImg->SetData(data); - + RefPtr img = new SurfaceTextureImage( + aVideoInfo->mSurfaceTexture, + gfx::IntSize(aVideoInfo->mDimensions.width, aVideoInfo->mDimensions.height), + gl::OriginPos::BottomLeft); container->SetCurrentImageInTransaction(img); return container.forget(); diff --git a/dom/plugins/ipc/PluginInstanceParent.cpp b/dom/plugins/ipc/PluginInstanceParent.cpp index a6fb579b9b..3b91c10864 100644 --- a/dom/plugins/ipc/PluginInstanceParent.cpp +++ b/dom/plugins/ipc/PluginInstanceParent.cpp @@ -635,18 +635,15 @@ PluginInstanceParent::RecvShow(const NPRect& updatedRect, updatedRect.bottom - updatedRect.top); surface->MarkDirty(ur); - ImageContainer *container = GetImageContainer(); - RefPtr image = container->CreateImage(ImageFormat::CAIRO_SURFACE); - NS_ASSERTION(image->GetFormat() == ImageFormat::CAIRO_SURFACE, "Wrong format?"); - CairoImage* cairoImage = static_cast(image.get()); - CairoImage::Data cairoData; - cairoData.mSize = surface->GetSize(); - cairoData.mSourceSurface = gfxPlatform::GetPlatform()->GetSourceSurfaceForSurface(nullptr, surface); - cairoImage->SetData(cairoData); + RefPtr sourceSurface = + gfxPlatform::GetPlatform()->GetSourceSurfaceForSurface(nullptr, surface); + RefPtr image = new CairoImage(surface->GetSize(), sourceSurface); nsAutoTArray imageList; imageList.AppendElement( ImageContainer::NonOwningImage(image)); + + ImageContainer *container = GetImageContainer(); container->SetCurrentImages(imageList); } else if (mImageContainer) { @@ -717,17 +714,8 @@ PluginInstanceParent::GetImageContainer(ImageContainer** aContainer) #ifdef XP_MACOSX if (ioSurface) { - RefPtr image = container->CreateImage(ImageFormat::MAC_IOSURFACE); - if (!image) { - return NS_ERROR_FAILURE; - } - - NS_ASSERTION(image->GetFormat() == ImageFormat::MAC_IOSURFACE, "Wrong format?"); - - MacIOSurfaceImage* pluginImage = static_cast(image.get()); - pluginImage->SetSurface(ioSurface); - - container->SetCurrentImageInTransaction(pluginImage); + RefPtr image = new MacIOSurfaceImage(ioSurface); + container->SetCurrentImageInTransaction(image); NS_IF_ADDREF(container); *aContainer = container; diff --git a/dom/tests/mochitest/general/test_interfaces.html b/dom/tests/mochitest/general/test_interfaces.html index bcc90946cb..7693921f40 100644 --- a/dom/tests/mochitest/general/test_interfaces.html +++ b/dom/tests/mochitest/general/test_interfaces.html @@ -1018,22 +1018,6 @@ var interfaceNamesInGlobalScope = {name: "SourceBuffer", desktop: true, linux: false, release: false}, // IMPORTANT: Do not change this list without review from a DOM peer! {name: "SourceBufferList", desktop: true, linux: false, release: false}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognition", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognitionError", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognitionAlternative", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognitionResult", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognitionResultList", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechRecognitionEvent", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechGrammar", b2g: true, nightly: true}, -// IMPORTANT: Do not change this list without review from a DOM peer! - {name: "SpeechGrammarList", b2g: true, nightly: true}, // IMPORTANT: Do not change this list without review from a DOM peer! {name: "SpeechSynthesisEvent", b2g: true}, // IMPORTANT: Do not change this list without review from a DOM peer! diff --git a/dom/webidl/SpeechGrammar.webidl b/dom/webidl/SpeechGrammar.webidl index 774a3c5481..8cc97d2ffc 100644 --- a/dom/webidl/SpeechGrammar.webidl +++ b/dom/webidl/SpeechGrammar.webidl @@ -11,7 +11,8 @@ */ [Constructor, - Pref="media.webspeech.recognition.enable"] + Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechGrammar { [Throws] attribute DOMString src; diff --git a/dom/webidl/SpeechGrammarList.webidl b/dom/webidl/SpeechGrammarList.webidl index 89d95767b9..f1b5775ae0 100644 --- a/dom/webidl/SpeechGrammarList.webidl +++ b/dom/webidl/SpeechGrammarList.webidl @@ -10,7 +10,8 @@ * liability, trademark and document use rules apply. */ -[Constructor, Pref="media.webspeech.recognition.enable"] +[Constructor, Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechGrammarList { readonly attribute unsigned long length; [Throws] diff --git a/dom/webidl/SpeechRecognition.webidl b/dom/webidl/SpeechRecognition.webidl index 556d972d1e..c7dc9422d6 100644 --- a/dom/webidl/SpeechRecognition.webidl +++ b/dom/webidl/SpeechRecognition.webidl @@ -11,18 +11,15 @@ */ [Constructor, - Pref="media.webspeech.recognition.enable"] + Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechRecognition : EventTarget { // recognition parameters - [Throws] attribute SpeechGrammarList grammars; - [Throws] attribute DOMString lang; [Throws] attribute boolean continuous; - [Throws] attribute boolean interimResults; - [Throws] attribute unsigned long maxAlternatives; [Throws] attribute DOMString serviceURI; diff --git a/dom/webidl/SpeechRecognitionAlternative.webidl b/dom/webidl/SpeechRecognitionAlternative.webidl index c5844e1a3a..f2692404d5 100644 --- a/dom/webidl/SpeechRecognitionAlternative.webidl +++ b/dom/webidl/SpeechRecognitionAlternative.webidl @@ -10,7 +10,8 @@ * liability, trademark and document use rules apply. */ -[Pref="media.webspeech.recognition.enable"] +[Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechRecognitionAlternative { readonly attribute DOMString transcript; readonly attribute float confidence; diff --git a/dom/webidl/SpeechRecognitionError.webidl b/dom/webidl/SpeechRecognitionError.webidl index ef57bbaafb..ddc74e88da 100644 --- a/dom/webidl/SpeechRecognitionError.webidl +++ b/dom/webidl/SpeechRecognitionError.webidl @@ -16,6 +16,7 @@ enum SpeechRecognitionErrorCode { }; [Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized", Constructor(DOMString type, optional SpeechRecognitionErrorInit eventInitDict)] interface SpeechRecognitionError : Event { diff --git a/dom/webidl/SpeechRecognitionEvent.webidl b/dom/webidl/SpeechRecognitionEvent.webidl index a464fcc70d..7a95323d77 100644 --- a/dom/webidl/SpeechRecognitionEvent.webidl +++ b/dom/webidl/SpeechRecognitionEvent.webidl @@ -6,6 +6,7 @@ interface nsISupports; [Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized", Constructor(DOMString type, optional SpeechRecognitionEventInit eventInitDict)] interface SpeechRecognitionEvent : Event { diff --git a/dom/webidl/SpeechRecognitionResult.webidl b/dom/webidl/SpeechRecognitionResult.webidl index 9fb7bdc6b7..c66bb9ca2f 100644 --- a/dom/webidl/SpeechRecognitionResult.webidl +++ b/dom/webidl/SpeechRecognitionResult.webidl @@ -10,7 +10,8 @@ * liability, trademark and document use rules apply. */ -[Pref="media.webspeech.recognition.enable"] +[Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechRecognitionResult { readonly attribute unsigned long length; getter SpeechRecognitionAlternative item(unsigned long index); diff --git a/dom/webidl/SpeechRecognitionResultList.webidl b/dom/webidl/SpeechRecognitionResultList.webidl index 184ef18337..ec182c1def 100644 --- a/dom/webidl/SpeechRecognitionResultList.webidl +++ b/dom/webidl/SpeechRecognitionResultList.webidl @@ -10,7 +10,8 @@ * liability, trademark and document use rules apply. */ -[Pref="media.webspeech.recognition.enable"] +[Pref="media.webspeech.recognition.enable", + Func="SpeechRecognition::IsAuthorized"] interface SpeechRecognitionResultList { readonly attribute unsigned long length; getter SpeechRecognitionResult item(unsigned long index); diff --git a/gfx/gl/DecomposeIntoNoRepeatTriangles.cpp b/gfx/gl/DecomposeIntoNoRepeatTriangles.cpp index a71ea2280b..829c201870 100644 --- a/gfx/gl/DecomposeIntoNoRepeatTriangles.cpp +++ b/gfx/gl/DecomposeIntoNoRepeatTriangles.cpp @@ -40,15 +40,8 @@ RectTriangles::addRect(GLfloat x0, GLfloat y0, GLfloat x1, GLfloat y1, static GLfloat WrapTexCoord(GLfloat v) { - // fmodf gives negative results for negative numbers; - // that is, fmodf(0.75, 1.0) == 0.75, but - // fmodf(-0.75, 1.0) == -0.75. For the negative case, - // the result we need is 0.25, so we add 1.0f. - if (v < 0.0f) { - return 1.0f + fmodf(v, 1.0f); - } - - return fmodf(v, 1.0f); + // This should return values in range [0, 1.0) + return v - floorf(v); } void diff --git a/gfx/gl/GLBlitHelper.cpp b/gfx/gl/GLBlitHelper.cpp index daca04b9e4..098c173dce 100644 --- a/gfx/gl/GLBlitHelper.cpp +++ b/gfx/gl/GLBlitHelper.cpp @@ -25,6 +25,11 @@ #include "GLLibraryEGL.h" #endif +#ifdef XP_MACOSX +#include "MacIOSurfaceImage.h" +#include "GLContextCGL.h" +#endif + using mozilla::layers::PlanarYCbCrImage; using mozilla::layers::PlanarYCbCrData; @@ -43,8 +48,10 @@ GLBlitHelper::GLBlitHelper(GLContext* gl) , mTextureTransformLoc(-1) , mTexExternalBlit_FragShader(0) , mTexYUVPlanarBlit_FragShader(0) + , mTexNV12PlanarBlit_FragShader(0) , mTexExternalBlit_Program(0) , mTexYUVPlanarBlit_Program(0) + , mTexNV12PlanarBlit_Program(0) , mFBO(0) , mSrcTexY(0) , mSrcTexCb(0) @@ -189,6 +196,33 @@ GLBlitHelper::InitTexQuadProgram(BlitType target) } \n\ "; +#ifdef XP_MACOSX + const char kTexNV12PlanarBlit_FragShaderSource[] = "\ + #extension GL_ARB_texture_rectangle : require \n\ + #ifdef GL_ES \n\ + precision mediump float \n\ + #endif \n\ + varying vec2 vTexCoord; \n\ + uniform sampler2DRect uYTexture; \n\ + uniform sampler2DRect uCbCrTexture; \n\ + uniform vec2 uYTexScale; \n\ + uniform vec2 uCbCrTexScale; \n\ + void main() \n\ + { \n\ + float y = texture2DRect(uYTexture, vTexCoord * uYTexScale).r; \n\ + float cb = texture2DRect(uCbCrTexture, vTexCoord * uCbCrTexScale).r; \n\ + float cr = texture2DRect(uCbCrTexture, vTexCoord * uCbCrTexScale).a; \n\ + y = (y - 0.06275) * 1.16438; \n\ + cb = cb - 0.50196; \n\ + cr = cr - 0.50196; \n\ + gl_FragColor.r = y + cr * 1.59603; \n\ + gl_FragColor.g = y - 0.81297 * cr - 0.39176 * cb; \n\ + gl_FragColor.b = y + cb * 2.01723; \n\ + gl_FragColor.a = 1.0; \n\ + } \n\ + "; +#endif + bool success = false; GLuint *programPtr; @@ -219,6 +253,13 @@ GLBlitHelper::InitTexQuadProgram(BlitType target) fragShaderPtr = &mTexYUVPlanarBlit_FragShader; fragShaderSource = kTexYUVPlanarBlit_FragShaderSource; break; +#ifdef XP_MACOSX + case ConvertMacIOSurfaceImage: + programPtr = &mTexNV12PlanarBlit_Program; + fragShaderPtr = &mTexNV12PlanarBlit_FragShader; + fragShaderSource = kTexNV12PlanarBlit_FragShaderSource; + break; +#endif default: return false; } @@ -373,6 +414,24 @@ GLBlitHelper::InitTexQuadProgram(BlitType target) mGL->fUniform1i(texCr, Channel_Cr); break; } + case ConvertMacIOSurfaceImage: { +#ifdef XP_MACOSX + GLint texY = mGL->fGetUniformLocation(program, "uYTexture"); + GLint texCbCr = mGL->fGetUniformLocation(program, "uCbCrTexture"); + mYTexScaleLoc = mGL->fGetUniformLocation(program, "uYTexScale"); + mCbCrTexScaleLoc= mGL->fGetUniformLocation(program, "uCbCrTexScale"); + + DebugOnly hasUniformLocations = texY != -1 && + texCbCr != -1 && + mYTexScaleLoc != -1 && + mCbCrTexScaleLoc != -1; + MOZ_ASSERT(hasUniformLocations, "uniforms not found"); + + mGL->fUniform1i(texY, Channel_Y); + mGL->fUniform1i(texCbCr, Channel_Cb); +#endif + break; + } } MOZ_ASSERT(mGL->fGetAttribLocation(program, "aPosition") == 0); mYFlipLoc = mGL->fGetUniformLocation(program, "uYflip"); @@ -455,6 +514,10 @@ GLBlitHelper::DeleteTexBlitProgram() mGL->fDeleteShader(mTexYUVPlanarBlit_FragShader); mTexYUVPlanarBlit_FragShader = 0; } + if (mTexNV12PlanarBlit_FragShader) { + mGL->fDeleteShader(mTexNV12PlanarBlit_FragShader); + mTexNV12PlanarBlit_FragShader = 0; + } if (mTexExternalBlit_Program) { mGL->fDeleteProgram(mTexExternalBlit_Program); mTexExternalBlit_Program = 0; @@ -463,6 +526,10 @@ GLBlitHelper::DeleteTexBlitProgram() mGL->fDeleteProgram(mTexYUVPlanarBlit_Program); mTexYUVPlanarBlit_Program = 0; } + if (mTexNV12PlanarBlit_Program) { + mGL->fDeleteProgram(mTexNV12PlanarBlit_Program); + mTexNV12PlanarBlit_Program = 0; + } } void @@ -617,7 +684,7 @@ GLBlitHelper::BlitGrallocImage(layers::GrallocImage* grallocImage) bool GLBlitHelper::BlitSurfaceTextureImage(layers::SurfaceTextureImage* stImage) { - AndroidSurfaceTexture* surfaceTexture = stImage->GetData()->mSurfTex; + AndroidSurfaceTexture* surfaceTexture = stImage->GetSurfaceTexture(); ScopedBindTextureUnit boundTU(mGL, LOCAL_GL_TEXTURE0); @@ -646,8 +713,8 @@ GLBlitHelper::BlitSurfaceTextureImage(layers::SurfaceTextureImage* stImage) bool GLBlitHelper::BlitEGLImageImage(layers::EGLImageImage* image) { - EGLImage eglImage = image->GetData()->mImage; - EGLSync eglSync = image->GetData()->mSync; + EGLImage eglImage = image->GetImage(); + EGLSync eglSync = image->GetSync(); if (eglSync) { EGLint status = sEGLLibrary.fClientWaitSync(EGL_DISPLAY(), eglSync, 0, LOCAL_EGL_FOREVER); @@ -706,6 +773,47 @@ GLBlitHelper::BlitPlanarYCbCrImage(layers::PlanarYCbCrImage* yuvImage) return true; } +#ifdef XP_MACOSX +bool +GLBlitHelper::BlitMacIOSurfaceImage(layers::MacIOSurfaceImage* ioImage) +{ + ScopedBindTextureUnit boundTU(mGL, LOCAL_GL_TEXTURE0); + MacIOSurface* surf = ioImage->GetSurface(); + + GLint oldTex[2]; + for (int i = 0; i < 2; i++) { + mGL->fActiveTexture(LOCAL_GL_TEXTURE0 + i); + mGL->fGetIntegerv(LOCAL_GL_TEXTURE_BINDING_2D, &oldTex[i]); + } + + GLuint textures[2]; + mGL->fGenTextures(2, textures); + + mGL->fActiveTexture(LOCAL_GL_TEXTURE0); + mGL->fBindTexture(LOCAL_GL_TEXTURE_RECTANGLE_ARB, textures[0]); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_RECTANGLE_ARB, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_RECTANGLE_ARB, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); + surf->CGLTexImageIOSurface2D(gl::GLContextCGL::Cast(mGL)->GetCGLContext(), 0); + mGL->fUniform2f(mYTexScaleLoc, surf->GetWidth(0), surf->GetHeight(0)); + + mGL->fActiveTexture(LOCAL_GL_TEXTURE1); + mGL->fBindTexture(LOCAL_GL_TEXTURE_RECTANGLE_ARB, textures[1]); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_RECTANGLE_ARB, LOCAL_GL_TEXTURE_WRAP_T, LOCAL_GL_CLAMP_TO_EDGE); + mGL->fTexParameteri(LOCAL_GL_TEXTURE_RECTANGLE_ARB, LOCAL_GL_TEXTURE_WRAP_S, LOCAL_GL_CLAMP_TO_EDGE); + surf->CGLTexImageIOSurface2D(gl::GLContextCGL::Cast(mGL)->GetCGLContext(), 1); + mGL->fUniform2f(mCbCrTexScaleLoc, surf->GetWidth(1), surf->GetHeight(1)); + + mGL->fDrawArrays(LOCAL_GL_TRIANGLE_STRIP, 0, 4); + for (int i = 0; i < 2; i++) { + mGL->fActiveTexture(LOCAL_GL_TEXTURE0 + i); + mGL->fBindTexture(LOCAL_GL_TEXTURE_2D, oldTex[i]); + } + + mGL->fDeleteTextures(2, textures); + return true; +} +#endif + bool GLBlitHelper::BlitImageToFramebuffer(layers::Image* srcImage, const gfx::IntSize& destSize, @@ -733,13 +841,18 @@ GLBlitHelper::BlitImageToFramebuffer(layers::Image* srcImage, #ifdef MOZ_WIDGET_ANDROID case ImageFormat::SURFACE_TEXTURE: type = ConvertSurfaceTexture; - srcOrigin = static_cast(srcImage)->GetData() - ->mOriginPos; + srcOrigin = srcImage->AsSurfaceTextureImage()->GetOriginPos(); break; case ImageFormat::EGLIMAGE: type = ConvertEGLImage; - srcOrigin = static_cast(srcImage)->GetData()->mOriginPos; + srcOrigin = srcImage->AsEGLImageImage()->GetOriginPos(); + break; +#endif +#ifdef XP_MACOSX + case ImageFormat::MAC_IOSURFACE: + type = ConvertMacIOSurfaceImage; + srcOrigin = OriginPos::TopLeft; break; #endif @@ -777,6 +890,11 @@ GLBlitHelper::BlitImageToFramebuffer(layers::Image* srcImage, return BlitEGLImageImage(static_cast(srcImage)); #endif +#ifdef XP_MACOSX + case ConvertMacIOSurfaceImage: + return BlitMacIOSurfaceImage(srcImage->AsMacIOSurfaceImage()); +#endif + default: return false; } diff --git a/gfx/gl/GLBlitHelper.h b/gfx/gl/GLBlitHelper.h index 224c87a145..7138ea4cc9 100644 --- a/gfx/gl/GLBlitHelper.h +++ b/gfx/gl/GLBlitHelper.h @@ -20,6 +20,7 @@ class Image; class PlanarYCbCrImage; class GrallocImage; class SurfaceTextureImage; +class MacIOSurfaceImage; class EGLImageImage; } // namespace layers @@ -58,7 +59,8 @@ class GLBlitHelper final ConvertGralloc, ConvertPlanarYCbCr, ConvertSurfaceTexture, - ConvertEGLImage + ConvertEGLImage, + ConvertMacIOSurfaceImage }; // The GLContext is the sole owner of the GLBlitHelper. GLContext* mGL; @@ -77,8 +79,10 @@ class GLBlitHelper final // Data for image blit path GLuint mTexExternalBlit_FragShader; GLuint mTexYUVPlanarBlit_FragShader; + GLuint mTexNV12PlanarBlit_FragShader; GLuint mTexExternalBlit_Program; GLuint mTexYUVPlanarBlit_Program; + GLuint mTexNV12PlanarBlit_Program; GLuint mFBO; GLuint mSrcTexY; GLuint mSrcTexCb; @@ -111,6 +115,9 @@ class GLBlitHelper final bool BlitSurfaceTextureImage(layers::SurfaceTextureImage* stImage); bool BlitEGLImageImage(layers::EGLImageImage* eglImage); #endif +#ifdef XP_MACOSX + bool BlitMacIOSurfaceImage(layers::MacIOSurfaceImage* ioImage); +#endif explicit GLBlitHelper(GLContext* gl); diff --git a/gfx/layers/Compositor.cpp b/gfx/layers/Compositor.cpp index a974b2c531..fb2edef94f 100644 --- a/gfx/layers/Compositor.cpp +++ b/gfx/layers/Compositor.cpp @@ -200,15 +200,8 @@ Compositor::FillRect(const gfx::Rect& aRect, const gfx::Color& aColor, static float WrapTexCoord(float v) { - // fmodf gives negative results for negative numbers; - // that is, fmodf(0.75, 1.0) == 0.75, but - // fmodf(-0.75, 1.0) == -0.75. For the negative case, - // the result we need is 0.25, so we add 1.0f. - if (v < 0.0f) { - return 1.0f + fmodf(v, 1.0f); - } - - return fmodf(v, 1.0f); + // This should return values in range [0, 1.0) + return v - floorf(v); } static void diff --git a/gfx/layers/CopyableCanvasLayer.cpp b/gfx/layers/CopyableCanvasLayer.cpp index 0fad0aa366..0ec37f6f2b 100644 --- a/gfx/layers/CopyableCanvasLayer.cpp +++ b/gfx/layers/CopyableCanvasLayer.cpp @@ -61,6 +61,7 @@ CopyableCanvasLayer::Initialize(const Data& aData) gfx::IntSize size(aData.mSize.width, aData.mSize.height); mGLFrontbuffer = SharedSurface_Basic::Wrap(aData.mGLContext, size, aData.mHasAlpha, aData.mFrontbufferGLTex); + mBufferProvider = aData.mBufferProvider; } } else if (aData.mBufferProvider) { mBufferProvider = aData.mBufferProvider; @@ -85,7 +86,7 @@ CopyableCanvasLayer::UpdateTarget(DrawTarget* aDestTarget) { if (mAsyncRenderer) { mSurface = mAsyncRenderer->GetSurface(); - } else if (mBufferProvider) { + } else if (!mGLFrontbuffer && mBufferProvider) { mSurface = mBufferProvider->GetSnapshot(); } @@ -100,7 +101,7 @@ CopyableCanvasLayer::UpdateTarget(DrawTarget* aDestTarget) return; } - if (mBufferProvider || mAsyncRenderer) { + if ((!mGLFrontbuffer && mBufferProvider) || mAsyncRenderer) { return; } diff --git a/gfx/layers/CopyableCanvasLayer.h b/gfx/layers/CopyableCanvasLayer.h index 58fd650677..7e188bbdb3 100644 --- a/gfx/layers/CopyableCanvasLayer.h +++ b/gfx/layers/CopyableCanvasLayer.h @@ -24,7 +24,7 @@ namespace mozilla { namespace gl { class SharedSurface; -} +} // namespace gl namespace layers { diff --git a/gfx/layers/D3D11ShareHandleImage.cpp b/gfx/layers/D3D11ShareHandleImage.cpp index 550f7208f0..4dc4dc5e70 100644 --- a/gfx/layers/D3D11ShareHandleImage.cpp +++ b/gfx/layers/D3D11ShareHandleImage.cpp @@ -16,20 +16,19 @@ namespace mozilla { namespace layers { -HRESULT -D3D11ShareHandleImage::SetData(const Data& aData) +D3D11ShareHandleImage::D3D11ShareHandleImage(const gfx::IntSize& aSize, + const gfx::IntRect& aRect) + : Image(nullptr, ImageFormat::D3D11_SHARE_HANDLE_TEXTURE), + mSize(aSize), + mPictureRect(aRect) { - mPictureRect = aData.mRegion; - mSize = aData.mSize; +} - mTextureClient = - aData.mAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8A8, - mSize); - if (!mTextureClient) { - return E_FAIL; - } - - return S_OK; +bool +D3D11ShareHandleImage::AllocateTexture(D3D11RecycleAllocator* aAllocator) +{ + mTextureClient = aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8A8, mSize); + return !!mTextureClient; } gfx::IntSize diff --git a/gfx/layers/D3D11ShareHandleImage.h b/gfx/layers/D3D11ShareHandleImage.h index 93836705c2..f66e8a018c 100644 --- a/gfx/layers/D3D11ShareHandleImage.h +++ b/gfx/layers/D3D11ShareHandleImage.h @@ -45,40 +45,22 @@ protected: // passed into SetData(), so that it can be accessed from other D3D devices. // This class also manages the synchronization of the copy, to ensure the // resource is ready to use. -class D3D11ShareHandleImage : public Image { +class D3D11ShareHandleImage final : public Image { public: + D3D11ShareHandleImage(const gfx::IntSize& aSize, + const gfx::IntRect& aRect); + ~D3D11ShareHandleImage() override {} - struct Data { - Data(D3D11RecycleAllocator* aAllocator, - const gfx::IntSize& aSize, - const gfx::IntRect& aRegion) - : mAllocator(aAllocator) - , mSize(aSize) - , mRegion(aRegion) {} - RefPtr mAllocator; - gfx::IntSize mSize; - gfx::IntRect mRegion; - }; - - D3D11ShareHandleImage() : Image(NULL, ImageFormat::D3D11_SHARE_HANDLE_TEXTURE), mSize(0, 0) {} - virtual ~D3D11ShareHandleImage() {} - - // Copies the surface into a sharable texture's surface, and initializes - // the image. - HRESULT SetData(const Data& aData); + bool AllocateTexture(D3D11RecycleAllocator* aAllocator); gfx::IntSize GetSize() override; - virtual already_AddRefed GetAsSourceSurface() override; - virtual TextureClient* GetTextureClient(CompositableClient* aClient) override; + virtual gfx::IntRect GetPictureRect() override { return mPictureRect; } ID3D11Texture2D* GetTexture() const; - virtual gfx::IntRect GetPictureRect() override { return mPictureRect; } - private: - gfx::IntSize mSize; gfx::IntRect mPictureRect; RefPtr mTextureClient; diff --git a/gfx/layers/D3D9SurfaceImage.cpp b/gfx/layers/D3D9SurfaceImage.cpp index d8ab4e58a3..84bac69f16 100644 --- a/gfx/layers/D3D9SurfaceImage.cpp +++ b/gfx/layers/D3D9SurfaceImage.cpp @@ -15,10 +15,11 @@ namespace mozilla { namespace layers { -D3D9SurfaceImage::D3D9SurfaceImage() +D3D9SurfaceImage::D3D9SurfaceImage(bool aIsFirstFrame) : Image(nullptr, ImageFormat::D3D9_RGB32_TEXTURE) , mSize(0, 0) , mValid(false) + , mIsFirstFrame(aIsFirstFrame) {} D3D9SurfaceImage::~D3D9SurfaceImage() @@ -26,11 +27,13 @@ D3D9SurfaceImage::~D3D9SurfaceImage() } HRESULT -D3D9SurfaceImage::SetData(const Data& aData) +D3D9SurfaceImage::AllocateAndCopy(D3D9RecycleAllocator* aAllocator, + IDirect3DSurface9* aSurface, + const gfx::IntRect& aRegion) { - NS_ENSURE_TRUE(aData.mSurface, E_POINTER); + NS_ENSURE_TRUE(aSurface, E_POINTER); HRESULT hr; - RefPtr surface = aData.mSurface; + RefPtr surface = aSurface; RefPtr device; hr = surface->GetDevice(getter_AddRefs(device)); @@ -53,10 +56,8 @@ D3D9SurfaceImage::SetData(const Data& aData) // DXVA surfaces aren't created sharable, so we need to copy the surface // to a sharable texture to that it's accessible to the layer manager's // device. - const gfx::IntRect& region = aData.mRegion; RefPtr textureClient = - aData.mAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8X8, - region.Size()); + aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8X8, aRegion.Size()); if (!textureClient) { return E_FAIL; } @@ -67,7 +68,7 @@ D3D9SurfaceImage::SetData(const Data& aData) return E_FAIL; } - RECT src = { region.x, region.y, region.x+region.width, region.y+region.height }; + RECT src = { aRegion.x, aRegion.y, aRegion.x+aRegion.width, aRegion.y+aRegion.height }; hr = device->StretchRect(surface, &src, textureSurface, nullptr, D3DTEXF_NONE); NS_ENSURE_TRUE(SUCCEEDED(hr), hr); @@ -81,9 +82,8 @@ D3D9SurfaceImage::SetData(const Data& aData) NS_ENSURE_TRUE(SUCCEEDED(hr), hr); mTextureClient = textureClient; - mSize = region.Size(); + mSize = aRegion.Size(); mQuery = query; - return S_OK; } @@ -103,7 +103,7 @@ D3D9SurfaceImage::EnsureSynchronized() return; } int iterations = 0; - while (iterations < 10) { + while (iterations < (mIsFirstFrame ? 100 : 10)) { HRESULT hr = query->GetData(nullptr, 0, D3DGETDATA_FLUSH); if (hr == S_FALSE) { Sleep(1); diff --git a/gfx/layers/D3D9SurfaceImage.h b/gfx/layers/D3D9SurfaceImage.h index fe3fe09809..b5a38447fc 100644 --- a/gfx/layers/D3D9SurfaceImage.h +++ b/gfx/layers/D3D9SurfaceImage.h @@ -47,27 +47,12 @@ protected: // resource is ready to use. class D3D9SurfaceImage : public Image { public: - - struct Data { - Data(IDirect3DSurface9* aSurface, - const gfx::IntRect& aRegion, - D3D9RecycleAllocator* aAllocator) - : mSurface(aSurface) - , mRegion(aRegion) - , mAllocator(aAllocator) - {} - - RefPtr mSurface; - gfx::IntRect mRegion; - RefPtr mAllocator; - }; - - D3D9SurfaceImage(); + explicit D3D9SurfaceImage(bool aIsFirstFrame); virtual ~D3D9SurfaceImage(); - // Copies the surface into a sharable texture's surface, and initializes - // the image. - HRESULT SetData(const Data& aData); + HRESULT AllocateAndCopy(D3D9RecycleAllocator* aAllocator, + IDirect3DSurface9* aSurface, + const gfx::IntRect& aRegion); // Returns the description of the shared surface. const D3DSURFACE_DESC& GetDesc() const; @@ -90,6 +75,7 @@ private: RefPtr mQuery; RefPtr mTextureClient; bool mValid; + bool mIsFirstFrame; }; } // namepace layers diff --git a/gfx/layers/GLImages.cpp b/gfx/layers/GLImages.cpp index c3d01029fd..0a2ed3ab8b 100644 --- a/gfx/layers/GLImages.cpp +++ b/gfx/layers/GLImages.cpp @@ -16,20 +16,32 @@ namespace layers { static RefPtr sSnapshotContext; +EGLImageImage::EGLImageImage(EGLImage aImage, EGLSync aSync, + const gfx::IntSize& aSize, const gl::OriginPos& aOrigin, + bool aOwns) + : GLImage(ImageFormat::EGLIMAGE), + mImage(aImage), + mSync(aSync), + mSize(aSize), + mPos(aOrigin), + mOwns(aOwns) +{ +} + EGLImageImage::~EGLImageImage() { - if (!mData.mOwns) { + if (!mOwns) { return; } - if (mData.mImage) { - sEGLLibrary.fDestroyImage(EGL_DISPLAY(), mData.mImage); - mData.mImage = nullptr; + if (mImage) { + sEGLLibrary.fDestroyImage(EGL_DISPLAY(), mImage); + mImage = nullptr; } - if (mData.mSync) { - sEGLLibrary.fDestroySync(EGL_DISPLAY(), mData.mSync); - mData.mSync = nullptr; + if (mSync) { + sEGLLibrary.fDestroySync(EGL_DISPLAY(), mSync); + mSync = nullptr; } } @@ -82,5 +94,17 @@ GLImage::GetAsSourceSurface() return source.forget(); } +#ifdef MOZ_WIDGET_ANDROID +SurfaceTextureImage::SurfaceTextureImage(gl::AndroidSurfaceTexture* aSurfTex, + const gfx::IntSize& aSize, + gl::OriginPos aOriginPos) + : GLImage(ImageFormat::SURFACE_TEXTURE), + mSurfaceTexture(aSurfTex), + mSize(aSize), + mOriginPos(aOriginPos) +{ +} +#endif + } // namespace layers } // namespace mozilla diff --git a/gfx/layers/GLImages.h b/gfx/layers/GLImages.h index 2e887b8c1b..d69dd00018 100644 --- a/gfx/layers/GLImages.h +++ b/gfx/layers/GLImages.h @@ -28,52 +28,60 @@ public: class EGLImageImage : public GLImage { public: - struct Data { - EGLImage mImage; - EGLSync mSync; - gfx::IntSize mSize; - gl::OriginPos mOriginPos; - bool mOwns; + EGLImageImage(EGLImage aImage, EGLSync aSync, + const gfx::IntSize& aSize, const gl::OriginPos& aOrigin, + bool aOwns); - Data() : mImage(nullptr), mSync(nullptr), mSize(0, 0), - mOriginPos(gl::OriginPos::TopLeft), mOwns(false) - { - } - }; + gfx::IntSize GetSize() override { return mSize; } + gl::OriginPos GetOriginPos() const { + return mPos; + } + EGLImage GetImage() const { + return mImage; + } + EGLSync GetSync() const { + return mSync; + } - void SetData(const Data& aData) { mData = aData; } - const Data* GetData() { return &mData; } - - gfx::IntSize GetSize() { return mData.mSize; } - - EGLImageImage() : GLImage(ImageFormat::EGLIMAGE) {} + EGLImageImage* AsEGLImageImage() override { + return this; + } protected: virtual ~EGLImageImage(); private: - Data mData; + EGLImage mImage; + EGLSync mSync; + gfx::IntSize mSize; + gl::OriginPos mPos; + bool mOwns; }; #ifdef MOZ_WIDGET_ANDROID class SurfaceTextureImage : public GLImage { public: - struct Data { - mozilla::gl::AndroidSurfaceTexture* mSurfTex; - gfx::IntSize mSize; - gl::OriginPos mOriginPos; - }; + SurfaceTextureImage(gl::AndroidSurfaceTexture* aSurfTex, + const gfx::IntSize& aSize, + gl::OriginPos aOriginPos); - void SetData(const Data& aData) { mData = aData; } - const Data* GetData() { return &mData; } + gfx::IntSize GetSize() override { return mSize; } + gl::AndroidSurfaceTexture* GetSurfaceTexture() const { + return mSurfaceTexture; + } + gl::OriginPos GetOriginPos() const { + return mOriginPos; + } - gfx::IntSize GetSize() { return mData.mSize; } - - SurfaceTextureImage() : GLImage(ImageFormat::SURFACE_TEXTURE) {} + SurfaceTextureImage* AsSurfaceTextureImage() override { + return this; + } private: - Data mData; + gl::AndroidSurfaceTexture* mSurfaceTexture; + gfx::IntSize mSize; + gl::OriginPos mOriginPos; }; #endif // MOZ_WIDGET_ANDROID diff --git a/gfx/layers/GrallocImages.cpp b/gfx/layers/GrallocImages.cpp index e3427001cd..29778ad520 100644 --- a/gfx/layers/GrallocImages.cpp +++ b/gfx/layers/GrallocImages.cpp @@ -147,11 +147,12 @@ GrallocImage::SetData(const Data& aData) return true; } -bool GrallocImage::SetData(const GrallocData& aData) +void +GrallocImage::SetData(TextureClient* aGraphicBuffer, const gfx::IntSize& aSize) { - mTextureClient = static_cast(aData.mGraphicBuffer.get()); - mSize = aData.mPicSize; - return true; + MOZ_ASSERT(aGraphicBuffer->AsGrallocTextureClientOGL()); + mTextureClient = aGraphicBuffer->AsGrallocTextureClientOGL(); + mSize = aSize; } /** diff --git a/gfx/layers/GrallocImages.h b/gfx/layers/GrallocImages.h index b7fe457543..82553b1b66 100644 --- a/gfx/layers/GrallocImages.h +++ b/gfx/layers/GrallocImages.h @@ -53,11 +53,6 @@ class GrallocImage : public RecyclingPlanarYCbCrImage typedef PlanarYCbCrData Data; static int32_t sColorIdMap[]; public: - struct GrallocData { - RefPtr mGraphicBuffer; - gfx::IntSize mPicSize; - }; - GrallocImage(); virtual ~GrallocImage(); @@ -72,7 +67,7 @@ public: * Share the SurfaceDescriptor without making the copy, in order * to support functioning in all different layer managers. */ - virtual bool SetData(const GrallocData& aData); + void SetData(TextureClient* aGraphicBuffer, const gfx::IntSize& aSize); // From [android 4.0.4]/hardware/msm7k/libgralloc-qsd8k/gralloc_priv.h enum { diff --git a/gfx/layers/ImageContainer.cpp b/gfx/layers/ImageContainer.cpp index dda00621a4..0b26f2eb6e 100644 --- a/gfx/layers/ImageContainer.cpp +++ b/gfx/layers/ImageContainer.cpp @@ -17,6 +17,8 @@ #include "mozilla/layers/PImageContainerChild.h" #include "mozilla/layers/ImageClient.h" // for ImageClient #include "mozilla/layers/LayersMessages.h" +#include "mozilla/layers/SharedPlanarYCbCrImage.h" +#include "mozilla/layers/SharedRGBImage.h" #include "nsISupportsUtils.h" // for NS_IF_ADDREF #include "YCbCrUtils.h" // for YCbCr conversions #ifdef MOZ_WIDGET_GONK @@ -31,14 +33,11 @@ #ifdef XP_MACOSX #include "mozilla/gfx/QuartzSupport.h" -#include "MacIOSurfaceImage.h" #endif #ifdef XP_WIN #include "gfxWindowsPlatform.h" #include -#include "D3D9SurfaceImage.h" -#include "D3D11ShareHandleImage.h" #endif namespace mozilla { @@ -52,63 +51,10 @@ Atomic Image::sSerialCounter(0); Atomic ImageContainer::sGenerationCounter(0); -already_AddRefed -ImageFactory::CreateImage(ImageFormat aFormat, - const gfx::IntSize &, - BufferRecycleBin *aRecycleBin) +RefPtr +ImageFactory::CreatePlanarYCbCrImage(const gfx::IntSize& aScaleHint, BufferRecycleBin *aRecycleBin) { - RefPtr img; -#ifdef MOZ_WIDGET_GONK - if (aFormat == ImageFormat::GRALLOC_PLANAR_YCBCR) { - img = new GrallocImage(); - return img.forget(); - } - if (aFormat == ImageFormat::OVERLAY_IMAGE) { - img = new OverlayImage(); - return img.forget(); - } -#endif -#if defined(MOZ_WIDGET_GONK) && defined(MOZ_B2G_CAMERA) && defined(MOZ_WEBRTC) - if (aFormat == ImageFormat::GONK_CAMERA_IMAGE) { - img = new GonkCameraImage(); - return img.forget(); - } -#endif - if (aFormat == ImageFormat::PLANAR_YCBCR) { - img = new RecyclingPlanarYCbCrImage(aRecycleBin); - return img.forget(); - } - if (aFormat == ImageFormat::CAIRO_SURFACE) { - img = new CairoImage(); - return img.forget(); - } -#ifdef MOZ_WIDGET_ANDROID - if (aFormat == ImageFormat::SURFACE_TEXTURE) { - img = new SurfaceTextureImage(); - return img.forget(); - } -#endif - if (aFormat == ImageFormat::EGLIMAGE) { - img = new EGLImageImage(); - return img.forget(); - } -#ifdef XP_MACOSX - if (aFormat == ImageFormat::MAC_IOSURFACE) { - img = new MacIOSurfaceImage(); - return img.forget(); - } -#endif -#ifdef XP_WIN - if (aFormat == ImageFormat::D3D11_SHARE_HANDLE_TEXTURE) { - img = new D3D11ShareHandleImage(); - return img.forget(); - } - if (aFormat == ImageFormat::D3D9_RGB32_TEXTURE) { - img = new D3D9SurfaceImage(); - return img.forget(); - } -#endif - return nullptr; + return new RecyclingPlanarYCbCrImage(aRecycleBin); } BufferRecycleBin::BufferRecycleBin() @@ -209,31 +155,42 @@ ImageContainer::~ImageContainer() } } -already_AddRefed -ImageContainer::CreateImage(ImageFormat aFormat) +RefPtr +ImageContainer::CreatePlanarYCbCrImage() { ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if (mImageClient && mImageClient->AsImageClientSingle()) { + return new SharedPlanarYCbCrImage(mImageClient); + } + return mImageFactory->CreatePlanarYCbCrImage(mScaleHint, mRecycleBin); +} + +RefPtr +ImageContainer::CreateSharedRGBImage() +{ + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if (!mImageClient || !mImageClient->AsImageClientSingle()) { + return nullptr; + } + return new SharedRGBImage(mImageClient); +} #ifdef MOZ_WIDGET_GONK - if (aFormat == ImageFormat::OVERLAY_IMAGE) { - if (mImageClient && mImageClient->GetTextureInfo().mCompositableType != CompositableType::IMAGE_OVERLAY) { - // If this ImageContainer is async but the image type mismatch, fix it here - if (ImageBridgeChild::IsCreated()) { - ImageBridgeChild::DispatchReleaseImageClient(mImageClient); - mImageClient = ImageBridgeChild::GetSingleton()->CreateImageClient( - CompositableType::IMAGE_OVERLAY, this).take(); - } +RefPtr +ImageContainer::CreateOverlayImage() +{ + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if (mImageClient && mImageClient->GetTextureInfo().mCompositableType != CompositableType::IMAGE_OVERLAY) { + // If this ImageContainer is async but the image type mismatch, fix it here + if (ImageBridgeChild::IsCreated()) { + ImageBridgeChild::DispatchReleaseImageClient(mImageClient); + mImageClient = ImageBridgeChild::GetSingleton()->CreateImageClient( + CompositableType::IMAGE_OVERLAY, this).take(); } } -#endif - if (mImageClient) { - RefPtr img = mImageClient->CreateImage(aFormat); - if (img) { - return img.forget(); - } - } - return mImageFactory->CreateImage(aFormat, mScaleHint, mRecycleBin); + return new OverlayImage(); } +#endif void ImageContainer::SetCurrentImageInternal(const nsTArray& aImages) @@ -603,8 +560,10 @@ PlanarYCbCrImage::GetAsSourceSurface() return surface.forget(); } -CairoImage::CairoImage() - : Image(nullptr, ImageFormat::CAIRO_SURFACE) +CairoImage::CairoImage(const gfx::IntSize& aSize, gfx::SourceSurface* aSourceSurface) + : Image(nullptr, ImageFormat::CAIRO_SURFACE), + mSize(aSize), + mSourceSurface(aSourceSurface) {} CairoImage::~CairoImage() diff --git a/gfx/layers/ImageContainer.h b/gfx/layers/ImageContainer.h index 4b92af1b66..fb08246346 100644 --- a/gfx/layers/ImageContainer.h +++ b/gfx/layers/ImageContainer.h @@ -103,6 +103,7 @@ class ImageCompositeNotification; class ImageContainerChild; class PImageContainerChild; class SharedPlanarYCbCrImage; +class PlanarYCbCrImage; class TextureClient; class CompositableClient; class GrallocImage; @@ -115,6 +116,17 @@ protected: ImageBackendData() {} }; +/* Forward declarations for Image derivatives. */ +class EGLImageImage; +class SharedRGBImage; +#ifdef MOZ_WIDGET_ANDROID +class SurfaceTextureImage; +#elif defined(XP_MACOSX) +class MacIOSurfaceImage; +#elif defined(MOZ_WIDGET_GONK) +class OverlayImage; +#endif + /** * A class representing a buffer of pixel data. The data can be in one * of various formats including YCbCr. @@ -161,11 +173,21 @@ public: virtual uint8_t* GetBuffer() { return nullptr; } /** - * For use with the CompositableClient only (so that the later can - * synchronize the TextureClient with the TextureHost). - */ + * For use with the CompositableClient only (so that the later can + * synchronize the TextureClient with the TextureHost). + */ virtual TextureClient* GetTextureClient(CompositableClient* aClient) { return nullptr; } + /* Access to derived classes. */ + virtual EGLImageImage* AsEGLImageImage() { return nullptr; } +#ifdef MOZ_WIDGET_ANDROID + virtual SurfaceTextureImage* AsSurfaceTextureImage() { return nullptr; } +#endif +#ifdef XP_MACOSX + virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; } +#endif + virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; } + protected: Image(void* aImplData, ImageFormat aFormat) : mImplData(aImplData), @@ -252,10 +274,9 @@ protected: ImageFactory() {} virtual ~ImageFactory() {} - virtual already_AddRefed CreateImage(ImageFormat aFormat, - const gfx::IntSize &aScaleHint, - BufferRecycleBin *aRecycleBin); - + virtual RefPtr CreatePlanarYCbCrImage( + const gfx::IntSize& aScaleHint, + BufferRecycleBin *aRecycleBin); }; /** @@ -292,16 +313,14 @@ public: typedef uint32_t FrameID; typedef uint32_t ProducerID; + RefPtr CreatePlanarYCbCrImage(); - /** - * Create an Image in one of the given formats. - * Picks the "best" format from the list and creates an Image of that - * format. - * Returns null if this backend does not support any of the formats. - * Can be called on any thread. This method takes mReentrantMonitor - * when accessing thread-shared state. - */ - B2G_ACL_EXPORT already_AddRefed CreateImage(ImageFormat aFormat); + // Factory methods for shared image types. + RefPtr CreateSharedRGBImage(); + +#ifdef MOZ_WIDGET_GONK + RefPtr CreateOverlayImage(); +#endif struct NonOwningImage { explicit NonOwningImage(Image* aImage = nullptr, @@ -704,6 +723,8 @@ public: virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const = 0; + PlanarYCbCrImage* AsPlanarYCbCrImage() { return this; } + protected: already_AddRefed GetAsSourceSurface(); @@ -748,22 +769,6 @@ protected: */ class CairoImage final : public Image { public: - struct Data { - gfx::IntSize mSize; - RefPtr mSourceSurface; - }; - - /** - * This can only be called on the main thread. It may add a reference - * to the surface (which will eventually be released on the main thread). - * The surface must not be modified after this call!!! - */ - void SetData(const Data& aData) - { - mSize = aData.mSize; - mSourceSurface = aData.mSourceSurface; - } - virtual already_AddRefed GetAsSourceSurface() override { RefPtr surface(mSourceSurface); @@ -774,11 +779,11 @@ public: virtual gfx::IntSize GetSize() override { return mSize; } - CairoImage(); + CairoImage(const gfx::IntSize& aSize, gfx::SourceSurface* aSourceSurface); ~CairoImage(); +private: gfx::IntSize mSize; - nsCountedRef mSourceSurface; nsDataHashtable > mTextureClients; }; diff --git a/gfx/layers/MacIOSurfaceImage.h b/gfx/layers/MacIOSurfaceImage.h index 31d33a2c18..0c131925ab 100644 --- a/gfx/layers/MacIOSurfaceImage.h +++ b/gfx/layers/MacIOSurfaceImage.h @@ -17,7 +17,11 @@ namespace layers { class MacIOSurfaceImage : public Image { public: - void SetSurface(MacIOSurface* aSurface) { mSurface = aSurface; } + explicit MacIOSurfaceImage(MacIOSurface* aSurface) + : Image(nullptr, ImageFormat::MAC_IOSURFACE), + mSurface(aSurface) + {} + MacIOSurface* GetSurface() { return mSurface; } gfx::IntSize GetSize() override { @@ -28,7 +32,9 @@ public: virtual TextureClient* GetTextureClient(CompositableClient* aClient) override; - MacIOSurfaceImage() : Image(nullptr, ImageFormat::MAC_IOSURFACE) {} + virtual MacIOSurfaceImage* AsMacIOSurfaceImage() override { + return this; + } private: RefPtr mSurface; diff --git a/gfx/layers/basic/BasicImages.cpp b/gfx/layers/basic/BasicImages.cpp index 39b554d7db..eda42930cc 100644 --- a/gfx/layers/basic/BasicImages.cpp +++ b/gfx/layers/basic/BasicImages.cpp @@ -79,17 +79,10 @@ class BasicImageFactory : public ImageFactory public: BasicImageFactory() {} - virtual already_AddRefed CreateImage(ImageFormat aFormat, - const gfx::IntSize &aScaleHint, - BufferRecycleBin *aRecycleBin) + virtual RefPtr + CreatePlanarYCbCrImage(const gfx::IntSize& aScaleHint, BufferRecycleBin* aRecycleBin) { - RefPtr image; - if (aFormat == ImageFormat::PLANAR_YCBCR) { - image = new BasicPlanarYCbCrImage(aScaleHint, gfxPlatform::GetPlatform()->GetOffscreenFormat(), aRecycleBin); - return image.forget(); - } - - return ImageFactory::CreateImage(aFormat, aScaleHint, aRecycleBin); + return new BasicPlanarYCbCrImage(aScaleHint, gfxPlatform::GetPlatform()->GetOffscreenFormat(), aRecycleBin); } }; diff --git a/gfx/layers/client/ImageClient.cpp b/gfx/layers/client/ImageClient.cpp index cc1ad1dbdd..c442932a40 100644 --- a/gfx/layers/client/ImageClient.cpp +++ b/gfx/layers/client/ImageClient.cpp @@ -21,8 +21,6 @@ #include "mozilla/layers/ISurfaceAllocator.h" #include "mozilla/layers/LayersSurfaces.h" // for SurfaceDescriptor, etc #include "mozilla/layers/ShadowLayers.h" // for ShadowLayerForwarder -#include "mozilla/layers/SharedPlanarYCbCrImage.h" -#include "mozilla/layers/SharedRGBImage.h" #include "mozilla/layers/TextureClient.h" // for TextureClient, etc #include "mozilla/layers/TextureClientOGL.h" // for SurfaceTextureClient #include "mozilla/mozalloc.h" // for operator delete, etc @@ -204,18 +202,17 @@ ImageClientSingle::UpdateImage(ImageContainer* aContainer, uint32_t aContentFlag gfx::IntSize size = image->GetSize(); if (image->GetFormat() == ImageFormat::EGLIMAGE) { - EGLImageImage* typedImage = static_cast(image); + EGLImageImage* typedImage = image->AsEGLImageImage(); texture = new EGLImageTextureClient(GetForwarder(), mTextureFlags, typedImage, size); #ifdef MOZ_WIDGET_ANDROID } else if (image->GetFormat() == ImageFormat::SURFACE_TEXTURE) { - SurfaceTextureImage* typedImage = static_cast(image); - const SurfaceTextureImage::Data* data = typedImage->GetData(); + SurfaceTextureImage* typedImage = image->AsSurfaceTextureImage(); texture = new SurfaceTextureClient(GetForwarder(), mTextureFlags, - data->mSurfTex, size, - data->mOriginPos); + typedImage->GetSurfaceTexture(), size, + typedImage->GetOriginPos()); #endif } else { MOZ_ASSERT(false, "Bad ImageFormat."); @@ -320,27 +317,6 @@ ImageClientBridge::UpdateImage(ImageContainer* aContainer, uint32_t aContentFlag return true; } -already_AddRefed -ImageClientSingle::CreateImage(ImageFormat aFormat) -{ - RefPtr img; - switch (aFormat) { - case ImageFormat::PLANAR_YCBCR: - img = new SharedPlanarYCbCrImage(this); - return img.forget(); - case ImageFormat::SHARED_RGB: - img = new SharedRGBImage(this); - return img.forget(); -#ifdef MOZ_WIDGET_GONK - case ImageFormat::GRALLOC_PLANAR_YCBCR: - img = new GrallocImage(); - return img.forget(); -#endif - default: - return nullptr; - } -} - #ifdef MOZ_WIDGET_GONK ImageClientOverlay::ImageClientOverlay(CompositableForwarder* aFwd, TextureFlags aFlags) @@ -375,20 +351,6 @@ ImageClientOverlay::UpdateImage(ImageContainer* aContainer, uint32_t aContentFla } return true; } - -already_AddRefed -ImageClientOverlay::CreateImage(ImageFormat aFormat) -{ - RefPtr img; - switch (aFormat) { - case ImageFormat::OVERLAY_IMAGE: - img = new OverlayImage(); - return img.forget(); - default: - return nullptr; - } -} - #endif } // namespace layers } // namespace mozilla diff --git a/gfx/layers/client/ImageClient.h b/gfx/layers/client/ImageClient.h index 3751c7338e..b9baacab23 100644 --- a/gfx/layers/client/ImageClient.h +++ b/gfx/layers/client/ImageClient.h @@ -29,6 +29,7 @@ class AsyncTransactionTracker; class Image; class ImageContainer; class ShadowableLayer; +class ImageClientSingle; /** * Image clients are used by basic image layers on the content thread, they @@ -56,8 +57,6 @@ public: */ virtual bool UpdateImage(ImageContainer* aContainer, uint32_t aContentFlags) = 0; - virtual already_AddRefed CreateImage(ImageFormat aFormat) = 0; - void SetLayer(ClientLayer* aLayer) { mLayer = aLayer; } ClientLayer* GetLayer() const { return mLayer; } @@ -72,6 +71,8 @@ public: void RemoveTextureWithWaiter(TextureClient* aTexture, AsyncTransactionWaiter* aAsyncTransactionWaiter = nullptr); + virtual ImageClientSingle* AsImageClientSingle() { return nullptr; } + protected: ImageClient(CompositableForwarder* aFwd, TextureFlags aFlags, CompositableType aType); @@ -99,10 +100,10 @@ public: virtual TextureInfo GetTextureInfo() const override; - virtual already_AddRefed CreateImage(ImageFormat aFormat) override; - virtual void FlushAllImages(AsyncTransactionWaiter* aAsyncTransactionWaiter) override; + ImageClientSingle* AsImageClientSingle() override { return this; } + protected: struct Buffer { RefPtr mTextureClient; @@ -135,12 +136,6 @@ public: MOZ_ASSERT(!aChild, "ImageClientBridge should not have IPDL actor"); } - virtual already_AddRefed CreateImage(ImageFormat aFormat) override - { - NS_WARNING("Should not create an image through an ImageClientBridge"); - return nullptr; - } - protected: uint64_t mAsyncContainerID; }; @@ -160,7 +155,6 @@ public: TextureFlags aFlags); virtual bool UpdateImage(ImageContainer* aContainer, uint32_t aContentFlags); - virtual already_AddRefed CreateImage(ImageFormat aFormat); TextureInfo GetTextureInfo() const override { return TextureInfo(CompositableType::IMAGE_OVERLAY); diff --git a/gfx/layers/client/TextureClient.h b/gfx/layers/client/TextureClient.h index 469efed1d8..0282498040 100644 --- a/gfx/layers/client/TextureClient.h +++ b/gfx/layers/client/TextureClient.h @@ -56,6 +56,7 @@ class TextureClientRecycleAllocator; class TextureClientPool; #endif class KeepAlive; +class GrallocTextureClientOGL; /** * TextureClient is the abstraction that allows us to share data between the @@ -234,6 +235,7 @@ public: } virtual TextureClientYCbCr* AsTextureClientYCbCr() { return nullptr; } + virtual GrallocTextureClientOGL* AsGrallocTextureClientOGL() { return nullptr; } /** * Locks the shared data, allowing the caller to get access to it. diff --git a/gfx/layers/ipc/SharedRGBImage.cpp b/gfx/layers/ipc/SharedRGBImage.cpp index 4061482f72..1a495ca3e8 100644 --- a/gfx/layers/ipc/SharedRGBImage.cpp +++ b/gfx/layers/ipc/SharedRGBImage.cpp @@ -41,19 +41,16 @@ CreateSharedRGBImage(ImageContainer *aImageContainer, return nullptr; } - RefPtr image = aImageContainer->CreateImage(ImageFormat::SHARED_RGB); - - if (!image) { + RefPtr rgbImage = aImageContainer->CreateSharedRGBImage(); + if (!rgbImage) { NS_WARNING("Failed to create SharedRGBImage"); return nullptr; } - - RefPtr rgbImage = static_cast(image.get()); if (!rgbImage->Allocate(aSize, gfx::ImageFormatToSurfaceFormat(aImageFormat))) { NS_WARNING("Failed to allocate a shared image"); return nullptr; } - return image.forget(); + return rgbImage.forget(); } SharedRGBImage::SharedRGBImage(ImageClient* aCompositable) diff --git a/gfx/layers/opengl/GrallocTextureClient.h b/gfx/layers/opengl/GrallocTextureClient.h index 2c5ae47bc6..1281489fd1 100644 --- a/gfx/layers/opengl/GrallocTextureClient.h +++ b/gfx/layers/opengl/GrallocTextureClient.h @@ -64,6 +64,10 @@ public: virtual void WaitForBufferOwnership(bool aWaitReleaseFence = true) override; + GrallocTextureClientOGL* AsGrallocTextureClientOGL() override { + return this; + } + void SetTextureFlags(TextureFlags aFlags) { AddFlags(aFlags); } gfx::IntSize GetSize() const override { return mSize; } diff --git a/gfx/layers/opengl/TextureClientOGL.cpp b/gfx/layers/opengl/TextureClientOGL.cpp index fe23227255..622454bd9b 100644 --- a/gfx/layers/opengl/TextureClientOGL.cpp +++ b/gfx/layers/opengl/TextureClientOGL.cpp @@ -34,7 +34,7 @@ EGLImageTextureClient::EGLImageTextureClient(ISurfaceAllocator* aAllocator, AddFlags(TextureFlags::DEALLOCATE_CLIENT); - if (aImage->GetData()->mOriginPos == gl::OriginPos::BottomLeft) { + if (aImage->GetOriginPos() == gl::OriginPos::BottomLeft) { AddFlags(TextureFlags::ORIGIN_BOTTOM_LEFT); } } @@ -45,10 +45,11 @@ EGLImageTextureClient::ToSurfaceDescriptor(SurfaceDescriptor& aOutDescriptor) MOZ_ASSERT(IsValid()); MOZ_ASSERT(IsAllocated()); - const EGLImageImage::Data* data = mImage->GetData(); const bool hasAlpha = true; - aOutDescriptor = EGLImageDescriptor((uintptr_t)data->mImage, (uintptr_t)data->mSync, - mSize, hasAlpha); + aOutDescriptor = + EGLImageDescriptor((uintptr_t)mImage->GetImage(), + (uintptr_t)mImage->GetSync(), + mImage->GetSize(), hasAlpha); return true; } diff --git a/image/RasterImage.cpp b/image/RasterImage.cpp index c7042c0b84..99eb61ff16 100644 --- a/image/RasterImage.cpp +++ b/image/RasterImage.cpp @@ -638,17 +638,11 @@ RasterImage::GetCurrentImage(ImageContainer* aContainer, uint32_t aFlags) return MakePair(drawResult, RefPtr()); } - CairoImage::Data cairoData; - GetWidth(&cairoData.mSize.width); - GetHeight(&cairoData.mSize.height); - cairoData.mSourceSurface = surface; - - RefPtr image = - aContainer->CreateImage(ImageFormat::CAIRO_SURFACE); - MOZ_ASSERT(image); - - static_cast(image.get())->SetData(cairoData); + IntSize size; + GetWidth(&size.width); + GetHeight(&size.height); + RefPtr image = new layers::CairoImage(size, surface); return MakePair(drawResult, Move(image)); } diff --git a/layout/base/FrameLayerBuilder.cpp b/layout/base/FrameLayerBuilder.cpp index e373bacdd2..114286bcb3 100644 --- a/layout/base/FrameLayerBuilder.cpp +++ b/layout/base/FrameLayerBuilder.cpp @@ -6141,13 +6141,8 @@ ContainerState::CreateMaskLayer(Layer *aLayer, // build the image and container container = aLayer->Manager()->CreateImageContainer(); NS_ASSERTION(container, "Could not create image container for mask layer."); - RefPtr image = container->CreateImage(ImageFormat::CAIRO_SURFACE); - NS_ASSERTION(image, "Could not create image container for mask layer."); - CairoImage::Data data; - data.mSize = surfaceSizeInt; - data.mSourceSurface = surface; - static_cast(image.get())->SetData(data); + RefPtr image = new CairoImage(surfaceSizeInt, surface); container->SetCurrentImageInTransaction(image); GetMaskLayerImageCache()->PutImage(newKey.forget(), container); diff --git a/layout/build/nsLayoutModule.cpp b/layout/build/nsLayoutModule.cpp index 038e05d640..ce7e5de03e 100644 --- a/layout/build/nsLayoutModule.cpp +++ b/layout/build/nsLayoutModule.cpp @@ -1271,7 +1271,7 @@ static const mozilla::Module::ContractIDEntry kLayoutContracts[] = { { NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "fake", &kNS_FAKE_SPEECH_RECOGNITION_SERVICE_CID }, #endif #ifdef MOZ_WEBSPEECH_POCKETSPHINX - { NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "pocketsphinx", &kNS_POCKETSPHINX_SPEECH_RECOGNITION_SERVICE_CID }, + { NS_SPEECH_RECOGNITION_SERVICE_CONTRACTID_PREFIX "pocketsphinx-en-US", &kNS_POCKETSPHINX_SPEECH_RECOGNITION_SERVICE_CID }, #endif #ifdef MOZ_WEBSPEECH { NS_SYNTHVOICEREGISTRY_CONTRACTID, &kNS_SYNTHVOICEREGISTRY_CID }, diff --git a/media/gmp-clearkey/0.1/AnnexB.cpp b/media/gmp-clearkey/0.1/AnnexB.cpp index a22d62a8e4..8b9b9ae121 100644 --- a/media/gmp-clearkey/0.1/AnnexB.cpp +++ b/media/gmp-clearkey/0.1/AnnexB.cpp @@ -1,9 +1,21 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "AnnexB.h" -#include "mozilla/Endian.h" +#include "Endian.h" using mozilla::BigEndian; diff --git a/media/gmp-clearkey/0.1/AnnexB.h b/media/gmp-clearkey/0.1/AnnexB.h index 297df06e28..81ca87e0cf 100644 --- a/media/gmp-clearkey/0.1/AnnexB.h +++ b/media/gmp-clearkey/0.1/AnnexB.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __AnnexB_h__ #define __AnnexB_h__ diff --git a/media/gmp-clearkey/0.1/ArrayUtils.h b/media/gmp-clearkey/0.1/ArrayUtils.h new file mode 100644 index 0000000000..c5e17689e7 --- /dev/null +++ b/media/gmp-clearkey/0.1/ArrayUtils.h @@ -0,0 +1,23 @@ +/* +* Copyright 2015, Mozilla Foundation and contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef __ArrayUtils_h__ +#define __ArrayUtils_h__ + +#define MOZ_ARRAY_LENGTH(array_) \ + (sizeof(array_)/sizeof(array_[0])) + +#endif diff --git a/media/gmp-clearkey/0.1/AudioDecoder.cpp b/media/gmp-clearkey/0.1/AudioDecoder.cpp index c038bf9824..3421651340 100644 --- a/media/gmp-clearkey/0.1/AudioDecoder.cpp +++ b/media/gmp-clearkey/0.1/AudioDecoder.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2013, Mozilla Foundation and contributors + * Copyright 2015, Mozilla Foundation and contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,8 @@ AudioDecoder::AudioDecoder(GMPAudioHost *aHostAPI) , mNumInputTasks(0) , mHasShutdown(false) { + // We drop the ref in DecodingComplete(). + AddRef(); } AudioDecoder::~AudioDecoder() @@ -84,9 +86,9 @@ AudioDecoder::Decode(GMPAudioSamples* aInput) AutoLock lock(mMutex); mNumInputTasks++; } - mWorkerThread->Post(WrapTask(this, - &AudioDecoder::DecodeTask, - aInput)); + mWorkerThread->Post(WrapTaskRefCounted(this, + &AudioDecoder::DecodeTask, + aInput)); } void @@ -258,8 +260,8 @@ AudioDecoder::Drain() return; } EnsureWorker(); - mWorkerThread->Post(WrapTask(this, - &AudioDecoder::DrainTask)); + mWorkerThread->Post(WrapTaskRefCounted(this, + &AudioDecoder::DrainTask)); } void @@ -270,24 +272,19 @@ AudioDecoder::DecodingComplete() } mHasShutdown = true; - // Worker thread might have dispatched more tasks to the main thread that need this object. - // Append another task to delete |this|. - GetPlatform()->runonmainthread(WrapTask(this, &AudioDecoder::Destroy)); + // Release the reference we added in the constructor. There may be + // WrapRefCounted tasks that also hold references to us, and keep + // us alive a little longer. + Release(); } void -AudioDecoder::Destroy() -{ - delete this; -} - -void -AudioDecoder::MaybeRunOnMainThread(gmp_task_args_base* aTask) +AudioDecoder::MaybeRunOnMainThread(GMPTask* aTask) { class MaybeRunTask : public GMPTask { public: - MaybeRunTask(AudioDecoder* aDecoder, gmp_task_args_base* aTask) + MaybeRunTask(AudioDecoder* aDecoder, GMPTask* aTask) : mDecoder(aDecoder), mTask(aTask) { } @@ -307,8 +304,8 @@ AudioDecoder::MaybeRunOnMainThread(gmp_task_args_base* aTask) } private: - AudioDecoder* mDecoder; - gmp_task_args_base* mTask; + RefPtr mDecoder; + GMPTask* mTask; }; GetPlatform()->runonmainthread(new MaybeRunTask(this, aTask)); diff --git a/media/gmp-clearkey/0.1/AudioDecoder.h b/media/gmp-clearkey/0.1/AudioDecoder.h index 4182d50fd6..98915bb7a7 100644 --- a/media/gmp-clearkey/0.1/AudioDecoder.h +++ b/media/gmp-clearkey/0.1/AudioDecoder.h @@ -1,5 +1,5 @@ /* - * Copyright 2013, Mozilla Foundation and contributors + * Copyright 2015, Mozilla Foundation and contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,16 +21,16 @@ #include "gmp-audio-host.h" #include "gmp-task-utils.h" #include "WMFAACDecoder.h" +#include "RefCounted.h" #include "mfobjects.h" class AudioDecoder : public GMPAudioDecoder + , public RefCounted { public: AudioDecoder(GMPAudioHost *aHostAPI); - virtual ~AudioDecoder(); - virtual void InitDecode(const GMPAudioCodec& aCodecSettings, GMPAudioDecoderCallback* aCallback) override; @@ -45,6 +45,7 @@ public: bool HasShutdown() { return mHasShutdown; } private: + virtual ~AudioDecoder(); void EnsureWorker(); @@ -56,8 +57,7 @@ private: HRESULT MFToGMPSample(IMFSample* aSample, GMPAudioSamples* aAudioFrame); - void MaybeRunOnMainThread(gmp_task_args_base* aTask); - void Destroy(); + void MaybeRunOnMainThread(GMPTask* aTask); GMPAudioHost *mHostAPI; // host-owned, invalid at DecodingComplete GMPAudioDecoderCallback* mCallback; // host-owned, invalid at DecodingComplete diff --git a/media/gmp-clearkey/0.1/ClearKeyBase64.cpp b/media/gmp-clearkey/0.1/ClearKeyBase64.cpp index 1afa4200ff..f0cbb22f73 100644 --- a/media/gmp-clearkey/0.1/ClearKeyBase64.cpp +++ b/media/gmp-clearkey/0.1/ClearKeyBase64.cpp @@ -1,13 +1,23 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public -* License, v. 2.0. If a copy of the MPL was not distributed with this -* file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "ClearKeyBase64.h" #include -#include "mozilla/ArrayUtils.h" - using namespace std; /** diff --git a/media/gmp-clearkey/0.1/ClearKeyBase64.h b/media/gmp-clearkey/0.1/ClearKeyBase64.h index 3e86d55427..8c424ad63a 100644 --- a/media/gmp-clearkey/0.1/ClearKeyBase64.h +++ b/media/gmp-clearkey/0.1/ClearKeyBase64.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public -* License, v. 2.0. If a copy of the MPL was not distributed with this -* file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyBase64_h__ #define __ClearKeyBase64_h__ diff --git a/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.cpp b/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.cpp index 76c0305189..81112576e0 100644 --- a/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.cpp +++ b/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.cpp @@ -1,19 +1,30 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include #include "ClearKeyDecryptionManager.h" #include "gmp-decryption.h" -#include "mozilla/Assertions.h" -#include "mozilla/Attributes.h" +#include class ClearKeyDecryptor : public RefCounted { public: - MOZ_IMPLICIT ClearKeyDecryptor(); + ClearKeyDecryptor(); void InitKey(const Key& aKey); bool HasKey() const { return !!mKey.size(); } @@ -84,7 +95,7 @@ ClearKeyDecryptionManager::HasKeyForKeyId(const KeyId& aKeyId) const const Key& ClearKeyDecryptionManager::GetDecryptionKey(const KeyId& aKeyId) { - MOZ_ASSERT(HasKeyForKeyId(aKeyId)); + assert(HasKeyForKeyId(aKeyId)); return mDecryptors[aKeyId]->DecryptionKey(); } @@ -111,7 +122,7 @@ void ClearKeyDecryptionManager::ReleaseKeyId(KeyId aKeyId) { CK_LOGD("ClearKeyDecryptionManager::ReleaseKeyId"); - MOZ_ASSERT(HasKeyForKeyId(aKeyId)); + assert(HasKeyForKeyId(aKeyId)); ClearKeyDecryptor* decryptor = mDecryptors[aKeyId]; if (!decryptor->Release()) { @@ -182,7 +193,7 @@ ClearKeyDecryptor::Decrypt(uint8_t* aBuffer, uint32_t aBufferSize, memcpy(&tmp[0], aBuffer, aBufferSize); } - MOZ_ASSERT(aMetadata->IVSize() == 8 || aMetadata->IVSize() == 16); + assert(aMetadata->IVSize() == 8 || aMetadata->IVSize() == 16); std::vector iv(aMetadata->IV(), aMetadata->IV() + aMetadata->IVSize()); iv.insert(iv.end(), CLEARKEY_KEY_LEN - aMetadata->IVSize(), 0); diff --git a/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.h b/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.h index 3fb2153bb6..ceaa249ccb 100644 --- a/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.h +++ b/media/gmp-clearkey/0.1/ClearKeyDecryptionManager.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyDecryptionManager_h__ #define __ClearKeyDecryptionManager_h__ diff --git a/media/gmp-clearkey/0.1/ClearKeyPersistence.cpp b/media/gmp-clearkey/0.1/ClearKeyPersistence.cpp index 77741dc0b4..93df08a0c1 100644 --- a/media/gmp-clearkey/0.1/ClearKeyPersistence.cpp +++ b/media/gmp-clearkey/0.1/ClearKeyPersistence.cpp @@ -1,20 +1,32 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "ClearKeyPersistence.h" #include "ClearKeyUtils.h" #include "ClearKeyStorage.h" #include "ClearKeySessionManager.h" -#include "mozilla/RefPtr.h" +#include "RefCounted.h" #include #include #include #include #include +#include -using namespace mozilla; using namespace std; // Whether we've loaded the persistent session ids from GMPStorage yet. @@ -36,7 +48,7 @@ ReadAllRecordsFromIterator(GMPRecordIterator* aRecordIterator, void* aUserArg, GMPErr aStatus) { - MOZ_ASSERT(sPersistentKeyState == LOADING); + assert(sPersistentKeyState == LOADING); if (GMP_SUCCEEDED(aStatus)) { // Extract the record names which are valid uint32_t's; they're // the persistent session ids. @@ -44,7 +56,7 @@ ReadAllRecordsFromIterator(GMPRecordIterator* aRecordIterator, uint32_t len = 0; while (GMP_SUCCEEDED(aRecordIterator->GetName(&name, &len))) { if (ClearKeyUtils::IsValidSessionId(name, len)) { - MOZ_ASSERT(name[len] == 0); + assert(name[len] == 0); sPersistentSessionIds.insert(atoi(name)); } aRecordIterator->NextRecord(); diff --git a/media/gmp-clearkey/0.1/ClearKeyPersistence.h b/media/gmp-clearkey/0.1/ClearKeyPersistence.h index 1bd3800e0b..5adeed1920 100644 --- a/media/gmp-clearkey/0.1/ClearKeyPersistence.h +++ b/media/gmp-clearkey/0.1/ClearKeyPersistence.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyPersistence_h__ #define __ClearKeyPersistence_h__ diff --git a/media/gmp-clearkey/0.1/ClearKeySession.cpp b/media/gmp-clearkey/0.1/ClearKeySession.cpp index 05cb29a61d..d6507a6514 100644 --- a/media/gmp-clearkey/0.1/ClearKeySession.cpp +++ b/media/gmp-clearkey/0.1/ClearKeySession.cpp @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "ClearKeyDecryptionManager.h" #include "ClearKeySession.h" @@ -9,7 +21,8 @@ #include "gmp-task-utils.h" #include "gmp-api/gmp-decryption.h" -#include "mozilla/Endian.h" +#include "Endian.h" +#include using namespace mozilla; @@ -29,7 +42,7 @@ ClearKeySession::~ClearKeySession() auto& keyIds = GetKeyIds(); for (auto it = keyIds.begin(); it != keyIds.end(); it++) { - MOZ_ASSERT(ClearKeyDecryptionManager::Get()->HasKeyForKeyId(*it)); + assert(ClearKeyDecryptionManager::Get()->HasKeyForKeyId(*it)); ClearKeyDecryptionManager::Get()->ReleaseKeyId(*it); mCallback->KeyStatusChanged(&mSessionId[0], mSessionId.size(), diff --git a/media/gmp-clearkey/0.1/ClearKeySession.h b/media/gmp-clearkey/0.1/ClearKeySession.h index 68f498aaf0..7756d7aafa 100644 --- a/media/gmp-clearkey/0.1/ClearKeySession.h +++ b/media/gmp-clearkey/0.1/ClearKeySession.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeySession_h__ #define __ClearKeySession_h__ diff --git a/media/gmp-clearkey/0.1/ClearKeySessionManager.cpp b/media/gmp-clearkey/0.1/ClearKeySessionManager.cpp index 78b583765f..17cb8ced1b 100644 --- a/media/gmp-clearkey/0.1/ClearKeySessionManager.cpp +++ b/media/gmp-clearkey/0.1/ClearKeySessionManager.cpp @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include @@ -17,9 +29,8 @@ #include #endif -#include "mozilla/Assertions.h" +#include -using namespace mozilla; using namespace std; ClearKeySessionManager::ClearKeySessionManager() @@ -37,7 +48,6 @@ ClearKeySessionManager::ClearKeySessionManager() ClearKeySessionManager::~ClearKeySessionManager() { CK_LOGD("ClearKeySessionManager dtor %p", this); - MOZ_ASSERT(!mRefCount); } static bool @@ -108,7 +118,7 @@ ClearKeySessionManager::CreateSession(uint32_t aCreateSessionToken, } string sessionId = ClearKeyPersistence::GetNewSessionId(aSessionType); - MOZ_ASSERT(mSessions.find(sessionId) == mSessions.end()); + assert(mSessions.find(sessionId) == mSessions.end()); ClearKeySession* session = new ClearKeySession(sessionId, mCallback, aSessionType); session->Init(aCreateSessionToken, aPromiseId, aInitData, aInitDataSize); @@ -192,10 +202,10 @@ ClearKeySessionManager::PersistentSessionDataLoaded(GMPErr aStatus, const uint8_t* base = aKeyData + 2 * CLEARKEY_KEY_LEN * i; KeyId keyId(base, base + CLEARKEY_KEY_LEN); - MOZ_ASSERT(keyId.size() == CLEARKEY_KEY_LEN); + assert(keyId.size() == CLEARKEY_KEY_LEN); Key key(base + CLEARKEY_KEY_LEN, base + 2 * CLEARKEY_KEY_LEN); - MOZ_ASSERT(key.size() == CLEARKEY_KEY_LEN); + assert(key.size() == CLEARKEY_KEY_LEN); session->AddKeyId(keyId); @@ -274,10 +284,10 @@ ClearKeySessionManager::Serialize(const ClearKeySession* aSession, if (!mDecryptionManager->HasKeyForKeyId(keyId)) { continue; } - MOZ_ASSERT(keyId.size() == CLEARKEY_KEY_LEN); + assert(keyId.size() == CLEARKEY_KEY_LEN); aOutKeyData.insert(aOutKeyData.end(), keyId.begin(), keyId.end()); const Key& key = mDecryptionManager->GetDecryptionKey(keyId); - MOZ_ASSERT(key.size() == CLEARKEY_KEY_LEN); + assert(key.size() == CLEARKEY_KEY_LEN); aOutKeyData.insert(aOutKeyData.end(), key.begin(), key.end()); } } @@ -298,7 +308,7 @@ ClearKeySessionManager::CloseSession(uint32_t aPromiseId, } ClearKeySession* session = itr->second; - MOZ_ASSERT(session); + assert(session); ClearInMemorySessionData(session); mCallback->ResolvePromise(aPromiseId); @@ -327,7 +337,7 @@ ClearKeySessionManager::RemoveSession(uint32_t aPromiseId, } ClearKeySession* session = itr->second; - MOZ_ASSERT(session); + assert(session); string sid = session->Id(); bool isPersistent = session->Type() == kGMPPersistentSession; ClearInMemorySessionData(session); @@ -376,9 +386,9 @@ ClearKeySessionManager::Decrypt(GMPBuffer* aBuffer, return; } - mThread->Post(WrapTask(this, - &ClearKeySessionManager::DoDecrypt, - aBuffer, aMetadata)); + mThread->Post(WrapTaskRefCounted(this, + &ClearKeySessionManager::DoDecrypt, + aBuffer, aMetadata)); } void diff --git a/media/gmp-clearkey/0.1/ClearKeySessionManager.h b/media/gmp-clearkey/0.1/ClearKeySessionManager.h index 5e007cba90..034efce3ec 100644 --- a/media/gmp-clearkey/0.1/ClearKeySessionManager.h +++ b/media/gmp-clearkey/0.1/ClearKeySessionManager.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyDecryptor_h__ #define __ClearKeyDecryptor_h__ @@ -14,8 +26,6 @@ #include "ClearKeySession.h" #include "ClearKeyUtils.h" #include "gmp-api/gmp-decryption.h" -#include "mozilla/Attributes.h" -#include "mozilla/RefPtr.h" #include "RefCounted.h" class ClearKeySessionManager final : public GMPDecryptor diff --git a/media/gmp-clearkey/0.1/ClearKeyStorage.cpp b/media/gmp-clearkey/0.1/ClearKeyStorage.cpp index b76d6e5757..0db51c9b7e 100644 --- a/media/gmp-clearkey/0.1/ClearKeyStorage.cpp +++ b/media/gmp-clearkey/0.1/ClearKeyStorage.cpp @@ -1,14 +1,26 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "ClearKeyStorage.h" #include "ClearKeyUtils.h" #include "gmp-task-utils.h" -#include "mozilla/Assertions.h" -#include "mozilla/ArrayUtils.h" +#include +#include "ArrayUtils.h" #include @@ -50,7 +62,7 @@ public: virtual void ReadComplete(GMPErr aStatus, const uint8_t* aData, uint32_t aDataSize) override { - MOZ_ASSERT(false, "Should not reach here."); + assert(false); // Should not reach here. } virtual void WriteComplete(GMPErr aStatus) override { @@ -115,7 +127,7 @@ public: */ static void Read(const std::string& aRecordName, ReadContinuation* aContinuation) { - MOZ_ASSERT(aContinuation); + assert(aContinuation); (new ReadRecordClient(aContinuation))->Do(aRecordName); } @@ -134,7 +146,7 @@ public: } virtual void WriteComplete(GMPErr aStatus) override { - MOZ_ASSERT(false, "Should not reach here."); + assert(false); // Should not reach here. } private: diff --git a/media/gmp-clearkey/0.1/ClearKeyStorage.h b/media/gmp-clearkey/0.1/ClearKeyStorage.h index 41f7c865a4..a1377097be 100644 --- a/media/gmp-clearkey/0.1/ClearKeyStorage.h +++ b/media/gmp-clearkey/0.1/ClearKeyStorage.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyStorage_h__ #define __ClearKeyStorage_h__ diff --git a/media/gmp-clearkey/0.1/ClearKeyUtils.cpp b/media/gmp-clearkey/0.1/ClearKeyUtils.cpp index 3d7e58406d..a15f7fed70 100644 --- a/media/gmp-clearkey/0.1/ClearKeyUtils.cpp +++ b/media/gmp-clearkey/0.1/ClearKeyUtils.cpp @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include @@ -10,9 +22,9 @@ #include "ClearKeyUtils.h" #include "ClearKeyBase64.h" -#include "mozilla/ArrayUtils.h" -#include "mozilla/Assertions.h" -#include "mozilla/Endian.h" +#include "ArrayUtils.h" +#include +#include "Endian.h" #include "openaes/oaes_lib.h" using namespace std; @@ -43,7 +55,7 @@ static void IncrementIV(vector& aIV) { using mozilla::BigEndian; - MOZ_ASSERT(aIV.size() == 16); + assert(aIV.size() == 16); BigEndian::writeUint64(&aIV[8], BigEndian::readUint64(&aIV[8]) + 1); } @@ -51,8 +63,8 @@ IncrementIV(vector& aIV) { ClearKeyUtils::DecryptAES(const vector& aKey, vector& aData, vector& aIV) { - MOZ_ASSERT(aIV.size() == CLEARKEY_KEY_LEN); - MOZ_ASSERT(aKey.size() == CLEARKEY_KEY_LEN); + assert(aIV.size() == CLEARKEY_KEY_LEN); + assert(aKey.size() == CLEARKEY_KEY_LEN); OAES_CTX* aes = oaes_alloc(); oaes_key_import_data(aes, &aKey[0], aKey.size()); @@ -65,7 +77,7 @@ ClearKeyUtils::DecryptAES(const vector& aKey, vector enc(encLen); oaes_encrypt(aes, &aIV[0], CLEARKEY_KEY_LEN, &enc[0], &encLen); - MOZ_ASSERT(encLen >= 2 * OAES_BLOCK_SIZE + CLEARKEY_KEY_LEN); + assert(encLen >= 2 * OAES_BLOCK_SIZE + CLEARKEY_KEY_LEN); size_t blockLen = min(aData.size() - i, CLEARKEY_KEY_LEN); for (size_t j = 0; j < blockLen; j++) { aData[i + j] ^= enc[2 * OAES_BLOCK_SIZE + j]; @@ -111,8 +123,7 @@ EncodeBase64Web(vector aBinary, string& aEncoded) // Cast idx to size_t before using it as an array-index, // to pacify clang 'Wchar-subscripts' warning: size_t idx = static_cast(out[i]); - MOZ_ASSERT(idx < MOZ_ARRAY_LENGTH(sAlphabet), - "out of bounds index for 'sAlphabet'"); + assert(idx < MOZ_ARRAY_LENGTH(sAlphabet)); // out of bounds index for 'sAlphabet' out[i] = sAlphabet[idx]; } @@ -181,7 +192,7 @@ ClearKeyUtils::MakeKeyRequest(const vector& aKeyIDs, string& aOutRequest, GMPSessionType aSessionType) { - MOZ_ASSERT(aKeyIDs.size() && aOutRequest.empty()); + assert(aKeyIDs.size() && aOutRequest.empty()); aOutRequest.append("{ \"kids\":["); for (size_t i = 0; i < aKeyIDs.size(); i++) { @@ -439,7 +450,7 @@ ParseKeys(ParserContext& aCtx, vector& aOutKeys) return false; } - MOZ_ASSERT(!key.mKey.empty() && !key.mKeyId.empty()); + assert(!key.mKey.empty() && !key.mKeyId.empty()); aOutKeys.push_back(key); uint8_t sym = PeekSymbol(aCtx); @@ -507,7 +518,7 @@ ClearKeyUtils::SessionTypeToString(GMPSessionType aSessionType) case kGMPTemporySession: return "temporary"; case kGMPPersistentSession: return "persistent"; default: { - MOZ_ASSERT(false, "Should not reach here."); + assert(false); // Should not reach here. return "invalid"; } } diff --git a/media/gmp-clearkey/0.1/ClearKeyUtils.h b/media/gmp-clearkey/0.1/ClearKeyUtils.h index 7700e7182e..4e359e13ef 100644 --- a/media/gmp-clearkey/0.1/ClearKeyUtils.h +++ b/media/gmp-clearkey/0.1/ClearKeyUtils.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __ClearKeyUtils_h__ #define __ClearKeyUtils_h__ diff --git a/media/gmp-clearkey/0.1/Endian.h b/media/gmp-clearkey/0.1/Endian.h new file mode 100644 index 0000000000..91b7174d49 --- /dev/null +++ b/media/gmp-clearkey/0.1/Endian.h @@ -0,0 +1,68 @@ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __Endian_h__ +#define __Endian_h__ + +#include + +namespace mozilla { + +class BigEndian { +public: + + static uint32_t readUint32(const void* aPtr) { + const uint8_t* p = reinterpret_cast(aPtr); + return uint32_t(p[0]) << 24 | + uint32_t(p[1]) << 16 | + uint32_t(p[2]) << 8 | + uint32_t(p[3]); + } + + static uint16_t readUint16(const void* aPtr) { + const uint8_t* p = reinterpret_cast(aPtr); + return uint32_t(p[0]) << 8 | + uint32_t(p[1]); + } + + static uint64_t readUint64(const void* aPtr) { + const uint8_t* p = reinterpret_cast(aPtr); + return uint64_t(p[0]) << 56 | + uint64_t(p[1]) << 48 | + uint64_t(p[2]) << 40 | + uint64_t(p[3]) << 32 | + uint64_t(p[4]) << 24 | + uint64_t(p[5]) << 16 | + uint64_t(p[6]) << 8 | + uint64_t(p[7]); + } + + static void writeUint64(void* aPtr, uint64_t aValue) { + uint8_t* p = reinterpret_cast(aPtr); + p[0] = uint8_t(aValue >> 56) & 0xff; + p[1] = uint8_t(aValue >> 48) & 0xff; + p[2] = uint8_t(aValue >> 40) & 0xff; + p[3] = uint8_t(aValue >> 32) & 0xff; + p[4] = uint8_t(aValue >> 24) & 0xff; + p[5] = uint8_t(aValue >> 16) & 0xff; + p[6] = uint8_t(aValue >> 8) & 0xff; + p[7] = uint8_t(aValue) & 0xff; + } +}; + +} + +#endif // __Endian_h__ diff --git a/media/gmp-clearkey/0.1/RefCounted.h b/media/gmp-clearkey/0.1/RefCounted.h index f5306759eb..62fcccf162 100644 --- a/media/gmp-clearkey/0.1/RefCounted.h +++ b/media/gmp-clearkey/0.1/RefCounted.h @@ -1,11 +1,27 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef __RefCount_h__ #define __RefCount_h__ -// Note: Not thread safe! +#include +#include +#include + +// Note: Thread safe. class RefCounted { public: void AddRef() { @@ -27,8 +43,38 @@ protected: } virtual ~RefCounted() { + assert(!mRefCount); } - uint32_t mRefCount; + std::atomic mRefCount; +}; + +template +class RefPtr { +public: + explicit RefPtr(T* aPtr) : mPtr(nullptr) { + Assign(aPtr); + } + ~RefPtr() { + Assign(nullptr); + } + T* operator->() const { return mPtr; } + + RefPtr& operator=(T* aVal) { + Assign(aVal); + return *this; + } + +private: + void Assign(T* aPtr) { + if (mPtr) { + mPtr->Release(); + } + mPtr = aPtr; + if (mPtr) { + aPtr->AddRef(); + } + } + T* mPtr; }; #endif // __RefCount_h__ diff --git a/media/gmp-clearkey/0.1/VideoDecoder.cpp b/media/gmp-clearkey/0.1/VideoDecoder.cpp index 3aefc2fa1e..eee9a584dc 100644 --- a/media/gmp-clearkey/0.1/VideoDecoder.cpp +++ b/media/gmp-clearkey/0.1/VideoDecoder.cpp @@ -21,9 +21,8 @@ #include "ClearKeyDecryptionManager.h" #include "ClearKeyUtils.h" #include "gmp-task-utils.h" -#include "mozilla/Endian.h" +#include "Endian.h" #include "VideoDecoder.h" -#include "mozilla/Move.h" using namespace wmf; @@ -36,6 +35,8 @@ VideoDecoder::VideoDecoder(GMPVideoHost *aHostAPI) , mSentExtraData(false) , mHasShutdown(false) { + // We drop the ref in DecodingComplete(). + AddRef(); } VideoDecoder::~VideoDecoder() @@ -113,9 +114,9 @@ VideoDecoder::Decode(GMPVideoEncodedFrame* aInputFrame, // Note: we don't need the codec specific info on a per-frame basis. // It's mostly useful for WebRTC use cases. - mWorkerThread->Post(WrapTask(this, - &VideoDecoder::DecodeTask, - aInputFrame)); + mWorkerThread->Post(WrapTaskRefCounted(this, + &VideoDecoder::DecodeTask, + aInputFrame)); } class AutoReleaseVideoFrame { @@ -198,13 +199,12 @@ VideoDecoder::DecodeTask(GMPVideoEncodedFrame* aInput) CK_LOGD("VideoDecoder::DecodeTask() output ret=0x%x\n", hr); if (hr == S_OK) { MaybeRunOnMainThread( - WrapTask(this, - &VideoDecoder::ReturnOutput, - CComPtr(mozilla::Move(output)), - mDecoder->GetFrameWidth(), - mDecoder->GetFrameHeight(), - mDecoder->GetStride())); - assert(!output.Get()); + WrapTaskRefCounted(this, + &VideoDecoder::ReturnOutput, + CComPtr(output), + mDecoder->GetFrameWidth(), + mDecoder->GetFrameHeight(), + mDecoder->GetStride())); } if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { AutoLock lock(mMutex); @@ -234,11 +234,20 @@ VideoDecoder::ReturnOutput(IMFSample* aSample, HRESULT hr; GMPVideoFrame* f = nullptr; - mHostAPI->CreateFrame(kGMPI420VideoFrame, &f); - if (!f) { + auto err = mHostAPI->CreateFrame(kGMPI420VideoFrame, &f); + if (GMP_FAILED(err) || !f) { CK_LOGE("Failed to create i420 frame!\n"); return; } + if (HasShutdown()) { + // Note: GMPVideoHost::CreateFrame() can process messages before returning, + // including a message that calls VideoDecoder::DecodingComplete(), i.e. + // we can shutdown during the call! + CK_LOGD("Shutdown while waiting on GMPVideoHost::CreateFrame()!\n"); + f->Destroy(); + return; + } + auto vf = static_cast(f); hr = SampleToVideoFrame(aSample, aWidth, aHeight, aStride, vf); @@ -293,9 +302,10 @@ VideoDecoder::SampleToVideoFrame(IMFSample* aSample, int32_t halfStride = (stride + 1) / 2; int32_t halfHeight = (aHeight + 1) / 2; - aVideoFrame->CreateEmptyFrame(stride, aHeight, stride, halfStride, halfStride); + auto err = aVideoFrame->CreateEmptyFrame(stride, aHeight, stride, halfStride, halfStride); + ENSURE(GMP_SUCCEEDED(err), E_FAIL); - auto err = aVideoFrame->SetWidth(aWidth); + err = aVideoFrame->SetWidth(aWidth); ENSURE(GMP_SUCCEEDED(err), E_FAIL); err = aVideoFrame->SetHeight(aHeight); ENSURE(GMP_SUCCEEDED(err), E_FAIL); @@ -357,13 +367,12 @@ VideoDecoder::DrainTask() CK_LOGD("VideoDecoder::DrainTask() output ret=0x%x\n", hr); if (hr == S_OK) { MaybeRunOnMainThread( - WrapTask(this, - &VideoDecoder::ReturnOutput, - CComPtr(mozilla::Move(output)), - mDecoder->GetFrameWidth(), - mDecoder->GetFrameHeight(), - mDecoder->GetStride())); - assert(!output.Get()); + WrapTaskRefCounted(this, + &VideoDecoder::ReturnOutput, + CComPtr(output), + mDecoder->GetFrameWidth(), + mDecoder->GetFrameHeight(), + mDecoder->GetStride())); } } MaybeRunOnMainThread(WrapTask(mCallback, &GMPVideoDecoderCallback::DrainComplete)); @@ -379,8 +388,8 @@ VideoDecoder::Drain() return; } EnsureWorker(); - mWorkerThread->Post(WrapTask(this, - &VideoDecoder::DrainTask)); + mWorkerThread->Post(WrapTaskRefCounted(this, + &VideoDecoder::DrainTask)); } void @@ -391,24 +400,19 @@ VideoDecoder::DecodingComplete() } mHasShutdown = true; - // Worker thread might have dispatched more tasks to the main thread that need this object. - // Append another task to delete |this|. - GetPlatform()->runonmainthread(WrapTask(this, &VideoDecoder::Destroy)); + // Release the reference we added in the constructor. There may be + // WrapRefCounted tasks that also hold references to us, and keep + // us alive a little longer. + Release(); } void -VideoDecoder::Destroy() -{ - delete this; -} - -void -VideoDecoder::MaybeRunOnMainThread(gmp_task_args_base* aTask) +VideoDecoder::MaybeRunOnMainThread(GMPTask* aTask) { class MaybeRunTask : public GMPTask { public: - MaybeRunTask(VideoDecoder* aDecoder, gmp_task_args_base* aTask) + MaybeRunTask(VideoDecoder* aDecoder, GMPTask* aTask) : mDecoder(aDecoder), mTask(aTask) { } @@ -428,8 +432,8 @@ VideoDecoder::MaybeRunOnMainThread(gmp_task_args_base* aTask) } private: - VideoDecoder* mDecoder; - gmp_task_args_base* mTask; + RefPtr mDecoder; + GMPTask* mTask; }; GetPlatform()->runonmainthread(new MaybeRunTask(this, aTask)); diff --git a/media/gmp-clearkey/0.1/VideoDecoder.h b/media/gmp-clearkey/0.1/VideoDecoder.h index e96dca5277..b2d153513a 100644 --- a/media/gmp-clearkey/0.1/VideoDecoder.h +++ b/media/gmp-clearkey/0.1/VideoDecoder.h @@ -25,12 +25,11 @@ #include "mfobjects.h" class VideoDecoder : public GMPVideoDecoder + , public RefCounted { public: VideoDecoder(GMPVideoHost *aHostAPI); - virtual ~VideoDecoder(); - virtual void InitDecode(const GMPVideoCodec& aCodecSettings, const uint8_t* aCodecSpecific, uint32_t aCodecSpecificLength, @@ -53,6 +52,8 @@ public: private: + virtual ~VideoDecoder(); + void EnsureWorker(); void DrainTask(); @@ -70,8 +71,7 @@ private: int32_t aStride, GMPVideoi420Frame* aVideoFrame); - void MaybeRunOnMainThread(gmp_task_args_base* aTask); - void Destroy(); + void MaybeRunOnMainThread(GMPTask* aTask); GMPVideoHost *mHostAPI; // host-owned, invalid at DecodingComplete GMPVideoDecoderCallback* mCallback; // host-owned, invalid at DecodingComplete diff --git a/media/gmp-clearkey/0.1/WMFSymbols.h b/media/gmp-clearkey/0.1/WMFSymbols.h index f6302a2335..378d1e025c 100644 --- a/media/gmp-clearkey/0.1/WMFSymbols.h +++ b/media/gmp-clearkey/0.1/WMFSymbols.h @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ MFPLAT_FUNC(MFCreateSample, "mfplat.dll"); MFPLAT_FUNC(MFCreateAlignedMemoryBuffer, "mfplat.dll"); diff --git a/media/gmp-clearkey/0.1/gmp-clearkey.cpp b/media/gmp-clearkey/0.1/gmp-clearkey.cpp index 5f0e1edfc0..ae19508cc2 100644 --- a/media/gmp-clearkey/0.1/gmp-clearkey.cpp +++ b/media/gmp-clearkey/0.1/gmp-clearkey.cpp @@ -1,6 +1,18 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include @@ -9,7 +21,6 @@ #include "ClearKeySessionManager.h" #include "gmp-api/gmp-decryption.h" #include "gmp-api/gmp-platform.h" -#include "mozilla/Attributes.h" #if defined(ENABLE_WMF) #include "WMFUtils.h" @@ -17,6 +28,12 @@ #include "VideoDecoder.h" #endif +#if defined(WIN32) +#define GMP_EXPORT __declspec(dllexport) +#else +#define GMP_EXPORT __attribute__((visibility("default"))) +#endif + static GMPPlatformAPI* sPlatform = nullptr; GMPPlatformAPI* GetPlatform() @@ -26,14 +43,14 @@ GetPlatform() extern "C" { -MOZ_EXPORT GMPErr +GMP_EXPORT GMPErr GMPInit(GMPPlatformAPI* aPlatformAPI) { sPlatform = aPlatformAPI; return GMPNoErr; } -MOZ_EXPORT GMPErr +GMP_EXPORT GMPErr GMPGetAPI(const char* aApiName, void* aHostAPI, void** aPluginAPI) { CK_LOGD("ClearKey GMPGetAPI |%s|", aApiName); @@ -58,7 +75,7 @@ GMPGetAPI(const char* aApiName, void* aHostAPI, void** aPluginAPI) return *aPluginAPI ? GMPNoErr : GMPNotImplementedErr; } -MOZ_EXPORT GMPErr +GMP_EXPORT GMPErr GMPShutdown(void) { CK_LOGD("ClearKey GMPShutdown"); diff --git a/media/gmp-clearkey/0.1/gmp-task-utils-generated.h b/media/gmp-clearkey/0.1/gmp-task-utils-generated.h index 5eec3374a2..c899656760 100644 --- a/media/gmp-clearkey/0.1/gmp-task-utils-generated.h +++ b/media/gmp-clearkey/0.1/gmp-task-utils-generated.h @@ -1,7 +1,20 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "RefCounted.h" // 0 arguments -- template class gmp_task_args_nm_0 : public gmp_task_args_base { @@ -1896,3 +1909,30 @@ gmp_task_args_m_14_retRun(); + } + virtual void Destroy() override { + mTask->Destroy(); + gmp_task_args_base::Destroy(); + } +private: + ~RefCountTaskWrapper() {} + + GMPTask* mTask; + RefPtr mRefCounted; +}; + +template +GMPTask* +WrapTaskRefCounted(Type* aType, Method aMethod, Args... args) +{ + GMPTask* t = WrapTask(aType, aMethod, args...); + return new RefCountTaskWrapper(t, aType); +} diff --git a/media/gmp-clearkey/0.1/gmp-task-utils.h b/media/gmp-clearkey/0.1/gmp-task-utils.h index 78a1393c64..55cae627d8 100644 --- a/media/gmp-clearkey/0.1/gmp-task-utils.h +++ b/media/gmp-clearkey/0.1/gmp-task-utils.h @@ -1,8 +1,18 @@ -/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ +/* + * Copyright 2015, Mozilla Foundation and contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ // Original author: ekr@rtfm.com diff --git a/media/gmp-clearkey/0.1/gtest/TestClearKeyUtils.cpp b/media/gmp-clearkey/0.1/gtest/TestClearKeyUtils.cpp index 865a225b1f..b31c50e420 100644 --- a/media/gmp-clearkey/0.1/gtest/TestClearKeyUtils.cpp +++ b/media/gmp-clearkey/0.1/gtest/TestClearKeyUtils.cpp @@ -10,10 +10,10 @@ #include #include "../ClearKeyBase64.cpp" +#include "../ArrayUtils.h" using namespace std; -using namespace mozilla; struct B64Test { const char* b64; diff --git a/media/libcubeb/tests/common.h b/media/libcubeb/tests/common.h index 47cc28cc7f..69445b86c6 100644 --- a/media/libcubeb/tests/common.h +++ b/media/libcubeb/tests/common.h @@ -27,4 +27,3 @@ void delay(unsigned int ms) #if !defined(M_PI) #define M_PI 3.14159265358979323846 #endif - diff --git a/media/libcubeb/tests/test_audio.cpp b/media/libcubeb/tests/test_audio.cpp index f2da917bef..69c166638b 100644 --- a/media/libcubeb/tests/test_audio.cpp +++ b/media/libcubeb/tests/test_audio.cpp @@ -19,6 +19,7 @@ #include "cubeb/cubeb.h" #include "common.h" +#include "TestHarness.h" #define MAX_NUM_CHANNELS 32 @@ -261,6 +262,8 @@ void run_channel_rate_test() int main(int argc, char *argv[]) { + ScopedXPCOM xpcom("test_audio"); + assert(run_panning_volume_test() == CUBEB_OK); run_channel_rate_test(); diff --git a/media/libcubeb/tests/test_latency.cpp b/media/libcubeb/tests/test_latency.cpp index c02cd61a20..c1b2ae2da1 100644 --- a/media/libcubeb/tests/test_latency.cpp +++ b/media/libcubeb/tests/test_latency.cpp @@ -5,11 +5,13 @@ #include #include #include - +#include "TestHarness.h" #define LOG(msg) fprintf(stderr, "%s\n", msg); int main(int argc, char * argv[]) { + ScopedXPCOM xpcom("test_latency"); + cubeb * ctx = NULL; int r; uint32_t max_channels; @@ -49,6 +51,5 @@ int main(int argc, char * argv[]) cubeb_destroy(ctx); LOG("cubeb_destroy ok"); - return EXIT_SUCCESS; } diff --git a/media/libcubeb/tests/test_sanity.cpp b/media/libcubeb/tests/test_sanity.cpp index 6841393fca..1d7e5562bf 100644 --- a/media/libcubeb/tests/test_sanity.cpp +++ b/media/libcubeb/tests/test_sanity.cpp @@ -14,6 +14,7 @@ #include #include #include "common.h" +#include "TestHarness.h" #if (defined(_WIN32) || defined(__WIN32__)) #define __func__ __FUNCTION__ @@ -571,6 +572,8 @@ int is_windows_7() int main(int argc, char * argv[]) { + ScopedXPCOM xpcom("test_sanity"); + test_init_destroy_context(); test_init_destroy_multiple_contexts(); test_context_variables(); @@ -607,5 +610,6 @@ main(int argc, char * argv[]) test_stream_destroy_pending_drain(); */ printf("\n"); + return 0; } diff --git a/media/libcubeb/tests/test_tone.cpp b/media/libcubeb/tests/test_tone.cpp index 52b8e1b3d2..ae50c69281 100644 --- a/media/libcubeb/tests/test_tone.cpp +++ b/media/libcubeb/tests/test_tone.cpp @@ -17,6 +17,7 @@ #include "cubeb/cubeb.h" #include "common.h" +#include "TestHarness.h" #define SAMPLE_FREQUENCY 48000 @@ -72,6 +73,8 @@ void state_cb(cubeb_stream *stream, void *user, cubeb_state state) int main(int argc, char *argv[]) { + ScopedXPCOM xpcom("test_tone"); + cubeb *ctx; cubeb_stream *stream; cubeb_stream_params params; diff --git a/media/libstagefright/binding/MoofParser.cpp b/media/libstagefright/binding/MoofParser.cpp index 4f519a9671..a667da1dc4 100644 --- a/media/libstagefright/binding/MoofParser.cpp +++ b/media/libstagefright/binding/MoofParser.cpp @@ -11,7 +11,7 @@ #include "mozilla/Logging.h" #if defined(MOZ_FMP4) -extern PRLogModuleInfo* GetDemuxerLog(); +extern mozilla::LogModule* GetDemuxerLog(); #define STRINGIFY(x) #x #define TOSTRING(x) STRINGIFY(x) diff --git a/media/webrtc/signaling/src/media-conduit/WebrtcGmpVideoCodec.cpp b/media/webrtc/signaling/src/media-conduit/WebrtcGmpVideoCodec.cpp index abf232eafd..a4eada8e3b 100644 --- a/media/webrtc/signaling/src/media-conduit/WebrtcGmpVideoCodec.cpp +++ b/media/webrtc/signaling/src/media-conduit/WebrtcGmpVideoCodec.cpp @@ -32,7 +32,7 @@ namespace mozilla { #endif #ifdef MOZILLA_INTERNAL_API -extern PRLogModuleInfo* GetGMPLog(); +extern mozilla::LogModule* GetGMPLog(); #else // For CPP unit tests PRLogModuleInfo* diff --git a/media/webrtc/signaling/src/media-conduit/WebrtcOMXH264VideoCodec.cpp b/media/webrtc/signaling/src/media-conduit/WebrtcOMXH264VideoCodec.cpp index 083b78b064..8492b11fee 100644 --- a/media/webrtc/signaling/src/media-conduit/WebrtcOMXH264VideoCodec.cpp +++ b/media/webrtc/signaling/src/media-conduit/WebrtcOMXH264VideoCodec.cpp @@ -524,12 +524,9 @@ public: return; } - layers::GrallocImage::GrallocData grallocData; - grallocData.mPicSize = buffer->GetSize(); - grallocData.mGraphicBuffer = buffer; - + gfx::IntSize picSize(buffer->GetSize()); nsAutoPtr grallocImage(new layers::GrallocImage()); - grallocImage->SetData(grallocData); + grallocImage->SetData(buffer, picSize); // Get timestamp of the frame about to render. int64_t timestamp = -1; @@ -547,12 +544,12 @@ public: MOZ_ASSERT(timestamp >= 0 && renderTimeMs >= 0); CODEC_LOGD("Decoder NewFrame: %dx%d, timestamp %lld, renderTimeMs %lld", - grallocData.mPicSize.width, grallocData.mPicSize.height, timestamp, renderTimeMs); + picSize.width, picSize.height, timestamp, renderTimeMs); nsAutoPtr videoFrame( new webrtc::TextureVideoFrame(new ImageNativeHandle(grallocImage.forget()), - grallocData.mPicSize.width, - grallocData.mPicSize.height, + picSize.width, + picSize.height, timestamp, renderTimeMs)); if (videoFrame != nullptr) { diff --git a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp index 05d0ed0038..c123b70945 100644 --- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp +++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp @@ -1481,12 +1481,10 @@ void MediaPipelineReceiveVideo::PipelineListener::RenderVideoFrame( if (buffer) { // Create a video frame using |buffer|. #ifdef MOZ_WIDGET_GONK - ImageFormat format = ImageFormat::GRALLOC_PLANAR_YCBCR; + RefPtr yuvImage = new GrallocImage(); #else - ImageFormat format = ImageFormat::PLANAR_YCBCR; + RefPtr yuvImage = image_container_->CreatePlanarYCbCrImage(); #endif - RefPtr image = image_container_->CreateImage(format); - PlanarYCbCrImage* yuvImage = static_cast(image.get()); uint8_t* frame = const_cast(static_cast (buffer)); PlanarYCbCrData yuvData; @@ -1507,7 +1505,7 @@ void MediaPipelineReceiveVideo::PipelineListener::RenderVideoFrame( return; } - image_ = image.forget(); + image_ = yuvImage; } #ifdef WEBRTC_GONK else { diff --git a/media/webrtc/trunk/webrtc/common_types.h b/media/webrtc/trunk/webrtc/common_types.h index 9284c594c2..17830a9c40 100644 --- a/media/webrtc/trunk/webrtc/common_types.h +++ b/media/webrtc/trunk/webrtc/common_types.h @@ -787,9 +787,10 @@ struct OverUseDetectorOptions { }; enum CPULoadState { - kLoadRelaxed, + kLoadRelaxed = 0, kLoadNormal, - kLoadStressed + kLoadStressed, + kLoadLast, }; class CPULoadStateObserver { diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index dea65bae7a..dad4622312 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -390,6 +390,9 @@ pref("media.gstreamer.enabled", true); pref("media.gstreamer.enable-blacklist", true); #endif #ifdef MOZ_APPLEMEDIA +#ifdef MOZ_WIDGET_UIKIT +pref("media.mp3.enabled", true); +#endif pref("media.apple.mp3.enabled", true); pref("media.apple.mp4.enabled", true); #endif diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index 25b493043a..49b4515f04 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -6288,6 +6288,48 @@ "n_values": 8, "description": "Type for media in getUserMedia calls (0=Camera, 1=Screen, 2=Application, 3=Window, 4=Browser, 5=Microphone, 6=AudioCapture, 7=Other)" }, + "WEBRTC_LOAD_STATE_RELAXED": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Relaxed load state in calls over 30 seconds." + }, + "WEBRTC_LOAD_STATE_RELAXED_SHORT": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Relaxed load state in calls 5-30 seconds." + }, + "WEBRTC_LOAD_STATE_NORMAL": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Normal load state in calls over 30 seconds." + }, + "WEBRTC_LOAD_STATE_NORMAL_SHORT": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Normal load state in calls over 5-30 seconds." + }, + "WEBRTC_LOAD_STATE_STRESSED": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Stressed load state in calls over 30 seconds." + }, + "WEBRTC_LOAD_STATE_STRESSED_SHORT": { + "expires_in_version": "never", + "kind": "linear", + "high": "100", + "n_buckets": "25", + "description": "Percentage of time spent in the Stressed load state in calls 5-30 seconds." + }, "DEVTOOLS_DEBUGGER_RDP_LOCAL_TRACERDETACH_MS": { "expires_in_version": "never", "kind": "exponential",