roytam1/palemoon27
1
0
Fork 0
mirror of https://github.com/roytam1/palemoon27.git synced 2026-05-26 14:18:48 +00:00
Code Issues Projects Releases Wiki Activity

import changes from `dev' branch of rmottola/Arctic-Fox:

Browse Source 
- Bug 1197669 - Part1 - 1.Enable the testcase on B2G. 2. fix the mimetype check in testcase. r=jwwang (9bbae092f)
- Bug 1197669 - Part2 - 1. Ensure the EOS is sent to MediaCodec. 2. mEndOfStream should be protected by monitor. r=sotaro (62b8e66b7)
- bug 1193922 skip SetIsOnlyNodeForContext after shutdown r=padenot (df991d0af)
- Minor (1d695f7c4)
- Bug 1197051 - Don't try to decode ahead while seeking - r=jya (7d8d877ad)
- Bug 1197075: P1. Revert "Bug 1197051 - Don't try to decode ahead while seeking - r=jya". r=edwin (e837b12e3)
- Bug 1143575. Push all available frames to the compositor. r=cpearce (109e54ae8)
- partial of  Bug 1197075: P2. Revert "Bug 1171257 - Add force decode ahead to MediaFormatReader r=jya,bholley". r=edwin (5f4859e47)
- Minor (a841ca407)
- Bug 1199155. Part 2 - fix includes and forward declarations. r=roc. (ca523595a)
- Bug 1199155. Part 1 - move dom/media/DecodedStream.* to dom/media/mediasink/ and implement the interface of MediaSink. r=roc. (90b376a24)
- Bug 1199155. Part 3 - rename mDecodedStream to mStreamSink. r=roc. (1aae54328)
- Bug 1203374. Part 1 - extract the code of computing canplaythrough so it is reusable. r=jya. (acc5fc8c6)
- Bug 1203374. Part 2 - duplicate the implementation of MediaDecoder::CanPlayThrough so MDSM can call its own CanPlayThrough() on its own thread. r=jya. (a042541a8)
- Bug 1182928 - Disable dormant mode for EME videos in Firefox Beta and Release. r=sotaro (ea5964ae9)
- Bug 1197022 - [EME] Disable dormant mode for EME videos on all channels. r=kentuckyfriedtakahe (742d00672)
- Bug 1179110 - Fix ComputePlaybackRate. r=jww (a43272af5)
- Bug 1178622 - Fix enum-to-string mismatch in MediaDecoder.cpp. r=jya. (b9da7ebf3)
- Bug 1203418. Part 1 - cache the results of ComputePlaybackRate() so they can be mirrored by MDSM. r=cpearce. (95f7ac068)
- Bug 1203418. Part 2 - duplicate the implementation of MediaDecoder::GetStatistics so MDSM can call it on its own thread. r=cpearce. (2755b85a7)
- Bug 1203418. Part 3 - ensure MDSM::mPlaybackOffset and MediaDecoder::mPlaybackPosition are mono-increasing to avoid "jitter" in calculating playback statistics. r=cpearce. (54dcb6219)
- Bug 1104616 - Proxy video capture access to the main process. r=jesup,glandium,mrbkap (cfdd08459)
- Bug 1200614 - Protect the capture engines array from concurrent access (during shutdown). r=jesup (c4dbd6e10)
- Bug 1200614 - Check whether engines are still alive when webrtc ops run. r=jesup (d42d3474f)
- Bug 1194640 - add NSPR logging of camera capabilities, r=jesup (c7365b916)
- Bug 1204413 - Make MediaDecoder::IsTransportSeekable run on the main thread. r=kinetik. (098868b0d)
- Bug 1198202 - Increase hardware video decoding fallback threshold. r=ajones (1059f9b0f)
- Bug 1201197 - add dedicated listener to enumerateDevices. r=jesup (a51477957)
- Bug 1199562. Part 1 - rename mAudioSink to mMediaSink as well as related member names. r=roc. (c2db1b4d4)
- Bug 1199562. Part 2 - replace usage of mStreamSink with mMediaSink in most cases. r=roc. (7aa0754c4)
- Bug 1199562. Part 3 - remove unused code. r=roc. (343c5be2f)
- Bug 1203877 - Remove MediaDecoder::UpdatePlaybackOffset. r=kinetik. (4476dd1f8)
- Bug 1204430. Part 2 - mirror MediaDecoder::mMediaSeekable. r=kinetik. (51ca1197d)
- Bug 1206576 - Dispatch some MDSM functions to hide its internal thread model. r=jya. (74b9a169c)
- Bug 1206574 - Remove AbstractMediaDecoder::IsShutdown(). r=cpearce. (4ed17a27f)
- Bug 1207017. Part 1 - fix coding styles. r=kinetik. (cc10a28c3)
- Bug 1207017. Part 2 - remove duplicated GetStateMachine(). r=kinetik. (19b663098)
- Bug 1200477 - Allow building with WebRTC disabled. r=glandium,mrbkap (56055c3c7)
This commit is contained in:
roytam1
2021-10-14 09:25:28 +08:00
parent edaff2b0dc
commit 0c01ffebb4
55 changed files with 3877 additions and 1340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dom/media/AbstractMediaDecoder.h
-3
View File
@@ -49,9 +49,6 @@ public:
// state.
virtual ReentrantMonitor& GetReentrantMonitor() = 0;
// Returns true if the decoder is shut down.
virtual bool IsShutdown() const = 0;
// A special version of the above for the ogg decoder that is allowed to be
// called cross-thread.
virtual bool IsOggDecoderShutdown() { return false; }
dom/media/MediaDecoder.cpp
+241 -207
View File
@@ -37,14 +37,6 @@ static const int DEFAULT_HEURISTIC_DORMANT_TIMEOUT_MSECS = 60000;
namespace mozilla {
// Number of estimated seconds worth of data we need to have buffered
// ahead of the current playback position before we allow the media decoder
// to report that it can play through the entire media without the decode
// catching up with the download. Having this margin make the
// MediaDecoder::CanPlayThrough() calculation more stable in the case of
// fluctuating bitrates.
static const int64_t CAN_PLAY_THROUGH_MARGIN = 1;
// The amount of instability we tollerate in calls to
// MediaDecoder::UpdateEstimatedMediaDuration(); changes of duration
// less than this are ignored, as they're assumed to be the result of
@@ -58,15 +50,20 @@ PRLogModuleInfo* gMediaDecoderLog;
#define DECODER_LOG(x, ...) \
MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, ("Decoder=%p " x, this, ##__VA_ARGS__))
static const char* const gPlayStateStr[] = {
"START",
"LOADING",
"PAUSED",
"PLAYING",
"SEEKING",
"ENDED",
"SHUTDOWN"
};
static const char*
ToPlayStateStr(MediaDecoder::PlayState aState)
{
switch (aState) {
case MediaDecoder::PLAY_STATE_START: return "START";
case MediaDecoder::PLAY_STATE_LOADING: return "LOADING";
case MediaDecoder::PLAY_STATE_PAUSED: return "PAUSED";
case MediaDecoder::PLAY_STATE_PLAYING: return "PLAYING";
case MediaDecoder::PLAY_STATE_ENDED: return "ENDED";
case MediaDecoder::PLAY_STATE_SHUTDOWN: return "SHUTDOWN";
default: MOZ_ASSERT_UNREACHABLE("Invalid playState.");
}
return "UNKNOWN";
}
class MediaMemoryTracker : public nsIMemoryReporter
{
@@ -138,7 +135,8 @@ NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter)
NS_IMPL_ISUPPORTS0(MediaDecoder)
void MediaDecoder::NotifyOwnerActivityChanged()
void
MediaDecoder::NotifyOwnerActivityChanged()
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -153,7 +151,20 @@ void MediaDecoder::NotifyOwnerActivityChanged()
StartDormantTimer();
}
void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
bool
MediaDecoder::IsHeuristicDormantSupported() const
{
return
#if defined(MOZ_EME)
// We disallow dormant for encrypted media until bug 1181864 is fixed.
mInfo &&
!mInfo->IsEncrypted() &&
#endif
mIsHeuristicDormantSupported;
}
void
MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
{
MOZ_ASSERT(NS_IsMainThread());
GetReentrantMonitor().AssertCurrentThreadIn();
@@ -168,9 +179,11 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
}
DECODER_LOG("UpdateDormantState aTimeout=%d aActivity=%d mIsDormant=%d "
"ownerActive=%d ownerHidden=%d mIsHeuristicDormant=%d mPlayState=%s",
"ownerActive=%d ownerHidden=%d mIsHeuristicDormant=%d "
"mPlayState=%s encrypted=%s",
aDormantTimeout, aActivity, mIsDormant, mOwner->IsActive(),
mOwner->IsHidden(), mIsHeuristicDormant, PlayStateStr());
mOwner->IsHidden(), mIsHeuristicDormant, PlayStateStr(),
(!mInfo ? "Unknown" : (mInfo->IsEncrypted() ? "1" : "0")));
bool prevDormant = mIsDormant;
mIsDormant = false;
@@ -182,10 +195,11 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
mIsDormant = true;
}
#endif
// Try to enable dormant by idle heuristic, when the owner is hidden.
bool prevHeuristicDormant = mIsHeuristicDormant;
mIsHeuristicDormant = false;
if (mIsHeuristicDormantSupported && mOwner->IsHidden()) {
if (IsHeuristicDormantSupported() && mOwner->IsHidden()) {
if (aDormantTimeout && !aActivity &&
(mPlayState == PLAY_STATE_PAUSED || IsEnded())) {
// Enable heuristic dormant
@@ -208,13 +222,7 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
if (mIsDormant) {
DECODER_LOG("UpdateDormantState() entering DORMANT state");
// enter dormant state
RefPtr<nsRunnable> event =
NS_NewRunnableMethodWithArg<bool>(
mDecoderStateMachine,
&MediaDecoderStateMachine::SetDormant,
true);
mDecoderStateMachine->OwnerThread()->Dispatch(event.forget());
mDecoderStateMachine->DispatchSetDormant(true);
if (IsEnded()) {
mWasEndedWhenEnteredDormant = true;
}
@@ -223,17 +231,12 @@ void MediaDecoder::UpdateDormantState(bool aDormantTimeout, bool aActivity)
} else {
DECODER_LOG("UpdateDormantState() leaving DORMANT state");
// exit dormant state
// trigger to state machine.
RefPtr<nsRunnable> event =
NS_NewRunnableMethodWithArg<bool>(
mDecoderStateMachine,
&MediaDecoderStateMachine::SetDormant,
false);
mDecoderStateMachine->OwnerThread()->Dispatch(event.forget());
mDecoderStateMachine->DispatchSetDormant(false);
}
}
void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure)
void
MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aClosure);
@@ -243,10 +246,11 @@ void MediaDecoder::DormantTimerExpired(nsITimer* aTimer, void* aClosure)
false /* aActivity */);
}
void MediaDecoder::StartDormantTimer()
void
MediaDecoder::StartDormantTimer()
{
MOZ_ASSERT(NS_IsMainThread());
if (!mIsHeuristicDormantSupported) {
if (!IsHeuristicDormantSupported()) {
return;
}
@@ -269,7 +273,8 @@ void MediaDecoder::StartDormantTimer()
nsITimer::TYPE_ONE_SHOT);
}
void MediaDecoder::CancelDormantTimer()
void
MediaDecoder::CancelDormantTimer()
{
MOZ_ASSERT(NS_IsMainThread());
if (mDormantTimer) {
@@ -277,7 +282,8 @@ void MediaDecoder::CancelDormantTimer()
}
}
void MediaDecoder::Pause()
void
MediaDecoder::Pause()
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -290,28 +296,32 @@ void MediaDecoder::Pause()
ChangeState(PLAY_STATE_PAUSED);
}
void MediaDecoder::SetVolume(double aVolume)
void
MediaDecoder::SetVolume(double aVolume)
{
MOZ_ASSERT(NS_IsMainThread());
mVolume = aVolume;
}
void MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream,
bool aFinishWhenEnded)
void
MediaDecoder::AddOutputStream(ProcessedMediaStream* aStream,
bool aFinishWhenEnded)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
mDecoderStateMachine->AddOutputStream(aStream, aFinishWhenEnded);
}
void MediaDecoder::RemoveOutputStream(MediaStream* aStream)
void
MediaDecoder::RemoveOutputStream(MediaStream* aStream)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load().");
mDecoderStateMachine->RemoveOutputStream(aStream);
}
double MediaDecoder::GetDuration()
double
MediaDecoder::GetDuration()
{
MOZ_ASSERT(NS_IsMainThread());
return mDuration;
@@ -324,14 +334,16 @@ MediaDecoder::CanonicalDurationOrNull()
return mDecoderStateMachine->CanonicalDuration();
}
void MediaDecoder::SetInfinite(bool aInfinite)
void
MediaDecoder::SetInfinite(bool aInfinite)
{
MOZ_ASSERT(NS_IsMainThread());
mInfiniteStream = aInfinite;
DurationChanged();
}
bool MediaDecoder::IsInfinite()
bool
MediaDecoder::IsInfinite()
{
MOZ_ASSERT(NS_IsMainThread());
return mInfiniteStream;
@@ -340,11 +352,8 @@ bool MediaDecoder::IsInfinite()
MediaDecoder::MediaDecoder() :
mWatchManager(this, AbstractThread::MainThread()),
mDormantSupported(false),
mDecoderPosition(0),
mPlaybackPosition(0),
mLogicalPosition(0.0),
mDuration(std::numeric_limits<double>::quiet_NaN()),
mMediaSeekable(true),
mReentrantMonitor("media.decoder"),
mIgnoreProgressData(false),
mInfiniteStream(false),
@@ -375,6 +384,8 @@ MediaDecoder::MediaDecoder() :
"MediaDecoder::mCurrentPosition (Mirror)"),
mStateMachineDuration(AbstractThread::MainThread(), NullableTimeUnit(),
"MediaDecoder::mStateMachineDuration (Mirror)"),
mPlaybackPosition(AbstractThread::MainThread(), 0,
"MediaDecoder::mPlaybackPosition (Mirror)"),
mVolume(AbstractThread::MainThread(), 0.0,
"MediaDecoder::mVolume (Canonical)"),
mPlaybackRate(AbstractThread::MainThread(), 1.0,
@@ -392,7 +403,15 @@ MediaDecoder::MediaDecoder() :
mLogicallySeeking(AbstractThread::MainThread(), false,
"MediaDecoder::mLogicallySeeking (Canonical)"),
mSameOriginMedia(AbstractThread::MainThread(), false,
"MediaDecoder::mSameOriginMedia (Canonical)")
"MediaDecoder::mSameOriginMedia (Canonical)"),
mPlaybackBytesPerSecond(AbstractThread::MainThread(), 0.0,
"MediaDecoder::mPlaybackBytesPerSecond (Canonical)"),
mPlaybackRateReliable(AbstractThread::MainThread(), true,
"MediaDecoder::mPlaybackRateReliable (Canonical)"),
mDecoderPosition(AbstractThread::MainThread(), 0,
"MediaDecoder::mDecoderPosition (Canonical)"),
mMediaSeekable(AbstractThread::MainThread(), true,
"MediaDecoder::mMediaSeekable (Canonical)")
{
MOZ_COUNT_CTOR(MediaDecoder);
MOZ_ASSERT(NS_IsMainThread());
@@ -423,7 +442,8 @@ MediaDecoder::MediaDecoder() :
mWatchManager.Watch(mLogicallySeeking, &MediaDecoder::SeekingChanged);
}
bool MediaDecoder::Init(MediaDecoderOwner* aOwner)
bool
MediaDecoder::Init(MediaDecoderOwner* aOwner)
{
MOZ_ASSERT(NS_IsMainThread());
mOwner = aOwner;
@@ -432,7 +452,8 @@ bool MediaDecoder::Init(MediaDecoderOwner* aOwner)
return true;
}
void MediaDecoder::Shutdown()
void
MediaDecoder::Shutdown()
{
MOZ_ASSERT(NS_IsMainThread());
@@ -472,7 +493,8 @@ MediaDecoder::~MediaDecoder()
MOZ_COUNT_DTOR(MediaDecoder);
}
nsresult MediaDecoder::OpenResource(nsIStreamListener** aStreamListener)
nsresult
MediaDecoder::OpenResource(nsIStreamListener** aStreamListener)
{
MOZ_ASSERT(NS_IsMainThread());
if (aStreamListener) {
@@ -491,8 +513,9 @@ nsresult MediaDecoder::OpenResource(nsIStreamListener** aStreamListener)
return NS_OK;
}
nsresult MediaDecoder::Load(nsIStreamListener** aStreamListener,
MediaDecoder* aCloneDonor)
nsresult
MediaDecoder::Load(nsIStreamListener** aStreamListener,
MediaDecoder* aCloneDonor)
{
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mResource, "Can't load without a MediaResource");
@@ -506,7 +529,8 @@ nsresult MediaDecoder::Load(nsIStreamListener** aStreamListener,
return InitializeStateMachine(aCloneDonor);
}
nsresult MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor)
nsresult
MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor)
{
MOZ_ASSERT(NS_IsMainThread());
NS_ASSERTION(mDecoderStateMachine, "Cannot initialize null state machine!");
@@ -523,7 +547,8 @@ nsresult MediaDecoder::InitializeStateMachine(MediaDecoder* aCloneDonor)
return NS_OK;
}
void MediaDecoder::SetStateMachineParameters()
void
MediaDecoder::SetStateMachineParameters()
{
MOZ_ASSERT(NS_IsMainThread());
if (mMinimizePreroll) {
@@ -533,7 +558,8 @@ void MediaDecoder::SetStateMachineParameters()
AbstractThread::MainThread(), this, &MediaDecoder::OnMetadataUpdate);
}
void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts()
void
MediaDecoder::SetMinimizePrerollUntilPlaybackStarts()
{
MOZ_ASSERT(NS_IsMainThread());
DECODER_LOG("SetMinimizePrerollUntilPlaybackStarts()");
@@ -544,7 +570,8 @@ void MediaDecoder::SetMinimizePrerollUntilPlaybackStarts()
MOZ_DIAGNOSTIC_ASSERT(!mDecoderStateMachine);
}
nsresult MediaDecoder::Play()
nsresult
MediaDecoder::Play()
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -566,7 +593,8 @@ nsresult MediaDecoder::Play()
return NS_OK;
}
nsresult MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType)
nsresult
MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType)
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -598,30 +626,33 @@ nsresult MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType)
return NS_OK;
}
void MediaDecoder::CallSeek(const SeekTarget& aTarget)
void
MediaDecoder::CallSeek(const SeekTarget& aTarget)
{
MOZ_ASSERT(NS_IsMainThread());
mSeekRequest.DisconnectIfExists();
mSeekRequest.Begin(InvokeAsync(mDecoderStateMachine->OwnerThread(),
mDecoderStateMachine.get(), __func__,
&MediaDecoderStateMachine::Seek, aTarget)
mSeekRequest.Begin(
mDecoderStateMachine->InvokeSeek(aTarget)
->Then(AbstractThread::MainThread(), __func__, this,
&MediaDecoder::OnSeekResolved, &MediaDecoder::OnSeekRejected));
}
double MediaDecoder::GetCurrentTime()
double
MediaDecoder::GetCurrentTime()
{
MOZ_ASSERT(NS_IsMainThread());
return mLogicalPosition;
}
already_AddRefed<nsIPrincipal> MediaDecoder::GetCurrentPrincipal()
already_AddRefed<nsIPrincipal>
MediaDecoder::GetCurrentPrincipal()
{
MOZ_ASSERT(NS_IsMainThread());
return mResource ? mResource->GetCurrentPrincipal() : nullptr;
}
void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata)
void
MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata)
{
MOZ_ASSERT(NS_IsMainThread());
RemoveMediaTracks();
@@ -632,9 +663,10 @@ void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata)
MediaDecoderEventVisibility::Observable);
}
void MediaDecoder::MetadataLoaded(nsAutoPtr<MediaInfo> aInfo,
nsAutoPtr<MetadataTags> aTags,
MediaDecoderEventVisibility aEventVisibility)
void
MediaDecoder::MetadataLoaded(nsAutoPtr<MediaInfo> aInfo,
nsAutoPtr<MetadataTags> aTags,
MediaDecoderEventVisibility aEventVisibility)
{
MOZ_ASSERT(NS_IsMainThread());
@@ -675,8 +707,9 @@ MediaDecoder::PlayStateStr()
}
}
void MediaDecoder::FirstFrameLoaded(nsAutoPtr<MediaInfo> aInfo,
MediaDecoderEventVisibility aEventVisibility)
void
MediaDecoder::FirstFrameLoaded(nsAutoPtr<MediaInfo> aInfo,
MediaDecoderEventVisibility aEventVisibility)
{
MOZ_ASSERT(NS_IsMainThread());
@@ -713,7 +746,8 @@ void MediaDecoder::FirstFrameLoaded(nsAutoPtr<MediaInfo> aInfo,
NotifySuspendedStatusChanged();
}
void MediaDecoder::ResetConnectionState()
void
MediaDecoder::ResetConnectionState()
{
MOZ_ASSERT(NS_IsMainThread());
if (mShuttingDown)
@@ -730,7 +764,8 @@ void MediaDecoder::ResetConnectionState()
Shutdown();
}
void MediaDecoder::NetworkError()
void
MediaDecoder::NetworkError()
{
MOZ_ASSERT(NS_IsMainThread());
if (mShuttingDown)
@@ -742,7 +777,8 @@ void MediaDecoder::NetworkError()
Shutdown();
}
void MediaDecoder::DecodeError()
void
MediaDecoder::DecodeError()
{
MOZ_ASSERT(NS_IsMainThread());
if (mShuttingDown)
@@ -754,33 +790,38 @@ void MediaDecoder::DecodeError()
Shutdown();
}
void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin)
void
MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin)
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
mSameOriginMedia = aSameOrigin;
}
bool MediaDecoder::IsSeeking() const
bool
MediaDecoder::IsSeeking() const
{
MOZ_ASSERT(NS_IsMainThread());
return mLogicallySeeking;
}
bool MediaDecoder::IsEndedOrShutdown() const
bool
MediaDecoder::IsEndedOrShutdown() const
{
MOZ_ASSERT(NS_IsMainThread());
return IsEnded() || mPlayState == PLAY_STATE_SHUTDOWN;
}
bool MediaDecoder::IsEnded() const
bool
MediaDecoder::IsEnded() const
{
MOZ_ASSERT(NS_IsMainThread());
return mPlayState == PLAY_STATE_ENDED ||
(mWasEndedWhenEnteredDormant && (mPlayState != PLAY_STATE_SHUTDOWN));
}
void MediaDecoder::PlaybackEnded()
void
MediaDecoder::PlaybackEnded()
{
MOZ_ASSERT(NS_IsMainThread());
@@ -804,70 +845,67 @@ void MediaDecoder::PlaybackEnded()
}
}
MediaDecoder::Statistics
MediaStatistics
MediaDecoder::GetStatistics()
{
Statistics result;
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
if (mResource) {
result.mDownloadRate =
mResource->GetDownloadRate(&result.mDownloadRateReliable);
result.mDownloadPosition =
mResource->GetCachedDataEnd(mDecoderPosition);
result.mTotalBytes = mResource->GetLength();
result.mPlaybackRate = ComputePlaybackRate(&result.mPlaybackRateReliable);
result.mDecoderPosition = mDecoderPosition;
result.mPlaybackPosition = mPlaybackPosition;
}
else {
result.mDownloadRate = 0;
result.mDownloadRateReliable = true;
result.mPlaybackRate = 0;
result.mPlaybackRateReliable = true;
result.mDecoderPosition = 0;
result.mPlaybackPosition = 0;
result.mDownloadPosition = 0;
result.mTotalBytes = 0;
}
MOZ_ASSERT(mResource);
MediaStatistics result;
result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable);
result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition);
result.mTotalBytes = mResource->GetLength();
result.mPlaybackRate = mPlaybackBytesPerSecond;
result.mPlaybackRateReliable = mPlaybackRateReliable;
result.mDecoderPosition = mDecoderPosition;
result.mPlaybackPosition = mPlaybackPosition;
return result;
}
double MediaDecoder::ComputePlaybackRate(bool* aReliable)
void
MediaDecoder::ComputePlaybackRate()
{
GetReentrantMonitor().AssertCurrentThreadIn();
MOZ_ASSERT(NS_IsMainThread() || OnStateMachineTaskQueue() || OnDecodeTaskQueue());
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
MOZ_ASSERT(mResource);
int64_t length = mResource ? mResource->GetLength() : -1;
int64_t length = mResource->GetLength();
if (!IsNaN(mDuration) && !mozilla::IsInfinite<double>(mDuration) && length >= 0) {
*aReliable = true;
return length * mDuration;
mPlaybackRateReliable = true;
mPlaybackBytesPerSecond = length / mDuration;
return;
}
return mPlaybackStatistics->GetRateAtLastStop(aReliable);
bool reliable = false;
mPlaybackBytesPerSecond = mPlaybackStatistics->GetRateAtLastStop(&reliable);
mPlaybackRateReliable = reliable;
}
void MediaDecoder::UpdatePlaybackRate()
void
MediaDecoder::UpdatePlaybackRate()
{
MOZ_ASSERT(NS_IsMainThread());
GetReentrantMonitor().AssertCurrentThreadIn();
if (!mResource)
return;
bool reliable;
uint32_t rate = uint32_t(ComputePlaybackRate(&reliable));
if (reliable) {
MOZ_ASSERT(mResource);
ComputePlaybackRate();
uint32_t rate = mPlaybackBytesPerSecond;
if (mPlaybackRateReliable) {
// Avoid passing a zero rate
rate = std::max(rate, 1u);
}
else {
} else {
// Set a minimum rate of 10,000 bytes per second ... sometimes we just
// don't have good data
rate = std::max(rate, 10000u);
}
mResource->SetPlaybackRate(rate);
}
void MediaDecoder::NotifySuspendedStatusChanged()
void
MediaDecoder::NotifySuspendedStatusChanged()
{
MOZ_ASSERT(NS_IsMainThread());
if (mResource && mOwner) {
@@ -876,7 +914,8 @@ void MediaDecoder::NotifySuspendedStatusChanged()
}
}
void MediaDecoder::NotifyBytesDownloaded()
void
MediaDecoder::NotifyBytesDownloaded()
{
MOZ_ASSERT(NS_IsMainThread());
{
@@ -888,7 +927,8 @@ void MediaDecoder::NotifyBytesDownloaded()
}
}
void MediaDecoder::NotifyDownloadEnded(nsresult aStatus)
void
MediaDecoder::NotifyDownloadEnded(nsresult aStatus)
{
MOZ_ASSERT(NS_IsMainThread());
@@ -917,7 +957,8 @@ void MediaDecoder::NotifyDownloadEnded(nsresult aStatus)
}
}
void MediaDecoder::NotifyPrincipalChanged()
void
MediaDecoder::NotifyPrincipalChanged()
{
MOZ_ASSERT(NS_IsMainThread());
if (mOwner) {
@@ -925,7 +966,8 @@ void MediaDecoder::NotifyPrincipalChanged()
}
}
void MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset)
void
MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset)
{
MOZ_ASSERT(NS_IsMainThread());
@@ -941,7 +983,8 @@ void MediaDecoder::NotifyBytesConsumed(int64_t aBytes, int64_t aOffset)
mDecoderPosition = aOffset + aBytes;
}
void MediaDecoder::OnSeekResolved(SeekResolveValue aVal)
void
MediaDecoder::OnSeekResolved(SeekResolveValue aVal)
{
MOZ_ASSERT(NS_IsMainThread());
mSeekRequest.Complete();
@@ -976,7 +1019,8 @@ void MediaDecoder::OnSeekResolved(SeekResolveValue aVal)
}
}
void MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility)
void
MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility)
{
MOZ_ASSERT(NS_IsMainThread());
if (mShuttingDown)
@@ -989,7 +1033,8 @@ void MediaDecoder::SeekingStarted(MediaDecoderEventVisibility aEventVisibility)
}
}
void MediaDecoder::ChangeState(PlayState aState)
void
MediaDecoder::ChangeState(PlayState aState)
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -1003,7 +1048,7 @@ void MediaDecoder::ChangeState(PlayState aState)
}
DECODER_LOG("ChangeState %s => %s",
gPlayStateStr[mPlayState], gPlayStateStr[aState]);
ToPlayStateStr(mPlayState), ToPlayStateStr(aState));
mPlayState = aState;
if (mPlayState == PLAY_STATE_PLAYING) {
@@ -1017,7 +1062,8 @@ void MediaDecoder::ChangeState(PlayState aState)
StartDormantTimer();
}
void MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisibility)
void
MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisibility)
{
MOZ_ASSERT(NS_IsMainThread());
if (mShuttingDown)
@@ -1044,7 +1090,8 @@ void MediaDecoder::UpdateLogicalPosition(MediaDecoderEventVisibility aEventVisib
}
}
void MediaDecoder::DurationChanged()
void
MediaDecoder::DurationChanged()
{
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
@@ -1079,7 +1126,8 @@ void MediaDecoder::DurationChanged()
}
}
void MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration)
void
MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration)
{
MOZ_ASSERT(NS_IsMainThread());
@@ -1099,7 +1147,8 @@ void MediaDecoder::UpdateEstimatedMediaDuration(int64_t aDuration)
mEstimatedDuration = Some(TimeUnit::FromMicroseconds(aDuration));
}
void MediaDecoder::SetMediaSeekable(bool aMediaSeekable) {
void
MediaDecoder::SetMediaSeekable(bool aMediaSeekable) {
MOZ_ASSERT(NS_IsMainThread());
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
mMediaSeekable = aMediaSeekable;
@@ -1108,18 +1157,20 @@ void MediaDecoder::SetMediaSeekable(bool aMediaSeekable) {
bool
MediaDecoder::IsTransportSeekable()
{
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
MOZ_ASSERT(NS_IsMainThread());
return GetResource()->IsTransportSeekable();
}
bool MediaDecoder::IsMediaSeekable()
bool
MediaDecoder::IsMediaSeekable()
{
MOZ_ASSERT(NS_IsMainThread());
NS_ENSURE_TRUE(GetStateMachine(), false);
ReentrantMonitorAutoEnter mon(GetReentrantMonitor());
return mMediaSeekable;
}
media::TimeIntervals MediaDecoder::GetSeekable()
media::TimeIntervals
MediaDecoder::GetSeekable()
{
MOZ_ASSERT(NS_IsMainThread());
// We can seek in buffered range if the media is seekable. Also, we can seek
@@ -1138,7 +1189,8 @@ media::TimeIntervals MediaDecoder::GetSeekable()
}
}
void MediaDecoder::SetFragmentEndTime(double aTime)
void
MediaDecoder::SetFragmentEndTime(double aTime)
{
MOZ_ASSERT(NS_IsMainThread());
if (mDecoderStateMachine) {
@@ -1146,7 +1198,8 @@ void MediaDecoder::SetFragmentEndTime(double aTime)
}
}
void MediaDecoder::Suspend()
void
MediaDecoder::Suspend()
{
MOZ_ASSERT(NS_IsMainThread());
if (mResource) {
@@ -1154,7 +1207,8 @@ void MediaDecoder::Suspend()
}
}
void MediaDecoder::Resume(bool aForceBuffering)
void
MediaDecoder::Resume(bool aForceBuffering)
{
MOZ_ASSERT(NS_IsMainThread());
if (mResource) {
@@ -1167,7 +1221,8 @@ void MediaDecoder::Resume(bool aForceBuffering)
}
}
void MediaDecoder::SetLoadInBackground(bool aLoadInBackground)
void
MediaDecoder::SetLoadInBackground(bool aLoadInBackground)
{
MOZ_ASSERT(NS_IsMainThread());
if (mResource) {
@@ -1175,18 +1230,14 @@ void MediaDecoder::SetLoadInBackground(bool aLoadInBackground)
}
}
void MediaDecoder::UpdatePlaybackOffset(int64_t aOffset)
{
GetReentrantMonitor().AssertCurrentThreadIn();
mPlaybackPosition = aOffset;
}
bool MediaDecoder::OnStateMachineTaskQueue() const
bool
MediaDecoder::OnStateMachineTaskQueue() const
{
return mDecoderStateMachine->OnTaskQueue();
}
void MediaDecoder::SetPlaybackRate(double aPlaybackRate)
void
MediaDecoder::SetPlaybackRate(double aPlaybackRate)
{
MOZ_ASSERT(NS_IsMainThread());
mPlaybackRate = aPlaybackRate;
@@ -1204,13 +1255,15 @@ void MediaDecoder::SetPlaybackRate(double aPlaybackRate)
}
}
void MediaDecoder::SetPreservesPitch(bool aPreservesPitch)
void
MediaDecoder::SetPreservesPitch(bool aPreservesPitch)
{
MOZ_ASSERT(NS_IsMainThread());
mPreservesPitch = aPreservesPitch;
}
bool MediaDecoder::OnDecodeTaskQueue() const {
bool
MediaDecoder::OnDecodeTaskQueue() const {
NS_WARN_IF_FALSE(mDecoderStateMachine, "mDecoderStateMachine is null");
return mDecoderStateMachine ? mDecoderStateMachine->OnDecodeTaskQueue() : false;
}
@@ -1228,32 +1281,38 @@ MediaDecoder::SetStateMachine(MediaDecoderStateMachine* aStateMachine)
mStateMachineIsShutdown.Connect(mDecoderStateMachine->CanonicalIsShutdown());
mNextFrameStatus.Connect(mDecoderStateMachine->CanonicalNextFrameStatus());
mCurrentPosition.Connect(mDecoderStateMachine->CanonicalCurrentPosition());
mPlaybackPosition.Connect(mDecoderStateMachine->CanonicalPlaybackOffset());
} else {
mStateMachineDuration.DisconnectIfConnected();
mBuffered.DisconnectIfConnected();
mStateMachineIsShutdown.DisconnectIfConnected();
mNextFrameStatus.DisconnectIfConnected();
mCurrentPosition.DisconnectIfConnected();
mPlaybackPosition.DisconnectIfConnected();
}
}
ReentrantMonitor& MediaDecoder::GetReentrantMonitor() {
ReentrantMonitor&
MediaDecoder::GetReentrantMonitor() {
return mReentrantMonitor;
}
ImageContainer* MediaDecoder::GetImageContainer()
ImageContainer*
MediaDecoder::GetImageContainer()
{
return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer() : nullptr;
}
void MediaDecoder::InvalidateWithFlags(uint32_t aFlags)
void
MediaDecoder::InvalidateWithFlags(uint32_t aFlags)
{
if (mVideoFrameContainer) {
mVideoFrameContainer->InvalidateWithFlags(aFlags);
}
}
void MediaDecoder::Invalidate()
void
MediaDecoder::Invalidate()
{
if (mVideoFrameContainer) {
mVideoFrameContainer->Invalidate();
@@ -1262,12 +1321,14 @@ void MediaDecoder::Invalidate()
// Constructs the time ranges representing what segments of the media
// are buffered and playable.
media::TimeIntervals MediaDecoder::GetBuffered() {
media::TimeIntervals
MediaDecoder::GetBuffered() {
MOZ_ASSERT(NS_IsMainThread());
return mBuffered.Ref();
}
size_t MediaDecoder::SizeOfVideoQueue() {
size_t
MediaDecoder::SizeOfVideoQueue() {
MOZ_ASSERT(NS_IsMainThread());
if (mDecoderStateMachine) {
return mDecoderStateMachine->SizeOfVideoQueue();
@@ -1275,7 +1336,8 @@ size_t MediaDecoder::SizeOfVideoQueue() {
return 0;
}
size_t MediaDecoder::SizeOfAudioQueue() {
size_t
MediaDecoder::SizeOfAudioQueue() {
MOZ_ASSERT(NS_IsMainThread());
if (mDecoderStateMachine) {
return mDecoderStateMachine->SizeOfAudioQueue();
@@ -1283,7 +1345,10 @@ size_t MediaDecoder::SizeOfAudioQueue() {
return 0;
}
void MediaDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aThrottleUpdates) {
void
MediaDecoder::NotifyDataArrived(uint32_t aLength,
int64_t aOffset,
bool aThrottleUpdates) {
MOZ_ASSERT(NS_IsMainThread());
if (mDecoderStateMachine) {
@@ -1296,7 +1361,8 @@ void MediaDecoder::NotifyDataArrived(uint32_t aLength, int64_t aOffset, bool aTh
}
// Provide access to the state machine object
MediaDecoderStateMachine* MediaDecoder::GetStateMachine() const {
MediaDecoderStateMachine*
MediaDecoder::GetStateMachine() const {
return mDecoderStateMachine;
}
@@ -1304,30 +1370,24 @@ void
MediaDecoder::NotifyWaitingForResourcesStatusChanged()
{
if (mDecoderStateMachine) {
RefPtr<nsRunnable> task =
NS_NewRunnableMethod(mDecoderStateMachine,
&MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged);
mDecoderStateMachine->OwnerThread()->Dispatch(task.forget());
mDecoderStateMachine->DispatchWaitingForResourcesStatusChanged();
}
}
bool MediaDecoder::IsShutdown() const {
MOZ_ASSERT(NS_IsMainThread());
NS_ENSURE_TRUE(GetStateMachine(), true);
return mStateMachineIsShutdown;
}
// Drop reference to state machine. Only called during shutdown dance.
void MediaDecoder::BreakCycles() {
void
MediaDecoder::BreakCycles() {
SetStateMachine(nullptr);
}
MediaDecoderOwner* MediaDecoder::GetMediaOwner() const
MediaDecoderOwner*
MediaDecoder::GetMediaOwner() const
{
return mOwner;
}
void MediaDecoder::FireTimeUpdate()
void
MediaDecoder::FireTimeUpdate()
{
MOZ_ASSERT(NS_IsMainThread());
if (!mOwner)
@@ -1335,7 +1395,8 @@ void MediaDecoder::FireTimeUpdate()
mOwner->FireTimeUpdate(true);
}
void MediaDecoder::PinForSeek()
void
MediaDecoder::PinForSeek()
{
MOZ_ASSERT(NS_IsMainThread());
MediaResource* resource = GetResource();
@@ -1346,7 +1407,8 @@ void MediaDecoder::PinForSeek()
resource->Pin();
}
void MediaDecoder::UnpinForSeek()
void
MediaDecoder::UnpinForSeek()
{
MOZ_ASSERT(NS_IsMainThread());
MediaResource* resource = GetResource();
@@ -1357,40 +1419,12 @@ void MediaDecoder::UnpinForSeek()
resource->Unpin();
}
bool MediaDecoder::CanPlayThrough()
bool
MediaDecoder::CanPlayThrough()
{
Statistics stats = GetStatistics();
MOZ_ASSERT(NS_IsMainThread());
NS_ENSURE_TRUE(mDecoderStateMachine, false);
if (mDecoderStateMachine->IsRealTime() ||
(stats.mTotalBytes < 0 && stats.mDownloadRateReliable) ||
(stats.mTotalBytes >= 0 && stats.mTotalBytes == stats.mDownloadPosition)) {
return true;
}
if (!stats.mDownloadRateReliable || !stats.mPlaybackRateReliable) {
return false;
}
int64_t bytesToDownload = stats.mTotalBytes - stats.mDownloadPosition;
int64_t bytesToPlayback = stats.mTotalBytes - stats.mPlaybackPosition;
double timeToDownload = bytesToDownload / stats.mDownloadRate;
double timeToPlay = bytesToPlayback / stats.mPlaybackRate;
if (timeToDownload > timeToPlay) {
// Estimated time to download is greater than the estimated time to play.
// We probably can't play through without having to stop to buffer.
return false;
}
// Estimated time to download is less than the estimated time to play.
// We can probably play through without having to buffer, but ensure that
// we've got a reasonable amount of data buffered after the current
// playback position, so that if the bitrate of the media fluctuates, or if
// our download rate or decode rate estimation is otherwise inaccurate,
// we don't suddenly discover that we need to buffer. This is particularly
// required near the start of the media, when not much data is downloaded.
int64_t readAheadMargin =
static_cast<int64_t>(stats.mPlaybackRate * CAN_PLAY_THROUGH_MARGIN);
return stats.mDownloadPosition > stats.mPlaybackPosition + readAheadMargin;
return mDecoderStateMachine->IsRealTime() || GetStatistics().CanPlayThrough();
}
#ifdef MOZ_RAW
dom/media/MediaDecoder.h
+45 -56
View File
@@ -199,11 +199,11 @@ destroying the MediaDecoder object.
#include "nsITimer.h"
#include "AbstractMediaDecoder.h"
#include "DecodedStream.h"
#include "MediaDecoderOwner.h"
#include "MediaEventSource.h"
#include "MediaMetadataManager.h"
#include "MediaResource.h"
#include "MediaStatistics.h"
#include "MediaStreamGraph.h"
#include "TimeUnits.h"
@@ -518,16 +518,13 @@ public:
bool OnDecodeTaskQueue() const override;
MediaDecoderStateMachine* GetStateMachine() { return mDecoderStateMachine; }
MediaDecoderStateMachine* GetStateMachine() const;
void SetStateMachine(MediaDecoderStateMachine* aStateMachine);
// Returns the monitor for other threads to synchronise access to
// state.
ReentrantMonitor& GetReentrantMonitor() override;
// Returns true if the decoder is shut down
bool IsShutdown() const final override;
// Constructs the time ranges representing what segments of the media
// are buffered and playable.
virtual media::TimeIntervals GetBuffered();
@@ -564,13 +561,15 @@ public:
// Records activity stopping on the channel.
void DispatchPlaybackStopped() {
nsRefPtr<MediaDecoder> self = this;
nsCOMPtr<nsIRunnable> r =
NS_NewRunnableFunction([self] () { self->mPlaybackStatistics->Stop(); });
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () {
self->mPlaybackStatistics->Stop();
self->ComputePlaybackRate();
});
AbstractThread::MainThread()->Dispatch(r.forget());
}
// The actual playback rate computation. The monitor must be held.
virtual double ComputePlaybackRate(bool* aReliable);
void ComputePlaybackRate();
// Returns true if we can play the entire media through without stopping
// to buffer, given the current download and playback rates.
@@ -648,13 +647,6 @@ public:
// position.
int64_t GetDownloadPosition();
// Updates the approximate byte offset which playback has reached. This is
// used to calculate the readyState transitions.
void UpdatePlaybackOffset(int64_t aOffset);
// Provide access to the state machine object
MediaDecoderStateMachine* GetStateMachine() const;
// Drop reference to state machine. Only called during shutdown dance.
virtual void BreakCycles();
@@ -701,37 +693,11 @@ public:
static bool IsAppleMP3Enabled();
#endif
struct Statistics {
// Estimate of the current playback rate (bytes/second).
double mPlaybackRate;
// Estimate of the current download rate (bytes/second). This
// ignores time that the channel was paused by Gecko.
double mDownloadRate;
// Total length of media stream in bytes; -1 if not known
int64_t mTotalBytes;
// Current position of the download, in bytes. This is the offset of
// the first uncached byte after the decoder position.
int64_t mDownloadPosition;
// Current position of decoding, in bytes (how much of the stream
// has been consumed)
int64_t mDecoderPosition;
// Current position of playback, in bytes
int64_t mPlaybackPosition;
// If false, then mDownloadRate cannot be considered a reliable
// estimate (probably because the download has only been running
// a short time).
bool mDownloadRateReliable;
// If false, then mPlaybackRate cannot be considered a reliable
// estimate (probably because playback has only been running
// a short time).
bool mPlaybackRateReliable;
};
// Return statistics. This is used for progress events and other things.
// This can be called from any thread. It's only a snapshot of the
// current state, since other threads might be changing the state
// at any time.
virtual Statistics GetStatistics();
MediaStatistics GetStatistics();
// Frame decoding/painting related performance counters.
// Threadsafe.
@@ -890,17 +856,6 @@ protected:
// Whether the decoder implementation supports dormant mode.
bool mDormantSupported;
// Current decoding position in the stream. This is where the decoder
// is up to consuming the stream. This is not adjusted during decoder
// seek operations, but it's updated at the end when we start playing
// back again.
int64_t mDecoderPosition;
// Current playback position in the stream. This is (approximately)
// where we're up to playing back the stream. This is not adjusted
// during decoder seek operations, but it's updated at the end when we
// start playing back again.
int64_t mPlaybackPosition;
// The logical playback position of the media resource in units of
// seconds. This corresponds to the "official position" in HTML5. Note that
// we need to store this as a double, rather than an int64_t (like
@@ -917,9 +872,6 @@ protected:
// Official duration of the media resource as observed by script.
double mDuration;
// True if the media is seekable (i.e. supports random access).
bool mMediaSeekable;
/******
* The following member variables can be accessed from any thread.
******/
@@ -948,8 +900,12 @@ private:
ReentrantMonitor mReentrantMonitor;
protected:
virtual void CallSeek(const SeekTarget& aTarget);
// Returns true if heuristic dormant is supported.
bool IsHeuristicDormantSupported() const;
MozPromiseRequestHolder<SeekPromise> mSeekRequest;
// True when seeking or otherwise moving the play position around in
@@ -1060,6 +1016,12 @@ protected:
// Duration of the media resource according to the state machine.
Mirror<media::NullableTimeUnit> mStateMachineDuration;
// Current playback position in the stream. This is (approximately)
// where we're up to playing back the stream. This is not adjusted
// during decoder seek operations, but it's updated at the end when we
// start playing back again.
Mirror<int64_t> mPlaybackPosition;
// Volume of playback. 0.0 = muted. 1.0 = full volume.
Canonical<double> mVolume;
@@ -1097,6 +1059,21 @@ protected:
// passed to MediaStreams when this is true.
Canonical<bool> mSameOriginMedia;
// Estimate of the current playback rate (bytes/second).
Canonical<double> mPlaybackBytesPerSecond;
// True if mPlaybackBytesPerSecond is a reliable estimate.
Canonical<bool> mPlaybackRateReliable;
// Current decoding position in the stream. This is where the decoder
// is up to consuming the stream. This is not adjusted during decoder
// seek operations, but it's updated at the end when we start playing
// back again.
Canonical<int64_t> mDecoderPosition;
// True if the media is seekable (i.e. supports random access).
Canonical<bool> mMediaSeekable;
public:
AbstractCanonical<media::NullableTimeUnit>* CanonicalDurationOrNull() override;
AbstractCanonical<double>* CanonicalVolume() {
@@ -1126,6 +1103,18 @@ public:
AbstractCanonical<bool>* CanonicalSameOriginMedia() {
return &mSameOriginMedia;
}
AbstractCanonical<double>* CanonicalPlaybackBytesPerSecond() {
return &mPlaybackBytesPerSecond;
}
AbstractCanonical<bool>* CanonicalPlaybackRateReliable() {
return &mPlaybackRateReliable;
}
AbstractCanonical<int64_t>* CanonicalDecoderPosition() {
return &mDecoderPosition;
}
AbstractCanonical<bool>* CanonicalMediaSeekable() {
return &mMediaSeekable;
}
};
} // namespace mozilla
dom/media/MediaDecoderReader.cpp
+2 -4
View File
@@ -295,8 +295,7 @@ public:
// Make sure ResetDecode hasn't been called in the mean time.
if (!mReader->mBaseVideoPromise.IsEmpty()) {
mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold,
/* aForceDecodeAhead = */ false);
mReader->RequestVideoData(/* aSkip = */ true, mTimeThreshold);
}
return NS_OK;
@@ -333,8 +332,7 @@ private:
nsRefPtr<MediaDecoderReader::VideoDataPromise>
MediaDecoderReader::RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead)
int64_t aTimeThreshold)
{
nsRefPtr<VideoDataPromise> p = mBaseVideoPromise.Ensure(__func__);
bool skip = aSkipToNextKeyframe;
dom/media/MediaDecoderReader.h
+1 -1
View File
@@ -149,7 +149,7 @@ public:
// If aSkipToKeyframe is true, the decode should skip ahead to the
// the next keyframe at or after aTimeThreshold microseconds.
virtual nsRefPtr<VideoDataPromise>
RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead);
RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold);
friend class ReRequestVideoWithSkipTask;
friend class ReRequestAudioTask;
dom/media/MediaDecoderStateMachine.cpp
+216 -207
View File
@@ -9,40 +9,42 @@
#include "mmsystem.h"
#endif
#include "mozilla/DebugOnly.h"
#include <algorithm>
#include <stdint.h>
#include "MediaDecoderStateMachine.h"
#include "MediaTimer.h"
#include "gfx2DGlue.h"
#include "mediasink/DecodedAudioDataSink.h"
#include "mediasink/AudioSinkWrapper.h"
#include "nsTArray.h"
#include "MediaDecoder.h"
#include "MediaDecoderReader.h"
#include "mozilla/MathAlgorithms.h"
#include "mediasink/DecodedStream.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/Logging.h"
#include "mozilla/mozalloc.h"
#include "VideoUtils.h"
#include "TimeUnits.h"
#include "nsDeque.h"
#include "AudioSegment.h"
#include "VideoSegment.h"
#include "ImageContainer.h"
#include "nsComponentManagerUtils.h"
#include "nsITimer.h"
#include "nsContentUtils.h"
#include "MediaShutdownManager.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Preferences.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/TaskQueue.h"
#include "nsIEventTarget.h"
#include "prenv.h"
#include "mozilla/Preferences.h"
#include "gfx2DGlue.h"
#include "nsPrintfCString.h"
#include "DOMMediaStream.h"
#include "DecodedStream.h"
#include "mozilla/Logging.h"
#include <algorithm>
#include "nsComponentManagerUtils.h"
#include "nsContentUtils.h"
#include "nsIEventTarget.h"
#include "nsITimer.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
#include "nsDeque.h"
#include "prenv.h"
#include "AudioSegment.h"
#include "DOMMediaStream.h"
#include "ImageContainer.h"
#include "MediaDecoder.h"
#include "MediaDecoderReader.h"
#include "MediaDecoderStateMachine.h"
#include "MediaShutdownManager.h"
#include "MediaTimer.h"
#include "TimeUnits.h"
#include "VideoSegment.h"
#include "VideoUtils.h"
namespace mozilla {
@@ -172,10 +174,11 @@ static int64_t DurationToUsecs(TimeDuration aDuration) {
static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
static const uint32_t SCARCE_VIDEO_QUEUE_SIZE = 1;
static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueHWAccelSize = MIN_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueSendToCompositorSize = VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
MediaDecoderReader* aReader,
@@ -215,12 +218,12 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mDropVideoUntilNextDiscontinuity(false),
mDecodeToSeekTarget(false),
mCurrentTimeBeforeSeek(0),
mCorruptFrames(30),
mCorruptFrames(60),
mDecodingFirstFrame(true),
mSentLoadedMetadataEvent(false),
mSentFirstFrameLoadedEvent(false),
mSentPlaybackEndedEvent(false),
mDecodedStream(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)),
mStreamSink(new DecodedStream(mTaskQueue, mAudioQueue, mVideoQueue)),
mResource(aDecoder->GetResource()),
mBuffered(mTaskQueue, TimeIntervals(),
"MediaDecoderStateMachine::mBuffered (Mirror)"),
@@ -241,6 +244,14 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
"MediaDecoderStateMachine::mPreservesPitch (Mirror)"),
mSameOriginMedia(mTaskQueue, false,
"MediaDecoderStateMachine::mSameOriginMedia (Mirror)"),
mPlaybackBytesPerSecond(mTaskQueue, 0.0,
"MediaDecoderStateMachine::mPlaybackBytesPerSecond (Mirror)"),
mPlaybackRateReliable(mTaskQueue, true,
"MediaDecoderStateMachine::mPlaybackRateReliable (Mirror)"),
mDecoderPosition(mTaskQueue, 0,
"MediaDecoderStateMachine::mDecoderPosition (Mirror)"),
mMediaSeekable(mTaskQueue, true,
"MediaDecoderStateMachine::mMediaSeekable (Mirror)"),
mDuration(mTaskQueue, NullableTimeUnit(),
"MediaDecoderStateMachine::mDuration (Canonical"),
mIsShutdown(mTaskQueue, false,
@@ -248,7 +259,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mNextFrameStatus(mTaskQueue, MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED,
"MediaDecoderStateMachine::mNextFrameStatus (Canonical)"),
mCurrentPosition(mTaskQueue, 0,
"MediaDecoderStateMachine::mCurrentPosition (Canonical)")
"MediaDecoderStateMachine::mCurrentPosition (Canonical)"),
mPlaybackOffset(mTaskQueue, 0,
"MediaDecoderStateMachine::mPlaybackOffset (Canonical)")
{
MOZ_COUNT_CTOR(MediaDecoderStateMachine);
NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
@@ -266,6 +279,9 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
Preferences::AddUintVarCache(&sVideoQueueHWAccelSize,
"media.video-queue.hw-accel-size",
MIN_VIDEO_QUEUE_SIZE);
Preferences::AddUintVarCache(&sVideoQueueSendToCompositorSize,
"media.video-queue.send-to-compositor-size",
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
}
mBufferingWait = IsRealTime() ? 0 : 15;
@@ -287,14 +303,7 @@ MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
nsRefPtr<MediaDecoderStateMachine> self = this;
auto audioSinkCreator = [self] () {
MOZ_ASSERT(self->OnTaskQueue());
return new DecodedAudioDataSink(
self->mAudioQueue, self->GetMediaTime(),
self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
};
mAudioSink = new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
mMediaSink = CreateAudioSink();
}
MediaDecoderStateMachine::~MediaDecoderStateMachine()
@@ -325,6 +334,10 @@ MediaDecoderStateMachine::InitializationTask()
mLogicalPlaybackRate.Connect(mDecoder->CanonicalPlaybackRate());
mPreservesPitch.Connect(mDecoder->CanonicalPreservesPitch());
mSameOriginMedia.Connect(mDecoder->CanonicalSameOriginMedia());
mPlaybackBytesPerSecond.Connect(mDecoder->CanonicalPlaybackBytesPerSecond());
mPlaybackRateReliable.Connect(mDecoder->CanonicalPlaybackRateReliable());
mDecoderPosition.Connect(mDecoder->CanonicalDecoderPosition());
mMediaSeekable.Connect(mDecoder->CanonicalMediaSeekable());
// Initialize watchers.
mWatchManager.Watch(mBuffered, &MediaDecoderStateMachine::BufferedRangeUpdated);
@@ -344,6 +357,19 @@ MediaDecoderStateMachine::InitializationTask()
SameOriginMediaChanged();
}
media::MediaSink*
MediaDecoderStateMachine::CreateAudioSink()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
auto audioSinkCreator = [self] () {
MOZ_ASSERT(self->OnTaskQueue());
return new DecodedAudioDataSink(
self->mAudioQueue, self->GetMediaTime(),
self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
};
return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
}
bool MediaDecoderStateMachine::HasFutureAudio()
{
MOZ_ASSERT(OnTaskQueue());
@@ -373,7 +399,7 @@ int64_t MediaDecoderStateMachine::GetDecodedAudioDuration()
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
int64_t audioDecoded = AudioQueue().Duration();
if (mAudioSink->IsStarted()) {
if (mMediaSink->IsStarted()) {
audioDecoded += AudioEndTime() - GetMediaTime();
}
return audioDecoded;
@@ -383,7 +409,6 @@ void MediaDecoderStateMachine::DiscardStreamData()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
MOZ_ASSERT(!mAudioSink->IsStarted(), "Should've been stopped in RunStateMachine()");
const auto clockTime = GetClock();
while (true) {
@@ -678,7 +703,7 @@ MediaDecoderStateMachine::OnAudioPopped(const nsRefPtr<MediaData>& aSample)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->UpdatePlaybackOffset(std::max<int64_t>(0, aSample->mOffset));
mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
UpdateNextFrameStatus();
DispatchAudioDecodeTaskIfNeeded();
}
@@ -688,7 +713,7 @@ MediaDecoderStateMachine::OnVideoPopped(const nsRefPtr<MediaData>& aSample)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecoder->UpdatePlaybackOffset(aSample->mOffset);
mPlaybackOffset = std::max(mPlaybackOffset.Ref(), aSample->mOffset);
UpdateNextFrameStatus();
DispatchVideoDecodeTaskIfNeeded();
}
@@ -1073,8 +1098,7 @@ void MediaDecoderStateMachine::MaybeStartPlayback()
SetPlayStartTime(TimeStamp::Now());
MOZ_ASSERT(IsPlaying());
StartAudioSink();
StartDecodedStream();
StartMediaSink();
DispatchDecodeTasksIfNeeded();
}
@@ -1148,8 +1172,7 @@ void MediaDecoderStateMachine::VolumeChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mAudioSink->SetVolume(mVolume);
mDecodedStream->SetVolume(mVolume);
mMediaSink->SetVolume(mVolume);
}
void MediaDecoderStateMachine::RecomputeDuration()
@@ -1184,7 +1207,16 @@ void MediaDecoderStateMachine::RecomputeDuration()
mDuration = Some(duration);
}
void MediaDecoderStateMachine::SetDormant(bool aDormant)
void
MediaDecoderStateMachine::DispatchSetDormant(bool aDormant)
{
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetDormant, aDormant);
OwnerThread()->Dispatch(r.forget());
}
void
MediaDecoderStateMachine::SetDormant(bool aDormant)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
@@ -1266,7 +1298,7 @@ void MediaDecoderStateMachine::Shutdown()
Reset();
mAudioSink->Shutdown();
mMediaSink->Shutdown();
// Shut down our start time rendezvous.
if (mStartTimeRendezvous) {
@@ -1329,7 +1361,16 @@ void MediaDecoderStateMachine::StartDecoding()
ScheduleStateMachine();
}
void MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged()
void
MediaDecoderStateMachine::DispatchWaitingForResourcesStatusChanged()
{
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethod(
this, &MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged);
OwnerThread()->Dispatch(r.forget());
}
void
MediaDecoderStateMachine::NotifyWaitingForResourcesStatusChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
@@ -1398,7 +1439,7 @@ void MediaDecoderStateMachine::SameOriginMediaChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mDecodedStream->SetSameOrigin(mSameOriginMedia);
mStreamSink->SetSameOrigin(mSameOriginMedia);
}
void MediaDecoderStateMachine::BufferedRangeUpdated()
@@ -1431,7 +1472,7 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
// We need to be able to seek both at a transport level and at a media level
// to seek.
if (!mDecoder->IsMediaSeekable()) {
if (!mMediaSeekable) {
DECODER_WARN("Seek() function should not be called on a non-seekable state machine");
return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true, __func__);
}
@@ -1457,15 +1498,22 @@ MediaDecoderStateMachine::Seek(SeekTarget aTarget)
return mPendingSeek.mPromise.Ensure(__func__);
}
void MediaDecoderStateMachine::StopAudioSink()
nsRefPtr<MediaDecoder::SeekPromise>
MediaDecoderStateMachine::InvokeSeek(SeekTarget aTarget)
{
return InvokeAsync(OwnerThread(), this, __func__,
&MediaDecoderStateMachine::Seek, aTarget);
}
void MediaDecoderStateMachine::StopMediaSink()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioSink->IsStarted()) {
DECODER_LOG("Stop AudioSink");
mAudioSink->Stop();
mAudioSinkPromise.DisconnectIfExists();
if (mMediaSink->IsStarted()) {
DECODER_LOG("Stop MediaSink");
mMediaSink->Stop();
mMediaSinkPromise.DisconnectIfExists();
}
}
@@ -1714,8 +1762,6 @@ MediaDecoderStateMachine::RequestVideoData()
bool skipToNextKeyFrame = mSentFirstFrameLoadedEvent &&
NeedToSkipToNextKeyframe();
int64_t currentTime = mState == DECODER_STATE_SEEKING ? 0 : GetMediaTime();
bool forceDecodeAhead = mSentFirstFrameLoadedEvent &&
static_cast<uint32_t>(VideoQueue().GetSize()) <= SCARCE_VIDEO_QUEUE_SIZE;
SAMPLE_LOG("Queueing video task - queued=%i, decoder-queued=%o, skip=%i, time=%lld",
VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), skipToNextKeyFrame,
@@ -1725,7 +1771,7 @@ MediaDecoderStateMachine::RequestVideoData()
mVideoDataRequest.Begin(
InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__,
&MediaDecoderReader::RequestVideoData,
skipToNextKeyFrame, currentTime, forceDecodeAhead)
skipToNextKeyFrame, currentTime)
->Then(OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnVideoDecoded,
&MediaDecoderStateMachine::OnVideoNotDecoded));
@@ -1733,7 +1779,7 @@ MediaDecoderStateMachine::RequestVideoData()
mVideoDataRequest.Begin(
InvokeAsync(DecodeTaskQueue(), mReader.get(), __func__,
&MediaDecoderReader::RequestVideoData,
skipToNextKeyFrame, currentTime, forceDecodeAhead)
skipToNextKeyFrame, currentTime)
->Then(OwnerThread(), __func__, mStartTimeRendezvous.get(),
&StartTimeRendezvous::ProcessFirstSample<VideoDataPromise>,
&StartTimeRendezvous::FirstSampleRejected<VideoData>)
@@ -1745,55 +1791,25 @@ MediaDecoderStateMachine::RequestVideoData()
}
void
MediaDecoderStateMachine::StartAudioSink()
MediaDecoderStateMachine::StartMediaSink()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioCaptured) {
MOZ_ASSERT(!mAudioSink->IsStarted());
return;
}
if (!mAudioSink->IsStarted()) {
if (!mMediaSink->IsStarted()) {
mAudioCompleted = false;
mAudioSink->Start(GetMediaTime(), mInfo);
mMediaSink->Start(GetMediaTime(), mInfo);
auto promise = mAudioSink->OnEnded(TrackInfo::kAudioTrack);
auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
if (promise) {
mAudioSinkPromise.Begin(promise->Then(
mMediaSinkPromise.Begin(promise->Then(
OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnAudioSinkComplete,
&MediaDecoderStateMachine::OnAudioSinkError));
&MediaDecoderStateMachine::OnMediaSinkComplete,
&MediaDecoderStateMachine::OnMediaSinkError));
}
}
}
void
MediaDecoderStateMachine::StopDecodedStream()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
mDecodedStream->StopPlayback();
mDecodedStreamPromise.DisconnectIfExists();
}
void
MediaDecoderStateMachine::StartDecodedStream()
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
// Tell DecodedStream to start playback with specified start time and media
// info. This is consistent with how we create AudioSink in StartAudioThread().
if (mAudioCaptured && !mDecodedStreamPromise.Exists()) {
mDecodedStreamPromise.Begin(
mDecodedStream->StartPlayback(GetMediaTime(), mInfo)->Then(
OwnerThread(), __func__, this,
&MediaDecoderStateMachine::OnDecodedStreamFinish,
&MediaDecoderStateMachine::OnDecodedStreamError));
}
}
int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
{
MOZ_ASSERT(OnTaskQueue());
@@ -1802,7 +1818,7 @@ int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
// The amount of audio we have decoded is the amount of audio data we've
// already decoded and pushed to the hardware, plus the amount of audio
// data waiting to be pushed to the hardware.
int64_t pushed = mAudioSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0;
int64_t pushed = mMediaSink->IsStarted() ? (AudioEndTime() - GetMediaTime()) : 0;
// Currently for real time streams, AudioQueue().Duration() produce
// wrong values (Bug 1114434), so we use frame counts to calculate duration.
@@ -1831,7 +1847,7 @@ bool MediaDecoderStateMachine::OutOfDecodedAudio()
MOZ_ASSERT(OnTaskQueue());
return IsAudioDecoding() && !AudioQueue().IsFinished() &&
AudioQueue().GetSize() == 0 &&
!mAudioSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
!mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack);
}
bool MediaDecoderStateMachine::HasLowUndecodedData()
@@ -2047,7 +2063,7 @@ MediaDecoderStateMachine::FinishDecodeFirstFrame()
DECODER_LOG("Media duration %lld, "
"transportSeekable=%d, mediaSeekable=%d",
Duration().ToMicroseconds(), mDecoder->IsTransportSeekable(), mDecoder->IsMediaSeekable());
Duration().ToMicroseconds(), mResource->IsTransportSeekable(), mMediaSeekable.Ref());
if (HasAudio() && !HasVideo() && !mSentFirstFrameLoadedEvent) {
// We're playing audio only. We don't need to worry about slow video
@@ -2179,6 +2195,15 @@ private:
nsRefPtr<MediaDecoderStateMachine> mStateMachine;
};
void
MediaDecoderStateMachine::DispatchShutdown()
{
mStreamSink->BeginShutdown();
nsCOMPtr<nsIRunnable> runnable =
NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown);
OwnerThread()->Dispatch(runnable.forget());
}
void
MediaDecoderStateMachine::FinishShutdown()
{
@@ -2205,10 +2230,16 @@ MediaDecoderStateMachine::FinishShutdown()
mLogicalPlaybackRate.DisconnectIfConnected();
mPreservesPitch.DisconnectIfConnected();
mSameOriginMedia.DisconnectIfConnected();
mPlaybackBytesPerSecond.DisconnectIfConnected();
mPlaybackRateReliable.DisconnectIfConnected();
mDecoderPosition.DisconnectIfConnected();
mMediaSeekable.DisconnectIfConnected();
mDuration.DisconnectAll();
mIsShutdown.DisconnectAll();
mNextFrameStatus.DisconnectAll();
mCurrentPosition.DisconnectAll();
mPlaybackOffset.DisconnectAll();
// Shut down the watch manager before shutting down our task queue.
mWatchManager.Shutdown();
@@ -2310,7 +2341,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
if (mReader->UseBufferingHeuristics()) {
TimeDuration elapsed = now - mBufferingStart;
bool isLiveStream = resource->IsLiveStream();
if ((isLiveStream || !mDecoder->CanPlayThrough()) &&
if ((isLiveStream || !CanPlayThrough()) &&
elapsed < TimeDuration::FromSeconds(mBufferingWait * mPlaybackRate) &&
(mQuickBuffering ? HasLowDecodedData(mQuickBufferingLowDataThresholdUsecs)
: HasLowUndecodedData(mBufferingWait * USECS_PER_S)) &&
@@ -2359,7 +2390,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
// end of the media, and so that we update the readyState.
if (VideoQueue().GetSize() > 1 ||
(HasAudio() && !mAudioCompleted) ||
(mAudioCaptured && !mDecodedStream->IsFinished()))
(mAudioCaptured && !mStreamSink->IsFinished()))
{
// Start playback if necessary to play the remaining media.
MaybeStartPlayback();
@@ -2396,8 +2427,7 @@ nsresult MediaDecoderStateMachine::RunStateMachine()
mSentPlaybackEndedEvent = true;
// MediaSink::GetEndTime() must be called before stopping playback.
StopAudioSink();
StopDecodedStream();
StopMediaSink();
}
return NS_OK;
@@ -2423,11 +2453,10 @@ MediaDecoderStateMachine::Reset()
mState == DECODER_STATE_DORMANT ||
mState == DECODER_STATE_DECODING_NONE);
// Stop the audio thread. Otherwise, AudioSink might be accessing AudioQueue
// Stop the audio thread. Otherwise, MediaSink might be accessing AudioQueue
// outside of the decoder monitor while we are clearing the queue and causes
// crash for no samples to be popped.
StopAudioSink();
StopDecodedStream();
StopMediaSink();
mVideoFrameEndTime = -1;
mDecodedVideoEndTime = -1;
@@ -2447,6 +2476,8 @@ MediaDecoderStateMachine::Reset()
mVideoWaitRequest.DisconnectIfExists();
mSeekRequest.DisconnectIfExists();
mPlaybackOffset = 0;
nsCOMPtr<nsIRunnable> resetTask =
NS_NewRunnableMethod(mReader, &MediaDecoderReader::ResetDecode);
DecodeTaskQueue()->Dispatch(resetTask.forget());
@@ -2472,8 +2503,8 @@ bool MediaDecoderStateMachine::CheckFrameValidity(VideoData* aData)
// only supports integer types.
mCorruptFrames.insert(10);
if (mReader->VideoIsHardwareAccelerated() &&
frameStats.GetPresentedFrames() > 30 &&
mCorruptFrames.mean() >= 1 /* 10% */) {
frameStats.GetPresentedFrames() > 60 &&
mCorruptFrames.mean() >= 2 /* 20% */) {
nsCOMPtr<nsIRunnable> task =
NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration);
DecodeTaskQueue()->Dispatch(task.forget());
@@ -2552,13 +2583,6 @@ void MediaDecoderStateMachine::RenderVideoFrames(int32_t aMaxFrames,
container->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
}
int64_t MediaDecoderStateMachine::GetStreamClock() const
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
return mDecodedStream->GetPosition();
}
int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
{
MOZ_ASSERT(OnTaskQueue());
@@ -2573,11 +2597,7 @@ int64_t MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
if (!IsPlaying()) {
clock_time = mPlayDuration;
} else {
if (mAudioCaptured) {
clock_time = GetStreamClock();
} else {
clock_time = mAudioSink->GetPosition(&t);
}
clock_time = mMediaSink->GetPosition(&t);
NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards.");
}
if (aTimeStamp) {
@@ -2633,7 +2653,7 @@ void MediaDecoderStateMachine::UpdateRenderedVideoFrames()
}
}
RenderVideoFrames(1, clockTime, nowTime);
RenderVideoFrames(sVideoQueueSendToCompositorSize, clockTime, nowTime);
// Check to see if we don't have enough data to play up to the next frame.
// If we don't, switch to buffering mode.
@@ -2828,6 +2848,28 @@ bool MediaDecoderStateMachine::JustExitedQuickBuffering()
(TimeStamp::Now() - mDecodeStartTime) < TimeDuration::FromMicroseconds(QUICK_BUFFER_THRESHOLD_USECS);
}
bool
MediaDecoderStateMachine::CanPlayThrough()
{
MOZ_ASSERT(OnTaskQueue());
return IsRealTime() || GetStatistics().CanPlayThrough();
}
MediaStatistics
MediaDecoderStateMachine::GetStatistics()
{
MOZ_ASSERT(OnTaskQueue());
MediaStatistics result;
result.mDownloadRate = mResource->GetDownloadRate(&result.mDownloadRateReliable);
result.mDownloadPosition = mResource->GetCachedDataEnd(mDecoderPosition);
result.mTotalBytes = mResource->GetLength();
result.mPlaybackRate = mPlaybackBytesPerSecond;
result.mPlaybackRateReliable = mPlaybackRateReliable;
result.mDecoderPosition = mDecoderPosition;
result.mPlaybackPosition = mPlaybackOffset;
return result;
}
void MediaDecoderStateMachine::StartBuffering()
{
MOZ_ASSERT(OnTaskQueue());
@@ -2858,7 +2900,7 @@ void MediaDecoderStateMachine::StartBuffering()
SetState(DECODER_STATE_BUFFERING);
DECODER_LOG("Changed state from DECODING to BUFFERING, decoded for %.3lfs",
decodeDuration.ToSeconds());
MediaDecoder::Statistics stats = mDecoder->GetStatistics();
MediaStatistics stats = GetStatistics();
DECODER_LOG("Playback rate: %.1lfKB/s%s download rate: %.1lfKB/s%s",
stats.mPlaybackRate/1024, stats.mPlaybackRateReliable ? "" : " (unreliable)",
stats.mDownloadRate/1024, stats.mDownloadRateReliable ? "" : " (unreliable)");
@@ -2869,12 +2911,7 @@ void MediaDecoderStateMachine::SetPlayStartTime(const TimeStamp& aTimeStamp)
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
mPlayStartTime = aTimeStamp;
mAudioSink->SetPlaying(!mPlayStartTime.IsNull());
// Have DecodedStream remember the playing state so it doesn't need to
// ask MDSM about IsPlaying(). Note we have to do this even before capture
// happens since capture could happen in the middle of playback.
mDecodedStream->SetPlaying(!mPlayStartTime.IsNull());
mMediaSink->SetPlaying(!mPlayStartTime.IsNull());
}
void MediaDecoderStateMachine::ScheduleStateMachineWithLockAndWakeDecoder()
@@ -2952,7 +2989,7 @@ MediaDecoderStateMachine::LogicalPlaybackRateChanged()
}
mPlaybackRate = mLogicalPlaybackRate;
mAudioSink->SetPlaybackRate(mPlaybackRate);
mMediaSink->SetPlaybackRate(mPlaybackRate);
ScheduleStateMachine();
}
@@ -2961,7 +2998,7 @@ void MediaDecoderStateMachine::PreservesPitchChanged()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
mAudioSink->SetPreservesPitch(mPreservesPitch);
mMediaSink->SetPreservesPitch(mPreservesPitch);
}
bool MediaDecoderStateMachine::IsShutdown()
@@ -2975,33 +3012,33 @@ MediaDecoderStateMachine::AudioEndTime() const
{
MOZ_ASSERT(OnTaskQueue());
AssertCurrentThreadInMonitor();
if (mAudioSink->IsStarted()) {
return mAudioSink->GetEndTime(TrackInfo::kAudioTrack);
} else if (mAudioCaptured) {
return mDecodedStream->AudioEndTime();
if (mMediaSink->IsStarted()) {
return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
}
MOZ_ASSERT(!HasAudio());
return -1;
}
void MediaDecoderStateMachine::OnAudioSinkComplete()
void MediaDecoderStateMachine::OnMediaSinkComplete()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
mAudioSinkPromise.Complete();
mAudioCompleted = true;
mMediaSinkPromise.Complete();
// Set true only when we have audio.
mAudioCompleted = mInfo.HasAudio();
// To notify PlaybackEnded as soon as possible.
ScheduleStateMachine();
}
void MediaDecoderStateMachine::OnAudioSinkError()
void MediaDecoderStateMachine::OnMediaSinkError()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
mAudioSinkPromise.Complete();
mAudioCompleted = true;
mMediaSinkPromise.Complete();
// Set true only when we have audio.
mAudioCompleted = mInfo.HasAudio();
// Make the best effort to continue playback when there is video.
if (HasVideo()) {
@@ -3014,31 +3051,41 @@ void MediaDecoderStateMachine::OnAudioSinkError()
}
void
MediaDecoderStateMachine::OnDecodedStreamFinish()
MediaDecoderStateMachine::SetAudioCaptured(bool aCaptured)
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
mDecodedStreamPromise.Complete();
if (mInfo.HasAudio()) {
mAudioCompleted = true;
if (aCaptured == mAudioCaptured) {
return;
}
// To notify PlaybackEnded as soon as possible.
// Backup current playback parameters.
MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams();
// Stop and shut down the existing sink.
StopMediaSink();
mMediaSink->Shutdown();
// Create a new sink according to whether audio is captured.
// TODO: We can't really create a new DecodedStream until OutputStreamManager
// is extracted. It is tricky that the implementation of DecodedStream
// happens to allow reuse after shutdown without creating a new one.
mMediaSink = aCaptured ? mStreamSink : CreateAudioSink();
// Restore playback parameters.
mMediaSink->SetPlaybackParams(params);
// Start the sink if we are already playing. Otherwise it will be
// handled in MaybeStartPlayback().
if (IsPlaying()) {
StartMediaSink();
}
mAudioCaptured = aCaptured;
ScheduleStateMachine();
}
void
MediaDecoderStateMachine::OnDecodedStreamError()
{
MOZ_ASSERT(OnTaskQueue());
ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
MOZ_ASSERT(mAudioCaptured, "Audio should be captured.");
mDecodedStreamPromise.Complete();
DecodeError();
}
uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
{
MOZ_ASSERT(OnTaskQueue());
@@ -3048,64 +3095,26 @@ uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const
: std::max<uint32_t>(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE);
}
void MediaDecoderStateMachine::DispatchAudioCaptured()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () -> void
{
MOZ_ASSERT(self->OnTaskQueue());
ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
if (!self->mAudioCaptured) {
// Stop the audio sink if it's running.
self->StopAudioSink();
self->mAudioCaptured = true;
// Start DecodedStream if we are already playing. Otherwise it will be
// handled in MaybeStartPlayback().
if (self->IsPlaying()) {
self->StartDecodedStream();
}
self->ScheduleStateMachine();
}
});
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::DispatchAudioUncaptured()
{
nsRefPtr<MediaDecoderStateMachine> self = this;
nsCOMPtr<nsIRunnable> r = NS_NewRunnableFunction([self] () -> void
{
MOZ_ASSERT(self->OnTaskQueue());
ReentrantMonitorAutoEnter mon(self->mDecoder->GetReentrantMonitor());
if (self->mAudioCaptured) {
self->StopDecodedStream();
// Start again the audio sink.
self->mAudioCaptured = false;
if (self->IsPlaying()) {
self->StartAudioSink();
}
self->ScheduleStateMachine();
}
});
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::AddOutputStream(ProcessedMediaStream* aStream,
bool aFinishWhenEnded)
{
MOZ_ASSERT(NS_IsMainThread());
DECODER_LOG("AddOutputStream aStream=%p!", aStream);
mDecodedStream->AddOutput(aStream, aFinishWhenEnded);
DispatchAudioCaptured();
mStreamSink->AddOutput(aStream, aFinishWhenEnded);
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetAudioCaptured, true);
OwnerThread()->Dispatch(r.forget());
}
void MediaDecoderStateMachine::RemoveOutputStream(MediaStream* aStream)
{
MOZ_ASSERT(NS_IsMainThread());
DECODER_LOG("RemoveOutputStream=%p!", aStream);
mDecodedStream->RemoveOutput(aStream);
if (!mDecodedStream->HasConsumers()) {
DispatchAudioUncaptured();
mStreamSink->RemoveOutput(aStream);
if (!mStreamSink->HasConsumers()) {
nsCOMPtr<nsIRunnable> r = NS_NewRunnableMethodWithArg<bool>(
this, &MediaDecoderStateMachine::SetAudioCaptured, false);
OwnerThread()->Dispatch(r.forget());
}
}
dom/media/MediaDecoderStateMachine.h
+53 -41
View File
@@ -93,8 +93,8 @@ hardware (via AudioStream).
#include "MediaDecoderOwner.h"
#include "MediaEventSource.h"
#include "MediaMetadataManager.h"
#include "MediaStatistics.h"
#include "MediaTimer.h"
#include "DecodedStream.h"
#include "ImageContainer.h"
namespace mozilla {
@@ -104,6 +104,7 @@ class MediaSink;
}
class AudioSegment;
class DecodedStream;
class TaskQueue;
extern PRLogModuleInfo* gMediaDecoderLog;
@@ -155,7 +156,7 @@ public:
void RemoveOutputStream(MediaStream* aStream);
// Set/Unset dormant state.
void SetDormant(bool aDormant);
void DispatchSetDormant(bool aDormant);
TimedMetadataEventSource& TimedMetadataEvent() {
return mMetadataManager.TimedMetadataEvent();
@@ -167,19 +168,18 @@ private:
// constructor immediately after the task queue is created.
void InitializationTask();
void DispatchAudioCaptured();
void DispatchAudioUncaptured();
void SetDormant(bool aDormant);
void SetAudioCaptured(bool aCaptured);
void NotifyWaitingForResourcesStatusChanged();
nsRefPtr<MediaDecoder::SeekPromise> Seek(SeekTarget aTarget);
void Shutdown();
public:
void DispatchShutdown()
{
mDecodedStream->Shutdown();
nsCOMPtr<nsIRunnable> runnable =
NS_NewRunnableMethod(this, &MediaDecoderStateMachine::Shutdown);
OwnerThread()->Dispatch(runnable.forget());
}
public:
void DispatchShutdown();
void FinishShutdown();
@@ -192,8 +192,7 @@ public:
bool OnTaskQueue() const;
// Seeks to the decoder to aTarget asynchronously.
// Must be called on the state machine thread.
nsRefPtr<MediaDecoder::SeekPromise> Seek(SeekTarget aTarget);
nsRefPtr<MediaDecoder::SeekPromise> InvokeSeek(SeekTarget aTarget);
// Clear the flag indicating that a playback position change event
// is currently queued. This is called from the main thread and must
@@ -212,8 +211,12 @@ private:
// immediately stop playback and buffer downloaded data. Called on
// the state machine thread.
void StartBuffering();
public:
bool CanPlayThrough();
MediaStatistics GetStatistics();
public:
void DispatchStartBuffering()
{
nsCOMPtr<nsIRunnable> runnable =
@@ -343,7 +346,7 @@ public:
// Called when the reader may have acquired the hardware resources required
// to begin decoding.
void NotifyWaitingForResourcesStatusChanged();
void DispatchWaitingForResourcesStatusChanged();
// Notifies the state machine that should minimize the number of samples
// decoded we preroll, until playback starts. The first time playback starts
@@ -457,8 +460,6 @@ protected:
// parties.
void UpdateNextFrameStatus();
int64_t GetStreamClock() const;
// Return the current time, either the audio clock if available (if the media
// has audio, and the playback is possible), or a clock for the video.
// Called on the state machine thread.
@@ -497,19 +498,17 @@ protected:
// state machine thread.
void UpdateRenderedVideoFrames();
// Stops the audio sink and shut it down.
media::MediaSink* CreateAudioSink();
// Stops the media sink and shut it down.
// The decoder monitor must be held with exactly one lock count.
// Called on the state machine thread.
void StopAudioSink();
void StopMediaSink();
// Create and start the audio sink.
// Create and start the media sink.
// The decoder monitor must be held with exactly one lock count.
// Called on the state machine thread.
void StartAudioSink();
void StopDecodedStream();
void StartDecodedStream();
void StartMediaSink();
// Notification method invoked when mPlayState changes.
void PlayStateChanged();
@@ -656,16 +655,12 @@ protected:
void SetPlayStartTime(const TimeStamp& aTimeStamp);
private:
// Resolved by the AudioSink to signal that all outstanding work is complete
// Resolved by the MediaSink to signal that all outstanding work is complete
// and the sink is shutting down.
void OnAudioSinkComplete();
void OnMediaSinkComplete();
// Rejected by the AudioSink to signal errors.
void OnAudioSinkError();
void OnDecodedStreamFinish();
void OnDecodedStreamError();
// Rejected by the MediaSink to signal errors.
void OnMediaSinkError();
// Return true if the video decoder's decode speed can not catch up the
// play time.
@@ -985,15 +980,15 @@ private:
// Media Fragment end time in microseconds. Access controlled by decoder monitor.
int64_t mFragmentEndTime;
// The audio sink resource. Used on the state machine thread.
nsRefPtr<media::MediaSink> mAudioSink;
// The media sink resource. Used on the state machine thread.
nsRefPtr<media::MediaSink> mMediaSink;
// The reader, don't call its methods with the decoder monitor held.
// This is created in the state machine's constructor.
nsRefPtr<MediaDecoderReader> mReader;
// The end time of the last audio frame that's been pushed onto the audio sink
// or DecodedStream in microseconds. This will approximately be the end time
// The end time of the last audio frame that's been pushed onto the media sink
// in microseconds. This will approximately be the end time
// of the audio stream, unless another frame is pushed to the hardware.
int64_t AudioEndTime() const;
@@ -1272,13 +1267,12 @@ private:
// Only written on the main thread while holding the monitor. Therefore it
// can be read on any thread while holding the monitor, or on the main thread
// without holding the monitor.
nsRefPtr<DecodedStream> mDecodedStream;
nsRefPtr<DecodedStream> mStreamSink;
// Media data resource from the decoder.
nsRefPtr<MediaResource> mResource;
MozPromiseRequestHolder<GenericPromise> mAudioSinkPromise;
MozPromiseRequestHolder<GenericPromise> mDecodedStreamPromise;
MozPromiseRequestHolder<GenericPromise> mMediaSinkPromise;
MediaEventListener mAudioQueueListener;
MediaEventListener mVideoQueueListener;
@@ -1313,6 +1307,18 @@ private:
// passed to MediaStreams when this is true.
Mirror<bool> mSameOriginMedia;
// Estimate of the current playback rate (bytes/second).
Mirror<double> mPlaybackBytesPerSecond;
// True if mPlaybackBytesPerSecond is a reliable estimate.
Mirror<bool> mPlaybackRateReliable;
// Current decoding position in the stream.
Mirror<int64_t> mDecoderPosition;
// True if the media is seekable (i.e. supports random access).
Mirror<bool> mMediaSeekable;
// Duration of the media. This is guaranteed to be non-null after we finish
// decoding the first frame.
Canonical<media::NullableTimeUnit> mDuration;
@@ -1329,6 +1335,9 @@ private:
// playback position.
Canonical<int64_t> mCurrentPosition;
// Current playback position in the stream in bytes.
Canonical<int64_t> mPlaybackOffset;
public:
AbstractCanonical<media::TimeIntervals>* CanonicalBuffered() {
return mReader->CanonicalBuffered();
@@ -1345,6 +1354,9 @@ public:
AbstractCanonical<int64_t>* CanonicalCurrentPosition() {
return &mCurrentPosition;
}
AbstractCanonical<int64_t>* CanonicalPlaybackOffset() {
return &mPlaybackOffset;
}
};
} // namespace mozilla
dom/media/MediaFormatReader.cpp
+2 -5
View File
@@ -508,8 +508,7 @@ MediaFormatReader::ShouldSkip(bool aSkipToNextKeyframe, media::TimeUnit aTimeThr
nsRefPtr<MediaDecoderReader::VideoDataPromise>
MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead)
int64_t aTimeThreshold)
{
MOZ_ASSERT(OnTaskQueue());
MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(), "No sample requests allowed while seeking");
@@ -535,7 +534,6 @@ MediaFormatReader::RequestVideoData(bool aSkipToNextKeyframe,
return VideoDataPromise::CreateAndReject(CANCELED, __func__);
}
mVideo.mForceDecodeAhead = aForceDecodeAhead;
media::TimeUnit timeThreshold{media::TimeUnit::FromMicroseconds(aTimeThreshold)};
if (ShouldSkip(aSkipToNextKeyframe, timeThreshold)) {
Flush(TrackInfo::kVideoTrack);
@@ -735,12 +733,11 @@ MediaFormatReader::NeedInput(DecoderData& aDecoder)
return
!aDecoder.mDraining &&
!aDecoder.mError &&
(aDecoder.HasPromise() || aDecoder.mForceDecodeAhead) &&
aDecoder.HasPromise() &&
!aDecoder.mDemuxRequest.Exists() &&
aDecoder.mOutput.IsEmpty() &&
(aDecoder.mInputExhausted || !aDecoder.mQueuedSamples.IsEmpty() ||
aDecoder.mTimeThreshold.isSome() ||
aDecoder.mForceDecodeAhead ||
aDecoder.mNumSamplesInput - aDecoder.mNumSamplesOutput < aDecoder.mDecodeAhead);
}
dom/media/MediaFormatReader.h
+1 -4
View File
@@ -35,7 +35,7 @@ public:
size_t SizeOfAudioQueueInFrames() override;
nsRefPtr<VideoDataPromise>
RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold, bool aForceDecodeAhead) override;
RequestVideoData(bool aSkipToNextKeyframe, int64_t aTimeThreshold) override;
nsRefPtr<AudioDataPromise> RequestAudioData() override;
@@ -187,7 +187,6 @@ private:
: mOwner(aOwner)
, mType(aType)
, mDecodeAhead(aDecodeAhead)
, mForceDecodeAhead(false)
, mUpdateScheduled(false)
, mDemuxEOS(false)
, mWaitingForData(false)
@@ -221,7 +220,6 @@ private:
// Only accessed from reader's task queue.
uint32_t mDecodeAhead;
bool mForceDecodeAhead;
bool mUpdateScheduled;
bool mDemuxEOS;
bool mWaitingForData;
@@ -276,7 +274,6 @@ private:
void ResetState()
{
MOZ_ASSERT(mOwner->OnTaskQueue());
mForceDecodeAhead = false;
mDemuxEOS = false;
mWaitingForData = false;
mReceivedNewData = false;
dom/media/MediaManager.cpp
+16 -3
View File
@@ -1382,6 +1382,8 @@ MediaManager::IsInMediaThread()
// NOTE: never Dispatch(....,NS_DISPATCH_SYNC) to the MediaManager
// thread from the MainThread, as we NS_DISPATCH_SYNC to MainThread
// from MediaManager thread.
// Guaranteed never to return nullptr.
/* static */ MediaManager*
MediaManager::Get() {
if (!sSingleton) {
@@ -2038,7 +2040,14 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow,
nsCOMPtr<nsIDOMGetUserMediaErrorCallback> onFailure(aOnFailure);
uint64_t windowId = aWindow->WindowID();
AddWindowID(windowId);
StreamListeners* listeners = AddWindowID(windowId);
// Create a disabled listener to act as a placeholder
nsRefPtr<GetUserMediaCallbackMediaStreamListener> listener =
new GetUserMediaCallbackMediaStreamListener(mMediaThread, windowId);
// No need for locking because we always do this in the main thread.
listeners->AppendElement(listener);
bool fake = Preferences::GetBool("media.navigator.streams.fake");
@@ -2046,11 +2055,15 @@ MediaManager::EnumerateDevices(nsPIDOMWindow* aWindow,
dom::MediaSourceEnum::Camera,
dom::MediaSourceEnum::Microphone,
fake);
p->Then([onSuccess](SourceSet*& aDevices) mutable {
p->Then([onSuccess, windowId, listener](SourceSet*& aDevices) mutable {
ScopedDeletePtr<SourceSet> devices(aDevices); // grab result
nsRefPtr<MediaManager> mgr = MediaManager_GetInstance();
mgr->RemoveFromWindowList(windowId, listener);
nsCOMPtr<nsIWritableVariant> array = MediaManager_ToJSArray(*devices);
onSuccess->OnSuccess(array);
}, [onFailure](MediaStreamError& reason) mutable {
}, [onFailure, windowId, listener](MediaStreamError& reason) mutable {
nsRefPtr<MediaManager> mgr = MediaManager_GetInstance();
mgr->RemoveFromWindowList(windowId, listener);
onFailure->OnError(&reason);
});
return NS_OK;
dom/media/MediaStatistics.h
+82
View File
@@ -0,0 +1,82 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MediaStatistics_h_
#define MediaStatistics_h_
namespace mozilla {
struct MediaStatistics {
// Estimate of the current playback rate (bytes/second).
double mPlaybackRate;
// Estimate of the current download rate (bytes/second). This
// ignores time that the channel was paused by Gecko.
double mDownloadRate;
// Total length of media stream in bytes; -1 if not known
int64_t mTotalBytes;
// Current position of the download, in bytes. This is the offset of
// the first uncached byte after the decoder position.
int64_t mDownloadPosition;
// Current position of decoding, in bytes (how much of the stream
// has been consumed)
int64_t mDecoderPosition;
// Current position of playback, in bytes
int64_t mPlaybackPosition;
// If false, then mDownloadRate cannot be considered a reliable
// estimate (probably because the download has only been running
// a short time).
bool mDownloadRateReliable;
// If false, then mPlaybackRate cannot be considered a reliable
// estimate (probably because playback has only been running
// a short time).
bool mPlaybackRateReliable;
bool CanPlayThrough()
{
// Number of estimated seconds worth of data we need to have buffered
// ahead of the current playback position before we allow the media decoder
// to report that it can play through the entire media without the decode
// catching up with the download. Having this margin make the
// CanPlayThrough() calculation more stable in the case of
// fluctuating bitrates.
static const int64_t CAN_PLAY_THROUGH_MARGIN = 1;
if ((mTotalBytes < 0 && mDownloadRateReliable) ||
(mTotalBytes >= 0 && mTotalBytes == mDownloadPosition)) {
return true;
}
if (!mDownloadRateReliable || !mPlaybackRateReliable) {
return false;
}
int64_t bytesToDownload = mTotalBytes - mDownloadPosition;
int64_t bytesToPlayback = mTotalBytes - mPlaybackPosition;
double timeToDownload = bytesToDownload / mDownloadRate;
double timeToPlay = bytesToPlayback / mPlaybackRate;
if (timeToDownload > timeToPlay) {
// Estimated time to download is greater than the estimated time to play.
// We probably can't play through without having to stop to buffer.
return false;
}
// Estimated time to download is less than the estimated time to play.
// We can probably play through without having to buffer, but ensure that
// we've got a reasonable amount of data buffered after the current
// playback position, so that if the bitrate of the media fluctuates, or if
// our download rate or decode rate estimation is otherwise inaccurate,
// we don't suddenly discover that we need to buffer. This is particularly
// required near the start of the media, when not much data is downloaded.
int64_t readAheadMargin =
static_cast<int64_t>(mPlaybackRate * CAN_PLAY_THROUGH_MARGIN);
return mDownloadPosition > mPlaybackPosition + readAheadMargin;
}
};
} // namespace mozilla
#endif // MediaStatistics_h_
dom/media/encoder/OmxTrackEncoder.cpp
+7 -9
View File
@@ -134,11 +134,8 @@ OmxVideoTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
layers::Image* img = (!mLastFrame.GetImage() || mLastFrame.GetForceBlack())
? nullptr : mLastFrame.GetImage();
rv = mEncoder->Encode(img, mFrameWidth, mFrameHeight, totalDurationUs,
OMXCodecWrapper::BUFFER_EOS);
OMXCodecWrapper::BUFFER_EOS, &mEosSetInEncoder);
NS_ENSURE_SUCCESS(rv, rv);
// Keep sending EOS signal until OMXVideoEncoder gets it.
mEosSetInEncoder = true;
}
// Dequeue an encoded frame from the output buffers of OMXCodecWrapper.
@@ -219,6 +216,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
PROFILER_LABEL("OmxAACAudioTrackEncoder", "GetEncodedTrack",
js::ProfileEntry::Category::OTHER);
AudioSegment segment;
bool EOS;
// Move all the samples from mRawSegment to segment. We only hold
// the monitor in this block.
{
@@ -234,14 +232,15 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
}
segment.AppendFrom(&mRawSegment);
EOS = mEndOfStream;
}
nsresult rv;
if (segment.GetDuration() == 0) {
// Notify EOS at least once, even if segment is empty.
if (mEndOfStream && !mEosSetInEncoder) {
mEosSetInEncoder = true;
rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS);
if (EOS && !mEosSetInEncoder) {
rv = mEncoder->Encode(segment, OMXCodecWrapper::BUFFER_EOS,
&mEosSetInEncoder);
NS_ENSURE_SUCCESS(rv, rv);
}
// Nothing to encode but encoder could still have encoded data for earlier
@@ -252,8 +251,7 @@ OmxAudioTrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
// OMX encoder has limited input buffers only so we have to feed input and get
// output more than once if there are too many samples pending in segment.
while (segment.GetDuration() > 0) {
rv = mEncoder->Encode(segment,
mEndOfStream ? OMXCodecWrapper::BUFFER_EOS : 0);
rv = mEncoder->Encode(segment, 0);
NS_ENSURE_SUCCESS(rv, rv);
rv = AppendEncodedFrames(aData);
dom/media/encoder/VP8TrackEncoder.cpp
+3 -1
View File
@@ -448,6 +448,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
{
PROFILER_LABEL("VP8TrackEncoder", "GetEncodedTrack",
js::ProfileEntry::Category::OTHER);
bool EOS;
{
// Move all the samples from mRawSegment to mSourceSegment. We only hold
// the monitor in this block.
@@ -463,6 +464,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
return NS_ERROR_FAILURE;
}
mSourceSegment.AppendFrom(&mRawSegment);
EOS = mEndOfStream;
}
VideoSegment::ChunkIterator iter(mSourceSegment);
@@ -536,7 +538,7 @@ VP8TrackEncoder::GetEncodedTrack(EncodedFrameContainer& aData)
VP8LOG("RemoveLeading %lld\n",totalProcessedDuration);
// End of stream, pull the rest frames in encoder.
if (mEndOfStream) {
if (EOS) {
VP8LOG("mEndOfStream is true\n");
mEncodingComplete = true;
if (vpx_codec_encode(mVPXContext, nullptr, mEncodedTimestamp,
dom/media/DecodedStream.cpp → dom/media/mediasink/DecodedStream.cpp
+79 -18
View File
@@ -4,6 +4,9 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/CheckedInt.h"
#include "mozilla/gfx/Point.h"
#include "AudioSegment.h"
#include "DecodedStream.h"
#include "MediaData.h"
@@ -358,7 +361,6 @@ DecodedStream::DecodedStream(AbstractThread* aOwnerThread,
: mOwnerThread(aOwnerThread)
, mShuttingDown(false)
, mPlaying(false)
, mVolume(1.0)
, mSameOrigin(false)
, mAudioQueue(aAudioQueue)
, mVideoQueue(aVideoQueue)
@@ -370,21 +372,52 @@ DecodedStream::~DecodedStream()
MOZ_ASSERT(mStartTime.isNothing(), "playback should've ended.");
}
const media::MediaSink::PlaybackParams&
DecodedStream::GetPlaybackParams() const
{
AssertOwnerThread();
return mParams;
}
void
DecodedStream::Shutdown()
DecodedStream::SetPlaybackParams(const PlaybackParams& aParams)
{
AssertOwnerThread();
mParams = aParams;
}
nsRefPtr<GenericPromise>
DecodedStream::OnEnded(TrackType aType)
{
AssertOwnerThread();
MOZ_ASSERT(mStartTime.isSome());
if (aType == TrackInfo::kAudioTrack) {
// TODO: we should return a promise which is resolved when the audio track
// is finished. For now this promise is resolved when the whole stream is
// finished.
return mFinishPromise;
}
// TODO: handle video track.
return nullptr;
}
void
DecodedStream::BeginShutdown()
{
MOZ_ASSERT(NS_IsMainThread());
mShuttingDown = true;
}
nsRefPtr<GenericPromise>
DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo)
void
DecodedStream::Start(int64_t aStartTime, const MediaInfo& aInfo)
{
AssertOwnerThread();
MOZ_ASSERT(mStartTime.isNothing(), "playback already started.");
mStartTime.emplace(aStartTime);
mInfo = aInfo;
mPlaying = true;
ConnectListener();
class R : public nsRunnable {
@@ -408,30 +441,33 @@ DecodedStream::StartPlayback(int64_t aStartTime, const MediaInfo& aInfo)
};
MozPromiseHolder<GenericPromise> promise;
nsRefPtr<GenericPromise> rv = promise.Ensure(__func__);
mFinishPromise = promise.Ensure(__func__);
nsCOMPtr<nsIRunnable> r = new R(this, &DecodedStream::CreateData, Move(promise));
AbstractThread::MainThread()->Dispatch(r.forget());
return rv.forget();
}
void DecodedStream::StopPlayback()
void
DecodedStream::Stop()
{
AssertOwnerThread();
// Playback didn't even start at all.
if (mStartTime.isNothing()) {
return;
}
MOZ_ASSERT(mStartTime.isSome(), "playback not started.");
mStartTime.reset();
DisconnectListener();
mFinishPromise = nullptr;
// Clear mData immediately when this playback session ends so we won't
// send data to the wrong stream in SendData() in next playback session.
DestroyData(Move(mData));
}
bool
DecodedStream::IsStarted() const
{
AssertOwnerThread();
return mStartTime.isSome();
}
void
DecodedStream::DestroyData(UniquePtr<DecodedStreamData> aData)
{
@@ -531,6 +567,12 @@ void
DecodedStream::SetPlaying(bool aPlaying)
{
AssertOwnerThread();
// Resume/pause matters only when playback started.
if (mStartTime.isNothing()) {
return;
}
mPlaying = aPlaying;
if (mData) {
mData->SetPlaying(aPlaying);
@@ -541,7 +583,21 @@ void
DecodedStream::SetVolume(double aVolume)
{
AssertOwnerThread();
mVolume = aVolume;
mParams.volume = aVolume;
}
void
DecodedStream::SetPlaybackRate(double aPlaybackRate)
{
AssertOwnerThread();
mParams.playbackRate = aPlaybackRate;
}
void
DecodedStream::SetPreservesPitch(bool aPreservesPitch)
{
AssertOwnerThread();
mParams.preservesPitch = aPreservesPitch;
}
void
@@ -815,7 +871,7 @@ DecodedStream::SendData()
}
InitTracks();
SendAudio(mVolume, mSameOrigin);
SendAudio(mParams.volume, mSameOrigin);
SendVideo(mSameOrigin);
AdvanceTracks();
@@ -829,26 +885,31 @@ DecodedStream::SendData()
}
int64_t
DecodedStream::AudioEndTime() const
DecodedStream::GetEndTime(TrackType aType) const
{
AssertOwnerThread();
if (mStartTime.isSome() && mInfo.HasAudio() && mData) {
if (aType == TrackInfo::kAudioTrack && mInfo.HasAudio() && mData) {
CheckedInt64 t = mStartTime.ref() +
FramesToUsecs(mData->mAudioFramesWritten, mInfo.mAudio.mRate);
if (t.isValid()) {
return t.value();
}
} else if (aType == TrackInfo::kVideoTrack && mData) {
return mData->mNextVideoTime;
}
return -1;
}
int64_t
DecodedStream::GetPosition() const
DecodedStream::GetPosition(TimeStamp* aTimeStamp) const
{
AssertOwnerThread();
// This is only called after MDSM starts playback. So mStartTime is
// guaranteed to be something.
MOZ_ASSERT(mStartTime.isSome());
if (aTimeStamp) {
*aTimeStamp = TimeStamp::Now();
}
return mStartTime.ref() + (mData ? mData->GetPosition() : 0);
}
dom/media/DecodedStream.h → dom/media/mediasink/DecodedStream.h
+29 -23
View File
@@ -10,15 +10,13 @@
#include "nsTArray.h"
#include "MediaEventSource.h"
#include "MediaInfo.h"
#include "MediaSink.h"
#include "mozilla/AbstractThread.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/Maybe.h"
#include "mozilla/MozPromise.h"
#include "mozilla/nsRefPtr.h"
#include "mozilla/ReentrantMonitor.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/gfx/Point.h"
namespace mozilla {
@@ -31,7 +29,7 @@ class MediaStreamGraph;
class OutputStreamListener;
class OutputStreamManager;
class ProcessedMediaStream;
class ReentrantMonitor;
class TimeStamp;
template <class T> class MediaQueue;
@@ -99,34 +97,41 @@ private:
nsTArray<OutputStreamData> mStreams;
};
class DecodedStream {
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecodedStream);
class DecodedStream : public media::MediaSink {
using media::MediaSink::PlaybackParams;
public:
DecodedStream(AbstractThread* aOwnerThread,
MediaQueue<MediaData>& aAudioQueue,
MediaQueue<MediaData>& aVideoQueue);
void Shutdown();
// MediaSink functions.
const PlaybackParams& GetPlaybackParams() const override;
void SetPlaybackParams(const PlaybackParams& aParams) override;
// Mimic MDSM::StartAudioThread.
// Must be called before any calls to SendData().
//
// Return a promise which will be resolved when the stream is finished
// or rejected if any error.
nsRefPtr<GenericPromise> StartPlayback(int64_t aStartTime,
const MediaInfo& aInfo);
// Mimic MDSM::StopAudioThread.
void StopPlayback();
nsRefPtr<GenericPromise> OnEnded(TrackType aType) override;
int64_t GetEndTime(TrackType aType) const override;
int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
bool HasUnplayedFrames(TrackType aType) const override
{
// TODO: implement this.
return false;
}
void SetVolume(double aVolume) override;
void SetPlaybackRate(double aPlaybackRate) override;
void SetPreservesPitch(bool aPreservesPitch) override;
void SetPlaying(bool aPlaying) override;
void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
void Stop() override;
bool IsStarted() const override;
// TODO: fix these functions that don't fit into the interface of MediaSink.
void BeginShutdown();
void AddOutput(ProcessedMediaStream* aStream, bool aFinishWhenEnded);
void RemoveOutput(MediaStream* aStream);
void SetPlaying(bool aPlaying);
void SetVolume(double aVolume);
void SetSameOrigin(bool aSameOrigin);
int64_t AudioEndTime() const;
int64_t GetPosition() const;
bool IsFinished() const;
bool HasConsumers() const;
@@ -164,10 +169,11 @@ private:
* Worker thread only members.
*/
UniquePtr<DecodedStreamData> mData;
nsRefPtr<GenericPromise> mFinishPromise;
bool mPlaying;
double mVolume;
bool mSameOrigin;
PlaybackParams mParams;
Maybe<int64_t> mStartTime;
MediaInfo mInfo;
dom/media/mediasink/moz.build
+1 -2
View File
@@ -7,8 +7,7 @@
UNIFIED_SOURCES += [
'AudioSinkWrapper.cpp',
'DecodedAudioDataSink.cpp',
'DecodedStream.cpp',
]
FINAL_LIBRARY = 'xul'
FAIL_ON_WARNINGS = True
dom/media/moz.build
+1 -2
View File
@@ -102,7 +102,6 @@ EXPORTS += [
'AudioStream.h',
'BufferMediaResource.h',
'CubebUtils.h',
'DecodedStream.h',
'DecoderTraits.h',
'DOMMediaStream.h',
'EncodedBufferCache.h',
@@ -125,6 +124,7 @@ EXPORTS += [
'MediaRecorder.h',
'MediaResource.h',
'MediaSegment.h',
'MediaStatistics.h',
'MediaStreamGraph.h',
'MediaTimer.h',
'MediaTrack.h',
@@ -200,7 +200,6 @@ UNIFIED_SOURCES += [
'AudioTrackList.cpp',
'CanvasCaptureMediaStream.cpp',
'CubebUtils.cpp',
'DecodedStream.cpp',
'DOMMediaStream.cpp',
'EncodedBufferCache.cpp',
'FileBlockCache.cpp',
dom/media/omx/MediaCodecReader.cpp
+1 -2
View File
@@ -353,8 +353,7 @@ MediaCodecReader::RequestAudioData()
nsRefPtr<MediaDecoderReader::VideoDataPromise>
MediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead)
int64_t aTimeThreshold)
{
MOZ_ASSERT(OnTaskQueue());
MOZ_ASSERT(HasVideo());
dom/media/omx/MediaCodecReader.h
+1 -2
View File
@@ -83,8 +83,7 @@ public:
// Disptach a DecodeVideoFrameTask to decode video data.
virtual nsRefPtr<VideoDataPromise>
RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead) override;
int64_t aTimeThreshold) override;
// Disptach a DecodeAduioDataTask to decode video data.
virtual nsRefPtr<AudioDataPromise> RequestAudioData() override;
dom/media/omx/MediaOmxCommonDecoder.cpp
+3 -18
View File
@@ -117,12 +117,7 @@ MediaOmxCommonDecoder::PauseStateMachine()
return;
}
// enter dormant state
RefPtr<nsRunnable> event =
NS_NewRunnableMethodWithArg<bool>(
GetStateMachine(),
&MediaDecoderStateMachine::SetDormant,
true);
GetStateMachine()->OwnerThread()->Dispatch(event.forget());
GetStateMachine()->DispatchSetDormant(true);
}
void
@@ -146,22 +141,12 @@ MediaOmxCommonDecoder::ResumeStateMachine()
SeekTarget::Accurate,
MediaDecoderEventVisibility::Suppressed);
// Call Seek of MediaDecoderStateMachine to suppress seek events.
RefPtr<nsRunnable> event =
NS_NewRunnableMethodWithArg<SeekTarget>(
GetStateMachine(),
&MediaDecoderStateMachine::Seek,
target);
GetStateMachine()->OwnerThread()->Dispatch(event.forget());
GetStateMachine()->InvokeSeek(target);
mNextState = mPlayState;
ChangeState(PLAY_STATE_LOADING);
// exit dormant state
event =
NS_NewRunnableMethodWithArg<bool>(
GetStateMachine(),
&MediaDecoderStateMachine::SetDormant,
false);
GetStateMachine()->OwnerThread()->Dispatch(event.forget());
GetStateMachine()->DispatchSetDormant(false);
UpdateLogicalPosition();
}
dom/media/omx/OMXCodecWrapper.cpp
+9 -3
View File
@@ -382,7 +382,7 @@ ConvertGrallocImageToNV12(GrallocImage* aSource, uint8_t* aDestination)
nsresult
OMXVideoEncoder::Encode(const Image* aImage, int aWidth, int aHeight,
int64_t aTimestamp, int aInputFlags)
int64_t aTimestamp, int aInputFlags, bool* aSendEOS)
{
MOZ_ASSERT(mStarted, "Configure() should be called before Encode().");
@@ -456,6 +456,9 @@ OMXVideoEncoder::Encode(const Image* aImage, int aWidth, int aHeight,
// Queue this input buffer.
result = mCodec->queueInputBuffer(index, 0, dstSize, aTimestamp, aInputFlags);
if (aSendEOS && (aInputFlags & BUFFER_EOS) && result == OK) {
*aSendEOS = true;
}
return result == OK ? NS_OK : NS_ERROR_FAILURE;
}
@@ -822,7 +825,8 @@ OMXAudioEncoder::~OMXAudioEncoder()
}
nsresult
OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags)
OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags,
bool* aSendEOS)
{
#ifndef MOZ_SAMPLE_TYPE_S16
#error MediaCodec accepts only 16-bit PCM data.
@@ -877,7 +881,9 @@ OMXAudioEncoder::Encode(AudioSegment& aSegment, int aInputFlags)
}
result = buffer.Enqueue(mTimestamp, flags);
NS_ENSURE_TRUE(result == OK, NS_ERROR_FAILURE);
if (aSendEOS && (aInputFlags & BUFFER_EOS)) {
*aSendEOS = true;
}
return NS_OK;
}
dom/media/omx/OMXCodecWrapper.h
+12 -2
View File
@@ -224,8 +224,13 @@ public:
* stream, set aInputFlags to BUFFER_EOS. Since encoder has limited buffers,
* this function might not be able to encode all chunks in one call, however
* it will remove chunks it consumes from aSegment.
* aSendEOS is the output to tell the caller EOS signal sent into MediaCodec
* because the signal might not be sent due to the dequeueInputBuffer timeout.
* And the value of aSendEOS won't be set to any default value, only set to
* true when EOS signal sent into MediaCodec.
*/
nsresult Encode(mozilla::AudioSegment& aSegment, int aInputFlags = 0);
nsresult Encode(mozilla::AudioSegment& aSegment, int aInputFlags = 0,
bool* aSendEOS = nullptr);
~OMXAudioEncoder();
protected:
@@ -299,9 +304,14 @@ public:
* semi-planar YUV420 format stored in the buffer of aImage. aTimestamp gives
* the frame timestamp/presentation time (in microseconds). To notify end of
* stream, set aInputFlags to BUFFER_EOS.
* aSendEOS is the output to tell the caller EOS signal sent into MediaCodec
* because the signal might not be sent due to the dequeueInputBuffer timeout.
* And the value of aSendEOS won't be set to any default value, only set to
* true when EOS signal sent into MediaCodec.
*/
nsresult Encode(const mozilla::layers::Image* aImage, int aWidth, int aHeight,
int64_t aTimestamp, int aInputFlags = 0);
int64_t aTimestamp, int aInputFlags = 0,
bool* aSendEOS = nullptr);
#if ANDROID_VERSION >= 18
/** Set encoding bitrate (in kbps). */
dom/media/omx/RtspMediaCodecReader.cpp
+2 -5
View File
@@ -82,13 +82,10 @@ RtspMediaCodecReader::RequestAudioData()
nsRefPtr<MediaDecoderReader::VideoDataPromise>
RtspMediaCodecReader::RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead)
int64_t aTimeThreshold)
{
EnsureActive();
return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe,
aTimeThreshold,
aForceDecodeAhead);
return MediaCodecReader::RequestVideoData(aSkipToNextKeyframe, aTimeThreshold);
}
nsRefPtr<MediaDecoderReader::MetadataPromise>
dom/media/omx/RtspMediaCodecReader.h
+1 -2
View File
@@ -53,8 +53,7 @@ public:
// Disptach a DecodeVideoFrameTask to decode video data.
virtual nsRefPtr<VideoDataPromise>
RequestVideoData(bool aSkipToNextKeyframe,
int64_t aTimeThreshold,
bool aForceDecodeAhead) override;
int64_t aTimeThreshold) override;
// Disptach a DecodeAudioDataTask to decode audio data.
virtual nsRefPtr<AudioDataPromise> RequestAudioData() override;
dom/media/systemservices/CamerasChild.cpp
+741
View File
@@ -0,0 +1,741 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "CamerasChild.h"
#include "CamerasUtils.h"
#include "webrtc/video_engine/include/vie_capture.h"
#undef FF
#include "mozilla/Assertions.h"
#include "mozilla/ipc/BackgroundChild.h"
#include "mozilla/ipc/PBackgroundChild.h"
#include "mozilla/Logging.h"
#include "mozilla/SyncRunnable.h"
#include "mozilla/WeakPtr.h"
#include "mozilla/unused.h"
#include "MediaUtils.h"
#include "nsThreadUtils.h"
#undef LOG
#undef LOG_ENABLED
PRLogModuleInfo *gCamerasChildLog;
#define LOG(args) MOZ_LOG(gCamerasChildLog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasChildLog, mozilla::LogLevel::Debug)
namespace mozilla {
namespace camera {
// We emulate the sync webrtc.org API with the help of singleton
// CamerasSingleton, which manages a pointer to an IPC object, a thread
// where IPC operations should run on, and a mutex.
// The static function Cameras() will use that Singleton to set up,
// if needed, both the thread and the associated IPC objects and return
// a pointer to the IPC object. Users can then do IPC calls on that object
// after dispatching them to aforementioned thread.
// 2 Threads are involved in this code:
// - the MediaManager thread, which will call the (static, sync API) functions
// through MediaEngineRemoteVideoSource
// - the Cameras IPC thread, which will be doing our IPC to the parent process
// via PBackground
// Our main complication is that we emulate a sync API while (having to do)
// async messaging. We dispatch the messages to another thread to send them
// async and hold a Monitor to wait for the result to be asynchronously received
// again. The requirement for async messaging originates on the parent side:
// it's not reasonable to block all PBackground IPC there while waiting for
// something like device enumeration to complete.
class CamerasSingleton {
public:
CamerasSingleton()
: mCamerasMutex("CamerasSingleton::mCamerasMutex"),
mCameras(nullptr),
mCamerasChildThread(nullptr) {
if (!gCamerasChildLog) {
gCamerasChildLog = PR_NewLogModule("CamerasChild");
}
LOG(("CamerasSingleton: %p", this));
}
~CamerasSingleton() {
LOG(("~CamerasSingleton: %p", this));
}
static CamerasSingleton& GetInstance() {
static CamerasSingleton instance;
return instance;
}
static OffTheBooksMutex& Mutex() {
return GetInstance().mCamerasMutex;
}
static CamerasChild*& Child() {
GetInstance().Mutex().AssertCurrentThreadOwns();
return GetInstance().mCameras;
}
static nsCOMPtr<nsIThread>& Thread() {
GetInstance().Mutex().AssertCurrentThreadOwns();
return GetInstance().mCamerasChildThread;
}
private:
// Reinitializing CamerasChild will change the pointers below.
// We don't want this to happen in the middle of preparing IPC.
// We will be alive on destruction, so this needs to be off the books.
mozilla::OffTheBooksMutex mCamerasMutex;
// This is owned by the IPC code, and the same code controls the lifetime.
// It will set and clear this pointer as appropriate in setup/teardown.
// We'd normally make this a WeakPtr but unfortunately the IPC code already
// uses the WeakPtr mixin in a protected base class of CamerasChild, and in
// any case the object becomes unusable as soon as IPC is tearing down, which
// will be before actual destruction.
CamerasChild* mCameras;
nsCOMPtr<nsIThread> mCamerasChildThread;
};
class InitializeIPCThread : public nsRunnable
{
public:
InitializeIPCThread()
: mCamerasChild(nullptr) {}
NS_IMETHOD Run() override {
// Try to get the PBackground handle
ipc::PBackgroundChild* existingBackgroundChild =
ipc::BackgroundChild::GetForCurrentThread();
// If it's not spun up yet, block until it is, and retry
if (!existingBackgroundChild) {
LOG(("No existingBackgroundChild"));
SynchronouslyCreatePBackground();
existingBackgroundChild =
ipc::BackgroundChild::GetForCurrentThread();
LOG(("BackgroundChild: %p", existingBackgroundChild));
}
// By now PBackground is guaranteed to be up
MOZ_RELEASE_ASSERT(existingBackgroundChild);
// Create CamerasChild
// We will be returning the resulting pointer (synchronously) to our caller.
mCamerasChild =
static_cast<mozilla::camera::CamerasChild*>(existingBackgroundChild->SendPCamerasConstructor());
return NS_OK;
}
CamerasChild* GetCamerasChild() {
MOZ_ASSERT(mCamerasChild);
return mCamerasChild;
}
private:
CamerasChild* mCamerasChild;
};
static CamerasChild*
Cameras() {
OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex());
if (!CamerasSingleton::Child()) {
MOZ_ASSERT(!NS_IsMainThread(), "Should not be on the main Thread");
if (!gCamerasChildLog) {
gCamerasChildLog = PR_NewLogModule("CamerasChild");
}
MOZ_ASSERT(!CamerasSingleton::Thread());
LOG(("No sCameras, setting up IPC Thread"));
nsresult rv = NS_NewNamedThread("Cameras IPC",
getter_AddRefs(CamerasSingleton::Thread()));
if (NS_FAILED(rv)) {
LOG(("Error launching IPC Thread"));
return nullptr;
}
// At this point we are in the MediaManager thread, and the thread we are
// dispatching to is the specific Cameras IPC thread that was just made
// above, so now we will fire off a runnable to run
// SynchronouslyCreatePBackground there, while we block in this thread.
// We block until the following happens in the Cameras IPC thread:
// 1) Creation of PBackground finishes
// 2) Creation of PCameras finishes by sending a message to the parent
nsRefPtr<InitializeIPCThread> runnable = new InitializeIPCThread();
nsRefPtr<SyncRunnable> sr = new SyncRunnable(runnable);
sr->DispatchToThread(CamerasSingleton::Thread());
CamerasSingleton::Child() = runnable->GetCamerasChild();
}
MOZ_ASSERT(CamerasSingleton::Child());
return CamerasSingleton::Child();
}
bool
CamerasChild::RecvReplyFailure(void)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = false;
monitor.Notify();
return true;
}
bool
CamerasChild::RecvReplySuccess(void)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
monitor.Notify();
return true;
}
int NumberOfCapabilities(CaptureEngine aCapEngine, const char* deviceUniqueIdUTF8)
{
return Cameras()->NumberOfCapabilities(aCapEngine, deviceUniqueIdUTF8);
}
bool
CamerasChild::RecvReplyNumberOfCapabilities(const int& numdev)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
mReplyInteger = numdev;
monitor.Notify();
return true;
}
bool
CamerasChild::DispatchToParent(nsIRunnable* aRunnable,
MonitorAutoLock& aMonitor)
{
{
OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex());
CamerasSingleton::Thread()->Dispatch(aRunnable, NS_DISPATCH_NORMAL);
}
// We can't see if the send worked, so we need to be able to bail
// out on shutdown (when it failed and we won't get a reply).
if (!mIPCIsAlive) {
return false;
}
// Guard against spurious wakeups.
mReceivedReply = false;
// Wait for a reply
do {
aMonitor.Wait();
} while (!mReceivedReply && mIPCIsAlive);
if (!mReplySuccess) {
return false;
}
return true;
}
int
CamerasChild::NumberOfCapabilities(CaptureEngine aCapEngine,
const char* deviceUniqueIdUTF8)
{
// Prevents multiple outstanding requests from happening.
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
LOG(("NumberOfCapabilities for %s", deviceUniqueIdUTF8));
nsCString unique_id(deviceUniqueIdUTF8);
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, unique_id]() -> nsresult {
if (this->SendNumberOfCapabilities(aCapEngine, unique_id)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
// Prevent concurrent use of the reply variables. Note
// that this is unlocked while waiting for the reply to be
// filled in, necessitating the first Mutex above.
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
LOG(("Get capture capability count failed"));
return 0;
}
LOG(("Capture capability count: %d", mReplyInteger));
return mReplyInteger;
}
int NumberOfCaptureDevices(CaptureEngine aCapEngine)
{
return Cameras()->NumberOfCaptureDevices(aCapEngine);
}
int
CamerasChild::NumberOfCaptureDevices(CaptureEngine aCapEngine)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine]() -> nsresult {
if (this->SendNumberOfCaptureDevices(aCapEngine)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
LOG(("Get NumberOfCaptureDevices failed"));
return 0;
}
LOG(("Capture Devices: %d", mReplyInteger));
return mReplyInteger;
}
bool
CamerasChild::RecvReplyNumberOfCaptureDevices(const int& numdev)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
mReplyInteger = numdev;
monitor.Notify();
return true;
}
int GetCaptureCapability(CaptureEngine aCapEngine, const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability)
{
return Cameras()->GetCaptureCapability(aCapEngine,
unique_idUTF8,
capability_number,
capability);
}
int
CamerasChild::GetCaptureCapability(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability)
{
MutexAutoLock requestLock(mRequestMutex);
LOG(("GetCaptureCapability: %s %d", unique_idUTF8, capability_number));
nsCString unique_id(unique_idUTF8);
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, unique_id, capability_number]() -> nsresult {
if (this->SendGetCaptureCapability(aCapEngine, unique_id, capability_number)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
return -1;
}
capability = mReplyCapability;
return 0;
}
bool
CamerasChild::RecvReplyGetCaptureCapability(const CaptureCapability& ipcCapability)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
mReplyCapability.width = ipcCapability.width();
mReplyCapability.height = ipcCapability.height();
mReplyCapability.maxFPS = ipcCapability.maxFPS();
mReplyCapability.expectedCaptureDelay = ipcCapability.expectedCaptureDelay();
mReplyCapability.rawType = static_cast<webrtc::RawVideoType>(ipcCapability.rawType());
mReplyCapability.codecType = static_cast<webrtc::VideoCodecType>(ipcCapability.codecType());
mReplyCapability.interlaced = ipcCapability.interlaced();
monitor.Notify();
return true;
}
int GetCaptureDevice(CaptureEngine aCapEngine,
unsigned int list_number, char* device_nameUTF8,
const unsigned int device_nameUTF8Length,
char* unique_idUTF8,
const unsigned int unique_idUTF8Length)
{
return Cameras()->GetCaptureDevice(aCapEngine,
list_number,
device_nameUTF8,
device_nameUTF8Length,
unique_idUTF8,
unique_idUTF8Length);
}
int
CamerasChild::GetCaptureDevice(CaptureEngine aCapEngine,
unsigned int list_number, char* device_nameUTF8,
const unsigned int device_nameUTF8Length,
char* unique_idUTF8,
const unsigned int unique_idUTF8Length)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, list_number]() -> nsresult {
if (this->SendGetCaptureDevice(aCapEngine, list_number)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
LOG(("GetCaptureDevice failed"));
return -1;
}
base::strlcpy(device_nameUTF8, mReplyDeviceName.get(), device_nameUTF8Length);
base::strlcpy(unique_idUTF8, mReplyDeviceID.get(), unique_idUTF8Length);
LOG(("Got %s name %s id", device_nameUTF8, unique_idUTF8));
return 0;
}
bool
CamerasChild::RecvReplyGetCaptureDevice(const nsCString& device_name,
const nsCString& device_id)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
mReplyDeviceName = device_name;
mReplyDeviceID = device_id;
monitor.Notify();
return true;
}
int AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int unique_idUTF8Length,
int& capture_id)
{
return Cameras()->AllocateCaptureDevice(aCapEngine,
unique_idUTF8,
unique_idUTF8Length,
capture_id);
}
int
CamerasChild::AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int unique_idUTF8Length,
int& capture_id)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
nsCString unique_id(unique_idUTF8);
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, unique_id]() -> nsresult {
if (this->SendAllocateCaptureDevice(aCapEngine, unique_id)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
LOG(("AllocateCaptureDevice failed"));
return -1;
}
LOG(("Capture Device allocated: %d", mReplyInteger));
capture_id = mReplyInteger;
return 0;
}
bool
CamerasChild::RecvReplyAllocateCaptureDevice(const int& numdev)
{
LOG((__PRETTY_FUNCTION__));
MonitorAutoLock monitor(mReplyMonitor);
mReceivedReply = true;
mReplySuccess = true;
mReplyInteger = numdev;
monitor.Notify();
return true;
}
int ReleaseCaptureDevice(CaptureEngine aCapEngine, const int capture_id)
{
return Cameras()->ReleaseCaptureDevice(aCapEngine, capture_id);
}
int
CamerasChild::ReleaseCaptureDevice(CaptureEngine aCapEngine,
const int capture_id)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, capture_id]() -> nsresult {
if (this->SendReleaseCaptureDevice(aCapEngine, capture_id)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
return -1;
}
return 0;
}
void
CamerasChild::AddCallback(const CaptureEngine aCapEngine, const int capture_id,
webrtc::ExternalRenderer* render)
{
MutexAutoLock lock(mCallbackMutex);
CapturerElement ce;
ce.engine = aCapEngine;
ce.id = capture_id;
ce.callback = render;
mCallbacks.AppendElement(ce);
}
void
CamerasChild::RemoveCallback(const CaptureEngine aCapEngine, const int capture_id)
{
MutexAutoLock lock(mCallbackMutex);
for (unsigned int i = 0; i < mCallbacks.Length(); i++) {
CapturerElement ce = mCallbacks[i];
if (ce.engine == aCapEngine && ce.id == capture_id) {
mCallbacks.RemoveElementAt(i);
break;
}
}
}
int StartCapture(CaptureEngine aCapEngine,
const int capture_id,
webrtc::CaptureCapability& webrtcCaps,
webrtc::ExternalRenderer* cb)
{
return Cameras()->StartCapture(aCapEngine,
capture_id,
webrtcCaps,
cb);
}
int
CamerasChild::StartCapture(CaptureEngine aCapEngine,
const int capture_id,
webrtc::CaptureCapability& webrtcCaps,
webrtc::ExternalRenderer* cb)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
AddCallback(aCapEngine, capture_id, cb);
CaptureCapability capCap(webrtcCaps.width,
webrtcCaps.height,
webrtcCaps.maxFPS,
webrtcCaps.expectedCaptureDelay,
webrtcCaps.rawType,
webrtcCaps.codecType,
webrtcCaps.interlaced);
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, capture_id, capCap]() -> nsresult {
if (this->SendStartCapture(aCapEngine, capture_id, capCap)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
return -1;
}
return 0;
}
int StopCapture(CaptureEngine aCapEngine, const int capture_id)
{
return Cameras()->StopCapture(aCapEngine, capture_id);
}
int
CamerasChild::StopCapture(CaptureEngine aCapEngine, const int capture_id)
{
MutexAutoLock requestLock(mRequestMutex);
LOG((__PRETTY_FUNCTION__));
nsCOMPtr<nsIRunnable> runnable =
media::NewRunnableFrom([this, aCapEngine, capture_id]() -> nsresult {
if (this->SendStopCapture(aCapEngine, capture_id)) {
return NS_OK;
}
return NS_ERROR_FAILURE;
});
MonitorAutoLock monitor(mReplyMonitor);
if (!DispatchToParent(runnable, monitor)) {
return -1;
}
RemoveCallback(aCapEngine, capture_id);
return 0;
}
void
Shutdown(void)
{
{
OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex());
if (!CamerasSingleton::Child()) {
// We don't want to cause everything to get fired up if we're
// really already shut down.
LOG(("Shutdown when already shut down"));
return;
}
}
Cameras()->Shutdown();
}
class ShutdownRunnable : public nsRunnable {
public:
ShutdownRunnable(nsRefPtr<nsRunnable> aReplyEvent,
nsIThread* aReplyThread)
: mReplyEvent(aReplyEvent), mReplyThread(aReplyThread) {};
NS_IMETHOD Run() override {
LOG(("Closing BackgroundChild"));
ipc::BackgroundChild::CloseForCurrentThread();
LOG(("PBackground thread exists, shutting down thread"));
mReplyThread->Dispatch(mReplyEvent, NS_DISPATCH_NORMAL);
return NS_OK;
}
private:
nsRefPtr<nsRunnable> mReplyEvent;
nsIThread* mReplyThread;
};
void
CamerasChild::Shutdown()
{
{
MonitorAutoLock monitor(mReplyMonitor);
mIPCIsAlive = false;
monitor.NotifyAll();
}
OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex());
if (CamerasSingleton::Thread()) {
LOG(("Dispatching actor deletion"));
// Delete the parent actor.
nsRefPtr<nsRunnable> deleteRunnable =
// CamerasChild (this) will remain alive and is only deleted by the
// IPC layer when SendAllDone returns.
media::NewRunnableFrom([this]() -> nsresult {
unused << this->SendAllDone();
return NS_OK;
});
CamerasSingleton::Thread()->Dispatch(deleteRunnable, NS_DISPATCH_NORMAL);
LOG(("PBackground thread exists, dispatching close"));
// Dispatch closing the IPC thread back to us when the
// BackgroundChild is closed.
nsRefPtr<nsRunnable> event =
new ThreadDestructor(CamerasSingleton::Thread());
nsRefPtr<ShutdownRunnable> runnable =
new ShutdownRunnable(event, NS_GetCurrentThread());
CamerasSingleton::Thread()->Dispatch(runnable, NS_DISPATCH_NORMAL);
} else {
LOG(("Shutdown called without PBackground thread"));
}
LOG(("Erasing sCameras & thread refs (original thread)"));
CamerasSingleton::Child() = nullptr;
CamerasSingleton::Thread() = nullptr;
}
bool
CamerasChild::RecvDeliverFrame(const int& capEngine,
const int& capId,
mozilla::ipc::Shmem&& shmem,
const int& size,
const uint32_t& time_stamp,
const int64_t& ntp_time,
const int64_t& render_time)
{
MutexAutoLock lock(mCallbackMutex);
CaptureEngine capEng = static_cast<CaptureEngine>(capEngine);
if (Callback(capEng, capId)) {
unsigned char* image = shmem.get<unsigned char>();
Callback(capEng, capId)->DeliverFrame(image, size,
time_stamp,
ntp_time, render_time,
nullptr);
} else {
LOG(("DeliverFrame called with dead callback"));
}
SendReleaseFrame(shmem);
return true;
}
bool
CamerasChild::RecvFrameSizeChange(const int& capEngine,
const int& capId,
const int& w, const int& h)
{
LOG((__PRETTY_FUNCTION__));
MutexAutoLock lock(mCallbackMutex);
CaptureEngine capEng = static_cast<CaptureEngine>(capEngine);
if (Callback(capEng, capId)) {
Callback(capEng, capId)->FrameSizeChange(w, h, 0);
} else {
LOG(("Frame size change with dead callback"));
}
return true;
}
void
CamerasChild::ActorDestroy(ActorDestroyReason aWhy)
{
MonitorAutoLock monitor(mReplyMonitor);
mIPCIsAlive = false;
// Hopefully prevent us from getting stuck
// on replies that'll never come.
monitor.NotifyAll();
}
CamerasChild::CamerasChild()
: mCallbackMutex("mozilla::cameras::CamerasChild::mCallbackMutex"),
mIPCIsAlive(true),
mRequestMutex("mozilla::cameras::CamerasChild::mRequestMutex"),
mReplyMonitor("mozilla::cameras::CamerasChild::mReplyMonitor")
{
if (!gCamerasChildLog) {
gCamerasChildLog = PR_NewLogModule("CamerasChild");
}
LOG(("CamerasChild: %p", this));
MOZ_COUNT_CTOR(CamerasChild);
}
CamerasChild::~CamerasChild()
{
LOG(("~CamerasChild: %p", this));
Shutdown();
MOZ_COUNT_DTOR(CamerasChild);
}
webrtc::ExternalRenderer* CamerasChild::Callback(CaptureEngine aCapEngine,
int capture_id)
{
for (unsigned int i = 0; i < mCallbacks.Length(); i++) {
CapturerElement ce = mCallbacks[i];
if (ce.engine == aCapEngine && ce.id == capture_id) {
return ce.callback;
}
}
return nullptr;
}
}
}
dom/media/systemservices/CamerasChild.h
+175
View File
@@ -0,0 +1,175 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_CamerasChild_h
#define mozilla_CamerasChild_h
#include "mozilla/Pair.h"
#include "mozilla/dom/ContentChild.h"
#include "mozilla/camera/PCamerasChild.h"
#include "mozilla/camera/PCamerasParent.h"
#include "mozilla/Mutex.h"
#include "nsCOMPtr.h"
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/common.h"
// Video Engine
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
namespace mozilla {
namespace ipc {
class BackgroundChildImpl;
}
namespace camera {
enum CaptureEngine : int {
InvalidEngine = 0,
ScreenEngine,
BrowserEngine,
WinEngine,
AppEngine,
CameraEngine,
MaxEngine
};
struct CapturerElement {
CaptureEngine engine;
int id;
webrtc::ExternalRenderer* callback;
};
// statically mirror webrtc.org ViECapture API
// these are called via MediaManager->MediaEngineRemoteVideoSource
// on the MediaManager thread
int NumberOfCapabilities(CaptureEngine aCapEngine,
const char* deviceUniqueIdUTF8);
int GetCaptureCapability(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability);
int NumberOfCaptureDevices(CaptureEngine aCapEngine);
int GetCaptureDevice(CaptureEngine aCapEngine,
unsigned int list_number, char* device_nameUTF8,
const unsigned int device_nameUTF8Length,
char* unique_idUTF8,
const unsigned int unique_idUTF8Length);
int AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int unique_idUTF8Length,
int& capture_id);
int ReleaseCaptureDevice(CaptureEngine aCapEngine,
const int capture_id);
int StartCapture(CaptureEngine aCapEngine,
const int capture_id, webrtc::CaptureCapability& capability,
webrtc::ExternalRenderer* func);
int StopCapture(CaptureEngine aCapEngine, const int capture_id);
void Shutdown(void);
class CamerasChild final : public PCamerasChild
{
friend class mozilla::ipc::BackgroundChildImpl;
public:
// We are owned by the PBackground thread only. CamerasSingleton
// takes a non-owning reference.
NS_INLINE_DECL_REFCOUNTING(CamerasChild)
// IPC messages recevied, received on the PBackground thread
// these are the actual callbacks with data
virtual bool RecvDeliverFrame(const int&, const int&, mozilla::ipc::Shmem&&,
const int&, const uint32_t&, const int64_t&,
const int64_t&) override;
virtual bool RecvFrameSizeChange(const int&, const int&,
const int& w, const int& h) override;
// these are response messages to our outgoing requests
virtual bool RecvReplyNumberOfCaptureDevices(const int&) override;
virtual bool RecvReplyNumberOfCapabilities(const int&) override;
virtual bool RecvReplyAllocateCaptureDevice(const int&) override;
virtual bool RecvReplyGetCaptureCapability(const CaptureCapability& capability) override;
virtual bool RecvReplyGetCaptureDevice(const nsCString& device_name,
const nsCString& device_id) override;
virtual bool RecvReplyFailure(void) override;
virtual bool RecvReplySuccess(void) override;
virtual void ActorDestroy(ActorDestroyReason aWhy) override;
// the webrtc.org ViECapture calls are mirrored here, but with access
// to a specific PCameras instance to communicate over. These also
// run on the MediaManager thread
int NumberOfCaptureDevices(CaptureEngine aCapEngine);
int NumberOfCapabilities(CaptureEngine aCapEngine,
const char* deviceUniqueIdUTF8);
int ReleaseCaptureDevice(CaptureEngine aCapEngine,
const int capture_id);
int StartCapture(CaptureEngine aCapEngine,
const int capture_id, webrtc::CaptureCapability& capability,
webrtc::ExternalRenderer* func);
int StopCapture(CaptureEngine aCapEngine, const int capture_id);
int AllocateCaptureDevice(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int unique_idUTF8Length,
int& capture_id);
int GetCaptureCapability(CaptureEngine aCapEngine,
const char* unique_idUTF8,
const unsigned int capability_number,
webrtc::CaptureCapability& capability);
int GetCaptureDevice(CaptureEngine aCapEngine,
unsigned int list_number, char* device_nameUTF8,
const unsigned int device_nameUTF8Length,
char* unique_idUTF8,
const unsigned int unique_idUTF8Length);
void Shutdown();
webrtc::ExternalRenderer* Callback(CaptureEngine aCapEngine, int capture_id);
void AddCallback(const CaptureEngine aCapEngine, const int capture_id,
webrtc::ExternalRenderer* render);
void RemoveCallback(const CaptureEngine aCapEngine, const int capture_id);
private:
CamerasChild();
~CamerasChild();
// Dispatch a Runnable to the PCamerasParent, by executing it on the
// decidecated Cameras IPC/PBackground thread.
bool DispatchToParent(nsIRunnable* aRunnable,
MonitorAutoLock& aMonitor);
nsTArray<CapturerElement> mCallbacks;
// Protects the callback arrays
Mutex mCallbackMutex;
bool mIPCIsAlive;
// Hold to prevent multiple outstanding requests. We don't use
// request IDs so we only support one at a time. Don't want try
// to use the webrtc.org API from multiple threads simultanously.
// The monitor below isn't sufficient for this, as it will drop
// the lock when Wait-ing for a response, allowing us to send a new
// request. The Notify on receiving the response will then unblock
// both waiters and one will be guaranteed to get the wrong result.
// Take this one before taking mReplyMonitor.
Mutex mRequestMutex;
// Hold to wait for an async response to our calls
Monitor mReplyMonitor;
// Async resposne valid?
bool mReceivedReply;
// Aynsc reponses data contents;
bool mReplySuccess;
int mReplyInteger;
webrtc::CaptureCapability mReplyCapability;
nsCString mReplyDeviceName;
nsCString mReplyDeviceID;
};
} // namespace camera
} // namespace mozilla
#endif // mozilla_CamerasChild_h
dom/media/systemservices/CamerasParent.cpp
+866
View File
@@ -0,0 +1,866 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "CamerasParent.h"
#include "CamerasUtils.h"
#include "MediaEngine.h"
#include "MediaUtils.h"
#include "mozilla/Assertions.h"
#include "mozilla/unused.h"
#include "mozilla/Logging.h"
#include "mozilla/ipc/BackgroundParent.h"
#include "nsThreadUtils.h"
#undef LOG
#undef LOG_ENABLED
PRLogModuleInfo *gCamerasParentLog;
#define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug)
namespace mozilla {
namespace camera {
// 3 threads are involved in this code:
// - the main thread for some setups, and occassionally for video capture setup
// calls that don't work correctly elsewhere.
// - the IPC thread on which PBackground is running and which receives and
// sends messages
// - a thread which will execute the actual (possibly slow) camera access
// called "VideoCapture". On Windows this is a thread with an event loop
// suitable for UI access.
class FrameSizeChangeRunnable : public nsRunnable {
public:
FrameSizeChangeRunnable(CamerasParent *aParent, CaptureEngine capEngine,
int cap_id, unsigned int aWidth, unsigned int aHeight)
: mParent(aParent), mCapEngine(capEngine), mCapId(cap_id),
mWidth(aWidth), mHeight(aHeight) {}
NS_IMETHOD Run() {
if (mParent->IsShuttingDown()) {
// Communication channel is being torn down
LOG(("FrameSizeChangeRunnable is active without active Child"));
mResult = 0;
return NS_OK;
}
if (!mParent->SendFrameSizeChange(mCapEngine, mCapId, mWidth, mHeight)) {
mResult = -1;
} else {
mResult = 0;
}
return NS_OK;
}
int GetResult() {
return mResult;
}
private:
nsRefPtr<CamerasParent> mParent;
CaptureEngine mCapEngine;
int mCapId;
unsigned int mWidth;
unsigned int mHeight;
int mResult;
};
int
CallbackHelper::FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams)
{
LOG(("CallbackHelper Video FrameSizeChange: %ux%u", w, h));
nsRefPtr<FrameSizeChangeRunnable> runnable =
new FrameSizeChangeRunnable(mParent, mCapEngine, mCapturerId, w, h);
MOZ_ASSERT(mParent);
nsIThread * thread = mParent->GetBackgroundThread();
MOZ_ASSERT(thread != nullptr);
thread->Dispatch(runnable, NS_DISPATCH_NORMAL);
return 0;
}
class DeliverFrameRunnable : public nsRunnable {
public:
DeliverFrameRunnable(CamerasParent *aParent,
CaptureEngine engine,
int cap_id,
ShmemBuffer buffer,
unsigned char* altbuffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time)
: mParent(aParent), mCapEngine(engine), mCapId(cap_id), mBuffer(Move(buffer)),
mSize(size), mTimeStamp(time_stamp), mNtpTime(ntp_time),
mRenderTime(render_time) {
// No ShmemBuffer (of the right size) was available, so make an
// extra buffer here. We have no idea when we are going to run and
// it will be potentially long after the webrtc frame callback has
// returned, so the copy needs to be no later than here.
// We will need to copy this back into a Shmem later on so we prefer
// using ShmemBuffers to avoid the extra copy.
if (altbuffer != nullptr) {
mAlternateBuffer.reset(new unsigned char[size]);
memcpy(mAlternateBuffer.get(), altbuffer, size);
}
};
NS_IMETHOD Run() {
if (mParent->IsShuttingDown()) {
// Communication channel is being torn down
mResult = 0;
return NS_OK;
}
if (!mParent->DeliverFrameOverIPC(mCapEngine, mCapId,
Move(mBuffer), mAlternateBuffer.get(),
mSize, mTimeStamp,
mNtpTime, mRenderTime)) {
mResult = -1;
} else {
mResult = 0;
}
return NS_OK;
}
int GetResult() {
return mResult;
}
private:
nsRefPtr<CamerasParent> mParent;
CaptureEngine mCapEngine;
int mCapId;
ShmemBuffer mBuffer;
mozilla::UniquePtr<unsigned char[]> mAlternateBuffer;
int mSize;
uint32_t mTimeStamp;
int64_t mNtpTime;
int64_t mRenderTime;
int mResult;
};
int
CamerasParent::DeliverFrameOverIPC(CaptureEngine cap_engine,
int cap_id,
ShmemBuffer buffer,
unsigned char* altbuffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time)
{
// No ShmemBuffers were available, so construct one now of the right size
// and copy into it. That is an extra copy, but we expect this to be
// the exceptional case, because we just assured the next call *will* have a
// buffer of the right size.
if (altbuffer != nullptr) {
// Get a shared memory buffer from the pool, at least size big
ShmemBuffer shMemBuff = mShmemPool.Get(this, size);
if (!shMemBuff.Valid()) {
LOG(("Video shmem is not writeable in DeliverFrame"));
// We can skip this frame if we run out of buffers, it's not a real error.
return 0;
}
// get() and Size() check for proper alignment of the segment
memcpy(shMemBuff.GetBytes(), altbuffer, size);
if (!SendDeliverFrame(cap_engine, cap_id,
shMemBuff.Get(), size,
time_stamp, ntp_time, render_time)) {
return -1;
}
} else {
// ShmemBuffer was available, we're all good. A single copy happened
// in the original webrtc callback.
if (!SendDeliverFrame(cap_engine, cap_id,
buffer.Get(), size,
time_stamp, ntp_time, render_time)) {
return -1;
}
}
return 0;
}
ShmemBuffer
CamerasParent::GetBuffer(size_t aSize)
{
return mShmemPool.GetIfAvailable(aSize);
}
int
CallbackHelper::DeliverFrame(unsigned char* buffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle)
{
// Get a shared memory buffer to copy the frame data into
ShmemBuffer shMemBuffer = mParent->GetBuffer(size);
if (!shMemBuffer.Valid()) {
// Either we ran out of buffers or they're not the right size yet
LOG(("Video shmem is not available in DeliverFrame"));
// We will do the copy into a(n extra) temporary buffer inside
// the DeliverFrameRunnable constructor.
} else {
// Shared memory buffers of the right size are available, do the copy here.
memcpy(shMemBuffer.GetBytes(), buffer, size);
// Mark the original buffer as cleared.
buffer = nullptr;
}
nsRefPtr<DeliverFrameRunnable> runnable =
new DeliverFrameRunnable(mParent, mCapEngine, mCapturerId,
Move(shMemBuffer), buffer, size, time_stamp,
ntp_time, render_time);
MOZ_ASSERT(mParent);
nsIThread* thread = mParent->GetBackgroundThread();
MOZ_ASSERT(thread != nullptr);
thread->Dispatch(runnable, NS_DISPATCH_NORMAL);
return 0;
}
bool
CamerasParent::RecvReleaseFrame(mozilla::ipc::Shmem&& s) {
mShmemPool.Put(ShmemBuffer(s));
return true;
}
bool
CamerasParent::SetupEngine(CaptureEngine aCapEngine)
{
EngineHelper *helper = &mEngines[aCapEngine];
// Already initialized
if (helper->mEngine) {
return true;
}
webrtc::CaptureDeviceInfo *captureDeviceInfo = nullptr;
switch (aCapEngine) {
case ScreenEngine:
captureDeviceInfo =
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen);
break;
case BrowserEngine:
captureDeviceInfo =
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Browser);
break;
case WinEngine:
captureDeviceInfo =
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Window);
break;
case AppEngine:
captureDeviceInfo =
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Application);
break;
case CameraEngine:
captureDeviceInfo =
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Camera);
break;
default:
LOG(("Invalid webrtc Video engine"));
MOZ_CRASH();
break;
}
helper->mConfig.Set<webrtc::CaptureDeviceInfo>(captureDeviceInfo);
helper->mEngine = webrtc::VideoEngine::Create(helper->mConfig);
if (!helper->mEngine) {
LOG(("VideoEngine::Create failed"));
return false;
}
helper->mPtrViEBase = webrtc::ViEBase::GetInterface(helper->mEngine);
if (!helper->mPtrViEBase) {
LOG(("ViEBase::GetInterface failed"));
return false;
}
if (helper->mPtrViEBase->Init() < 0) {
LOG(("ViEBase::Init failed"));
return false;
}
helper->mPtrViECapture = webrtc::ViECapture::GetInterface(helper->mEngine);
if (!helper->mPtrViECapture) {
LOG(("ViECapture::GetInterface failed"));
return false;
}
helper->mPtrViERender = webrtc::ViERender::GetInterface(helper->mEngine);
if (!helper->mPtrViERender) {
LOG(("ViERender::GetInterface failed"));
return false;
}
return true;
}
void
CamerasParent::CloseEngines()
{
{
MutexAutoLock lock(mCallbackMutex);
// Stop the callers
while (mCallbacks.Length()) {
auto capEngine = mCallbacks[0]->mCapEngine;
auto capNum = mCallbacks[0]->mCapturerId;
LOG(("Forcing shutdown of engine %d, capturer %d", capEngine, capNum));
{
MutexAutoUnlock unlock(mCallbackMutex);
RecvStopCapture(capEngine, capNum);
RecvReleaseCaptureDevice(capEngine, capNum);
}
// The callbacks list might have changed while we released the lock,
// but note that due to the loop construct this will not break us.
}
}
{
MutexAutoLock lock(mEngineMutex);
for (int i = 0; i < CaptureEngine::MaxEngine; i++) {
if (mEngines[i].mEngineIsRunning) {
LOG(("Being closed down while engine %d is running!", i));
}
if (mEngines[i].mPtrViERender) {
mEngines[i].mPtrViERender->Release();
mEngines[i].mPtrViERender = nullptr;
}
if (mEngines[i].mPtrViECapture) {
mEngines[i].mPtrViECapture->Release();
mEngines[i].mPtrViECapture = nullptr;
}
if(mEngines[i].mPtrViEBase) {
mEngines[i].mPtrViEBase->Release();
mEngines[i].mPtrViEBase = nullptr;
}
}
}
}
bool
CamerasParent::EnsureInitialized(int aEngine)
{
LOG((__PRETTY_FUNCTION__));
CaptureEngine capEngine = static_cast<CaptureEngine>(aEngine);
if (!SetupEngine(capEngine)) {
return false;
}
return true;
}
// Dispatch the runnable to do the camera operation on the
// specific Cameras thread, preventing us from blocking, and
// chain a runnable to send back the result on the IPC thread.
// It would be nice to get rid of the code duplication here,
// perhaps via Promises.
bool
CamerasParent::RecvNumberOfCaptureDevices(const int& aCapEngine)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("RecvNumberOfCaptureDevices fails to initialize"));
unused << SendReplyFailure();
return false;
}
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine]() -> nsresult {
MutexAutoLock lock(self->mEngineMutex);
int num = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
num = self->mEngines[aCapEngine].mPtrViECapture->NumberOfCaptureDevices();
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, num]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (num < 0) {
LOG(("RecvNumberOfCaptureDevices couldn't find devices"));
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
} else {
LOG(("RecvNumberOfCaptureDevices: %d", num));
unused << self->SendReplyNumberOfCaptureDevices(num);
return NS_OK;
}
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvNumberOfCapabilities(const int& aCapEngine,
const nsCString& unique_id)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("RecvNumberOfCapabilities fails to initialize"));
unused << SendReplyFailure();
return false;
}
LOG(("Getting caps for %s", unique_id.get()));
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, unique_id, aCapEngine]() -> nsresult {
MutexAutoLock lock(self->mEngineMutex);
int num = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
num =
self->mEngines[aCapEngine].mPtrViECapture->NumberOfCapabilities(
unique_id.get(),
MediaEngineSource::kMaxUniqueIdLength);
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, num]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (num < 0) {
LOG(("RecvNumberOfCapabilities couldn't find capabilities"));
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
} else {
LOG(("RecvNumberOfCapabilities: %d", num));
}
unused << self->SendReplyNumberOfCapabilities(num);
return NS_OK;
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvGetCaptureCapability(const int &aCapEngine,
const nsCString& unique_id,
const int& num)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Fails to initialize"));
unused << SendReplyFailure();
return false;
}
LOG(("RecvGetCaptureCapability: %s %d", unique_id.get(), num));
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, unique_id, aCapEngine, num]() -> nsresult {
webrtc::CaptureCapability webrtcCaps;
MutexAutoLock lock(self->mEngineMutex);
int error = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureCapability(
unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, num, webrtcCaps);
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, webrtcCaps, error]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
CaptureCapability capCap(webrtcCaps.width,
webrtcCaps.height,
webrtcCaps.maxFPS,
webrtcCaps.expectedCaptureDelay,
webrtcCaps.rawType,
webrtcCaps.codecType,
webrtcCaps.interlaced);
LOG(("Capability: %u %u %u %u %d %d",
webrtcCaps.width,
webrtcCaps.height,
webrtcCaps.maxFPS,
webrtcCaps.expectedCaptureDelay,
webrtcCaps.rawType,
webrtcCaps.codecType));
if (error) {
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
}
unused << self->SendReplyGetCaptureCapability(capCap);
return NS_OK;
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvGetCaptureDevice(const int& aCapEngine,
const int& aListNumber)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Fails to initialize"));
unused << SendReplyFailure();
return false;
}
LOG(("RecvGetCaptureDevice"));
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, aListNumber]() -> nsresult {
char deviceName[MediaEngineSource::kMaxDeviceNameLength];
char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength];
nsCString name;
nsCString uniqueId;
MutexAutoLock lock(self->mEngineMutex);
int error = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
error = self->mEngines[aCapEngine].mPtrViECapture->GetCaptureDevice(aListNumber,
deviceName,
sizeof(deviceName),
deviceUniqueId,
sizeof(deviceUniqueId));
}
if (!error) {
name.Assign(deviceName);
uniqueId.Assign(deviceUniqueId);
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, error, name, uniqueId]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (error) {
LOG(("GetCaptureDevice failed: %d", error));
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
}
LOG(("Returning %s name %s id", name.get(), uniqueId.get()));
unused << self->SendReplyGetCaptureDevice(name, uniqueId);
return NS_OK;
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvAllocateCaptureDevice(const int& aCapEngine,
const nsCString& unique_id)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Fails to initialize"));
unused << SendReplyFailure();
return false;
}
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, unique_id]() -> nsresult {
int numdev = -1;
MutexAutoLock lock(self->mEngineMutex);
int error = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
error = self->mEngines[aCapEngine].mPtrViECapture->AllocateCaptureDevice(
unique_id.get(), MediaEngineSource::kMaxUniqueIdLength, numdev);
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, numdev, error]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (error) {
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
} else {
LOG(("Allocated device nr %d", numdev));
unused << self->SendReplyAllocateCaptureDevice(numdev);
return NS_OK;
}
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvReleaseCaptureDevice(const int& aCapEngine,
const int& numdev)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Fails to initialize"));
unused << SendReplyFailure();
return false;
}
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, numdev]() -> nsresult {
LOG(("RecvReleaseCamera device nr %d", numdev));
MutexAutoLock lock(self->mEngineMutex);
int error = -1;
if (self->mEngines[aCapEngine].mPtrViECapture) {
error = self->mEngines[aCapEngine].mPtrViECapture->ReleaseCaptureDevice(numdev);
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, error, numdev]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (error) {
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
} else {
unused << self->SendReplySuccess();
LOG(("Freed device nr %d", numdev));
return NS_OK;
}
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
#ifndef XP_MACOSX
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
#else
// Mac OS X hangs on shutdown if we don't do this on the main thread.
NS_DispatchToMainThread(webrtc_runnable);
#endif
return true;
}
bool
CamerasParent::RecvStartCapture(const int& aCapEngine,
const int& capnum,
const CaptureCapability& ipcCaps)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Failure to initialize"));
unused << SendReplyFailure();
return false;
}
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, capnum, ipcCaps]() -> nsresult {
CallbackHelper** cbh;
webrtc::ExternalRenderer* render;
EngineHelper* helper = nullptr;
int error;
{
MutexAutoLock lockCallback(self->mCallbackMutex);
cbh = self->mCallbacks.AppendElement(
new CallbackHelper(static_cast<CaptureEngine>(aCapEngine), capnum, self));
render = static_cast<webrtc::ExternalRenderer*>(*cbh);
}
{
MutexAutoLock lockEngine(self->mEngineMutex);
if (self->mEngines[aCapEngine].mPtrViECapture) {
helper = &self->mEngines[aCapEngine];
error =
helper->mPtrViERender->AddRenderer(capnum, webrtc::kVideoI420, render);
} else {
error = -1;
}
if (!error) {
error = helper->mPtrViERender->StartRender(capnum);
}
webrtc::CaptureCapability capability;
capability.width = ipcCaps.width();
capability.height = ipcCaps.height();
capability.maxFPS = ipcCaps.maxFPS();
capability.expectedCaptureDelay = ipcCaps.expectedCaptureDelay();
capability.rawType = static_cast<webrtc::RawVideoType>(ipcCaps.rawType());
capability.codecType = static_cast<webrtc::VideoCodecType>(ipcCaps.codecType());
capability.interlaced = ipcCaps.interlaced();
if (!error) {
error = helper->mPtrViECapture->StartCapture(capnum, capability);
}
if (!error) {
helper->mEngineIsRunning = true;
}
}
nsRefPtr<nsIRunnable> ipc_runnable =
media::NewRunnableFrom([self, error]() -> nsresult {
if (self->IsShuttingDown()) {
return NS_ERROR_FAILURE;
}
if (!error) {
unused << self->SendReplySuccess();
return NS_OK;
} else {
unused << self->SendReplyFailure();
return NS_ERROR_FAILURE;
}
});
self->mPBackgroundThread->Dispatch(ipc_runnable, NS_DISPATCH_NORMAL);
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return true;
}
bool
CamerasParent::RecvStopCapture(const int& aCapEngine,
const int& capnum)
{
LOG((__PRETTY_FUNCTION__));
if (!EnsureInitialized(aCapEngine)) {
LOG(("Failure to initialize"));
unused << SendReplyFailure();
return false;
}
nsRefPtr<CamerasParent> self(this);
nsRefPtr<nsRunnable> webrtc_runnable =
media::NewRunnableFrom([self, aCapEngine, capnum]() -> nsresult {
{
MutexAutoLock lock(self->mEngineMutex);
// We only need to check mPtrViECapture as all other engines are guaranteed
// to be nulled under the same lock.
if (self->mEngines[aCapEngine].mPtrViECapture) {
self->mEngines[aCapEngine].mPtrViECapture->StopCapture(capnum);
self->mEngines[aCapEngine].mPtrViERender->StopRender(capnum);
self->mEngines[aCapEngine].mPtrViERender->RemoveRenderer(capnum);
self->mEngines[aCapEngine].mEngineIsRunning = false;
}
}
MutexAutoLock lock(self->mCallbackMutex);
for (unsigned int i = 0; i < self->mCallbacks.Length(); i++) {
if (self->mCallbacks[i]->mCapEngine == aCapEngine
&& self->mCallbacks[i]->mCapturerId == capnum) {
delete self->mCallbacks[i];
self->mCallbacks.RemoveElementAt(i);
break;
}
}
return NS_OK;
});
mVideoCaptureThread->message_loop()->PostTask(FROM_HERE, new RunnableTask(webrtc_runnable));
return SendReplySuccess();
}
bool
CamerasParent::RecvAllDone()
{
LOG((__PRETTY_FUNCTION__));
// Don't try to send anything to the child now
mChildIsAlive = false;
return Send__delete__(this);
}
void CamerasParent::DoShutdown()
{
LOG((__PRETTY_FUNCTION__));
CloseEngines();
{
MutexAutoLock lock(mEngineMutex);
for (int i = 0; i < CaptureEngine::MaxEngine; i++) {
if (mEngines[i].mEngine) {
mEngines[i].mEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mEngines[i].mEngine);
mEngines[i].mEngine = nullptr;
}
}
}
mShmemPool.Cleanup(this);
mPBackgroundThread = nullptr;
if (mVideoCaptureThread) {
if (mVideoCaptureThread->IsRunning()) {
mVideoCaptureThread->Stop();
}
delete mVideoCaptureThread;
mVideoCaptureThread = nullptr;
}
}
void
CamerasParent::ActorDestroy(ActorDestroyReason aWhy)
{
// No more IPC from here
LOG((__PRETTY_FUNCTION__));
// We don't want to receive callbacks or anything if we can't
// forward them anymore anyway.
mChildIsAlive = false;
mDestroyed = true;
CloseEngines();
}
CamerasParent::CamerasParent()
: mCallbackMutex("CamerasParent.mCallbackMutex"),
mEngineMutex("CamerasParent.mEngineMutex"),
mShmemPool(CaptureEngine::MaxEngine),
mVideoCaptureThread(nullptr),
mChildIsAlive(true),
mDestroyed(false)
{
if (!gCamerasParentLog) {
gCamerasParentLog = PR_NewLogModule("CamerasParent");
}
LOG(("CamerasParent: %p", this));
mPBackgroundThread = NS_GetCurrentThread();
MOZ_ASSERT(mPBackgroundThread != nullptr, "GetCurrentThread failed");
LOG(("Spinning up WebRTC Cameras Thread"));
mVideoCaptureThread = new base::Thread("VideoCapture");
base::Thread::Options options;
#if defined(_WIN32)
options.message_loop_type = MessageLoop::TYPE_MOZILLA_NONMAINUITHREAD;
#else
options.message_loop_type = MessageLoop::TYPE_MOZILLA_NONMAINTHREAD;
#endif
if (!mVideoCaptureThread->StartWithOptions(options)) {
MOZ_CRASH();
}
MOZ_COUNT_CTOR(CamerasParent);
}
CamerasParent::~CamerasParent()
{
LOG(("~CamerasParent: %p", this));
MOZ_COUNT_DTOR(CamerasParent);
DoShutdown();
}
already_AddRefed<CamerasParent>
CamerasParent::Create() {
mozilla::ipc::AssertIsOnBackgroundThread();
nsRefPtr<CamerasParent> camerasParent = new CamerasParent();
return camerasParent.forget();
}
}
}
dom/media/systemservices/CamerasParent.h
+147
View File
@@ -0,0 +1,147 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_CamerasParent_h
#define mozilla_CamerasParent_h
#include "mozilla/dom/ContentParent.h"
#include "mozilla/camera/PCamerasParent.h"
#include "mozilla/ipc/Shmem.h"
#include "mozilla/ShmemPool.h"
// conflicts with #include of scoped_ptr.h
#undef FF
#include "webrtc/common.h"
// Video Engine
#include "webrtc/video_engine/include/vie_base.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "CamerasChild.h"
#include "base/thread.h"
namespace mozilla {
namespace camera {
class CamerasParent;
class CallbackHelper : public webrtc::ExternalRenderer
{
public:
CallbackHelper(CaptureEngine aCapEng, int aCapId, CamerasParent *aParent)
: mCapEngine(aCapEng), mCapturerId(aCapId), mParent(aParent) {};
// ViEExternalRenderer implementation. These callbacks end up
// running on the VideoCapture thread.
virtual int FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams) override;
virtual int DeliverFrame(unsigned char* buffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle) override;
virtual bool IsTextureSupported() override { return false; };
friend CamerasParent;
private:
CaptureEngine mCapEngine;
int mCapturerId;
CamerasParent *mParent;
};
class EngineHelper
{
public:
EngineHelper() :
mEngine(nullptr), mPtrViEBase(nullptr), mPtrViECapture(nullptr),
mPtrViERender(nullptr), mEngineIsRunning(false) {};
webrtc::VideoEngine *mEngine;
webrtc::ViEBase *mPtrViEBase;
webrtc::ViECapture *mPtrViECapture;
webrtc::ViERender *mPtrViERender;
// The webrtc code keeps a reference to this one.
webrtc::Config mConfig;
// Engine alive
bool mEngineIsRunning;
};
class CamerasParent : public PCamerasParent
{
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CamerasParent);
public:
static already_AddRefed<CamerasParent> Create();
// Messages received form the child. These run on the IPC/PBackground thread.
virtual bool RecvAllocateCaptureDevice(const int&, const nsCString&) override;
virtual bool RecvReleaseCaptureDevice(const int&, const int &) override;
virtual bool RecvNumberOfCaptureDevices(const int&) override;
virtual bool RecvNumberOfCapabilities(const int&, const nsCString&) override;
virtual bool RecvGetCaptureCapability(const int&, const nsCString&, const int&) override;
virtual bool RecvGetCaptureDevice(const int&, const int&) override;
virtual bool RecvStartCapture(const int&, const int&, const CaptureCapability&) override;
virtual bool RecvStopCapture(const int&, const int&) override;
virtual bool RecvReleaseFrame(mozilla::ipc::Shmem&&) override;
virtual bool RecvAllDone() override;
virtual void ActorDestroy(ActorDestroyReason aWhy) override;
nsIThread* GetBackgroundThread() { return mPBackgroundThread; };
bool IsShuttingDown() { return !mChildIsAlive || mDestroyed; };
ShmemBuffer GetBuffer(size_t aSize);
// helper to forward to the PBackground thread
int DeliverFrameOverIPC(CaptureEngine capEng,
int cap_id,
ShmemBuffer buffer,
unsigned char* altbuffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time);
CamerasParent();
protected:
virtual ~CamerasParent();
bool SetupEngine(CaptureEngine aCapEngine);
void CloseEngines();
bool EnsureInitialized(int aEngine);
void DoShutdown();
EngineHelper mEngines[CaptureEngine::MaxEngine];
nsTArray<CallbackHelper*> mCallbacks;
// Protects the callback arrays
Mutex mCallbackMutex;
// Protects the engines array
Mutex mEngineMutex;
// image buffers
mozilla::ShmemPool mShmemPool;
// PBackground parent thread
nsCOMPtr<nsIThread> mPBackgroundThread;
// video processing thread - where webrtc.org capturer code runs
base::Thread* mVideoCaptureThread;
// Shutdown handling
bool mChildIsAlive;
bool mDestroyed;
};
PCamerasParent* CreateCamerasParent();
} // namespace camera
} // namespace mozilla
#endif // mozilla_CameraParent_h
dom/media/systemservices/CamerasUtils.cpp
+81
View File
@@ -0,0 +1,81 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/unused.h"
#include "nsThreadUtils.h"
#include "nsCOMPtr.h"
#include "mozilla/Assertions.h"
#include "mozilla/ipc/BackgroundChild.h"
#include "mozilla/ipc/PBackgroundChild.h"
#include "nsIIPCBackgroundChildCreateCallback.h"
namespace mozilla {
namespace camera {
class WorkerBackgroundChildCallback final :
public nsIIPCBackgroundChildCreateCallback
{
bool* mDone;
public:
explicit WorkerBackgroundChildCallback(bool* aDone)
: mDone(aDone)
{
MOZ_ASSERT(!NS_IsMainThread());
MOZ_ASSERT(mDone);
}
NS_DECL_ISUPPORTS
private:
~WorkerBackgroundChildCallback() { }
virtual void
ActorCreated(PBackgroundChild* aActor) override
{
*mDone = true;
}
virtual void
ActorFailed() override
{
*mDone = true;
}
};
NS_IMPL_ISUPPORTS(WorkerBackgroundChildCallback, nsIIPCBackgroundChildCreateCallback)
nsresult
SynchronouslyCreatePBackground()
{
using mozilla::ipc::BackgroundChild;
MOZ_ASSERT(!BackgroundChild::GetForCurrentThread());
bool done = false;
nsCOMPtr<nsIIPCBackgroundChildCreateCallback> callback =
new WorkerBackgroundChildCallback(&done);
if (NS_WARN_IF(!BackgroundChild::GetOrCreateForCurrentThread(callback))) {
return NS_ERROR_FAILURE;
}
nsIThread *thread = NS_GetCurrentThread();
while (!done) {
if (NS_WARN_IF(!NS_ProcessNextEvent(thread, true /* aMayWait */))) {
return NS_ERROR_FAILURE;
}
}
if (NS_WARN_IF(!BackgroundChild::GetForCurrentThread())) {
return NS_ERROR_FAILURE;
}
return NS_OK;
}
}
}
dom/media/systemservices/CamerasUtils.h
+60
View File
@@ -0,0 +1,60 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_CameraUtils_h
#define mozilla_CameraUtils_h
#include "nsThreadUtils.h"
#include "nsCOMPtr.h"
#include "mozilla/UniquePtr.h"
#include "base/thread.h"
namespace mozilla {
namespace camera {
nsresult SynchronouslyCreatePBackground();
class ThreadDestructor : public nsRunnable
{
DISALLOW_COPY_AND_ASSIGN(ThreadDestructor);
public:
explicit ThreadDestructor(nsIThread* aThread)
: mThread(aThread) {}
NS_IMETHOD Run() override
{
if (mThread) {
mThread->Shutdown();
}
return NS_OK;
}
private:
~ThreadDestructor() {}
nsCOMPtr<nsIThread> mThread;
};
class RunnableTask : public Task
{
public:
explicit RunnableTask(nsRunnable* aRunnable)
: mRunnable(aRunnable) {}
void Run() override {
mRunnable->Run();
}
private:
~RunnableTask() {}
nsRefPtr<nsRunnable> mRunnable;
};
}
}
#endif // mozilla_CameraUtils_h
dom/media/systemservices/PCameras.ipdl
+60
View File
@@ -0,0 +1,60 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
include protocol PContent;
include protocol PBackground;
namespace mozilla {
namespace camera {
struct CaptureCapability
{
int width;
int height;
int maxFPS;
int expectedCaptureDelay;
int rawType;
int codecType;
bool interlaced;
};
async protocol PCameras
{
manager PBackground;
child:
async FrameSizeChange(int capEngine, int cap_id, int w, int h);
// transfers ownership of |buffer| from parent to child
async DeliverFrame(int capEngine, int cap_id,
Shmem buffer, int size, uint32_t time_stamp,
int64_t ntp_time, int64_t render_time);
async ReplyNumberOfCaptureDevices(int numdev);
async ReplyNumberOfCapabilities(int numdev);
async ReplyAllocateCaptureDevice(int numdev);
async ReplyGetCaptureCapability(CaptureCapability cap);
async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id);
async ReplyFailure();
async ReplySuccess();
async __delete__();
parent:
async NumberOfCaptureDevices(int engine);
async NumberOfCapabilities(int engine, nsCString deviceUniqueIdUTF8);
async GetCaptureCapability(int engine, nsCString unique_idUTF8, int capability_number);
async GetCaptureDevice(int engine, int num);
async AllocateCaptureDevice(int engine, nsCString unique_idUTF8);
async ReleaseCaptureDevice(int engine, int numdev);
async StartCapture(int engine, int numdev, CaptureCapability capability);
async StopCapture(int engine, int numdev);
// transfers frame back
async ReleaseFrame(Shmem s);
// Ask parent to delete us
async AllDone();
};
} // namespace camera
} // namespace mozilla
dom/media/systemservices/ShmemPool.cpp
+129
View File
@@ -0,0 +1,129 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "mozilla/Assertions.h"
#include "mozilla/Logging.h"
#include "mozilla/ShmemPool.h"
#include "mozilla/Move.h"
#undef LOG
#undef LOG_ENABLED
extern PRLogModuleInfo *gCamerasParentLog;
#define LOG(args) MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, args)
#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug)
namespace mozilla {
ShmemPool::ShmemPool(size_t aPoolSize)
: mMutex("mozilla::ShmemPool"),
mPoolFree(aPoolSize)
#ifdef DEBUG
,mMaxPoolUse(0)
#endif
{
mShmemPool.SetLength(aPoolSize);
}
mozilla::ShmemBuffer ShmemPool::GetIfAvailable(size_t aSize)
{
MutexAutoLock lock(mMutex);
// Pool is empty, don't block caller.
if (mPoolFree == 0) {
// This isn't initialized, so will be understood as an error.
return ShmemBuffer();
}
ShmemBuffer& res = mShmemPool[mPoolFree - 1];
if (!res.mInitialized) {
return ShmemBuffer();
}
MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?");
if (res.mShmem.Size<char>() < aSize) {
return ShmemBuffer();
}
mPoolFree--;
#ifdef DEBUG
size_t poolUse = mShmemPool.Length() - mPoolFree;
if (poolUse > mMaxPoolUse) {
mMaxPoolUse = poolUse;
LOG(("Maximum ShmemPool use increased: %d buffers", mMaxPoolUse));
}
#endif
return Move(res);
}
template <class T>
mozilla::ShmemBuffer ShmemPool::Get(T* aInstance, size_t aSize)
{
MutexAutoLock lock(mMutex);
// Pool is empty, don't block caller.
if (mPoolFree == 0) {
// This isn't initialized, so will be understood as an error.
return ShmemBuffer();
}
ShmemBuffer res = Move(mShmemPool[mPoolFree - 1]);
if (!res.mInitialized) {
LOG(("Initiaizing new Shmem in pool"));
aInstance->AllocShmem(aSize, SharedMemory::TYPE_BASIC, &res.mShmem);
res.mInitialized = true;
}
MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?");
// Prepare buffer, increase size if needed (we never shrink as we don't
// maintain seperate sized pools and we don't want to keep reallocating)
if (res.mShmem.Size<char>() < aSize) {
LOG(("Size change/increase in Shmem Pool"));
aInstance->DeallocShmem(res.mShmem);
// this may fail; always check return value
if (!aInstance->AllocShmem(aSize, SharedMemory::TYPE_BASIC, &res.mShmem)) {
LOG(("Failure allocating new size Shmem buffer"));
return ShmemBuffer();
}
}
mPoolFree--;
return res;
}
void ShmemPool::Put(ShmemBuffer&& aShmem)
{
MutexAutoLock lock(mMutex);
MOZ_ASSERT(mPoolFree < mShmemPool.Length());
mShmemPool[mPoolFree] = Move(aShmem);
mPoolFree++;
}
template <class T>
void ShmemPool::Cleanup(T* aInstance)
{
MutexAutoLock lock(mMutex);
for (size_t i = 0; i < mShmemPool.Length(); i++) {
if (mShmemPool[i].mInitialized) {
aInstance->DeallocShmem(mShmemPool[i].Get());
mShmemPool[i].mInitialized = false;
}
}
}
ShmemPool::~ShmemPool()
{
#ifdef DEBUG
for (size_t i = 0; i < mShmemPool.Length(); i++) {
MOZ_ASSERT(!mShmemPool[i].Valid());
}
#endif
}
} // namespace mozilla
dom/media/systemservices/ShmemPool.h
+85
View File
@@ -0,0 +1,85 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_ShmemPool_h
#define mozilla_ShmemPool_h
#include "mozilla/ipc/Shmem.h"
#include "mozilla/Mutex.h"
namespace mozilla {
class ShmemPool;
class ShmemBuffer {
public:
ShmemBuffer() : mInitialized(false) {}
explicit ShmemBuffer(mozilla::ipc::Shmem aShmem) {
mInitialized = true;
mShmem = aShmem;
}
ShmemBuffer(ShmemBuffer&& rhs) {
mInitialized = rhs.mInitialized;
mShmem = Move(rhs.mShmem);
}
ShmemBuffer& operator=(ShmemBuffer&& rhs) {
MOZ_ASSERT(&rhs != this, "self-moves are prohibited");
mInitialized = rhs.mInitialized;
mShmem = Move(rhs.mShmem);
return *this;
}
// No copies allowed
ShmemBuffer(const ShmemBuffer&) = delete;
ShmemBuffer& operator=(const ShmemBuffer&) = delete;
bool Valid() {
return mInitialized;
}
char* GetBytes() {
return mShmem.get<char>();
}
mozilla::ipc::Shmem& Get() {
return mShmem;
}
private:
friend class ShmemPool;
bool mInitialized;
mozilla::ipc::Shmem mShmem;
};
class ShmemPool {
public:
explicit ShmemPool(size_t aPoolSize);
~ShmemPool();
// We need to use the allocation/deallocation functions
// of a specific IPC child/parent instance.
template <class T> void Cleanup(T* aInstance);
// These 2 differ in what thread they can run on. GetIfAvailable
// can run anywhere but won't allocate if the right size isn't available.
ShmemBuffer GetIfAvailable(size_t aSize);
template <class T> ShmemBuffer Get(T* aInstance, size_t aSize);
void Put(ShmemBuffer&& aShmem);
private:
Mutex mMutex;
size_t mPoolFree;
#ifdef DEBUG
size_t mMaxPoolUse;
#endif
nsTArray<ShmemBuffer> mShmemPool;
};
} // namespace mozilla
#endif // mozilla_ShmemPool_h
dom/media/systemservices/moz.build
+32 -15
View File
@@ -5,15 +5,25 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
if CONFIG['MOZ_WEBRTC']:
EXPORTS += ['LoadManager.h',
'LoadManagerFactory.h',
'LoadMonitor.h',
EXPORTS += [
'CamerasChild.h',
'CamerasParent.h',
'CamerasUtils.h',
'LoadManager.h',
'LoadManagerFactory.h',
'LoadMonitor.h',
]
UNIFIED_SOURCES += ['LoadManager.cpp',
'LoadManagerFactory.cpp',
'LoadMonitor.cpp',
UNIFIED_SOURCES += [
'CamerasChild.cpp',
'CamerasParent.cpp',
'CamerasUtils.cpp',
'LoadManager.cpp',
'LoadManagerFactory.cpp',
'LoadMonitor.cpp',
'ShmemPool.cpp',
]
LOCAL_INCLUDES += [
'/media/webrtc/signaling',
'/media/webrtc/trunk',
]
@@ -37,6 +47,11 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'gonk':
]
]
if CONFIG['_MSC_VER']:
DEFINES['__PRETTY_FUNCTION__'] = '__FUNCSIG__'
EXPORTS.mozilla += ['ShmemPool.h',]
EXPORTS.mozilla.media += ['MediaChild.h',
'MediaParent.h',
'MediaSystemResourceClient.h',
@@ -46,18 +61,20 @@ EXPORTS.mozilla.media += ['MediaChild.h',
'MediaSystemResourceMessageUtils.h',
'MediaSystemResourceService.h',
'MediaSystemResourceTypes.h',
'MediaUtils.h'
'MediaUtils.h',
]
UNIFIED_SOURCES += ['MediaChild.cpp',
'MediaParent.cpp',
'MediaSystemResourceClient.cpp',
'MediaSystemResourceManager.cpp',
'MediaSystemResourceManagerChild.cpp',
'MediaSystemResourceManagerParent.cpp',
'MediaSystemResourceService.cpp',
'MediaUtils.cpp',
UNIFIED_SOURCES += [
'MediaChild.cpp',
'MediaParent.cpp',
'MediaSystemResourceClient.cpp',
'MediaSystemResourceManager.cpp',
'MediaSystemResourceManagerChild.cpp',
'MediaSystemResourceManagerParent.cpp',
'MediaSystemResourceService.cpp',
'MediaUtils.cpp',
]
IPDL_SOURCES += [
'PCameras.ipdl',
'PMedia.ipdl',
'PMediaSystemResourceManager.ipdl',
]
dom/media/test/mochitest.ini
-1
View File
@@ -392,7 +392,6 @@ tags=msg
tags=msg
[test_mediarecorder_record_gum_video_timeslice.html]
tags=msg
skip-if = buildapp == 'b2g' || toolkit == 'android' # mimetype check, bug 969289
[test_mediarecorder_record_immediate_stop.html]
tags=msg
[test_mediarecorder_record_no_timeslice.html]
dom/media/test/test_mediarecorder_record_gum_video_timeslice.html
+4 -6
View File
@@ -40,12 +40,10 @@ function startTest() {
is(evt.type, 'dataavailable',
'Event type should dataavailable');
ok(evt.data.size >= 0,
'Blob data size received is greater than or equal to zero');
is(evt.data.type, 'video/webm',
'Blob data received should have type = ' + 'video/webm');
is(mediaRecorder.mimeType, 'video/webm',
'Mime type in ondataavailable = ' + mediaRecorder.mimeType);
'Blob data size ' + evt.data.size + ' received is greater than or equal to zero');
is(mediaRecorder.mimeType, evt.data.type,
'Mime type in MediaRecorder and ondataavailable : '
+ mediaRecorder.mimeType + ' == ' + evt.data.type);
// We'll stop recording upon the 1st blob being received
if (dataAvailableCount === 1) {
dom/media/webaudio/AudioContext.cpp
+5 -2
View File
@@ -944,8 +944,11 @@ AudioContext::UpdateNodeCount(int32_t aDelta)
bool firstNode = mNodeCount == 0;
mNodeCount += aDelta;
MOZ_ASSERT(mNodeCount >= 0);
// mDestinationNode may be null when we're destroying nodes unlinked by CC
if (!firstNode && mDestination) {
// mDestinationNode may be null when we're destroying nodes unlinked by CC.
// Skipping unnecessary calls after shutdown avoids RunInStableState events
// getting stuck in CycleCollectedJSRuntime during final cycle collection
// (bug 1200514).
if (!firstNode && mDestination && !mIsShutDown) {
mDestination->SetIsOnlyNodeForContext(mNodeCount == 1);
}
}
dom/media/webaudio/BufferDecoder.cpp
-7
View File
@@ -45,13 +45,6 @@ BufferDecoder::GetReentrantMonitor()
return mReentrantMonitor;
}
bool
BufferDecoder::IsShutdown() const
{
// BufferDecoder cannot be shut down.
return false;
}
bool
BufferDecoder::OnStateMachineTaskQueue() const
{
dom/media/webaudio/BufferDecoder.h
-2
View File
@@ -33,8 +33,6 @@ public:
virtual ReentrantMonitor& GetReentrantMonitor() final override;
virtual bool IsShutdown() const final override;
virtual bool OnStateMachineTaskQueue() const final override;
virtual bool OnDecodeTaskQueue() const final override;
dom/media/webrtc/MediaEngineCameraVideoSource.cpp
+46 -3
View File
@@ -162,6 +162,49 @@ MediaEngineCameraVideoSource::LogConstraints(
c.mFrameRate.mIdeal.WasPassed()? c.mFrameRate.mIdeal.Value() : 0));
}
void
MediaEngineCameraVideoSource::LogCapability(const char* aHeader,
const webrtc::CaptureCapability &aCapability, uint32_t aDistance)
{
// RawVideoType and VideoCodecType media/webrtc/trunk/webrtc/common_types.h
static const char* const types[] = {
"I420",
"YV12",
"YUY2",
"UYVY",
"IYUV",
"ARGB",
"RGB24",
"RGB565",
"ARGB4444",
"ARGB1555",
"MJPEG",
"NV12",
"NV21",
"BGRA",
"Unknown type"
};
static const char* const codec[] = {
"VP8",
"VP9",
"H264",
"I420",
"RED",
"ULPFEC",
"Generic codec",
"Unknown codec"
};
LOG(("%s: %4u x %4u x %2u maxFps, %s, %s. Distance = %lu",
aHeader, aCapability.width, aCapability.height, aCapability.maxFPS,
types[std::min(std::max(uint32_t(0), uint32_t(aCapability.rawType)),
uint32_t(sizeof(types) / sizeof(*types) - 1))],
codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.codecType)),
uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
aDistance));
}
bool
MediaEngineCameraVideoSource::ChooseCapability(
const MediaTrackConstraints &aConstraints,
@@ -195,6 +238,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
webrtc::CaptureCapability cap;
GetCapability(candidate.mIndex, cap);
candidate.mDistance = GetFitnessDistance(cap, aConstraints, false, aDeviceId);
LogCapability("Capability", cap, candidate.mDistance);
if (candidate.mDistance == UINT32_MAX) {
candidateSet.RemoveElementAt(i);
} else {
@@ -234,6 +278,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
// Any remaining multiples all have the same distance. A common case of this
// occurs when no ideal is specified. Lean toward defaults.
uint32_t sameDistance = candidateSet[0].mDistance;
{
MediaTrackConstraintSet prefs;
prefs.mWidth.SetAsLong() = aPrefs.GetWidth();
@@ -268,9 +313,7 @@ MediaEngineCameraVideoSource::ChooseCapability(
GetCapability(candidateSet[0].mIndex, mCapability);
}
LOG(("chose cap %dx%d @%dfps codec %d raw %d",
mCapability.width, mCapability.height, mCapability.maxFPS,
mCapability.codecType, mCapability.rawType));
LogCapability("Chosen capability", mCapability, sameDistance);
return true;
}
dom/media/webrtc/MediaEngineCameraVideoSource.h
+3 -6
View File
@@ -30,7 +30,6 @@ public:
, mHasDirectListeners(false)
, mCaptureIndex(aIndex)
, mTrackID(0)
, mFps(-1)
{}
@@ -50,10 +49,6 @@ public:
return false;
}
virtual const dom::MediaSourceEnum GetMediaSource() override {
return dom::MediaSourceEnum::Camera;
}
virtual nsresult TakePhoto(PhotoCallback* aCallback) override
{
return NS_ERROR_NOT_IMPLEMENTED;
@@ -89,6 +84,9 @@ protected:
static void TrimLessFitCandidates(CapabilitySet& set);
static void LogConstraints(const dom::MediaTrackConstraintSet& aConstraints,
bool aAdvanced);
static void LogCapability(const char* aHeader,
const webrtc::CaptureCapability &aCapability,
uint32_t aDistance);
virtual size_t NumCapabilities();
virtual void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut);
bool ChooseCapability(const dom::MediaTrackConstraints &aConstraints,
@@ -119,7 +117,6 @@ protected:
bool mHasDirectListeners;
int mCaptureIndex;
TrackID mTrackID;
int mFps; // Track rate (30 fps by default)
webrtc::CaptureCapability mCapability; // Doesn't work on OS X.
dom/media/webrtc/MediaEngineGonkVideoSource.h
+3
View File
@@ -70,6 +70,9 @@ public:
SourceMediaStream* aSource,
TrackID aId,
StreamTime aDesiredTime) override;
virtual const dom::MediaSourceEnum GetMediaSource() override {
return dom::MediaSourceEnum::Camera;
}
void OnHardwareStateChange(HardwareState aState, nsresult aReason) override;
void GetRotation();
dom/media/webrtc/MediaEngineRemoteVideoSource.cpp
+398
View File
@@ -0,0 +1,398 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaEngineRemoteVideoSource.h"
#include "mozilla/RefPtr.h"
#include "VideoUtils.h"
#include "nsIPrefService.h"
#include "MediaTrackConstraints.h"
#include "CamerasChild.h"
extern PRLogModuleInfo* GetMediaManagerLog();
#define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
#define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
namespace mozilla {
using dom::ConstrainLongRange;
NS_IMPL_ISUPPORTS0(MediaEngineRemoteVideoSource)
MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
int aIndex, mozilla::camera::CaptureEngine aCapEngine,
dom::MediaSourceEnum aMediaSource, const char* aMonitorName)
: MediaEngineCameraVideoSource(aIndex, aMonitorName),
mMediaSource(aMediaSource),
mCapEngine(aCapEngine)
{
MOZ_ASSERT(aMediaSource != dom::MediaSourceEnum::Other);
Init();
}
void
MediaEngineRemoteVideoSource::Init()
{
LOG((__PRETTY_FUNCTION__));
char deviceName[kMaxDeviceNameLength];
char uniqueId[kMaxUniqueIdLength];
if (mozilla::camera::GetCaptureDevice(mCapEngine,
mCaptureIndex,
deviceName, kMaxDeviceNameLength,
uniqueId, kMaxUniqueIdLength)) {
LOG(("Error initializing RemoteVideoSource (GetCaptureDevice)"));
return;
}
SetName(NS_ConvertUTF8toUTF16(deviceName));
SetUUID(uniqueId);
mInitDone = true;
return;
}
void
MediaEngineRemoteVideoSource::Shutdown()
{
LOG((__PRETTY_FUNCTION__));
if (!mInitDone) {
return;
}
if (mState == kStarted) {
SourceMediaStream *source;
bool empty;
while (1) {
{
MonitorAutoLock lock(mMonitor);
empty = mSources.IsEmpty();
if (empty) {
break;
}
source = mSources[0];
}
Stop(source, kVideoTrack); // XXX change to support multiple tracks
}
MOZ_ASSERT(mState == kStopped);
}
if (mState == kAllocated || mState == kStopped) {
Deallocate();
}
mozilla::camera::Shutdown();
mState = kReleased;
mInitDone = false;
return;
}
nsresult
MediaEngineRemoteVideoSource::Allocate(const dom::MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId)
{
LOG((__PRETTY_FUNCTION__));
if (mState == kReleased && mInitDone) {
// Note: if shared, we don't allow a later opener to affect the resolution.
// (This may change depending on spec changes for Constraints/settings)
if (!ChooseCapability(aConstraints, aPrefs, aDeviceId)) {
return NS_ERROR_UNEXPECTED;
}
if (mozilla::camera::AllocateCaptureDevice(mCapEngine,
GetUUID().get(),
kMaxUniqueIdLength, mCaptureIndex)) {
return NS_ERROR_FAILURE;
}
mState = kAllocated;
LOG(("Video device %d allocated", mCaptureIndex));
} else if (MOZ_LOG_TEST(GetMediaManagerLog(), mozilla::LogLevel::Debug)) {
MonitorAutoLock lock(mMonitor);
if (mSources.IsEmpty()) {
LOG(("Video device %d reallocated", mCaptureIndex));
} else {
LOG(("Video device %d allocated shared", mCaptureIndex));
}
}
return NS_OK;
}
nsresult
MediaEngineRemoteVideoSource::Deallocate()
{
LOG((__FUNCTION__));
bool empty;
{
MonitorAutoLock lock(mMonitor);
empty = mSources.IsEmpty();
}
if (empty) {
if (mState != kStopped && mState != kAllocated) {
return NS_ERROR_FAILURE;
}
mozilla::camera::ReleaseCaptureDevice(mCapEngine, mCaptureIndex);
mState = kReleased;
LOG(("Video device %d deallocated", mCaptureIndex));
} else {
LOG(("Video device %d deallocated but still in use", mCaptureIndex));
}
return NS_OK;
}
nsresult
MediaEngineRemoteVideoSource::Start(SourceMediaStream* aStream, TrackID aID)
{
LOG((__PRETTY_FUNCTION__));
if (!mInitDone || !aStream) {
LOG(("No stream or init not done"));
return NS_ERROR_FAILURE;
}
{
MonitorAutoLock lock(mMonitor);
mSources.AppendElement(aStream);
}
aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED);
if (mState == kStarted) {
return NS_OK;
}
mImageContainer = layers::LayerManager::CreateImageContainer();
mState = kStarted;
mTrackID = aID;
if (mozilla::camera::StartCapture(mCapEngine,
mCaptureIndex, mCapability, this)) {
LOG(("StartCapture failed"));
return NS_ERROR_FAILURE;
}
return NS_OK;
}
nsresult
MediaEngineRemoteVideoSource::Stop(mozilla::SourceMediaStream* aSource,
mozilla::TrackID aID)
{
LOG((__PRETTY_FUNCTION__));
{
MonitorAutoLock lock(mMonitor);
if (!mSources.RemoveElement(aSource)) {
// Already stopped - this is allowed
return NS_OK;
}
aSource->EndTrack(aID);
if (!mSources.IsEmpty()) {
return NS_OK;
}
if (mState != kStarted) {
return NS_ERROR_FAILURE;
}
mState = kStopped;
// Drop any cached image so we don't start with a stale image on next
// usage
mImage = nullptr;
}
mozilla::camera::StopCapture(mCapEngine, mCaptureIndex);
return NS_OK;
}
void
MediaEngineRemoteVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID, StreamTime aDesiredTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta);
}
}
int
MediaEngineRemoteVideoSource::FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams)
{
mWidth = w;
mHeight = h;
LOG(("MediaEngineRemoteVideoSource Video FrameSizeChange: %ux%u", w, h));
return 0;
}
int
MediaEngineRemoteVideoSource::DeliverFrame(unsigned char* buffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Create a video frame and append it to the track.
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, ntp_time, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
// Push the frame into the MSG with a minimal duration. This will likely
// mean we'll still get NotifyPull calls which will then return the same
// frame again with a longer duration. However, this means we won't
// fail to get the frame in and drop frames.
// XXX The timestamp for the frame should be based on the Capture time,
// not the MSG time, and MSG should never, ever block on a (realtime)
// video frame (or even really for streaming - audio yes, video probably no).
// Note that MediaPipeline currently ignores the timestamps from MSG
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
if (mSources[i]) {
AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
}
}
return 0;
}
size_t
MediaEngineRemoteVideoSource::NumCapabilities()
{
int num = mozilla::camera::NumberOfCapabilities(mCapEngine, GetUUID().get());
if (num > 0) {
return num;
}
switch(mMediaSource) {
case dom::MediaSourceEnum::Camera:
#ifdef XP_MACOSX
// Mac doesn't support capabilities.
//
// Hardcode generic desktop capabilities modeled on OSX camera.
// Note: Values are empirically picked to be OSX friendly, as on OSX, values
// other than these cause the source to not produce.
if (mHardcodedCapabilities.IsEmpty()) {
for (int i = 0; i < 9; i++) {
webrtc::CaptureCapability c;
c.width = 1920 - i*128;
c.height = 1080 - i*72;
c.maxFPS = 30;
mHardcodedCapabilities.AppendElement(c);
}
for (int i = 0; i < 16; i++) {
webrtc::CaptureCapability c;
c.width = 640 - i*40;
c.height = 480 - i*30;
c.maxFPS = 30;
mHardcodedCapabilities.AppendElement(c);
}
}
break;
#endif
default:
webrtc::CaptureCapability c;
// The default for devices that don't return discrete capabilities: treat
// them as supporting all capabilities orthogonally. E.g. screensharing.
c.width = 0; // 0 = accept any value
c.height = 0;
c.maxFPS = 0;
mHardcodedCapabilities.AppendElement(c);
break;
}
return mHardcodedCapabilities.Length();
}
void
MediaEngineRemoteVideoSource::GetCapability(size_t aIndex,
webrtc::CaptureCapability& aOut)
{
if (!mHardcodedCapabilities.IsEmpty()) {
MediaEngineCameraVideoSource::GetCapability(aIndex, aOut);
}
mozilla::camera::GetCaptureCapability(mCapEngine,
GetUUID().get(),
aIndex,
aOut);
}
void MediaEngineRemoteVideoSource::Refresh(int aIndex) {
// NOTE: mCaptureIndex might have changed when allocated!
// Use aIndex to update information, but don't change mCaptureIndex!!
// Caller looked up this source by uniqueId, so it shouldn't change
char deviceName[kMaxDeviceNameLength];
char uniqueId[kMaxUniqueIdLength];
if (mozilla::camera::GetCaptureDevice(mCapEngine,
aIndex,
deviceName, sizeof(deviceName),
uniqueId, sizeof(uniqueId))) {
return;
}
SetName(NS_ConvertUTF8toUTF16(deviceName));
#ifdef DEBUG
MOZ_ASSERT(GetUUID().Equals(uniqueId));
#endif
}
}
dom/media/webrtc/MediaEngineRemoteVideoSource.h
+101
View File
@@ -0,0 +1,101 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_
#define MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_
#include "prcvar.h"
#include "prthread.h"
#include "nsIThread.h"
#include "nsIRunnable.h"
#include "mozilla/Mutex.h"
#include "mozilla/Monitor.h"
#include "nsCOMPtr.h"
#include "nsThreadUtils.h"
#include "DOMMediaStream.h"
#include "nsDirectoryServiceDefs.h"
#include "nsComponentManagerUtils.h"
#include "VideoUtils.h"
#include "MediaEngineCameraVideoSource.h"
#include "VideoSegment.h"
#include "AudioSegment.h"
#include "StreamBuffer.h"
#include "MediaStreamGraph.h"
#include "MediaEngineWrapper.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
// WebRTC library includes follow
#include "webrtc/common.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "CamerasChild.h"
#include "NullTransport.h"
namespace mozilla {
/**
* The WebRTC implementation of the MediaEngine interface.
*/
class MediaEngineRemoteVideoSource : public MediaEngineCameraVideoSource,
public webrtc::ExternalRenderer
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
// ExternalRenderer
virtual int FrameSizeChange(unsigned int w, unsigned int h,
unsigned int streams) override;
virtual int DeliverFrame(unsigned char* buffer,
int size,
uint32_t time_stamp,
int64_t ntp_time,
int64_t render_time,
void *handle) override;
virtual bool IsTextureSupported() override { return false; };
// MediaEngineCameraVideoSource
MediaEngineRemoteVideoSource(int aIndex, mozilla::camera::CaptureEngine aCapEngine,
dom::MediaSourceEnum aMediaSource,
const char* aMonitorName = "RemoteVideo.Monitor");
virtual nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId) override;
virtual nsresult Deallocate() override;;
virtual nsresult Start(SourceMediaStream*, TrackID) override;
virtual nsresult Stop(SourceMediaStream*, TrackID) override;
virtual void NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aId,
StreamTime aDesiredTime) override;
virtual const dom::MediaSourceEnum GetMediaSource() override {
return mMediaSource;
}
void Refresh(int aIndex);
virtual void Shutdown() override;
protected:
~MediaEngineRemoteVideoSource() { Shutdown(); }
private:
// Initialize the needed Video engine interfaces.
void Init();
size_t NumCapabilities() override;
void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) override;
dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen)
mozilla::camera::CaptureEngine mCapEngine;
};
}
#endif /* MEDIAENGINE_REMOTE_VIDEO_SOURCE_H_ */
dom/media/webrtc/MediaEngineWebRTC.cpp
+37 -118
View File
@@ -1,9 +1,12 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set sw=2 ts=8 et ft=cpp : */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsIPrefService.h"
#include "nsIPrefBranch.h"
#include "CamerasUtils.h"
#include "CSFLog.h"
#include "prenv.h"
@@ -23,6 +26,8 @@ GetUserMediaLog()
#include "ImageContainer.h"
#include "nsIComponentRegistrar.h"
#include "MediaEngineTabVideoSource.h"
#include "MediaEngineRemoteVideoSource.h"
#include "CamerasChild.h"
#include "nsITabSource.h"
#include "MediaTrackConstraints.h"
@@ -42,18 +47,9 @@ GetUserMediaLog()
namespace mozilla {
MediaEngineWebRTC::MediaEngineWebRTC(MediaEnginePrefs &aPrefs)
: mMutex("mozilla::MediaEngineWebRTC")
, mScreenEngine(nullptr)
, mBrowserEngine(nullptr)
, mWinEngine(nullptr)
, mAppEngine(nullptr)
, mVideoEngine(nullptr)
, mVoiceEngine(nullptr)
, mVideoEngineInit(false)
, mAudioEngineInit(false)
, mScreenEngineInit(false)
, mBrowserEngineInit(false)
, mAppEngineInit(false)
: mMutex("mozilla::MediaEngineWebRTC"),
mVoiceEngine(nullptr),
mAudioEngineInit(false)
{
#ifndef MOZ_B2G_CAMERA
nsCOMPtr<nsIComponentRegistrar> compMgr;
@@ -122,11 +118,7 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
return;
#else
ScopedCustomReleasePtr<webrtc::ViEBase> ptrViEBase;
ScopedCustomReleasePtr<webrtc::ViECapture> ptrViECapture;
webrtc::Config configSet;
webrtc::VideoEngine *videoEngine = nullptr;
bool *videoEngineInit = nullptr;
mozilla::camera::CaptureEngine capEngine = mozilla::camera::InvalidEngine;
#ifdef MOZ_WIDGET_ANDROID
// get the JVM
@@ -140,74 +132,24 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
switch (aMediaSource) {
case dom::MediaSourceEnum::Window:
mWinEngineConfig.Set<webrtc::CaptureDeviceInfo>(
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Window));
if (!mWinEngine) {
if (!(mWinEngine = webrtc::VideoEngine::Create(mWinEngineConfig))) {
return;
}
}
videoEngine = mWinEngine;
videoEngineInit = &mWinEngineInit;
capEngine = mozilla::camera::WinEngine;
break;
case dom::MediaSourceEnum::Application:
mAppEngineConfig.Set<webrtc::CaptureDeviceInfo>(
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Application));
if (!mAppEngine) {
if (!(mAppEngine = webrtc::VideoEngine::Create(mAppEngineConfig))) {
return;
}
}
videoEngine = mAppEngine;
videoEngineInit = &mAppEngineInit;
capEngine = mozilla::camera::AppEngine;
break;
case dom::MediaSourceEnum::Screen:
mScreenEngineConfig.Set<webrtc::CaptureDeviceInfo>(
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Screen));
if (!mScreenEngine) {
if (!(mScreenEngine = webrtc::VideoEngine::Create(mScreenEngineConfig))) {
return;
}
}
videoEngine = mScreenEngine;
videoEngineInit = &mScreenEngineInit;
capEngine = mozilla::camera::ScreenEngine;
break;
case dom::MediaSourceEnum::Browser:
mBrowserEngineConfig.Set<webrtc::CaptureDeviceInfo>(
new webrtc::CaptureDeviceInfo(webrtc::CaptureDeviceType::Browser));
if (!mBrowserEngine) {
if (!(mBrowserEngine = webrtc::VideoEngine::Create(mBrowserEngineConfig))) {
return;
}
}
videoEngine = mBrowserEngine;
videoEngineInit = &mBrowserEngineInit;
capEngine = mozilla::camera::BrowserEngine;
break;
case dom::MediaSourceEnum::Camera:
// fall through
default:
if (!mVideoEngine) {
if (!(mVideoEngine = webrtc::VideoEngine::Create())) {
return;
}
}
videoEngine = mVideoEngine;
videoEngineInit = &mVideoEngineInit;
capEngine = mozilla::camera::CameraEngine;
break;
default:
// BOOM
MOZ_CRASH("No valid video engine");
break;
}
ptrViEBase = webrtc::ViEBase::GetInterface(videoEngine);
if (!ptrViEBase) {
return;
}
if (ptrViEBase->Init() < 0) {
return;
}
*videoEngineInit = true;
ptrViECapture = webrtc::ViECapture::GetInterface(videoEngine);
if (!ptrViECapture) {
return;
}
/**
@@ -218,7 +160,8 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
* for a given instance of the engine. Likewise, if a device was plugged out,
* mVideoSources must be updated.
*/
int num = ptrViECapture->NumberOfCaptureDevices();
int num;
num = mozilla::camera::NumberOfCaptureDevices(capEngine);
if (num <= 0) {
return;
}
@@ -230,27 +173,29 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
// paranoia
deviceName[0] = '\0';
uniqueId[0] = '\0';
int error = ptrViECapture->GetCaptureDevice(i, deviceName,
sizeof(deviceName), uniqueId,
sizeof(uniqueId));
int error;
error = mozilla::camera::GetCaptureDevice(capEngine,
i, deviceName,
sizeof(deviceName), uniqueId,
sizeof(uniqueId));
if (error) {
LOG((" VieCapture:GetCaptureDevice: Failed %d",
ptrViEBase->LastError() ));
LOG(("camera:GetCaptureDevice: Failed %d", error ));
continue;
}
#ifdef DEBUG
LOG((" Capture Device Index %d, Name %s", i, deviceName));
webrtc::CaptureCapability cap;
int numCaps = ptrViECapture->NumberOfCapabilities(uniqueId,
MediaEngineSource::kMaxUniqueIdLength);
int numCaps = mozilla::camera::NumberOfCapabilities(capEngine,
uniqueId);
LOG(("Number of Capabilities %d", numCaps));
for (int j = 0; j < numCaps; j++) {
if (ptrViECapture->GetCaptureCapability(uniqueId,
MediaEngineSource::kMaxUniqueIdLength,
j, cap ) != 0 ) {
break;
if (mozilla::camera::GetCaptureCapability(capEngine,
uniqueId,
j, cap ) != 0 ) {
break;
}
LOG(("type=%d width=%d height=%d maxFPS=%d",
cap.rawType, cap.width, cap.height, cap.maxFPS ));
@@ -267,10 +212,10 @@ MediaEngineWebRTC::EnumerateVideoDevices(dom::MediaSourceEnum aMediaSource,
NS_ConvertUTF8toUTF16 uuid(uniqueId);
if (mVideoSources.Get(uuid, getter_AddRefs(vSource))) {
// We've already seen this device, just refresh and append.
static_cast<MediaEngineWebRTCVideoSource*>(vSource.get())->Refresh(i);
static_cast<MediaEngineRemoteVideoSource*>(vSource.get())->Refresh(i);
aVSources->AppendElement(vSource.get());
} else {
vSource = new MediaEngineWebRTCVideoSource(videoEngine, i, aMediaSource);
vSource = new MediaEngineRemoteVideoSource(i, capEngine, aMediaSource);
mVideoSources.Put(uuid, vSource); // Hashtable takes ownership.
aVSources->AppendElement(vSource);
}
@@ -414,40 +359,14 @@ MediaEngineWebRTC::Shutdown()
mVideoSources.Clear();
mAudioSources.Clear();
// Clear callbacks before we go away since the engines may outlive us
if (mVideoEngine) {
mVideoEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mVideoEngine);
}
if (mScreenEngine) {
mScreenEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mScreenEngine);
}
if (mWinEngine) {
mWinEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mWinEngine);
}
if (mBrowserEngine) {
mBrowserEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mBrowserEngine);
}
if (mAppEngine) {
mAppEngine->SetTraceCallback(nullptr);
webrtc::VideoEngine::Delete(mAppEngine);
}
if (mVoiceEngine) {
mVoiceEngine->SetTraceCallback(nullptr);
webrtc::VoiceEngine::Delete(mVoiceEngine);
}
mVideoEngine = nullptr;
mVoiceEngine = nullptr;
mScreenEngine = nullptr;
mWinEngine = nullptr;
mBrowserEngine = nullptr;
mAppEngine = nullptr;
mozilla::camera::Shutdown();
if (mThread) {
mThread->Shutdown();
dom/media/webrtc/MediaEngineWebRTC.h
+5 -101
View File
@@ -48,91 +48,13 @@
#include "webrtc/video_engine/include/vie_codec.h"
#include "webrtc/video_engine/include/vie_render.h"
#include "webrtc/video_engine/include/vie_capture.h"
#include "CamerasChild.h"
#include "NullTransport.h"
#include "AudioOutputObserver.h"
namespace mozilla {
/**
* The WebRTC implementation of the MediaEngine interface.
*/
class MediaEngineWebRTCVideoSource : public MediaEngineCameraVideoSource
, public webrtc::ExternalRenderer
{
public:
NS_DECL_THREADSAFE_ISUPPORTS
// ViEExternalRenderer.
virtual int FrameSizeChange(unsigned int w, unsigned int h, unsigned int streams) override;
virtual int DeliverFrame(unsigned char* buffer,
int size,
uint32_t time_stamp,
int64_t ntp_time_ms,
int64_t render_time,
void *handle) override;
/**
* Does DeliverFrame() support a null buffer and non-null handle
* (video texture)?
* XXX Investigate! Especially for Android/B2G
*/
virtual bool IsTextureSupported() override { return false; }
MediaEngineWebRTCVideoSource(webrtc::VideoEngine* aVideoEnginePtr, int aIndex,
dom::MediaSourceEnum aMediaSource = dom::MediaSourceEnum::Camera)
: MediaEngineCameraVideoSource(aIndex, "WebRTCCamera.Monitor")
, mVideoEngine(aVideoEnginePtr)
, mMinFps(-1)
, mMediaSource(aMediaSource)
{
MOZ_ASSERT(aVideoEnginePtr);
MOZ_ASSERT(aMediaSource != dom::MediaSourceEnum::Other);
Init();
}
virtual nsresult Allocate(const dom::MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
const nsString& aDeviceId) override;
virtual nsresult Deallocate() override;
virtual nsresult Start(SourceMediaStream*, TrackID) override;
virtual nsresult Stop(SourceMediaStream*, TrackID) override;
virtual void NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aId,
StreamTime aDesiredTime) override;
virtual const dom::MediaSourceEnum GetMediaSource() override {
return mMediaSource;
}
virtual nsresult TakePhoto(PhotoCallback* aCallback) override
{
return NS_ERROR_NOT_IMPLEMENTED;
}
void Refresh(int aIndex);
virtual void Shutdown() override;
protected:
~MediaEngineWebRTCVideoSource() { Shutdown(); }
private:
// Initialize the needed Video engine interfaces.
void Init();
// Engine variables.
webrtc::VideoEngine* mVideoEngine; // Weak reference, don't free.
ScopedCustomReleasePtr<webrtc::ViEBase> mViEBase;
ScopedCustomReleasePtr<webrtc::ViECapture> mViECapture;
ScopedCustomReleasePtr<webrtc::ViERender> mViERender;
int mMinFps; // Min rate we want to accept
dom::MediaSourceEnum mMediaSource; // source of media (camera | application | screen)
size_t NumCapabilities() override;
void GetCapability(size_t aIndex, webrtc::CaptureCapability& aOut) override;
};
class MediaEngineWebRTCAudioCaptureSource : public MediaEngineAudioSource
{
public:
@@ -266,10 +188,10 @@ public:
int16_t audio10ms[], int length,
int samplingFreq, bool isStereo) override;
NS_DECL_THREADSAFE_ISUPPORTS
virtual void Shutdown() override;
NS_DECL_THREADSAFE_ISUPPORTS
protected:
~MediaEngineWebRTCMicrophoneSource() { Shutdown(); }
@@ -330,29 +252,11 @@ private:
nsCOMPtr<nsIThread> mThread;
// gUM runnables can e.g. Enumerate from multiple threads
Mutex mMutex;
// protected with mMutex:
webrtc::VideoEngine* mScreenEngine;
webrtc::VideoEngine* mBrowserEngine;
webrtc::VideoEngine* mWinEngine;
webrtc::VideoEngine* mAppEngine;
webrtc::VideoEngine* mVideoEngine;
webrtc::VoiceEngine* mVoiceEngine;
// specialized configurations
webrtc::Config mAppEngineConfig;
webrtc::Config mWinEngineConfig;
webrtc::Config mScreenEngineConfig;
webrtc::Config mBrowserEngineConfig;
// Need this to avoid unneccesary WebRTC calls while enumerating.
bool mVideoEngineInit;
bool mAudioEngineInit;
bool mScreenEngineInit;
bool mBrowserEngineInit;
bool mWinEngineInit;
bool mAppEngineInit;
bool mHasTabVideoSource;
// Store devices we've already seen in a hashtable for quick return.
dom/media/webrtc/MediaEngineWebRTCVideo.cpp
-449
View File
@@ -1,449 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaEngineWebRTC.h"
#include "Layers.h"
#include "ImageTypes.h"
#include "ImageContainer.h"
#include "mozilla/layers/GrallocTextureClient.h"
#include "nsMemory.h"
#include "mtransport/runnable_utils.h"
#include "MediaTrackConstraints.h"
namespace mozilla {
using namespace mozilla::gfx;
using dom::ConstrainLongRange;
using dom::ConstrainDoubleRange;
using dom::MediaTrackConstraintSet;
extern PRLogModuleInfo* GetMediaManagerLog();
#define LOG(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Debug, msg)
#define LOGFRAME(msg) MOZ_LOG(GetMediaManagerLog(), mozilla::LogLevel::Verbose, msg)
/**
* Webrtc video source.
*/
NS_IMPL_ISUPPORTS0(MediaEngineWebRTCVideoSource)
int
MediaEngineWebRTCVideoSource::FrameSizeChange(
unsigned int w, unsigned int h, unsigned int streams)
{
mWidth = w;
mHeight = h;
LOG(("Video FrameSizeChange: %ux%u", w, h));
return 0;
}
// ViEExternalRenderer Callback. Process every incoming frame here.
int
MediaEngineWebRTCVideoSource::DeliverFrame(
unsigned char* buffer, int size, uint32_t time_stamp,
int64_t ntp_time_ms, int64_t render_time, void *handle)
{
// Check for proper state.
if (mState != kStarted) {
LOG(("DeliverFrame: video not started"));
return 0;
}
if (mWidth*mHeight + 2*(((mWidth+1)/2)*((mHeight+1)/2)) != size) {
MOZ_ASSERT(false, "Wrong size frame in DeliverFrame!");
return 0;
}
// Create a video frame and append it to the track.
nsRefPtr<layers::Image> image = mImageContainer->CreateImage(ImageFormat::PLANAR_YCBCR);
layers::PlanarYCbCrImage* videoImage = static_cast<layers::PlanarYCbCrImage*>(image.get());
uint8_t* frame = static_cast<uint8_t*> (buffer);
const uint8_t lumaBpp = 8;
const uint8_t chromaBpp = 4;
// Take lots of care to round up!
layers::PlanarYCbCrData data;
data.mYChannel = frame;
data.mYSize = IntSize(mWidth, mHeight);
data.mYStride = (mWidth * lumaBpp + 7)/ 8;
data.mCbCrStride = (mWidth * chromaBpp + 7) / 8;
data.mCbChannel = frame + mHeight * data.mYStride;
data.mCrChannel = data.mCbChannel + ((mHeight+1)/2) * data.mCbCrStride;
data.mCbCrSize = IntSize((mWidth+1)/ 2, (mHeight+1)/ 2);
data.mPicX = 0;
data.mPicY = 0;
data.mPicSize = IntSize(mWidth, mHeight);
data.mStereoMode = StereoMode::MONO;
videoImage->SetData(data);
#ifdef DEBUG
static uint32_t frame_num = 0;
LOGFRAME(("frame %d (%dx%d); timestamp %u, ntp_time %lu, render_time %lu", frame_num++,
mWidth, mHeight, time_stamp, ntp_time_ms, render_time));
#endif
// we don't touch anything in 'this' until here (except for snapshot,
// which has it's own lock)
MonitorAutoLock lock(mMonitor);
// implicitly releases last image
mImage = image.forget();
// Push the frame into the MSG with a minimal duration. This will likely
// mean we'll still get NotifyPull calls which will then return the same
// frame again with a longer duration. However, this means we won't
// fail to get the frame in and drop frames.
// XXX The timestamp for the frame should be based on the Capture time,
// not the MSG time, and MSG should never, ever block on a (realtime)
// video frame (or even really for streaming - audio yes, video probably no).
// Note that MediaPipeline currently ignores the timestamps from MSG
uint32_t len = mSources.Length();
for (uint32_t i = 0; i < len; i++) {
if (mSources[i]) {
AppendToTrack(mSources[i], mImage, mTrackID, 1); // shortest possible duration
}
}
return 0;
}
// Called if the graph thinks it's running out of buffered video; repeat
// the last frame for whatever minimum period it think it needs. Note that
// this means that no *real* frame can be inserted during this period.
void
MediaEngineWebRTCVideoSource::NotifyPull(MediaStreamGraph* aGraph,
SourceMediaStream* aSource,
TrackID aID,
StreamTime aDesiredTime)
{
VideoSegment segment;
MonitorAutoLock lock(mMonitor);
// B2G does AddTrack, but holds kStarted until the hardware changes state.
// So mState could be kReleased here. We really don't care about the state,
// though.
StreamTime delta = aDesiredTime - aSource->GetEndOfAppendedData(aID);
LOGFRAME(("NotifyPull, desired = %ld, delta = %ld %s", (int64_t) aDesiredTime,
(int64_t) delta, mImage.get() ? "" : "<null>"));
// Bug 846188 We may want to limit incoming frames to the requested frame rate
// mFps - if you want 30FPS, and the camera gives you 60FPS, this could
// cause issues.
// We may want to signal if the actual frame rate is below mMinFPS -
// cameras often don't return the requested frame rate especially in low
// light; we should consider surfacing this so that we can switch to a
// lower resolution (which may up the frame rate)
// Don't append if we've already provided a frame that supposedly goes past the current aDesiredTime
// Doing so means a negative delta and thus messes up handling of the graph
if (delta > 0) {
// nullptr images are allowed
AppendToTrack(aSource, mImage, aID, delta);
}
}
size_t
MediaEngineWebRTCVideoSource::NumCapabilities()
{
int num = mViECapture->NumberOfCapabilities(GetUUID().get(), kMaxUniqueIdLength);
if (num > 0) {
return num;
}
switch (mMediaSource) {
case dom::MediaSourceEnum::Camera:
#ifdef XP_MACOSX
// Mac doesn't support capabilities.
//
// Hardcode generic desktop capabilities modeled on OSX camera.
// Note: Values are empirically picked to be OSX friendly, as on OSX,
// values other than these cause the source to not produce.
if (mHardcodedCapabilities.IsEmpty()) {
for (int i = 0; i < 9; i++) {
webrtc::CaptureCapability c;
c.width = 1920 - i*128;
c.height = 1080 - i*72;
c.maxFPS = 30;
mHardcodedCapabilities.AppendElement(c);
}
for (int i = 0; i < 16; i++) {
webrtc::CaptureCapability c;
c.width = 640 - i*40;
c.height = 480 - i*30;
c.maxFPS = 30;
mHardcodedCapabilities.AppendElement(c);
}
}
break;
#endif
default:
// The default for devices that don't return discrete capabilities: treat
// them as supporting all capabilities orthogonally. E.g. screensharing.
webrtc::CaptureCapability c;
c.width = 0; // 0 = accept any value
c.height = 0;
c.maxFPS = 0;
mHardcodedCapabilities.AppendElement(c);
break;
}
return mHardcodedCapabilities.Length();
}
void
MediaEngineWebRTCVideoSource::GetCapability(size_t aIndex,
webrtc::CaptureCapability& aOut)
{
if (!mHardcodedCapabilities.IsEmpty()) {
MediaEngineCameraVideoSource::GetCapability(aIndex, aOut);
}
mViECapture->GetCaptureCapability(GetUUID().get(), kMaxUniqueIdLength, aIndex, aOut);
}
nsresult
MediaEngineWebRTCVideoSource::Allocate(const dom::MediaTrackConstraints &aConstraints,
const MediaEnginePrefs &aPrefs,
const nsString& aDeviceId)
{
LOG((__FUNCTION__));
if (mState == kReleased && mInitDone) {
// Note: if shared, we don't allow a later opener to affect the resolution.
// (This may change depending on spec changes for Constraints/settings)
if (!ChooseCapability(aConstraints, aPrefs, aDeviceId)) {
return NS_ERROR_UNEXPECTED;
}
if (mViECapture->AllocateCaptureDevice(GetUUID().get(),
kMaxUniqueIdLength, mCaptureIndex)) {
return NS_ERROR_FAILURE;
}
mState = kAllocated;
LOG(("Video device %d allocated", mCaptureIndex));
} else if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
MonitorAutoLock lock(mMonitor);
if (mSources.IsEmpty()) {
LOG(("Video device %d reallocated", mCaptureIndex));
} else {
LOG(("Video device %d allocated shared", mCaptureIndex));
}
}
return NS_OK;
}
nsresult
MediaEngineWebRTCVideoSource::Deallocate()
{
LOG((__FUNCTION__));
bool empty;
{
MonitorAutoLock lock(mMonitor);
empty = mSources.IsEmpty();
}
if (empty) {
// If empty, no callbacks to deliver data should be occuring
if (mState != kStopped && mState != kAllocated) {
return NS_ERROR_FAILURE;
}
#ifdef XP_MACOSX
// Bug 829907 - on mac, in shutdown, the mainthread stops processing
// 'native' events, and the QTKit code uses events to the main native CFRunLoop
// in order to provide thread safety. In order to avoid this locking us up,
// release the ViE capture device synchronously on MainThread (so the native
// event isn't needed).
// XXX Note if MainThread Dispatch()es NS_DISPATCH_SYNC to us we can deadlock.
// XXX It might be nice to only do this if we're in shutdown... Hard to be
// sure when that is though.
// Thread safety: a) we call this synchronously, and don't use ViECapture from
// another thread anywhere else, b) ViEInputManager::DestroyCaptureDevice() grabs
// an exclusive object lock and deletes it in a critical section, so all in all
// this should be safe threadwise.
NS_DispatchToMainThread(WrapRunnable(mViECapture.get(),
&webrtc::ViECapture::ReleaseCaptureDevice,
mCaptureIndex),
NS_DISPATCH_SYNC);
#else
mViECapture->ReleaseCaptureDevice(mCaptureIndex);
#endif
mState = kReleased;
LOG(("Video device %d deallocated", mCaptureIndex));
} else {
LOG(("Video device %d deallocated but still in use", mCaptureIndex));
}
return NS_OK;
}
nsresult
MediaEngineWebRTCVideoSource::Start(SourceMediaStream* aStream, TrackID aID)
{
LOG((__FUNCTION__));
int error = 0;
if (!mInitDone || !aStream) {
return NS_ERROR_FAILURE;
}
{
MonitorAutoLock lock(mMonitor);
mSources.AppendElement(aStream);
}
aStream->AddTrack(aID, 0, new VideoSegment(), SourceMediaStream::ADDTRACK_QUEUED);
if (mState == kStarted) {
return NS_OK;
}
mImageContainer = layers::LayerManager::CreateImageContainer();
mState = kStarted;
mTrackID = aID;
error = mViERender->AddRenderer(mCaptureIndex, webrtc::kVideoI420, (webrtc::ExternalRenderer*)this);
if (error == -1) {
return NS_ERROR_FAILURE;
}
error = mViERender->StartRender(mCaptureIndex);
if (error == -1) {
return NS_ERROR_FAILURE;
}
if (mViECapture->StartCapture(mCaptureIndex, mCapability) < 0) {
return NS_ERROR_FAILURE;
}
return NS_OK;
}
nsresult
MediaEngineWebRTCVideoSource::Stop(SourceMediaStream *aSource, TrackID aID)
{
LOG((__FUNCTION__));
{
MonitorAutoLock lock(mMonitor);
if (!mSources.RemoveElement(aSource)) {
// Already stopped - this is allowed
return NS_OK;
}
aSource->EndTrack(aID);
if (!mSources.IsEmpty()) {
return NS_OK;
}
if (mState != kStarted) {
return NS_ERROR_FAILURE;
}
mState = kStopped;
// Drop any cached image so we don't start with a stale image on next
// usage
mImage = nullptr;
}
mViERender->StopRender(mCaptureIndex);
mViERender->RemoveRenderer(mCaptureIndex);
mViECapture->StopCapture(mCaptureIndex);
return NS_OK;
}
void
MediaEngineWebRTCVideoSource::Init()
{
// fix compile warning for these being unused. (remove once used)
(void) mFps;
(void) mMinFps;
LOG((__FUNCTION__));
if (mVideoEngine == nullptr) {
return;
}
mViEBase = webrtc::ViEBase::GetInterface(mVideoEngine);
if (mViEBase == nullptr) {
return;
}
// Get interfaces for capture, render for now
mViECapture = webrtc::ViECapture::GetInterface(mVideoEngine);
mViERender = webrtc::ViERender::GetInterface(mVideoEngine);
if (mViECapture == nullptr || mViERender == nullptr) {
return;
}
char deviceName[kMaxDeviceNameLength];
char uniqueId[kMaxUniqueIdLength];
if (mViECapture->GetCaptureDevice(mCaptureIndex,
deviceName, kMaxDeviceNameLength,
uniqueId, kMaxUniqueIdLength)) {
return;
}
SetName(NS_ConvertUTF8toUTF16(deviceName));
SetUUID(uniqueId);
mInitDone = true;
}
void
MediaEngineWebRTCVideoSource::Shutdown()
{
LOG((__FUNCTION__));
if (!mInitDone) {
return;
}
if (mState == kStarted) {
SourceMediaStream *source;
bool empty;
while (1) {
{
MonitorAutoLock lock(mMonitor);
empty = mSources.IsEmpty();
if (empty) {
break;
}
source = mSources[0];
}
Stop(source, kVideoTrack); // XXX change to support multiple tracks
}
MOZ_ASSERT(mState == kStopped);
}
if (mState == kAllocated || mState == kStopped) {
Deallocate();
}
mViECapture = nullptr;
mViERender = nullptr;
mViEBase = nullptr;
mState = kReleased;
mInitDone = false;
}
void MediaEngineWebRTCVideoSource::Refresh(int aIndex) {
// NOTE: mCaptureIndex might have changed when allocated!
// Use aIndex to update information, but don't change mCaptureIndex!!
// Caller looked up this source by uniqueId, so it shouldn't change
char deviceName[kMaxDeviceNameLength];
char uniqueId[kMaxUniqueIdLength];
if (mViECapture->GetCaptureDevice(aIndex,
deviceName, sizeof(deviceName),
uniqueId, sizeof(uniqueId))) {
return;
}
SetName(NS_ConvertUTF8toUTF16(deviceName));
#ifdef DEBUG
MOZ_ASSERT(GetUUID().Equals(uniqueId));
#endif
}
} // namespace mozilla
dom/media/webrtc/moz.build
+10 -10
View File
@@ -21,15 +21,16 @@ EXPORTS += [
if CONFIG['MOZ_WEBRTC']:
EXPORTS += ['AudioOutputObserver.h',
'MediaEngineRemoteVideoSource.h',
'MediaEngineWebRTC.h']
EXPORTS.mozilla.dom += [ 'RTCIdentityProviderRegistrar.h' ]
UNIFIED_SOURCES += [
'MediaEngineCameraVideoSource.cpp',
'MediaEngineRemoteVideoSource.cpp',
'MediaEngineTabVideoSource.cpp',
'MediaEngineWebRTCAudio.cpp',
'MediaEngineWebRTCVideo.cpp',
'MediaTrackConstraints.cpp',
'RTCCertificate.cpp',
'RTCCertificate.cpp',
'RTCIdentityProviderRegistrar.cpp',
]
# MediaEngineWebRTC.cpp needs to be built separately.
@@ -78,9 +79,9 @@ include('/ipc/chromium/chromium-config.mozbuild')
# defined, which complains about important MOZ_EXPORT attributes for
# android API types
if CONFIG['GNU_CC'] or CONFIG['CLANG_CL']:
CXXFLAGS += [
'-Wno-error=attributes'
]
CXXFLAGS += [
'-Wno-error=attributes'
]
FINAL_LIBRARY = 'xul'
if CONFIG['OS_ARCH'] == 'WINNT':
@@ -88,8 +89,7 @@ if CONFIG['OS_ARCH'] == 'WINNT':
if CONFIG['_MSC_VER']:
CXXFLAGS += [
'-wd4275', # non dll-interface class used as base for dll-interface class
]
FAIL_ON_WARNINGS = True
CXXFLAGS += [
'-wd4275', # non dll-interface class used as base for dll-interface class
]
DEFINES['__PRETTY_FUNCTION__'] = '__FUNCSIG__'
ipc/glue/BackgroundChildImpl.cpp
+27
View File
@@ -8,6 +8,10 @@
#include "BroadcastChannelChild.h"
#include "ServiceWorkerManagerChild.h"
#include "FileDescriptorSetChild.h"
#ifdef MOZ_WEBRTC
#include "CamerasChild.h"
#endif
#include "mozilla/media/MediaChild.h"
#include "mozilla/Assertions.h"
#include "mozilla/dom/PBlobChild.h"
#include "mozilla/dom/asmjscache/AsmJSCache.h"
@@ -265,6 +269,29 @@ BackgroundChildImpl::DeallocPBroadcastChannelChild(
return true;
}
camera::PCamerasChild*
BackgroundChildImpl::AllocPCamerasChild()
{
#ifdef MOZ_WEBRTC
nsRefPtr<camera::CamerasChild> agent =
new camera::CamerasChild();
return agent.forget().take();
#else
return nullptr;
#endif
}
bool
BackgroundChildImpl::DeallocPCamerasChild(camera::PCamerasChild *aActor)
{
#ifdef MOZ_WEBRTC
nsRefPtr<camera::CamerasChild> child =
dont_AddRef(static_cast<camera::CamerasChild*>(aActor));
MOZ_ASSERT(aActor);
#endif
return true;
}
// -----------------------------------------------------------------------------
// ServiceWorkerManager
// -----------------------------------------------------------------------------
ipc/glue/BackgroundChildImpl.h
+6
View File
@@ -71,6 +71,12 @@ protected:
virtual bool
DeallocPFileDescriptorSetChild(PFileDescriptorSetChild* aActor) override;
virtual PCamerasChild*
AllocPCamerasChild() override;
virtual bool
DeallocPCamerasChild(PCamerasChild* aActor) override;
virtual PVsyncChild*
AllocPVsyncChild() override;
ipc/glue/BackgroundParentImpl.cpp
+33
View File
@@ -6,6 +6,10 @@
#include "BroadcastChannelParent.h"
#include "FileDescriptorSetParent.h"
#ifdef MOZ_WEBRTC
#include "CamerasParent.h"
#endif
#include "mozilla/media/MediaParent.h"
#include "mozilla/AppProcessChecker.h"
#include "mozilla/Assertions.h"
#include "mozilla/dom/ContentParent.h"
@@ -282,6 +286,35 @@ BackgroundParentImpl::DeallocPVsyncParent(PVsyncParent* aActor)
return true;
}
camera::PCamerasParent*
BackgroundParentImpl::AllocPCamerasParent()
{
AssertIsInMainProcess();
AssertIsOnBackgroundThread();
#ifdef MOZ_WEBRTC
nsRefPtr<mozilla::camera::CamerasParent> actor =
mozilla::camera::CamerasParent::Create();
return actor.forget().take();
#else
return nullptr;
#endif
}
bool
BackgroundParentImpl::DeallocPCamerasParent(camera::PCamerasParent *aActor)
{
AssertIsInMainProcess();
AssertIsOnBackgroundThread();
MOZ_ASSERT(aActor);
#ifdef MOZ_WEBRTC
nsRefPtr<mozilla::camera::CamerasParent> actor =
dont_AddRef(static_cast<mozilla::camera::CamerasParent*>(aActor));
#endif
return true;
}
namespace {
class InitUDPSocketParentCallback final : public nsRunnable
ipc/glue/BackgroundParentImpl.h
+6
View File
@@ -101,6 +101,12 @@ protected:
virtual bool
DeallocPServiceWorkerManagerParent(PServiceWorkerManagerParent* aActor) override;
virtual PCamerasParent*
AllocPCamerasParent() override;
virtual bool
DeallocPCamerasParent(PCamerasParent* aActor) override;
virtual bool
RecvShutdownServiceWorkerRegistrar() override;
ipc/glue/PBackground.ipdl
+4
View File
@@ -12,6 +12,7 @@ include protocol PCacheStorage;
include protocol PCacheStreamControl;
include protocol PFileDescriptorSet;
include protocol PMessagePort;
include protocol PCameras;
include protocol PNuwa;
include protocol PServiceWorkerManager;
include protocol PUDPSocket;
@@ -47,6 +48,7 @@ sync protocol PBackground
manages PCacheStreamControl;
manages PFileDescriptorSet;
manages PMessagePort;
manages PCameras;
manages PNuwa;
manages PServiceWorkerManager;
manages PUDPSocket;
@@ -60,6 +62,8 @@ parent:
PVsync();
PCameras();
PUDPSocket(OptionalPrincipalInfo pInfo, nsCString filter);
PBroadcastChannel(PrincipalInfo pInfo, nsCString origin, nsString channel,
bool privateBrowsing);
ipc/glue/moz.build
+1
View File
@@ -139,6 +139,7 @@ LOCAL_INCLUDES += [
'/dom/broadcastchannel',
'/dom/indexedDB',
'/dom/workers',
'/media/webrtc/trunk',
'/xpcom/build',
]
modules/libpref/init/all.js
+4
View File
@@ -515,6 +515,10 @@ pref("media.autoplay.allowscripted", true);
// MediaDecoderReader's mVideoQueue.
pref("media.video-queue.default-size", 10);
// The maximum number of queued frames to send to the compositor.
// By default, send all of them.
pref("media.video-queue.send-to-compositor-size", 9999);
// Whether to disable the video stats to prevent fingerprinting
pref("media.video_stats.enabled", true);