diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h index af74cb69ae..bbb768a2de 100644 --- a/js/src/gc/GCRuntime.h +++ b/js/src/gc/GCRuntime.h @@ -969,8 +969,6 @@ class GCRuntime void sweepZones(FreeOp* fop, bool lastGC); void decommitAllWithoutUnlocking(const AutoLockGC& lock); void startDecommit(); - bool sweepBackgroundFinalizePhaseInParallel(ZoneList& zones, const FinalizePhase& phase, - Arena** emptyArenas); void queueZonesForBackgroundSweep(ZoneList& zones); void sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks); void assertBackgroundSweepingFinished(); @@ -984,7 +982,7 @@ class GCRuntime [[nodiscard]] bool relocateArenas(Zone* zone, JS::gcreason::Reason reason, Arena*& relocatedListOut, SliceBudget& sliceBudget); void updateTypeDescrObjects(MovingTracer* trc, Zone* zone); - void updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds); + void updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds, size_t bgTaskCount); void updateAllCellPointers(MovingTracer* trc, Zone* zone); void updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess& lock); void protectAndHoldArenas(Arena* arenaList); diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp index f30406ed4a..8ffb0a5339 100644 --- a/js/src/jsgc.cpp +++ b/js/src/jsgc.cpp @@ -2118,7 +2118,7 @@ ArenasToUpdate::next(AutoLockHelperThreadState& lock) // Find the next arena to update. // // This iterates through the GC thing kinds filtered by shouldProcessKind(), - // and then through the arenas of that kind. All state is held in the + // and then through thea arenas of that kind. All state is held in the // object and we just return when we find an arena. for (; kind < AllocKind::LIMIT; kind = nextAllocKind(kind)) { @@ -2208,12 +2208,21 @@ UpdatePointersTask::run() } // namespace gc } // namespace js -static const size_t MinCellUpdateBackgroundTasks = 1; +static const size_t MinCellUpdateBackgroundTasks = 2; static const size_t MaxCellUpdateBackgroundTasks = 8; -static bool -CanUpdateKindInBackground(AllocKind kind) +static size_t +CellUpdateBackgroundTaskCount() { + if (!CanUseExtraThreads()) + return 0; + + size_t targetTaskCount = HelperThreadState().cpuCount / 2; + return Min(Max(targetTaskCount, MinCellUpdateBackgroundTasks), MaxCellUpdateBackgroundTasks); +} + +static bool +CanUpdateKindInBackground(AllocKind kind) { // We try to update as many GC things in parallel as we can, but there are // kinds for which this might not be safe: // - we assume JSObjects that are foreground finalized are not safe to @@ -2225,34 +2234,6 @@ CanUpdateKindInBackground(AllocKind kind) return true; } -static size_t -CountBackgroundUpdateArenas(Zone* zone, AllocKinds kinds) -{ - size_t arenaCount = 0; - for (AllocKind kind : kinds) { - MOZ_ASSERT(CanUpdateKindInBackground(kind)); - for (Arena* arena = zone->arenas.getFirstArena(kind); arena; arena = arena->next) - arenaCount++; - } - return arenaCount; -} - -static size_t -CellUpdateBackgroundTaskCount(Zone* zone, AllocKinds kinds) -{ - if (!CanUseExtraThreads() || kinds.isEmpty()) - return 0; - - size_t arenaCount = CountBackgroundUpdateArenas(zone, kinds); - if (arenaCount < UpdatePointersTask::MaxArenasToProcess * 2) - return 0; - - size_t targetTaskCount = HelperThreadState().cpuCount / 2; - size_t workTaskCount = arenaCount / UpdatePointersTask::MaxArenasToProcess; - targetTaskCount = Min(targetTaskCount, workTaskCount); - return Min(Max(targetTaskCount, MinCellUpdateBackgroundTasks), MaxCellUpdateBackgroundTasks); -} - static AllocKinds ForegroundUpdateKinds(AllocKinds kinds) { @@ -2273,15 +2254,10 @@ GCRuntime::updateTypeDescrObjects(MovingTracer* trc, Zone* zone) } void -GCRuntime::updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds) +GCRuntime::updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds, size_t bgTaskCount) { - MOZ_ASSERT(trc); - - AllocKinds fgKinds = ForegroundUpdateKinds(kinds); + AllocKinds fgKinds = bgTaskCount == 0 ? kinds : ForegroundUpdateKinds(kinds); AllocKinds bgKinds = kinds - fgKinds; - size_t bgTaskCount = CellUpdateBackgroundTaskCount(zone, bgKinds); - if (bgTaskCount == 0) - fgKinds = kinds; ArenasToUpdate fgArenas(zone, fgKinds); ArenasToUpdate bgArenas(zone, bgKinds); @@ -2376,13 +2352,15 @@ GCRuntime::updateAllCellPointers(MovingTracer* trc, Zone* zone) { AutoDisableProxyCheck noProxyCheck(rt); // These checks assert when run in parallel. - updateCellPointers(trc, zone, UpdatePhaseMisc); + size_t bgTaskCount = CellUpdateBackgroundTaskCount(); + + updateCellPointers(trc, zone, UpdatePhaseMisc, bgTaskCount); // Update TypeDescrs before all other objects as typed objects access these // objects when we trace them. updateTypeDescrObjects(trc, zone); - updateCellPointers(trc, zone, UpdatePhaseObjects); + updateCellPointers(trc, zone, UpdatePhaseObjects, bgTaskCount); } /* @@ -2688,17 +2666,6 @@ ArenaLists::backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty) lists->backgroundFinalizeState[thingKind] = BFS_DONE; } -void -ArenaLists::backgroundFinalizePhase(FreeOp* fop, const FinalizePhase& phase, Arena** empty) -{ - for (auto kind : phase.kinds) { - Arena* arenas = arenaListsToSweep[kind]; - MOZ_RELEASE_ASSERT(uintptr_t(arenas) != uintptr_t(-1)); - if (arenas) - backgroundFinalize(fop, arenas, empty); - } -} - void ArenaLists::queueForegroundObjectsForSweep(FreeOp* fop) { @@ -3037,138 +3004,6 @@ js::gc::BackgroundDecommitTask::run() } } -class BackgroundFinalizeTask : public GCParallelTaskHelper -{ - Zone* zone_; - const FinalizePhase* phase_; - Arena* emptyArenas_; - - BackgroundFinalizeTask(const BackgroundFinalizeTask&) = delete; - - public: - BackgroundFinalizeTask(Zone* zone, const FinalizePhase* phase) - : zone_(zone), - phase_(phase), - emptyArenas_(nullptr) - {} - - BackgroundFinalizeTask(BackgroundFinalizeTask&& other) - : GCParallelTaskHelper(mozilla::Move(other)), - zone_(other.zone_), - phase_(other.phase_), - emptyArenas_(other.emptyArenas_) - { - other.emptyArenas_ = nullptr; - } - - void run() { - AutoSetThreadIsSweeping threadIsSweeping; - finalize(); - } - - void runAlreadySweeping() { -#ifdef DEBUG - MOZ_ASSERT(CurrentThreadIsGCSweeping()); -#endif - finalize(); - } - - private: - void finalize() { - FreeOp fop(nullptr); - zone_->arenas.backgroundFinalizePhase(&fop, *phase_, &emptyArenas_); - } - - public: - Arena* takeEmptyArenas() { - Arena* empty = emptyArenas_; - emptyArenas_ = nullptr; - return empty; - } -}; - -using BackgroundFinalizeTaskVector = - Vector; - -static size_t -IdleHelperThreadCount(const AutoLockHelperThreadState&) -{ - if (!HelperThreadState().threads) - return 0; - - size_t idle = 0; - for (const auto& thread : *HelperThreadState().threads) { - if (thread.idle()) - idle++; - } - return idle; -} - -static void -PrependArenaList(Arena** head, Arena* arenas) -{ - if (!arenas) - return; - - Arena* tail = arenas; - while (tail->next) - tail = tail->next; - tail->next = *head; - *head = arenas; -} - -bool -GCRuntime::sweepBackgroundFinalizePhaseInParallel(ZoneList& zones, const FinalizePhase& phase, - Arena** emptyArenas) -{ - if (!CanUseExtraThreads()) - return false; - - size_t zoneCount = 0; - for (Zone* zone = zones.front(); zone; zone = zone->nextZone()) - zoneCount++; - - if (zoneCount < 2) - return false; - - BackgroundFinalizeTaskVector tasks; - if (!tasks.reserve(zoneCount)) - return false; - - for (Zone* zone = zones.front(); zone; zone = zone->nextZone()) - tasks.infallibleEmplaceBack(zone, &phase); - - size_t tasksStarted = 0; - - { - AutoLockHelperThreadState helperLock; - - // sweepBackgroundThings() itself runs as a GC helper task. Do not queue - // nested GC parallel tasks unless at least one other helper is idle. - if (IdleHelperThreadCount(helperLock) == 0) - return false; - - for (; tasksStarted < tasks.length(); tasksStarted++) { - if (!tasks[tasksStarted].startWithLockHeld(helperLock)) - break; - } - - { - AutoUnlockHelperThreadState unlock(helperLock); - for (size_t i = tasksStarted; i < tasks.length(); i++) - tasks[i].runAlreadySweeping(); - } - - for (size_t i = 0; i < tasksStarted; i++) - tasks[i].joinWithLockHeld(helperLock); - } - - for (auto& task : tasks) - PrependArenaList(emptyArenas, task.takeEmptyArenas()); - - return true; -} - void GCRuntime::sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks) { @@ -3181,12 +3016,14 @@ GCRuntime::sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks) Arena* emptyArenas = nullptr; FreeOp fop(nullptr); for (unsigned phase = 0 ; phase < ArrayLength(BackgroundFinalizePhases) ; ++phase) { - const FinalizePhase& finalizePhase = BackgroundFinalizePhases[phase]; - if (sweepBackgroundFinalizePhaseInParallel(zones, finalizePhase, &emptyArenas)) - continue; - - for (Zone* zone = zones.front(); zone; zone = zone->nextZone()) - zone->arenas.backgroundFinalizePhase(&fop, finalizePhase, &emptyArenas); + for (Zone* zone = zones.front(); zone; zone = zone->nextZone()) { + for (auto kind : BackgroundFinalizePhases[phase].kinds) { + Arena* arenas = zone->arenas.arenaListsToSweep[kind]; + MOZ_RELEASE_ASSERT(uintptr_t(arenas) != uintptr_t(-1)); + if (arenas) + ArenaLists::backgroundFinalize(&fop, arenas, &emptyArenas); + } + } } AutoLockGC lock(rt); diff --git a/js/src/jsgc.h b/js/src/jsgc.h index f6d3208d9e..521dea05c6 100644 --- a/js/src/jsgc.h +++ b/js/src/jsgc.h @@ -804,7 +804,6 @@ class ArenaLists bool foregroundFinalize(FreeOp* fop, AllocKind thingKind, SliceBudget& sliceBudget, SortedArenaList& sweepList); - void backgroundFinalizePhase(FreeOp* fop, const FinalizePhase& phase, Arena** empty); static void backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty); // When finalizing arenas, whether to keep empty arenas on the list or