/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * * Copyright 2014 Mozilla Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "asmjs/AsmJSModule.h" #include "mozilla/Atomics.h" #include "mozilla/BinarySearch.h" #include "mozilla/Compression.h" #include "mozilla/EnumeratedRange.h" #include "mozilla/PodOperations.h" #include "mozilla/TaggedAnonymousMemory.h" #include "mozilla/Vector.h" #include "jslibmath.h" #include "jsmath.h" #include "jsprf.h" #include "builtin/AtomicsObject.h" #include "frontend/Parser.h" #include "jit/IonCode.h" #ifdef JS_ION_PERF # include "jit/PerfSpewer.h" #endif #include "js/Class.h" #include "js/Conversions.h" #include "js/MemoryMetrics.h" #include "vm/Time.h" #include "jsobjinlines.h" #include "frontend/ParseNode-inl.h" #include "jit/MacroAssembler-inl.h" #include "vm/ArrayBufferObject-inl.h" #include "vm/Stack-inl.h" using namespace js; using namespace js::jit; using namespace js::wasm; using namespace js::frontend; using mozilla::Atomic; using mozilla::BinarySearch; using mozilla::Compression::LZ4; using mozilla::MakeEnumeratedRange; using mozilla::MallocSizeOf; using mozilla::PodCopy; using mozilla::PodEqual; using mozilla::PodZero; using mozilla::Swap; using JS::GenericNaN; // Limit the number of concurrent wasm code allocations per process. Note that // on Linux, the real maximum is ~32k, as each module requires 2 maps (RW/RX), // and the kernel's default max_map_count is ~65k. static Atomic wasmCodeAllocations(0); static const uint32_t MaxWasmCodeAllocations = 16384; static uint8_t* AllocateExecutableMemory(ExclusiveContext* cx, size_t bytes) { // bytes is a multiple of the system's page size, but not necessarily // a multiple of ExecutableCodePageSize. bytes = JS_ROUNDUP(bytes, ExecutableCodePageSize); void* p = nullptr; if (wasmCodeAllocations++ < MaxWasmCodeAllocations) p = AllocateExecutableMemory(bytes, ProtectionSetting::Writable); if (!p) { wasmCodeAllocations--; ReportOutOfMemory(cx); } return (uint8_t*)p; } AsmJSModule::AsmJSModule(ScriptSource* scriptSource, uint32_t srcStart, uint32_t srcBodyStart, bool strict, bool canUseSignalHandlers) : srcStart_(srcStart), srcBodyStart_(srcBodyStart), scriptSource_(scriptSource), globalArgumentName_(nullptr), importArgumentName_(nullptr), bufferArgumentName_(nullptr), code_(nullptr), interruptExit_(nullptr), prevLinked_(nullptr), nextLinked_(nullptr), dynamicallyLinked_(false), loadedFromCache_(false), profilingEnabled_(false), interrupted_(false) { mozilla::PodZero(&pod); pod.globalBytes_ = sInitialGlobalDataBytes; pod.minHeapLength_ = RoundUpToNextValidAsmJSHeapLength(0); pod.maxHeapLength_ = 0x80000000; pod.strict_ = strict; pod.canUseSignalHandlers_ = canUseSignalHandlers; // AsmJSCheckedImmediateRange should be defined to be at most the minimum // heap length so that offsets can be folded into bounds checks. MOZ_ASSERT(pod.minHeapLength_ - AsmJSCheckedImmediateRange <= pod.minHeapLength_); scriptSource_->incref(); } AsmJSModule::~AsmJSModule() { MOZ_ASSERT(!interrupted_); scriptSource_->decref(); if (code_) { for (unsigned i = 0; i < numExits(); i++) { AsmJSModule::ExitDatum& exitDatum = exit(i).datum(*this); if (!exitDatum.baselineScript) continue; jit::DependentAsmJSModuleExit exit(this, i); exitDatum.baselineScript->removeDependentAsmJSModule(exit); } uint32_t size = JS_ROUNDUP(pod.totalBytes_, ExecutableCodePageSize); MOZ_ASSERT(wasmCodeAllocations > 0); wasmCodeAllocations--; DeallocateExecutableMemory(code_, size); } if (prevLinked_) *prevLinked_ = nextLinked_; if (nextLinked_) nextLinked_->prevLinked_ = prevLinked_; } void AsmJSModule::trace(JSTracer* trc) { for (Global& global : globals_) global.trace(trc); for (Exit& exit : exits_) { if (exit.datum(*this).fun) TraceEdge(trc, &exit.datum(*this).fun, "asm.js imported function"); } for (ExportedFunction& exp : exports_) exp.trace(trc); for (Name& name : names_) TraceManuallyBarrieredEdge(trc, &name.name(), "asm.js module function name"); #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) for (ProfiledFunction& profiledFunction : profiledFunctions_) profiledFunction.trace(trc); #endif if (globalArgumentName_) TraceManuallyBarrieredEdge(trc, &globalArgumentName_, "asm.js global argument name"); if (importArgumentName_) TraceManuallyBarrieredEdge(trc, &importArgumentName_, "asm.js import argument name"); if (bufferArgumentName_) TraceManuallyBarrieredEdge(trc, &bufferArgumentName_, "asm.js buffer argument name"); if (maybeHeap_) TraceEdge(trc, &maybeHeap_, "asm.js heap"); } void AsmJSModule::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* asmJSModuleCode, size_t* asmJSModuleData) { *asmJSModuleCode += pod.totalBytes_; *asmJSModuleData += mallocSizeOf(this) + globals_.sizeOfExcludingThis(mallocSizeOf) + exits_.sizeOfExcludingThis(mallocSizeOf) + exports_.sizeOfExcludingThis(mallocSizeOf) + callSites_.sizeOfExcludingThis(mallocSizeOf) + codeRanges_.sizeOfExcludingThis(mallocSizeOf) + names_.sizeOfExcludingThis(mallocSizeOf) + heapAccesses_.sizeOfExcludingThis(mallocSizeOf) + #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) profiledFunctions_.sizeOfExcludingThis(mallocSizeOf) + #endif staticLinkData_.sizeOfExcludingThis(mallocSizeOf); } struct CallSiteRetAddrOffset { const CallSiteVector& callSites; explicit CallSiteRetAddrOffset(const CallSiteVector& callSites) : callSites(callSites) {} uint32_t operator[](size_t index) const { return callSites[index].returnAddressOffset(); } }; const CallSite* AsmJSModule::lookupCallSite(void* returnAddress) const { MOZ_ASSERT(isFinished()); uint32_t target = ((uint8_t*)returnAddress) - code_; size_t lowerBound = 0; size_t upperBound = callSites_.length(); size_t match; if (!BinarySearch(CallSiteRetAddrOffset(callSites_), lowerBound, upperBound, target, &match)) return nullptr; return &callSites_[match]; } namespace js { // Create an ordering on CodeRange and pc offsets suitable for BinarySearch. // Stick these in the same namespace as AsmJSModule so that argument-dependent // lookup will find it. bool operator==(size_t pcOffset, const AsmJSModule::CodeRange& rhs) { return pcOffset >= rhs.begin() && pcOffset < rhs.end(); } bool operator<=(const AsmJSModule::CodeRange& lhs, const AsmJSModule::CodeRange& rhs) { return lhs.begin() <= rhs.begin(); } bool operator<(size_t pcOffset, const AsmJSModule::CodeRange& rhs) { return pcOffset < rhs.begin(); } } // namespace js const AsmJSModule::CodeRange* AsmJSModule::lookupCodeRange(void* pc) const { MOZ_ASSERT(isFinished()); uint32_t target = ((uint8_t*)pc) - code_; size_t lowerBound = 0; size_t upperBound = codeRanges_.length(); size_t match; if (!BinarySearch(codeRanges_, lowerBound, upperBound, target, &match)) return nullptr; return &codeRanges_[match]; } struct HeapAccessOffset { const HeapAccessVector& accesses; explicit HeapAccessOffset(const HeapAccessVector& accesses) : accesses(accesses) {} uintptr_t operator[](size_t index) const { return accesses[index].insnOffset(); } }; const HeapAccess* AsmJSModule::lookupHeapAccess(void* pc) const { MOZ_ASSERT(isFinished()); MOZ_ASSERT(containsFunctionPC(pc)); uint32_t target = ((uint8_t*)pc) - code_; size_t lowerBound = 0; size_t upperBound = heapAccesses_.length(); size_t match; if (!BinarySearch(HeapAccessOffset(heapAccesses_), lowerBound, upperBound, target, &match)) return nullptr; return &heapAccesses_[match]; } bool AsmJSModule::finish(ExclusiveContext* cx, TokenStream& tokenStream, MacroAssembler& masm) { MOZ_ASSERT(!isFinished()); uint32_t endBeforeCurly = tokenStream.currentToken().pos.end; TokenPos pos; if (!tokenStream.peekTokenPos(&pos, TokenStream::Operand)) return false; uint32_t endAfterCurly = pos.end; MOZ_ASSERT(endBeforeCurly >= srcBodyStart_); MOZ_ASSERT(endAfterCurly >= srcBodyStart_); pod.srcLength_ = endBeforeCurly - srcStart_; pod.srcLengthWithRightBrace_ = endAfterCurly - srcStart_; // Start global data on a new page so JIT code may be given independent // protection flags. pod.codeBytes_ = AlignBytes(masm.bytesNeeded(), AsmJSPageSize); MOZ_ASSERT(pod.functionBytes_ <= pod.codeBytes_); // The entire region is allocated via mmap/VirtualAlloc which requires // units of pages. pod.totalBytes_ = AlignBytes(pod.codeBytes_ + pod.globalBytes_, AsmJSPageSize); MOZ_ASSERT(!code_); code_ = AllocateExecutableMemory(cx, pod.totalBytes_); if (!code_) return false; // Delay flushing until dynamic linking. The flush-inhibited range is set within // masm.executableCopy. AutoFlushICache afc("CheckModule", /* inhibit = */ true); // Copy the code from the MacroAssembler into its final resting place in the // AsmJSModule. MOZ_ASSERT(uintptr_t(code_) % AsmJSPageSize == 0); masm.executableCopy(code_); // c.f. JitCode::copyFrom MOZ_ASSERT(masm.jumpRelocationTableBytes() == 0); MOZ_ASSERT(masm.dataRelocationTableBytes() == 0); MOZ_ASSERT(masm.preBarrierTableBytes() == 0); MOZ_ASSERT(!masm.hasSelfReference()); // Heap-access metadata used for link-time patching and fault-handling. heapAccesses_ = masm.extractHeapAccesses(); // Call-site metadata used for stack unwinding. const CallSiteAndTargetVector& callSites = masm.callSites(); if (!callSites_.appendAll(callSites)) return false; // Absolute link metadata: absolute addresses that refer to some fixed // address in the address space. AbsoluteLinkArray& absoluteLinks = staticLinkData_.absoluteLinks; for (size_t i = 0; i < masm.numAsmJSAbsoluteLinks(); i++) { AsmJSAbsoluteLink src = masm.asmJSAbsoluteLink(i); if (!absoluteLinks[src.target].append(src.patchAt.offset())) return false; } // Relative link metadata: absolute addresses that refer to another point within // the asm.js module. // CodeLabels are used for switch cases and loads from floating-point / // SIMD values in the constant pool. for (size_t i = 0; i < masm.numCodeLabels(); i++) { CodeLabel cl = masm.codeLabel(i); RelativeLink link(RelativeLink::CodeLabel); link.patchAtOffset = masm.labelToPatchOffset(*cl.patchAt()); link.targetOffset = cl.target()->offset(); if (!staticLinkData_.relativeLinks.append(link)) return false; } #if defined(JS_CODEGEN_X86) // Global data accesses in x86 need to be patched with the absolute // address of the global. Globals are allocated sequentially after the // code section so we can just use an RelativeLink. for (size_t i = 0; i < masm.numAsmJSGlobalAccesses(); i++) { AsmJSGlobalAccess a = masm.asmJSGlobalAccess(i); RelativeLink link(RelativeLink::RawPointer); link.patchAtOffset = masm.labelToPatchOffset(a.patchAt); link.targetOffset = offsetOfGlobalData() + a.globalDataOffset; if (!staticLinkData_.relativeLinks.append(link)) return false; } #endif #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) // On MIPS we need to update all the long jumps because they contain an // absolute adress. The values are correctly patched for the current address // space, but not after serialization or profiling-mode toggling. for (size_t i = 0; i < masm.numLongJumps(); i++) { size_t off = masm.longJump(i); RelativeLink link(RelativeLink::InstructionImmediate); link.patchAtOffset = off; link.targetOffset = Assembler::ExtractInstructionImmediate(code_ + off) - uintptr_t(code_); if (!staticLinkData_.relativeLinks.append(link)) return false; } #endif #if defined(JS_CODEGEN_X64) // Global data accesses on x64 use rip-relative addressing and thus do // not need patching after deserialization. for (size_t i = 0; i < masm.numAsmJSGlobalAccesses(); i++) { AsmJSGlobalAccess a = masm.asmJSGlobalAccess(i); masm.patchAsmJSGlobalAccess(a.patchAt, code_, globalData(), a.globalDataOffset); } #endif return true; } void AsmJSModule::setAutoFlushICacheRange() { MOZ_ASSERT(isFinished()); AutoFlushICache::setRange(uintptr_t(code_), pod.codeBytes_); } static void AsmJSReportOverRecursed() { JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); ReportOverRecursed(cx); } static void OnDetached() { // See hasDetachedHeap comment in LinkAsmJS. JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_OUT_OF_MEMORY); } static void OnOutOfBounds() { JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_BAD_INDEX); } static void OnImpreciseConversion() { JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_SIMD_FAILED_CONVERSION); } static bool AsmJSHandleExecutionInterrupt() { AsmJSActivation* act = JSRuntime::innermostAsmJSActivation(); act->module().setInterrupted(true); bool ret = CheckForInterrupt(act->cx()); act->module().setInterrupted(false); return ret; } static int32_t CoerceInPlace_ToInt32(MutableHandleValue val) { JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); int32_t i32; if (!ToInt32(cx, val, &i32)) return false; val.set(Int32Value(i32)); return true; } static int32_t CoerceInPlace_ToNumber(MutableHandleValue val) { JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx(); double dbl; if (!ToNumber(cx, val, &dbl)) return false; val.set(DoubleValue(dbl)); return true; } static bool TryEnablingJit(JSContext* cx, AsmJSModule& module, HandleFunction fun, uint32_t exitIndex, int32_t argc, Value* argv) { if (!fun->hasScript()) return true; // Test if the function is JIT compiled. JSScript* script = fun->nonLazyScript(); if (!script->hasBaselineScript()) { MOZ_ASSERT(!script->hasIonScript()); return true; } // Don't enable jit entry when we have a pending ion builder. // Take the interpreter path which will link it and enable // the fast path on the next call. if (script->baselineScript()->hasPendingIonBuilder()) return true; // Currently we can't rectify arguments. Therefore disabling if argc is too low. if (fun->nargs() > size_t(argc)) return true; // Ensure the argument types are included in the argument TypeSets stored in // the TypeScript. This is necessary for Ion, because the FFI exit will // use the skip-arg-checks entry point. // // Note that the TypeScript is never discarded while the script has a // BaselineScript, so if those checks hold now they must hold at least until // the BaselineScript is discarded and when that happens the FFI exit is // patched back. if (!TypeScript::ThisTypes(script)->hasType(TypeSet::UndefinedType())) return true; for (uint32_t i = 0; i < fun->nargs(); i++) { StackTypeSet* typeset = TypeScript::ArgTypes(script, i); TypeSet::Type type = TypeSet::DoubleType(); if (!argv[i].isDouble()) type = TypeSet::PrimitiveType(argv[i].extractNonDoubleType()); if (!typeset->hasType(type)) return true; } // The exit may have become optimized while executing the FFI. AsmJSModule::Exit& exit = module.exit(exitIndex); if (exit.isOptimized(module)) return true; BaselineScript* baselineScript = script->baselineScript(); if (!baselineScript->addDependentAsmJSModule(cx, DependentAsmJSModuleExit(&module, exitIndex))) return false; exit.optimize(module, baselineScript); return true; } static bool InvokeFromAsmJS(AsmJSActivation* activation, int32_t exitIndex, int32_t argc, Value* argv, MutableHandleValue rval) { JSContext* cx = activation->cx(); AsmJSModule& module = activation->module(); RootedFunction fun(cx, module.exit(exitIndex).datum(module).fun); RootedValue fval(cx, ObjectValue(*fun)); if (!Invoke(cx, UndefinedValue(), fval, argc, argv, rval)) return false; return TryEnablingJit(cx, module, fun, exitIndex, argc, argv); } // Use an int32_t return type instead of bool since bool does not have a // specified width and the caller is assuming a word-sized return. static int32_t InvokeFromAsmJS_Ignore(int32_t exitIndex, int32_t argc, Value* argv) { AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation(); JSContext* cx = activation->cx(); RootedValue rval(cx); return InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval); } // Use an int32_t return type instead of bool since bool does not have a // specified width and the caller is assuming a word-sized return. static int32_t InvokeFromAsmJS_ToInt32(int32_t exitIndex, int32_t argc, Value* argv) { AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation(); JSContext* cx = activation->cx(); RootedValue rval(cx); if (!InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval)) return false; int32_t i32; if (!ToInt32(cx, rval, &i32)) return false; argv[0] = Int32Value(i32); return true; } // Use an int32_t return type instead of bool since bool does not have a // specified width and the caller is assuming a word-sized return. static int32_t InvokeFromAsmJS_ToNumber(int32_t exitIndex, int32_t argc, Value* argv) { AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation(); JSContext* cx = activation->cx(); RootedValue rval(cx); if (!InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval)) return false; double dbl; if (!ToNumber(cx, rval, &dbl)) return false; argv[0] = DoubleValue(dbl); return true; } #if defined(JS_CODEGEN_ARM) extern "C" { extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int); extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int); } #endif template static inline void* FuncCast(F* pf) { return JS_FUNC_TO_DATA_PTR(void*, pf); } static void* RedirectCall(void* fun, ABIFunctionType type) { #ifdef JS_SIMULATOR fun = Simulator::RedirectNativeFunction(fun, type); #endif return fun; } static void* AddressOf(SymbolicAddress imm, ExclusiveContext* cx) { switch (imm) { case SymbolicAddress::Runtime: return cx->runtimeAddressForJit(); case SymbolicAddress::RuntimeInterruptUint32: return cx->runtimeAddressOfInterruptUint32(); case SymbolicAddress::StackLimit: return cx->stackLimitAddressForJitCode(StackForUntrustedScript); case SymbolicAddress::ReportOverRecursed: return RedirectCall(FuncCast(AsmJSReportOverRecursed), Args_General0); case SymbolicAddress::OnDetached: return RedirectCall(FuncCast(OnDetached), Args_General0); case SymbolicAddress::OnOutOfBounds: return RedirectCall(FuncCast(OnOutOfBounds), Args_General0); case SymbolicAddress::OnImpreciseConversion: return RedirectCall(FuncCast(OnImpreciseConversion), Args_General0); case SymbolicAddress::HandleExecutionInterrupt: return RedirectCall(FuncCast(AsmJSHandleExecutionInterrupt), Args_General0); case SymbolicAddress::InvokeFromAsmJS_Ignore: return RedirectCall(FuncCast(InvokeFromAsmJS_Ignore), Args_General3); case SymbolicAddress::InvokeFromAsmJS_ToInt32: return RedirectCall(FuncCast(InvokeFromAsmJS_ToInt32), Args_General3); case SymbolicAddress::InvokeFromAsmJS_ToNumber: return RedirectCall(FuncCast(InvokeFromAsmJS_ToNumber), Args_General3); case SymbolicAddress::CoerceInPlace_ToInt32: return RedirectCall(FuncCast(CoerceInPlace_ToInt32), Args_General1); case SymbolicAddress::CoerceInPlace_ToNumber: return RedirectCall(FuncCast(CoerceInPlace_ToNumber), Args_General1); case SymbolicAddress::ToInt32: return RedirectCall(FuncCast(JS::ToInt32), Args_Int_Double); #if defined(JS_CODEGEN_ARM) case SymbolicAddress::aeabi_idivmod: return RedirectCall(FuncCast(__aeabi_idivmod), Args_General2); case SymbolicAddress::aeabi_uidivmod: return RedirectCall(FuncCast(__aeabi_uidivmod), Args_General2); case SymbolicAddress::AtomicCmpXchg: return RedirectCall(FuncCast(js::atomics_cmpxchg_asm_callout), Args_General4); case SymbolicAddress::AtomicXchg: return RedirectCall(FuncCast(js::atomics_xchg_asm_callout), Args_General3); case SymbolicAddress::AtomicFetchAdd: return RedirectCall(FuncCast(js::atomics_add_asm_callout), Args_General3); case SymbolicAddress::AtomicFetchSub: return RedirectCall(FuncCast(js::atomics_sub_asm_callout), Args_General3); case SymbolicAddress::AtomicFetchAnd: return RedirectCall(FuncCast(js::atomics_and_asm_callout), Args_General3); case SymbolicAddress::AtomicFetchOr: return RedirectCall(FuncCast(js::atomics_or_asm_callout), Args_General3); case SymbolicAddress::AtomicFetchXor: return RedirectCall(FuncCast(js::atomics_xor_asm_callout), Args_General3); #endif case SymbolicAddress::ModD: return RedirectCall(FuncCast(NumberMod), Args_Double_DoubleDouble); case SymbolicAddress::SinD: #ifdef _WIN64 // Workaround a VS 2013 sin issue, see math_sin_uncached. return RedirectCall(FuncCast(js::math_sin_uncached), Args_Double_Double); #else return RedirectCall(FuncCast(sin), Args_Double_Double); #endif case SymbolicAddress::CosD: return RedirectCall(FuncCast(cos), Args_Double_Double); case SymbolicAddress::TanD: return RedirectCall(FuncCast(tan), Args_Double_Double); case SymbolicAddress::ASinD: return RedirectCall(FuncCast(asin), Args_Double_Double); case SymbolicAddress::ACosD: return RedirectCall(FuncCast(acos), Args_Double_Double); case SymbolicAddress::ATanD: return RedirectCall(FuncCast(atan), Args_Double_Double); case SymbolicAddress::CeilD: return RedirectCall(FuncCast(ceil), Args_Double_Double); case SymbolicAddress::CeilF: return RedirectCall(FuncCast(ceilf), Args_Float32_Float32); case SymbolicAddress::FloorD: return RedirectCall(FuncCast(floor), Args_Double_Double); case SymbolicAddress::FloorF: return RedirectCall(FuncCast(floorf), Args_Float32_Float32); case SymbolicAddress::ExpD: return RedirectCall(FuncCast(exp), Args_Double_Double); case SymbolicAddress::LogD: return RedirectCall(FuncCast(log), Args_Double_Double); case SymbolicAddress::PowD: return RedirectCall(FuncCast(ecmaPow), Args_Double_DoubleDouble); case SymbolicAddress::ATan2D: return RedirectCall(FuncCast(ecmaAtan2), Args_Double_DoubleDouble); case SymbolicAddress::Limit: break; } MOZ_CRASH("Bad SymbolicAddress"); } void AsmJSModule::staticallyLink(ExclusiveContext* cx) { MOZ_ASSERT(isFinished()); // Process staticLinkData_ MOZ_ASSERT(staticLinkData_.pod.interruptExitOffset != 0); interruptExit_ = code_ + staticLinkData_.pod.interruptExitOffset; MOZ_ASSERT(staticLinkData_.pod.outOfBoundsExitOffset != 0); outOfBoundsExit_ = code_ + staticLinkData_.pod.outOfBoundsExitOffset; for (size_t i = 0; i < staticLinkData_.relativeLinks.length(); i++) { RelativeLink link = staticLinkData_.relativeLinks[i]; uint8_t* patchAt = code_ + link.patchAtOffset; uint8_t* target = code_ + link.targetOffset; // In the case of long-jumps on MIPS and possibly future cases, a // RelativeLink is used to patch a pointer to the function entry. If // profiling is enabled (by cloning a module with profiling enabled), // the target should be the profiling entry. if (profilingEnabled_) { const CodeRange* codeRange = lookupCodeRange(target); if (codeRange && codeRange->isFunction() && link.targetOffset == codeRange->entry()) target = code_ + codeRange->profilingEntry(); } if (link.isRawPointerPatch()) *(uint8_t**)(patchAt) = target; else Assembler::PatchInstructionImmediate(patchAt, PatchedImmPtr(target)); } for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) { const OffsetVector& offsets = staticLinkData_.absoluteLinks[imm]; for (size_t i = 0; i < offsets.length(); i++) { uint8_t* patchAt = code_ + offsets[i]; void* target = AddressOf(imm, cx); // Builtin calls are another case where, when profiling is enabled, // we must point to the profiling entry. Builtin builtin; if (profilingEnabled_ && ImmediateIsBuiltin(imm, &builtin)) { const CodeRange* codeRange = lookupCodeRange(patchAt); if (codeRange->isFunction()) target = code_ + staticLinkData_.pod.builtinThunkOffsets[builtin]; } Assembler::PatchDataWithValueCheck(CodeLocationLabel(patchAt), PatchedImmPtr(target), PatchedImmPtr((void*)-1)); } } // Initialize global data segment *(double*)(globalData() + NaN64GlobalDataOffset) = GenericNaN(); *(float*)(globalData() + NaN32GlobalDataOffset) = GenericNaN(); for (size_t tableIndex = 0; tableIndex < staticLinkData_.funcPtrTables.length(); tableIndex++) { FuncPtrTable& funcPtrTable = staticLinkData_.funcPtrTables[tableIndex]; const OffsetVector& offsets = funcPtrTable.elemOffsets(); auto array = reinterpret_cast(globalData() + funcPtrTable.globalDataOffset()); for (size_t elemIndex = 0; elemIndex < offsets.length(); elemIndex++) { uint8_t* target = code_ + offsets[elemIndex]; if (profilingEnabled_) target = code_ + lookupCodeRange(target)->profilingEntry(); array[elemIndex] = target; } } for (AsmJSModule::Exit& exit : exits_) exit.initDatum(*this); } void AsmJSModule::initHeap(Handle heap, JSContext* cx) { MOZ_ASSERT_IF(heap->is(), heap->as().isAsmJS()); MOZ_ASSERT(IsValidAsmJSHeapLength(heap->byteLength())); MOZ_ASSERT(dynamicallyLinked_); MOZ_ASSERT(!maybeHeap_); maybeHeap_ = heap; // heapDatum() may point to shared memory but that memory is only // accessed from maybeHeap(), which wraps it, and from // hasDetachedHeap(), which checks it for null. heapDatum() = heap->dataPointerEither().unwrap(/*safe - explained above*/); #if defined(JS_CODEGEN_X86) uint8_t* heapOffset = heap->dataPointerEither().unwrap(/*safe - used for value*/); uint32_t heapLength = heap->byteLength(); for (unsigned i = 0; i < heapAccesses_.length(); i++) { const HeapAccess& access = heapAccesses_[i]; // An access is out-of-bounds iff // ptr + offset + data-type-byte-size > heapLength // i.e. ptr > heapLength - data-type-byte-size - offset. // data-type-byte-size and offset are already included in the addend // so we just have to add the heap length here. if (access.hasLengthCheck()) X86Encoding::AddInt32(access.patchLengthAt(code_), heapLength); void* addr = access.patchHeapPtrImmAt(code_); uint32_t disp = reinterpret_cast(X86Encoding::GetPointer(addr)); MOZ_ASSERT(disp <= INT32_MAX); X86Encoding::SetPointer(addr, (void*)(heapOffset + disp)); } #elif defined(JS_CODEGEN_X64) // Even with signal handling being used for most bounds checks, there may be // atomic operations that depend on explicit checks. // // If we have any explicit bounds checks, we need to patch the heap length // checks at the right places. All accesses that have been recorded are the // only ones that need bound checks (see also // CodeGeneratorX64::visitAsmJS{Load,Store,CompareExchange,Exchange,AtomicBinop}Heap) uint32_t heapLength = heap->byteLength(); for (size_t i = 0; i < heapAccesses_.length(); i++) { const HeapAccess& access = heapAccesses_[i]; // See comment above for x86 codegen. if (access.hasLengthCheck()) X86Encoding::AddInt32(access.patchLengthAt(code_), heapLength); } #elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) uint32_t heapLength = heap->byteLength(); for (unsigned i = 0; i < heapAccesses_.length(); i++) { jit::Assembler::UpdateBoundsCheck(heapLength, (jit::Instruction*)(heapAccesses_[i].insnOffset() + code_)); } #endif } void AsmJSModule::restoreHeapToInitialState(ArrayBufferObjectMaybeShared* maybePrevBuffer) { #if defined(JS_CODEGEN_X86) if (maybePrevBuffer) { // Subtract out the base-pointer added by AsmJSModule::initHeap. uint8_t* ptrBase = maybePrevBuffer->dataPointerEither().unwrap(/*safe - used for value*/); uint32_t heapLength = maybePrevBuffer->byteLength(); for (unsigned i = 0; i < heapAccesses_.length(); i++) { const HeapAccess& access = heapAccesses_[i]; // Subtract the heap length back out, leaving the raw displacement in place. if (access.hasLengthCheck()) X86Encoding::AddInt32(access.patchLengthAt(code_), -heapLength); void* addr = access.patchHeapPtrImmAt(code_); uint8_t* ptr = reinterpret_cast(X86Encoding::GetPointer(addr)); MOZ_ASSERT(ptr >= ptrBase); X86Encoding::SetPointer(addr, (void*)(ptr - ptrBase)); } } #elif defined(JS_CODEGEN_X64) if (maybePrevBuffer) { uint32_t heapLength = maybePrevBuffer->byteLength(); for (unsigned i = 0; i < heapAccesses_.length(); i++) { const HeapAccess& access = heapAccesses_[i]; // See comment above for x86 codegen. if (access.hasLengthCheck()) X86Encoding::AddInt32(access.patchLengthAt(code_), -heapLength); } } #endif maybeHeap_ = nullptr; heapDatum() = nullptr; } void AsmJSModule::restoreToInitialState(ArrayBufferObjectMaybeShared* maybePrevBuffer, uint8_t* prevCode, ExclusiveContext* cx) { #ifdef DEBUG // Put the absolute links back to -1 so PatchDataWithValueCheck assertions // in staticallyLink are valid. for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) { void* callee = AddressOf(imm, cx); // If we are in profiling mode, calls to builtins will have been patched // by setProfilingEnabled to be calls to thunks. Builtin builtin; void* profilingCallee = profilingEnabled_ && ImmediateIsBuiltin(imm, &builtin) ? prevCode + staticLinkData_.pod.builtinThunkOffsets[builtin] : nullptr; const AsmJSModule::OffsetVector& offsets = staticLinkData_.absoluteLinks[imm]; for (size_t i = 0; i < offsets.length(); i++) { uint8_t* caller = code_ + offsets[i]; void* originalValue = profilingCallee && !lookupCodeRange(caller)->isThunk() ? profilingCallee : callee; Assembler::PatchDataWithValueCheck(CodeLocationLabel(caller), PatchedImmPtr((void*)-1), PatchedImmPtr(originalValue)); } } #endif restoreHeapToInitialState(maybePrevBuffer); } namespace { class MOZ_STACK_CLASS AutoMutateCode { AutoWritableJitCode awjc_; AutoFlushICache afc_; public: AutoMutateCode(JSContext* cx, AsmJSModule& module, const char* name) : awjc_(cx->runtime(), module.codeBase(), module.codeBytes()), afc_(name) { module.setAutoFlushICacheRange(); } }; } // namespace bool AsmJSModule::detachHeap(JSContext* cx) { MOZ_ASSERT(isDynamicallyLinked()); MOZ_ASSERT(maybeHeap_); // Content JS should not be able to run (and detach heap) from within an // interrupt callback, but in case it does, fail. Otherwise, the heap can // change at an arbitrary instruction and break the assumption below. if (interrupted_) { JS_ReportError(cx, "attempt to detach from inside interrupt handler"); return false; } // Even if this->active(), to reach here, the activation must have called // out via an FFI stub. FFI stubs check if heapDatum() is null on reentry // and throw an exception if so. MOZ_ASSERT_IF(active(), activation()->exitReason().kind() == ExitReason::Jit || activation()->exitReason().kind() == ExitReason::Slow); AutoMutateCode amc(cx, *this, "AsmJSModule::detachHeap"); restoreHeapToInitialState(maybeHeap_); MOZ_ASSERT(hasDetachedHeap()); return true; } bool js::OnDetachAsmJSArrayBuffer(JSContext* cx, Handle buffer) { for (AsmJSModule* m = cx->runtime()->linkedAsmJSModules; m; m = m->nextLinked()) { if (buffer == m->maybeHeapBufferObject() && !m->detachHeap(cx)) return false; } return true; } static void AsmJSModuleObject_finalize(FreeOp* fop, JSObject* obj) { fop->delete_(&obj->as().module()); } static void AsmJSModuleObject_trace(JSTracer* trc, JSObject* obj) { obj->as().module().trace(trc); } const Class AsmJSModuleObject::class_ = { "AsmJSModuleObject", JSCLASS_IS_ANONYMOUS | JSCLASS_DELAY_METADATA_CALLBACK | JSCLASS_HAS_RESERVED_SLOTS(AsmJSModuleObject::RESERVED_SLOTS), nullptr, /* addProperty */ nullptr, /* delProperty */ nullptr, /* getProperty */ nullptr, /* setProperty */ nullptr, /* enumerate */ nullptr, /* resolve */ nullptr, /* mayResolve */ AsmJSModuleObject_finalize, nullptr, /* call */ nullptr, /* hasInstance */ nullptr, /* construct */ AsmJSModuleObject_trace }; AsmJSModuleObject* AsmJSModuleObject::create(ExclusiveContext* cx, ScopedJSDeletePtr* module) { AutoSetNewObjectMetadata metadata(cx); JSObject* obj = NewObjectWithGivenProto(cx, &AsmJSModuleObject::class_, nullptr); if (!obj) return nullptr; AsmJSModuleObject* nobj = &obj->as(); nobj->setReservedSlot(MODULE_SLOT, PrivateValue(module->forget())); return nobj; } AsmJSModule& AsmJSModuleObject::module() const { MOZ_ASSERT(is()); return *(AsmJSModule*)getReservedSlot(MODULE_SLOT).toPrivate(); } static inline uint8_t* WriteBytes(uint8_t* dst, const void* src, size_t nbytes) { memcpy(dst, src, nbytes); return dst + nbytes; } static inline const uint8_t* ReadBytes(const uint8_t* src, void* dst, size_t nbytes) { memcpy(dst, src, nbytes); return src + nbytes; } template static inline uint8_t* WriteScalar(uint8_t* dst, T t) { memcpy(dst, &t, sizeof(t)); return dst + sizeof(t); } template static inline const uint8_t* ReadScalar(const uint8_t* src, T* dst) { memcpy(dst, src, sizeof(*dst)); return src + sizeof(*dst); } static size_t SerializedNameSize(PropertyName* name) { size_t s = sizeof(uint32_t); if (name) s += name->length() * (name->hasLatin1Chars() ? sizeof(Latin1Char) : sizeof(char16_t)); return s; } size_t AsmJSModule::Name::serializedSize() const { return SerializedNameSize(name_); } static uint8_t* SerializeName(uint8_t* cursor, PropertyName* name) { MOZ_ASSERT_IF(name, !name->empty()); if (name) { static_assert(JSString::MAX_LENGTH <= INT32_MAX, "String length must fit in 31 bits"); uint32_t length = name->length(); uint32_t lengthAndEncoding = (length << 1) | uint32_t(name->hasLatin1Chars()); cursor = WriteScalar(cursor, lengthAndEncoding); JS::AutoCheckCannotGC nogc; if (name->hasLatin1Chars()) cursor = WriteBytes(cursor, name->latin1Chars(nogc), length * sizeof(Latin1Char)); else cursor = WriteBytes(cursor, name->twoByteChars(nogc), length * sizeof(char16_t)); } else { cursor = WriteScalar(cursor, 0); } return cursor; } uint8_t* AsmJSModule::Name::serialize(uint8_t* cursor) const { return SerializeName(cursor, name_); } template static const uint8_t* DeserializeChars(ExclusiveContext* cx, const uint8_t* cursor, size_t length, PropertyName** name) { Vector tmp(cx); CharT* src; if ((size_t(cursor) & (sizeof(CharT) - 1)) != 0) { // Align 'src' for AtomizeChars. if (!tmp.resize(length)) return nullptr; memcpy(tmp.begin(), cursor, length * sizeof(CharT)); src = tmp.begin(); } else { src = (CharT*)cursor; } JSAtom* atom = AtomizeChars(cx, src, length); if (!atom) return nullptr; *name = atom->asPropertyName(); return cursor + length * sizeof(CharT); } static const uint8_t* DeserializeName(ExclusiveContext* cx, const uint8_t* cursor, PropertyName** name) { uint32_t lengthAndEncoding; cursor = ReadScalar(cursor, &lengthAndEncoding); uint32_t length = lengthAndEncoding >> 1; if (length == 0) { *name = nullptr; return cursor; } bool latin1 = lengthAndEncoding & 0x1; return latin1 ? DeserializeChars(cx, cursor, length, name) : DeserializeChars(cx, cursor, length, name); } const uint8_t* AsmJSModule::Name::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { return DeserializeName(cx, cursor, &name_); } bool AsmJSModule::Name::clone(ExclusiveContext* cx, Name* out) const { out->name_ = name_; return true; } template size_t SerializedVectorSize(const mozilla::Vector& vec) { size_t size = sizeof(uint32_t); for (size_t i = 0; i < vec.length(); i++) size += vec[i].serializedSize(); return size; } template uint8_t* SerializeVector(uint8_t* cursor, const mozilla::Vector& vec) { cursor = WriteScalar(cursor, vec.length()); for (size_t i = 0; i < vec.length(); i++) cursor = vec[i].serialize(cursor); return cursor; } template const uint8_t* DeserializeVector(ExclusiveContext* cx, const uint8_t* cursor, mozilla::Vector* vec) { uint32_t length; cursor = ReadScalar(cursor, &length); if (!vec->resize(length)) return nullptr; for (size_t i = 0; i < vec->length(); i++) { if (!(cursor = (*vec)[i].deserialize(cx, cursor))) return nullptr; } return cursor; } template bool CloneVector(ExclusiveContext* cx, const mozilla::Vector& in, mozilla::Vector* out) { if (!out->resize(in.length())) return false; for (size_t i = 0; i < in.length(); i++) { if (!in[i].clone(cx, &(*out)[i])) return false; } return true; } template size_t SerializedPodVectorSize(const mozilla::Vector& vec) { return sizeof(uint32_t) + vec.length() * sizeof(T); } template uint8_t* SerializePodVector(uint8_t* cursor, const mozilla::Vector& vec) { cursor = WriteScalar(cursor, vec.length()); cursor = WriteBytes(cursor, vec.begin(), vec.length() * sizeof(T)); return cursor; } template const uint8_t* DeserializePodVector(ExclusiveContext* cx, const uint8_t* cursor, mozilla::Vector* vec) { uint32_t length; cursor = ReadScalar(cursor, &length); if (!vec->resize(length)) return nullptr; cursor = ReadBytes(cursor, vec->begin(), length * sizeof(T)); return cursor; } template bool ClonePodVector(ExclusiveContext* cx, const mozilla::Vector& in, mozilla::Vector* out) { if (!out->resize(in.length())) return false; PodCopy(out->begin(), in.begin(), in.length()); return true; } size_t SerializedSigSize(const MallocSig& sig) { return sizeof(ExprType) + SerializedPodVectorSize(sig.args()); } uint8_t* SerializeSig(uint8_t* cursor, const MallocSig& sig) { cursor = WriteScalar(cursor, sig.ret()); cursor = SerializePodVector(cursor, sig.args()); return cursor; } const uint8_t* DeserializeSig(ExclusiveContext* cx, const uint8_t* cursor, MallocSig* sig) { ExprType ret; cursor = ReadScalar(cursor, &ret); MallocSig::ArgVector args; cursor = DeserializePodVector(cx, cursor, &args); if (!cursor) return nullptr; sig->init(Move(args), ret); return cursor; } bool CloneSig(ExclusiveContext* cx, const MallocSig& sig, MallocSig* out) { MallocSig::ArgVector args; if (!ClonePodVector(cx, sig.args(), &args)) return false; out->init(Move(args), sig.ret()); return true; } uint8_t* AsmJSModule::Global::serialize(uint8_t* cursor) const { cursor = WriteBytes(cursor, &pod, sizeof(pod)); cursor = SerializeName(cursor, name_); return cursor; } size_t AsmJSModule::Global::serializedSize() const { return sizeof(pod) + SerializedNameSize(name_); } const uint8_t* AsmJSModule::Global::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = ReadBytes(cursor, &pod, sizeof(pod))) && (cursor = DeserializeName(cx, cursor, &name_)); return cursor; } bool AsmJSModule::Global::clone(ExclusiveContext* cx, Global* out) const { *out = *this; return true; } uint8_t* AsmJSModule::Exit::serialize(uint8_t* cursor) const { cursor = SerializeSig(cursor, sig_); cursor = WriteBytes(cursor, &pod, sizeof(pod)); return cursor; } size_t AsmJSModule::Exit::serializedSize() const { return SerializedSigSize(sig_) + sizeof(pod); } const uint8_t* AsmJSModule::Exit::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = DeserializeSig(cx, cursor, &sig_)) && (cursor = ReadBytes(cursor, &pod, sizeof(pod))); return cursor; } bool AsmJSModule::Exit::clone(ExclusiveContext* cx, Exit* out) const { out->pod = pod; return CloneSig(cx, sig_, &out->sig_); } uint8_t* AsmJSModule::ExportedFunction::serialize(uint8_t* cursor) const { cursor = SerializeName(cursor, name_); cursor = SerializeName(cursor, maybeFieldName_); cursor = SerializeSig(cursor, sig_); cursor = WriteBytes(cursor, &pod, sizeof(pod)); return cursor; } size_t AsmJSModule::ExportedFunction::serializedSize() const { return SerializedNameSize(name_) + SerializedNameSize(maybeFieldName_) + SerializedSigSize(sig_) + sizeof(pod); } const uint8_t* AsmJSModule::ExportedFunction::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = DeserializeName(cx, cursor, &name_)) && (cursor = DeserializeName(cx, cursor, &maybeFieldName_)) && (cursor = DeserializeSig(cx, cursor, &sig_)) && (cursor = ReadBytes(cursor, &pod, sizeof(pod))); return cursor; } bool AsmJSModule::ExportedFunction::clone(ExclusiveContext* cx, ExportedFunction* out) const { out->name_ = name_; out->maybeFieldName_ = maybeFieldName_; out->pod = pod; return CloneSig(cx, sig_, &out->sig_); } AsmJSModule::CodeRange::CodeRange(uint32_t lineNumber, AsmJSFunctionOffsets offsets) : nameIndex_(UINT32_MAX), lineNumber_(lineNumber) { PodZero(&u); // zero padding for Valgrind u.kind_ = Function; MOZ_ASSERT(offsets.nonProfilingEntry - offsets.begin <= UINT8_MAX); begin_ = offsets.begin; u.func.beginToEntry_ = offsets.nonProfilingEntry - begin_; MOZ_ASSERT(offsets.nonProfilingEntry < offsets.profilingReturn); MOZ_ASSERT(offsets.profilingReturn - offsets.profilingJump <= UINT8_MAX); MOZ_ASSERT(offsets.profilingReturn - offsets.profilingEpilogue <= UINT8_MAX); profilingReturn_ = offsets.profilingReturn; u.func.profilingJumpToProfilingReturn_ = profilingReturn_ - offsets.profilingJump; u.func.profilingEpilogueToProfilingReturn_ = profilingReturn_ - offsets.profilingEpilogue; MOZ_ASSERT(offsets.nonProfilingEntry < offsets.end); end_ = offsets.end; } AsmJSModule::CodeRange::CodeRange(Kind kind, AsmJSOffsets offsets) : nameIndex_(0), lineNumber_(0), begin_(offsets.begin), profilingReturn_(0), end_(offsets.end) { PodZero(&u); // zero padding for Valgrind u.kind_ = kind; MOZ_ASSERT(begin_ <= end_); MOZ_ASSERT(u.kind_ == Entry || u.kind_ == Inline); } AsmJSModule::CodeRange::CodeRange(Kind kind, AsmJSProfilingOffsets offsets) : nameIndex_(0), lineNumber_(0), begin_(offsets.begin), profilingReturn_(offsets.profilingReturn), end_(offsets.end) { PodZero(&u); // zero padding for Valgrind u.kind_ = kind; MOZ_ASSERT(begin_ < profilingReturn_); MOZ_ASSERT(profilingReturn_ < end_); MOZ_ASSERT(u.kind_ == JitFFI || u.kind_ == SlowFFI || u.kind_ == Interrupt); } AsmJSModule::CodeRange::CodeRange(Builtin builtin, AsmJSProfilingOffsets offsets) : nameIndex_(0), lineNumber_(0), begin_(offsets.begin), profilingReturn_(offsets.profilingReturn), end_(offsets.end) { PodZero(&u); // zero padding for Valgrind u.kind_ = Thunk; u.thunk.target_ = uint16_t(builtin); MOZ_ASSERT(begin_ < profilingReturn_); MOZ_ASSERT(profilingReturn_ < end_); } #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) size_t AsmJSModule::ProfiledFunction::serializedSize() const { return SerializedNameSize(name) + sizeof(pod); } uint8_t* AsmJSModule::ProfiledFunction::serialize(uint8_t* cursor) const { cursor = SerializeName(cursor, name); cursor = WriteBytes(cursor, &pod, sizeof(pod)); return cursor; } const uint8_t* AsmJSModule::ProfiledFunction::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = DeserializeName(cx, cursor, &name)) && (cursor = ReadBytes(cursor, &pod, sizeof(pod))); return cursor; } #endif size_t AsmJSModule::AbsoluteLinkArray::serializedSize() const { size_t size = 0; for (const OffsetVector& offsets : *this) size += SerializedPodVectorSize(offsets); return size; } uint8_t* AsmJSModule::AbsoluteLinkArray::serialize(uint8_t* cursor) const { for (const OffsetVector& offsets : *this) cursor = SerializePodVector(cursor, offsets); return cursor; } const uint8_t* AsmJSModule::AbsoluteLinkArray::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { for (OffsetVector& offsets : *this) { cursor = DeserializePodVector(cx, cursor, &offsets); if (!cursor) return nullptr; } return cursor; } bool AsmJSModule::AbsoluteLinkArray::clone(ExclusiveContext* cx, AbsoluteLinkArray* out) const { for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) { if (!ClonePodVector(cx, (*this)[imm], &(*out)[imm])) return false; } return true; } size_t AsmJSModule::AbsoluteLinkArray::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { size_t size = 0; for (const OffsetVector& offsets : *this) size += offsets.sizeOfExcludingThis(mallocSizeOf); return size; } size_t AsmJSModule::FuncPtrTable::serializedSize() const { return sizeof(pod) + SerializedPodVectorSize(elemOffsets_); } uint8_t* AsmJSModule::FuncPtrTable::serialize(uint8_t* cursor) const { cursor = WriteBytes(cursor, &pod, sizeof(pod)); cursor = SerializePodVector(cursor, elemOffsets_); return cursor; } const uint8_t* AsmJSModule::FuncPtrTable::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = ReadBytes(cursor, &pod, sizeof(pod))) && (cursor = DeserializePodVector(cx, cursor, &elemOffsets_)); return cursor; } bool AsmJSModule::FuncPtrTable::clone(ExclusiveContext* cx, FuncPtrTable* out) const { out->pod = pod; return ClonePodVector(cx, elemOffsets_, &out->elemOffsets_); } size_t AsmJSModule::FuncPtrTable::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { return elemOffsets_.sizeOfExcludingThis(mallocSizeOf); } size_t AsmJSModule::StaticLinkData::serializedSize() const { return sizeof(pod) + SerializedPodVectorSize(relativeLinks) + absoluteLinks.serializedSize() + SerializedVectorSize(funcPtrTables); } uint8_t* AsmJSModule::StaticLinkData::serialize(uint8_t* cursor) const { cursor = WriteBytes(cursor, &pod, sizeof(pod)); cursor = SerializePodVector(cursor, relativeLinks); cursor = absoluteLinks.serialize(cursor); cursor = SerializeVector(cursor, funcPtrTables); return cursor; } const uint8_t* AsmJSModule::StaticLinkData::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = ReadBytes(cursor, &pod, sizeof(pod))) && (cursor = DeserializePodVector(cx, cursor, &relativeLinks)) && (cursor = absoluteLinks.deserialize(cx, cursor)) && (cursor = DeserializeVector(cx, cursor, &funcPtrTables)); return cursor; } bool AsmJSModule::StaticLinkData::clone(ExclusiveContext* cx, StaticLinkData* out) const { out->pod = pod; return ClonePodVector(cx, relativeLinks, &out->relativeLinks) && absoluteLinks.clone(cx, &out->absoluteLinks) && CloneVector(cx, funcPtrTables, &out->funcPtrTables); } size_t AsmJSModule::StaticLinkData::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { size_t size = relativeLinks.sizeOfExcludingThis(mallocSizeOf) + absoluteLinks.sizeOfExcludingThis(mallocSizeOf) + funcPtrTables.sizeOfExcludingThis(mallocSizeOf); for (const FuncPtrTable& table : funcPtrTables) size += table.sizeOfExcludingThis(mallocSizeOf); return size; } size_t AsmJSModule::serializedSize() const { return sizeof(pod) + pod.codeBytes_ + SerializedNameSize(globalArgumentName_) + SerializedNameSize(importArgumentName_) + SerializedNameSize(bufferArgumentName_) + SerializedVectorSize(globals_) + SerializedVectorSize(exits_) + SerializedVectorSize(exports_) + SerializedPodVectorSize(callSites_) + SerializedPodVectorSize(codeRanges_) + SerializedVectorSize(names_) + SerializedPodVectorSize(heapAccesses_) + #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) SerializedVectorSize(profiledFunctions_) + #endif staticLinkData_.serializedSize(); } uint8_t* AsmJSModule::serialize(uint8_t* cursor) const { MOZ_ASSERT(!dynamicallyLinked_); MOZ_ASSERT(!loadedFromCache_); MOZ_ASSERT(!profilingEnabled_); MOZ_ASSERT(!interrupted_); cursor = WriteBytes(cursor, &pod, sizeof(pod)); cursor = WriteBytes(cursor, code_, pod.codeBytes_); cursor = SerializeName(cursor, globalArgumentName_); cursor = SerializeName(cursor, importArgumentName_); cursor = SerializeName(cursor, bufferArgumentName_); cursor = SerializeVector(cursor, globals_); cursor = SerializeVector(cursor, exits_); cursor = SerializeVector(cursor, exports_); cursor = SerializePodVector(cursor, callSites_); cursor = SerializePodVector(cursor, codeRanges_); cursor = SerializeVector(cursor, names_); cursor = SerializePodVector(cursor, heapAccesses_); #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) cursor = SerializeVector(cursor, profiledFunctions_); #endif cursor = staticLinkData_.serialize(cursor); return cursor; } const uint8_t* AsmJSModule::deserialize(ExclusiveContext* cx, const uint8_t* cursor) { // To avoid GC-during-deserialization corner cases, prevent atoms from // being collected. AutoKeepAtoms aka(cx->perThreadData); (cursor = ReadBytes(cursor, &pod, sizeof(pod))) && (code_ = AllocateExecutableMemory(cx, pod.totalBytes_)) && (cursor = ReadBytes(cursor, code_, pod.codeBytes_)) && (cursor = DeserializeName(cx, cursor, &globalArgumentName_)) && (cursor = DeserializeName(cx, cursor, &importArgumentName_)) && (cursor = DeserializeName(cx, cursor, &bufferArgumentName_)) && (cursor = DeserializeVector(cx, cursor, &globals_)) && (cursor = DeserializeVector(cx, cursor, &exits_)) && (cursor = DeserializeVector(cx, cursor, &exports_)) && (cursor = DeserializePodVector(cx, cursor, &callSites_)) && (cursor = DeserializePodVector(cx, cursor, &codeRanges_)) && (cursor = DeserializeVector(cx, cursor, &names_)) && (cursor = DeserializePodVector(cx, cursor, &heapAccesses_)) && #if defined(MOZ_VTUNE) || defined(JS_ION_PERF) (cursor = DeserializeVector(cx, cursor, &profiledFunctions_)) && #endif (cursor = staticLinkData_.deserialize(cx, cursor)); loadedFromCache_ = true; return cursor; } bool AsmJSModule::clone(JSContext* cx, ScopedJSDeletePtr* moduleOut) const { *moduleOut = cx->new_(scriptSource_, srcStart_, srcBodyStart_, pod.strict_, pod.canUseSignalHandlers_); if (!*moduleOut) return false; AsmJSModule& out = **moduleOut; // Mirror the order of serialize/deserialize in cloning: out.pod = pod; out.code_ = AllocateExecutableMemory(cx, pod.totalBytes_); if (!out.code_) return false; memcpy(out.code_, code_, pod.codeBytes_); out.globalArgumentName_ = globalArgumentName_; out.importArgumentName_ = importArgumentName_; out.bufferArgumentName_ = bufferArgumentName_; if (!CloneVector(cx, globals_, &out.globals_) || !CloneVector(cx, exits_, &out.exits_) || !CloneVector(cx, exports_, &out.exports_) || !ClonePodVector(cx, callSites_, &out.callSites_) || !ClonePodVector(cx, codeRanges_, &out.codeRanges_) || !CloneVector(cx, names_, &out.names_) || !ClonePodVector(cx, heapAccesses_, &out.heapAccesses_) || !staticLinkData_.clone(cx, &out.staticLinkData_)) { return false; } out.loadedFromCache_ = loadedFromCache_; out.profilingEnabled_ = profilingEnabled_; if (profilingEnabled_) { if (!out.profilingLabels_.resize(profilingLabels_.length())) return false; for (size_t i = 0; i < profilingLabels_.length(); i++) { out.profilingLabels_[i] = DuplicateString(cx, profilingLabels_[i].get()); if (!out.profilingLabels_[i]) return false; } } // Delay flushing until dynamic linking. AutoFlushICache afc("AsmJSModule::clone", /* inhibit = */ true); out.setAutoFlushICacheRange(); out.restoreToInitialState(maybeHeap_, code_, cx); out.staticallyLink(cx); return true; } bool AsmJSModule::changeHeap(Handle newHeap, JSContext* cx) { MOZ_ASSERT(hasArrayView()); // Content JS should not be able to run (and change heap) from within an // interrupt callback, but in case it does, fail to change heap. Otherwise, // the heap can change at every single instruction which would prevent // future optimizations like heap-base hoisting. if (interrupted_) return false; AutoMutateCode amc(cx, *this, "AsmJSModule::changeHeap"); restoreHeapToInitialState(maybeHeap_); initHeap(newHeap, cx); return true; } size_t AsmJSModule::heapLength() const { MOZ_ASSERT(isDynamicallyLinked()); return maybeHeap_ ? maybeHeap_->byteLength() : 0; } void AsmJSModule::setProfilingEnabled(bool enabled, JSContext* cx) { MOZ_ASSERT(isDynamicallyLinked()); if (profilingEnabled_ == enabled) return; // When enabled, generate profiling labels for every name in names_ that is // the name of some Function CodeRange. This involves malloc() so do it now // since, once we start sampling, we'll be in a signal-handing context where // we cannot malloc. if (enabled) { profilingLabels_.resize(names_.length()); const char* filename = scriptSource_->filename(); JS::AutoCheckCannotGC nogc; for (size_t i = 0; i < codeRanges_.length(); i++) { CodeRange& cr = codeRanges_[i]; if (!cr.isFunction()) continue; unsigned lineno = cr.functionLineNumber(); PropertyName* name = names_[cr.functionNameIndex()].name(); profilingLabels_[cr.functionNameIndex()].reset( name->hasLatin1Chars() ? JS_smprintf("%s (%s:%u)", name->latin1Chars(nogc), filename, lineno) : JS_smprintf("%hs (%s:%u)", name->twoByteChars(nogc), filename, lineno)); } } else { profilingLabels_.clear(); } AutoMutateCode amc(cx, *this, "AsmJSModule::setProfilingEnabled"); // Patch all internal (asm.js->asm.js) callsites to call the profiling // prologues: for (size_t i = 0; i < callSites_.length(); i++) { CallSite& cs = callSites_[i]; if (cs.kind() != CallSite::Relative) continue; uint8_t* callerRetAddr = code_ + cs.returnAddressOffset(); #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) void* callee = X86Encoding::GetRel32Target(callerRetAddr); #elif defined(JS_CODEGEN_ARM) uint8_t* caller = callerRetAddr - 4; Instruction* callerInsn = reinterpret_cast(caller); BOffImm calleeOffset; callerInsn->as()->extractImm(&calleeOffset); void* callee = calleeOffset.getDest(callerInsn); #elif defined(JS_CODEGEN_ARM64) MOZ_CRASH(); void* callee = nullptr; (void)callerRetAddr; #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) uint8_t* instr = callerRetAddr - Assembler::PatchWrite_NearCallSize(); void* callee = (void*)Assembler::ExtractInstructionImmediate(instr); #elif defined(JS_CODEGEN_NONE) MOZ_CRASH(); void* callee = nullptr; #else # error "Missing architecture" #endif const CodeRange* codeRange = lookupCodeRange(callee); if (codeRange->kind() != CodeRange::Function) continue; uint8_t* profilingEntry = code_ + codeRange->profilingEntry(); uint8_t* entry = code_ + codeRange->entry(); MOZ_ASSERT_IF(profilingEnabled_, callee == profilingEntry); MOZ_ASSERT_IF(!profilingEnabled_, callee == entry); uint8_t* newCallee = enabled ? profilingEntry : entry; #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) X86Encoding::SetRel32(callerRetAddr, newCallee); #elif defined(JS_CODEGEN_ARM) new (caller) InstBLImm(BOffImm(newCallee - caller), Assembler::Always); #elif defined(JS_CODEGEN_ARM64) (void)newCallee; MOZ_CRASH(); #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) Assembler::PatchInstructionImmediate(instr, PatchedImmPtr(newCallee)); #elif defined(JS_CODEGEN_NONE) MOZ_CRASH(); #else # error "Missing architecture" #endif } // Update all the addresses in the function-pointer tables to point to the // profiling prologues: for (FuncPtrTable& funcPtrTable : staticLinkData_.funcPtrTables) { auto array = reinterpret_cast(globalData() + funcPtrTable.globalDataOffset()); for (size_t i = 0; i < funcPtrTable.elemOffsets().length(); i++) { void* callee = array[i]; const CodeRange* codeRange = lookupCodeRange(callee); void* profilingEntry = code_ + codeRange->profilingEntry(); void* entry = code_ + codeRange->entry(); MOZ_ASSERT_IF(profilingEnabled_, callee == profilingEntry); MOZ_ASSERT_IF(!profilingEnabled_, callee == entry); if (enabled) array[i] = profilingEntry; else array[i] = entry; } } // Replace all the nops in all the epilogues of asm.js functions with jumps // to the profiling epilogues. for (size_t i = 0; i < codeRanges_.length(); i++) { CodeRange& cr = codeRanges_[i]; if (!cr.isFunction()) continue; uint8_t* jump = code_ + cr.profilingJump(); uint8_t* profilingEpilogue = code_ + cr.profilingEpilogue(); #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) // An unconditional jump with a 1 byte offset immediate has the opcode // 0x90. The offset is relative to the address of the instruction after // the jump. 0x66 0x90 is the canonical two-byte nop. ptrdiff_t jumpImmediate = profilingEpilogue - jump - 2; MOZ_ASSERT(jumpImmediate > 0 && jumpImmediate <= 127); if (enabled) { MOZ_ASSERT(jump[0] == 0x66); MOZ_ASSERT(jump[1] == 0x90); jump[0] = 0xeb; jump[1] = jumpImmediate; } else { MOZ_ASSERT(jump[0] == 0xeb); MOZ_ASSERT(jump[1] == jumpImmediate); jump[0] = 0x66; jump[1] = 0x90; } #elif defined(JS_CODEGEN_ARM) if (enabled) { MOZ_ASSERT(reinterpret_cast(jump)->is()); new (jump) InstBImm(BOffImm(profilingEpilogue - jump), Assembler::Always); } else { MOZ_ASSERT(reinterpret_cast(jump)->is()); new (jump) InstNOP(); } #elif defined(JS_CODEGEN_ARM64) (void)jump; (void)profilingEpilogue; MOZ_CRASH(); #elif defined(JS_CODEGEN_MIPS32) Instruction* instr = (Instruction*)jump; if (enabled) { Assembler::WriteLuiOriInstructions(instr, instr->next(), ScratchRegister, (uint32_t)profilingEpilogue); instr[2] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr); } else { instr[0].makeNop(); instr[1].makeNop(); instr[2].makeNop(); } #elif defined(JS_CODEGEN_MIPS64) Instruction* instr = (Instruction*)jump; if (enabled) { Assembler::WriteLoad64Instructions(instr, ScratchRegister, (uint64_t)profilingEpilogue); instr[4] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr); } else { instr[0].makeNop(); instr[1].makeNop(); instr[2].makeNop(); instr[3].makeNop(); instr[4].makeNop(); } #elif defined(JS_CODEGEN_NONE) MOZ_CRASH(); #else # error "Missing architecture" #endif } // Replace all calls to builtins with calls to profiling thunks that push a // frame pointer. Since exit unwinding always starts at the caller of fp, // this avoids losing the innermost asm.js function. for (auto builtin : MakeEnumeratedRange(Builtin::Limit)) { auto imm = BuiltinToImmediate(builtin); const OffsetVector& offsets = staticLinkData_.absoluteLinks[imm]; void* from = AddressOf(imm, nullptr); void* to = code_ + staticLinkData_.pod.builtinThunkOffsets[builtin]; if (!enabled) Swap(from, to); for (size_t j = 0; j < offsets.length(); j++) { uint8_t* caller = code_ + offsets[j]; const AsmJSModule::CodeRange* codeRange = lookupCodeRange(caller); if (codeRange->isThunk()) continue; MOZ_ASSERT(codeRange->isFunction()); Assembler::PatchDataWithValueCheck(CodeLocationLabel(caller), PatchedImmPtr(to), PatchedImmPtr(from)); } } profilingEnabled_ = enabled; } static bool GetCPUID(uint32_t* cpuId) { enum Arch { X86 = 0x1, X64 = 0x2, ARM = 0x3, MIPS = 0x4, MIPS64 = 0x5, ARCH_BITS = 3 }; #if defined(JS_CODEGEN_X86) MOZ_ASSERT(uint32_t(CPUInfo::GetSSEVersion()) <= (UINT32_MAX >> ARCH_BITS)); *cpuId = X86 | (uint32_t(CPUInfo::GetSSEVersion()) << ARCH_BITS); return true; #elif defined(JS_CODEGEN_X64) MOZ_ASSERT(uint32_t(CPUInfo::GetSSEVersion()) <= (UINT32_MAX >> ARCH_BITS)); *cpuId = X64 | (uint32_t(CPUInfo::GetSSEVersion()) << ARCH_BITS); return true; #elif defined(JS_CODEGEN_ARM) MOZ_ASSERT(GetARMFlags() <= (UINT32_MAX >> ARCH_BITS)); *cpuId = ARM | (GetARMFlags() << ARCH_BITS); return true; #elif defined(JS_CODEGEN_MIPS32) MOZ_ASSERT(GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS)); *cpuId = MIPS | (GetMIPSFlags() << ARCH_BITS); return true; #elif defined(JS_CODEGEN_MIPS64) MOZ_ASSERT(GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS)); *cpuId = MIPS64 | (GetMIPSFlags() << ARCH_BITS); return true; #else return false; #endif } class MachineId { uint32_t cpuId_; JS::BuildIdCharVector buildId_; public: bool extractCurrentState(ExclusiveContext* cx) { if (!cx->asmJSCacheOps().buildId) return false; if (!cx->asmJSCacheOps().buildId(&buildId_)) return false; if (!GetCPUID(&cpuId_)) return false; return true; } size_t serializedSize() const { return sizeof(uint32_t) + SerializedPodVectorSize(buildId_); } uint8_t* serialize(uint8_t* cursor) const { cursor = WriteScalar(cursor, cpuId_); cursor = SerializePodVector(cursor, buildId_); return cursor; } const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) { (cursor = ReadScalar(cursor, &cpuId_)) && (cursor = DeserializePodVector(cx, cursor, &buildId_)); return cursor; } bool operator==(const MachineId& rhs) const { return cpuId_ == rhs.cpuId_ && buildId_.length() == rhs.buildId_.length() && PodEqual(buildId_.begin(), rhs.buildId_.begin(), buildId_.length()); } bool operator!=(const MachineId& rhs) const { return !(*this == rhs); } }; struct PropertyNameWrapper { PropertyName* name; PropertyNameWrapper() : name(nullptr) {} explicit PropertyNameWrapper(PropertyName* name) : name(name) {} size_t serializedSize() const { return SerializedNameSize(name); } uint8_t* serialize(uint8_t* cursor) const { return SerializeName(cursor, name); } const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) { return DeserializeName(cx, cursor, &name); } }; class ModuleChars { protected: uint32_t isFunCtor_; Vector funCtorArgs_; public: static uint32_t beginOffset(AsmJSParser& parser) { return parser.pc->maybeFunction->pn_pos.begin; } static uint32_t endOffset(AsmJSParser& parser) { TokenPos pos(0, 0); // initialize to silence GCC warning MOZ_ALWAYS_TRUE(parser.tokenStream.peekTokenPos(&pos, TokenStream::Operand)); return pos.end; } }; class ModuleCharsForStore : ModuleChars { uint32_t uncompressedSize_; uint32_t compressedSize_; Vector compressedBuffer_; public: bool init(AsmJSParser& parser) { MOZ_ASSERT(beginOffset(parser) < endOffset(parser)); uncompressedSize_ = (endOffset(parser) - beginOffset(parser)) * sizeof(char16_t); size_t maxCompressedSize = LZ4::maxCompressedSize(uncompressedSize_); if (maxCompressedSize < uncompressedSize_) return false; if (!compressedBuffer_.resize(maxCompressedSize)) return false; const char16_t* chars = parser.tokenStream.rawCharPtrAt(beginOffset(parser)); const char* source = reinterpret_cast(chars); size_t compressedSize = LZ4::compress(source, uncompressedSize_, compressedBuffer_.begin()); if (!compressedSize || compressedSize > UINT32_MAX) return false; compressedSize_ = compressedSize; // For a function statement or named function expression: // function f(x,y,z) { abc } // the range [beginOffset, endOffset) captures the source: // f(x,y,z) { abc } // An unnamed function expression captures the same thing, sans 'f'. // Since asm.js modules do not contain any free variables, equality of // [beginOffset, endOffset) is sufficient to guarantee identical code // generation, modulo MachineId. // // For functions created with 'new Function', function arguments are // not present in the source so we must manually explicitly serialize // and match the formals as a Vector of PropertyName. isFunCtor_ = parser.pc->isFunctionConstructorBody(); if (isFunCtor_) { unsigned numArgs; ParseNode* arg = FunctionArgsList(parser.pc->maybeFunction, &numArgs); for (unsigned i = 0; i < numArgs; i++, arg = arg->pn_next) { if (!funCtorArgs_.append(arg->name())) return false; } } return true; } size_t serializedSize() const { return sizeof(uint32_t) + sizeof(uint32_t) + compressedSize_ + sizeof(uint32_t) + (isFunCtor_ ? SerializedVectorSize(funCtorArgs_) : 0); } uint8_t* serialize(uint8_t* cursor) const { cursor = WriteScalar(cursor, uncompressedSize_); cursor = WriteScalar(cursor, compressedSize_); cursor = WriteBytes(cursor, compressedBuffer_.begin(), compressedSize_); cursor = WriteScalar(cursor, isFunCtor_); if (isFunCtor_) cursor = SerializeVector(cursor, funCtorArgs_); return cursor; } }; class ModuleCharsForLookup : ModuleChars { Vector chars_; public: const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) { uint32_t uncompressedSize; cursor = ReadScalar(cursor, &uncompressedSize); uint32_t compressedSize; cursor = ReadScalar(cursor, &compressedSize); if (!chars_.resize(uncompressedSize / sizeof(char16_t))) return nullptr; const char* source = reinterpret_cast(cursor); char* dest = reinterpret_cast(chars_.begin()); if (!LZ4::decompress(source, dest, uncompressedSize)) return nullptr; cursor += compressedSize; cursor = ReadScalar(cursor, &isFunCtor_); if (isFunCtor_) cursor = DeserializeVector(cx, cursor, &funCtorArgs_); return cursor; } bool match(AsmJSParser& parser) const { const char16_t* parseBegin = parser.tokenStream.rawCharPtrAt(beginOffset(parser)); const char16_t* parseLimit = parser.tokenStream.rawLimit(); MOZ_ASSERT(parseLimit >= parseBegin); if (uint32_t(parseLimit - parseBegin) < chars_.length()) return false; if (!PodEqual(chars_.begin(), parseBegin, chars_.length())) return false; if (isFunCtor_ != parser.pc->isFunctionConstructorBody()) return false; if (isFunCtor_) { // For function statements, the closing } is included as the last // character of the matched source. For Function constructor, // parsing terminates with EOF which we must explicitly check. This // prevents // new Function('"use asm"; function f() {} return f') // from incorrectly matching // new Function('"use asm"; function f() {} return ff') if (parseBegin + chars_.length() != parseLimit) return false; unsigned numArgs; ParseNode* arg = FunctionArgsList(parser.pc->maybeFunction, &numArgs); if (funCtorArgs_.length() != numArgs) return false; for (unsigned i = 0; i < funCtorArgs_.length(); i++, arg = arg->pn_next) { if (funCtorArgs_[i].name != arg->name()) return false; } } return true; } }; struct ScopedCacheEntryOpenedForWrite { ExclusiveContext* cx; const size_t serializedSize; uint8_t* memory; intptr_t handle; ScopedCacheEntryOpenedForWrite(ExclusiveContext* cx, size_t serializedSize) : cx(cx), serializedSize(serializedSize), memory(nullptr), handle(-1) {} ~ScopedCacheEntryOpenedForWrite() { if (memory) cx->asmJSCacheOps().closeEntryForWrite(serializedSize, memory, handle); } }; JS::AsmJSCacheResult js::StoreAsmJSModuleInCache(AsmJSParser& parser, const AsmJSModule& module, ExclusiveContext* cx) { MachineId machineId; if (!machineId.extractCurrentState(cx)) return JS::AsmJSCache_InternalError; ModuleCharsForStore moduleChars; if (!moduleChars.init(parser)) return JS::AsmJSCache_InternalError; size_t serializedSize = machineId.serializedSize() + moduleChars.serializedSize() + module.serializedSize(); JS::OpenAsmJSCacheEntryForWriteOp open = cx->asmJSCacheOps().openEntryForWrite; if (!open) return JS::AsmJSCache_Disabled_Internal; const char16_t* begin = parser.tokenStream.rawCharPtrAt(ModuleChars::beginOffset(parser)); const char16_t* end = parser.tokenStream.rawCharPtrAt(ModuleChars::endOffset(parser)); bool installed = parser.options().installedFile; ScopedCacheEntryOpenedForWrite entry(cx, serializedSize); JS::AsmJSCacheResult openResult = open(cx->global(), installed, begin, end, serializedSize, &entry.memory, &entry.handle); if (openResult != JS::AsmJSCache_Success) return openResult; uint8_t* cursor = entry.memory; cursor = machineId.serialize(cursor); cursor = moduleChars.serialize(cursor); cursor = module.serialize(cursor); MOZ_ASSERT(cursor == entry.memory + serializedSize); return JS::AsmJSCache_Success; } struct ScopedCacheEntryOpenedForRead { ExclusiveContext* cx; size_t serializedSize; const uint8_t* memory; intptr_t handle; explicit ScopedCacheEntryOpenedForRead(ExclusiveContext* cx) : cx(cx), serializedSize(0), memory(nullptr), handle(0) {} ~ScopedCacheEntryOpenedForRead() { if (memory) cx->asmJSCacheOps().closeEntryForRead(serializedSize, memory, handle); } }; bool js::LookupAsmJSModuleInCache(ExclusiveContext* cx, AsmJSParser& parser, ScopedJSDeletePtr* moduleOut, ScopedJSFreePtr* compilationTimeReport) { int64_t usecBefore = PRMJ_Now(); MachineId machineId; if (!machineId.extractCurrentState(cx)) return true; JS::OpenAsmJSCacheEntryForReadOp open = cx->asmJSCacheOps().openEntryForRead; if (!open) return true; const char16_t* begin = parser.tokenStream.rawCharPtrAt(ModuleChars::beginOffset(parser)); const char16_t* limit = parser.tokenStream.rawLimit(); ScopedCacheEntryOpenedForRead entry(cx); if (!open(cx->global(), begin, limit, &entry.serializedSize, &entry.memory, &entry.handle)) return true; const uint8_t* cursor = entry.memory; MachineId cachedMachineId; cursor = cachedMachineId.deserialize(cx, cursor); if (!cursor) return false; if (machineId != cachedMachineId) return true; ModuleCharsForLookup moduleChars; cursor = moduleChars.deserialize(cx, cursor); if (!moduleChars.match(parser)) return true; uint32_t srcStart = parser.pc->maybeFunction->pn_body->pn_pos.begin; uint32_t srcBodyStart = parser.tokenStream.currentToken().pos.end; bool strict = parser.pc->sc->strict() && !parser.pc->sc->hasExplicitUseStrict(); // canUseSignalHandlers will be clobbered when deserializing and checked below ScopedJSDeletePtr module( cx->new_(parser.ss, srcStart, srcBodyStart, strict, /* canUseSignalHandlers = */ false)); if (!module) return false; cursor = module->deserialize(cx, cursor); if (!cursor) return false; bool atEnd = cursor == entry.memory + entry.serializedSize; MOZ_ASSERT(atEnd, "Corrupt cache file"); if (!atEnd) return true; if (module->canUseSignalHandlers() != cx->canUseSignalHandlers()) return true; if (!parser.tokenStream.advance(module->srcEndBeforeCurly())) return false; { // Delay flushing until dynamic linking. AutoFlushICache afc("LookupAsmJSModuleInCache", /* inhibit = */ true); module->setAutoFlushICacheRange(); module->staticallyLink(cx); } int64_t usecAfter = PRMJ_Now(); int ms = (usecAfter - usecBefore) / PRMJ_USEC_PER_MSEC; *compilationTimeReport = JS_smprintf("loaded from cache in %dms", ms); *moduleOut = module.forget(); return true; }