From ebb7e1061ce9c6a2fab120416221ee356172fc1b Mon Sep 17 00:00:00 2001 From: Alexander Markov Date: Wed, 26 Feb 2025 14:01:44 -0800 Subject: [PATCH] [vm] Use _setjmp instead of setjmp on MacOS/iOS setjmp is extremely expensive on MacOS/iOS as it always saves signal mask. Introduce DART_SETJMP / DART_LONGJMP macros to use _setjmp instead of setjmp on MacOS/iOS. DeltaBlue on Mac/arm64 on the interpreter: 179585.3 us -> 58347.9 us. (3x faster) TEST=ci Issue: https://github.com/dart-lang/sdk/issues/60205 Change-Id: I2122f2eb4d5de66ae2ef904a7034af1ed09f1d07 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/412320 Reviewed-by: Ryan Macnak Commit-Queue: Alexander Markov --- runtime/platform/globals.h | 9 +++++++++ runtime/vm/allocation_test.cc | 4 ++-- runtime/vm/bootstrap.cc | 2 +- runtime/vm/class_finalizer.cc | 4 ++-- runtime/vm/compiler/aot/precompiler.cc | 4 ++-- runtime/vm/compiler/backend/inliner.cc | 2 +- runtime/vm/compiler/jit/compiler.cc | 6 +++--- runtime/vm/exceptions.cc | 2 +- runtime/vm/heap/scavenger.cc | 8 ++++---- runtime/vm/interpreter.cc | 6 +++--- runtime/vm/isolate_reload.cc | 2 +- runtime/vm/kernel.cc | 6 +++--- runtime/vm/kernel_loader.cc | 4 ++-- runtime/vm/longjump.cc | 2 +- runtime/vm/longjump_test.cc | 2 +- runtime/vm/message_snapshot.cc | 2 +- runtime/vm/object_graph_copy.cc | 2 +- runtime/vm/simulator_arm.cc | 4 ++-- runtime/vm/simulator_arm64.cc | 4 ++-- runtime/vm/simulator_riscv.cc | 4 ++-- runtime/vm/thread.h | 2 +- runtime/vm/thread_test.cc | 2 +- runtime/vm/type_testing_stubs.cc | 2 +- runtime/vm/unit_test.h | 2 +- 24 files changed, 48 insertions(+), 39 deletions(-) diff --git a/runtime/platform/globals.h b/runtime/platform/globals.h index 124b85be8d6..2996d476b08 100644 --- a/runtime/platform/globals.h +++ b/runtime/platform/globals.h @@ -297,6 +297,15 @@ struct simd128_value_t { #error Automatic compiler detection failed. #endif +#if defined(__APPLE__) +// Avoid expensive saving of sigmask in setjmp/longjmp. +#define DART_SETJMP _setjmp +#define DART_LONGJMP _longjmp +#else +#define DART_SETJMP setjmp +#define DART_LONGJMP longjmp +#endif + #if !defined(TARGET_ARCH_ARM) && !defined(TARGET_ARCH_X64) && \ !defined(TARGET_ARCH_IA32) && !defined(TARGET_ARCH_ARM64) && \ !defined(TARGET_ARCH_RISCV32) && !defined(TARGET_ARCH_RISCV64) diff --git a/runtime/vm/allocation_test.cc b/runtime/vm/allocation_test.cc index 5d4e1541116..fb70b66da65 100644 --- a/runtime/vm/allocation_test.cc +++ b/runtime/vm/allocation_test.cc @@ -92,7 +92,7 @@ static void StackAllocatedLongJumpHelper(int* ptr, LongJumpScope* jump) { ISOLATE_UNIT_TEST_CASE(StackAllocatedLongJump) { LongJumpScope jump; int data = 1; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { StackAllocatedLongJumpHelper(&data, &jump); UNREACHABLE(); } else { @@ -145,7 +145,7 @@ ISOLATE_UNIT_TEST_CASE(StackResourceLongJump) { { LongJumpScope jump; int data = 1; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { StackResourceLongJumpHelper(&data, &jump); UNREACHABLE(); } else { diff --git a/runtime/vm/bootstrap.cc b/runtime/vm/bootstrap.cc index 401b710cef5..fcc8b26228d 100644 --- a/runtime/vm/bootstrap.cc +++ b/runtime/vm/bootstrap.cc @@ -114,7 +114,7 @@ static ErrorPtr BootstrapFromKernelSingleProgram( std::unique_ptr program) { Zone* zone = thread->zone(); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { kernel::KernelLoader loader(program.get(), /*uri_to_source_table=*/nullptr); auto isolate_group = thread->isolate_group(); diff --git a/runtime/vm/class_finalizer.cc b/runtime/vm/class_finalizer.cc index b63685c50fc..2dc4e6f6d9d 100644 --- a/runtime/vm/class_finalizer.cc +++ b/runtime/vm/class_finalizer.cc @@ -199,7 +199,7 @@ bool ClassFinalizer::ProcessPendingClasses() { } LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { GrowableObjectArray& class_array = GrowableObjectArray::Handle(); class_array = object_store->pending_classes(); ASSERT(!class_array.IsNull()); @@ -827,7 +827,7 @@ ErrorPtr ClassFinalizer::LoadClassMembers(const Class& cls) { ASSERT(!cls.is_finalized()); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { cls.EnsureDeclarationLoaded(); ASSERT(cls.is_type_finalized()); ClassFinalizer::FinalizeClass(cls); diff --git a/runtime/vm/compiler/aot/precompiler.cc b/runtime/vm/compiler/aot/precompiler.cc index b0827b62b06..888aa693a07 100644 --- a/runtime/vm/compiler/aot/precompiler.cc +++ b/runtime/vm/compiler/aot/precompiler.cc @@ -364,7 +364,7 @@ static void Jump(const Error& error) { ErrorPtr Precompiler::CompileAll() { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { Precompiler precompiler(Thread::Current()); precompiler.DoCompileAll(); precompiler.ReportStats(); @@ -3486,7 +3486,7 @@ bool PrecompileParsedFunctionHelper::GenerateCode(FlowGraph* flow_graph) { while (!done) { LongJumpScope jump; - const intptr_t val = setjmp(*jump.Set()); + const intptr_t val = DART_SETJMP(*jump.Set()); if (val == 0) { // Even in bare instructions mode we don't directly add objects into // the global object pool because code generation can bail out diff --git a/runtime/vm/compiler/backend/inliner.cc b/runtime/vm/compiler/backend/inliner.cc index 2909c6915fc..2ce98a5d1b7 100644 --- a/runtime/vm/compiler/backend/inliner.cc +++ b/runtime/vm/compiler/backend/inliner.cc @@ -1278,7 +1278,7 @@ class CallSiteInliner : public ValueObject { // Install bailout jump. LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { // Load IC data for the callee. ZoneGrowableArray* ic_data_array = new (Z) ZoneGrowableArray(); diff --git a/runtime/vm/compiler/jit/compiler.cc b/runtime/vm/compiler/jit/compiler.cc index 055d6a08c0f..b1e5f5fab3d 100644 --- a/runtime/vm/compiler/jit/compiler.cc +++ b/runtime/vm/compiler/jit/compiler.cc @@ -498,7 +498,7 @@ CodePtr CompileParsedFunctionHelper::Compile() { while (!done) { *result = Code::null(); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { FlowGraph* flow_graph = nullptr; ZoneGrowableArray* ic_data_array = nullptr; @@ -673,7 +673,7 @@ static ObjectPtr CompileFunctionHelper(const Function& function, ASSERT(!FLAG_precompiled_mode); ASSERT(!optimized || function.WasCompiled() || function.ForceOptimize()); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { StackZone stack_zone(thread); Zone* const zone = stack_zone.GetZone(); const bool trace_compiler = @@ -889,7 +889,7 @@ void Compiler::ComputeLocalVarDescriptors(const Code& code) { Zone* zone = thread->zone(); CompilerState state(thread, /*is_aot=*/false, /*is_optimizing=*/false); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { ParsedFunction* parsed_function = new ParsedFunction(thread, Function::ZoneHandle(zone, function.ptr())); ZoneGrowableArray* ic_data_array = diff --git a/runtime/vm/exceptions.cc b/runtime/vm/exceptions.cc index 41533d7b539..0be70fb7f9c 100644 --- a/runtime/vm/exceptions.cc +++ b/runtime/vm/exceptions.cc @@ -671,7 +671,7 @@ NO_SANITIZE_SAFE_STACK // This function manipulates the safestack pointer. tsan_utils->exception_pc = program_counter; tsan_utils->exception_sp = stack_pointer; tsan_utils->exception_fp = frame_pointer; - longjmp(*(tsan_utils->setjmp_buffer), 1); + DART_LONGJMP(*(tsan_utils->setjmp_buffer), 1); } #endif // defined(USING_THREAD_SANITIZER) diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc index 2f760915b87..c1739a53528 100644 --- a/runtime/vm/heap/scavenger.cc +++ b/runtime/vm/heap/scavenger.cc @@ -282,7 +282,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor, page_space_->AcquireLock(freelist_); LongJumpScope jump(thread_); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { scavenger_->IterateRoots(this); } else { ASSERT(scavenger_->abort_); @@ -291,7 +291,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor, void ProcessSurvivors() { LongJumpScope jump(thread_); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { // Iterate until all work has been drained. do { ProcessToSpace(); @@ -305,7 +305,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor, void ProcessAll() { TIMELINE_FUNCTION_GC_DURATION(thread_, "ProcessToSpace"); LongJumpScope jump(thread_); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { do { do { ProcessToSpace(); @@ -320,7 +320,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor, void ProcessWeakProperties() { LongJumpScope jump(thread_); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { ProcessWeakPropertiesScoped(); } else { ASSERT(scavenger_->abort_); diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc index a66bee9bf5c..8e89d734fc7 100644 --- a/runtime/vm/interpreter.cc +++ b/runtime/vm/interpreter.cc @@ -51,7 +51,7 @@ class InterpreterSetjmpBuffer { void Longjmp() { // "This" is now the last setjmp buffer. interpreter_->set_last_setjmp_buffer(this); - longjmp(buffer_, 1); + DART_LONGJMP(buffer_, 1); } explicit InterpreterSetjmpBuffer(Interpreter* interpreter) { @@ -532,7 +532,7 @@ static DART_NOINLINE bool InvokeRuntime(Thread* thread, RuntimeFunction drt, const NativeArguments& args) { InterpreterSetjmpBuffer buffer(interpreter); - if (!setjmp(buffer.buffer_)) { + if (!DART_SETJMP(buffer.buffer_)) { thread->set_vm_tag(reinterpret_cast(drt)); drt(args); thread->set_vm_tag(VMTag::kDartInterpretedTagId); @@ -581,7 +581,7 @@ DART_NOINLINE bool Interpreter::InvokeCompiled(Thread* thread, Exit(thread, *FP, call_top + 1, *pc); { InterpreterSetjmpBuffer buffer(this); - if (!setjmp(buffer.buffer_)) { + if (!DART_SETJMP(buffer.buffer_)) { #if defined(USING_SIMULATOR) // We need to beware that bouncing between the interpreter and the // simulator may exhaust the C stack before exhausting either the diff --git a/runtime/vm/isolate_reload.cc b/runtime/vm/isolate_reload.cc index 75f5e40c619..63133bf4746 100644 --- a/runtime/vm/isolate_reload.cc +++ b/runtime/vm/isolate_reload.cc @@ -1218,7 +1218,7 @@ ObjectPtr ProgramReloadContext::ReloadPhase2LoadKernel( Thread* thread = Thread::Current(); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { const Object& tmp = kernel::KernelLoader::LoadEntireProgram(program); if (tmp.IsError()) { return tmp.ptr(); diff --git a/runtime/vm/kernel.cc b/runtime/vm/kernel.cc index f5e73535666..8114a513395 100644 --- a/runtime/vm/kernel.cc +++ b/runtime/vm/kernel.cc @@ -382,7 +382,7 @@ ObjectPtr EvaluateStaticConstFieldInitializer(const Field& field) { ASSERT(field.is_static() && field.is_const()); LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { Thread* thread = Thread::Current(); Zone* zone = thread->zone(); TranslationHelper helper(thread); @@ -468,7 +468,7 @@ ObjectPtr EvaluateMetadata(const Library& library, intptr_t kernel_offset, bool is_annotations_offset) { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { Thread* thread = Thread::Current(); Zone* zone = thread->zone(); TranslationHelper helper(thread); @@ -572,7 +572,7 @@ ObjectPtr ParameterDescriptorBuilder::BuildParameterDescriptor( ObjectPtr BuildParameterDescriptor(const Function& function) { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { Thread* thread = Thread::Current(); Zone* zone = thread->zone(); diff --git a/runtime/vm/kernel_loader.cc b/runtime/vm/kernel_loader.cc index 1c0c7b7b0bd..635dad7363a 100644 --- a/runtime/vm/kernel_loader.cc +++ b/runtime/vm/kernel_loader.cc @@ -526,7 +526,7 @@ ObjectPtr KernelLoader::LoadProgram(bool process_pending_classes) { } LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { // Note that `problemsAsJson` on Component is implicitly skipped. const intptr_t length = program_->library_count(); for (intptr_t i = 0; i < length; i++) { @@ -652,7 +652,7 @@ void KernelLoader::FindModifiedLibraries(Program* program, intptr_t* p_num_procedures) { LongJumpScope jump; Zone* zone = Thread::Current()->zone(); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { if (force_reload) { // If a reload is being forced we mark all libraries as having // been modified. diff --git a/runtime/vm/longjump.cc b/runtime/vm/longjump.cc index 28a8e47a14b..d3970c5d005 100644 --- a/runtime/vm/longjump.cc +++ b/runtime/vm/longjump.cc @@ -38,7 +38,7 @@ void LongJumpScope::Jump(int value) { // Destruct all the active StackResource objects. StackResource::UnwindAbove(thread, top_); - longjmp(environment_, value); + DART_LONGJMP(environment_, value); UNREACHABLE(); } diff --git a/runtime/vm/longjump_test.cc b/runtime/vm/longjump_test.cc index a32b2ef1be1..e89140cd148 100644 --- a/runtime/vm/longjump_test.cc +++ b/runtime/vm/longjump_test.cc @@ -19,7 +19,7 @@ ISOLATE_UNIT_TEST_CASE(LongJump) { LongJumpScope* base = Thread::Current()->long_jump_base(); { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { LongJumpHelper(&jump); UNREACHABLE(); } else { diff --git a/runtime/vm/message_snapshot.cc b/runtime/vm/message_snapshot.cc index 61b2e273c95..0b94c6dcce6 100644 --- a/runtime/vm/message_snapshot.cc +++ b/runtime/vm/message_snapshot.cc @@ -3483,7 +3483,7 @@ ObjectPtr ReadMessage(Thread* thread, Message* message) { } else { RELEASE_ASSERT(message->IsSnapshot()); LongJumpScope jump(thread); - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { MessageDeserializer deserializer(thread, message); return deserializer.Deserialize(); } else { diff --git a/runtime/vm/object_graph_copy.cc b/runtime/vm/object_graph_copy.cc index 09e2cb71a29..561a8a212f5 100644 --- a/runtime/vm/object_graph_copy.cc +++ b/runtime/vm/object_graph_copy.cc @@ -2441,7 +2441,7 @@ class ObjectGraphCopier : public StackResource { { LongJumpScope jump; // e.g. for OOMs. - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { result = CopyObjectGraphInternal(root, &exception_msg); // Any allocated external typed data must have finalizers attached so // memory will get free()ed. diff --git a/runtime/vm/simulator_arm.cc b/runtime/vm/simulator_arm.cc index 979fe7a8d92..9d7be40cc05 100644 --- a/runtime/vm/simulator_arm.cc +++ b/runtime/vm/simulator_arm.cc @@ -55,7 +55,7 @@ class SimulatorSetjmpBuffer { void Longjmp() { // "This" is now the last setjmp buffer. simulator_->set_last_setjmp_buffer(this); - longjmp(buffer_, 1); + DART_LONGJMP(buffer_, 1); } explicit SimulatorSetjmpBuffer(Simulator* sim) { @@ -1416,7 +1416,7 @@ void Simulator::SupervisorCall(Instr* instr) { case Instr::kSimulatorRedirectCode: { SimulatorSetjmpBuffer buffer(this); - if (!setjmp(buffer.buffer_)) { + if (!DART_SETJMP(buffer.buffer_)) { int32_t saved_lr = get_register(LR); Redirection* redirection = Redirection::FromSvcInstruction(instr); uword external = redirection->external_function(); diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index e879c870937..2ddc632bb96 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -58,7 +58,7 @@ class SimulatorSetjmpBuffer { void Longjmp() { // "This" is now the last setjmp buffer. simulator_->set_last_setjmp_buffer(this); - longjmp(buffer_, 1); + DART_LONGJMP(buffer_, 1); } explicit SimulatorSetjmpBuffer(Simulator* sim) { @@ -1677,7 +1677,7 @@ void Simulator::DoRedirectedCall(Instr* instr) { memory_.FlushAll(); SimulatorSetjmpBuffer buffer(this); - if (!setjmp(buffer.buffer_)) { + if (!DART_SETJMP(buffer.buffer_)) { int64_t saved_lr = get_register(LR); Redirection* redirection = Redirection::FromHltInstruction(instr); uword external = redirection->external_function(); diff --git a/runtime/vm/simulator_riscv.cc b/runtime/vm/simulator_riscv.cc index c4d1289fa5e..9d3c204b0bc 100644 --- a/runtime/vm/simulator_riscv.cc +++ b/runtime/vm/simulator_riscv.cc @@ -43,7 +43,7 @@ class SimulatorSetjmpBuffer { void Longjmp() { // "This" is now the last setjmp buffer. simulator_->set_last_setjmp_buffer(this); - longjmp(buffer_, 1); + DART_LONGJMP(buffer_, 1); } explicit SimulatorSetjmpBuffer(Simulator* sim) { @@ -2109,7 +2109,7 @@ void Simulator::InterpretECALL(Instr instr) { memory_.FlushAll(); SimulatorSetjmpBuffer buffer(this); - if (!setjmp(buffer.buffer_)) { + if (!DART_SETJMP(buffer.buffer_)) { uintx_t saved_ra = get_xreg(RA); Redirection* redirection = Redirection::FromECallInstruction(pc_); uword external = redirection->external_function(); diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index 93079bb773d..4e3f53f0ca7 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -321,7 +321,7 @@ struct TsanUtils { // implementation. // -> See https://dartbug.com/47472#issuecomment-948235479 for details. #if defined(USING_THREAD_SANITIZER) - void* setjmp_function = reinterpret_cast(&setjmp); + void* setjmp_function = reinterpret_cast(&DART_SETJMP); #else // MSVC (on Windows) is not happy with getting address of purely intrinsic. void* setjmp_function = nullptr; diff --git a/runtime/vm/thread_test.cc b/runtime/vm/thread_test.cc index c33fbe8169a..caba19be394 100644 --- a/runtime/vm/thread_test.cc +++ b/runtime/vm/thread_test.cc @@ -1034,7 +1034,7 @@ static void RunLockerWithLongJumpTest() { LockType lock; for (intptr_t i = 0; i < kNumIterations; ++i) { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { LockerType locker(Thread::Current(), &lock); execution_count = execution_count + 1; Thread::Current()->long_jump_base()->Jump( diff --git a/runtime/vm/type_testing_stubs.cc b/runtime/vm/type_testing_stubs.cc index b6caf441f41..3d01449a4c8 100644 --- a/runtime/vm/type_testing_stubs.cc +++ b/runtime/vm/type_testing_stubs.cc @@ -223,7 +223,7 @@ static CodePtr RetryCompilationWithFarBranches( volatile intptr_t far_branch_level = 0; while (true) { LongJumpScope jump; - if (setjmp(*jump.Set()) == 0) { + if (DART_SETJMP(*jump.Set()) == 0) { // To use the already-defined __ Macro ! compiler::Assembler assembler(nullptr, far_branch_level); return fun(assembler); diff --git a/runtime/vm/unit_test.h b/runtime/vm/unit_test.h index 6f535912cbf..76ab4c6dbce 100644 --- a/runtime/vm/unit_test.h +++ b/runtime/vm/unit_test.h @@ -103,7 +103,7 @@ volatile intptr_t far_branch_level = 0; \ while (true) { \ LongJumpScope jump(thread); \ - if (setjmp(*jump.Set()) == 0) { \ + if (DART_SETJMP(*jump.Set()) == 0) { \ compiler::ObjectPoolBuilder object_pool_builder; \ compiler::Assembler assembler(&object_pool_builder, far_branch_level); \ AssemblerTest test("" #name, &assembler, thread->zone()); \