From 78d280a2dfc1c12b63a67b76f8e19b4ae4238d04 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Wed, 24 Sep 2025 13:14:32 -0700 Subject: [PATCH] [vm] Symbolize JIT Dart frames for TSAN. TEST=tsan Bug: https://github.com/dart-lang/sdk/issues/61352 Change-Id: I0e64e563b19bbd7b67e6f275dbc373db08b9d2c7 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/449980 Reviewed-by: Alexander Aprelev Commit-Queue: Ryan Macnak --- runtime/platform/thread_sanitizer.h | 10 + .../vm/compiler/assembler/assembler_arm64.cc | 50 ---- .../vm/compiler/assembler/assembler_arm64.h | 2 - .../vm/compiler/assembler/assembler_riscv.cc | 50 ---- .../vm/compiler/assembler/assembler_riscv.h | 2 - .../vm/compiler/assembler/assembler_x64.cc | 50 ---- runtime/vm/compiler/assembler/assembler_x64.h | 2 - runtime/vm/compiler/backend/il.cc | 92 ++++++- runtime/vm/compiler/backend/il.h | 4 + runtime/vm/compiler/backend/il_arm64.cc | 58 ++++- runtime/vm/compiler/backend/il_riscv.cc | 56 ++++ runtime/vm/compiler/backend/il_x64.cc | 43 +++- runtime/vm/object.cc | 4 +- runtime/vm/runtime_entry.cc | 35 ++- runtime/vm/tsan_symbolize.cc | 241 ++++++++++++++++++ runtime/vm/tsan_symbolize.h | 16 ++ runtime/vm/vm_sources.gni | 2 + 17 files changed, 549 insertions(+), 168 deletions(-) create mode 100644 runtime/vm/tsan_symbolize.cc create mode 100644 runtime/vm/tsan_symbolize.h diff --git a/runtime/platform/thread_sanitizer.h b/runtime/platform/thread_sanitizer.h index 12c0efdbaee..3b4f3ccf317 100644 --- a/runtime/platform/thread_sanitizer.h +++ b/runtime/platform/thread_sanitizer.h @@ -35,6 +35,16 @@ extern "C" void __tsan_write2(void* addr); extern "C" void __tsan_write4(void* addr); extern "C" void __tsan_write8(void* addr); extern "C" void __tsan_write16(void* addr); +extern "C" void __tsan_read1_pc(void* addr, void* pc); +extern "C" void __tsan_read2_pc(void* addr, void* pc); +extern "C" void __tsan_read4_pc(void* addr, void* pc); +extern "C" void __tsan_read8_pc(void* addr, void* pc); +extern "C" void __tsan_read16_pc(void* addr, void* pc); +extern "C" void __tsan_write1_pc(void* addr, void* pc); +extern "C" void __tsan_write2_pc(void* addr, void* pc); +extern "C" void __tsan_write4_pc(void* addr, void* pc); +extern "C" void __tsan_write8_pc(void* addr, void* pc); +extern "C" void __tsan_write16_pc(void* addr, void* pc); extern "C" void __tsan_func_entry(void* pc); extern "C" void __tsan_func_exit(); #else diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index 8f8452dd667..39dc14025bd 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -353,56 +353,6 @@ void Assembler::TsanStoreRelease(Register src, } } -void Assembler::TsanRead(Register addr, intptr_t size) { - Comment("TsanRead"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(R0, addr); - switch (size) { - case 1: - rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - -void Assembler::TsanWrite(Register addr, intptr_t size) { - Comment("TsanWrite"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(R0, addr); - switch (size) { - case 1: - rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - void Assembler::TsanFuncEntry(bool preserve_registers) { Comment("TsanFuncEntry"); LeafRuntimeScope rt(this, /*frame_size=*/0, preserve_registers); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index d46ab7343e9..a12aa1d33ad 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -504,8 +504,6 @@ class Assembler : public AssemblerBase { void TsanLoadAcquire(Register dst, Register addr, OperandSize size); void TsanStoreRelease(Register src, Register addr, OperandSize size); - void TsanRead(Register addr, intptr_t size); - void TsanWrite(Register addr, intptr_t size); void TsanFuncEntry(bool preserve_registers = true); void TsanFuncExit(bool preserve_registers = true); diff --git a/runtime/vm/compiler/assembler/assembler_riscv.cc b/runtime/vm/compiler/assembler/assembler_riscv.cc index 92ed0b467bc..3ed95a5f4ad 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.cc +++ b/runtime/vm/compiler/assembler/assembler_riscv.cc @@ -3172,56 +3172,6 @@ void Assembler::TsanStoreRelease(Register src, } } -void Assembler::TsanRead(Register addr, intptr_t size) { - Comment("TsanRead"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(A0, addr); - switch (size) { - case 1: - rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - -void Assembler::TsanWrite(Register addr, intptr_t size) { - Comment("TsanWrite"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(A0, addr); - switch (size) { - case 1: - rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - void Assembler::TsanFuncEntry(bool preserve_registers) { Comment("TsanFuncEntry"); LeafRuntimeScope rt(this, /*frame_size=*/0, preserve_registers); diff --git a/runtime/vm/compiler/assembler/assembler_riscv.h b/runtime/vm/compiler/assembler/assembler_riscv.h index 855a19d2c91..144a9493ee6 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.h +++ b/runtime/vm/compiler/assembler/assembler_riscv.h @@ -1009,8 +1009,6 @@ class Assembler : public MicroAssembler { void TsanLoadAcquire(Register dst, const Address& address, OperandSize size); void TsanStoreRelease(Register src, const Address& address, OperandSize size); - void TsanRead(Register addr, intptr_t size); - void TsanWrite(Register addr, intptr_t size); void TsanFuncEntry(bool preserve_registers = true); void TsanFuncExit(bool preserve_registers = true); diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc index 1b3e3555cb5..3936cfbe761 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.cc +++ b/runtime/vm/compiler/assembler/assembler_x64.cc @@ -2210,56 +2210,6 @@ void Assembler::TsanStoreRelease(Register src, Address addr, OperandSize size) { } } -void Assembler::TsanRead(Register addr, intptr_t size) { - Comment("TsanRead"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(CallingConventions::kArg1Reg, addr); - switch (size) { - case 1: - rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - -void Assembler::TsanWrite(Register addr, intptr_t size) { - Comment("TsanWrite"); - LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true); - MoveRegister(CallingConventions::kArg1Reg, addr); - switch (size) { - case 1: - rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1); - break; - case 2: - rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1); - break; - case 4: - rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1); - break; - case 8: - rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1); - break; - case 16: - rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1); - break; - default: - UNREACHABLE(); - } -} - void Assembler::TsanFuncEntry(bool preserve_registers) { Comment("TsanFuncEntry"); LeafRuntimeScope rt(this, /*frame_size=*/0, preserve_registers); diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index f71dac18582..cdf120281b8 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -1163,8 +1163,6 @@ class Assembler : public AssemblerBase { void TsanLoadAcquire(Register dst, Address addr, OperandSize size); void TsanStoreRelease(Register src, Address addr, OperandSize size); - void TsanRead(Register addr, intptr_t size); - void TsanWrite(Register addr, intptr_t size); void TsanFuncEntry(bool preserve_registers = true); void TsanFuncExit(bool preserve_registers = true); diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 366db145bbc..87a8e92e914 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -4684,6 +4684,30 @@ void LoadFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); auto const rep = slot().representation(); + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer && + !slot().is_no_sanitize_thread() && + memory_order() == compiler::Assembler::kRelaxedNonAtomic) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + intptr_t tag = slot().has_untagged_instance() ? 0 : kHeapObjectTag; + __ AddImmediate(CallingConventions::ArgumentRegisters[0], instance_reg, + slot().offset_in_bytes() - tag); + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanRead1RuntimeEntry; + case 2: + return kTsanRead2RuntimeEntry; + case 4: + return kTsanRead4RuntimeEntry; + case 8: + return kTsanRead8RuntimeEntry; + case 16: + return kTsanRead16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + if (calls_initializer()) { __ LoadFromSlot(locs()->out(0).reg(), instance_reg, slot(), memory_order_); EmitNativeCodeForInitializerCall(compiler); @@ -7297,11 +7321,11 @@ static LocationSummary* MakeTsanLocationSummary(Zone* zone, return result; } -static void EmitTsanNativeCode( - FlowGraphCompiler* compiler, - LocationSummary* locs, - std::function move_parameters) { - const Register saved_sp = locs->temp(0).reg(); +static void EmitTsanCall(FlowGraphCompiler* compiler, + Instruction* instr, + RegisterSet spill_set, + Register saved_sp, + std::function move_parameters) { ASSERT(IsCalleeSavedRegister(saved_sp)); ASSERT(IsCalleeSavedRegister(THR)); ASSERT(IsCalleeSavedRegister(PP)); @@ -7316,6 +7340,7 @@ static void EmitTsanNativeCode( ASSERT(IsCalleeSavedRegister(DISPATCH_TABLE_REG)); #endif + __ PushRegisters(spill_set); __ MoveRegister(saved_sp, SPREG); #if defined(TARGET_ARCH_ARM64) __ AndImmediate(CSP, SP, ~(OS::ActivationFrameAlignment() - 1)); @@ -7327,6 +7352,8 @@ static void EmitTsanNativeCode( __ Store(TMP, compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ CallCFunction(TMP); + compiler->AddCurrentDescriptor(UntaggedPcDescriptors::kOther, DeoptId::kNone, + instr->source()); __ LoadImmediate(TMP, VMTag::kDartTagId); __ Store(TMP, compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); @@ -7334,6 +7361,31 @@ static void EmitTsanNativeCode( #if defined(TARGET_ARCH_ARM64) __ SetupCSPFromThread(THR); #endif + __ PopRegisters(spill_set); +} + +static void EmitTsanCall(FlowGraphCompiler* compiler, + Instruction* instr, + std::function move_parameters) { + EmitTsanCall(compiler, instr, RegisterSet(), instr->locs()->temp(0).reg(), + move_parameters); +} + +void EmitTsanCallUnopt(FlowGraphCompiler* compiler, + Instruction* instr, + std::function move_parameters) { + intptr_t cpu_reg_mask = 0; + intptr_t fpu_reg_mask = 0; + LocationSummary* locs = instr->locs(); + for (intptr_t i = 0, n = locs->input_count(); i < n; i++) { + if (locs->in(i).IsRegister()) { + cpu_reg_mask |= 1 << locs->in(i).reg(); + } else if (locs->in(i).IsFpuRegister()) { + fpu_reg_mask |= 1 << locs->in(i).fpu_reg(); + } + } + EmitTsanCall(compiler, instr, RegisterSet(cpu_reg_mask, fpu_reg_mask), + CALLEE_SAVED_TEMP, move_parameters); } LocationSummary* TsanFuncEntryExitInstr::MakeLocationSummary(Zone* zone, @@ -7344,7 +7396,7 @@ LocationSummary* TsanFuncEntryExitInstr::MakeLocationSummary(Zone* zone, void TsanFuncEntryExitInstr::EmitNativeCode(FlowGraphCompiler* compiler) { if (!compiler->flow_graph().graph_entry()->NeedsFrame()) return; - EmitTsanNativeCode(compiler, locs(), [&]() -> const RuntimeEntry& { + EmitTsanCall(compiler, this, [&]() -> const RuntimeEntry& { if (kind_ == kEntry) { __ Load( CallingConventions::ArgumentRegisters[0], @@ -7364,7 +7416,7 @@ LocationSummary* TsanReadWriteInstr::MakeLocationSummary(Zone* zone, } void TsanReadWriteInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - EmitTsanNativeCode(compiler, locs(), [&]() -> const RuntimeEntry& { + EmitTsanCall(compiler, this, [&]() -> const RuntimeEntry& { const Register instance_reg = locs()->in(0).reg(); intptr_t tag = slot().has_untagged_instance() ? 0 : kHeapObjectTag; __ AddImmediate(CallingConventions::ArgumentRegisters[0], instance_reg, @@ -7413,7 +7465,7 @@ LocationSummary* TsanReadWriteIndexedInstr::MakeLocationSummary( } void TsanReadWriteIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - EmitTsanNativeCode(compiler, locs(), [&]() -> const RuntimeEntry& { + EmitTsanCall(compiler, this, [&]() -> const RuntimeEntry& { const Register array_reg = locs()->in(kArrayPos).reg(); Register index_reg = locs()->in(kIndexPos).reg(); @@ -8022,6 +8074,30 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); const Representation rep = slot().representation(); + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer && + !slot().is_no_sanitize_thread() && + memory_order() == compiler::Assembler::kRelaxedNonAtomic) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + intptr_t tag = slot().has_untagged_instance() ? 0 : kHeapObjectTag; + __ AddImmediate(CallingConventions::ArgumentRegisters[0], instance_reg, + slot().offset_in_bytes() - tag); + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanWrite1RuntimeEntry; + case 2: + return kTsanWrite2RuntimeEntry; + case 4: + return kTsanWrite4RuntimeEntry; + case 8: + return kTsanWrite8RuntimeEntry; + case 16: + return kTsanWrite16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + #if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_RISCV32) || \ defined(TARGET_ARCH_RISCV64) if (locs()->in(kValuePos).IsConstant() && diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 026780e3271..dd96b2d7887 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -11799,6 +11799,10 @@ inline bool Value::CanBe(const Object& value) { #undef DECLARE_CUSTOM_SERIALIZATION #undef DECLARE_EMPTY_SERIALIZATION +void EmitTsanCallUnopt(FlowGraphCompiler* compiler, + Instruction* instr, + std::function move_parameters); + } // namespace dart #endif // RUNTIME_VM_COMPILER_BACKEND_IL_H_ diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 845de1cec8c..46356060940 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -1916,6 +1916,35 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Location index = locs()->in(kIndexPos); auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); + + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + if (index.IsRegister()) { + __ ComputeElementAddressForRegIndex(R0, IsUntagged(), class_id(), + index_scale(), index_unboxed_, + array, index.reg()); + } else { + __ ComputeElementAddressForIntIndex( + R0, IsUntagged(), class_id(), index_scale(), array, + Smi::Cast(index.constant()).Value()); + } + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanRead1RuntimeEntry; + case 2: + return kTsanRead2RuntimeEntry; + case 4: + return kTsanRead4RuntimeEntry; + case 8: + return kTsanRead8RuntimeEntry; + case 16: + return kTsanRead16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + compiler::Address element_address(TMP); // Bad address. element_address = index.IsRegister() ? __ ElementAddressForRegIndex( @@ -2074,11 +2103,38 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Location index = locs()->in(1); const Register temp = locs()->temp(0).reg(); compiler::Address element_address(TMP); // Bad address. - auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); ASSERT(RequiredInputRepresentation(2) == Boxing::NativeRepresentation(rep)); + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + if (index.IsRegister()) { + __ ComputeElementAddressForRegIndex(R0, IsUntagged(), class_id(), + index_scale(), index_unboxed_, + array, index.reg()); + } else { + __ ComputeElementAddressForIntIndex( + R0, IsUntagged(), class_id(), index_scale(), array, + Smi::Cast(index.constant()).Value()); + } + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanWrite1RuntimeEntry; + case 2: + return kTsanWrite2RuntimeEntry; + case 4: + return kTsanWrite4RuntimeEntry; + case 8: + return kTsanWrite8RuntimeEntry; + case 16: + return kTsanWrite16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + // Deal with a special case separately. if (class_id() == kArrayCid && ShouldEmitStoreBarrier()) { if (index.IsRegister()) { diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index a721bf323fe..6a9b171fed7 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -1999,6 +1999,34 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + if (index.IsRegister()) { + __ ComputeElementAddressForRegIndex(A0, IsUntagged(), class_id(), + index_scale(), index_unboxed_, + array, index.reg()); + } else { + __ ComputeElementAddressForIntIndex( + A0, IsUntagged(), class_id(), index_scale(), array, + Smi::Cast(index.constant()).Value()); + } + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanRead1RuntimeEntry; + case 2: + return kTsanRead2RuntimeEntry; + case 4: + return kTsanRead4RuntimeEntry; + case 8: + return kTsanRead8RuntimeEntry; + case 16: + return kTsanRead16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + compiler::Address element_address(TMP); // Bad address. element_address = index.IsRegister() ? __ ElementAddressForRegIndex( @@ -2243,6 +2271,34 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + if (index.IsRegister()) { + __ ComputeElementAddressForRegIndex(A0, IsUntagged(), class_id(), + index_scale(), index_unboxed_, + array, index.reg()); + } else { + __ ComputeElementAddressForIntIndex( + A0, IsUntagged(), class_id(), index_scale(), array, + Smi::Cast(index.constant()).Value()); + } + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanWrite1RuntimeEntry; + case 2: + return kTsanWrite2RuntimeEntry; + case 4: + return kTsanWrite4RuntimeEntry; + case 8: + return kTsanWrite8RuntimeEntry; + case 16: + return kTsanWrite16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + // Deal with a special case separately. if (class_id() == kArrayCid && ShouldEmitStoreBarrier()) { if (index.IsRegister()) { diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 5bbbb798800..14974665b35 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -1839,6 +1839,27 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Smi::Cast(index.constant()).Value()); auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); + + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + __ leaq(CallingConventions::ArgumentRegisters[0], element_address); + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanRead1RuntimeEntry; + case 2: + return kTsanRead2RuntimeEntry; + case 4: + return kTsanRead4RuntimeEntry; + case 8: + return kTsanRead8RuntimeEntry; + case 16: + return kTsanRead16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + ASSERT(representation() == Boxing::NativeRepresentation(rep)); if (RepresentationUtils::IsUnboxedInteger(rep)) { Register result = locs()->out(0).reg(); @@ -2022,9 +2043,29 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) { : compiler::Assembler::ElementAddressForIntIndex( IsUntagged(), class_id(), index_scale_, array, Smi::Cast(index.constant()).Value()); - auto const rep = RepresentationUtils::RepresentationOfArrayElement(class_id()); + + if (!compiler->is_optimizing() && FLAG_target_thread_sanitizer) { + EmitTsanCallUnopt(compiler, this, [&]() -> const RuntimeEntry& { + __ leaq(CallingConventions::ArgumentRegisters[0], element_address); + switch (RepresentationUtils::ValueSize(rep)) { + case 1: + return kTsanWrite1RuntimeEntry; + case 2: + return kTsanWrite2RuntimeEntry; + case 4: + return kTsanWrite4RuntimeEntry; + case 8: + return kTsanWrite8RuntimeEntry; + case 16: + return kTsanWrite16RuntimeEntry; + default: + UNREACHABLE(); + } + }); + } + ASSERT(RequiredInputRepresentation(2) == Boxing::NativeRepresentation(rep)); if (IsClampedTypedDataBaseClassId(class_id())) { ASSERT(rep == kUnboxedUint8); diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 9d9f4b7a999..96802dd621e 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -6,7 +6,6 @@ #include -#include "compiler/method_recognizer.h" #include "include/dart_api.h" #include "lib/integers.h" #include "lib/stacktrace.h" @@ -27,6 +26,7 @@ #include "vm/compiler/assembler/disassembler.h" #include "vm/compiler/assembler/disassembler_kbc.h" #include "vm/compiler/jit/compiler.h" +#include "vm/compiler/method_recognizer.h" #include "vm/compiler/runtime_api.h" #include "vm/cpu.h" #include "vm/dart.h" @@ -71,6 +71,7 @@ #include "vm/tags.h" #include "vm/thread_registry.h" #include "vm/timeline.h" +#include "vm/tsan_symbolize.h" #include "vm/type_testing_stubs.h" #include "vm/zone_text_buffer.h" @@ -18925,6 +18926,7 @@ void Code::NotifyCodeObservers(const Function& function, NotifyCodeObservers(name, code, optimized); } #endif + RegisterTsanSymbolize(code); } void Code::NotifyCodeObservers(const char* name, diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc index dbfecb1c82f..d5e1ce3c3b9 100644 --- a/runtime/vm/runtime_entry.cc +++ b/runtime/vm/runtime_entry.cc @@ -5298,17 +5298,49 @@ extern "C" void __tsan_func_entry(void* pc) { extern "C" void __tsan_func_exit() { UNREACHABLE(); } +#else +#define CASE(x) \ + extern "C" __attribute__((disable_sanitizer_instrumentation)) void \ + jit_tsan_##x(void* addr) { \ + __tsan_##x##_pc( \ + addr, reinterpret_cast( \ + reinterpret_cast(__builtin_return_address(0)) | \ + (1ULL << 60))); \ + } + +CASE(read1) +CASE(read2) +CASE(read4) +CASE(read8) +CASE(read16) +CASE(write1) +CASE(write2) +CASE(write4) +CASE(write8) +CASE(write16) +#undef CASE #endif // These runtime entries are defined even when not using MSAN / TSAN to keep // offsets on Thread consistent. - DEFINE_LEAF_RUNTIME_ENTRY(MsanUnpoison, 2, __msan_unpoison); DEFINE_LEAF_RUNTIME_ENTRY(MsanUnpoisonParam, 1, __msan_unpoison_param); DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Load, 2, __tsan_atomic32_load); DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Store, 3, __tsan_atomic32_store); DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Load, 2, __tsan_atomic64_load); DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Store, 3, __tsan_atomic64_store); +#if defined(USING_THREAD_SANITIZER) && !defined(DART_PRECOMPILED_RUNTIME) +DEFINE_LEAF_RUNTIME_ENTRY(TsanRead1, 1, jit_tsan_read1); +DEFINE_LEAF_RUNTIME_ENTRY(TsanRead2, 1, jit_tsan_read2); +DEFINE_LEAF_RUNTIME_ENTRY(TsanRead4, 1, jit_tsan_read4); +DEFINE_LEAF_RUNTIME_ENTRY(TsanRead8, 1, jit_tsan_read8); +DEFINE_LEAF_RUNTIME_ENTRY(TsanRead16, 1, jit_tsan_read16); +DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite1, 1, jit_tsan_write1); +DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite2, 1, jit_tsan_write2); +DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite4, 1, jit_tsan_write4); +DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite8, 1, jit_tsan_write8); +DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite16, 1, jit_tsan_write16); +#else DEFINE_LEAF_RUNTIME_ENTRY(TsanRead1, 1, __tsan_read1); DEFINE_LEAF_RUNTIME_ENTRY(TsanRead2, 1, __tsan_read2); DEFINE_LEAF_RUNTIME_ENTRY(TsanRead4, 1, __tsan_read4); @@ -5319,6 +5351,7 @@ DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite2, 1, __tsan_write2); DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite4, 1, __tsan_write4); DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite8, 1, __tsan_write8); DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite16, 1, __tsan_write16); +#endif DEFINE_LEAF_RUNTIME_ENTRY(TsanFuncEntry, 1, __tsan_func_entry); DEFINE_LEAF_RUNTIME_ENTRY(TsanFuncExit, 0, __tsan_func_exit); diff --git a/runtime/vm/tsan_symbolize.cc b/runtime/vm/tsan_symbolize.cc new file mode 100644 index 00000000000..78f6a6e0994 --- /dev/null +++ b/runtime/vm/tsan_symbolize.cc @@ -0,0 +1,241 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "vm/tsan_symbolize.h" + +#include "platform/atomic.h" +#include "platform/thread_sanitizer.h" +#include "vm/code_descriptors.h" +#include "vm/object.h" + +namespace dart { + +#if defined(USING_THREAD_SANITIZER) && !defined(DART_PRECOMPILED_RUNTIME) + +struct TsanLineNumberProgram { + TsanLineNumberProgram* next; + uintptr_t pc; + uint32_t size; + uint8_t stream[]; +}; +static std::atomic head = nullptr; + +enum : uint8_t { + OP_STOP = 0, + OP_PUSH_FUNCTION, + OP_POP_FUNCTION, + OP_ADVANCE_PC, + OP_CHANGE_POSITION, +}; + +static void WriteString(GrowableArray& out, const char* str) { + for (intptr_t i = 0, len = strlen(str); i <= len; i++) { + out.Add(str[i]); + } +} + +void RegisterTsanSymbolize(const Code& code) { + if (!code.IsFunctionCode()) { + // TODO(rmacnak): Do something for stubs? + return; + } + + const CodeSourceMap& map = CodeSourceMap::Handle(code.code_source_map()); + RELEASE_ASSERT(!map.IsNull()); + const Array& functions = Array::Handle(code.inlined_id_to_function()); + Function& function = Function::Handle(code.function()); + Script& script = Script::Handle(function.script()); + String& url = String::Handle(script.url()); + GrowableArray function_stack(8); + GrowableArray token_positions(8); + function_stack.Add(&Function::Handle(function.ptr())); + token_positions.Add(CodeSourceMapBuilder::kInitialPosition); + + GrowableArray out(256); + out.Add(OP_PUSH_FUNCTION); + WriteString(out, function.QualifiedUserVisibleNameCString()); + WriteString(out, url.ToCString()); + + ReadStream stream(map.Data(), map.Length()); + while (stream.PendingBytes() > 0) { + int32_t arg; + switch (CodeSourceMapOps::Read(&stream, &arg)) { + case CodeSourceMapOps::kChangePosition: { + const TokenPosition& old_token = token_positions.Last(); + token_positions.Last() = TokenPosition::Deserialize( + Utils::AddWithWrapAround(arg, old_token.Serialize())); + + TokenPosition pos = token_positions.Last(); + if (pos.IsNoSource()) { + pos = function_stack.Last()->token_pos(); + } + + intptr_t line = -1; + intptr_t column = -1; + script = function_stack.Last()->script(); + script.GetTokenLocation(pos, &line, &column); + + out.Add(OP_CHANGE_POSITION); + out.Add((line >> 0) & 0xFF); + out.Add((line >> 8) & 0xFF); + out.Add((column >> 0) & 0xFF); + out.Add((column >> 8) & 0xFF); + break; + } + case CodeSourceMapOps::kAdvancePC: { + out.Add(OP_ADVANCE_PC); + out.Add((arg >> 0) & 0xFF); + out.Add((arg >> 8) & 0xFF); + out.Add((arg >> 16) & 0xFF); + out.Add((arg >> 24) & 0xFF); + break; + } + case CodeSourceMapOps::kPushFunction: { + function ^= functions.At(arg); + function_stack.Add(&Function::Handle(function.ptr())); + token_positions.Add(CodeSourceMapBuilder::kInitialPosition); + + out.Add(OP_PUSH_FUNCTION); + WriteString(out, function.QualifiedUserVisibleNameCString()); + script = function_stack.Last()->script(); + url = script.url(); + WriteString(out, url.ToCString()); + break; + } + case CodeSourceMapOps::kPopFunction: { + function_stack.RemoveLast(); + token_positions.RemoveLast(); + out.Add(OP_POP_FUNCTION); + break; + } + case CodeSourceMapOps::kNullCheck: { + break; + } + default: + UNREACHABLE(); + } + } + out.Add(OP_STOP); + + TsanLineNumberProgram* lnp = reinterpret_cast( + malloc(sizeof(TsanLineNumberProgram) + out.length())); + lnp->next = nullptr; + lnp->pc = code.PayloadStart(); + lnp->size = code.Size(); + memcpy(&lnp->stream, out.data(), out.length()); // NOLINT + + TsanLineNumberProgram* old_head = head.load(std::memory_order_acquire); + do { + lnp->next = old_head; + } while ( + !head.compare_exchange_weak(old_head, lnp, std::memory_order_acq_rel)); +} + +typedef void (*AddFrame)(void* ctxt, + const char* function_name, + const char* file, + int line, + int column); + +// It would be nice to implement this by a heap walk to find the Code object and +// then symbolize using our normal PC descriptors, etc, but this function must +// not call any function that has been instrumented by TSAN or it might deadlock +// during __tsan_func_entry. +extern "C" __attribute__((disable_sanitizer_instrumentation)) void +__tsan_symbolize_external_ex(uintptr_t pc, AddFrame add_frame, void* ctxt) { + constexpr uintptr_t kExternalPCBit = 1ULL << 60; + const uword lookup_pc = pc & ~kExternalPCBit; + + for (TsanLineNumberProgram* lnp = head.load(std::memory_order_acquire); + lnp != nullptr; lnp = lnp->next) { + if ((lookup_pc >= lnp->pc) && ((lookup_pc - lnp->pc) < lnp->size)) { + // Greater than the default value of --inlining_depth_threshold. + constexpr intptr_t kMaxDepth = 32; + const char* names[kMaxDepth]; + const char* files[kMaxDepth]; + int32_t lines[kMaxDepth]; + int32_t columns[kMaxDepth]; + intptr_t depth = 0; + uword lnp_pc = lnp->pc; + + uint8_t* cursor = &lnp->stream[0]; + for (;;) { + switch (*cursor++) { + case OP_PUSH_FUNCTION: { + const char* name = reinterpret_cast(cursor); + while (*cursor++ != 0) { + } + const char* file = reinterpret_cast(cursor); + while (*cursor++ != 0) { + } + + names[depth] = name; + files[depth] = file; + lines[depth] = -1; + columns[depth] = -1; + + depth++; + if (depth >= kMaxDepth) { + FATAL("LNP overflow"); + } + break; + } + case OP_POP_FUNCTION: { + depth--; + break; + } + case OP_CHANGE_POSITION: { + uint8_t a = *cursor++; + uint8_t b = *cursor++; + uint8_t c = *cursor++; + uint8_t d = *cursor++; + lines[depth - 1] = a | (b << 8); + columns[depth - 1] = c | (d << 8); + break; + } + case OP_ADVANCE_PC: { + uint8_t a = *cursor++; + uint8_t b = *cursor++; + uint8_t c = *cursor++; + uint8_t d = *cursor++; + uint32_t disp = a | (b << 8) | (c << 16) | (d << 24); + lnp_pc += disp; + if (lookup_pc <= lnp_pc) { +#if 0 + char s[17]; + s[16] = 0; + for (intptr_t k = 0; k < 16; k++) { + s[k] = "0123456789abcdef"[lookup_pc >> ((15 - k) << 2)) & 0xFF]; + } + add_frame(ctxt, "lookup_pc", s, 0, 0); + for (intptr_t k = 0; k < 16; k++) { + s[k] = "0123456789abcdef"[lnp_pc >> ((15 - k) << 2)) & 0xFF]; + } + add_frame(ctxt, "lnp_pc", s, 0, 0); +#endif + + for (intptr_t i = depth - 1; i >= 0; i--) { + add_frame(ctxt, names[i], files[i], lines[i], columns[i]); + } + return; + } + break; + } + case OP_STOP: { + FATAL("pc in function but outside LNP"); + } + default: + UNREACHABLE(); + } + } + UNREACHABLE(); + } + } + add_frame(ctxt, "dart-code-lookup-failed", nullptr, 0, 0); +} +#else +void RegisterTsanSymbolize(const Code& code) {} +#endif + +} // namespace dart diff --git a/runtime/vm/tsan_symbolize.h b/runtime/vm/tsan_symbolize.h new file mode 100644 index 00000000000..fc597d5bb1f --- /dev/null +++ b/runtime/vm/tsan_symbolize.h @@ -0,0 +1,16 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_VM_TSAN_SYMBOLIZE_H_ +#define RUNTIME_VM_TSAN_SYMBOLIZE_H_ + +#include "vm/object.h" + +namespace dart { + +void RegisterTsanSymbolize(const Code& code); + +} // namespace dart + +#endif // RUNTIME_VM_TSAN_SYMBOLIZE_H_ diff --git a/runtime/vm/vm_sources.gni b/runtime/vm/vm_sources.gni index 3e50c6a5582..56a9942aeba 100644 --- a/runtime/vm/vm_sources.gni +++ b/runtime/vm/vm_sources.gni @@ -346,6 +346,8 @@ vm_sources = [ "token.h", "token_position.cc", "token_position.h", + "tsan_symbolize.cc", + "tsan_symbolize.h", "type_testing_stubs.cc", "type_testing_stubs.h", "unicode.cc",