dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, compiler] Add TSAN instrumentation to Dart field access.

Browse Source 
Allows TSAN to detect data races involving Dart fields.

TEST=tsan
Cq-Include-Trybots: luci.dart.try:vm-tsan-linux-release-x64-try,vm-tsan-linux-release-arm64-try,iso-stress-linux-arm64-try,iso-stress-linux-x64-try
Change-Id: Ic7a6c7e6c1810adf79b41e5c0ae891132f368a61
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/439143
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2025-08-06 09:18:43 -07:00
committed by Commit Queue
parent f1a6d1cc38
commit e6053248c3
32 changed files with 949 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/docs/pragmas.md
+1
View File
@@ -21,6 +21,7 @@ These pragmas are part of the VM's API and are safe for use in external code.
| `vm:awaiter-link` | [Specifying variable to follow for awaiter stack unwinding](awaiter_stack_traces.md) |
| `vm:deeply-immutable` | [Specifying a class and all its subtypes are deeply immutable](deeply_immutable.md) |
| `vm:align-loops` | Tells compiler to align all loop headers inside the function to an architecture specific boundary: currently 32 bytes on X64 and ARM64 (except Apple Silicon, which explicitly discourages aligning branch targets) |
| `vm:no-sanitize-thread` | Disable ThreadSanitizer instrumentation |
## Unsafe pragmas for general use
runtime/platform/thread_sanitizer.h
+12 -2
View File
@@ -17,14 +17,24 @@
#if defined(USING_THREAD_SANITIZER)
#define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
extern "C" void __tsan_atomic32_load(uint32_t* addr, int order);
extern "C" uint32_t __tsan_atomic32_load(uint32_t* addr, int order);
extern "C" void __tsan_atomic32_store(uint32_t* addr,
uint32_t value,
int order);
extern "C" void __tsan_atomic64_load(uint64_t* addr, int order);
extern "C" uint64_t __tsan_atomic64_load(uint64_t* addr, int order);
extern "C" void __tsan_atomic64_store(uint64_t* addr,
uint64_t value,
int order);
extern "C" void __tsan_read1(void* addr);
extern "C" void __tsan_read2(void* addr);
extern "C" void __tsan_read4(void* addr);
extern "C" void __tsan_read8(void* addr);
extern "C" void __tsan_read16(void* addr);
extern "C" void __tsan_write1(void* addr);
extern "C" void __tsan_write2(void* addr);
extern "C" void __tsan_write4(void* addr);
extern "C" void __tsan_write8(void* addr);
extern "C" void __tsan_write16(void* addr);
#else
#define NO_SANITIZE_THREAD
#endif
runtime/tests/vm/dart/tsan/array_data_race_test.dart
+66
View File
@@ -0,0 +1,66 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// VMOptions=--experimental-shared-data
import "dart:io";
import "dart:isolate";
import "package:expect/expect.dart";
@pragma("vm:shared")
List<dynamic> box = List<dynamic>.filled(1, 0);
@pragma("vm:never-inline")
noopt() {}
@pragma("vm:never-inline")
dataRaceFromMain() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox[0] += 1;
noopt();
}
}
@pragma("vm:never-inline")
dataRaceFromChild() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox[0] += 1;
noopt();
}
}
child(_) {
dataRaceFromChild();
}
main(List<String> arguments) {
if (arguments.contains("--testee")) {
print(box); // side effect initialization
Isolate.spawn(child, null);
dataRaceFromMain();
return;
}
var exec = Platform.executable;
var args = [
...Platform.executableArguments,
Platform.script.toFilePath(),
"--testee",
];
print("+ $exec ${args.join(' ')}");
var result = Process.runSync(exec, args);
print("Command stdout:");
print(result.stdout);
print("Command stderr:");
print(result.stderr);
Expect.notEquals(0, result.exitCode);
Expect.contains("ThreadSanitizer: data race", result.stderr);
Expect.contains("of size 8", result.stderr);
Expect.contains("dataRaceFromMain", result.stderr);
Expect.contains("dataRaceFromChild", result.stderr);
}
runtime/tests/vm/dart/tsan/field_data_race_no_sanitize_test.dart
+67
View File
@@ -0,0 +1,67 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// VMOptions=--experimental-shared-data
import "dart:io";
import "dart:isolate";
import "package:expect/expect.dart";
class Box {
@pragma("vm:no-sanitize-thread") // __attribute__((no_sanitize("thread")))
int foo = 0;
}
@pragma("vm:shared")
Box box = Box();
@pragma("vm:never-inline")
noopt() {}
@pragma("vm:never-inline")
dataRaceFromMain() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox.foo += 1;
noopt();
}
}
@pragma("vm:never-inline")
dataRaceFromChild() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox.foo += 1;
noopt();
}
}
child(_) {
dataRaceFromChild();
}
main(List<String> arguments) {
if (arguments.contains("--testee")) {
print(box); // side effect initialization
Isolate.spawn(child, null);
dataRaceFromMain();
return;
}
var exec = Platform.executable;
var args = [
...Platform.executableArguments,
Platform.script.toFilePath(),
"--testee",
];
print("+ $exec ${args.join(' ')}");
var result = Process.runSync(exec, args);
print("Command stdout:");
print(result.stdout);
print("Command stderr:");
print(result.stderr);
Expect.equals(0, result.exitCode);
}
runtime/tests/vm/dart/tsan/field_data_race_test.dart
+70
View File
@@ -0,0 +1,70 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// VMOptions=--experimental-shared-data
import "dart:io";
import "dart:isolate";
import "package:expect/expect.dart";
class Box {
int foo = 0;
}
@pragma("vm:shared")
Box box = Box();
@pragma("vm:never-inline")
noopt() {}
@pragma("vm:never-inline")
dataRaceFromMain() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox.foo += 1;
noopt();
}
}
@pragma("vm:never-inline")
dataRaceFromChild() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox.foo += 1;
noopt();
}
}
child(_) {
dataRaceFromChild();
}
main(List<String> arguments) {
if (arguments.contains("--testee")) {
print(box); // side effect initialization
Isolate.spawn(child, null);
dataRaceFromMain();
return;
}
var exec = Platform.executable;
var args = [
...Platform.executableArguments,
Platform.script.toFilePath(),
"--testee",
];
print("+ $exec ${args.join(' ')}");
var result = Process.runSync(exec, args);
print("Command stdout:");
print(result.stdout);
print("Command stderr:");
print(result.stderr);
Expect.notEquals(0, result.exitCode);
Expect.contains("ThreadSanitizer: data race", result.stderr);
Expect.contains("of size 8", result.stderr);
Expect.contains("dataRaceFromMain", result.stderr);
Expect.contains("dataRaceFromChild", result.stderr);
}
runtime/tests/vm/dart/tsan/typed_data_race_test.dart
+67
View File
@@ -0,0 +1,67 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// VMOptions=--experimental-shared-data
import "dart:io";
import "dart:isolate";
import "dart:typed_data";
import "package:expect/expect.dart";
@pragma("vm:shared")
Uint8List box = Uint8List(1);
@pragma("vm:never-inline")
noopt() {}
@pragma("vm:never-inline")
dataRaceFromMain() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox[0] += 1;
noopt();
}
}
@pragma("vm:never-inline")
dataRaceFromChild() {
final localBox = box;
for (var i = 0; i < 1000000; i++) {
localBox[0] += 1;
noopt();
}
}
child(_) {
dataRaceFromChild();
}
main(List<String> arguments) {
if (arguments.contains("--testee")) {
print(box); // side effect initialization
Isolate.spawn(child, null);
dataRaceFromMain();
return;
}
var exec = Platform.executable;
var args = [
...Platform.executableArguments,
Platform.script.toFilePath(),
"--testee",
];
print("+ $exec ${args.join(' ')}");
var result = Process.runSync(exec, args);
print("Command stdout:");
print(result.stdout);
print("Command stderr:");
print(result.stderr);
Expect.notEquals(0, result.exitCode);
Expect.contains("ThreadSanitizer: data race", result.stderr);
Expect.contains("of size 1", result.stderr);
Expect.contains("dataRaceFromMain", result.stderr);
Expect.contains("dataRaceFromChild", result.stderr);
}
runtime/tests/vm/vm.status
+3
View File
@@ -392,6 +392,9 @@ dart/finalizer/finalizer_isolate_groups_run_gc_test: SkipByDesign # uses spawnUr
dart/isolates/send_object_to_spawn_uri_isolate_test: SkipByDesign # uses spawnUri
dart/issue32950_test: SkipByDesign # uses spawnUri.
[ $runtime != dart_precompiled || $sanitizer != tsan ]
dart/tsan/*: SkipByDesign
[ $runtime != dart_precompiled || $sanitizer != msan && $sanitizer != tsan ]
dart/sanitizer_compatibility_test: SkipByDesign
runtime/vm/app_snapshot.cc
+2 -2
View File
@@ -2204,7 +2204,7 @@ class FieldSerializationCluster : public SerializationCluster {
s->Write<int8_t>(field->untag()->static_type_exactness_state_);
s->Write<uint32_t>(field->untag()->kernel_offset_);
}
s->Write<uint16_t>(field->untag()->kind_bits_);
s->Write<uint32_t>(field->untag()->kind_bits_);
// Write out either the initial static value or field offset.
if (Field::StaticBit::decode(field->untag()->kind_bits_)) {
@@ -2267,7 +2267,7 @@ class FieldDeserializationCluster : public DeserializationCluster {
#endif // defined(TARGET_ARCH_X64)
field->untag()->kernel_offset_ = d.Read<uint32_t>();
#endif
field->untag()->kind_bits_ = d.Read<uint16_t>();
field->untag()->kind_bits_ = d.Read<uint32_t>();
field->untag()->host_offset_or_field_id_ =
static_cast<SmiPtr>(d.ReadRef());
runtime/vm/compiler/assembler/assembler_arm64.cc
+52
View File
@@ -287,6 +287,7 @@ void Assembler::Align(intptr_t alignment, intptr_t offset) {
}
void Assembler::TsanLoadAcquire(Register dst, Register addr, OperandSize size) {
Comment("TsanLoadAcquire");
RegisterSet registers(kDartVolatileCpuRegs & ~(1 << dst),
kAllFpuRegistersList);
@@ -326,6 +327,7 @@ void Assembler::TsanLoadAcquire(Register dst, Register addr, OperandSize size) {
void Assembler::TsanStoreRelease(Register src,
Register addr,
OperandSize size) {
Comment("TsanStoreRelease");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
if (src == R0) {
@@ -351,6 +353,56 @@ void Assembler::TsanStoreRelease(Register src,
}
}
void Assembler::TsanRead(Register addr, intptr_t size) {
Comment("TsanRead");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(R0, addr);
switch (size) {
case 1:
rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::TsanWrite(Register addr, intptr_t size) {
Comment("TsanWrite");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(R0, addr);
switch (size) {
case 1:
rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
static int CountLeadingZeros(uint64_t value, int width) {
if (width == 64) return Utils::CountLeadingZeros64(value);
if (width == 32) return Utils::CountLeadingZeros32(value);
runtime/vm/compiler/assembler/assembler_arm64.h
+2
View File
@@ -504,6 +504,8 @@ class Assembler : public AssemblerBase {
void TsanLoadAcquire(Register dst, Register addr, OperandSize size);
void TsanStoreRelease(Register src, Register addr, OperandSize size);
void TsanRead(Register addr, intptr_t size);
void TsanWrite(Register addr, intptr_t size);
void LoadAcquire(Register dst,
const Address& address,
runtime/vm/compiler/assembler/assembler_riscv.cc
+53 -1
View File
@@ -3089,6 +3089,7 @@ void Assembler::TsanLoadAcquire(Register dst,
ASSERT(addr.base() != FP);
ASSERT(dst != SP);
ASSERT(dst != FP);
Comment("TsanLoadAcquire");
RegisterSet registers(kDartVolatileCpuRegs & ~(1 << dst),
kAbiVolatileFpuRegs);
@@ -3144,6 +3145,7 @@ void Assembler::TsanStoreRelease(Register src,
ASSERT(addr.base() != FP);
ASSERT(src != SP);
ASSERT(src != FP);
Comment("TsanStoreRelease");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
@@ -3170,6 +3172,56 @@ void Assembler::TsanStoreRelease(Register src,
}
}
void Assembler::TsanRead(Register addr, intptr_t size) {
Comment("TsanRead");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(A0, addr);
switch (size) {
case 1:
rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::TsanWrite(Register addr, intptr_t size) {
Comment("TsanWrite");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(A0, addr);
switch (size) {
case 1:
rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::LoadAcquire(Register dst,
const Address& address,
OperandSize size) {
@@ -4848,7 +4900,7 @@ LeafRuntimeScope::LeafRuntimeScope(Assembler* assembler,
void LeafRuntimeScope::Call(const RuntimeEntry& entry,
intptr_t argument_count) {
ASSERT(argument_count == entry.argument_count());
__ lx(TMP2, compiler::Address(THR, entry.OffsetFromThread()));
__ Load(TMP2, compiler::Address(THR, entry.OffsetFromThread()));
__ sx(TMP2, compiler::Address(THR, target::Thread::vm_tag_offset()));
__ jalr(TMP2);
__ LoadImmediate(TMP2, VMTag::kDartTagId);
runtime/vm/compiler/assembler/assembler_riscv.h
+2
View File
@@ -1009,6 +1009,8 @@ class Assembler : public MicroAssembler {
void TsanLoadAcquire(Register dst, const Address& address, OperandSize size);
void TsanStoreRelease(Register src, const Address& address, OperandSize size);
void TsanRead(Register addr, intptr_t size);
void TsanWrite(Register addr, intptr_t size);
void LoadAcquire(Register dst,
const Address& address,
runtime/vm/compiler/assembler/assembler_x64.cc
+52
View File
@@ -2125,6 +2125,7 @@ void Assembler::TsanLoadAcquire(Register dst, Address addr, OperandSize size) {
ASSERT(addr.base() != RBP);
ASSERT(dst != RSP);
ASSERT(dst != RBP);
Comment("TsanLoadAcquire");
RegisterSet registers(CallingConventions::kVolatileCpuRegisters & ~(1 << dst),
CallingConventions::kVolatileXmmRegisters);
@@ -2170,6 +2171,7 @@ void Assembler::TsanStoreRelease(Register src, Address addr, OperandSize size) {
ASSERT(addr.base() != RBP);
ASSERT(src != RSP);
ASSERT(src != RBP);
Comment("TsanStoreRelease");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
@@ -2196,6 +2198,56 @@ void Assembler::TsanStoreRelease(Register src, Address addr, OperandSize size) {
}
}
void Assembler::TsanRead(Register addr, intptr_t size) {
Comment("TsanRead");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(CallingConventions::kArg1Reg, addr);
switch (size) {
case 1:
rt.Call(kTsanRead1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanRead2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanRead4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanRead8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanRead16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::TsanWrite(Register addr, intptr_t size) {
Comment("TsanStoreRelease");
LeafRuntimeScope rt(this, /*frame_size=*/0, /*preserve_registers=*/true);
MoveRegister(CallingConventions::kArg1Reg, addr);
switch (size) {
case 1:
rt.Call(kTsanWrite1RuntimeEntry, /*argument_count=*/1);
break;
case 2:
rt.Call(kTsanWrite2RuntimeEntry, /*argument_count=*/1);
break;
case 4:
rt.Call(kTsanWrite4RuntimeEntry, /*argument_count=*/1);
break;
case 8:
rt.Call(kTsanWrite8RuntimeEntry, /*argument_count=*/1);
break;
case 16:
rt.Call(kTsanWrite16RuntimeEntry, /*argument_count=*/1);
break;
default:
UNREACHABLE();
}
}
void Assembler::RestoreCodePointer() {
movq(CODE_REG,
Address(RBP, target::frame_layout.code_from_fp * target::kWordSize));
runtime/vm/compiler/assembler/assembler_x64.h
+2
View File
@@ -1161,6 +1161,8 @@ class Assembler : public AssemblerBase {
void TsanLoadAcquire(Register dst, Address addr, OperandSize size);
void TsanStoreRelease(Register src, Address addr, OperandSize size);
void TsanRead(Register addr, intptr_t size);
void TsanWrite(Register addr, intptr_t size);
void LoadAcquire(Register dst,
const Address& address,
runtime/vm/compiler/backend/il.cc
+22 -2
View File
@@ -4682,8 +4682,18 @@ void LoadFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
ASSERT(OffsetInBytes() >= 0); // Field is finalized.
// For fields on Dart objects, the offset must point after the header.
ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance());
auto const rep = slot().representation();
#if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_RISCV64) || \
defined(TARGET_ARCH_X64)
if (FLAG_target_thread_sanitizer && !slot().is_no_sanitize_thread() &&
memory_order_ == compiler::Assembler::kRelaxedNonAtomic) {
intptr_t tag = slot().has_untagged_instance() ? 0 : kHeapObjectTag;
__ AddImmediate(TMP, instance_reg, slot().offset_in_bytes() - tag);
__ TsanRead(TMP, RepresentationUtils::ValueSize(rep));
}
#endif
if (calls_initializer()) {
__ LoadFromSlot(locs()->out(0).reg(), instance_reg, slot(), memory_order_);
EmitNativeCodeForInitializerCall(compiler);
@@ -7809,8 +7819,18 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
ASSERT(OffsetInBytes() >= 0); // Field is finalized.
// For fields on Dart objects, the offset must point after the header.
ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance());
const Representation rep = slot().representation();
#if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_RISCV64) || \
defined(TARGET_ARCH_X64)
if (FLAG_target_thread_sanitizer && !slot().is_no_sanitize_thread() &&
memory_order_ == compiler::Assembler::kRelaxedNonAtomic) {
intptr_t tag = slot().has_untagged_instance() ? 0 : kHeapObjectTag;
__ AddImmediate(TMP, instance_reg, slot().offset_in_bytes() - tag);
__ TsanWrite(TMP, RepresentationUtils::ValueSize(rep));
}
#endif
#if defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_RISCV32) || \
defined(TARGET_ARCH_RISCV64)
if (locs()->in(kValuePos).IsConstant() &&
runtime/vm/compiler/backend/il_arm64.cc
+28 -2
View File
@@ -1914,6 +1914,21 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// The array register points to the backing store for external arrays.
const Register array = locs()->in(kArrayPos).reg();
const Location index = locs()->in(kIndexPos);
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
if (FLAG_target_thread_sanitizer) {
if (index.IsRegister()) {
__ ComputeElementAddressForRegIndex(TMP, IsUntagged(), class_id(),
index_scale(), index_unboxed_, array,
index.reg());
} else {
__ ComputeElementAddressForIntIndex(TMP, IsUntagged(), class_id(),
index_scale(), array,
Smi::Cast(index.constant()).Value());
}
__ TsanRead(TMP, RepresentationUtils::ValueSize(rep));
}
compiler::Address element_address(TMP); // Bad address.
element_address = index.IsRegister()
@@ -1923,8 +1938,6 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
: __ ElementAddressForIntIndex(
IsUntagged(), class_id(), index_scale(), array,
Smi::Cast(index.constant()).Value());
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(representation() == Boxing::NativeRepresentation(rep));
if (RepresentationUtils::IsUnboxedInteger(rep)) {
const Register result = locs()->out(0).reg();
@@ -2080,6 +2093,19 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(RequiredInputRepresentation(2) == Boxing::NativeRepresentation(rep));
if (FLAG_target_thread_sanitizer) {
if (index.IsRegister()) {
__ ComputeElementAddressForRegIndex(TMP, IsUntagged(), class_id(),
index_scale(), index_unboxed_, array,
index.reg());
} else {
__ ComputeElementAddressForIntIndex(TMP, IsUntagged(), class_id(),
index_scale(), array,
Smi::Cast(index.constant()).Value());
}
__ TsanWrite(TMP, RepresentationUtils::ValueSize(rep));
}
// Deal with a special case separately.
if (class_id() == kArrayCid && ShouldEmitStoreBarrier()) {
if (index.IsRegister()) {
runtime/vm/compiler/backend/il_riscv.cc
+30 -4
View File
@@ -1996,6 +1996,21 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// The array register points to the backing store for external arrays.
const Register array = locs()->in(kArrayPos).reg();
const Location index = locs()->in(kIndexPos);
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
if (FLAG_target_thread_sanitizer) {
if (index.IsRegister()) {
__ ComputeElementAddressForRegIndex(TMP, IsUntagged(), class_id(),
index_scale(), index_unboxed_, array,
index.reg());
} else {
__ ComputeElementAddressForIntIndex(TMP, IsUntagged(), class_id(),
index_scale(), array,
Smi::Cast(index.constant()).Value());
}
__ TsanRead(TMP, RepresentationUtils::ValueSize(rep));
}
compiler::Address element_address(TMP); // Bad address.
element_address = index.IsRegister()
@@ -2006,8 +2021,6 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
IsUntagged(), class_id(), index_scale(), array,
Smi::Cast(index.constant()).Value());
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(representation() == Boxing::NativeRepresentation(rep));
if (RepresentationUtils::IsUnboxedInteger(rep)) {
#if XLEN == 32
@@ -2240,6 +2253,21 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
const Location index = locs()->in(1);
const Register temp = locs()->temp(0).reg();
compiler::Address element_address(TMP); // Bad address.
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
if (FLAG_target_thread_sanitizer) {
if (index.IsRegister()) {
__ ComputeElementAddressForRegIndex(TMP, IsUntagged(), class_id(),
index_scale(), index_unboxed_, array,
index.reg());
} else {
__ ComputeElementAddressForIntIndex(TMP, IsUntagged(), class_id(),
index_scale(), array,
Smi::Cast(index.constant()).Value());
}
__ TsanWrite(TMP, RepresentationUtils::ValueSize(rep));
}
// Deal with a special case separately.
if (class_id() == kArrayCid && ShouldEmitStoreBarrier()) {
@@ -2265,8 +2293,6 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
IsUntagged(), class_id(), index_scale(), array,
Smi::Cast(index.constant()).Value());
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(RequiredInputRepresentation(2) == Boxing::NativeRepresentation(rep));
if (IsClampedTypedDataBaseClassId(class_id())) {
if (locs()->in(2).IsConstant()) {
runtime/vm/compiler/backend/il_x64.cc
+12 -1
View File
@@ -1837,10 +1837,15 @@ void LoadIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
: compiler::Assembler::ElementAddressForIntIndex(
IsUntagged(), class_id(), index_scale_, array,
Smi::Cast(index.constant()).Value());
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(representation() == Boxing::NativeRepresentation(rep));
if (FLAG_target_thread_sanitizer) {
__ leaq(TMP, element_address);
__ TsanRead(TMP, RepresentationUtils::ValueSize(rep));
}
if (RepresentationUtils::IsUnboxedInteger(rep)) {
Register result = locs()->out(0).reg();
__ Load(result, element_address, RepresentationUtils::OperandSize(rep));
@@ -2027,6 +2032,12 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
auto const rep =
RepresentationUtils::RepresentationOfArrayElement(class_id());
ASSERT(RequiredInputRepresentation(2) == Boxing::NativeRepresentation(rep));
if (FLAG_target_thread_sanitizer) {
__ leaq(TMP, element_address);
__ TsanWrite(TMP, RepresentationUtils::ValueSize(rep));
}
if (IsClampedTypedDataBaseClassId(class_id())) {
ASSERT(rep == kUnboxedUint8);
if (locs()->in(2).IsConstant()) {
runtime/vm/compiler/backend/slot.cc
+6 -1
View File
@@ -72,6 +72,9 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) {
(Slot::IsImmutableBit::encode(true) | Slot::IsWeakBit::encode(false))
#define FIELD_FLAGS_VAR \
(Slot::IsImmutableBit::encode(false) | Slot::IsWeakBit::encode(false))
#define FIELD_FLAGS_VAR_NOSANITIZETHREAD \
(Slot::IsImmutableBit::encode(false) | Slot::IsWeakBit::encode(false) | \
Slot::IsNoSanitizeThreadBit::encode(true))
#define FIELD_FLAGS_WEAK \
(Slot::IsImmutableBit::encode(false) | Slot::IsWeakBit::encode(true))
#define DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD(ClassName, UnderlyingType, \
@@ -195,6 +198,7 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) {
#undef FIELD_FLAGS_FINAL
#undef FIELD_FLAGS_VAR
#undef FIELD_FLAGS_VAR_NOSANITIZETHREAD
#undef FIELD_FLAGS_WEAK
default:
UNREACHABLE();
@@ -428,7 +432,8 @@ const Slot& Slot::Get(const Field& field,
IsGuardedBit::encode(used_guarded_state) |
IsCompressedBit::encode(
compiler::target::Class::HasCompressedPointers(owner)) |
IsNonTaggedBit::encode(is_unboxed),
IsNonTaggedBit::encode(is_unboxed) |
IsNoSanitizeThreadBit::encode(field.is_no_sanitize_thread()),
compiler::target::Field::OffsetOf(field), &field, type, rep,
field_guard_state);
runtime/vm/compiler/backend/slot.h
+8 -2
View File
@@ -73,7 +73,7 @@ class ParsedFunction;
V(ReceivePort, UntaggedReceivePort, send_port, SendPort, FINAL) \
V(ReceivePort, UntaggedReceivePort, handler, Closure, VAR) \
V(ImmutableLinkedHashBase, UntaggedLinkedHashBase, index, \
TypedDataUint32Array, VAR) \
TypedDataUint32Array, VAR_NOSANITIZETHREAD) \
V(Instance, UntaggedInstance, native_fields_array, Dynamic, VAR) \
V(SuspendState, UntaggedSuspendState, function_data, Dynamic, VAR) \
V(SuspendState, UntaggedSuspendState, then_callback, Closure, VAR) \
@@ -101,7 +101,7 @@ class ParsedFunction;
// that) or like a non-final field.
#define NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \
V(Array, UntaggedArray, length, Smi, FINAL) \
V(Closure, UntaggedClosure, hash, Smi, VAR) \
V(Closure, UntaggedClosure, hash, Smi, VAR_NOSANITIZETHREAD) \
V(GrowableObjectArray, UntaggedGrowableObjectArray, length, Smi, VAR) \
V(TypedDataBase, UntaggedTypedDataBase, length, Smi, FINAL) \
V(TypedDataView, UntaggedTypedDataView, offset_in_bytes, Smi, FINAL) \
@@ -535,6 +535,10 @@ class Slot : public ZoneAllocated {
return MayContainInnerPointerBit::decode(flags_);
}
bool is_no_sanitize_thread() const {
return IsNoSanitizeThreadBit::decode(flags_);
}
// Type information about values that can be read from this slot.
CompileType type() const { return type_; }
@@ -635,6 +639,8 @@ class Slot : public ZoneAllocated {
BitField<decltype(flags_), bool, IsNonTaggedBit::kNextBit, 1>;
using HasUntaggedInstanceBit =
BitField<decltype(flags_), bool, MayContainInnerPointerBit::kNextBit, 1>;
using IsNoSanitizeThreadBit =
BitField<decltype(flags_), bool, HasUntaggedInstanceBit::kNextBit, 1>;
friend class SlotCache;
};
runtime/vm/compiler/frontend/kernel_to_il.cc
+2 -1
View File
@@ -1247,7 +1247,8 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod(
break;
case MethodRecognizer::kRecord_fieldNames:
body += LoadObjectStore();
body += LoadNativeField(Slot::ObjectStore_record_field_names());
body += LoadNativeField(Slot::ObjectStore_record_field_names(), false,
compiler::Assembler::kAcquire);
body += LoadLocal(parsed_function_->RawParameterVariable(0));
body += LoadNativeField(Slot::Record_shape());
body += IntConstant(compiler::target::RecordShape::kFieldNamesIndexShift);
runtime/vm/compiler/runtime_offsets_extracted.h
+299 -299
View File
File diff suppressed because it is too large Load Diff
runtime/vm/heap/marker.cc
+1 -1
View File
@@ -73,7 +73,7 @@ class MarkingVisitor : public ObjectPointerVisitor {
// Reset the next pointer in the weak property.
cur_weak->untag()->next_seen_by_gc_ = WeakProperty::null();
if (raw_key->IsImmediateObject() || raw_key->untag()->IsMarked()) {
ObjectPtr raw_val = cur_weak->untag()->value();
ObjectPtr raw_val = cur_weak->untag()->value_ignore_race();
if (!raw_val->IsImmediateObject() && !raw_val->untag()->IsMarked()) {
more_to_mark = true;
}
runtime/vm/kernel_loader.cc
+8
View File
@@ -1032,6 +1032,8 @@ void KernelLoader::FinishTopLevelClassLoading(
field.set_is_extension_member(is_extension_member);
field.set_is_extension_type_member(is_extension_type_member);
field.set_is_shared(SharedPragma::decode(pragma_bits));
field.set_is_no_sanitize_thread(
NoSanitizeThreadPragma::decode(pragma_bits));
const AbstractType& type = T.BuildType(); // read type.
field.SetFieldType(type);
ReadInferredType(field, field_offset + library_kernel_offset_);
@@ -1459,6 +1461,8 @@ void KernelLoader::FinishClassLoading(const Class& klass,
field.set_is_extension_member(is_extension_member);
field.set_is_extension_type_member(is_extension_type_member);
field.set_is_shared(SharedPragma::decode(pragma_bits));
field.set_is_no_sanitize_thread(
NoSanitizeThreadPragma::decode(pragma_bits));
ReadInferredType(field, field_offset + library_kernel_offset_);
CheckForInitializer(field);
// Static fields with initializers are implicitly late.
@@ -1769,6 +1773,10 @@ void KernelLoader::ReadVMAnnotations(const Library& library,
}
*pragma_bits = SharedPragma::update(true, *pragma_bits);
}
if (constant_reader.IsStringConstant(name_index,
"vm:no-sanitize-thread")) {
*pragma_bits = NoSanitizeThreadPragma::update(true, *pragma_bits);
}
if (constant_reader.IsStringConstant(name_index,
"dyn-module:extendable")) {
*pragma_bits = DynModuleExtendablePragma::update(true, *pragma_bits);
runtime/vm/kernel_loader.h
+3 -1
View File
@@ -233,8 +233,10 @@ class KernelLoader : public ValueObject {
using FfiNativePragma =
BitField<uint32_t, bool, DeeplyImmutablePragma::kNextBit>;
using SharedPragma = BitField<uint32_t, bool, FfiNativePragma::kNextBit>;
using DynModuleExtendablePragma =
using NoSanitizeThreadPragma =
BitField<uint32_t, bool, SharedPragma::kNextBit>;
using DynModuleExtendablePragma =
BitField<uint32_t, bool, NoSanitizeThreadPragma::kNextBit>;
using DynModuleImplicitlyExtendablePragma =
BitField<uint32_t, bool, DynModuleExtendablePragma::kNextBit>;
using DynModuleCanBeOverriddenPragma =
runtime/vm/object.cc
+3 -4
View File
@@ -27392,7 +27392,7 @@ void RegExp::set_num_bracket_expressions(intptr_t value) const {
}
void RegExp::set_capture_name_map(const Array& array) const {
untag()->set_capture_name_map(array.ptr());
untag()->set_capture_name_map<std::memory_order_release>(array.ptr());
}
RegExpPtr RegExp::New(Zone* zone, Heap::Space space) {
@@ -28511,8 +28511,7 @@ RecordShape RecordShape::Register(Thread* thread,
IsolateGroup* isolate_group = thread->isolate_group();
ObjectStore* object_store = isolate_group->object_store();
if (object_store->record_field_names<std::memory_order_acquire>() ==
Array::null()) {
if (object_store->record_field_names() == Array::null()) {
// First-time initialization.
SafepointWriteRwLocker ml(thread, isolate_group->program_lock());
if (object_store->record_field_names() == Array::null()) {
@@ -28525,7 +28524,7 @@ RecordShape RecordShape::Register(Thread* thread,
object_store->set_record_field_names_map(map.Release());
const auto& table = Array::Handle(zone, Array::New(16));
table.SetAt(0, Object::empty_array());
object_store->set_record_field_names<std::memory_order_release>(table);
object_store->set_record_field_names(table);
}
}
runtime/vm/object.h
+12 -1
View File
@@ -4537,6 +4537,13 @@ class Field : public Object {
}
bool is_shared() const { return untag()->kind_bits_.Read<SharedBit>(); }
void set_is_no_sanitize_thread(bool value) const {
untag()->kind_bits_.UpdateBool<NoSanitizeThreadBit>(value);
}
bool is_no_sanitize_thread() const {
return untag()->kind_bits_.Read<NoSanitizeThreadBit>();
}
#if defined(DART_DYNAMIC_MODULES)
bool is_declared_in_bytecode() const;
#else
@@ -4948,6 +4955,8 @@ class Field : public Object {
using SharedBit = BitField<decltype(UntaggedField::kind_bits_),
bool,
HasInitializerBit::kNextBit>;
using NoSanitizeThreadBit =
BitField<decltype(UntaggedField::kind_bits_), bool, SharedBit::kNextBit>;
// Force this field's guard to be dynamic and deoptimize dependent code.
void ForceDynamicGuardedCidAndLength() const;
@@ -12966,7 +12975,9 @@ class RegExp : public Instance {
intptr_t num_bracket_expressions() const {
return untag()->num_bracket_expressions_;
}
ArrayPtr capture_name_map() const { return untag()->capture_name_map(); }
ArrayPtr capture_name_map() const {
return untag()->capture_name_map<std::memory_order_acquire>();
}
TypedDataPtr bytecode(bool is_one_byte, bool sticky) const {
if (sticky) {
runtime/vm/object_store.h
+1 -1
View File
@@ -169,7 +169,7 @@ class ObjectPointerVisitor;
RW(Array, closure_functions_table) \
RW(GrowableObjectArray, pending_classes) \
RW(Array, record_field_names_map) \
ARW_RELAXED(Array, record_field_names) \
ARW_AR(Array, record_field_names) \
RW(Instance, stack_overflow) \
RW(Instance, out_of_memory) \
RW(Function, growable_list_factory) \
runtime/vm/raw_object.h
+11 -1
View File
@@ -584,6 +584,13 @@ class UntaggedObject {
->load(order);
return static_cast<type>(v.Decompress(heap_base()));
}
template <typename type, typename compressed_type>
NO_SANITIZE_THREAD type
LoadCompressedPointerIgnoreRace(compressed_type const* addr) const {
compressed_type v =
*reinterpret_cast<compressed_type*>(const_cast<compressed_type*>(addr));
return static_cast<type>(v.Decompress(heap_base()));
}
uword heap_base() const {
return reinterpret_cast<uword>(this) & kHeapBaseMask;
@@ -907,6 +914,9 @@ inline intptr_t ObjectPtr::GetClassId() const {
type name() const { \
return LoadCompressedPointer<type, Compressed##type, order>(&name##_); \
} \
type name##_ignore_race() const { \
return LoadCompressedPointerIgnoreRace<type, Compressed##type>(&name##_); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
StoreCompressedPointer<type, Compressed##type, order>(&name##_, value); \
@@ -1603,7 +1613,7 @@ class UntaggedField : public UntaggedObject {
int8_t static_type_exactness_state_;
// static, final, const, has initializer....
AtomicBitFieldContainer<uint16_t> kind_bits_;
AtomicBitFieldContainer<uint32_t> kind_bits_;
#if !defined(DART_PRECOMPILED_RUNTIME)
// for instance fields, the offset in words in the target architecture
runtime/vm/runtime_entry.cc
+40
View File
@@ -4884,6 +4884,36 @@ extern "C" void __tsan_atomic64_store(uint64_t* addr,
int order) {
UNREACHABLE();
}
extern "C" void __tsan_read1(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_read2(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_read4(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_read8(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_read16(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_write1(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_write2(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_write4(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_write8(void* addr) {
UNREACHABLE();
}
extern "C" void __tsan_write16(void* addr) {
UNREACHABLE();
}
#endif
// These runtime entries are defined even when not using MSAN / TSAN to keep
@@ -4895,5 +4925,15 @@ DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Load, 2, __tsan_atomic32_load);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic32Store, 3, __tsan_atomic32_store);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Load, 2, __tsan_atomic64_load);
DEFINE_LEAF_RUNTIME_ENTRY(TsanAtomic64Store, 3, __tsan_atomic64_store);
DEFINE_LEAF_RUNTIME_ENTRY(TsanRead1, 1, __tsan_read1);
DEFINE_LEAF_RUNTIME_ENTRY(TsanRead2, 1, __tsan_read2);
DEFINE_LEAF_RUNTIME_ENTRY(TsanRead4, 1, __tsan_read4);
DEFINE_LEAF_RUNTIME_ENTRY(TsanRead8, 1, __tsan_read8);
DEFINE_LEAF_RUNTIME_ENTRY(TsanRead16, 1, __tsan_read16);
DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite1, 1, __tsan_write1);
DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite2, 1, __tsan_write2);
DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite4, 1, __tsan_write4);
DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite8, 1, __tsan_write8);
DEFINE_LEAF_RUNTIME_ENTRY(TsanWrite16, 1, __tsan_write16);
} // namespace dart
runtime/vm/runtime_entry_list.h
+10
View File
@@ -131,6 +131,16 @@ namespace dart {
V(void, TsanAtomic32Store, void*, uint32_t, int) \
V(uint64_t, TsanAtomic64Load, void*, int) \
V(void, TsanAtomic64Store, void*, uint64_t, int) \
V(void, TsanRead1, void*) \
V(void, TsanRead2, void*) \
V(void, TsanRead4, void*) \
V(void, TsanRead8, void*) \
V(void, TsanRead16, void*) \
V(void, TsanWrite1, void*) \
V(void, TsanWrite2, void*) \
V(void, TsanWrite4, void*) \
V(void, TsanWrite8, void*) \
V(void, TsanWrite16, void*) \
V(bool, TryDoubleAsInteger, Thread*) \
V(void*, MemoryMove, void*, const void*, size_t)
sdk/lib/_internal/vm/lib/function.dart
+2
View File
@@ -51,5 +51,7 @@ final class _Closure implements Function {
// This initializer makes _hash field nullable even without constructor
// compilation.
@pragma("vm:entry-point")
// Harmless race lazily computing the hash.
@pragma("vm:no-sanitize-thread")
var _hash = null;
}