dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Switchable call site updates without RunWithStoppedMutators.

Browse Source 
TEST=iso-stress
Change-Id: Idb00490737e22348d6609291f052ac6785a42424
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/441864
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2025-10-01 10:06:20 -07:00
committed by Commit Queue
parent 99a370942b
commit 158398dc26
11 changed files with 61 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/code_patcher.h
-5
View File
@@ -77,11 +77,6 @@ class CodePatcher : public AllStatic {
const Code& caller_code,
const Object& data,
const Code& target);
static void PatchSwitchableCallAtWithMutatorsStopped(Thread* thread,
uword return_address,
const Code& caller_code,
const Object& data,
const Code& target);
static ObjectPtr GetSwitchableCallDataAt(uword return_address,
const Code& caller_code);
static ObjectPtr GetSwitchableCallTargetAt(uword return_address,
runtime/vm/code_patcher_arm.cc
+9 -18
View File
@@ -86,28 +86,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
auto thread = Thread::Current();
// Ensure all threads are suspended as we update data and target pair.
thread->isolate_group()->RunWithStoppedMutators([&]() {
PatchSwitchableCallAtWithMutatorsStopped(thread, return_address,
caller_code, data, target);
});
}
void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
Thread* thread,
uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
// First update target to a stub that does not read 'data' so that concurrent
// Dart execution cannot observe the new stub with the old data or the old
// stub with the new data.
if (FLAG_precompiled_mode) {
BareSwitchableCallPattern call(return_address);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
} else {
SwitchableCallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
}
}
runtime/vm/code_patcher_arm64.cc
+9 -18
View File
@@ -114,28 +114,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
auto thread = Thread::Current();
// Ensure all threads are suspended as we update data and target pair.
thread->isolate_group()->RunWithStoppedMutators([&]() {
PatchSwitchableCallAtWithMutatorsStopped(thread, return_address,
caller_code, data, target);
});
}
void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
Thread* thread,
uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
// First update target to a stub that does not read 'data' so that concurrent
// Dart execution cannot observe the new stub with the old data or the old
// stub with the new data.
if (FLAG_precompiled_mode) {
BareSwitchableCallPattern call(return_address);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
} else {
SwitchableCallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
}
}
runtime/vm/code_patcher_riscv.cc
+9 -18
View File
@@ -123,28 +123,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
auto thread = Thread::Current();
// Ensure all threads are suspended as we update data and target pair.
thread->isolate_group()->RunWithStoppedMutators([&]() {
PatchSwitchableCallAtWithMutatorsStopped(thread, return_address,
caller_code, data, target);
});
}
void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
Thread* thread,
uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
// First update target to a stub that does not read 'data' so that concurrent
// Dart execution cannot observe the new stub with the old data or the old
// stub with the new data.
if (FLAG_precompiled_mode) {
BareSwitchableCallPattern call(return_address);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
} else {
SwitchableCallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
}
}
runtime/vm/code_patcher_x64.cc
+13 -22
View File
@@ -296,7 +296,7 @@ class SwitchableCallBase : public ValueObject {
ObjectPtr data() const { return object_pool_.ObjectAt(data_index()); }
void SetData(const Object& data) const {
void SetDataRelease(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(data_index(), data);
// No need to flush the instruction cache, since the code is not modified.
@@ -358,7 +358,7 @@ class SwitchableCall : public SwitchableCallBase {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index_)).IsCode());
}
void SetTarget(const Code& target) const {
void SetTargetRelease(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(target_index(), target);
// No need to flush the instruction cache, since the code is not modified.
@@ -435,10 +435,10 @@ class BareSwitchableCall : public SwitchableCallBase {
ObjectPool::EntryType::kImmediate);
}
void SetTarget(const Code& target) const {
void SetTargetRelease(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_index()) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
object_pool_.SetRawValueAt<std::memory_order_release>(
target_index(), target.MonomorphicEntryPoint());
}
@@ -518,28 +518,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
auto thread = Thread::Current();
// Ensure all threads are suspended as we update data and target pair.
thread->isolate_group()->RunWithStoppedMutators([&]() {
PatchSwitchableCallAtWithMutatorsStopped(thread, return_address,
caller_code, data, target);
});
}
void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped(
Thread* thread,
uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
// First update target to a stub that does not read 'data' so that concurrent
// Dart execution cannot observe the new stub with the old data or the old
// stub with the new data.
if (FLAG_precompiled_mode) {
BareSwitchableCall call(return_address);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
} else {
SwitchableCall call(return_address, caller_code);
call.SetData(data);
call.SetTarget(target);
call.SetTargetRelease(StubCode::SwitchableCallMiss());
call.SetDataRelease(data);
call.SetTargetRelease(target);
}
}
runtime/vm/instructions_arm.cc
+4 -4
View File
@@ -247,7 +247,7 @@ ObjectPtr SwitchableCallPatternBase::data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void SwitchableCallPatternBase::SetData(const Object& data) const {
void SwitchableCallPatternBase::SetDataRelease(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
@@ -271,7 +271,7 @@ ObjectPtr SwitchableCallPattern::target() const {
return object_pool_.ObjectAt(target_pool_index_);
}
void SwitchableCallPattern::SetTarget(const Code& target) const {
void SwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
@@ -297,10 +297,10 @@ uword BareSwitchableCallPattern::target_entry() const {
return object_pool_.RawValueAt(target_pool_index_);
}
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
object_pool_.SetRawValueAt<std::memory_order_release>(
target_pool_index_, target.MonomorphicEntryPoint());
}
runtime/vm/instructions_arm.h
+3 -3
View File
@@ -124,7 +124,7 @@ class SwitchableCallPatternBase : public ValueObject {
explicit SwitchableCallPatternBase(const ObjectPool& object_pool);
ObjectPtr data() const;
void SetData(const Object& data) const;
void SetDataRelease(const Object& data) const;
protected:
const ObjectPool& object_pool_;
@@ -144,7 +144,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase {
SwitchableCallPattern(uword pc, const Code& code);
ObjectPtr target() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern);
@@ -159,7 +159,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase {
explicit BareSwitchableCallPattern(uword pc);
uword target_entry() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern);
runtime/vm/instructions_arm64.cc
+4 -4
View File
@@ -431,7 +431,7 @@ ObjectPtr SwitchableCallPatternBase::data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void SwitchableCallPatternBase::SetData(const Object& data) const {
void SwitchableCallPatternBase::SetDataRelease(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
@@ -457,7 +457,7 @@ ObjectPtr SwitchableCallPattern::target() const {
return object_pool_.ObjectAt(target_pool_index_);
}
void SwitchableCallPattern::SetTarget(const Code& target) const {
void SwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
@@ -484,10 +484,10 @@ uword BareSwitchableCallPattern::target_entry() const {
return object_pool_.RawValueAt(target_pool_index_);
}
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
object_pool_.SetRawValueAt<std::memory_order_release>(
target_pool_index_, target.MonomorphicEntryPoint());
}
runtime/vm/instructions_arm64.h
+3 -3
View File
@@ -134,7 +134,7 @@ class SwitchableCallPatternBase : public ValueObject {
explicit SwitchableCallPatternBase(const ObjectPool& object_pool);
ObjectPtr data() const;
void SetData(const Object& data) const;
void SetDataRelease(const Object& data) const;
protected:
const ObjectPool& object_pool_;
@@ -154,7 +154,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase {
SwitchableCallPattern(uword pc, const Code& code);
ObjectPtr target() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern);
@@ -169,7 +169,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase {
explicit BareSwitchableCallPattern(uword pc);
uword target_entry() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern);
runtime/vm/instructions_riscv.cc
+4 -4
View File
@@ -330,7 +330,7 @@ ObjectPtr SwitchableCallPatternBase::data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void SwitchableCallPatternBase::SetData(const Object& data) const {
void SwitchableCallPatternBase::SetDataRelease(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
@@ -360,7 +360,7 @@ ObjectPtr SwitchableCallPattern::target() const {
return object_pool_.ObjectAt(target_pool_index_);
}
void SwitchableCallPattern::SetTarget(const Code& target) const {
void SwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
@@ -390,10 +390,10 @@ uword BareSwitchableCallPattern::target_entry() const {
return object_pool_.RawValueAt(target_pool_index_);
}
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
object_pool_.SetRawValueAt<std::memory_order_release>(
target_pool_index_, target.MonomorphicEntryPoint());
}
runtime/vm/instructions_riscv.h
+3 -3
View File
@@ -121,7 +121,7 @@ class SwitchableCallPatternBase : public ValueObject {
explicit SwitchableCallPatternBase(const ObjectPool& object_pool);
ObjectPtr data() const;
void SetData(const Object& data) const;
void SetDataRelease(const Object& data) const;
protected:
const ObjectPool& object_pool_;
@@ -141,7 +141,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase {
SwitchableCallPattern(uword pc, const Code& code);
ObjectPtr target() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern);
@@ -156,7 +156,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase {
explicit BareSwitchableCallPattern(uword pc);
uword target_entry() const;
void SetTarget(const Code& target) const;
void SetTargetRelease(const Code& target) const;
private:
DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern);