diff --git a/runtime/vm/code_patcher.h b/runtime/vm/code_patcher.h index 8b8aab0fd49..d5a010e178a 100644 --- a/runtime/vm/code_patcher.h +++ b/runtime/vm/code_patcher.h @@ -77,11 +77,6 @@ class CodePatcher : public AllStatic { const Code& caller_code, const Object& data, const Code& target); - static void PatchSwitchableCallAtWithMutatorsStopped(Thread* thread, - uword return_address, - const Code& caller_code, - const Object& data, - const Code& target); static ObjectPtr GetSwitchableCallDataAt(uword return_address, const Code& caller_code); static ObjectPtr GetSwitchableCallTargetAt(uword return_address, diff --git a/runtime/vm/code_patcher_arm.cc b/runtime/vm/code_patcher_arm.cc index f918be3906b..421adcfe6cd 100644 --- a/runtime/vm/code_patcher_arm.cc +++ b/runtime/vm/code_patcher_arm.cc @@ -86,28 +86,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address, const Code& caller_code, const Object& data, const Code& target) { - auto thread = Thread::Current(); - // Ensure all threads are suspended as we update data and target pair. - thread->isolate_group()->RunWithStoppedMutators([&]() { - PatchSwitchableCallAtWithMutatorsStopped(thread, return_address, - caller_code, data, target); - }); -} - -void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( - Thread* thread, - uword return_address, - const Code& caller_code, - const Object& data, - const Code& target) { + // First update target to a stub that does not read 'data' so that concurrent + // Dart execution cannot observe the new stub with the old data or the old + // stub with the new data. if (FLAG_precompiled_mode) { BareSwitchableCallPattern call(return_address); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } else { SwitchableCallPattern call(return_address, caller_code); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } } diff --git a/runtime/vm/code_patcher_arm64.cc b/runtime/vm/code_patcher_arm64.cc index 10518a894ab..9740f0f75f2 100644 --- a/runtime/vm/code_patcher_arm64.cc +++ b/runtime/vm/code_patcher_arm64.cc @@ -114,28 +114,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address, const Code& caller_code, const Object& data, const Code& target) { - auto thread = Thread::Current(); - // Ensure all threads are suspended as we update data and target pair. - thread->isolate_group()->RunWithStoppedMutators([&]() { - PatchSwitchableCallAtWithMutatorsStopped(thread, return_address, - caller_code, data, target); - }); -} - -void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( - Thread* thread, - uword return_address, - const Code& caller_code, - const Object& data, - const Code& target) { + // First update target to a stub that does not read 'data' so that concurrent + // Dart execution cannot observe the new stub with the old data or the old + // stub with the new data. if (FLAG_precompiled_mode) { BareSwitchableCallPattern call(return_address); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } else { SwitchableCallPattern call(return_address, caller_code); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } } diff --git a/runtime/vm/code_patcher_riscv.cc b/runtime/vm/code_patcher_riscv.cc index b85d165fa2e..f5f95b6ac13 100644 --- a/runtime/vm/code_patcher_riscv.cc +++ b/runtime/vm/code_patcher_riscv.cc @@ -123,28 +123,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address, const Code& caller_code, const Object& data, const Code& target) { - auto thread = Thread::Current(); - // Ensure all threads are suspended as we update data and target pair. - thread->isolate_group()->RunWithStoppedMutators([&]() { - PatchSwitchableCallAtWithMutatorsStopped(thread, return_address, - caller_code, data, target); - }); -} - -void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( - Thread* thread, - uword return_address, - const Code& caller_code, - const Object& data, - const Code& target) { + // First update target to a stub that does not read 'data' so that concurrent + // Dart execution cannot observe the new stub with the old data or the old + // stub with the new data. if (FLAG_precompiled_mode) { BareSwitchableCallPattern call(return_address); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } else { SwitchableCallPattern call(return_address, caller_code); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } } diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc index c5abf415fe9..f60a0062dc3 100644 --- a/runtime/vm/code_patcher_x64.cc +++ b/runtime/vm/code_patcher_x64.cc @@ -296,7 +296,7 @@ class SwitchableCallBase : public ValueObject { ObjectPtr data() const { return object_pool_.ObjectAt(data_index()); } - void SetData(const Object& data) const { + void SetDataRelease(const Object& data) const { ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode()); object_pool_.SetObjectAt(data_index(), data); // No need to flush the instruction cache, since the code is not modified. @@ -358,7 +358,7 @@ class SwitchableCall : public SwitchableCallBase { ASSERT(Object::Handle(object_pool_.ObjectAt(target_index_)).IsCode()); } - void SetTarget(const Code& target) const { + void SetTargetRelease(const Code& target) const { ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode()); object_pool_.SetObjectAt(target_index(), target); // No need to flush the instruction cache, since the code is not modified. @@ -435,10 +435,10 @@ class BareSwitchableCall : public SwitchableCallBase { ObjectPool::EntryType::kImmediate); } - void SetTarget(const Code& target) const { + void SetTargetRelease(const Code& target) const { ASSERT(object_pool_.TypeAt(target_index()) == ObjectPool::EntryType::kImmediate); - object_pool_.SetRawValueAt( + object_pool_.SetRawValueAt( target_index(), target.MonomorphicEntryPoint()); } @@ -518,28 +518,19 @@ void CodePatcher::PatchSwitchableCallAt(uword return_address, const Code& caller_code, const Object& data, const Code& target) { - auto thread = Thread::Current(); - // Ensure all threads are suspended as we update data and target pair. - thread->isolate_group()->RunWithStoppedMutators([&]() { - PatchSwitchableCallAtWithMutatorsStopped(thread, return_address, - caller_code, data, target); - }); -} - -void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( - Thread* thread, - uword return_address, - const Code& caller_code, - const Object& data, - const Code& target) { + // First update target to a stub that does not read 'data' so that concurrent + // Dart execution cannot observe the new stub with the old data or the old + // stub with the new data. if (FLAG_precompiled_mode) { BareSwitchableCall call(return_address); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } else { SwitchableCall call(return_address, caller_code); - call.SetData(data); - call.SetTarget(target); + call.SetTargetRelease(StubCode::SwitchableCallMiss()); + call.SetDataRelease(data); + call.SetTargetRelease(target); } } diff --git a/runtime/vm/instructions_arm.cc b/runtime/vm/instructions_arm.cc index 205f86e3717..de009e207dc 100644 --- a/runtime/vm/instructions_arm.cc +++ b/runtime/vm/instructions_arm.cc @@ -247,7 +247,7 @@ ObjectPtr SwitchableCallPatternBase::data() const { return object_pool_.ObjectAt(data_pool_index_); } -void SwitchableCallPatternBase::SetData(const Object& data) const { +void SwitchableCallPatternBase::SetDataRelease(const Object& data) const { ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode()); object_pool_.SetObjectAt(data_pool_index_, data); } @@ -271,7 +271,7 @@ ObjectPtr SwitchableCallPattern::target() const { return object_pool_.ObjectAt(target_pool_index_); } -void SwitchableCallPattern::SetTarget(const Code& target) const { +void SwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode()); object_pool_.SetObjectAt(target_pool_index_, target); @@ -297,10 +297,10 @@ uword BareSwitchableCallPattern::target_entry() const { return object_pool_.RawValueAt(target_pool_index_); } -void BareSwitchableCallPattern::SetTarget(const Code& target) const { +void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(object_pool_.TypeAt(target_pool_index_) == ObjectPool::EntryType::kImmediate); - object_pool_.SetRawValueAt( + object_pool_.SetRawValueAt( target_pool_index_, target.MonomorphicEntryPoint()); } diff --git a/runtime/vm/instructions_arm.h b/runtime/vm/instructions_arm.h index c3328a59c78..9f406e2cd34 100644 --- a/runtime/vm/instructions_arm.h +++ b/runtime/vm/instructions_arm.h @@ -124,7 +124,7 @@ class SwitchableCallPatternBase : public ValueObject { explicit SwitchableCallPatternBase(const ObjectPool& object_pool); ObjectPtr data() const; - void SetData(const Object& data) const; + void SetDataRelease(const Object& data) const; protected: const ObjectPool& object_pool_; @@ -144,7 +144,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase { SwitchableCallPattern(uword pc, const Code& code); ObjectPtr target() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); @@ -159,7 +159,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase { explicit BareSwitchableCallPattern(uword pc); uword target_entry() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern); diff --git a/runtime/vm/instructions_arm64.cc b/runtime/vm/instructions_arm64.cc index c0f5db4d38b..c18dc714cbe 100644 --- a/runtime/vm/instructions_arm64.cc +++ b/runtime/vm/instructions_arm64.cc @@ -431,7 +431,7 @@ ObjectPtr SwitchableCallPatternBase::data() const { return object_pool_.ObjectAt(data_pool_index_); } -void SwitchableCallPatternBase::SetData(const Object& data) const { +void SwitchableCallPatternBase::SetDataRelease(const Object& data) const { ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode()); object_pool_.SetObjectAt(data_pool_index_, data); } @@ -457,7 +457,7 @@ ObjectPtr SwitchableCallPattern::target() const { return object_pool_.ObjectAt(target_pool_index_); } -void SwitchableCallPattern::SetTarget(const Code& target) const { +void SwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode()); object_pool_.SetObjectAt(target_pool_index_, target); @@ -484,10 +484,10 @@ uword BareSwitchableCallPattern::target_entry() const { return object_pool_.RawValueAt(target_pool_index_); } -void BareSwitchableCallPattern::SetTarget(const Code& target) const { +void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(object_pool_.TypeAt(target_pool_index_) == ObjectPool::EntryType::kImmediate); - object_pool_.SetRawValueAt( + object_pool_.SetRawValueAt( target_pool_index_, target.MonomorphicEntryPoint()); } diff --git a/runtime/vm/instructions_arm64.h b/runtime/vm/instructions_arm64.h index 6b3d373546d..90bb6cd2be2 100644 --- a/runtime/vm/instructions_arm64.h +++ b/runtime/vm/instructions_arm64.h @@ -134,7 +134,7 @@ class SwitchableCallPatternBase : public ValueObject { explicit SwitchableCallPatternBase(const ObjectPool& object_pool); ObjectPtr data() const; - void SetData(const Object& data) const; + void SetDataRelease(const Object& data) const; protected: const ObjectPool& object_pool_; @@ -154,7 +154,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase { SwitchableCallPattern(uword pc, const Code& code); ObjectPtr target() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); @@ -169,7 +169,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase { explicit BareSwitchableCallPattern(uword pc); uword target_entry() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern); diff --git a/runtime/vm/instructions_riscv.cc b/runtime/vm/instructions_riscv.cc index ee81f345beb..7a97ce5ec11 100644 --- a/runtime/vm/instructions_riscv.cc +++ b/runtime/vm/instructions_riscv.cc @@ -330,7 +330,7 @@ ObjectPtr SwitchableCallPatternBase::data() const { return object_pool_.ObjectAt(data_pool_index_); } -void SwitchableCallPatternBase::SetData(const Object& data) const { +void SwitchableCallPatternBase::SetDataRelease(const Object& data) const { ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode()); object_pool_.SetObjectAt(data_pool_index_, data); } @@ -360,7 +360,7 @@ ObjectPtr SwitchableCallPattern::target() const { return object_pool_.ObjectAt(target_pool_index_); } -void SwitchableCallPattern::SetTarget(const Code& target) const { +void SwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode()); object_pool_.SetObjectAt(target_pool_index_, target); @@ -390,10 +390,10 @@ uword BareSwitchableCallPattern::target_entry() const { return object_pool_.RawValueAt(target_pool_index_); } -void BareSwitchableCallPattern::SetTarget(const Code& target) const { +void BareSwitchableCallPattern::SetTargetRelease(const Code& target) const { ASSERT(object_pool_.TypeAt(target_pool_index_) == ObjectPool::EntryType::kImmediate); - object_pool_.SetRawValueAt( + object_pool_.SetRawValueAt( target_pool_index_, target.MonomorphicEntryPoint()); } diff --git a/runtime/vm/instructions_riscv.h b/runtime/vm/instructions_riscv.h index 99a9ce0edf9..af6925dd464 100644 --- a/runtime/vm/instructions_riscv.h +++ b/runtime/vm/instructions_riscv.h @@ -121,7 +121,7 @@ class SwitchableCallPatternBase : public ValueObject { explicit SwitchableCallPatternBase(const ObjectPool& object_pool); ObjectPtr data() const; - void SetData(const Object& data) const; + void SetDataRelease(const Object& data) const; protected: const ObjectPool& object_pool_; @@ -141,7 +141,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase { SwitchableCallPattern(uword pc, const Code& code); ObjectPtr target() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); @@ -156,7 +156,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase { explicit BareSwitchableCallPattern(uword pc); uword target_entry() const; - void SetTarget(const Code& target) const; + void SetTargetRelease(const Code& target) const; private: DISALLOW_COPY_AND_ASSIGN(BareSwitchableCallPattern);