diff --git a/runtime/tests/vm/dart/profiler_memmove_test.dart b/runtime/tests/vm/dart/profiler_memmove_test.dart new file mode 100644 index 00000000000..91c79114abe --- /dev/null +++ b/runtime/tests/vm/dart/profiler_memmove_test.dart @@ -0,0 +1,29 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// VMOptions=--profiler --profile-vm=true +// VMOptions=--profiler --profile-vm=false + +import "dart:typed_data"; + +main() { + Uint8List a = new Uint8List(2 << 20); + Uint8List b = new Uint8List(2 << 20); + + for (int i = 0; i < a.length; i++) { + a[i] = i; + } + + for (int i = 0; i < 1000; i++) { + b.setRange(0, a.length, a); // Implemented via memmove. + a.setRange(0, b.length, b); // Implemented via memmove. + } + + for (int i = 0; i < a.length; i++) { + if (a[i] != (i & 0xFF)) throw "A"; + } + for (int i = 0; i < a.length; i++) { + if (b[i] != (i & 0xFF)) throw "A"; + } +} diff --git a/runtime/vm/compiler/backend/constant_propagator.cc b/runtime/vm/compiler/backend/constant_propagator.cc index 101bd1b5265..6c1cd743e83 100644 --- a/runtime/vm/compiler/backend/constant_propagator.cc +++ b/runtime/vm/compiler/backend/constant_propagator.cc @@ -776,7 +776,7 @@ void ConstantPropagator::VisitFfiCall(FfiCallInstr* instr) { SetValue(instr, non_constant_); } -void ConstantPropagator::VisitCCall(CCallInstr* instr) { +void ConstantPropagator::VisitLeafRuntimeCall(LeafRuntimeCallInstr* instr) { SetValue(instr, non_constant_); } diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 666154b219a..e8ee6f4fb63 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -7923,7 +7923,7 @@ DEFINE_BACKEND(LoadThread, (Register out)) { __ MoveRegister(out, THR); } -LocationSummary* CCallInstr::MakeLocationSummaryInternal( +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummaryInternal( Zone* zone, const RegList temps) const { LocationSummary* summary = @@ -7966,7 +7966,7 @@ LocationSummary* CCallInstr::MakeLocationSummaryInternal( return summary; } -CCallInstr::CCallInstr( +LeafRuntimeCallInstr::LeafRuntimeCallInstr( Representation return_representation, const ZoneGrowableArray& argument_representations, const compiler::ffi::NativeCallingConvention& native_calling_convention, @@ -7985,7 +7985,7 @@ CCallInstr::CCallInstr( #endif } -CCallInstr* CCallInstr::Make( +LeafRuntimeCallInstr* LeafRuntimeCallInstr::Make( Zone* zone, Representation return_representation, const ZoneGrowableArray& argument_representations, @@ -7996,13 +7996,14 @@ CCallInstr* CCallInstr::Make( const auto& native_calling_convention = compiler::ffi::NativeCallingConvention::FromSignature( zone, native_function_type); - return new (zone) CCallInstr(return_representation, argument_representations, - native_calling_convention, std::move(inputs)); + return new (zone) + LeafRuntimeCallInstr(return_representation, argument_representations, + native_calling_convention, std::move(inputs)); } -void CCallInstr::EmitParamMoves(FlowGraphCompiler* compiler, - Register saved_fp, - Register temp0) { +void LeafRuntimeCallInstr::EmitParamMoves(FlowGraphCompiler* compiler, + Register saved_fp, + Register temp0) { if (native_calling_convention_.StackTopInBytes() == 0) { return; } diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 56896878d1f..f7a616809e8 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -435,7 +435,7 @@ struct InstrAttrs { M(AssertBoolean, _) \ M(ClosureCall, _) \ M(FfiCall, _) \ - M(CCall, kNoGC) \ + M(LeafRuntimeCall, kNoGC) \ M(InstanceCall, _) \ M(PolymorphicInstanceCall, _) \ M(DispatchTableCall, _) \ @@ -6132,15 +6132,15 @@ class FfiCallInstr : public VariadicDefinition { }; // Has the target address in a register passed as the last input in IL. -class CCallInstr : public VariadicDefinition { +class LeafRuntimeCallInstr : public VariadicDefinition { public: - static CCallInstr* Make( + static LeafRuntimeCallInstr* Make( Zone* zone, Representation return_representation, const ZoneGrowableArray& argument_representations, InputsArray&& inputs); - DECLARE_INSTRUCTION(CCall) + DECLARE_INSTRUCTION(LeafRuntimeCall) LocationSummary* MakeLocationSummaryInternal(Zone* zone, const RegList temps) const; @@ -6197,10 +6197,10 @@ class CCallInstr : public VariadicDefinition { PRINT_OPERANDS_TO_SUPPORT - DECLARE_CUSTOM_SERIALIZATION(CCallInstr) + DECLARE_CUSTOM_SERIALIZATION(LeafRuntimeCallInstr) private: - CCallInstr( + LeafRuntimeCallInstr( Representation return_representation, const ZoneGrowableArray& argument_representations, const compiler::ffi::NativeCallingConvention& native_calling_convention, @@ -6211,7 +6211,7 @@ class CCallInstr : public VariadicDefinition { const ZoneGrowableArray& argument_representations_; // Not serialized. const compiler::ffi::NativeCallingConvention& native_calling_convention_; - DISALLOW_COPY_AND_ASSIGN(CCallInstr); + DISALLOW_COPY_AND_ASSIGN(LeafRuntimeCallInstr); }; class DebugStepCheckInstr : public TemplateInstruction<0, NoThrow> { diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 70b281dd814..9fb5058cc94 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -2042,15 +2042,16 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { #define R(r) (1 << r) -LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, - bool is_optimizing) const { +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummary( + Zone* zone, + bool is_optimizing) const { constexpr Register saved_fp = CallingConventions::kSecondNonArgumentRegister; return MakeLocationSummaryInternal(zone, (R(saved_fp))); } #undef R -void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { +void LeafRuntimeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register saved_fp = locs()->temp(0).reg(); const Register temp0 = TMP; @@ -2062,7 +2063,12 @@ void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { EmitParamMoves(compiler, saved_fp, temp0); const Register target_address = locs()->in(TargetAddressIndex()).reg(); + __ str(target_address, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ CallCFunction(target_address); + __ LoadImmediate(temp0, VMTag::kDartTagId); + __ str(temp0, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ LeaveCFrame(); } diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 7ea48499c8f..c21595076b0 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -1805,8 +1805,9 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { #define R(r) (1 << r) -LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, - bool is_optimizing) const { +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummary( + Zone* zone, + bool is_optimizing) const { // Compare FfiCallInstr's use of kFfiAnyNonAbiRegister. constexpr Register saved_csp = CallingConventions::kFfiAnyNonAbiRegister; ASSERT(IsAbiPreservedRegister(saved_csp)); @@ -1815,7 +1816,7 @@ LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, #undef R -void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { +void LeafRuntimeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register saved_fp = TMP2; const Register temp0 = TMP; const Register saved_csp = locs()->temp(0).reg(); @@ -1831,7 +1832,12 @@ void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { EmitParamMoves(compiler, saved_fp, temp0); const Register target_address = locs()->in(TargetAddressIndex()).reg(); + __ str(target_address, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ CallCFunction(target_address); + __ LoadImmediate(temp0, VMTag::kDartTagId); + __ str(temp0, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); // We don't use the DartSP, we leave the frame after this immediately. // However, we need set CSP to a 16 byte aligned value far above the SP. diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index eb6c8bfa981..4c65fc3c958 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -1460,8 +1460,9 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { #define R(r) (1 << r) -LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, - bool is_optimizing) const { +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummary( + Zone* zone, + bool is_optimizing) const { constexpr Register saved_fp = CallingConventions::kSecondNonArgumentRegister; constexpr Register temp0 = CallingConventions::kFfiAnyNonAbiRegister; static_assert(saved_fp < temp0, "Unexpected ordering of registers in set."); @@ -1470,7 +1471,7 @@ LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, #undef R -void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { +void LeafRuntimeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register saved_fp = locs()->temp(0).reg(); const Register temp0 = locs()->temp(1).reg(); @@ -1481,7 +1482,10 @@ void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { EmitParamMoves(compiler, saved_fp, temp0); const Register target_address = locs()->in(TargetAddressIndex()).reg(); + __ movl(compiler::Assembler::VMTagAddress(), target_address); __ CallCFunction(target_address); + __ movl(compiler::Assembler::VMTagAddress(), + compiler::Immediate(VMTag::kDartTagId)); __ LeaveCFrame(); } diff --git a/runtime/vm/compiler/backend/il_printer.cc b/runtime/vm/compiler/backend/il_printer.cc index 1ddb60591cd..3de08eec7b3 100644 --- a/runtime/vm/compiler/backend/il_printer.cc +++ b/runtime/vm/compiler/backend/il_printer.cc @@ -1407,7 +1407,7 @@ void FfiCallInstr::PrintOperandsTo(BaseTextBuffer* f) const { } } -void CCallInstr::PrintOperandsTo(BaseTextBuffer* f) const { +void LeafRuntimeCallInstr::PrintOperandsTo(BaseTextBuffer* f) const { f->AddString("target_address="); InputAt(TargetAddressIndex())->PrintTo(f); for (intptr_t i = 0, n = argument_representations_.length(); i < n; ++i) { diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index eee8e1b4f32..68a22f049b6 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -1948,8 +1948,9 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { #define R(r) (1 << r) -LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, - bool is_optimizing) const { +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummary( + Zone* zone, + bool is_optimizing) const { constexpr Register saved_fp = CallingConventions::kSecondNonArgumentRegister; constexpr Register temp0 = CallingConventions::kFfiAnyNonAbiRegister; static_assert(saved_fp < temp0, "Unexpected ordering of registers in set."); @@ -1960,7 +1961,7 @@ LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, #undef R -void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { +void LeafRuntimeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register saved_fp = locs()->temp(0).reg(); const Register temp0 = locs()->temp(1).reg(); @@ -1974,7 +1975,12 @@ void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register target_address = locs()->in(TargetAddressIndex()).reg(); // I.e., no use of A3/A4/A5. RELEASE_ASSERT(native_calling_convention_.argument_locations().length() < 4); + __ sx(target_address, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ CallCFunction(target_address); + __ li(temp0, VMTag::kDartTagId); + __ sx(temp0, + compiler::Address(THR, compiler::target::Thread::vm_tag_offset())); __ LeaveCFrame(); // Also restores PP=A5. } diff --git a/runtime/vm/compiler/backend/il_serializer.cc b/runtime/vm/compiler/backend/il_serializer.cc index a4d59308e53..803887219c9 100644 --- a/runtime/vm/compiler/backend/il_serializer.cc +++ b/runtime/vm/compiler/backend/il_serializer.cc @@ -2164,13 +2164,13 @@ PhiInstr::PhiInstr(FlowGraphDeserializer* d) is_alive_(d->Read()), is_receiver_(d->Read()) {} -void CCallInstr::WriteTo(FlowGraphSerializer* s) { +void LeafRuntimeCallInstr::WriteTo(FlowGraphSerializer* s) { VariadicDefinition::WriteTo(s); s->Write(return_representation_); s->Write&>(argument_representations_); } -CCallInstr::CCallInstr(FlowGraphDeserializer* d) +LeafRuntimeCallInstr::LeafRuntimeCallInstr(FlowGraphDeserializer* d) : VariadicDefinition(d), return_representation_(d->Read()), argument_representations_( diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 1b013ac869e..d4c70953551 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -1696,15 +1696,16 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { #define R(r) (1 << r) -LocationSummary* CCallInstr::MakeLocationSummary(Zone* zone, - bool is_optimizing) const { +LocationSummary* LeafRuntimeCallInstr::MakeLocationSummary( + Zone* zone, + bool is_optimizing) const { constexpr Register saved_fp = CallingConventions::kSecondNonArgumentRegister; return MakeLocationSummaryInternal(zone, (R(saved_fp))); } #undef R -void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { +void LeafRuntimeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register saved_fp = locs()->temp(0).reg(); const Register temp0 = TMP; @@ -1717,7 +1718,10 @@ void CCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { EmitParamMoves(compiler, saved_fp, temp0); const Register target_address = locs()->in(TargetAddressIndex()).reg(); + __ movq(compiler::Assembler::VMTagAddress(), target_address); __ CallCFunction(target_address); + __ movq(compiler::Assembler::VMTagAddress(), + compiler::Immediate(VMTag::kDartTagId)); __ LeaveCFrame(); } diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index e3626a961af..93bada89f9f 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -416,7 +416,7 @@ Fragment FlowGraphBuilder::FfiCall( return body; } -Fragment FlowGraphBuilder::CallRuntimeEntry( +Fragment FlowGraphBuilder::CallLeafRuntimeEntry( const RuntimeEntry& entry, Representation return_representation, const ZoneGrowableArray& argument_representations) { @@ -427,7 +427,7 @@ Fragment FlowGraphBuilder::CallRuntimeEntry( const intptr_t num_arguments = argument_representations.length() + 1; InputsArray arguments = GetArguments(num_arguments); - auto* const call = CCallInstr::Make( + auto* const call = LeafRuntimeCallInstr::Make( Z, return_representation, argument_representations, std::move(arguments)); Push(call); body <<= call; @@ -2230,7 +2230,7 @@ Fragment FlowGraphBuilder::BuildTypedDataMemMove(const Function& function, arg_reps->Add(size_rep); // memmove(dest, src, n) call_memmove += - CallRuntimeEntry(kMemoryMoveRuntimeEntry, kUntagged, *arg_reps); + CallLeafRuntimeEntry(kMemoryMoveRuntimeEntry, kUntagged, *arg_reps); // The returned address is unused. call_memmove += Drop(); call_memmove += DropTemporary(&length_in_bytes); @@ -5259,7 +5259,8 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToNative( arg_reps->Add(kUntagged); // Allocate a new handle in the top handle scope. - body += CallRuntimeEntry(kAllocateHandleRuntimeEntry, kUntagged, *arg_reps); + body += + CallLeafRuntimeEntry(kAllocateHandleRuntimeEntry, kUntagged, *arg_reps); LocalVariable* handle = MakeTemporary("handle"); @@ -5464,8 +5465,8 @@ Fragment FlowGraphBuilder::FfiCallFunctionBody( body += LoadThread(); // argument. arg_reps->Add(kUntagged); - body += - CallRuntimeEntry(kEnterHandleScopeRuntimeEntry, kUntagged, *arg_reps); + body += CallLeafRuntimeEntry(kEnterHandleScopeRuntimeEntry, kUntagged, + *arg_reps); } // Allocate typed data before FfiCall and pass it in to ffi call if needed. @@ -5530,8 +5531,8 @@ Fragment FlowGraphBuilder::FfiCallFunctionBody( code += LoadThread(); // argument. arg_reps->Add(kUntagged); - code += - CallRuntimeEntry(kExitHandleScopeRuntimeEntry, kUntagged, *arg_reps); + code += CallLeafRuntimeEntry(kExitHandleScopeRuntimeEntry, kUntagged, + *arg_reps); code += Drop(); return code; }; diff --git a/runtime/vm/compiler/frontend/kernel_to_il.h b/runtime/vm/compiler/frontend/kernel_to_il.h index 75c09a46e29..aded1e8cb86 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.h +++ b/runtime/vm/compiler/frontend/kernel_to_il.h @@ -211,7 +211,7 @@ class FlowGraphBuilder : public BaseFlowGraphBuilder { Fragment FfiCall(const compiler::ffi::CallMarshaller& marshaller, bool is_leaf); - Fragment CallRuntimeEntry( + Fragment CallLeafRuntimeEntry( const RuntimeEntry& entry, Representation return_representation, const ZoneGrowableArray& argument_representations);