diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index 3b41386dda3..eeaab460200 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -475,7 +475,9 @@ void Assembler::LoadObjectHelper(Register dst, bool is_unique) { ASSERT(IsOriginalObject(object)); word offset = 0; - if (target::CanLoadFromThread(object, &offset)) { + if (IsSameObject(compiler::NullObject(), object)) { + mov(dst, NULL_REG); + } else if (target::CanLoadFromThread(object, &offset)) { ldr(dst, Address(THR, offset)); } else if (CanLoadFromObjectPool(object)) { const int32_t offset = target::ObjectPool::element_offset( @@ -499,7 +501,9 @@ void Assembler::LoadUniqueObject(Register dst, const Object& object) { void Assembler::CompareObject(Register reg, const Object& object) { ASSERT(IsOriginalObject(object)); word offset = 0; - if (target::CanLoadFromThread(object, &offset)) { + if (IsSameObject(compiler::NullObject(), object)) { + CompareRegisters(reg, NULL_REG); + } else if (target::CanLoadFromThread(object, &offset)) { ldr(TMP, Address(THR, offset)); CompareRegisters(reg, TMP); } else if (CanLoadFromObjectPool(object)) { @@ -1083,8 +1087,12 @@ void Assembler::StoreIntoObjectNoBarrier(Register object, ASSERT(IsOriginalObject(value)); ASSERT(IsNotTemporaryScopedHandle(value)); // No store buffer update. - LoadObject(TMP2, value); - str(TMP2, dest); + if (IsSameObject(compiler::NullObject(), value)) { + str(NULL_REG, dest); + } else { + LoadObject(TMP2, value); + str(TMP2, dest); + } } void Assembler::StoreIntoObjectOffsetNoBarrier(Register object, @@ -1182,6 +1190,12 @@ void Assembler::RestoreCodePointer() { CheckCodePointer(); } +void Assembler::RestorePinnedRegisters() { + ldr(BARRIER_MASK, + compiler::Address(THR, target::Thread::write_barrier_mask_offset())); + ldr(NULL_REG, compiler::Address(THR, target::Thread::object_null_offset())); +} + void Assembler::CheckCodePointer() { #ifdef DEBUG if (!FLAG_check_code_pointer) { diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 4018c882a1c..719b520b340 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -1515,8 +1515,12 @@ class Assembler : public AssemblerBase { uint32_t offset); void PushObject(const Object& object) { - LoadObject(TMP, object); - Push(TMP); + if (IsSameObject(compiler::NullObject(), object)) { + Push(NULL_REG); + } else { + LoadObject(TMP, object); + Push(TMP); + } } void PushImmediate(int64_t immediate) { LoadImmediate(TMP, immediate); @@ -1556,6 +1560,10 @@ class Assembler : public AssemblerBase { void CheckCodePointer(); void RestoreCodePointer(); + // Restores the values of the registers that are blocked to cache some values + // e.g. BARRIER_MASK and NULL_REG. + void RestorePinnedRegisters(); + void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister); void EnterOsrFrame(intptr_t extra_size, Register new_pp = kNoRegister); void LeaveDartFrame(RestorePP restore_pp = kRestoreCallerPP); diff --git a/runtime/vm/compiler/assembler/assembler_arm64_test.cc b/runtime/vm/compiler/assembler/assembler_arm64_test.cc index 13a69a8b046..310e20e3138 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64_test.cc @@ -2398,15 +2398,18 @@ static void EnterTestFrame(Assembler* assembler) { __ Push(CODE_REG); __ Push(THR); __ Push(BARRIER_MASK); + __ Push(NULL_REG); __ TagAndPushPP(); __ ldr(CODE_REG, Address(R0, VMHandles::kOffsetOfRawPtrInHandle)); __ mov(THR, R1); __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); + __ ldr(NULL_REG, Address(THR, Thread::object_null_offset())); __ LoadPoolPointer(PP); } static void LeaveTestFrame(Assembler* assembler) { __ PopAndUntagPP(); + __ Pop(NULL_REG); __ Pop(BARRIER_MASK); __ Pop(THR); __ Pop(CODE_REG); @@ -2481,6 +2484,39 @@ ASSEMBLER_TEST_RUN(LoadObjectNull, test) { EXPECT_EQ(Object::null(), test->InvokeWithCodeAndThread()); } +// PushObject null. +ASSEMBLER_TEST_GENERATE(PushObjectNull, assembler) { + __ SetupDartSP(); + EnterTestFrame(assembler); + __ PushObject(Object::null_object()); + __ Pop(R0); + LeaveTestFrame(assembler); + __ RestoreCSP(); + __ ret(); +} + +ASSEMBLER_TEST_RUN(PushObjectNull, test) { + EXPECT_EQ(Object::null(), test->InvokeWithCodeAndThread()); +} + +// CompareObject null. +ASSEMBLER_TEST_GENERATE(CompareObjectNull, assembler) { + __ SetupDartSP(); + EnterTestFrame(assembler); + __ LoadObject(R0, Object::bool_true()); + __ LoadObject(R1, Object::bool_false()); + __ ldr(R2, Address(THR, Thread::object_null_offset())); + __ CompareObject(R2, Object::null_object()); + __ csel(R0, R0, R1, EQ); + LeaveTestFrame(assembler); + __ RestoreCSP(); + __ ret(); +} + +ASSEMBLER_TEST_RUN(CompareObjectNull, test) { + EXPECT_EQ(Bool::True().raw(), test->InvokeWithCodeAndThread()); +} + ASSEMBLER_TEST_GENERATE(LoadObjectTrue, assembler) { __ SetupDartSP(); EnterTestFrame(assembler); diff --git a/runtime/vm/compiler/assembler/disassembler_arm64.cc b/runtime/vm/compiler/assembler/disassembler_arm64.cc index 7ed5a443c15..0a75a1716c6 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm64.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm64.cc @@ -86,7 +86,7 @@ void ARM64Decoder::Print(const char* str) { static const char* reg_names[kNumberOfCpuRegisters] = { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "ip0", "ip1", "r18", "sp", "r20", "r21", - "r22", "r23", "r24", "r25", "thr", "pp", "ctx", "fp", "lr", "r31", + "nr", "r23", "r24", "r25", "thr", "pp", "ctx", "fp", "lr", "r31", }; // Print the register name according to the active name converter. diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index bd504f70dea..5f4adbd91fe 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -940,10 +940,8 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ blr(TMP); } - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - compiler::Address( - THR, compiler::target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Although PP is a callee-saved register, it may have been moved by the GC. __ LeaveDartFrame(compiler::kRestoreCallerPP); @@ -1102,10 +1100,8 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // Now that we have THR, we can set CSP. __ SetupCSPFromThread(THR); - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - compiler::Address( - THR, compiler::target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Save the current VMTag on the stack. __ LoadFromOffset(R0, THR, compiler::target::Thread::vm_tag_offset()); diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h index 7210759bfb1..6fe7f0659f9 100644 --- a/runtime/vm/compiler/runtime_offsets_extracted.h +++ b/runtime/vm/compiler/runtime_offsets_extracted.h @@ -197,12 +197,12 @@ static constexpr dart::compiler::target::word Thread_array_write_barrier_entry_point_offset = 204; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 88; -static constexpr dart::compiler::target::word - Thread_bootstrap_native_wrapper_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 256; static constexpr dart::compiler::target::word Thread_bool_false_offset = 112; static constexpr dart::compiler::target::word Thread_bool_true_offset = 108; +static constexpr dart::compiler::target::word + Thread_bootstrap_native_wrapper_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 208; static constexpr dart::compiler::target::word @@ -553,12 +553,12 @@ static constexpr dart::compiler::target::word Thread_array_write_barrier_entry_point_offset = 392; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 176; -static constexpr dart::compiler::target::word - Thread_bootstrap_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 496; static constexpr dart::compiler::target::word Thread_bool_false_offset = 208; static constexpr dart::compiler::target::word Thread_bool_true_offset = 200; +static constexpr dart::compiler::target::word + Thread_bootstrap_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 400; static constexpr dart::compiler::target::word @@ -909,12 +909,12 @@ static constexpr dart::compiler::target::word Thread_array_write_barrier_entry_point_offset = 204; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 88; -static constexpr dart::compiler::target::word - Thread_bootstrap_native_wrapper_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 256; static constexpr dart::compiler::target::word Thread_bool_false_offset = 112; static constexpr dart::compiler::target::word Thread_bool_true_offset = 108; +static constexpr dart::compiler::target::word + Thread_bootstrap_native_wrapper_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 208; static constexpr dart::compiler::target::word @@ -1252,26 +1252,26 @@ static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8; static constexpr dart::compiler::target::word Thread_AllocateArray_entry_point_offset = 576; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 1352; + 1344; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 1360; + 1352; static constexpr dart::compiler::target::word Thread_array_write_barrier_code_offset = 224; static constexpr dart::compiler::target::word Thread_array_write_barrier_entry_point_offset = 392; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 176; -static constexpr dart::compiler::target::word - Thread_bootstrap_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 496; static constexpr dart::compiler::target::word Thread_bool_false_offset = 208; static constexpr dart::compiler::target::word Thread_bool_true_offset = 200; +static constexpr dart::compiler::target::word + Thread_bootstrap_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 400; static constexpr dart::compiler::target::word Thread_call_to_runtime_stub_offset = 264; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1424; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1416; static constexpr dart::compiler::target::word Thread_optimize_entry_offset = 456; static constexpr dart::compiler::target::word Thread_optimize_stub_offset = 312; @@ -1287,7 +1287,7 @@ static constexpr dart::compiler::target::word Thread_end_offset = 128; static constexpr dart::compiler::target::word Thread_enter_safepoint_stub_offset = 360; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 1392; + 1384; static constexpr dart::compiler::target::word Thread_exit_safepoint_stub_offset = 368; static constexpr dart::compiler::target::word @@ -1307,7 +1307,7 @@ static constexpr dart::compiler::target::word Thread_float_not_address_offset = static constexpr dart::compiler::target::word Thread_float_zerow_address_offset = 568; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 1368; + 1360; static constexpr dart::compiler::target::word Thread_interpret_call_entry_point_offset = 504; static constexpr dart::compiler::target::word @@ -1342,11 +1342,11 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_object_null_offset = 192; static constexpr dart::compiler::target::word Thread_predefined_symbols_address_offset = 512; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1376; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1368; static constexpr dart::compiler::target::word - Thread_saved_shadow_call_stack_offset = 1384; + Thread_saved_shadow_call_stack_offset = 1376; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 1400; + 1392; static constexpr dart::compiler::target::word Thread_slow_type_test_stub_offset = 344; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72; @@ -1378,7 +1378,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 96; static constexpr dart::compiler::target::word Thread_callback_code_offset = - 1408; + 1400; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 16; static constexpr dart::compiler::target::word TwoByteString_data_offset = 16; @@ -1414,7 +1414,7 @@ static constexpr dart::compiler::target::word Thread_write_barrier_wrappers_thread_offset[] = { 1176, 1184, 1192, 1200, 1208, 1216, 1224, 1232, 1240, 1248, 1256, 1264, 1272, 1280, 1288, -1, -1, -1, -1, 1296, 1304, 1312, - 1320, 1328, 1336, 1344, -1, -1, -1, -1, -1, -1}; + -1, 1320, 1328, 1336, -1, -1, -1, -1, -1, -1}; static constexpr dart::compiler::target::word Array_header_size = 24; static constexpr dart::compiler::target::word Context_header_size = 24; static constexpr dart::compiler::target::word Double_InstanceSize = 16; diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 52fc21cb9e0..afebf6d8b3a 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -147,9 +147,8 @@ void StubCodeCompiler::GenerateCallToRuntimeStub(Assembler* assembler) { __ mov(SP, CSP); __ mov(CSP, R25); - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Retval is next to 1st argument. // Mark that the thread is executing Dart code. @@ -624,9 +623,8 @@ static void GenerateCallNativeWithWrapperStub(Assembler* assembler, __ mov(SP, CSP); __ mov(CSP, R25); - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartCompiledTagId); @@ -1179,13 +1177,12 @@ void StubCodeCompiler::GenerateAllocateArrayStub(Assembler* assembler) { __ AddImmediate(R1, R0, target::Array::data_offset() - kHeapObjectTag); // R1: iterator which initially points to the start of the variable // data area to be initialized. - __ LoadObject(TMP, NullObject()); Label loop, done; __ Bind(&loop); // TODO(cshapiro): StoreIntoObjectNoBarrier __ CompareRegisters(R1, R7); __ b(&done, CS); - __ str(TMP, Address(R1)); // Store if unsigned lower. + __ str(NULL_REG, Address(R1)); // Store if unsigned lower. __ AddImmediate(R1, target::kWordSize); __ b(&loop); // Loop until R1 == R7. __ Bind(&done); @@ -1257,9 +1254,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) { __ mov(THR, R3); } - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Save the current VMTag on the stack. __ LoadFromOffset(R4, THR, target::Thread::vm_tag_offset()); @@ -1409,9 +1405,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeFromBytecodeStub( __ mov(THR, R3); } - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Save the current VMTag on the stack. __ LoadFromOffset(R4, THR, target::Thread::vm_tag_offset()); @@ -2583,9 +2578,8 @@ void StubCodeCompiler::GenerateInterpretCallStub(Assembler* assembler) { __ mov(SP, CSP); __ mov(CSP, R25); - // Refresh write barrier mask. - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartCompiledTagId); @@ -3072,8 +3066,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub(Assembler* assembler) { #elif defined(USING_SHADOW_CALL_STACK) #error Unimplemented #endif - __ ldr(BARRIER_MASK, - Address(THR, target::Thread::write_barrier_mask_offset())); + // Refresh pinned registers values (inc. write barrier mask and null object). + __ RestorePinnedRegisters(); // Set the tag. __ LoadImmediate(R2, VMTag::kDartCompiledTagId); __ StoreToOffset(R2, THR, target::Thread::vm_tag_offset()); diff --git a/runtime/vm/constants_arm64.cc b/runtime/vm/constants_arm64.cc index 95a52c0c06d..7248ef17030 100644 --- a/runtime/vm/constants_arm64.cc +++ b/runtime/vm/constants_arm64.cc @@ -10,7 +10,7 @@ namespace arch_arm64 { const char* cpu_reg_names[kNumberOfCpuRegisters] = { "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18", "r19", "r20", "r21", - "r22", "r23", "r24", "ip0", "ip1", "pp", "ctx", "fp", "lr", "r31", + "nr", "r23", "r24", "ip0", "ip1", "pp", "ctx", "fp", "lr", "r31", }; const char* fpu_reg_names[kNumberOfFpuRegisters] = { diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index cca7779ba1b..d141f818faf 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -36,7 +36,7 @@ enum Register { R19 = 19, R20 = 20, R21 = 21, - R22 = 22, + R22 = 22, // NULL_REG R23 = 23, R24 = 24, R25 = 25, @@ -124,6 +124,7 @@ const Register THR = R26; // Caches current thread in generated code. const Register CALLEE_SAVED_TEMP = R19; const Register CALLEE_SAVED_TEMP2 = R20; const Register BARRIER_MASK = R28; +const Register NULL_REG = R22; // Caches NullObject() value. // ABI for catch-clause entry point. const Register kExceptionObjectReg = R0; @@ -173,9 +174,9 @@ const int kAbiPreservedFpuRegCount = 8; const intptr_t kReservedCpuRegisters = (1 << SPREG) | // Dart SP (1 << FPREG) | (1 << TMP) | (1 << TMP2) | (1 << PP) | (1 << THR) | - (1 << LR) | (1 << BARRIER_MASK) | (1 << R31) | // C++ SP + (1 << LR) | (1 << BARRIER_MASK) | (1 << NULL_REG) | (1 << R31) | // C++ SP (1 << R18); -constexpr intptr_t kNumberOfReservedCpuRegisters = 10; +constexpr intptr_t kNumberOfReservedCpuRegisters = 11; // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = kAllCpuRegistersList & ~kReservedCpuRegisters;