diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index 6add2969554..3b77f84223a 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -48,6 +48,9 @@ dart/inline_stack_frame_test: Fail # Skip until we stabilize language tests. *: Skip +[ $arch == x64 && $mode == debug ] +cc/FindCodeObject: Timeout # Issue 13144 + [ $arch == simarm ] dart/isolate_mirror_local_test: Skip diff --git a/runtime/vm/assembler_x64.cc b/runtime/vm/assembler_x64.cc index 7b1315d5eca..8d9d716dec4 100644 --- a/runtime/vm/assembler_x64.cc +++ b/runtime/vm/assembler_x64.cc @@ -65,6 +65,47 @@ void CPUFeatures::InitOnce() { #undef __ +Assembler::Assembler(bool use_far_branches) + : buffer_(), + object_pool_(GrowableObjectArray::Handle()), + patchable_pool_entries_(), + prologue_offset_(-1), + comments_() { + // Far branching mode is only needed and implemented for MIPS and ARM. + ASSERT(!use_far_branches); + if (Isolate::Current() != Dart::vm_isolate()) { + object_pool_ = GrowableObjectArray::New(Heap::kOld); + + // These objects and labels need to be accessible through every pool-pointer + // at the same index. + object_pool_.Add(Object::Handle(), Heap::kOld); + patchable_pool_entries_.Add(kNotPatchable); + + object_pool_.Add(Bool::True(), Heap::kOld); + patchable_pool_entries_.Add(kNotPatchable); + + object_pool_.Add(Bool::False(), Heap::kOld); + patchable_pool_entries_.Add(kNotPatchable); + + if (StubCode::UpdateStoreBuffer_entry() != NULL) { + FindExternalLabel(&StubCode::UpdateStoreBufferLabel(), kNotPatchable); + patchable_pool_entries_.Add(kNotPatchable); + } else { + object_pool_.Add(Object::Handle(), Heap::kOld); + patchable_pool_entries_.Add(kNotPatchable); + } + + if (StubCode::CallToRuntime_entry() != NULL) { + FindExternalLabel(&StubCode::CallToRuntimeLabel(), kNotPatchable); + patchable_pool_entries_.Add(kNotPatchable); + } else { + object_pool_.Add(Object::Handle(), Heap::kOld); + patchable_pool_entries_.Add(kNotPatchable); + } + } +} + + void Assembler::InitializeMemoryWithBreakpoints(uword data, int length) { memset(reinterpret_cast(data), Instr::kBreakPointInstruction, length); } @@ -95,25 +136,45 @@ void Assembler::call(Label* label) { } +void Assembler::LoadExternalLabel(Register dst, + const ExternalLabel* label, + Patchability patchable, + Register pp) { + const int32_t offset = + Array::element_offset(FindExternalLabel(label, patchable)); + LoadWordFromPoolOffset(dst, pp, offset - kHeapObjectTag); +} + + void Assembler::call(const ExternalLabel* label) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); + { // Encode movq(TMP, Immediate(label->address())), but always as imm64. + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitRegisterREX(TMP, REX_W); + EmitUint8(0xB8 | (TMP & 7)); + EmitInt64(label->address()); + } + call(TMP); +} + + +void Assembler::CallPatchable(const ExternalLabel* label) { intptr_t call_start = buffer_.GetPosition(); - - // Encode movq(TMP, Immediate(label->address())), but always as imm64. - EmitRegisterREX(TMP, REX_W); - EmitUint8(0xB8 | (TMP & 7)); - EmitInt64(label->address()); - - // Encode call(TMP). - Operand operand(TMP); - EmitOperandREX(2, operand, REX_NONE); - EmitUint8(0xFF); - EmitOperand(2, operand); - + LoadExternalLabel(TMP, label, kPatchable, PP); + call(TMP); ASSERT((buffer_.GetPosition() - call_start) == kCallExternalLabelSize); } +void Assembler::Call(const ExternalLabel* label, Register pp) { + if (Isolate::Current() == Dart::vm_isolate()) { + call(label); + } else { + LoadExternalLabel(TMP, label, kNotPatchable, pp); + call(TMP); + } +} + + void Assembler::pushq(Register reg) { AssemblerBuffer::EnsureCapacity ensured(&buffer_); EmitRegisterREX(reg, REX_NONE); @@ -1960,6 +2021,15 @@ void Assembler::j(Condition condition, const ExternalLabel* label) { } +void Assembler::J(Condition condition, const ExternalLabel* label, + Register pp) { + Label no_jump; + j(static_cast(condition ^ 1), &no_jump); // Negate condition. + Jmp(label, pp); + Bind(&no_jump); +} + + void Assembler::jmp(Register reg) { AssemblerBuffer::EnsureCapacity ensured(&buffer_); Operand operand(reg); @@ -1994,24 +2064,30 @@ void Assembler::jmp(Label* label, bool near) { void Assembler::jmp(const ExternalLabel* label) { - AssemblerBuffer::EnsureCapacity ensured(&buffer_); + { // Encode movq(TMP, Immediate(label->address())), but always as imm64. + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitRegisterREX(TMP, REX_W); + EmitUint8(0xB8 | (TMP & 7)); + EmitInt64(label->address()); + } + jmp(TMP); +} + + +void Assembler::JmpPatchable(const ExternalLabel* label, Register pp) { intptr_t call_start = buffer_.GetPosition(); - - // Encode movq(TMP, Immediate(label->address())), but always as imm64. - EmitRegisterREX(TMP, REX_W); - EmitUint8(0xB8 | (TMP & 7)); - EmitInt64(label->address()); - - // Encode jmp(TMP). - Operand operand(TMP); - EmitOperandREX(4, operand, REX_NONE); - EmitUint8(0xFF); - EmitOperand(4, operand); - + LoadExternalLabel(TMP, label, kPatchable, pp); + jmp(TMP); ASSERT((buffer_.GetPosition() - call_start) == kCallExternalLabelSize); } +void Assembler::Jmp(const ExternalLabel* label, Register pp) { + LoadExternalLabel(TMP, label, kNotPatchable, pp); + jmp(TMP); +} + + void Assembler::lock() { AssemblerBuffer::EnsureCapacity ensured(&buffer_); EmitUint8(0xF0); @@ -2091,49 +2167,103 @@ void Assembler::Drop(intptr_t stack_elements) { } -void Assembler::LoadObject(Register dst, const Object& object) { - if (object.IsSmi() || object.InVMHeap()) { - movq(dst, Immediate(reinterpret_cast(object.raw()))); +intptr_t Assembler::FindObject(const Object& obj, Patchability patchable) { + // The object pool cannot be used in the vm isolate. + ASSERT(Isolate::Current() != Dart::vm_isolate()); + ASSERT(!object_pool_.IsNull()); + + // TODO(zra): This can be slow. Add a hash map from obj.raw() to + // object pool indexes to speed lookup. + for (int i = 0; i < object_pool_.Length(); i++) { + if ((object_pool_.At(i) == obj.raw()) && + (patchable_pool_entries_[i] != kPatchable)) { + return i; + } + } + object_pool_.Add(obj, Heap::kOld); + patchable_pool_entries_.Add(patchable); + return object_pool_.Length() - 1; +} + + +intptr_t Assembler::FindExternalLabel(const ExternalLabel* label, + Patchability patchable) { + // The object pool cannot be used in the vm isolate. + ASSERT(Isolate::Current() != Dart::vm_isolate()); + ASSERT(!object_pool_.IsNull()); + const uword address = label->address(); + ASSERT(Utils::IsAligned(address, 4)); + // The address is stored in the object array as a RawSmi. + const Smi& smi = Smi::Handle(reinterpret_cast(address)); + if (patchable == kNotPatchable) { + return FindObject(smi, kNotPatchable); + } + // If the call is patchable, do not reuse an existing entry since each + // reference may be patched independently. + object_pool_.Add(smi, Heap::kOld); + patchable_pool_entries_.Add(patchable); + return object_pool_.Length() - 1; +} + + +bool Assembler::CanLoadFromObjectPool(const Object& object) { + return !object.IsSmi() && // Not a Smi + // Not in the VMHeap, OR is one of the VMHeap objects we put in every + // object pool. + (!object.InVMHeap() || (object.raw() == Object::null()) || + (object.raw() == Bool::True().raw()) || + (object.raw() == Bool::False().raw())) && + object.IsNotTemporaryScopedHandle() && + object.IsOld(); +} + + +void Assembler::LoadWordFromPoolOffset(Register dst, Register pp, + int32_t offset) { + // This sequence must be of fixed size. AddressBaseImm32 + // forces the address operand to use a fixed-size imm32 encoding. + movq(dst, Address::AddressBaseImm32(pp, offset)); +} + + +void Assembler::LoadObject(Register dst, const Object& object, Register pp) { + if (CanLoadFromObjectPool(object)) { + const int32_t offset = + Array::element_offset(FindObject(object, kNotPatchable)); + LoadWordFromPoolOffset(dst, pp, offset - kHeapObjectTag); } else { - ASSERT(object.IsNotTemporaryScopedHandle()); - ASSERT(object.IsOld()); - AssemblerBuffer::EnsureCapacity ensured(&buffer_); - EmitRegisterREX(dst, REX_W); - EmitUint8(0xB8 | (dst & 7)); - buffer_.EmitObject(object); + movq(dst, Immediate(reinterpret_cast(object.raw()))); } } void Assembler::StoreObject(const Address& dst, const Object& object) { - if (object.IsSmi() || object.InVMHeap()) { - movq(dst, Immediate(reinterpret_cast(object.raw()))); - } else { - ASSERT(object.IsNotTemporaryScopedHandle()); - ASSERT(object.IsOld()); - LoadObject(TMP, object); + if (CanLoadFromObjectPool(object)) { + LoadObject(TMP, object, PP); movq(dst, TMP); + } else { + movq(dst, Immediate(reinterpret_cast(object.raw()))); } } void Assembler::PushObject(const Object& object) { - if (object.IsSmi() || object.InVMHeap()) { - pushq(Immediate(reinterpret_cast(object.raw()))); - } else { - LoadObject(TMP, object); + if (CanLoadFromObjectPool(object)) { + LoadObject(TMP, object, PP); pushq(TMP); + } else { + pushq(Immediate(reinterpret_cast(object.raw()))); } } void Assembler::CompareObject(Register reg, const Object& object) { - if (object.IsSmi() || object.InVMHeap()) { - cmpq(reg, Immediate(reinterpret_cast(object.raw()))); - } else { + if (CanLoadFromObjectPool(object)) { ASSERT(reg != TMP); - LoadObject(TMP, object); + LoadObject(TMP, object, PP); cmpq(reg, TMP); + } else { + cmpq(reg, Immediate(reinterpret_cast(object.raw()))); } } @@ -2196,7 +2326,7 @@ void Assembler::StoreIntoObject(Register object, if (object != RAX) { movq(RAX, object); } - call(&StubCode::UpdateStoreBufferLabel()); + Call(&StubCode::UpdateStoreBufferLabel(), PP); if (value != RAX) popq(RAX); Bind(&done); } @@ -2298,6 +2428,23 @@ void Assembler::LeaveFrame() { } +void Assembler::LeaveFrameWithPP() { + movq(PP, Address(RBP, -2 * kWordSize)); + LeaveFrame(); +} + + +void Assembler::ReturnPatchable() { + // This sequence must have a fixed size so that it can be patched by the + // debugger. + intptr_t start = buffer_.GetPosition(); + LeaveFrameWithPP(); + ret(); + nop(4); + ASSERT((buffer_.GetPosition() - start) == 13); +} + + void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) { // Reserve space for arguments and align frame before entering // the C++ world. @@ -2378,18 +2525,59 @@ void Assembler::CallRuntime(const RuntimeEntry& entry, } +void Assembler::LoadPoolPointer(Register pp) { + Label next; + call(&next); + Bind(&next); + + // Load new pool pointer. + const intptr_t object_pool_pc_dist = + Instructions::HeaderSize() - Instructions::object_pool_offset() + + CodeSize(); + popq(pp); + movq(pp, Address(pp, -object_pool_pc_dist)); +} + + void Assembler::EnterDartFrame(intptr_t frame_size) { EnterFrame(0); + Label dart_entry; call(&dart_entry); Bind(&dart_entry); // The runtime system assumes that the code marker address is // kEntryPointToPcMarkerOffset bytes from the entry. If there is any code // generated before entering the frame, the address needs to be adjusted. + const intptr_t object_pool_pc_dist = + Instructions::HeaderSize() - Instructions::object_pool_offset() + + CodeSize(); const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize(); if (offset != 0) { addq(Address(RSP, 0), Immediate(offset)); } + // Save caller's pool pointer + pushq(PP); + + // Load callee's pool pointer. + movq(PP, Address(RSP, 1 * kWordSize)); + movq(PP, Address(PP, -object_pool_pc_dist - offset)); + + if (frame_size != 0) { + subq(RSP, Immediate(frame_size)); + } +} + + +void Assembler::EnterDartFrameWithInfo(intptr_t frame_size, + Register new_pp, Register new_pc) { + if (new_pc == kNoRegister) { + EnterDartFrame(0); + } else { + EnterFrame(0); + pushq(new_pc); + pushq(PP); + movq(PP, new_pp); + } if (frame_size != 0) { subq(RSP, Immediate(frame_size)); } @@ -2401,18 +2589,32 @@ void Assembler::EnterDartFrame(intptr_t frame_size) { // pointer is already set up. The PC marker is not correct for the // optimized function and there may be extra space for spill slots to // allocate. -void Assembler::EnterOsrFrame(intptr_t extra_size) { - Label dart_entry; - call(&dart_entry); - Bind(&dart_entry); - // The runtime system assumes that the code marker address is - // kEntryPointToPcMarkerOffset bytes from the entry. Since there is no - // code to set up the frame pointer, the address needs to be adjusted. - const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize(); - if (offset != 0) { - addq(Address(RSP, 0), Immediate(offset)); +void Assembler::EnterOsrFrame(intptr_t extra_size, + Register new_pp, Register new_pc) { + if (new_pc == kNoRegister) { + Label dart_entry; + call(&dart_entry); + Bind(&dart_entry); + // The runtime system assumes that the code marker address is + // kEntryPointToPcMarkerOffset bytes from the entry. Since there is no + // code to set up the frame pointer, the address needs to be adjusted. + const intptr_t object_pool_pc_dist = + Instructions::HeaderSize() - Instructions::object_pool_offset() + + CodeSize(); + const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize(); + if (offset != 0) { + addq(Address(RSP, 0), Immediate(offset)); + } + + // Load callee's pool pointer. + movq(PP, Address(RSP, 0)); + movq(PP, Address(PP, -object_pool_pc_dist - offset)); + + popq(Address(RBP, kPcMarkerSlotFromFp * kWordSize)); + } else { + movq(Address(RBP, kPcMarkerSlotFromFp * kWordSize), new_pc); + movq(PP, new_pp); } - popq(Address(RBP, kPcMarkerSlotFromFp * kWordSize)); if (extra_size != 0) { subq(RSP, Immediate(extra_size)); } @@ -2425,6 +2627,14 @@ void Assembler::EnterStubFrame() { } +void Assembler::EnterStubFrameWithPP() { + EnterFrame(0); + pushq(Immediate(0)); // Push 0 in the saved PC area for stub frames. + pushq(PP); // Save caller's pool pointer + LoadPoolPointer(PP); +} + + void Assembler::TryAllocate(const Class& cls, Label* failure, bool near_jump, @@ -2647,7 +2857,6 @@ const char* Assembler::FpuRegisterName(FpuRegister reg) { return xmm_reg_names[reg]; } - } // namespace dart #endif // defined TARGET_ARCH_X64 diff --git a/runtime/vm/assembler_x64.h b/runtime/vm/assembler_x64.h index f4c945f426a..96921ee7cd9 100644 --- a/runtime/vm/assembler_x64.h +++ b/runtime/vm/assembler_x64.h @@ -212,6 +212,20 @@ class Address : public Operand { Operand::operator=(other); return *this; } + + static Address AddressBaseImm32(Register base, int32_t disp) { + return Address(base, disp, true); + } + + private: + Address(Register base, int32_t disp, bool fixed) { + ASSERT(fixed); + SetModRM(2, base); + if ((base & 7) == RSP) { + SetSIB(TIMES_1, RSP, base); + } + SetDisp32(disp); + } }; @@ -321,14 +335,8 @@ class CPUFeatures : public AllStatic { class Assembler : public ValueObject { public: - explicit Assembler(bool use_far_branches = false) - : buffer_(), - object_pool_(GrowableObjectArray::Handle()), - prologue_offset_(-1), - comments_() { - // This mode is only needed and implemented for MIPS and ARM. - ASSERT(!use_far_branches); - } + explicit Assembler(bool use_far_branches = false); + ~Assembler() { } static const bool kNearJump = true; @@ -342,7 +350,7 @@ class Assembler : public ValueObject { void call(Label* label); void call(const ExternalLabel* label); - static const intptr_t kCallExternalLabelSize = 13; + static const intptr_t kCallExternalLabelSize = 10; void pushq(Register reg); void pushq(const Address& address); @@ -652,7 +660,17 @@ class Assembler : public ValueObject { void Drop(intptr_t stack_elements); - void LoadObject(Register dst, const Object& object); + enum Patchability { + kPatchable, + kNotPatchable, + }; + + void LoadObject(Register dst, const Object& obj, Register pp); + void JmpPatchable(const ExternalLabel* label, Register pp); + void Jmp(const ExternalLabel* label, Register pp); + void J(Condition condition, const ExternalLabel* label, Register pp); + void CallPatchable(const ExternalLabel* label); + void Call(const ExternalLabel* label, Register pp); void StoreObject(const Address& dst, const Object& obj); void PushObject(const Object& object); void CompareObject(Register reg, const Object& object); @@ -680,6 +698,8 @@ class Assembler : public ValueObject { void EnterFrame(intptr_t frame_space); void LeaveFrame(); + void LeaveFrameWithPP(); + void ReturnPatchable(); void ReserveAlignedFrameSpace(intptr_t frame_space); // Create a frame for calling into runtime that preserves all volatile @@ -731,6 +751,8 @@ class Assembler : public ValueObject { buffer_.FinalizeInstructions(region); } + void LoadPoolPointer(Register pp); + // Set up a Dart frame on entry with a frame pointer and PC information to // enable easy access to the RawInstruction object of code corresponding // to this frame. @@ -739,6 +761,7 @@ class Assembler : public ValueObject { // ret PC // saved RBP <=== RBP // pc (used to derive the RawInstruction Object of the dart code) + // saved PP // locals space <=== RSP // ..... // This code sets this up with the sequence: @@ -746,13 +769,17 @@ class Assembler : public ValueObject { // movq rbp, rsp // call L // L: + // ... + // pushq r15 // ..... void EnterDartFrame(intptr_t frame_size); + void EnterDartFrameWithInfo(intptr_t frame_size, + Register new_pp, Register new_pc); // Set up a Dart frame for a function compiled for on-stack replacement. // The frame layout is a normal Dart frame, but the frame is partially set // up on entry (it is the frame of the unoptimized code). - void EnterOsrFrame(intptr_t extra_size); + void EnterOsrFrame(intptr_t extra_size, Register new_pp, Register new_pc); // Set up a stub frame so that the stack traversal code can easily identify // a stub frame. @@ -768,8 +795,9 @@ class Assembler : public ValueObject { // pushq immediate(0) // ..... void EnterStubFrame(); + void EnterStubFrameWithPP(); - // Instruction pattern from entrypoint is used in dart frame prologs + // Instruction pattern from entrypoint is used in dart frame prologues // to set up the frame and save a PC which can be used to figure out the // RawInstruction object corresponding to the code running in the frame. // entrypoint: @@ -802,7 +830,13 @@ class Assembler : public ValueObject { private: AssemblerBuffer buffer_; - GrowableObjectArray& object_pool_; // Object pool is not used on x64. + + // Objects and jump targets. + GrowableObjectArray& object_pool_; + + // Patchability of pool entries. + GrowableArray patchable_pool_entries_; + int prologue_offset_; class CodeComment : public ZoneAllocated { @@ -822,6 +856,16 @@ class Assembler : public ValueObject { GrowableArray comments_; + intptr_t FindObject(const Object& obj, Patchability patchable); + intptr_t FindExternalLabel(const ExternalLabel* label, + Patchability patchable); + void LoadExternalLabel(Register dst, + const ExternalLabel* label, + Patchability patchable, + Register pp); + bool CanLoadFromObjectPool(const Object& object); + void LoadWordFromPoolOffset(Register dst, Register pp, int32_t offset); + inline void EmitUint8(uint8_t value); inline void EmitInt32(int32_t value); inline void EmitInt64(int64_t value); diff --git a/runtime/vm/assembler_x64_test.cc b/runtime/vm/assembler_x64_test.cc index cbad36f427d..2d148cdd9a3 100644 --- a/runtime/vm/assembler_x64_test.cc +++ b/runtime/vm/assembler_x64_test.cc @@ -200,6 +200,28 @@ ASSEMBLER_TEST_GENERATE(AddressingModes, assembler) { __ movq(RAX, Address(R13, R10, TIMES_2, 256 * kWordSize)); __ movq(RAX, Address(R13, R12, TIMES_2, 256 * kWordSize)); __ movq(RAX, Address(R13, R13, TIMES_2, 256 * kWordSize)); + + __ movq(RAX, Address::AddressBaseImm32(RSP, 0)); + __ movq(RAX, Address::AddressBaseImm32(RBP, 0)); + __ movq(RAX, Address::AddressBaseImm32(RAX, 0)); + __ movq(RAX, Address::AddressBaseImm32(R10, 0)); + __ movq(RAX, Address::AddressBaseImm32(R12, 0)); + __ movq(RAX, Address::AddressBaseImm32(R13, 0)); + __ movq(R10, Address::AddressBaseImm32(RAX, 0)); + + __ movq(RAX, Address::AddressBaseImm32(RSP, kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(RBP, kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(RAX, kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R10, kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R12, kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R13, kWordSize)); + + __ movq(RAX, Address::AddressBaseImm32(RSP, -kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(RBP, -kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(RAX, -kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R10, -kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R12, -kWordSize)); + __ movq(RAX, Address::AddressBaseImm32(R13, -kWordSize)); } @@ -2159,14 +2181,15 @@ ASSEMBLER_TEST_GENERATE(TestObjectCompare, assembler) { ObjectStore* object_store = Isolate::Current()->object_store(); const Object& obj = Object::ZoneHandle(object_store->smi_class()); Label fail; - __ LoadObject(RAX, obj); + __ EnterDartFrame(0); + __ LoadObject(RAX, obj, PP); __ CompareObject(RAX, obj); __ j(NOT_EQUAL, &fail); - __ LoadObject(RCX, obj); + __ LoadObject(RCX, obj, PP); __ CompareObject(RCX, obj); __ j(NOT_EQUAL, &fail); const Smi& smi = Smi::ZoneHandle(Smi::New(15)); - __ LoadObject(RCX, smi); + __ LoadObject(RCX, smi, PP); __ CompareObject(RCX, smi); __ j(NOT_EQUAL, &fail); __ pushq(RAX); @@ -2180,9 +2203,11 @@ ASSEMBLER_TEST_GENERATE(TestObjectCompare, assembler) { __ CompareObject(RCX, smi); __ j(NOT_EQUAL, &fail); __ movl(RAX, Immediate(1)); // OK + __ LeaveFrameWithPP(); __ ret(); __ Bind(&fail); __ movl(RAX, Immediate(0)); // Fail. + __ LeaveFrameWithPP(); __ ret(); } @@ -2393,12 +2418,14 @@ ASSEMBLER_TEST_RUN(SquareRootDouble, test) { // Called from assembler_test.cc. ASSEMBLER_TEST_GENERATE(StoreIntoObject, assembler) { + __ EnterDartFrame(0); __ pushq(CTX); __ movq(CTX, RDI); __ StoreIntoObject(RDX, FieldAddress(RDX, GrowableObjectArray::data_offset()), RSI); __ popq(CTX); + __ LeaveFrameWithPP(); __ ret(); } diff --git a/runtime/vm/code_patcher.cc b/runtime/vm/code_patcher.cc index 986ade43e01..6fda035f404 100644 --- a/runtime/vm/code_patcher.cc +++ b/runtime/vm/code_patcher.cc @@ -29,11 +29,11 @@ void CodePatcher::PatchEntry(const Code& code) { const uword patch_addr = code.GetPcForDeoptId(Isolate::kNoDeoptId, PcDescriptors::kEntryPatch); ASSERT(patch_addr != 0); - JumpPattern jmp_entry(patch_addr); + JumpPattern jmp_entry(patch_addr, code); ASSERT(!jmp_entry.IsValid()); const uword patch_buffer = code.GetPatchCodePc(); ASSERT(patch_buffer != 0); - JumpPattern jmp_patch(patch_buffer); + JumpPattern jmp_patch(patch_buffer, code); ASSERT(jmp_patch.IsValid()); const uword jump_target = jmp_patch.TargetAddress(); SwapCode(jmp_patch.pattern_length_in_bytes(), @@ -49,13 +49,13 @@ void CodePatcher::RestoreEntry(const Code& code) { const uword patch_addr = code.GetPcForDeoptId(Isolate::kNoDeoptId, PcDescriptors::kEntryPatch); ASSERT(patch_addr != 0); - JumpPattern jmp_entry(patch_addr); + JumpPattern jmp_entry(patch_addr, code); ASSERT(jmp_entry.IsValid()); const uword jump_target = jmp_entry.TargetAddress(); const uword patch_buffer = code.GetPatchCodePc(); ASSERT(patch_buffer != 0); // 'patch_buffer' contains original entry code. - JumpPattern jmp_patch(patch_buffer); + JumpPattern jmp_patch(patch_buffer, code); ASSERT(!jmp_patch.IsValid()); SwapCode(jmp_patch.pattern_length_in_bytes(), reinterpret_cast(patch_addr), @@ -72,7 +72,7 @@ bool CodePatcher::CodeIsPatchable(const Code& code) { if (patch_addr == 0) { return true; } - JumpPattern jmp_entry(patch_addr); + JumpPattern jmp_entry(patch_addr, code); if (code.Size() < (jmp_entry.pattern_length_in_bytes() * 2)) { return false; } diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc index ae307d57d42..5782527c6f7 100644 --- a/runtime/vm/code_patcher_x64.cc +++ b/runtime/vm/code_patcher_x64.cc @@ -16,53 +16,60 @@ namespace dart { // The expected pattern of a Dart unoptimized call (static and instance): -// 00: 48 bb imm64 mov RBX, ic-data -// 10: 49 bb imm64 mov R11, target_address -// 20: 41 ff d3 call R11 -// 23 <- return address +// 00: 49 8b 9f imm32 mov RBX, [PP + off] +// 07: 4d 8b 9f imm32 mov R11, [PP + off] +// 14: 41 ff d3 call R11 +// 17 <- return address class UnoptimizedCall : public ValueObject { public: - explicit UnoptimizedCall(uword return_address) - : start_(return_address - kCallPatternSize) { + UnoptimizedCall(uword return_address, const Code& code) + : start_(return_address - kCallPatternSize), + object_pool_(Array::Handle(code.ObjectPool())) { ASSERT(IsValid(return_address)); - ASSERT((kCallPatternSize - 10) == Assembler::kCallExternalLabelSize); + ASSERT((kCallPatternSize - 7) == Assembler::kCallExternalLabelSize); } - static const int kCallPatternSize = 23; + static const int kCallPatternSize = 17; static bool IsValid(uword return_address) { uint8_t* code_bytes = reinterpret_cast(return_address - kCallPatternSize); - return (code_bytes[00] == 0x48) && (code_bytes[01] == 0xBB) && - (code_bytes[10] == 0x49) && (code_bytes[11] == 0xBB) && - (code_bytes[20] == 0x41) && (code_bytes[21] == 0xFF) && - (code_bytes[22] == 0xD3); + return (code_bytes[0] == 0x49) && (code_bytes[1] == 0x8B) && + (code_bytes[2] == 0x9F) && + (code_bytes[7] == 0x4D) && (code_bytes[8] == 0x8B) && + (code_bytes[9] == 0x9F) && + (code_bytes[14] == 0x41) && (code_bytes[15] == 0xFF) && + (code_bytes[16] == 0xD3); } RawObject* ic_data() const { - return *reinterpret_cast(start_ + 0 + 2); + int index = InstructionPattern::IndexFromPPLoad(start_ + 3); + return object_pool_.At(index); } uword target() const { - return *reinterpret_cast(start_ + 10 + 2); + int index = InstructionPattern::IndexFromPPLoad(start_ + 10); + return reinterpret_cast(object_pool_.At(index)); } void set_target(uword target) const { - uword* target_addr = reinterpret_cast(start_ + 10 + 2); - *target_addr = target; - CPU::FlushICache(start_ + 10, 2 + 8); + int index = InstructionPattern::IndexFromPPLoad(start_ + 10); + const Smi& smi = Smi::Handle(reinterpret_cast(target)); + object_pool_.SetAt(index, smi); + // No need to flush the instruction cache, since the code is not modified. } private: uword start_; + const Array& object_pool_; DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedCall); }; class InstanceCall : public UnoptimizedCall { public: - explicit InstanceCall(uword return_address) - : UnoptimizedCall(return_address) { + InstanceCall(uword return_address, const Code& code) + : UnoptimizedCall(return_address, code) { #if defined(DEBUG) ICData& test_ic_data = ICData::Handle(); test_ic_data ^= ic_data(); @@ -77,8 +84,8 @@ class InstanceCall : public UnoptimizedCall { class UnoptimizedStaticCall : public UnoptimizedCall { public: - explicit UnoptimizedStaticCall(uword return_address) - : UnoptimizedCall(return_address) { + UnoptimizedStaticCall(uword return_address, const Code& code) + : UnoptimizedCall(return_address, code) { #if defined(DEBUG) ICData& test_ic_data = ICData::Handle(); test_ic_data ^= ic_data(); @@ -92,50 +99,54 @@ class UnoptimizedStaticCall : public UnoptimizedCall { // The expected pattern of a dart static call: -// mov R10, arguments_descriptor_array (10 bytes) (optional in polym. calls) -// mov R11, target_address (10 bytes) -// call R11 (3 bytes) +// 00 mov R10, arguments_descriptor_array (10 bytes) (optional in polym. calls) +// 11: 4d 8b 9f imm32 mov R11, [PP + off] +// 16: call R11 (3 bytes) // <- return address class StaticCall : public ValueObject { public: - explicit StaticCall(uword return_address) - : start_(return_address - kCallPatternSize) { + explicit StaticCall(uword return_address, const Code& code) + : start_(return_address - kCallPatternSize), + object_pool_(Array::Handle(code.ObjectPool())) { ASSERT(IsValid(return_address)); ASSERT(kCallPatternSize == Assembler::kCallExternalLabelSize); } - static const int kCallPatternSize = 13; + static const int kCallPatternSize = 10; static bool IsValid(uword return_address) { uint8_t* code_bytes = reinterpret_cast(return_address - kCallPatternSize); - return (code_bytes[00] == 0x49) && (code_bytes[01] == 0xBB) && - (code_bytes[10] == 0x41) && (code_bytes[11] == 0xFF) && - (code_bytes[12] == 0xD3); + return (code_bytes[0] == 0x4D) && (code_bytes[1] == 0x8B) && + (code_bytes[2] == 0x9F) && + (code_bytes[7] == 0x41) && (code_bytes[8] == 0xFF) && + (code_bytes[9] == 0xD3); } uword target() const { - return *reinterpret_cast(start_ + 2); + int index = InstructionPattern::IndexFromPPLoad(start_ + 3); + return reinterpret_cast(object_pool_.At(index)); } void set_target(uword target) const { - uword* target_addr = reinterpret_cast(start_ + 2); - *target_addr = target; - CPU::FlushICache(start_, 2 + 8); + int index = InstructionPattern::IndexFromPPLoad(start_ + 3); + const Smi& smi = Smi::Handle(reinterpret_cast(target)); + object_pool_.SetAt(index, smi); + // No need to flush the instruction cache, since the code is not modified. } private: uword start_; - + const Array& object_pool_; DISALLOW_IMPLICIT_CONSTRUCTORS(StaticCall); }; // The expected code pattern of a dart closure call: -// 00: 49 ba imm64 mov R10, immediate 2 ; 10 bytes -// 10: 49 bb imm64 mov R11, target_address ; 10 bytes -// 20: 41 ff d3 call R11 ; 3 bytes -// 23: <- return_address +// 00: 49 ba imm64 mov R10, immediate 2 ; 10 bytes +// 10: 4d 8b 9f imm32 mov R11, [PP + off] +// 17: 41 ff d3 call R11 ; 3 bytes +// 20: <- return_address class ClosureCall : public ValueObject { public: explicit ClosureCall(uword return_address) @@ -147,9 +158,10 @@ class ClosureCall : public ValueObject { uint8_t* code_bytes = reinterpret_cast(return_address - kCallPatternSize); return (code_bytes[00] == 0x49) && (code_bytes[01] == 0xBA) && - (code_bytes[10] == 0x49) && (code_bytes[11] == 0xBB) && - (code_bytes[20] == 0x41) && (code_bytes[21] == 0xFF) && - (code_bytes[22] == 0xD3); + (code_bytes[10] == 0x4D) && (code_bytes[11] == 0x8B) && + (code_bytes[12] == 0x9F) && + (code_bytes[17] == 0x41) && (code_bytes[18] == 0xFF) && + (code_bytes[19] == 0xD3); } RawArray* arguments_descriptor() const { @@ -157,7 +169,7 @@ class ClosureCall : public ValueObject { } private: - static const int kCallPatternSize = 10 + 10 + 3; + static const int kCallPatternSize = 10 + 7 + 3; uword start_; DISALLOW_IMPLICIT_CONSTRUCTORS(ClosureCall); }; @@ -174,7 +186,7 @@ RawArray* CodePatcher::GetClosureArgDescAt(uword return_address, uword CodePatcher::GetStaticCallTargetAt(uword return_address, const Code& code) { ASSERT(code.ContainsInstructionAt(return_address)); - StaticCall call(return_address); + StaticCall call(return_address, code); return call.target(); } @@ -183,7 +195,7 @@ void CodePatcher::PatchStaticCallAt(uword return_address, const Code& code, uword new_target) { ASSERT(code.ContainsInstructionAt(return_address)); - StaticCall call(return_address); + StaticCall call(return_address, code); call.set_target(new_target); } @@ -192,7 +204,7 @@ void CodePatcher::PatchInstanceCallAt(uword return_address, const Code& code, uword new_target) { ASSERT(code.ContainsInstructionAt(return_address)); - InstanceCall call(return_address); + InstanceCall call(return_address, code); call.set_target(new_target); } @@ -201,7 +213,7 @@ uword CodePatcher::GetInstanceCallAt(uword return_address, const Code& code, ICData* ic_data) { ASSERT(code.ContainsInstructionAt(return_address)); - InstanceCall call(return_address); + InstanceCall call(return_address, code); if (ic_data != NULL) { *ic_data ^= call.ic_data(); } @@ -227,7 +239,7 @@ void CodePatcher::InsertCallAt(uword start, uword target) { RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( uword return_address, const Code& code, ICData* ic_data_result) { ASSERT(code.ContainsInstructionAt(return_address)); - UnoptimizedStaticCall static_call(return_address); + UnoptimizedStaticCall static_call(return_address, code); ICData& ic_data = ICData::Handle(); ic_data ^= static_call.ic_data(); if (ic_data_result != NULL) { diff --git a/runtime/vm/code_patcher_x64_test.cc b/runtime/vm/code_patcher_x64_test.cc index 202c27a3b0b..67561dbc6a5 100644 --- a/runtime/vm/code_patcher_x64_test.cc +++ b/runtime/vm/code_patcher_x64_test.cc @@ -39,10 +39,10 @@ ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) { 15, 1)); - __ LoadObject(RBX, ic_data); + __ LoadObject(RBX, ic_data, PP); ExternalLabel target_label( "InlineCache", StubCode::OneArgCheckInlineCacheEntryPoint()); - __ call(&target_label); + __ CallPatchable(&target_label); __ ret(); } diff --git a/runtime/vm/constants_x64.h b/runtime/vm/constants_x64.h index 3fa9526bd08..6839b677d8c 100644 --- a/runtime/vm/constants_x64.h +++ b/runtime/vm/constants_x64.h @@ -85,8 +85,9 @@ enum RexBits { // Register aliases. const Register TMP = R11; // Used as scratch register by the assembler. -const Register CTX = R15; // Caches current context in generated code. -const Register PP = kNoRegister; // No object pool pointer. +const Register CTX = R14; // Caches current context in generated code. +// Caches object pool pointer in generated code. +const Register PP = R15; const Register SPREG = RSP; // Stack pointer register. const Register FPREG = RBP; // Frame pointer register. const Register ICREG = RBX; // IC data register. diff --git a/runtime/vm/debugger_x64.cc b/runtime/vm/debugger_x64.cc index 29d3b33c4dd..f9b31c79bdc 100644 --- a/runtime/vm/debugger_x64.cc +++ b/runtime/vm/debugger_x64.cc @@ -7,6 +7,7 @@ #include "vm/debugger.h" +#include "vm/assembler.h" #include "vm/cpu.h" #include "vm/stub_code.h" @@ -42,16 +43,15 @@ RawObject* ActivationFrame::GetClosureObject(intptr_t num_actual_args) { void CodeBreakpoint::PatchFunctionReturn() { uint8_t* code = reinterpret_cast(pc_ - 13); - // movq %rbp,%rsp - ASSERT((code[0] == 0x48) && (code[1] == 0x89) && (code[2] == 0xec)); - ASSERT(code[3] == 0x5d); // popq %rbp - ASSERT(code[4] == 0xc3); // ret - // Next 8 bytes are nop instructions - ASSERT((code[5] == 0x90) && (code[6] == 0x90) && - (code[7] == 0x90) && (code[8] == 0x90) && - (code[9] == 0x90) && (code[10] == 0x90) && - (code[11] == 0x90) && (code[12] == 0x90)); - // Smash code with call instruction and relative target address. + ASSERT((code[0] == 0x4c) && (code[1] == 0x8b) && (code[2] == 0x7d) && + (code[3] == 0xf0)); // movq r15,[rbp-0x10] + ASSERT((code[4] == 0x48) && (code[5] == 0x89) && + (code[6] == 0xec)); // mov rsp, rbp + ASSERT(code[7] == 0x5d); // pop rbp + ASSERT(code[8] == 0xc3); // ret + ASSERT((code[9] == 0x0F) && (code[10] == 0x1F) && (code[11] == 0x40) && + (code[12] == 0x00)); // nops + // Smash code with call instruction and relative target address. uword stub_addr = StubCode::BreakpointReturnEntryPoint(); code[0] = 0x49; code[1] = 0xbb; @@ -66,19 +66,13 @@ void CodeBreakpoint::PatchFunctionReturn() { void CodeBreakpoint::RestoreFunctionReturn() { uint8_t* code = reinterpret_cast(pc_ - 13); ASSERT((code[0] == 0x49) && (code[1] == 0xbb)); - code[0] = 0x48; // movq %rbp,%rsp - code[1] = 0x89; - code[2] = 0xec; - code[3] = 0x5d; // popq %rbp - code[4] = 0xc3; // ret - code[5] = 0x90; // nop - code[6] = 0x90; // nop - code[7] = 0x90; // nop - code[8] = 0x90; // nop - code[9] = 0x90; // nop - code[10] = 0x90; // nop - code[11] = 0x90; // nop - code[12] = 0x90; // nop + + MemoryRegion code_region(reinterpret_cast(pc_ - 13), 13); + Assembler assembler; + + assembler.ReturnPatchable(); + assembler.FinalizeInstructions(code_region); + CPU::FlushICache(pc_ - 13, 13); } diff --git a/runtime/vm/find_code_object_test.cc b/runtime/vm/find_code_object_test.cc index ea82a810c47..ad1835a8ee7 100644 --- a/runtime/vm/find_code_object_test.cc +++ b/runtime/vm/find_code_object_test.cc @@ -14,7 +14,7 @@ namespace dart { TEST_CASE(FindCodeObject) { -#if defined(TARGET_ARCH_IA32) +#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) const int kLoopCount = 50000; #else const int kLoopCount = 25000; diff --git a/runtime/vm/flow_graph_compiler.cc b/runtime/vm/flow_graph_compiler.cc index 834d0efc261..5941c9d5bbe 100644 --- a/runtime/vm/flow_graph_compiler.cc +++ b/runtime/vm/flow_graph_compiler.cc @@ -633,48 +633,6 @@ void FlowGraphCompiler::GenerateStaticCall(intptr_t deopt_id, } -void FlowGraphCompiler::EmitUnoptimizedStaticCall( - const Function& target_function, - const Array& arguments_descriptor, - intptr_t argument_count, - intptr_t deopt_id, - intptr_t token_pos, - LocationSummary* locs) { - // TODO(srdjan): Improve performance of function recognition. - MethodRecognizer::Kind recognized_kind = - MethodRecognizer::RecognizeKind(target_function); - int num_args_checked = 0; - if ((recognized_kind == MethodRecognizer::kMathMin) || - (recognized_kind == MethodRecognizer::kMathMax)) { - num_args_checked = 2; - } - const ICData& ic_data = ICData::ZoneHandle( - ICData::New(parsed_function().function(), // Caller function. - String::Handle(target_function.name()), - arguments_descriptor, - deopt_id, - num_args_checked)); // No arguments checked. - ic_data.AddTarget(target_function); - uword label_address = 0; - if (ic_data.num_args_tested() == 0) { - label_address = StubCode::ZeroArgsUnoptimizedStaticCallEntryPoint(); - } else if (ic_data.num_args_tested() == 2) { - label_address = StubCode::TwoArgsUnoptimizedStaticCallEntryPoint(); - } else { - UNIMPLEMENTED(); - } - ExternalLabel target_label("StaticCallICStub", label_address); - assembler()->LoadObject(ICREG, ic_data); - GenerateDartCall(deopt_id, - token_pos, - &target_label, - PcDescriptors::kUnoptStaticCall, - locs); - assembler()->Drop(argument_count); -} - - - void FlowGraphCompiler::GenerateNumberTypeCheck(Register kClassIdReg, const AbstractType& type, Label* is_instance_lbl, diff --git a/runtime/vm/flow_graph_compiler_arm.cc b/runtime/vm/flow_graph_compiler_arm.cc index d129f736872..0854649de2a 100644 --- a/runtime/vm/flow_graph_compiler_arm.cc +++ b/runtime/vm/flow_graph_compiler_arm.cc @@ -1385,6 +1385,47 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + // TODO(srdjan): Improve performance of function recognition. + MethodRecognizer::Kind recognized_kind = + MethodRecognizer::RecognizeKind(target_function); + int num_args_checked = 0; + if ((recognized_kind == MethodRecognizer::kMathMin) || + (recognized_kind == MethodRecognizer::kMathMax)) { + num_args_checked = 2; + } + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + num_args_checked)); // No arguments checked. + ic_data.AddTarget(target_function); + uword label_address = 0; + if (ic_data.num_args_tested() == 0) { + label_address = StubCode::ZeroArgsUnoptimizedStaticCallEntryPoint(); + } else if (ic_data.num_args_tested() == 2) { + label_address = StubCode::TwoArgsUnoptimizedStaticCallEntryPoint(); + } else { + UNIMPLEMENTED(); + } + ExternalLabel target_label("StaticCallICStub", label_address); + __ LoadObject(R5, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &target_label, + PcDescriptors::kUnoptStaticCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitOptimizedStaticCall( const Function& function, const Array& arguments_descriptor, diff --git a/runtime/vm/flow_graph_compiler_ia32.cc b/runtime/vm/flow_graph_compiler_ia32.cc index fcd55f04ac2..151ca9f8949 100644 --- a/runtime/vm/flow_graph_compiler_ia32.cc +++ b/runtime/vm/flow_graph_compiler_ia32.cc @@ -1280,6 +1280,47 @@ void FlowGraphCompiler::GenerateCallRuntime(intptr_t token_pos, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + // TODO(srdjan): Improve performance of function recognition. + MethodRecognizer::Kind recognized_kind = + MethodRecognizer::RecognizeKind(target_function); + int num_args_checked = 0; + if ((recognized_kind == MethodRecognizer::kMathMin) || + (recognized_kind == MethodRecognizer::kMathMax)) { + num_args_checked = 2; + } + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + num_args_checked)); // No arguments checked. + ic_data.AddTarget(target_function); + uword label_address = 0; + if (ic_data.num_args_tested() == 0) { + label_address = StubCode::ZeroArgsUnoptimizedStaticCallEntryPoint(); + } else if (ic_data.num_args_tested() == 2) { + label_address = StubCode::TwoArgsUnoptimizedStaticCallEntryPoint(); + } else { + UNIMPLEMENTED(); + } + ExternalLabel target_label("StaticCallICStub", label_address); + __ LoadObject(ECX, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &target_label, + PcDescriptors::kUnoptStaticCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitOptimizedInstanceCall( ExternalLabel* target_label, const ICData& ic_data, diff --git a/runtime/vm/flow_graph_compiler_mips.cc b/runtime/vm/flow_graph_compiler_mips.cc index a377ab6d226..8e510511a19 100644 --- a/runtime/vm/flow_graph_compiler_mips.cc +++ b/runtime/vm/flow_graph_compiler_mips.cc @@ -1432,6 +1432,47 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + // TODO(srdjan): Improve performance of function recognition. + MethodRecognizer::Kind recognized_kind = + MethodRecognizer::RecognizeKind(target_function); + int num_args_checked = 0; + if ((recognized_kind == MethodRecognizer::kMathMin) || + (recognized_kind == MethodRecognizer::kMathMax)) { + num_args_checked = 2; + } + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + num_args_checked)); // No arguments checked. + ic_data.AddTarget(target_function); + uword label_address = 0; + if (ic_data.num_args_tested() == 0) { + label_address = StubCode::ZeroArgsUnoptimizedStaticCallEntryPoint(); + } else if (ic_data.num_args_tested() == 2) { + label_address = StubCode::TwoArgsUnoptimizedStaticCallEntryPoint(); + } else { + UNIMPLEMENTED(); + } + ExternalLabel target_label("StaticCallICStub", label_address); + __ LoadObject(S5, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &target_label, + PcDescriptors::kUnoptStaticCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitOptimizedStaticCall( const Function& function, const Array& arguments_descriptor, diff --git a/runtime/vm/flow_graph_compiler_x64.cc b/runtime/vm/flow_graph_compiler_x64.cc index 6106c0fe427..b82c70439d8 100644 --- a/runtime/vm/flow_graph_compiler_x64.cc +++ b/runtime/vm/flow_graph_compiler_x64.cc @@ -61,10 +61,9 @@ RawDeoptInfo* CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler, // The real frame starts here. builder->MarkFrameStart(); - // Callee's PC marker is not used anymore. Pass Function::null() to set to 0. + // Current PP, FP, and PC. + builder->AddPp(current->function(), slot_ix++); builder->AddPcMarker(Function::Handle(), slot_ix++); - - // Current FP and PC. builder->AddCallerFp(slot_ix++); builder->AddReturnAddress(current->function(), deopt_id(), slot_ix++); @@ -80,13 +79,14 @@ RawDeoptInfo* CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler, builder->AddCopy(current->ValueAt(i), current->LocationAt(i), slot_ix++); } - // Current PC marker and caller FP. - builder->AddPcMarker(current->function(), slot_ix++); - builder->AddCallerFp(slot_ix++); - Environment* previous = current; current = current->outer(); while (current != NULL) { + // PP, FP, and PC. + builder->AddPp(current->function(), slot_ix++); + builder->AddPcMarker(previous->function(), slot_ix++); + builder->AddCallerFp(slot_ix++); + // For any outer environment the deopt id is that of the call instruction // which is recorded in the outer environment. builder->AddReturnAddress(current->function(), @@ -110,10 +110,6 @@ RawDeoptInfo* CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler, slot_ix++); } - // PC marker and caller FP. - builder->AddPcMarker(current->function(), slot_ix++); - builder->AddCallerFp(slot_ix++); - // Iterate on the outer environment. previous = current; current = current->outer(); @@ -121,7 +117,11 @@ RawDeoptInfo* CompilerDeoptInfo::CreateDeoptInfo(FlowGraphCompiler* compiler, // The previous pointer is now the outermost environment. ASSERT(previous != NULL); - // For the outermost environment, set caller PC. + // For the outermost environment, set caller PC, caller PP, and caller FP. + builder->AddCallerPp(slot_ix++); + // PC marker. + builder->AddPcMarker(previous->function(), slot_ix++); + builder->AddCallerFp(slot_ix++); builder->AddCallerPc(slot_ix++); // For the outermost environment, set the incoming arguments. @@ -146,7 +146,7 @@ void CompilerDeoptInfoWithStub::GenerateCode(FlowGraphCompiler* compiler, ASSERT(deopt_env() != NULL); - __ call(&StubCode::DeoptimizeLabel()); + __ Call(&StubCode::DeoptimizeLabel(), PP); set_pc_offset(assem->CodeSize()); __ int3(); #undef __ @@ -160,10 +160,8 @@ void CompilerDeoptInfoWithStub::GenerateCode(FlowGraphCompiler* compiler, void FlowGraphCompiler::GenerateBoolToJump(Register bool_register, Label* is_true, Label* is_false) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label fall_through; - __ cmpq(bool_register, raw_null); + __ CompareObject(bool_register, Object::Handle()); __ j(EQUAL, &fall_through, Assembler::kNearJump); __ CompareObject(bool_register, Bool::True()); __ j(EQUAL, is_true); @@ -182,22 +180,20 @@ RawSubtypeTestCache* FlowGraphCompiler::GenerateCallSubtypeTestStub( Label* is_not_instance_lbl) { const SubtypeTestCache& type_test_cache = SubtypeTestCache::ZoneHandle(SubtypeTestCache::New()); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ LoadObject(temp_reg, type_test_cache); + __ LoadObject(temp_reg, type_test_cache, PP); __ pushq(temp_reg); // Subtype test cache. __ pushq(instance_reg); // Instance. if (test_kind == kTestTypeOneArg) { ASSERT(type_arguments_reg == kNoRegister); - __ pushq(raw_null); - __ call(&StubCode::Subtype1TestCacheLabel()); + __ PushObject(Object::Handle()); + __ Call(&StubCode::Subtype1TestCacheLabel(), PP); } else if (test_kind == kTestTypeTwoArgs) { ASSERT(type_arguments_reg == kNoRegister); - __ pushq(raw_null); - __ call(&StubCode::Subtype2TestCacheLabel()); + __ PushObject(Object::Handle()); + __ Call(&StubCode::Subtype2TestCacheLabel(), PP); } else if (test_kind == kTestTypeThreeArgs) { __ pushq(type_arguments_reg); - __ call(&StubCode::Subtype3TestCacheLabel()); + __ Call(&StubCode::Subtype3TestCacheLabel(), PP); } else { UNREACHABLE(); } @@ -342,11 +338,9 @@ bool FlowGraphCompiler::GenerateInstantiatedTypeNoArgumentsTest( } if (type.IsFunctionType()) { // Check if instance is a closure. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); __ LoadClassById(R13, kClassIdReg); __ movq(R13, FieldAddress(R13, Class::signature_function_offset())); - __ cmpq(R13, raw_null); + __ CompareObject(R13, Object::Handle()); __ j(NOT_EQUAL, is_instance_lbl); } // Custom checking for numbers (Smi, Mint, Bigint and Double). @@ -409,15 +403,13 @@ RawSubtypeTestCache* FlowGraphCompiler::GenerateUninstantiatedTypeTest( __ Comment("UninstantiatedTypeTest"); ASSERT(!type.IsInstantiated()); // Skip check if destination is a dynamic type. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); if (type.IsTypeParameter()) { const TypeParameter& type_param = TypeParameter::Cast(type); // Load instantiator (or null) and instantiator type arguments on stack. __ movq(RDX, Address(RSP, 0)); // Get instantiator type arguments. // RDX: instantiator type arguments. // Check if type argument is dynamic. - __ cmpq(RDX, raw_null); + __ CompareObject(RDX, Object::Handle()); __ j(EQUAL, is_instance_lbl); // Can handle only type arguments that are instances of TypeArguments. // (runtime checks canonicalize type arguments). @@ -430,7 +422,7 @@ RawSubtypeTestCache* FlowGraphCompiler::GenerateUninstantiatedTypeTest( // Check if type argument is dynamic. __ CompareObject(RDI, Type::ZoneHandle(Type::DynamicType())); __ j(EQUAL, is_instance_lbl); - __ cmpq(RDI, raw_null); + __ CompareObject(RDI, Object::Handle()); __ j(EQUAL, is_instance_lbl); const Type& object_type = Type::ZoneHandle(Type::ObjectType()); __ CompareObject(RDI, object_type); @@ -570,8 +562,6 @@ void FlowGraphCompiler::GenerateInstanceOf(intptr_t token_pos, LocationSummary* locs) { ASSERT(type.IsFinalized() && !type.IsMalformed() && !type.IsMalbounded()); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label is_instance, is_not_instance; __ pushq(RCX); // Store instantiator on stack. __ pushq(RDX); // Store instantiator type arguments. @@ -585,7 +575,7 @@ void FlowGraphCompiler::GenerateInstanceOf(intptr_t token_pos, // We can only inline this null check if the type is instantiated at compile // time, since an uninstantiated type at compile time could be Object or // dynamic at run time. - __ cmpq(RAX, raw_null); + __ CompareObject(RAX, Object::Handle()); __ j(EQUAL, &is_not_instance); } @@ -605,7 +595,7 @@ void FlowGraphCompiler::GenerateInstanceOf(intptr_t token_pos, __ PushObject(type); // Push the type. __ pushq(RCX); // TODO(srdjan): Pass instantiator instead of null. __ pushq(RDX); // Instantiator type arguments. - __ LoadObject(RAX, test_cache); + __ LoadObject(RAX, test_cache, PP); __ pushq(RAX); GenerateCallRuntime(token_pos, deopt_id, @@ -617,21 +607,21 @@ void FlowGraphCompiler::GenerateInstanceOf(intptr_t token_pos, __ Drop(5); if (negate_result) { __ popq(RDX); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ cmpq(RDX, RAX); __ j(NOT_EQUAL, &done, Assembler::kNearJump); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); } else { __ popq(RAX); } __ jmp(&done, Assembler::kNearJump); } __ Bind(&is_not_instance); - __ LoadObject(RAX, Bool::Get(negate_result)); + __ LoadObject(RAX, Bool::Get(negate_result), PP); __ jmp(&done, Assembler::kNearJump); __ Bind(&is_instance); - __ LoadObject(RAX, Bool::Get(!negate_result)); + __ LoadObject(RAX, Bool::Get(!negate_result), PP); __ Bind(&done); __ popq(RDX); // Remove pushed instantiator type arguments. __ popq(RCX); // Remove pushed instantiator. @@ -664,10 +654,8 @@ void FlowGraphCompiler::GenerateAssertAssignable(intptr_t token_pos, __ pushq(RCX); // Store instantiator. __ pushq(RDX); // Store instantiator type arguments. // A null object is always assignable and is returned as result. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label is_assignable, runtime_call; - __ cmpq(RAX, raw_null); + __ CompareObject(RAX, Object::Handle()); __ j(EQUAL, &is_assignable); if (!FLAG_eliminate_type_checks || dst_type.IsMalformed()) { @@ -720,7 +708,7 @@ void FlowGraphCompiler::GenerateAssertAssignable(intptr_t token_pos, __ pushq(RCX); // Instantiator. __ pushq(RDX); // Instantiator type arguments. __ PushObject(dst_name); // Push the name of the destination. - __ LoadObject(RAX, test_cache); + __ LoadObject(RAX, test_cache, PP); __ pushq(RAX); GenerateCallRuntime(token_pos, deopt_id, kTypeCheckRuntimeEntry, 6, locs); // Pop the parameters supplied to the runtime entry. The result of the @@ -765,7 +753,7 @@ void FlowGraphCompiler::EmitTrySyncMove(intptr_t dest_offset, __ pushq(RAX); *push_emitted = true; } - __ LoadObject(RAX, loc.constant()); + __ LoadObject(RAX, loc.constant(), PP); __ movq(dest, RAX); } else if (loc.IsRegister()) { if (*push_emitted && loc.reg() == RAX) { @@ -895,8 +883,6 @@ void FlowGraphCompiler::CopyParameters() { __ j(POSITIVE, &loop, Assembler::kNearJump); // Copy or initialize optional named arguments. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label all_arguments_processed; #ifdef DEBUG const bool check_correct_named_args = true; @@ -958,7 +944,7 @@ void FlowGraphCompiler::CopyParameters() { const Object& value = Object::ZoneHandle( parsed_function().default_parameter_values().At( param_pos - num_fixed_params)); - __ LoadObject(RAX, value); + __ LoadObject(RAX, value, PP); __ Bind(&assign_optional_parameter); // Assign RAX to fp[kFirstLocalSlotFromFp - param_pos]. // We do not use the final allocation index of the variable here, i.e. @@ -973,7 +959,8 @@ void FlowGraphCompiler::CopyParameters() { if (check_correct_named_args) { // Check that RDI now points to the null terminator in the arguments // descriptor. - __ cmpq(Address(RDI, 0), raw_null); + __ LoadObject(TMP, Object::Handle(), PP); + __ cmpq(Address(RDI, 0), TMP); __ j(EQUAL, &all_arguments_processed, Assembler::kNearJump); } } else { @@ -992,7 +979,7 @@ void FlowGraphCompiler::CopyParameters() { // Load RAX with default argument. const Object& value = Object::ZoneHandle( parsed_function().default_parameter_values().At(i)); - __ LoadObject(RAX, value); + __ LoadObject(RAX, value, PP); // Assign RAX to fp[kFirstLocalSlotFromFp - param_pos]. // We do not use the final allocation index of the variable here, i.e. // scope->VariableAt(i)->index(), because captured variables still need @@ -1018,8 +1005,8 @@ void FlowGraphCompiler::CopyParameters() { const ICData& ic_data = ICData::ZoneHandle( ICData::New(function, Symbols::Call(), Object::empty_array(), Isolate::kNoDeoptId, kNumArgsChecked)); - __ LoadObject(RBX, ic_data); - __ LeaveFrame(); // The arguments are still on the stack. + __ LoadObject(RBX, ic_data, PP); + __ LeaveFrameWithPP(); // The arguments are still on the stack. __ jmp(&StubCode::CallNoSuchMethodFunctionLabel()); // The noSuchMethod call may return to the caller, but not here. __ int3(); @@ -1037,12 +1024,13 @@ void FlowGraphCompiler::CopyParameters() { // R10 : arguments descriptor array. __ movq(RCX, FieldAddress(R10, ArgumentsDescriptor::count_offset())); __ SmiUntag(RCX); + __ LoadObject(R12, Object::Handle(), PP); Label null_args_loop, null_args_loop_condition; __ jmp(&null_args_loop_condition, Assembler::kNearJump); const Address original_argument_addr( RBP, RCX, TIMES_8, (kParamEndSlotFromFp + 1) * kWordSize); __ Bind(&null_args_loop); - __ movq(original_argument_addr, raw_null); + __ movq(original_argument_addr, R12); __ Bind(&null_args_loop_condition); __ decq(RCX); __ j(POSITIVE, &null_args_loop, Assembler::kNearJump); @@ -1067,20 +1055,43 @@ void FlowGraphCompiler::GenerateInlinedSetter(intptr_t offset) { __ movq(RAX, Address(RSP, 2 * kWordSize)); // Receiver. __ movq(RBX, Address(RSP, 1 * kWordSize)); // Value. __ StoreIntoObject(RAX, FieldAddress(RAX, offset), RBX); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(RAX, raw_null); + __ LoadObject(RAX, Object::Handle(), PP); __ ret(); } void FlowGraphCompiler::EmitFrameEntry() { const Function& function = parsed_function().function(); + Register new_pp = kNoRegister; + Register new_pc = kNoRegister; if (CanOptimizeFunction() && function.is_optimizable() && (!is_optimizing() || may_reoptimize())) { const Register function_reg = RDI; - __ LoadObject(function_reg, function); + new_pp = R13; + new_pc = R12; + + Label next; + __ nop(4); // Need a fixed size sequence on frame entry. + __ call(&next); + __ Bind(&next); + + const intptr_t object_pool_pc_dist = + Instructions::HeaderSize() - Instructions::object_pool_offset() + + __ CodeSize(); + const intptr_t offset = + Assembler::kEntryPointToPcMarkerOffset - __ CodeSize(); + __ popq(new_pc); + if (offset != 0) { + __ addq(new_pc, Immediate(offset)); + } + + // Load callee's pool pointer. + __ movq(new_pp, Address(new_pc, -object_pool_pc_dist - offset)); + + // Load function object using the callee's pool pointer. + __ LoadObject(function_reg, function, new_pp); + // Patch point is after the eventually inlined function object. AddCurrentDescriptor(PcDescriptors::kEntryPatch, Isolate::kNoDeoptId, @@ -1096,8 +1107,30 @@ void FlowGraphCompiler::EmitFrameEntry() { Immediate(FLAG_optimization_counter_threshold)); } ASSERT(function_reg == RDI); - __ j(GREATER_EQUAL, &StubCode::OptimizeFunctionLabel()); + __ J(GREATER_EQUAL, &StubCode::OptimizeFunctionLabel(), R13); } else if (!flow_graph().IsCompiledForOsr()) { + // We have to load the PP here too because a load of an external label + // may be patched at the AddCurrentDescriptor below. + new_pp = R13; + new_pc = R12; + + Label next; + __ nop(4); // Need a fixed size sequence on frame entry. + __ call(&next); + __ Bind(&next); + + const intptr_t object_pool_pc_dist = + Instructions::HeaderSize() - Instructions::object_pool_offset() + + __ CodeSize(); + const intptr_t offset = + Assembler::kEntryPointToPcMarkerOffset - __ CodeSize(); + __ popq(new_pc); + if (offset != 0) { + __ addq(new_pc, Immediate(offset)); + } + + // Load callee's pool pointer. + __ movq(new_pp, Address(new_pc, -object_pool_pc_dist - offset)); AddCurrentDescriptor(PcDescriptors::kEntryPatch, Isolate::kNoDeoptId, 0); // No token position. @@ -1108,10 +1141,10 @@ void FlowGraphCompiler::EmitFrameEntry() { - flow_graph().num_stack_locals() - flow_graph().num_copied_params(); ASSERT(extra_slots >= 0); - __ EnterOsrFrame(extra_slots * kWordSize); + __ EnterOsrFrame(extra_slots * kWordSize, new_pp, new_pc); } else { ASSERT(StackSize() >= 0); - __ EnterDartFrame(StackSize() * kWordSize); + __ EnterDartFrameWithInfo(StackSize() * kWordSize, new_pp, new_pc); } } @@ -1165,8 +1198,8 @@ void FlowGraphCompiler::CompileGraph() { const ICData& ic_data = ICData::ZoneHandle( ICData::New(function, name, Object::empty_array(), Isolate::kNoDeoptId, kNumArgsChecked)); - __ LoadObject(RBX, ic_data); - __ LeaveFrame(); // The arguments are still on the stack. + __ LoadObject(RBX, ic_data, PP); + __ LeaveFrameWithPP(); // The arguments are still on the stack. __ jmp(&StubCode::CallNoSuchMethodFunctionLabel()); // The noSuchMethod call may return to the caller, but not here. __ int3(); @@ -1184,9 +1217,7 @@ void FlowGraphCompiler::CompileGraph() { if (!is_optimizing() && (num_locals > 0)) { __ Comment("Initialize spill slots"); const intptr_t slot_base = parsed_function().first_stack_local_index(); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(RAX, raw_null); + __ LoadObject(RAX, Object::Handle(), PP); for (intptr_t i = 0; i < num_locals; ++i) { // Subtract index i (locals lie at lower addresses than RBP). __ movq(Address(RBP, (slot_base - i) * kWordSize), RAX); @@ -1213,11 +1244,15 @@ void FlowGraphCompiler::CompileGraph() { AddCurrentDescriptor(PcDescriptors::kPatchCode, Isolate::kNoDeoptId, 0); // No token position. - __ jmp(&StubCode::FixCallersTargetLabel()); + // This is patched up to a point in FrameEntry where the PP for the + // current function is in R13 instead of PP. + __ JmpPatchable(&StubCode::FixCallersTargetLabel(), R13); + + // TOOD(zra): Is this descriptor used? AddCurrentDescriptor(PcDescriptors::kLazyDeoptJump, Isolate::kNoDeoptId, 0); // No token position. - __ jmp(&StubCode::DeoptimizeLazyLabel()); + __ Jmp(&StubCode::DeoptimizeLazyLabel(), PP); } @@ -1225,7 +1260,7 @@ void FlowGraphCompiler::GenerateCall(intptr_t token_pos, const ExternalLabel* label, PcDescriptors::Kind kind, LocationSummary* locs) { - __ call(label); + __ Call(label, PP); AddCurrentDescriptor(kind, Isolate::kNoDeoptId, token_pos); RecordSafepoint(locs); } @@ -1236,7 +1271,7 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id, const ExternalLabel* label, PcDescriptors::Kind kind, LocationSummary* locs) { - __ call(label); + __ CallPatchable(label); AddCurrentDescriptor(kind, deopt_id, token_pos); RecordSafepoint(locs); // Marks either the continuation point in unoptimized code or the @@ -1275,6 +1310,47 @@ void FlowGraphCompiler::GenerateCallRuntime(intptr_t token_pos, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + // TODO(srdjan): Improve performance of function recognition. + MethodRecognizer::Kind recognized_kind = + MethodRecognizer::RecognizeKind(target_function); + int num_args_checked = 0; + if ((recognized_kind == MethodRecognizer::kMathMin) || + (recognized_kind == MethodRecognizer::kMathMax)) { + num_args_checked = 2; + } + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + num_args_checked)); // No arguments checked. + ic_data.AddTarget(target_function); + uword label_address = 0; + if (ic_data.num_args_tested() == 0) { + label_address = StubCode::ZeroArgsUnoptimizedStaticCallEntryPoint(); + } else if (ic_data.num_args_tested() == 2) { + label_address = StubCode::TwoArgsUnoptimizedStaticCallEntryPoint(); + } else { + UNIMPLEMENTED(); + } + ExternalLabel target_label("StaticCallICStub", label_address); + __ LoadObject(RBX, ic_data, PP); + GenerateDartCall(deopt_id, + token_pos, + &target_label, + PcDescriptors::kUnoptStaticCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitOptimizedInstanceCall( ExternalLabel* target_label, const ICData& ic_data, @@ -1288,8 +1364,8 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall( // top-level function (parsed_function().function()) which could be // reoptimized and which counter needs to be incremented. // Pass the function explicitly, it is used in IC stub. - __ LoadObject(RDI, parsed_function().function()); - __ LoadObject(RBX, ic_data); + __ LoadObject(RDI, parsed_function().function(), PP); + __ LoadObject(RBX, ic_data, PP); GenerateDartCall(deopt_id, token_pos, target_label, @@ -1305,7 +1381,7 @@ void FlowGraphCompiler::EmitInstanceCall(ExternalLabel* target_label, intptr_t deopt_id, intptr_t token_pos, LocationSummary* locs) { - __ LoadObject(RBX, ic_data); + __ LoadObject(RBX, ic_data, PP); GenerateDartCall(deopt_id, token_pos, target_label, @@ -1341,7 +1417,7 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( // RAX: class ID of the receiver (smi). __ Bind(&load_cache); - __ LoadObject(RBX, cache); + __ LoadObject(RBX, cache, PP); __ movq(RDI, FieldAddress(RBX, MegamorphicCache::buckets_offset())); __ movq(RBX, FieldAddress(RBX, MegamorphicCache::mask_offset())); // RDI: cache buckets array. @@ -1373,8 +1449,8 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( __ movq(RAX, FieldAddress(RDI, RCX, TIMES_8, base + kWordSize)); __ movq(RAX, FieldAddress(RAX, Function::code_offset())); __ movq(RAX, FieldAddress(RAX, Code::instructions_offset())); - __ LoadObject(RBX, ic_data); - __ LoadObject(R10, arguments_descriptor); + __ LoadObject(RBX, ic_data, PP); + __ LoadObject(R10, arguments_descriptor, PP); __ addq(RAX, Immediate(Instructions::HeaderSize() - kHeapObjectTag)); __ call(RAX); AddCurrentDescriptor(PcDescriptors::kOther, Isolate::kNoDeoptId, token_pos); @@ -1391,7 +1467,7 @@ void FlowGraphCompiler::EmitOptimizedStaticCall( intptr_t deopt_id, intptr_t token_pos, LocationSummary* locs) { - __ LoadObject(R10, arguments_descriptor); + __ LoadObject(R10, arguments_descriptor, PP); // Do not use the code from the function, but let the code be patched so that // we can record the outgoing edges to other code. GenerateDartCall(deopt_id, @@ -1424,9 +1500,9 @@ void FlowGraphCompiler::EmitEqualityRegConstCompare(Register reg, __ pushq(reg); __ PushObject(obj); if (is_optimizing()) { - __ call(&StubCode::OptimizedIdenticalWithNumberCheckLabel()); + __ CallPatchable(&StubCode::OptimizedIdenticalWithNumberCheckLabel()); } else { - __ call(&StubCode::UnoptimizedIdenticalWithNumberCheckLabel()); + __ CallPatchable(&StubCode::UnoptimizedIdenticalWithNumberCheckLabel()); } AddCurrentDescriptor(PcDescriptors::kRuntimeCall, Isolate::kNoDeoptId, @@ -1448,9 +1524,9 @@ void FlowGraphCompiler::EmitEqualityRegRegCompare(Register left, __ pushq(left); __ pushq(right); if (is_optimizing()) { - __ call(&StubCode::OptimizedIdenticalWithNumberCheckLabel()); + __ CallPatchable(&StubCode::OptimizedIdenticalWithNumberCheckLabel()); } else { - __ call(&StubCode::UnoptimizedIdenticalWithNumberCheckLabel()); + __ CallPatchable(&StubCode::UnoptimizedIdenticalWithNumberCheckLabel()); } AddCurrentDescriptor(PcDescriptors::kRuntimeCall, Isolate::kNoDeoptId, @@ -1540,7 +1616,7 @@ void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, const Array& arguments_descriptor = Array::ZoneHandle(ArgumentsDescriptor::New(argument_count, argument_names)); - __ LoadObject(R10, arguments_descriptor); + __ LoadObject(R10, arguments_descriptor, PP); for (intptr_t i = 0; i < len; i++) { const bool is_last_check = (i == (len - 1)); Label next_test; @@ -1582,7 +1658,6 @@ void FlowGraphCompiler::EmitDoubleCompareBranch(Condition true_condition, } - void FlowGraphCompiler::EmitDoubleCompareBool(Condition true_condition, FpuRegister left, FpuRegister right, @@ -1592,10 +1667,10 @@ void FlowGraphCompiler::EmitDoubleCompareBool(Condition true_condition, assembler()->j(PARITY_EVEN, &is_false, Assembler::kNearJump); // NaN false; assembler()->j(true_condition, &is_true, Assembler::kNearJump); assembler()->Bind(&is_false); - assembler()->LoadObject(result, Bool::False()); + assembler()->LoadObject(result, Bool::False(), PP); assembler()->jmp(&done); assembler()->Bind(&is_true); - assembler()->LoadObject(result, Bool::True()); + assembler()->LoadObject(result, Bool::True(), PP); assembler()->Bind(&done); } @@ -1716,7 +1791,7 @@ void ParallelMoveResolver::EmitMove(int index) { if (constant.IsSmi() && (Smi::Cast(constant).Value() == 0)) { __ xorq(destination.reg(), destination.reg()); } else { - __ LoadObject(destination.reg(), constant); + __ LoadObject(destination.reg(), constant, PP); } } else { ASSERT(destination.IsStackSlot()); diff --git a/runtime/vm/instructions_arm.cc b/runtime/vm/instructions_arm.cc index c896e112297..4ae5936a51e 100644 --- a/runtime/vm/instructions_arm.cc +++ b/runtime/vm/instructions_arm.cc @@ -163,7 +163,7 @@ void CallPattern::InsertAt(uword pc, uword target_address) { } -JumpPattern::JumpPattern(uword pc) : pc_(pc) { } +JumpPattern::JumpPattern(uword pc, const Code& code) : pc_(pc) { } bool JumpPattern::IsValid() const { diff --git a/runtime/vm/instructions_arm.h b/runtime/vm/instructions_arm.h index 98230ae2f71..59cb2595866 100644 --- a/runtime/vm/instructions_arm.h +++ b/runtime/vm/instructions_arm.h @@ -50,7 +50,7 @@ class CallPattern : public ValueObject { class JumpPattern : public ValueObject { public: - explicit JumpPattern(uword pc); + JumpPattern(uword pc, const Code& code); static const int kLengthInBytes = 3 * Instr::kInstrSize; diff --git a/runtime/vm/instructions_arm_test.cc b/runtime/vm/instructions_arm_test.cc index 0b2388430ba..dec458d5ccc 100644 --- a/runtime/vm/instructions_arm_test.cc +++ b/runtime/vm/instructions_arm_test.cc @@ -38,10 +38,11 @@ ASSEMBLER_TEST_GENERATE(Jump, assembler) { ASSEMBLER_TEST_RUN(Jump, test) { - JumpPattern jump1(test->entry()); + JumpPattern jump1(test->entry(), test->code()); EXPECT_EQ(StubCode::InstanceFunctionLookupLabel().address(), jump1.TargetAddress()); - JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes()); + JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes(), + test->code()); EXPECT_EQ(StubCode::AllocateArrayLabel().address(), jump2.TargetAddress()); uword target1 = jump1.TargetAddress(); diff --git a/runtime/vm/instructions_ia32.h b/runtime/vm/instructions_ia32.h index 6f8af00d5ca..365ac265309 100644 --- a/runtime/vm/instructions_ia32.h +++ b/runtime/vm/instructions_ia32.h @@ -11,6 +11,7 @@ #endif #include "vm/allocation.h" +#include "vm/object.h" namespace dart { @@ -85,7 +86,7 @@ class CallPattern : public CallOrJumpPattern { class JumpPattern : public CallOrJumpPattern { public: - explicit JumpPattern(uword pc) : CallOrJumpPattern(pc) {} + JumpPattern(uword pc, const Code& code) : CallOrJumpPattern(pc) {} private: virtual const int* pattern() const; diff --git a/runtime/vm/instructions_ia32_test.cc b/runtime/vm/instructions_ia32_test.cc index 3ceade778e6..e449e212b40 100644 --- a/runtime/vm/instructions_ia32_test.cc +++ b/runtime/vm/instructions_ia32_test.cc @@ -35,10 +35,11 @@ ASSEMBLER_TEST_GENERATE(Jump, assembler) { ASSEMBLER_TEST_RUN(Jump, test) { - JumpPattern jump1(test->entry()); + JumpPattern jump1(test->entry(), test->code()); EXPECT_EQ(StubCode::InstanceFunctionLookupLabel().address(), jump1.TargetAddress()); - JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes()); + JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes(), + test->code()); EXPECT_EQ(StubCode::AllocateArrayLabel().address(), jump2.TargetAddress()); uword target1 = jump1.TargetAddress(); diff --git a/runtime/vm/instructions_mips.cc b/runtime/vm/instructions_mips.cc index 1f79afd9ec3..7215ec08f6f 100644 --- a/runtime/vm/instructions_mips.cc +++ b/runtime/vm/instructions_mips.cc @@ -177,7 +177,7 @@ void CallPattern::InsertAt(uword pc, uword target_address) { } -JumpPattern::JumpPattern(uword pc) : pc_(pc) { } +JumpPattern::JumpPattern(uword pc, const Code& code) : pc_(pc) { } bool JumpPattern::IsValid() const { diff --git a/runtime/vm/instructions_mips.h b/runtime/vm/instructions_mips.h index fc2ef10dfa7..7cecc700e58 100644 --- a/runtime/vm/instructions_mips.h +++ b/runtime/vm/instructions_mips.h @@ -50,7 +50,7 @@ class CallPattern : public ValueObject { class JumpPattern : public ValueObject { public: - explicit JumpPattern(uword pc); + JumpPattern(uword pc, const Code& code); // lui; ori; jr; nop (in delay slot) = 4. static const int kLengthInBytes = 4*Instr::kInstrSize; diff --git a/runtime/vm/instructions_mips_test.cc b/runtime/vm/instructions_mips_test.cc index 4add9ef2892..7348db8ec5f 100644 --- a/runtime/vm/instructions_mips_test.cc +++ b/runtime/vm/instructions_mips_test.cc @@ -39,10 +39,11 @@ ASSEMBLER_TEST_GENERATE(Jump, assembler) { ASSEMBLER_TEST_RUN(Jump, test) { - JumpPattern jump1(test->entry()); + JumpPattern jump1(test->entry(), test->code()); EXPECT_EQ(StubCode::InstanceFunctionLookupLabel().address(), jump1.TargetAddress()); - JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes()); + JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes(), + test->code()); EXPECT_EQ(StubCode::AllocateArrayLabel().address(), jump2.TargetAddress()); uword target1 = jump1.TargetAddress(); diff --git a/runtime/vm/instructions_x64.cc b/runtime/vm/instructions_x64.cc index 62a4aa46179..0f78807306b 100644 --- a/runtime/vm/instructions_x64.cc +++ b/runtime/vm/instructions_x64.cc @@ -11,6 +11,13 @@ namespace dart { +intptr_t InstructionPattern::IndexFromPPLoad(uword start) { + int32_t offset = *reinterpret_cast(start); + offset += kHeapObjectTag; + return (offset - Array::data_offset()) / kWordSize; +} + + bool InstructionPattern::TestBytesWith(const int* data, int num_bytes) const { ASSERT(data != NULL); const uint8_t* byte_array = reinterpret_cast(start_); @@ -24,13 +31,13 @@ bool InstructionPattern::TestBytesWith(const int* data, int num_bytes) const { } -uword CallOrJumpPattern::TargetAddress() const { +uword CallPattern::TargetAddress() const { ASSERT(IsValid()); return *reinterpret_cast(start() + 2); } -void CallOrJumpPattern::SetTargetAddress(uword target) const { +void CallPattern::SetTargetAddress(uword target) const { ASSERT(IsValid()); *reinterpret_cast(start() + 2) = target; CPU::FlushICache(start() + 2, kWordSize); @@ -46,11 +53,27 @@ const int* CallPattern::pattern() const { } +uword JumpPattern::TargetAddress() const { + ASSERT(IsValid()); + int index = InstructionPattern::IndexFromPPLoad(start() + 3); + return reinterpret_cast(object_pool_.At(index)); +} + + +void JumpPattern::SetTargetAddress(uword target) const { + ASSERT(IsValid()); + int index = InstructionPattern::IndexFromPPLoad(start() + 3); + const Smi& smi = Smi::Handle(reinterpret_cast(target)); + object_pool_.SetAt(index, smi); + // No need to flush the instruction cache, since the code is not modified. +} + + const int* JumpPattern::pattern() const { - // movq $target, TMP - // jmpq TMP + // 00: 4d 8b 9d imm32 mov R11, [R13 + off] + // 07: 41 ff e3 jmpq R11 static const int kJumpPattern[kLengthInBytes] = - {0x49, 0xBB, -1, -1, -1, -1, -1, -1, -1, -1, 0x41, 0xFF, 0xE3}; + {0x4D, 0x8B, -1, -1, -1, -1, -1, 0x41, 0xFF, 0xE3}; return kJumpPattern; } diff --git a/runtime/vm/instructions_x64.h b/runtime/vm/instructions_x64.h index c4fc155e7d2..8b91e7d399a 100644 --- a/runtime/vm/instructions_x64.h +++ b/runtime/vm/instructions_x64.h @@ -11,6 +11,7 @@ #endif #include "vm/allocation.h" +#include "vm/object.h" namespace dart { @@ -37,6 +38,8 @@ class InstructionPattern : public ValueObject { virtual const int* pattern() const = 0; virtual int pattern_length_in_bytes() const = 0; + static intptr_t IndexFromPPLoad(uword start); + protected: uword start() const { return start_; } @@ -52,46 +55,47 @@ class InstructionPattern : public ValueObject { }; -class CallOrJumpPattern : public InstructionPattern { +class CallPattern : public InstructionPattern { public: - virtual int pattern_length_in_bytes() const { + CallPattern(uword pc, const Code& code) + : InstructionPattern(pc), + code_(code) {} + static int InstructionLength() { return kLengthInBytes; } uword TargetAddress() const; void SetTargetAddress(uword new_target) const; - - protected: - explicit CallOrJumpPattern(uword pc) : InstructionPattern(pc) {} - static const int kLengthInBytes = 13; - - private: - DISALLOW_COPY_AND_ASSIGN(CallOrJumpPattern); -}; - - -class CallPattern : public CallOrJumpPattern { - public: - explicit CallPattern(uword pc) : CallOrJumpPattern(pc) {} - static int InstructionLength() { + virtual int pattern_length_in_bytes() const { return kLengthInBytes; } private: + static const int kLengthInBytes = 13; virtual const int* pattern() const; + const Code& code_; DISALLOW_COPY_AND_ASSIGN(CallPattern); }; -class JumpPattern : public CallOrJumpPattern { +class JumpPattern : public InstructionPattern { public: - explicit JumpPattern(uword pc) : CallOrJumpPattern(pc) {} + JumpPattern(uword pc, const Code& code) + : InstructionPattern(pc), + object_pool_(Array::Handle(code.ObjectPool())) {} static int InstructionLength() { return kLengthInBytes; } + uword TargetAddress() const; + void SetTargetAddress(uword new_target) const; + virtual int pattern_length_in_bytes() const { + return kLengthInBytes; + } private: + static const int kLengthInBytes = 10; virtual const int* pattern() const; + const Array& object_pool_; DISALLOW_COPY_AND_ASSIGN(JumpPattern); }; diff --git a/runtime/vm/instructions_x64_test.cc b/runtime/vm/instructions_x64_test.cc index fb46518364b..72d41ea9182 100644 --- a/runtime/vm/instructions_x64_test.cc +++ b/runtime/vm/instructions_x64_test.cc @@ -21,25 +21,28 @@ ASSEMBLER_TEST_GENERATE(Call, assembler) { ASSEMBLER_TEST_RUN(Call, test) { - CallPattern call(test->entry()); + CallPattern call(test->entry(), test->code()); EXPECT_EQ(StubCode::InstanceFunctionLookupLabel().address(), call.TargetAddress()); } ASSEMBLER_TEST_GENERATE(Jump, assembler) { - __ jmp(&StubCode::InstanceFunctionLookupLabel()); - __ jmp(&StubCode::AllocateArrayLabel()); + __ EnterDartFrame(0); // 20 bytes + __ JmpPatchable(&StubCode::InstanceFunctionLookupLabel(), PP); + __ JmpPatchable(&StubCode::AllocateArrayLabel(), PP); + __ LeaveFrameWithPP(); __ ret(); } ASSEMBLER_TEST_RUN(Jump, test) { - JumpPattern jump1(test->entry()); + JumpPattern jump1(test->entry() + 20, test->code()); jump1.IsValid(); EXPECT_EQ(StubCode::InstanceFunctionLookupLabel().address(), jump1.TargetAddress()); - JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes()); + JumpPattern jump2(test->entry() + jump1.pattern_length_in_bytes() + 20, + test->code()); EXPECT_EQ(StubCode::AllocateArrayLabel().address(), jump2.TargetAddress()); uword target1 = jump1.TargetAddress(); diff --git a/runtime/vm/intermediate_language_x64.cc b/runtime/vm/intermediate_language_x64.cc index 5c670f819ef..ec9c31bc116 100644 --- a/runtime/vm/intermediate_language_x64.cc +++ b/runtime/vm/intermediate_language_x64.cc @@ -96,20 +96,8 @@ void ReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Bind(&done); } #endif - __ LeaveFrame(); - __ ret(); - // Generate 8 bytes of NOPs so that the debugger can patch the - // return pattern with a call to the debug stub. - // Note that the nop(8) byte pattern is not recognized by the debugger. - __ nop(1); - __ nop(1); - __ nop(1); - __ nop(1); - __ nop(1); - __ nop(1); - __ nop(1); - __ nop(1); + __ ReturnPatchable(); compiler->AddCurrentDescriptor(PcDescriptors::kReturn, Isolate::kNoDeoptId, token_pos()); @@ -315,7 +303,7 @@ void ConstantInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // The register allocator drops constant definitions that have no uses. if (!locs()->out().IsInvalid()) { Register result = locs()->out().reg(); - __ LoadObject(result, value()); + __ LoadObject(result, value(), PP); } } @@ -465,12 +453,11 @@ static void EmitEqualityAsInstanceCall(FlowGraphCompiler* compiler, const Array& kNoArgumentNames = Object::null_array(); const int kNumArgumentsChecked = 2; - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label check_identity; - __ cmpq(Address(RSP, 0 * kWordSize), raw_null); + __ LoadObject(TMP, Object::Handle(), PP); + __ cmpq(Address(RSP, 0 * kWordSize), TMP); __ j(EQUAL, &check_identity); - __ cmpq(Address(RSP, 1 * kWordSize), raw_null); + __ cmpq(Address(RSP, 1 * kWordSize), TMP); __ j(EQUAL, &check_identity); ICData& equality_ic_data = ICData::ZoneHandle(original_ic_data.raw()); @@ -511,10 +498,10 @@ static void EmitEqualityAsInstanceCall(FlowGraphCompiler* compiler, __ popq(RDX); __ cmpq(RAX, RDX); __ j(EQUAL, &is_true); - __ LoadObject(RAX, Bool::Get(kind != Token::kEQ)); + __ LoadObject(RAX, Bool::Get(kind != Token::kEQ), PP); __ jmp(&equality_done); __ Bind(&is_true); - __ LoadObject(RAX, Bool::Get(kind == Token::kEQ)); + __ LoadObject(RAX, Bool::Get(kind == Token::kEQ), PP); if (kind == Token::kNE) { // Skip not-equal result conversion. __ jmp(&equality_done); @@ -524,7 +511,7 @@ static void EmitEqualityAsInstanceCall(FlowGraphCompiler* compiler, // necessary. Register ic_data_reg = locs->temp(0).reg(); ASSERT(ic_data_reg == RBX); // Stub depends on it. - __ LoadObject(ic_data_reg, equality_ic_data); + __ LoadObject(ic_data_reg, equality_ic_data, PP); compiler->GenerateCall(token_pos, &StubCode::EqualityWithNullArgLabel(), PcDescriptors::kRuntimeCall, @@ -537,10 +524,10 @@ static void EmitEqualityAsInstanceCall(FlowGraphCompiler* compiler, // Negate the condition: true label returns false and vice versa. __ CompareObject(RAX, Bool::True()); __ j(EQUAL, &true_label, Assembler::kNearJump); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ jmp(&done, Assembler::kNearJump); __ Bind(&true_label); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ Bind(&done); } __ Bind(&equality_done); @@ -610,10 +597,10 @@ static void EmitEqualityAsPolymorphicCall(FlowGraphCompiler* compiler, Register result = locs->out().reg(); Label load_true; __ j(cond, &load_true, Assembler::kNearJump); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ jmp(&done); __ Bind(&load_true); - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); } } else { const int kNumberOfArguments = 2; @@ -629,10 +616,10 @@ static void EmitEqualityAsPolymorphicCall(FlowGraphCompiler* compiler, Label false_label; __ CompareObject(RAX, Bool::True()); __ j(EQUAL, &false_label, Assembler::kNearJump); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ jmp(&done); __ Bind(&false_label); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); } } else { if (branch->is_checked()) { @@ -666,12 +653,11 @@ static void EmitCheckedStrictEqual(FlowGraphCompiler* compiler, __ testq(left, Immediate(kSmiTagMask)); __ j(ZERO, deopt); // 'left' is not Smi. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + Label identity_compare; - __ cmpq(right, raw_null); + __ CompareObject(right, Object::Handle()); __ j(EQUAL, &identity_compare); - __ cmpq(left, raw_null); + __ CompareObject(left, Object::Handle()); __ j(EQUAL, &identity_compare); __ LoadClassId(temp, left); @@ -692,10 +678,10 @@ static void EmitCheckedStrictEqual(FlowGraphCompiler* compiler, Register result = locs.out().reg(); __ j(EQUAL, &is_equal, Assembler::kNearJump); // Not equal. - __ LoadObject(result, Bool::Get(kind != Token::kEQ)); + __ LoadObject(result, Bool::Get(kind != Token::kEQ), PP); __ jmp(&done, Assembler::kNearJump); __ Bind(&is_equal); - __ LoadObject(result, Bool::Get(kind == Token::kEQ)); + __ LoadObject(result, Bool::Get(kind == Token::kEQ), PP); __ Bind(&done); } else { Condition cond = TokenKindToSmiCondition(kind); @@ -718,12 +704,11 @@ static void EmitGenericEqualityCompare(FlowGraphCompiler* compiler, ASSERT(!ic_data.IsNull() && (ic_data.NumberOfChecks() > 0)); Register left = locs->in(0).reg(); Register right = locs->in(1).reg(); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + Label done, identity_compare, non_null_compare; - __ cmpq(right, raw_null); + __ CompareObject(right, Object::Handle()); __ j(EQUAL, &identity_compare, Assembler::kNearJump); - __ cmpq(left, raw_null); + __ CompareObject(left, Object::Handle()); __ j(NOT_EQUAL, &non_null_compare, Assembler::kNearJump); // Comparison with NULL is "===". __ Bind(&identity_compare); @@ -735,10 +720,10 @@ static void EmitGenericEqualityCompare(FlowGraphCompiler* compiler, Register result = locs->out().reg(); Label load_true; __ j(cond, &load_true, Assembler::kNearJump); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ jmp(&done); __ Bind(&load_true); - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); } __ jmp(&done); __ Bind(&non_null_compare); // Receiver is not null. @@ -796,10 +781,10 @@ static void EmitSmiComparisonOp(FlowGraphCompiler* compiler, Register result = locs.out().reg(); Label done, is_true; __ j(true_condition, &is_true); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ jmp(&done); __ Bind(&is_true); - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); __ Bind(&done); } } @@ -1527,7 +1512,7 @@ void GuardFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { ASSERT((field_reg != value_reg) && (field_reg != value_cid_reg)); } - __ LoadObject(field_reg, Field::ZoneHandle(field().raw())); + __ LoadObject(field_reg, Field::ZoneHandle(field().raw()), PP); FieldAddress field_cid_operand(field_reg, Field::guarded_cid_offset()); FieldAddress field_nullability_operand( @@ -1711,7 +1696,7 @@ void GuardFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { } } else { if (field_reg != kNoRegister) { - __ LoadObject(field_reg, Field::ZoneHandle(field().raw())); + __ LoadObject(field_reg, Field::ZoneHandle(field().raw()), PP); } if (value_cid == kDynamicCid) { @@ -1747,9 +1732,7 @@ void GuardFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { if (field().is_nullable() && (field_cid != kNullCid)) { __ j(EQUAL, &ok); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ cmpq(value_reg, raw_null); + __ CompareObject(value_reg, Object::Handle()); } if (ok_is_fall_through) { @@ -1874,7 +1857,7 @@ void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Register value = locs()->in(0).reg(); Register temp = locs()->temp(0).reg(); - __ LoadObject(temp, field()); + __ LoadObject(temp, field(), PP); if (this->value()->NeedsStoreBuffer()) { __ StoreIntoObject(temp, FieldAddress(temp, Field::value_offset()), value, CanValueBeSmi()); @@ -2028,9 +2011,7 @@ void InstantiateTypeArgumentsInstr::EmitNativeCode( Label type_arguments_instantiated; const intptr_t len = type_arguments().Length(); if (type_arguments().IsRawInstantiatedRaw(len)) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ cmpq(instantiator_reg, raw_null); + __ CompareObject(instantiator_reg, Object::Handle()); __ j(EQUAL, &type_arguments_instantiated, Assembler::kNearJump); } // Instantiate non-null type arguments. @@ -2078,14 +2059,13 @@ void ExtractConstructorTypeArgumentsInstr::EmitNativeCode( // the type arguments. Label type_arguments_instantiated; ASSERT(type_arguments().IsRawInstantiatedRaw(type_arguments().Length())); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ cmpq(instantiator_reg, raw_null); + + __ CompareObject(instantiator_reg, Object::Handle()); __ j(EQUAL, &type_arguments_instantiated, Assembler::kNearJump); // Instantiate non-null type arguments. // In the non-factory case, we rely on the allocation stub to // instantiate the type arguments. - __ LoadObject(result_reg, type_arguments()); + __ LoadObject(result_reg, type_arguments(), PP); // result_reg: uninstantiated type arguments. __ Bind(&type_arguments_instantiated); @@ -2120,10 +2100,9 @@ void ExtractConstructorInstantiatorInstr::EmitNativeCode( // instantiated from null becomes a vector of dynamic, then use null as // the type arguments and do not pass the instantiator. ASSERT(type_arguments().IsRawInstantiatedRaw(type_arguments().Length())); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + Label instantiator_not_null; - __ cmpq(instantiator_reg, raw_null); + __ CompareObject(instantiator_reg, Object::Handle()); __ j(NOT_EQUAL, &instantiator_not_null, Assembler::kNearJump); // Null was used in VisitExtractConstructorTypeArguments as the // instantiated type arguments, no proper instantiator needed. @@ -2199,6 +2178,10 @@ void CatchBlockEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { compiler->assembler()->CodeSize(), catch_handler_types_, needs_stacktrace()); + + // Restore the pool pointer. + __ LoadPoolPointer(PP); + if (HasParallelMove()) { compiler->parallel_move_resolver()->EmitNativeCode(parallel_move()); } @@ -2279,7 +2262,7 @@ void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // In unoptimized code check the usage counter to trigger OSR at loop // stack checks. Use progressively higher thresholds for more deeply // nested loops to attempt to hit outer loops with OSR when possible. - __ LoadObject(temp, compiler->parsed_function().function()); + __ LoadObject(temp, compiler->parsed_function().function(), PP); intptr_t threshold = FLAG_optimization_counter_threshold * (loop_depth() + 1); __ cmpq(FieldAddress(temp, Function::usage_counter_offset()), @@ -3726,10 +3709,10 @@ void Uint32x4GetFlagInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ addq(RSP, Immediate(16)); __ testl(result, result); __ j(NOT_ZERO, &non_zero, Assembler::kNearJump); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ jmp(&done); __ Bind(&non_zero); - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); __ Bind(&done); } @@ -4247,9 +4230,9 @@ void InvokeMathCFunctionInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Label check_base_is_one; // Check if exponent is 0.0 -> return 1.0; - __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(0))); + __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(0)), PP); __ movsd(zero_temp, FieldAddress(temp, Double::value_offset())); - __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(1))); + __ LoadObject(temp, Double::ZoneHandle(Double::NewCanonical(1)), PP); __ movsd(result, FieldAddress(temp, Double::value_offset())); // 'result' contains 1.0. __ comisd(exp, zero_temp); @@ -4347,9 +4330,8 @@ void CheckClassInstr::EmitNativeCode(FlowGraphCompiler* compiler) { if (IsNullCheck()) { Label* deopt = compiler->AddDeoptStub(deopt_id(), kDeoptCheckClass); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ cmpq(locs()->in(0).reg(), raw_null); + __ CompareObject(locs()->in(0).reg(), + Object::Handle()); __ j(EQUAL, deopt); return; } @@ -4559,7 +4541,7 @@ void TargetEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { counter.SetAt(0, Smi::Handle(Smi::New(0))); Label done; __ Comment("Edge counter"); - __ LoadObject(RAX, counter); + __ LoadObject(RAX, counter, PP); __ addq(FieldAddress(RAX, Array::element_offset(0)), Immediate(Smi::RawValue(1))); __ j(NO_OVERFLOW, &done); @@ -4590,7 +4572,7 @@ void GotoInstr::EmitNativeCode(FlowGraphCompiler* compiler) { counter.SetAt(0, Smi::Handle(Smi::New(0))); Label done; __ Comment("Edge counter"); - __ LoadObject(RAX, counter); + __ LoadObject(RAX, counter, PP); __ addq(FieldAddress(RAX, Array::element_offset(0)), Immediate(Smi::RawValue(1))); __ j(NO_OVERFLOW, &done); @@ -4673,7 +4655,7 @@ void StrictCompareInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const bool result = (kind() == Token::kEQ_STRICT) ? left.constant().raw() == right.constant().raw() : left.constant().raw() != right.constant().raw(); - __ LoadObject(locs()->out().reg(), Bool::Get(result)); + __ LoadObject(locs()->out().reg(), Bool::Get(result), PP); return; } if (left.IsConstant()) { @@ -4697,10 +4679,10 @@ void StrictCompareInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Label load_true, done; Condition true_condition = (kind() == Token::kEQ_STRICT) ? EQUAL : NOT_EQUAL; __ j(true_condition, &load_true, Assembler::kNearJump); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ jmp(&done, Assembler::kNearJump); __ Bind(&load_true); - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); __ Bind(&done); } @@ -4759,7 +4741,7 @@ void ClosureCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Array& arguments_descriptor = Array::ZoneHandle(ArgumentsDescriptor::New(argument_count, argument_names())); - __ LoadObject(temp_reg, arguments_descriptor); + __ LoadObject(temp_reg, arguments_descriptor, PP); ASSERT(temp_reg == R10); compiler->GenerateDartCall(deopt_id(), token_pos(), @@ -4782,10 +4764,10 @@ void BooleanNegateInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Register result = locs()->out().reg(); Label done; - __ LoadObject(result, Bool::True()); + __ LoadObject(result, Bool::True(), PP); __ CompareRegisters(result, value); __ j(NOT_EQUAL, &done, Assembler::kNearJump); - __ LoadObject(result, Bool::False()); + __ LoadObject(result, Bool::False(), PP); __ Bind(&done); } diff --git a/runtime/vm/intrinsifier_x64.cc b/runtime/vm/intrinsifier_x64.cc index dc647ac794e..a688c330a66 100644 --- a/runtime/vm/intrinsifier_x64.cc +++ b/runtime/vm/intrinsifier_x64.cc @@ -117,15 +117,14 @@ void Intrinsifier::ObjectArray_Allocate(Assembler* assembler) { // RCX: new object end address. // RDI: iterator which initially points to the start of the variable // data area to be initialized. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + __ LoadObject(R12, Object::Handle(), PP); __ leaq(RDI, FieldAddress(RAX, sizeof(RawArray))); Label done; Label init_loop; __ Bind(&init_loop); __ cmpq(RDI, RCX); __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); - __ movq(Address(RDI, 0), raw_null); + __ movq(Address(RDI, 0), R12); __ addq(RDI, Immediate(kWordSize)); __ jmp(&init_loop, Assembler::kNearJump); __ Bind(&done); @@ -392,9 +391,7 @@ void Intrinsifier::GrowableArray_add(Assembler* assembler) { __ StoreIntoObject(RDX, FieldAddress(RDX, RCX, TIMES_4, Array::data_offset()), RAX); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(RAX, raw_null); + __ LoadObject(RAX, Object::Handle(), PP); __ ret(); __ Bind(&fall_through); } @@ -879,10 +876,10 @@ static void CompareIntegers(Assembler* assembler, Condition true_condition) { // RAX contains the right argument. __ cmpq(Address(RSP, + 2 * kWordSize), RAX); __ j(true_condition, &true_label, Assembler::kNearJump); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&true_label); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); __ Bind(&fall_through); } @@ -919,34 +916,39 @@ void Intrinsifier::Integer_greaterEqualThan(Assembler* assembler) { void Intrinsifier::Integer_equalToInteger(Assembler* assembler) { Label fall_through, true_label, check_for_mint; // For integer receiver '===' check first. - __ movq(RAX, Address(RSP, + 1 * kWordSize)); - __ movq(RCX, Address(RSP, + 2 * kWordSize)); + // Entering a dart frame so we can use the PP for loading True and False. + __ EnterDartFrame(0); + __ movq(RAX, Address(RSP, + 4 * kWordSize)); + __ movq(RCX, Address(RSP, + 5 * kWordSize)); __ cmpq(RAX, RCX); __ j(EQUAL, &true_label, Assembler::kNearJump); __ orq(RAX, RCX); __ testq(RAX, Immediate(kSmiTagMask)); __ j(NOT_ZERO, &check_for_mint, Assembler::kNearJump); // Both arguments are smi, '===' is good enough. - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); + __ LeaveFrameWithPP(); __ ret(); __ Bind(&true_label); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); + __ LeaveFrameWithPP(); __ ret(); // At least one of the arguments was not Smi. Label receiver_not_smi; __ Bind(&check_for_mint); - __ movq(RAX, Address(RSP, + 2 * kWordSize)); // Receiver. + __ movq(RAX, Address(RSP, + 5 * kWordSize)); // Receiver. __ testq(RAX, Immediate(kSmiTagMask)); __ j(NOT_ZERO, &receiver_not_smi); // Left (receiver) is Smi, return false if right is not Double. // Note that an instance of Mint or Bigint never contains a value that can be // represented by Smi. - __ movq(RAX, Address(RSP, + 1 * kWordSize)); + __ movq(RAX, Address(RSP, + 4 * kWordSize)); __ CompareClassId(RAX, kDoubleCid); __ j(EQUAL, &fall_through); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); + __ LeaveFrameWithPP(); __ ret(); __ Bind(&receiver_not_smi); @@ -954,14 +956,17 @@ void Intrinsifier::Integer_equalToInteger(Assembler* assembler) { __ CompareClassId(RAX, kMintCid); __ j(NOT_EQUAL, &fall_through); // Receiver is Mint, return false if right is Smi. - __ movq(RAX, Address(RSP, + 1 * kWordSize)); // Right argument. + __ movq(RAX, Address(RSP, + 4 * kWordSize)); // Right argument. __ testq(RAX, Immediate(kSmiTagMask)); __ j(NOT_ZERO, &fall_through); - __ LoadObject(RAX, Bool::False()); // Smi == Mint -> false. + // Smi == Mint -> false. + __ LoadObject(RAX, Bool::False(), PP); + __ LeaveFrameWithPP(); __ ret(); // TODO(srdjan): Implement Mint == Mint comparison. __ Bind(&fall_through); + __ LeaveFrameWithPP(); } @@ -1041,10 +1046,10 @@ static void CompareDoubles(Assembler* assembler, Condition true_condition) { __ j(true_condition, &is_true, Assembler::kNearJump); // Fall through false. __ Bind(&is_false); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&is_true); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); __ Bind(&is_smi); __ SmiUntag(RAX); @@ -1178,10 +1183,10 @@ void Intrinsifier::Double_getIsNaN(Assembler* assembler) { __ movsd(XMM0, FieldAddress(RAX, Double::value_offset())); __ comisd(XMM0, XMM0); __ j(PARITY_EVEN, &is_true, Assembler::kNearJump); // NaN -> true; - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&is_true); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); } @@ -1196,10 +1201,10 @@ void Intrinsifier::Double_getIsNegative(Assembler* assembler) { __ j(EQUAL, &is_zero, Assembler::kNearJump); // Check for negative zero. __ j(ABOVE_EQUAL, &is_false, Assembler::kNearJump); // >= 0 -> false. __ Bind(&is_true); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); __ Bind(&is_false); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&is_zero); // Check for negative zero (get the sign bit). @@ -1347,17 +1352,23 @@ void Intrinsifier::Random_nextState(Assembler* assembler) { } - // Identity comparison. void Intrinsifier::Object_equal(Assembler* assembler) { Label is_true; - __ movq(RAX, Address(RSP, + 1 * kWordSize)); - __ cmpq(RAX, Address(RSP, + 2 * kWordSize)); + // This intrinsic is used from the API even when we have not entered any + // Dart frame, yet, so the PP would otherwise be null in this case unless + // we enter a Dart frame here. + __ EnterDartFrame(0); + __ movq(RAX, Address(RSP, + 4 * kWordSize)); + __ cmpq(RAX, Address(RSP, + 5 * kWordSize)); __ j(EQUAL, &is_true, Assembler::kNearJump); - __ LoadObject(RAX, Bool::False()); + __ movq(RAX, Immediate(reinterpret_cast(Bool::False().raw()))); + __ LoadObject(RAX, Bool::False(), PP); + __ LeaveFrameWithPP(); __ ret(); __ Bind(&is_true); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); + __ LeaveFrameWithPP(); __ ret(); } @@ -1417,10 +1428,10 @@ void Intrinsifier::String_getIsEmpty(Assembler* assembler) { __ movq(RAX, FieldAddress(RAX, String::length_offset())); __ cmpq(RAX, Immediate(Smi::RawValue(0))); __ j(EQUAL, &is_true, Assembler::kNearJump); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&is_true); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); } diff --git a/runtime/vm/object_x64_test.cc b/runtime/vm/object_x64_test.cc index 74c0e5cd529..42e872c2182 100644 --- a/runtime/vm/object_x64_test.cc +++ b/runtime/vm/object_x64_test.cc @@ -34,7 +34,9 @@ void GenerateIncrement(Assembler* assembler) { void GenerateEmbedStringInCode(Assembler* assembler, const char* str) { const String& string_object = String::ZoneHandle(String::New(str, Heap::kOld)); - __ LoadObject(RAX, string_object); + __ EnterDartFrame(0); + __ LoadObject(RAX, string_object, PP); + __ LeaveFrameWithPP(); __ ret(); } @@ -43,7 +45,7 @@ void GenerateEmbedStringInCode(Assembler* assembler, const char* str) { // This is used to test Embedded Smi objects in the instructions. void GenerateEmbedSmiInCode(Assembler* assembler, intptr_t value) { const Smi& smi_object = Smi::ZoneHandle(Smi::New(value)); - __ LoadObject(RAX, smi_object); + __ LoadObject(RAX, smi_object, PP); __ ret(); } diff --git a/runtime/vm/runtime_entry_x64.cc b/runtime/vm/runtime_entry_x64.cc index 13fa2a1b666..568373822ba 100644 --- a/runtime/vm/runtime_entry_x64.cc +++ b/runtime/vm/runtime_entry_x64.cc @@ -30,7 +30,7 @@ void RuntimeEntry::Call(Assembler* assembler, intptr_t argument_count) const { // informative error message. __ movq(RBX, Immediate(GetEntryPoint())); __ movq(R10, Immediate(argument_count)); - __ call(&StubCode::CallToRuntimeLabel()); + __ Call(&StubCode::CallToRuntimeLabel(), PP); } } diff --git a/runtime/vm/stack_frame_x64.h b/runtime/vm/stack_frame_x64.h index fb6e11e04f7..33c87b666a6 100644 --- a/runtime/vm/stack_frame_x64.h +++ b/runtime/vm/stack_frame_x64.h @@ -11,11 +11,14 @@ namespace dart { | | <- TOS Callee frame | ... | + | saved PP | + | callee's PC marker | | saved RBP | (RBP of current frame) | saved PC | (PC of current frame) +--------------------+ Current frame | ... | <- RSP of current frame | first local | + | caller's PP | | PC marker | (current frame's code entry + offset) | caller's RBP | <- RBP of current frame | caller's ret addr | (PC of caller frame) @@ -24,21 +27,22 @@ Caller frame | last parameter | <- RSP of caller frame | ... | */ -static const int kDartFrameFixedSize = 3; // PC marker, RBP, PC. +static const int kDartFrameFixedSize = 4; // PC marker, RBP, PP, PC. static const int kSavedPcSlotFromSp = -1; -static const int kFirstLocalSlotFromFp = -2; + +static const int kFirstLocalSlotFromFp = -3; +static const int kSavedCallerPpSlotFromFp = -2; static const int kPcMarkerSlotFromFp = -1; static const int kSavedCallerFpSlotFromFp = 0; static const int kSavedCallerPcSlotFromFp = 1; + static const int kParamEndSlotFromFp = 1; // One slot past last parameter. static const int kCallerSpSlotFromFp = 2; - -// No pool pointer on X64 (indicated by aliasing saved fp). -static const int kSavedCallerPpSlotFromFp = kSavedCallerFpSlotFromFp; +static const int kSavedAboveReturnAddress = 3; // Saved above return address. // Entry and exit frame layout. -static const int kSavedContextSlotFromEntryFp = -9; -static const int kExitLinkSlotFromEntryFp = -8; +static const int kSavedContextSlotFromEntryFp = -10; +static const int kExitLinkSlotFromEntryFp = -9; } // namespace dart diff --git a/runtime/vm/stub_code.h b/runtime/vm/stub_code.h index e4f18a23d83..3e185ee0973 100644 --- a/runtime/vm/stub_code.h +++ b/runtime/vm/stub_code.h @@ -182,7 +182,8 @@ class StubCode { StubEntry* name##_entry_; STUB_CODE_LIST(STUB_CODE_ENTRY); #undef STUB_CODE_ENTRY - // This dummy field is needed so that we can intialize the stubs from a macro. + // This dummy field is needed so that we can initialize + // the stubs from a macro. void* dummy_; // Generate the stub and finalize the generated code into the stub diff --git a/runtime/vm/stub_code_x64.cc b/runtime/vm/stub_code_x64.cc index 3039a97613b..111e2e75503 100644 --- a/runtime/vm/stub_code_x64.cc +++ b/runtime/vm/stub_code_x64.cc @@ -83,9 +83,8 @@ void StubCode::GenerateCallToRuntimeStub(Assembler* assembler) { __ movq(RBX, Address(CTX, Isolate::top_context_offset())); // Reset Context pointer in Isolate structure. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(Address(CTX, Isolate::top_context_offset()), raw_null); + __ LoadObject(R12, Object::Handle(), PP); + __ movq(Address(CTX, Isolate::top_context_offset()), R12); // Cache Context pointer into CTX while executing Dart code. __ movq(CTX, RBX); @@ -172,9 +171,8 @@ void StubCode::GenerateCallNativeCFunctionStub(Assembler* assembler) { __ movq(R8, Address(CTX, Isolate::top_context_offset())); // Reset Context pointer in Isolate structure. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(Address(CTX, Isolate::top_context_offset()), raw_null); + __ LoadObject(R12, Object::Handle(), PP); + __ movq(Address(CTX, Isolate::top_context_offset()), R12); // Cache Context pointer into CTX while executing Dart code. __ movq(CTX, R8); @@ -240,9 +238,8 @@ void StubCode::GenerateCallBootstrapCFunctionStub(Assembler* assembler) { __ movq(R8, Address(CTX, Isolate::top_context_offset())); // Reset Context pointer in Isolate structure. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ movq(Address(CTX, Isolate::top_context_offset()), raw_null); + __ LoadObject(R12, Object::Handle(), PP); + __ movq(Address(CTX, Isolate::top_context_offset()), R12); // Cache Context pointer into CTX while executing Dart code. __ movq(CTX, R8); @@ -255,11 +252,10 @@ void StubCode::GenerateCallBootstrapCFunctionStub(Assembler* assembler) { // Input parameters: // R10: arguments descriptor array. void StubCode::GenerateCallStaticFunctionStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); __ EnterStubFrame(); __ pushq(R10); // Preserve arguments descriptor array. - __ pushq(raw_null); // Setup space on stack for return value. + // Setup space on stack for return value. + __ PushObject(Object::Handle()); __ CallRuntime(kPatchStaticCallRuntimeEntry, 0); __ popq(RAX); // Get Code object result. __ popq(R10); // Restore arguments descriptor array. @@ -276,11 +272,10 @@ void StubCode::GenerateCallStaticFunctionStub(Assembler* assembler) { // (invalid because its function was optimized or deoptimized). // R10: arguments descriptor array. void StubCode::GenerateFixCallersTargetStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); __ EnterStubFrame(); __ pushq(R10); // Preserve arguments descriptor array. - __ pushq(raw_null); // Setup space on stack for return value. + // Setup space on stack for return value. + __ PushObject(Object::Handle()); __ CallRuntime(kFixCallersTargetRuntimeEntry, 0); __ popq(RAX); // Get Code object. __ popq(R10); // Restore arguments descriptor array. @@ -296,11 +291,9 @@ void StubCode::GenerateFixCallersTargetStub(Assembler* assembler) { // R10: smi-tagged argument count, may be zero. // RBP[kParamEndSlotFromFp + 1]: last argument. static void PushArgumentsArray(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - + __ LoadObject(R12, Object::Handle(), PP); // Allocate array to store arguments of caller. - __ movq(RBX, raw_null); // Null element type for raw Array. + __ movq(RBX, R12); // Null element type for raw Array. __ call(&StubCode::AllocateArrayLabel()); __ SmiUntag(R10); // RAX: newly allocated array. @@ -330,18 +323,15 @@ static void PushArgumentsArray(Assembler* assembler) { // called, the stub accesses the receiver from this location directly // when trying to resolve the call. void StubCode::GenerateInstanceFunctionLookupStub(Assembler* assembler) { - __ EnterStubFrame(); - - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ pushq(raw_null); // Space for the return value. + __ EnterStubFrameWithPP(); + __ PushObject(Object::Handle()); // Space for the return value. // Push the receiver as an argument. Load the smi-tagged argument // count into R13 to index the receiver in the stack. There are - // three words (null, stub's pc marker, saved fp) above the return + // four words (null, stub's pc marker, saved pp, saved fp) above the return // address. __ movq(R13, FieldAddress(R10, ArgumentsDescriptor::count_offset())); - __ pushq(Address(RSP, R13, TIMES_4, (3 * kWordSize))); + __ pushq(Address(RSP, R13, TIMES_4, (4 * kWordSize))); __ pushq(RBX); // Pass IC data object. __ pushq(R10); // Pass arguments descriptor array. @@ -355,7 +345,7 @@ void StubCode::GenerateInstanceFunctionLookupStub(Assembler* assembler) { // Remove arguments. __ Drop(4); __ popq(RAX); // Get result into RAX. - __ LeaveFrame(); + __ LeaveFrameWithPP(); __ ret(); } @@ -378,7 +368,9 @@ DECLARE_LEAF_RUNTIME_ENTRY(void, DeoptimizeFillFrame, uword last_fp); // - Fill the unoptimized frame. // - Materialize objects that require allocation (e.g. Double instances). // GC can occur only after frame is fully rewritten. -// Stack after EnterDartFrame(0) below: +// Stack after EnterDartFrame(0, PP, kNoRegister) below: +// +------------------+ +// | Saved PP | <- PP // +------------------+ // | PC marker | <- TOS // +------------------+ @@ -391,8 +383,12 @@ DECLARE_LEAF_RUNTIME_ENTRY(void, DeoptimizeFillFrame, uword last_fp); // Parts of the code cannot GC, part of the code can GC. static void GenerateDeoptimizationSequence(Assembler* assembler, bool preserve_result) { - // Leaf runtime function DeoptimizeCopyFrame expects a Dart frame. - __ EnterDartFrame(0); + // DeoptimizeCopyFrame expects a Dart frame, i.e. EnterDartFrame(0), but there + // is no need to set the correct PC marker or load PP, since they get patched. + __ EnterFrame(0); + __ pushq(Immediate(0)); + __ pushq(PP); + // The code in this frame may not cause GC. kDeoptimizeCopyFrameRuntimeEntry // and kDeoptimizeFillFrameRuntimeEntry are leaf runtime calls. const intptr_t saved_result_slot_from_fp = @@ -422,14 +418,21 @@ static void GenerateDeoptimizationSequence(Assembler* assembler, __ movq(RBX, Address(RBP, saved_result_slot_from_fp * kWordSize)); } + // There is a Dart Frame on the stack. We just need the PP. + __ movq(PP, Address(RBP, -2 * kWordSize)); __ LeaveFrame(); + __ popq(RCX); // Preserve return address. __ movq(RSP, RBP); // Discard optimized frame. __ subq(RSP, RAX); // Reserve space for deoptimized frame. __ pushq(RCX); // Restore return address. - // Leaf runtime function DeoptimizeFillFrame expects a Dart frame. - __ EnterDartFrame(0); + // DeoptimizeFillFrame expects a Dart frame, i.e. EnterDartFrame(0), but there + // is no need to set the correct PC marker or load PP, since they get patched. + __ EnterFrame(0); + __ pushq(Immediate(0)); + __ pushq(PP); + if (preserve_result) { __ pushq(RBX); // Preserve result as first local. } @@ -441,6 +444,8 @@ static void GenerateDeoptimizationSequence(Assembler* assembler, __ movq(RBX, Address(RBP, kFirstLocalSlotFromFp * kWordSize)); } // Code above cannot cause GC. + // There is a Dart Frame on the stack. We just need the PP. + __ movq(PP, Address(RBP, -2 * kWordSize)); __ LeaveFrame(); // Frame is fully rewritten at this point and it is safe to perform a GC. @@ -486,20 +491,20 @@ void StubCode::GenerateDeoptimizeStub(Assembler* assembler) { void StubCode::GenerateMegamorphicMissStub(Assembler* assembler) { - __ EnterStubFrame(); + __ EnterStubFrameWithPP(); // Load the receiver into RAX. The argument count in the arguments // descriptor in R10 is a smi. __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset())); - // Two words (saved fp, stub's pc marker) in the stack above the return - // address. - __ movq(RAX, Address(RSP, RAX, TIMES_4, 2 * kWordSize)); + // Three words (saved pp, saved fp, stub's pc marker) + // in the stack above the return address. + __ movq(RAX, Address(RSP, RAX, TIMES_4, + kSavedAboveReturnAddress * kWordSize)); // Preserve IC data and arguments descriptor. __ pushq(RBX); __ pushq(R10); - const Immediate& raw_null = - Immediate(reinterpret_cast(Instructions::null())); - __ pushq(raw_null); // Space for the result of the runtime call. + // Space for the result of the runtime call. + __ PushObject(Object::Handle()); __ pushq(RAX); // Receiver. __ pushq(RBX); // IC data. __ pushq(R10); // Arguments descriptor. @@ -511,10 +516,10 @@ void StubCode::GenerateMegamorphicMissStub(Assembler* assembler) { __ popq(RAX); // Return value from the runtime call (instructions). __ popq(R10); // Restore arguments descriptor. __ popq(RBX); // Restore IC data. - __ LeaveFrame(); + __ LeaveFrameWithPP(); Label lookup; - __ cmpq(RAX, raw_null); + __ CompareObject(RAX, Object::Handle()); __ j(EQUAL, &lookup, Assembler::kNearJump); __ addq(RAX, Immediate(Instructions::HeaderSize() - kHeapObjectTag)); __ jmp(RAX); @@ -532,8 +537,6 @@ void StubCode::GenerateMegamorphicMissStub(Assembler* assembler) { // The newly allocated object is returned in RAX. void StubCode::GenerateAllocateArrayStub(Assembler* assembler) { Label slow_case; - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); if (FLAG_inline_alloc) { // Compute the size to be allocated, it is based on the array length @@ -622,13 +625,14 @@ void StubCode::GenerateAllocateArrayStub(Assembler* assembler) { __ leaq(RBX, FieldAddress(RAX, Array::data_offset())); // RBX: iterator which initially points to the start of the variable // data area to be initialized. + __ LoadObject(R13, Object::Handle(), PP); Label done; Label init_loop; __ Bind(&init_loop); __ cmpq(RBX, R12); __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); // TODO(cshapiro): StoreIntoObjectNoBarrier - __ movq(Address(RBX, 0), raw_null); + __ movq(Address(RBX, 0), R13); __ addq(RBX, Immediate(kWordSize)); __ jmp(&init_loop, Assembler::kNearJump); __ Bind(&done); @@ -645,7 +649,8 @@ void StubCode::GenerateAllocateArrayStub(Assembler* assembler) { // Create a stub frame as we are pushing some objects on the stack before // calling into the runtime. __ EnterStubFrame(); - __ pushq(raw_null); // Setup space on stack for return value. + // Setup space on stack for return value. + __ PushObject(Object::Handle()); __ pushq(R10); // Array length as Smi. __ pushq(RBX); // Element type. __ CallRuntime(kAllocateArrayRuntimeEntry, 2); @@ -663,17 +668,16 @@ void StubCode::GenerateAllocateArrayStub(Assembler* assembler) { // called, the stub accesses the closure from this location directly // when trying to resolve the call. void StubCode::GenerateCallClosureFunctionStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - // Load num_args. __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset())); // Load closure object in R13. __ movq(R13, Address(RSP, RAX, TIMES_4, 0)); // RAX is a Smi. + __ LoadObject(R12, Object::Handle(), PP); + // Verify that R13 is a closure by checking its class. Label not_closure; - __ cmpq(R13, raw_null); + __ cmpq(R13, R12); // Not a closure, but null object. __ j(EQUAL, ¬_closure); __ testq(R13, Immediate(kSmiTagMask)); @@ -682,7 +686,7 @@ void StubCode::GenerateCallClosureFunctionStub(Assembler* assembler) { // class.signature_function() is not null. __ LoadClass(RAX, R13); __ movq(RAX, FieldAddress(RAX, Class::signature_function_offset())); - __ cmpq(RAX, raw_null); + __ cmpq(RAX, R12); // Actual class is not a closure class. __ j(EQUAL, ¬_closure, Assembler::kNearJump); @@ -694,7 +698,7 @@ void StubCode::GenerateCallClosureFunctionStub(Assembler* assembler) { // Load closure function code in RAX. __ movq(RAX, FieldAddress(RBX, Function::code_offset())); - __ cmpq(RAX, raw_null); + __ cmpq(RAX, R12); Label function_compiled; __ j(NOT_EQUAL, &function_compiled, Assembler::kNearJump); @@ -733,8 +737,8 @@ void StubCode::GenerateCallClosureFunctionStub(Assembler* assembler) { // Create a stub frame as we are pushing some objects on the stack before // calling into the runtime. __ EnterStubFrame(); - - __ pushq(raw_null); // Setup space on stack for result from call. + // Setup space on stack for result from call. + __ pushq(R12); __ pushq(R10); // Arguments descriptor. // Load smi-tagged arguments array length, including the non-closure. __ movq(R10, FieldAddress(R10, ArgumentsDescriptor::count_offset())); @@ -761,12 +765,12 @@ void StubCode::GenerateCallClosureFunctionStub(Assembler* assembler) { // RCX : new context containing the current isolate pointer. void StubCode::GenerateInvokeDartCodeStub(Assembler* assembler) { // Save frame pointer coming in. - __ EnterFrame(0); + __ EnterStubFrame(); // Save arguments descriptor array and new context. - const intptr_t kArgumentsDescOffset = -1 * kWordSize; + const intptr_t kArgumentsDescOffset = -2 * kWordSize; __ pushq(RSI); - const intptr_t kNewContextOffset = -2 * kWordSize; + const intptr_t kNewContextOffset = -3 * kWordSize; __ pushq(RCX); // Save C++ ABI callee-saved registers. @@ -792,7 +796,7 @@ void StubCode::GenerateInvokeDartCodeStub(Assembler* assembler) { // StackFrameIterator reads the top exit frame info saved in this frame. // The constant kExitLinkSlotFromEntryFp must be kept in sync with the // code below. - ASSERT(kExitLinkSlotFromEntryFp == -8); + ASSERT(kExitLinkSlotFromEntryFp == -9); __ movq(RAX, Address(R8, Isolate::top_exit_frame_info_offset())); __ pushq(RAX); __ movq(Address(R8, Isolate::top_exit_frame_info_offset()), Immediate(0)); @@ -804,7 +808,7 @@ void StubCode::GenerateInvokeDartCodeStub(Assembler* assembler) { // EntryFrame::SavedContext reads the context saved in this frame. // The constant kSavedContextSlotFromEntryFp must be kept in sync with // the code below. - ASSERT(kSavedContextSlotFromEntryFp == -9); + ASSERT(kSavedContextSlotFromEntryFp == -10); __ movq(RAX, Address(R8, Isolate::top_context_offset())); __ pushq(RAX); @@ -881,8 +885,7 @@ void StubCode::GenerateInvokeDartCodeStub(Assembler* assembler) { // Output: // RAX: new allocated RawContext object. void StubCode::GenerateAllocateContextStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + __ LoadObject(R12, Object::Handle(), PP); if (FLAG_inline_alloc) { const Class& context_class = Class::ZoneHandle(Object::context_class()); Label slow_case; @@ -957,12 +960,10 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { // R13: Isolate, not an object. __ movq(FieldAddress(RAX, Context::isolate_offset()), R13); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); // Setup the parent field. // RAX: new object. // R10: number of context variables. - __ movq(FieldAddress(RAX, Context::parent_offset()), raw_null); + __ movq(FieldAddress(RAX, Context::parent_offset()), R12); // Initialize the context variables. // RAX: new object. @@ -974,7 +975,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { __ jmp(&entry, Assembler::kNearJump); __ Bind(&loop); __ decq(R10); - __ movq(Address(R13, R10, TIMES_8, 0), raw_null); + __ movq(Address(R13, R10, TIMES_8, 0), R12); __ Bind(&entry); __ cmpq(R10, Immediate(0)); __ j(NOT_EQUAL, &loop, Assembler::kNearJump); @@ -988,7 +989,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { } // Create a stub frame. __ EnterStubFrame(); - __ pushq(raw_null); // Setup space on stack for the return value. + __ pushq(R12); // Setup space on stack for the return value. __ SmiTag(R10); __ pushq(R10); // Push number of context variables. __ CallRuntime(kAllocateContextRuntimeEntry, 1); // Allocate context. @@ -1072,8 +1073,6 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, const Class& cls) { const intptr_t kObjectTypeArgumentsOffset = 2 * kWordSize; const intptr_t kInstantiatorTypeArgumentsOffset = 1 * kWordSize; - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); // The generated code is different if the class is parameterized. const bool is_cls_parameterized = cls.HasTypeArguments(); ASSERT(!cls.HasTypeArguments() || @@ -1085,6 +1084,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, const intptr_t instance_size = cls.instance_size(); ASSERT(instance_size > 0); const intptr_t type_args_size = InstantiatedTypeArguments::InstanceSize(); + __ LoadObject(R12, Object::Handle(), PP); if (FLAG_inline_alloc && Heap::IsAllocatableInNewSpace(instance_size + type_args_size)) { Label slow_case; @@ -1168,9 +1168,6 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, __ movq(Address(RAX, Instance::tags_offset()), Immediate(tags)); // Initialize the remaining words of the object. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - // RAX: new object start. // RBX: next object start. // RDI: new object type arguments (if is_cls_parameterized). @@ -1181,7 +1178,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, for (intptr_t current_offset = sizeof(RawObject); current_offset < instance_size; current_offset += kWordSize) { - __ movq(Address(RAX, current_offset), raw_null); + __ movq(Address(RAX, current_offset), R12); } } else { __ leaq(RCX, Address(RAX, sizeof(RawObject))); @@ -1195,7 +1192,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, __ Bind(&init_loop); __ cmpq(RCX, RBX); __ j(ABOVE_EQUAL, &done, Assembler::kNearJump); - __ movq(Address(RCX, 0), raw_null); + __ movq(Address(RCX, 0), R12); __ addq(RCX, Immediate(kWordSize)); __ jmp(&init_loop, Assembler::kNearJump); __ Bind(&done); @@ -1217,14 +1214,14 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, __ movq(RDX, Address(RSP, kInstantiatorTypeArgumentsOffset)); } // Create a stub frame. - __ EnterStubFrame(); - __ pushq(raw_null); // Setup space on stack for return value. + __ EnterStubFrameWithPP(); + __ pushq(R12); // Setup space on stack for return value. __ PushObject(cls); // Push class of object to be allocated. if (is_cls_parameterized) { __ pushq(RAX); // Push type arguments of object to be allocated. __ pushq(RDX); // Push type arguments of instantiator. } else { - __ pushq(raw_null); // Push null type arguments. + __ pushq(R12); // Push null type arguments. __ pushq(Immediate(Smi::RawValue(StubCode::kNoInstantiator))); } __ CallRuntime(kAllocateObjectRuntimeEntry, 3); // Allocate object. @@ -1234,7 +1231,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, __ popq(RAX); // Pop result (newly allocated object). // RAX: new object // Restore the frame pointer. - __ LeaveFrame(); + __ LeaveFrameWithPP(); __ ret(); } @@ -1246,16 +1243,17 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // RSP : points to return address. void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, const Function& func) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); ASSERT(func.IsClosureFunction()); ASSERT(!func.IsImplicitStaticClosureFunction()); const bool is_implicit_instance_closure = func.IsImplicitInstanceClosureFunction(); const Class& cls = Class::ZoneHandle(func.signature_class()); const bool has_type_arguments = cls.HasTypeArguments(); - const intptr_t kTypeArgumentsOffset = 1 * kWordSize; - const intptr_t kReceiverOffset = 2 * kWordSize; + + __ EnterStubFrameWithPP(); // Uses pool pointer to refer to function. + __ LoadObject(R12, Object::Handle(), PP); + const intptr_t kTypeArgumentsOffset = 4 * kWordSize; + const intptr_t kReceiverOffset = 5 * kWordSize; const intptr_t closure_size = Closure::InstanceSize(); const intptr_t context_size = Context::InstanceSize(1); // Captured receiver. if (FLAG_inline_alloc && @@ -1298,7 +1296,8 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, // RAX: new closure object. // RBX: new context object (only if is_implicit_closure). // R13: next object start. - __ LoadObject(R10, func); // Load function of closure to be allocated. + // Load function of closure to be allocated. + __ LoadObject(R10, func, PP); __ movq(Address(RAX, Closure::function_offset()), R10); // Setup the context for this closure. @@ -1320,7 +1319,7 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, __ movq(Address(RBX, Context::isolate_offset()), R10); // Set the parent to null. - __ movq(Address(RBX, Context::parent_offset()), raw_null); + __ movq(Address(RBX, Context::parent_offset()), R12); // Initialize the context variable to the receiver. __ movq(R10, Address(RSP, kReceiverOffset)); @@ -1340,6 +1339,7 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, // Done allocating and initializing the instance. // RAX: new object. __ addq(RAX, Immediate(kHeapObjectTag)); + __ LeaveFrameWithPP(); __ ret(); __ Bind(&slow_case); @@ -1350,9 +1350,8 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, if (is_implicit_instance_closure) { __ movq(RAX, Address(RSP, kReceiverOffset)); } - // Create the stub frame. - __ EnterStubFrame(); - __ pushq(raw_null); // Setup space on stack for the return value. + + __ pushq(R12); // Setup space on stack for the return value. __ PushObject(func); if (is_implicit_instance_closure) { __ pushq(RAX); // Receiver. @@ -1360,7 +1359,7 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, if (has_type_arguments) { __ pushq(RCX); // Push type arguments of closure to be allocated. } else { - __ pushq(raw_null); // Push null type arguments. + __ pushq(R12); // Push null type arguments. } if (is_implicit_instance_closure) { __ CallRuntime(kAllocateImplicitInstanceClosureRuntimeEntry, 3); @@ -1375,7 +1374,7 @@ void StubCode::GenerateAllocationStubForClosure(Assembler* assembler, __ popq(RAX); // Pop the result. // RAX: New closure object. // Restore the calling frame. - __ LeaveFrame(); + __ LeaveFrameWithPP(); __ ret(); } @@ -1395,9 +1394,8 @@ void StubCode::GenerateCallNoSuchMethodFunctionStub(Assembler* assembler) { __ movq(R13, FieldAddress(R10, ArgumentsDescriptor::count_offset())); __ movq(RAX, Address(RBP, R13, TIMES_4, kParamEndSlotFromFp * kWordSize)); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ pushq(raw_null); // Setup space on stack for result from noSuchMethod. + __ LoadObject(R12, Object::Handle(), PP); + __ pushq(R12); // Setup space on stack for result from noSuchMethod. __ pushq(RAX); // Receiver. __ pushq(RBX); // IC data array. __ pushq(R10); // Arguments descriptor array. @@ -1545,8 +1543,7 @@ void StubCode::GenerateNArgsCheckInlineCacheStub( __ j(NOT_EQUAL, &loop, Assembler::kNearJump); // IC miss. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); + __ LoadObject(R12, Object::Handle(), PP); // Compute address of arguments (first read number of arguments from // arguments descriptor array and then compute address on the stack). __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset())); @@ -1554,7 +1551,7 @@ void StubCode::GenerateNArgsCheckInlineCacheStub( __ EnterStubFrame(); __ pushq(R10); // Preserve arguments descriptor array. __ pushq(RBX); // Preserve IC data object. - __ pushq(raw_null); // Setup space on stack for result (target code object). + __ pushq(R12); // Setup space on stack for result (target code object). // Push call arguments. for (intptr_t i = 0; i < num_args; i++) { __ movq(RCX, Address(RAX, -kWordSize * i)); @@ -1571,7 +1568,7 @@ void StubCode::GenerateNArgsCheckInlineCacheStub( __ popq(R10); // Restore arguments descriptor array. __ LeaveFrame(); Label call_target_function; - __ cmpq(RAX, raw_null); + __ cmpq(RAX, R12); __ j(NOT_EQUAL, &call_target_function, Assembler::kNearJump); // NoSuchMethod or closure. // Mark IC call that it may be a closure call that does not collect @@ -1737,13 +1734,12 @@ void StubCode::GenerateZeroArgsUnoptimizedStaticCallStub(Assembler* assembler) { Immediate(Smi::RawValue(Smi::kMaxValue))); __ Bind(&increment_done); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); Label target_is_compiled; // Get function and call it, if possible. __ movq(R13, Address(R12, target_offset)); __ movq(RAX, FieldAddress(R13, Function::code_offset())); - __ cmpq(RAX, raw_null); + __ LoadObject(R12, Object::Handle(), PP); + __ cmpq(RAX, R12); __ j(NOT_EQUAL, &target_is_compiled, Assembler::kNearJump); __ EnterStubFrame(); @@ -1783,9 +1779,8 @@ void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { __ pushq(R10); // Room for result. Debugger stub returns address of the // unpatched runtime stub. - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ pushq(raw_null); // Room for result. + __ LoadObject(R12, Object::Handle(), PP); + __ pushq(R12); // Room for result. __ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0); __ popq(RAX); // Address of original. __ popq(R10); // Restore arguments. @@ -1798,11 +1793,10 @@ void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { // RBX: ICData (unoptimized static call) // TOS(0): return address (Dart code). void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); __ EnterStubFrame(); + __ LoadObject(R12, Object::Handle(), PP); __ pushq(RBX); // Preserve IC data for unoptimized call. - __ pushq(raw_null); // Room for result. + __ pushq(R12); // Room for result. __ CallRuntime(kBreakpointStaticHandlerRuntimeEntry, 0); __ popq(RAX); // Code object. __ popq(RBX); // Restore IC data. @@ -1827,7 +1821,7 @@ void StubCode::GenerateBreakpointReturnStub(Assembler* assembler) { __ LeaveFrame(); __ popq(R11); // discard return address of call to this stub. - __ LeaveFrame(); + __ LeaveFrameWithPP(); __ ret(); } @@ -1868,17 +1862,16 @@ void StubCode::GenerateBreakpointDynamicStub(Assembler* assembler) { // Result in RCX: null -> not found, otherwise result (true or false). static void GenerateSubtypeNTestCacheStub(Assembler* assembler, int n) { ASSERT((1 <= n) && (n <= 3)); - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); const intptr_t kInstantiatorTypeArgumentsInBytes = 1 * kWordSize; const intptr_t kInstanceOffsetInBytes = 2 * kWordSize; const intptr_t kCacheOffsetInBytes = 3 * kWordSize; __ movq(RAX, Address(RSP, kInstanceOffsetInBytes)); + __ LoadObject(R12, Object::Handle(), PP); if (n > 1) { __ LoadClass(R10, RAX); // Compute instance type arguments into R13. Label has_no_type_arguments; - __ movq(R13, raw_null); + __ movq(R13, R12); __ movq(RDI, FieldAddress(R10, Class::type_arguments_field_offset_in_words_offset())); __ cmpq(RDI, Immediate(Class::kNoTypeArguments)); @@ -1900,7 +1893,7 @@ static void GenerateSubtypeNTestCacheStub(Assembler* assembler, int n) { __ SmiTag(R10); __ Bind(&loop); __ movq(RDI, Address(RDX, kWordSize * SubtypeTestCache::kInstanceClassId)); - __ cmpq(RDI, raw_null); + __ cmpq(RDI, R12); __ j(EQUAL, ¬_found, Assembler::kNearJump); __ cmpq(RDI, R10); if (n == 1) { @@ -1927,7 +1920,7 @@ static void GenerateSubtypeNTestCacheStub(Assembler* assembler, int n) { __ jmp(&loop, Assembler::kNearJump); // Fall through to not found. __ Bind(¬_found); - __ movq(RCX, raw_null); + __ movq(RCX, R12); __ ret(); __ Bind(&found); @@ -2060,10 +2053,10 @@ void StubCode::GenerateEqualityWithNullArgStub(Assembler* assembler) { __ movq(RAX, Address(RSP, 1 * kWordSize)); __ cmpq(RAX, Address(RSP, 2 * kWordSize)); __ j(EQUAL, &true_label, Assembler::kNearJump); - __ LoadObject(RAX, Bool::False()); + __ LoadObject(RAX, Bool::False(), PP); __ ret(); __ Bind(&true_label); - __ LoadObject(RAX, Bool::True()); + __ LoadObject(RAX, Bool::True(), PP); __ ret(); __ Bind(&get_class_id_as_smi); @@ -2081,7 +2074,7 @@ void StubCode::GenerateEqualityWithNullArgStub(Assembler* assembler) { __ Bind(&update_ic_data); - // RCX: ICData + // RBX: ICData __ movq(RAX, Address(RSP, 1 * kWordSize)); __ movq(R13, Address(RSP, 2 * kWordSize)); __ EnterStubFrame(); @@ -2100,11 +2093,10 @@ void StubCode::GenerateEqualityWithNullArgStub(Assembler* assembler) { // RDI: function to be reoptimized. // R10: argument descriptor (preserved). void StubCode::GenerateOptimizeFunctionStub(Assembler* assembler) { - const Immediate& raw_null = - Immediate(reinterpret_cast(Object::null())); - __ EnterStubFrame(); + __ EnterStubFrameWithPP(); + __ LoadObject(R12, Object::Handle(), PP); __ pushq(R10); - __ pushq(raw_null); // Setup space on stack for return value. + __ pushq(R12); // Setup space on stack for return value. __ pushq(RDI); __ CallRuntime(kOptimizeInvokedFunctionRuntimeEntry, 1); __ popq(RAX); // Disard argument. @@ -2112,7 +2104,7 @@ void StubCode::GenerateOptimizeFunctionStub(Assembler* assembler) { __ popq(R10); // Restore argument descriptor. __ movq(RAX, FieldAddress(RAX, Code::instructions_offset())); __ addq(RAX, Immediate(Instructions::HeaderSize() - kHeapObjectTag)); - __ LeaveFrame(); + __ LeaveFrameWithPP(); __ jmp(RAX); __ int3(); } diff --git a/runtime/vm/stub_code_x64_test.cc b/runtime/vm/stub_code_x64_test.cc index 769134db93e..3efc152d0ef 100644 --- a/runtime/vm/stub_code_x64_test.cc +++ b/runtime/vm/stub_code_x64_test.cc @@ -45,8 +45,8 @@ static void GenerateCallToCallRuntimeStub(Assembler* assembler, const Object& result = Object::ZoneHandle(); const Context& context = Context::ZoneHandle(Context::New(0, Heap::kOld)); ASSERT(context.isolate() == Isolate::Current()); - __ enter(Immediate(0)); - __ LoadObject(CTX, context); + __ EnterStubFrameWithPP(); + __ LoadObject(CTX, context, PP); __ PushObject(result); // Push Null object for return value. __ PushObject(smi1); // Push argument 1 smi1. __ PushObject(smi2); // Push argument 2 smi2. @@ -54,7 +54,7 @@ static void GenerateCallToCallRuntimeStub(Assembler* assembler, __ CallRuntime(kTestSmiSubRuntimeEntry, argc); // Call SmiSub runtime func. __ AddImmediate(RSP, Immediate(argc * kWordSize)); __ popq(RAX); // Pop return value from return slot. - __ leave(); + __ LeaveFrameWithPP(); __ ret(); } @@ -84,8 +84,8 @@ static void GenerateCallToCallLeafRuntimeStub(Assembler* assembler, const Smi& smi2 = Smi::ZoneHandle(Smi::New(value2)); __ enter(Immediate(0)); __ ReserveAlignedFrameSpace(0); - __ LoadObject(RDI, smi1); // Set up argument 1 smi1. - __ LoadObject(RSI, smi2); // Set up argument 2 smi2. + __ LoadObject(RDI, smi1, PP); // Set up argument 1 smi1. + __ LoadObject(RSI, smi2, PP); // Set up argument 2 smi2. __ CallRuntime(kTestLeafSmiAddRuntimeEntry, 2); // Call SmiAdd runtime func. __ leave(); __ ret(); // Return value is in RAX.