From 8e7ffafbafc8203361111ddcafe0e0fcc372edf8 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Thu, 17 Oct 2019 21:45:38 +0000 Subject: [PATCH] Reland "[vm, compiler] Reduce alignment of Instructions and remove some debugging trap instructions." Further adjustments for simarm_x64. Flutter Gallery: ARM32 Instructions(CodeSize): 6979424 -> 6771856 (-2.97%) Total(CodeSize): 10855163 -> 10363319 (-4.53%) ARM64 Instructions(CodeSize): 7334368 -> 6817312 (-7.05%) Total(CodeSize): 11505069 -> 10771431 (-6.37%) Bug: https://github.com/dart-lang/sdk/issues/37103 Bug: https://github.com/dart-lang/sdk/issues/38452 Change-Id: I2bb44e7b2c002d53830b66cf4aedc4455c815326 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119440 Commit-Queue: Ryan Macnak Reviewed-by: Martin Kustermann --- .../vm/compiler/backend/code_statistics.cc | 5 +- .../backend/flow_graph_compiler_arm.cc | 2 + .../backend/flow_graph_compiler_arm64.cc | 2 + .../backend/flow_graph_compiler_ia32.cc | 2 + .../backend/flow_graph_compiler_x64.cc | 2 + runtime/vm/compiler/backend/il_arm.cc | 2 + runtime/vm/compiler/backend/il_arm64.cc | 2 + runtime/vm/compiler/backend/il_ia32.cc | 2 + runtime/vm/compiler/backend/il_x64.cc | 2 + runtime/vm/compiler/relocation.cc | 15 ++- runtime/vm/compiler/runtime_api.cc | 6 +- .../vm/compiler/runtime_offsets_extracted.h | 12 +- runtime/vm/heap/pages.cc | 2 +- runtime/vm/heap/pages.h | 2 +- runtime/vm/image_snapshot.cc | 119 ++++++++++-------- runtime/vm/image_snapshot.h | 5 +- runtime/vm/object.cc | 15 +++ runtime/vm/object.h | 28 +---- runtime/vm/os.h | 8 -- runtime/vm/os_android.cc | 19 --- runtime/vm/os_fuchsia.cc | 19 --- runtime/vm/os_linux.cc | 19 --- runtime/vm/os_macos.cc | 19 --- runtime/vm/os_test.cc | 1 - runtime/vm/os_win.cc | 12 -- runtime/vm/pointer_tagging.h | 4 + runtime/vm/raw_object.h | 9 -- runtime/vm/snapshot.h | 2 +- 28 files changed, 133 insertions(+), 204 deletions(-) diff --git a/runtime/vm/compiler/backend/code_statistics.cc b/runtime/vm/compiler/backend/code_statistics.cc index e0c2a548c6f..f18339cbcff 100644 --- a/runtime/vm/compiler/backend/code_statistics.cc +++ b/runtime/vm/compiler/backend/code_statistics.cc @@ -167,9 +167,10 @@ void CodeStatistics::Finalize() { intptr_t function_size = assembler_->CodeSize(); unaccounted_bytes_ = function_size - instruction_bytes_; ASSERT(unaccounted_bytes_ >= 0); + + const intptr_t unaligned_bytes = Instructions::HeaderSize() + function_size; alignment_bytes_ = - Utils::RoundUp(function_size, OS::PreferredCodeAlignment()) - - function_size; + Utils::RoundUp(unaligned_bytes, kObjectAlignment) - unaligned_bytes; assembler_ = NULL; } diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc index cfe60c784ce..7143db91128 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc @@ -952,7 +952,9 @@ void FlowGraphCompiler::CompileGraph() { VisitBlocks(); +#if defined(DEBUG) __ bkpt(0); +#endif if (!skip_body_compilation()) { ASSERT(assembler()->constant_pool_allowed()); diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc index 407fd55ae09..980cc1ec33b 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc @@ -920,7 +920,9 @@ void FlowGraphCompiler::CompileGraph() { VisitBlocks(); +#if defined(DEBUG) __ brk(0); +#endif if (!skip_body_compilation()) { ASSERT(assembler()->constant_pool_allowed()); diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc b/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc index ba4bf3ee5b3..b73de0875e0 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc @@ -839,7 +839,9 @@ void FlowGraphCompiler::CompileGraph() { VisitBlocks(); if (!skip_body_compilation()) { +#if defined(DEBUG) __ int3(); +#endif GenerateDeferredCode(); } } diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc index 08fd79ee57a..59af52e328b 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc @@ -935,7 +935,9 @@ void FlowGraphCompiler::CompileGraph() { ASSERT(!block_order().is_empty()); VisitBlocks(); +#if defined(DEBUG) __ int3(); +#endif if (!skip_body_compilation()) { ASSERT(assembler()->constant_pool_allowed()); diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 8a986f70ecc..7d823c5b210 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -510,7 +510,9 @@ static void EmitAssertBoolean(Register reg, compiler->GenerateRuntimeCall(token_pos, deopt_id, kNonBoolTypeErrorRuntimeEntry, 1, locs); // We should never return here. +#if defined(DEBUG) __ bkpt(0); +#endif __ Bind(&done); } diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 1d934ec5fb5..0e1d13dd864 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -499,7 +499,9 @@ static void EmitAssertBoolean(Register reg, compiler->GenerateRuntimeCall(token_pos, deopt_id, kNonBoolTypeErrorRuntimeEntry, 1, locs); // We should never return here. +#if defined(DEBUG) __ brk(0); +#endif __ Bind(&done); } diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index b6daf52e3af..7fc34baa965 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -398,7 +398,9 @@ static void EmitAssertBoolean(Register reg, compiler->GenerateRuntimeCall(token_pos, deopt_id, kNonBoolTypeErrorRuntimeEntry, 1, locs); // We should never return here. +#if defined(DEBUG) __ int3(); +#endif __ Bind(&done); } diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index dc394bf8991..89bb9ab7aef 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -527,7 +527,9 @@ static void EmitAssertBoolean(Register reg, compiler->GenerateRuntimeCall(token_pos, deopt_id, kNonBoolTypeErrorRuntimeEntry, 1, locs); // We should never return here. +#if defined(DEBUG) __ int3(); +#endif __ Bind(&done); } diff --git a/runtime/vm/compiler/relocation.cc b/runtime/vm/compiler/relocation.cc index af363e62ad5..0f4f567d647 100644 --- a/runtime/vm/compiler/relocation.cc +++ b/runtime/vm/compiler/relocation.cc @@ -21,9 +21,12 @@ DEFINE_FLAG(bool, false, "Generate always trampolines (for testing purposes)."); -// The trampolines will have a 1-word object header in front of them. +// The trampolines will be disguised as FreeListElement objects, with a 1-word +// object header in front of the jump code. const intptr_t kOffsetInTrampoline = kWordSize; -const intptr_t kTrampolineSize = OS::kMaxPreferredCodeAlignment; +const intptr_t kTrampolineSize = Utils::RoundUp( + kOffsetInTrampoline + PcRelativeTrampolineJumpPattern::kLengthInBytes, + kObjectAlignment); CodeRelocator::CodeRelocator(Thread* thread, GrowableArray* code_objects, @@ -408,7 +411,12 @@ bool CodeRelocator::IsTargetInRangeFor(UnresolvedCall* unresolved_call, static void MarkAsFreeListElement(uint8_t* trampoline_bytes, intptr_t trampoline_length) { uint32_t tags = 0; +#if defined(IS_SIMARM_X64) + // Account for difference in kObjectAlignment between host and target. + tags = RawObject::SizeTag::update(trampoline_length * 2, tags); +#else tags = RawObject::SizeTag::update(trampoline_length, tags); +#endif tags = RawObject::ClassIdTag::update(kFreeListElement, tags); tags = RawObject::OldBit::update(true, tags); tags = RawObject::OldAndNotMarkedBit::update(true, tags); @@ -464,9 +472,6 @@ void CodeRelocator::BuildTrampolinesForAlmostOutOfRangeCalls() { // buffer. auto trampoline_bytes = new uint8_t[kTrampolineSize]; memset(trampoline_bytes, 0x00, kTrampolineSize); - ASSERT((kOffsetInTrampoline + - PcRelativeTrampolineJumpPattern::kLengthInBytes) < - kTrampolineSize); auto unresolved_trampoline = new UnresolvedTrampoline{ unresolved_call->callee, unresolved_call->offset_into_target, diff --git a/runtime/vm/compiler/runtime_api.cc b/runtime/vm/compiler/runtime_api.cc index e7f6f80d980..e8db7d02c62 100644 --- a/runtime/vm/compiler/runtime_api.cc +++ b/runtime/vm/compiler/runtime_api.cc @@ -505,11 +505,7 @@ const word StoreBufferBlock::kSize = dart::StoreBufferBlock::kSize; const word MarkingStackBlock::kSize = dart::MarkingStackBlock::kSize; word Instructions::HeaderSize() { - intptr_t alignment = OS::PreferredCodeAlignment(); - intptr_t aligned_size = - Utils::RoundUp(Instructions::UnalignedHeaderSize(), alignment); - ASSERT(aligned_size == alignment); - return aligned_size; + return Utils::RoundUp(Instructions::UnalignedHeaderSize(), target::kWordSize); } #if !defined(TARGET_ARCH_DBC) diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h index 0965a3ca6f0..62a38bad9a7 100644 --- a/runtime/vm/compiler/runtime_offsets_extracted.h +++ b/runtime/vm/compiler/runtime_offsets_extracted.h @@ -357,7 +357,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 16; + 12; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 16; @@ -718,7 +718,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 24; + 16; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 32; @@ -1071,7 +1071,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 16; + 12; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 16; @@ -1433,7 +1433,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 24; + 16; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 32; @@ -1722,7 +1722,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 24; + 16; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 32; @@ -2008,7 +2008,7 @@ static constexpr dart::compiler::target::word Double_InstanceSize = 16; static constexpr dart::compiler::target::word Float32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Float64x2_InstanceSize = 24; static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize = - 16; + 12; static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24; static constexpr dart::compiler::target::word Mint_InstanceSize = 16; static constexpr dart::compiler::target::word NativeArguments_StructSize = 16; diff --git a/runtime/vm/heap/pages.cc b/runtime/vm/heap/pages.cc index 17446eb8916..f662bee1dd0 100644 --- a/runtime/vm/heap/pages.cc +++ b/runtime/vm/heap/pages.cc @@ -1322,6 +1322,7 @@ void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) { // memory->end()). uword offset = HeapPage::ObjectStartOffset(); pointer = reinterpret_cast(reinterpret_cast(pointer) - offset); + ASSERT(Utils::IsAligned(pointer, kObjectAlignment)); size += offset; VirtualMemory* memory = VirtualMemory::ForImagePage(pointer, size); @@ -1334,7 +1335,6 @@ void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) { page->forwarding_page_ = NULL; page->card_table_ = NULL; if (is_executable) { - ASSERT(Utils::IsAligned(pointer, OS::PreferredCodeAlignment())); page->type_ = HeapPage::kExecutable; } else { page->type_ = HeapPage::kData; diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h index b723b636d87..b9b4824a85f 100644 --- a/runtime/vm/heap/pages.h +++ b/runtime/vm/heap/pages.h @@ -65,7 +65,7 @@ class HeapPage { void WriteProtect(bool read_only); static intptr_t ObjectStartOffset() { - return Utils::RoundUp(sizeof(HeapPage), OS::kMaxPreferredCodeAlignment); + return Utils::RoundUp(sizeof(HeapPage), kMaxObjectAlignment); } // Warning: This does not work for objects on image pages because image pages diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc index 4f2cac1c6ad..2d2fde18949 100644 --- a/runtime/vm/image_snapshot.cc +++ b/runtime/vm/image_snapshot.cc @@ -220,12 +220,9 @@ static intptr_t PcDescriptorsSizeInSnapshot(intptr_t len) { compiler::target::ObjectAlignment::kObjectAlignment); } -static constexpr intptr_t kSimarmX64InstructionsAlignment = - 2 * compiler::target::ObjectAlignment::kObjectAlignment; static intptr_t InstructionsSizeInSnapshot(intptr_t len) { - const intptr_t header_size = Utils::RoundUp(3 * compiler::target::kWordSize, - kSimarmX64InstructionsAlignment); - return header_size + Utils::RoundUp(len, kSimarmX64InstructionsAlignment); + return Utils::RoundUp(compiler::target::Instructions::HeaderSize() + len, + compiler::target::ObjectAlignment::kObjectAlignment); } intptr_t ImageWriter::SizeInSnapshot(RawObject* raw_object) { @@ -399,15 +396,15 @@ void ImageWriter::Write(WriteStream* clustered_stream, bool vm) { } void ImageWriter::WriteROData(WriteStream* stream) { - stream->Align(OS::kMaxPreferredCodeAlignment); + stream->Align(kMaxObjectAlignment); // Heap page starts here. intptr_t section_start = stream->Position(); stream->WriteWord(next_data_offset_); // Data length. - COMPILE_ASSERT(OS::kMaxPreferredCodeAlignment >= kObjectAlignment); - stream->Align(OS::kMaxPreferredCodeAlignment); + COMPILE_ASSERT(kMaxObjectAlignment >= kObjectAlignment); + stream->Align(kMaxObjectAlignment); ASSERT(stream->Position() - section_start == Image::kHeaderSize); @@ -521,6 +518,7 @@ void AssemblyImageWriter::Finalize() { #endif } +#if !defined(DART_PRECOMPILED_RUNTIME) static void EnsureAssemblerIdentifier(char* label) { for (char c = *label; c != '\0'; c = *++label) { if (((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || @@ -531,8 +529,8 @@ static void EnsureAssemblerIdentifier(char* label) { } } -const char* NameOfStubIsolateSpecificStub(ObjectStore* object_store, - const Code& code) { +static const char* NameOfStubIsolateSpecificStub(ObjectStore* object_store, + const Code& code) { if (code.raw() == object_store->build_method_extractor_code()) { return "_iso_stub_BuildMethodExtractorStub"; } else if (code.raw() == object_store->null_error_stub_with_fpu_regs_stub()) { @@ -553,8 +551,12 @@ const char* NameOfStubIsolateSpecificStub(ObjectStore* object_store, } return nullptr; } +#endif // !defined(DART_PRECOMPILED_RUNTIME) void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { +#if defined(DART_PRECOMPILED_RUNTIME) + UNREACHABLE(); +#else Zone* zone = Thread::Current()->zone(); #if defined(DART_PRECOMPILER) @@ -568,7 +570,7 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { assembly_stream_.Print(".globl %s\n", instructions_symbol); // Start snapshot at page boundary. - ASSERT(VirtualMemory::PageSize() >= OS::kMaxPreferredCodeAlignment); + ASSERT(VirtualMemory::PageSize() >= kMaxObjectAlignment); assembly_stream_.Print(".balign %" Pd ", 0\n", VirtualMemory::PageSize()); assembly_stream_.Print("%s:\n", instructions_symbol); @@ -639,9 +641,8 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { SizeInSnapshot(insns.raw())); } - ASSERT(insns.raw()->HeapSize() % sizeof(uint64_t) == 0); - - // 1. Write from the header to the entry point. + // 1. Write from the object start to the payload start. This includes the + // object header and the fixed fields. { NoSafepointScope no_safepoint; @@ -667,15 +668,13 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { text_offset += sizeof(compiler::target::uword); WriteWordLiteralText(insns.raw_ptr()->unchecked_entrypoint_pc_offset_); text_offset += sizeof(compiler::target::uword); - WriteWordLiteralText(0); - text_offset += sizeof(compiler::target::uword); #else // defined(IS_SIMARM_X64) - uword beginning = reinterpret_cast(insns.raw_ptr()); - uword entry = beginning + Instructions::HeaderSize(); + uword object_start = reinterpret_cast(insns.raw_ptr()); + uword payload_start = insns.PayloadStart(); WriteWordLiteralText(marked_tags); - beginning += sizeof(uword); + object_start += sizeof(uword); text_offset += sizeof(uword); - text_offset += WriteByteSequence(beginning, entry); + text_offset += WriteByteSequence(object_start, payload_start); #endif // defined(IS_SIMARM_X64) ASSERT((text_offset - instr_start) == @@ -723,27 +722,23 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { #endif { - // 3. Write from the entry point to the end. + // 3. Write from the payload start to payload end. NoSafepointScope no_safepoint; - uword beginning = reinterpret_cast(insns.raw_ptr()); - uword entry = beginning + Instructions::HeaderSize(); - uword payload_size = insns.raw()->HeapSize() - insns.HeaderSize(); - uword end = entry + payload_size; - - ASSERT(Utils::IsAligned(beginning, sizeof(uword))); - ASSERT(Utils::IsAligned(entry, sizeof(uword))); - ASSERT(Utils::IsAligned(end, sizeof(uword))); + const uword payload_start = insns.PayloadStart(); + const uword payload_size = + Utils::RoundUp(insns.Size(), sizeof(compiler::target::uword)); + const uword payload_end = payload_start + payload_size; #if defined(DART_PRECOMPILER) PcDescriptors::Iterator iterator(descriptors, RawPcDescriptors::kBSSRelocation); uword next_reloc_offset = iterator.MoveNext() ? iterator.PcOffset() : -1; - for (uword cursor = entry; cursor < end; + for (uword cursor = payload_start; cursor < payload_end; cursor += sizeof(compiler::target::uword)) { compiler::target::uword data = *reinterpret_cast(cursor); - if ((cursor - entry) == next_reloc_offset) { + if ((cursor - payload_start) == next_reloc_offset) { assembly_stream_.Print("%s %s - (.) + %" Pd "\n", kLiteralPrefix, bss_symbol, /*addend=*/data); next_reloc_offset = iterator.MoveNext() ? iterator.PcOffset() : -1; @@ -751,13 +746,31 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { WriteWordLiteralText(data); } } - text_offset += end - entry; + text_offset += payload_size; #else - text_offset += WriteByteSequence(entry, end); + text_offset += WriteByteSequence(payload_start, payload_end); #endif + + // 4. Write from the payload end to object end. Note we can't simply copy + // from the object because the host object may have less alignment filler + // than the target object in the cross-word case. + uword unaligned_size = + compiler::target::Instructions::HeaderSize() + payload_size; + uword alignment_size = + Utils::RoundUp(unaligned_size, + compiler::target::ObjectAlignment::kObjectAlignment) - + unaligned_size; + while (alignment_size > 0) { + WriteWordLiteralText(compiler::Assembler::GetBreakInstructionFiller()); + alignment_size -= sizeof(compiler::target::uword); + text_offset += sizeof(compiler::target::uword); + } + ASSERT(kWordSize != compiler::target::kWordSize || (text_offset - instr_start) == insns.raw()->HeapSize()); } + + ASSERT((text_offset - instr_start) == SizeInSnapshot(insns.raw())); } FrameUnwindEpilogue(); @@ -784,12 +797,12 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { const char* data_symbol = vm ? "_kDartVmSnapshotData" : "_kDartIsolateSnapshotData"; assembly_stream_.Print(".globl %s\n", data_symbol); - assembly_stream_.Print(".balign %" Pd ", 0\n", - OS::kMaxPreferredCodeAlignment); + assembly_stream_.Print(".balign %" Pd ", 0\n", kMaxObjectAlignment); assembly_stream_.Print("%s:\n", data_symbol); uword buffer = reinterpret_cast(clustered_stream->buffer()); intptr_t length = clustered_stream->bytes_written(); WriteByteSequence(buffer, buffer + length); +#endif // !defined(DART_PRECOMPILED_RUNTIME) } void AssemblyImageWriter::FrameUnwindPrologue() { @@ -964,14 +977,16 @@ void BlobImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { const Instructions& insns = *instructions_[i].insns_; AutoTraceImage(insns, 0, &this->instructions_blob_stream_); - uword beginning = reinterpret_cast(insns.raw_ptr()); - uword entry = beginning + Instructions::HeaderSize(); - uword payload_size = insns.Size(); - payload_size = Utils::RoundUp(payload_size, OS::PreferredCodeAlignment()); - uword end = entry + payload_size; + uword object_start = reinterpret_cast(insns.raw_ptr()); + uword payload_start = insns.PayloadStart(); + uword payload_size = + Utils::RoundUp( + compiler::target::Instructions::HeaderSize() + insns.Size(), + compiler::target::ObjectAlignment::kObjectAlignment) - + compiler::target::Instructions::HeaderSize(); + uword object_end = payload_start + payload_size; - ASSERT(Utils::IsAligned(beginning, sizeof(uword))); - ASSERT(Utils::IsAligned(entry, sizeof(uword))); + ASSERT(Utils::IsAligned(payload_start, sizeof(uword))); #ifdef DART_PRECOMPILER const Code& code = *instructions_[i].code_; @@ -1005,22 +1020,23 @@ void BlobImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { insns.raw_ptr()->size_and_flags_); instructions_blob_stream_.WriteFixed( insns.raw_ptr()->unchecked_entrypoint_pc_offset_); - instructions_blob_stream_.Align(kSimarmX64InstructionsAlignment); payload_stream_start = instructions_blob_stream_.Position(); instructions_blob_stream_.WriteBytes( reinterpret_cast(insns.PayloadStart()), insns.Size()); - instructions_blob_stream_.Align(kSimarmX64InstructionsAlignment); + instructions_blob_stream_.Align( + compiler::target::ObjectAlignment::kObjectAlignment); const intptr_t end_offset = instructions_blob_stream_.bytes_written(); text_offset += (end_offset - start_offset); - USE(end); + USE(object_start); + USE(object_end); #else // defined(IS_SIMARM_X64) payload_stream_start = instructions_blob_stream_.Position() + - (insns.PayloadStart() - beginning); + (insns.PayloadStart() - object_start); instructions_blob_stream_.WriteWord(marked_tags); text_offset += sizeof(uword); - beginning += sizeof(uword); - text_offset += WriteByteSequence(beginning, end); + object_start += sizeof(uword); + text_offset += WriteByteSequence(object_start, object_end); #endif // defined(IS_SIMARM_X64) #if defined(DART_PRECOMPILER) @@ -1103,9 +1119,8 @@ ImageReader::ImageReader(const uint8_t* data_image, RawApiError* ImageReader::VerifyAlignment() const { if (!Utils::IsAligned(data_image_, kObjectAlignment) || !Utils::IsAligned(shared_data_image_, kObjectAlignment) || - !Utils::IsAligned(instructions_image_, OS::PreferredCodeAlignment()) || - !Utils::IsAligned(shared_instructions_image_, - OS::PreferredCodeAlignment())) { + !Utils::IsAligned(instructions_image_, kMaxObjectAlignment) || + !Utils::IsAligned(shared_instructions_image_, kMaxObjectAlignment)) { return ApiError::New( String::Handle(String::New("Snapshot is misaligned", Heap::kOld)), Heap::kOld); @@ -1114,7 +1129,7 @@ RawApiError* ImageReader::VerifyAlignment() const { } RawInstructions* ImageReader::GetInstructionsAt(int32_t offset) const { - ASSERT(Utils::IsAligned(offset, OS::PreferredCodeAlignment())); + ASSERT(Utils::IsAligned(offset, kObjectAlignment)); RawObject* result; if (offset < 0) { diff --git a/runtime/vm/image_snapshot.h b/runtime/vm/image_snapshot.h index deb94ef7942..e8d0ea46f07 100644 --- a/runtime/vm/image_snapshot.h +++ b/runtime/vm/image_snapshot.h @@ -35,7 +35,7 @@ class RawObject; class Image : ValueObject { public: explicit Image(const void* raw_memory) : raw_memory_(raw_memory) { - ASSERT(Utils::IsAligned(raw_memory, OS::kMaxPreferredCodeAlignment)); + ASSERT(Utils::IsAligned(raw_memory, kMaxObjectAlignment)); } void* object_start() const { @@ -53,7 +53,8 @@ class Image : ValueObject { } static constexpr intptr_t kHeaderFields = 2; - static const intptr_t kHeaderSize = OS::kMaxPreferredCodeAlignment; + static constexpr intptr_t kHeaderSize = kMaxObjectAlignment; + COMPILE_ASSERT((kHeaderFields * compiler::target::kWordSize) <= kHeaderSize); private: const void* raw_memory_; // The symbol kInstructionsSnapshot. diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 51e0b8f7e69..38d97aeeb98 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -12587,6 +12587,21 @@ const char* Instructions::ToCString() const { return "Instructions"; } +CodeStatistics* Instructions::stats() const { +#if defined(DART_PRECOMPILER) + return reinterpret_cast( + Thread::Current()->heap()->GetPeer(raw())); +#else + return nullptr; +#endif +} + +void Instructions::set_stats(CodeStatistics* stats) const { +#if defined(DART_PRECOMPILER) + Thread::Current()->heap()->SetPeer(raw(), stats); +#endif +} + // Encode integer |value| in SLEB128 format and store into |data|. static void EncodeSLEB128(GrowableArray* data, intptr_t value) { bool is_last_part = false; diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 1595e428a82..a3a118a5032 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -4842,7 +4842,7 @@ class Instructions : public Object { static const intptr_t kMaxElements = (kMaxInt32 - (sizeof(RawInstructions) + sizeof(RawObject) + - (2 * OS::kMaxPreferredCodeAlignment))); + (2 * kMaxObjectAlignment))); static intptr_t InstanceSize() { ASSERT(sizeof(RawInstructions) == @@ -4851,18 +4851,11 @@ class Instructions : public Object { } static intptr_t InstanceSize(intptr_t size) { - intptr_t instructions_size = - Utils::RoundUp(size, OS::PreferredCodeAlignment()); - intptr_t result = instructions_size + HeaderSize(); - ASSERT(result % OS::PreferredCodeAlignment() == 0); - return result; + return Utils::RoundUp(HeaderSize() + size, kObjectAlignment); } static intptr_t HeaderSize() { - const intptr_t alignment = OS::PreferredCodeAlignment(); - const intptr_t aligned_size = - Utils::RoundUp(sizeof(RawInstructions), alignment); - return aligned_size; + return Utils::RoundUp(sizeof(RawInstructions), kWordSize); } static RawInstructions* FromPayloadStart(uword payload_start) { @@ -4880,19 +4873,8 @@ class Instructions : public Object { return memcmp(a->ptr(), b->ptr(), InstanceSize(Size(a))) == 0; } - CodeStatistics* stats() const { -#if defined(DART_PRECOMPILER) - return raw_ptr()->stats_; -#else - return nullptr; -#endif - } - - void set_stats(CodeStatistics* stats) const { -#if defined(DART_PRECOMPILER) - StoreNonPointer(&raw_ptr()->stats_, stats); -#endif - } + CodeStatistics* stats() const; + void set_stats(CodeStatistics* stats) const; uword unchecked_entrypoint_pc_offset() const { return raw_ptr()->unchecked_entrypoint_pc_offset_; diff --git a/runtime/vm/os.h b/runtime/vm/os.h index af98eb1c729..a5fb7391994 100644 --- a/runtime/vm/os.h +++ b/runtime/vm/os.h @@ -67,14 +67,6 @@ class OS { // the platform doesn't care. Guaranteed to be a power of two. static intptr_t ActivationFrameAlignment(); - // This constant is guaranteed to be greater or equal to the - // preferred code alignment on all platforms. - static const int kMaxPreferredCodeAlignment = 32; - - // Returns the preferred code alignment or zero if - // the platform doesn't care. Guaranteed to be a power of two. - static intptr_t PreferredCodeAlignment(); - // Returns number of available processor cores. static int NumberOfAvailableProcessors(); diff --git a/runtime/vm/os_android.cc b/runtime/vm/os_android.cc index 38168ad399c..222ae792f15 100644 --- a/runtime/vm/os_android.cc +++ b/runtime/vm/os_android.cc @@ -201,25 +201,6 @@ intptr_t OS::ActivationFrameAlignment() { return alignment; } -intptr_t OS::PreferredCodeAlignment() { -#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_DBC) - const int kMinimumAlignment = 32; -#elif defined(TARGET_ARCH_ARM) - const int kMinimumAlignment = 16; -#else -#error Unsupported architecture. -#endif - intptr_t alignment = kMinimumAlignment; - // TODO(5411554): Allow overriding default code alignment for - // testing purposes. - // Flags::DebugIsInt("codealign", &alignment); - ASSERT(Utils::IsPowerOfTwo(alignment)); - ASSERT(alignment >= kMinimumAlignment); - ASSERT(alignment <= OS::kMaxPreferredCodeAlignment); - return alignment; -} - int OS::NumberOfAvailableProcessors() { return sysconf(_SC_NPROCESSORS_ONLN); } diff --git a/runtime/vm/os_fuchsia.cc b/runtime/vm/os_fuchsia.cc index 26411037129..a0bfa909ef9 100644 --- a/runtime/vm/os_fuchsia.cc +++ b/runtime/vm/os_fuchsia.cc @@ -145,25 +145,6 @@ intptr_t OS::ActivationFrameAlignment() { return alignment; } -intptr_t OS::PreferredCodeAlignment() { -#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_DBC) - const int kMinimumAlignment = 32; -#elif defined(TARGET_ARCH_ARM) - const int kMinimumAlignment = 16; -#else -#error Unsupported architecture. -#endif - intptr_t alignment = kMinimumAlignment; - // TODO(5411554): Allow overriding default code alignment for - // testing purposes. - // Flags::DebugIsInt("codealign", &alignment); - ASSERT(Utils::IsPowerOfTwo(alignment)); - ASSERT(alignment >= kMinimumAlignment); - ASSERT(alignment <= OS::kMaxPreferredCodeAlignment); - return alignment; -} - int OS::NumberOfAvailableProcessors() { return sysconf(_SC_NPROCESSORS_CONF); } diff --git a/runtime/vm/os_linux.cc b/runtime/vm/os_linux.cc index 9baba7e499e..b1a9231dffc 100644 --- a/runtime/vm/os_linux.cc +++ b/runtime/vm/os_linux.cc @@ -513,25 +513,6 @@ intptr_t OS::ActivationFrameAlignment() { return alignment; } -intptr_t OS::PreferredCodeAlignment() { -#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_DBC) - const int kMinimumAlignment = 32; -#elif defined(TARGET_ARCH_ARM) - const int kMinimumAlignment = 16; -#else -#error Unsupported architecture. -#endif - intptr_t alignment = kMinimumAlignment; - // TODO(5411554): Allow overriding default code alignment for - // testing purposes. - // Flags::DebugIsInt("codealign", &alignment); - ASSERT(Utils::IsPowerOfTwo(alignment)); - ASSERT(alignment >= kMinimumAlignment); - ASSERT(alignment <= OS::kMaxPreferredCodeAlignment); - return alignment; -} - int OS::NumberOfAvailableProcessors() { return sysconf(_SC_NPROCESSORS_ONLN); } diff --git a/runtime/vm/os_macos.cc b/runtime/vm/os_macos.cc index 4788b6cb0dc..9fc4bb0cd3e 100644 --- a/runtime/vm/os_macos.cc +++ b/runtime/vm/os_macos.cc @@ -151,25 +151,6 @@ intptr_t OS::ActivationFrameAlignment() { #endif // HOST_OS_IOS } -intptr_t OS::PreferredCodeAlignment() { -#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_DBC) - const int kMinimumAlignment = 32; -#elif defined(TARGET_ARCH_ARM) - const int kMinimumAlignment = 16; -#else -#error Unsupported architecture. -#endif - intptr_t alignment = kMinimumAlignment; - // TODO(5411554): Allow overriding default code alignment for - // testing purposes. - // Flags::DebugIsInt("codealign", &alignment); - ASSERT(Utils::IsPowerOfTwo(alignment)); - ASSERT(alignment >= kMinimumAlignment); - ASSERT(alignment <= OS::kMaxPreferredCodeAlignment); - return alignment; -} - int OS::NumberOfAvailableProcessors() { return sysconf(_SC_NPROCESSORS_ONLN); } diff --git a/runtime/vm/os_test.cc b/runtime/vm/os_test.cc index 406edf6209e..331b3d2a823 100644 --- a/runtime/vm/os_test.cc +++ b/runtime/vm/os_test.cc @@ -35,7 +35,6 @@ VM_UNIT_TEST_CASE_WITH_EXPECTATION(SNPrint_BadArgs, "Crash") { VM_UNIT_TEST_CASE(OsFuncs) { EXPECT(Utils::IsPowerOfTwo(OS::ActivationFrameAlignment())); - EXPECT(Utils::IsPowerOfTwo(OS::PreferredCodeAlignment())); int procs = OS::NumberOfAvailableProcessors(); EXPECT_LE(1, procs); } diff --git a/runtime/vm/os_win.cc b/runtime/vm/os_win.cc index c0f8e39a2b0..736d9b3b0bb 100644 --- a/runtime/vm/os_win.cc +++ b/runtime/vm/os_win.cc @@ -190,18 +190,6 @@ intptr_t OS::ActivationFrameAlignment() { #endif } -intptr_t OS::PreferredCodeAlignment() { - ASSERT(32 <= OS::kMaxPreferredCodeAlignment); -#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64) || \ - defined(TARGET_ARCH_ARM64) || defined(TARGET_ARCH_DBC) - return 32; -#elif defined(TARGET_ARCH_ARM) - return 16; -#else -#error Unsupported architecture. -#endif -} - int OS::NumberOfAvailableProcessors() { SYSTEM_INFO info; GetSystemInfo(&info); diff --git a/runtime/vm/pointer_tagging.h b/runtime/vm/pointer_tagging.h index e4a59e617ef..487f3f8e123 100644 --- a/runtime/vm/pointer_tagging.h +++ b/runtime/vm/pointer_tagging.h @@ -51,6 +51,10 @@ static constexpr intptr_t kObjectAlignmentLog2 = static constexpr intptr_t kObjectAlignmentMask = HostObjectAlignment::kObjectAlignmentMask; +// The largest value of kObjectAlignment across all configurations. +static constexpr intptr_t kMaxObjectAlignment = 16; +COMPILE_ASSERT(kMaxObjectAlignment >= kObjectAlignment); + // On all targets heap pointers are tagged by set least significant bit. // // To recover address of the actual heap object kHeapObjectTag needs to be diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index c6243307cd0..b70f95c89ee 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -1482,15 +1482,6 @@ class RawInstructions : public RawObject { uint32_t size_and_flags_; uint32_t unchecked_entrypoint_pc_offset_; - // There is a gap between size_and_flags_ and the entry point - // because we align entry point by 4 words on all platforms. - // This allows us to have a free field here without affecting - // the aligned size of the Instructions object header. - // This also means that entry point offset is the same - // whether this field is included or excluded. - // TODO(37103): This field should be removed. - CodeStatistics* stats_; - // Variable length data follows here. uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); } diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index 61b90a94575..1242943f2a8 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h @@ -199,7 +199,7 @@ class Snapshot { if (!IncludesCode(kind())) { return NULL; } - uword offset = Utils::RoundUp(length(), OS::kMaxPreferredCodeAlignment); + uword offset = Utils::RoundUp(length(), kMaxObjectAlignment); return Addr() + offset; }