From 2362cd5508b7d74e40e29b5274bcc92ae445e2b7 Mon Sep 17 00:00:00 2001 From: Tess Strickland Date: Fri, 2 Jul 2021 17:25:34 +0000 Subject: [PATCH] [vm] Reorder ELF sections more sensibly. Previous CLs made it so that we don't need to calculate memory offsets until finalization. This means that we can now reorder sections as desired and then calculate the memory offsets at the same time as the file offsets. Thus, we reorder them so that writable, non-executable allocated sections come first (due to requirements by some loaders to not sandwich writable segments between non-writable ones), followed by non-writable, non-executable allocated sections, followed by non-writable, executable sections, and finally unallocated sections (to avoid differences in memory offsets between snapshots and separate debugging information). This ordering minimizes the number of segments that need to be created (5 or 6, depending on whether a build ID exists). Since we can reorder, we also delay the creation of the build ID (if generated) and BSS sections until finalization, instead of creating them up front in the constructor. TEST=Further refactorings, so existing tests. Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-linux-debug-simarm_x64-try,vm-kernel-precomp-linux-debug-x64-try,vm-kernel-precomp-linux-release-simarm-try,vm-kernel-precomp-linux-release-simarm64-try,vm-kernel-precomp-dwarf-linux-product-x64-try Change-Id: Id4142a021c2bb800938e8bc711b1b8a7529bff51 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/205780 Commit-Queue: Tess Strickland Reviewed-by: Slava Egorov --- runtime/vm/elf.cc | 752 ++++++++++++++++++------------------ runtime/vm/elf.h | 69 ++-- runtime/vm/image_snapshot.h | 2 + 3 files changed, 417 insertions(+), 406 deletions(-) diff --git a/runtime/vm/elf.cc b/runtime/vm/elf.cc index 059c2c4025c..a0734284881 100644 --- a/runtime/vm/elf.cc +++ b/runtime/vm/elf.cc @@ -134,17 +134,11 @@ class Section : public ZoneAllocated { } bool IsWritable() const { return (flags & elf::SHF_WRITE) == elf::SHF_WRITE; } - // Returns whether new content can be added to a section. + // Returns whether the size of a section can change. bool HasBeenFinalized() const { - if (IsAllocated()) { - // The contents of a section that is allocated (part of a segment) must - // not change after the section is added. - return memory_offset_is_set(); - } else { - // Unallocated sections can have new content added until we calculate - // file offsets. - return file_offset_is_set(); - } + // Sections can grow or shrink up until Elf::ComputeOffsets has been run, + // which sets the file offset (and memory offset for allocated sections). + return file_offset_is_set(); } virtual const BitsContainer* AsBitsContainer() const { return nullptr; } @@ -222,18 +216,11 @@ class Segment : public ZoneAllocated { // Unlike sections, we don't have a reserved segment with the null type, // so we never should pass this value. ASSERT(segment_type != elf::ProgramHeaderType::PT_NULL); - // All segments should have at least one section. The first one is added - // during initialization. Unlike others added later, it should already have - // a memory offset since we use it to determine the segment memory offset. + // All segments should have at least one section. ASSERT(initial_section->IsAllocated()); - ASSERT(initial_section->memory_offset_is_set()); - // Make sure the memory offset chosen for the initial section is consistent - // with the alignment for the segment. - ASSERT(Utils::IsAligned(initial_section->memory_offset(), Alignment(type))); sections_.Add(initial_section); if (type == elf::ProgramHeaderType::PT_LOAD) { ASSERT(initial_section->load_segment == nullptr); - initial_section->load_segment = this; } } @@ -276,68 +263,41 @@ class Segment : public ZoneAllocated { #endif } - // Adds the given section to this segment. - // - // Returns whether the Section could be added to the segment. If not, a - // new segment will need to be created for this section. - // - // Sets the memory offset of the section if added. + // Adds a given section to the end of this segment. Returns whether the + // section was successfully added. bool Add(Section* section) { + ASSERT(section != nullptr); // We only add additional sections to load segments. ASSERT(type == elf::ProgramHeaderType::PT_LOAD); - ASSERT(section != nullptr); - // Only sections with the allocate flag set should be added to segments, - // and sections with already-set memory offsets cannot be added. - ASSERT(section->IsAllocated()); - ASSERT(!section->memory_offset_is_set()); + // Don't use this to change a section's segment. ASSERT(section->load_segment == nullptr); - switch (sections_.Last()->type) { - // We only use SHT_NULL sections as pseudo sections that will not appear - // in the final ELF file. Don't pack sections into these segments, as we - // may remove/replace the segments during finalization. - case elf::SectionHeaderType::SHT_NULL: - // If the last section in the segments is NOBITS, then we don't add it, - // as otherwise we'll be guaranteed the file offset and memory offset - // won't be page aligned without padding. - case elf::SectionHeaderType::SHT_NOBITS: - return false; - default: - break; - } - // We don't add if the W or X bits don't match. + // We only add sections with the same executable and writable bits. if (IsExecutable() != section->IsExecutable() || IsWritable() != section->IsWritable()) { return false; } - auto const start_address = Utils::RoundUp(MemoryEnd(), section->alignment); - section->set_memory_offset(start_address); sections_.Add(section); section->load_segment = this; return true; } - void Replace(Section* old_section, Section* new_section) { - ASSERT(old_section->load_segment == this); - // All these must be true for replacement to be safe. - ASSERT_EQUAL(static_cast(old_section->type), - static_cast(new_section->type)); - ASSERT_EQUAL(old_section->MemorySize(), new_section->MemorySize()); - ASSERT_EQUAL(old_section->IsExecutable(), new_section->IsExecutable()); - ASSERT_EQUAL(old_section->IsWritable(), new_section->IsWritable()); - ASSERT(old_section->memory_offset_is_set()); - ASSERT(!new_section->memory_offset_is_set()); - for (intptr_t i = 0; i < sections_.length(); i++) { - auto const section = sections_[i]; - if (section != old_section) { - continue; - } - new_section->set_memory_offset(old_section->memory_offset()); - sections_[i] = new_section; - new_section->load_segment = this; - old_section->load_segment = nullptr; - return; + bool Merge(Segment* other) { + ASSERT(other != nullptr); + // We only add additional sections to load segments. + ASSERT(type == elf::ProgramHeaderType::PT_LOAD); + // We only merge segments with the same executable and writable bits. + if (IsExecutable() != other->IsExecutable() || + IsWritable() != other->IsWritable()) { + return false; } - UNREACHABLE(); + for (auto* section : other->sections_) { + // Don't merge segments where the memory offsets have already been + // calculated. + ASSERT(!section->memory_offset_is_set()); + sections_.Add(section); + section->load_segment = this; + } + return true; } intptr_t FileOffset() const { return sections_[0]->file_offset(); } @@ -374,7 +334,7 @@ class Segment : public ZoneAllocated { const intptr_t flags; private: - GrowableArray sections_; + GrowableArray sections_; }; // Represents the first entry in the section table, which should only contain @@ -524,17 +484,23 @@ class Symbol : public ZoneAllocated { intptr_t name, intptr_t binding, intptr_t type, - intptr_t section_index, + intptr_t initial_section_index, intptr_t size) : name_index(name), binding(binding), type(type), size(size), - section_index(section_index), + section_index(initial_section_index), cstr_(cstr) {} - void Finalize(intptr_t offset) { offset_ = offset; } + void Finalize(intptr_t final_section_index, intptr_t offset) { + ASSERT(!HasBeenFinalized()); // No symbol should be re-finalized. + section_index = final_section_index; + offset_ = offset; + } + bool HasBeenFinalized() const { return offset_ != kNotFinalizedMarker; } intptr_t offset() const { + ASSERT(HasBeenFinalized()); // Only the reserved initial symbol should have an offset of 0. ASSERT_EQUAL(type == elf::STT_NOTYPE, offset_ == 0); return offset_; @@ -563,13 +529,18 @@ class Symbol : public ZoneAllocated { const intptr_t binding; const intptr_t type; const intptr_t size; - const intptr_t section_index; + // Is set twice: once in Elf::AddSection to the section's initial index into + // sections_, and then in Elf::FinalizeSymbols to the section's final index + // into sections_ after reordering. + intptr_t section_index; private: - friend class SymbolHashTable; // For cstr_ access. + static const intptr_t kNotFinalizedMarker = -1; const char* const cstr_; - intptr_t offset_ = 0; + intptr_t offset_ = kNotFinalizedMarker; + + friend class SymbolHashTable; // For cstr_ access. }; class SymbolTable : public Section { @@ -588,8 +559,10 @@ class SymbolTable : public Section { entry_size = sizeof(elf::Symbol); // The first symbol table entry is reserved and must be all zeros. // (String tables always have the empty string at the 0th index.) - AddSymbol(/*name=*/"", elf::STB_LOCAL, elf::STT_NOTYPE, /*section_index=*/0, + const char* const kReservedName = ""; + AddSymbol(kReservedName, elf::STB_LOCAL, elf::STT_NOTYPE, elf::SHN_UNDEF, /*size=*/0); + FinalizeSymbol(kReservedName, elf::SHN_UNDEF, /*offset=*/0); } intptr_t FileSize() const { return Length() * entry_size; } @@ -632,12 +605,14 @@ class SymbolTable : public Section { } } - void FinalizeSymbol(const char* name, intptr_t offset) { + void FinalizeSymbol(const char* name, + intptr_t final_section_index, + intptr_t offset) { const intptr_t name_index = table_->Lookup(name); ASSERT(name_index != StringTable::kNotIndexed); Symbol* symbol = by_name_index_.Lookup(name_index); ASSERT(symbol != nullptr); - symbol->Finalize(offset); + symbol->Finalize(final_section_index, offset); } intptr_t Length() const { return symbols_.length(); } @@ -675,7 +650,6 @@ class SymbolHashTable : public Section { /*allocate=*/true, /*executable=*/false, /*writable=*/false) { - link = symtab->index(); entry_size = sizeof(int32_t); nchain_ = symtab->Length(); @@ -723,25 +697,24 @@ class SymbolHashTable : public Section { class DynamicTable : public Section { public: - DynamicTable(Zone* zone, - StringTable* strtab, - SymbolTable* symtab, - SymbolHashTable* hash) + explicit DynamicTable(Zone* zone) : Section(elf::SectionHeaderType::SHT_DYNAMIC, /*allocate=*/true, /*executable=*/false, /*writable=*/true) { - link = strtab->index(); entry_size = sizeof(elf::DynamicEntry); - AddEntry(zone, elf::DynamicEntryType::DT_HASH, hash->memory_offset()); - AddEntry(zone, elf::DynamicEntryType::DT_STRTAB, strtab->memory_offset()); - AddEntry(zone, elf::DynamicEntryType::DT_STRSZ, strtab->MemorySize()); - AddEntry(zone, elf::DynamicEntryType::DT_SYMTAB, symtab->memory_offset()); + // Entries that are not constants are fixed during Elf::Finalize(). + AddEntry(zone, elf::DynamicEntryType::DT_HASH, kInvalidEntry); + AddEntry(zone, elf::DynamicEntryType::DT_STRTAB, kInvalidEntry); + AddEntry(zone, elf::DynamicEntryType::DT_STRSZ, kInvalidEntry); + AddEntry(zone, elf::DynamicEntryType::DT_SYMTAB, kInvalidEntry); AddEntry(zone, elf::DynamicEntryType::DT_SYMENT, sizeof(elf::Symbol)); AddEntry(zone, elf::DynamicEntryType::DT_NULL, 0); } + static constexpr intptr_t kInvalidEntry = -1; + intptr_t FileSize() const { return entries_.length() * entry_size; } intptr_t MemorySize() const { return FileSize(); } @@ -755,6 +728,7 @@ class DynamicTable : public Section { Entry(elf::DynamicEntryType tag, intptr_t value) : tag(tag), value(value) {} void Write(ElfWriteStream* stream) { + ASSERT(value != kInvalidEntry); const intptr_t start = stream->Position(); #if defined(TARGET_ARCH_IS_32_BIT) stream->WriteWord(static_cast(tag)); @@ -775,6 +749,24 @@ class DynamicTable : public Section { entries_.Add(entry); } + void FinalizeEntry(elf::DynamicEntryType tag, intptr_t value) { + for (auto* entry : entries_) { + if (entry->tag == tag) { + entry->value = value; + break; + } + } + } + + void FinalizeEntries(StringTable* strtab, + SymbolTable* symtab, + SymbolHashTable* hash) { + FinalizeEntry(elf::DynamicEntryType::DT_HASH, hash->memory_offset()); + FinalizeEntry(elf::DynamicEntryType::DT_STRTAB, strtab->memory_offset()); + FinalizeEntry(elf::DynamicEntryType::DT_STRSZ, strtab->MemorySize()); + FinalizeEntry(elf::DynamicEntryType::DT_SYMTAB, symtab->memory_offset()); + } + private: GrowableArray entries_; }; @@ -884,10 +876,26 @@ class BitsContainer : public Section { // as an absolute offset in the ELF memory space. if (reloc.source_symbol != nullptr) { const Symbol* const source_symbol = symtab->Find(reloc.source_symbol); + ASSERT(source_symbol != nullptr); source_address += source_symbol->offset(); } if (reloc.target_symbol != nullptr) { const Symbol* const target_symbol = symtab->Find(reloc.target_symbol); + if (target_symbol == nullptr) { + ASSERT_EQUAL(strcmp(reloc.target_symbol, kSnapshotBuildIdAsmSymbol), + 0); + ASSERT_EQUAL(reloc.target_offset, 0); + ASSERT_EQUAL(reloc.source_offset, 0); + ASSERT_EQUAL(reloc.size_in_bytes, compiler::target::kWordSize); + // TODO(dartbug.com/43516): Special case for snapshots with deferred + // sections that handles the build ID relocation in an + // InstructionsSection when there is no build ID. + const word to_write = Image::kNoRelocatedAddress; + stream->WriteBytes(reinterpret_cast(&to_write), + reloc.size_in_bytes); + current_pos = reloc.section_offset + reloc.size_in_bytes; + continue; + } target_address += target_symbol->offset(); } ASSERT(reloc.size_in_bytes <= kWordSize); @@ -917,8 +925,70 @@ class BitsContainer : public Section { const ZoneGrowableArray* const symbols_; }; -// We assume that the final program table fits in a single page of memory. -static constexpr intptr_t kProgramTableSegmentSize = Elf::kPageSize; +Elf::Elf(Zone* zone, BaseWriteStream* stream, Type type, Dwarf* dwarf) + : zone_(zone), + unwrapped_stream_(stream), + type_(type), + dwarf_(dwarf), + shstrtab_(new (zone) StringTable(zone, /*allocate=*/false)), + dynstrtab_(new (zone) StringTable(zone, /*allocate=*/true)), + dynsym_(new (zone) SymbolTable(zone, dynstrtab_, /*dynamic=*/true)), + strtab_(new (zone_) StringTable(zone_, /*allocate=*/false)), + symtab_(new (zone_) SymbolTable(zone, strtab_, /*dynamic=*/false)) { + // Separate debugging information should always have a Dwarf object. + ASSERT(type_ == Type::Snapshot || dwarf_ != nullptr); + // Assumed by various offset logic in this file. + ASSERT_EQUAL(unwrapped_stream_->Position(), 0); +} + +void Elf::AddSection(Section* section, + const char* name, + const char* symbol_name) { + ASSERT(section_table_file_size_ < 0); + ASSERT(!shstrtab_->HasBeenFinalized()); + section->set_name(shstrtab_->AddString(name)); + // We do not set the section index yet, that will be done during Finalize(). + sections_.Add(section); + // We do set the initial section index in initialized symbols for quick lookup + // until reordering happens. + const intptr_t initial_section_index = sections_.length() - 1; + if (symbol_name != nullptr) { + ASSERT(section->IsAllocated()); + section->symbol_name = symbol_name; + // While elf::STT_SECTION might seem more appropriate, section symbols are + // usually local and dlsym won't return them. + ASSERT(!dynsym_->HasBeenFinalized()); + dynsym_->AddSymbol(symbol_name, elf::STB_GLOBAL, elf::STT_FUNC, + initial_section_index, section->MemorySize()); + // Some tools assume the static symbol table is a superset of the dynamic + // symbol table when it exists (see dartbug.com/41783). + ASSERT(!symtab_->HasBeenFinalized()); + symtab_->AddSymbol(symbol_name, elf::STB_GLOBAL, elf::STT_FUNC, + initial_section_index, section->FileSize()); + } + if (auto const container = section->AsBitsContainer()) { + if (container->symbols() != nullptr) { + ASSERT(section->IsAllocated()); + for (const auto& symbol_data : *container->symbols()) { + ASSERT(!symtab_->HasBeenFinalized()); + symtab_->AddSymbol(symbol_data.name, elf::STB_LOCAL, symbol_data.type, + initial_section_index, symbol_data.size); + } + } + } +} + +void Elf::AddText(const char* name, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbols) { + auto const image = + new (zone_) BitsContainer(type_, /*executable=*/true, + /*writable=*/false, size, bytes, relocations, + symbols, ImageWriter::kTextAlignment); + AddSection(image, ".text", name); +} // Here, both VM and isolate will be compiled into a single snapshot. // In assembly generation, each serialized text section gets a separate @@ -932,37 +1002,17 @@ static constexpr intptr_t kBssIsolateSize = BSS::kIsolateEntryCount * compiler::target::kWordSize; static constexpr intptr_t kBssSize = kBssVmSize + kBssIsolateSize; -// For the build ID, we generate a 128-bit hash, where each 32 bits is a hash of -// the contents of the following segments in order: -// -// .text(VM) | .text(Isolate) | .rodata(VM) | .rodata(Isolate) -static constexpr const char* kBuildIdSegmentNames[]{ - kVmSnapshotInstructionsAsmSymbol, - kIsolateSnapshotInstructionsAsmSymbol, - kVmSnapshotDataAsmSymbol, - kIsolateSnapshotDataAsmSymbol, -}; -static constexpr intptr_t kBuildIdSegmentNamesLength = - ARRAY_SIZE(kBuildIdSegmentNames); -// Includes the note name, but not the description. -static constexpr intptr_t kBuildIdHeaderSize = - sizeof(elf::Note) + sizeof(elf::ELF_NOTE_GNU); - -Elf::Elf(Zone* zone, BaseWriteStream* stream, Type type, Dwarf* dwarf) - : zone_(zone), - unwrapped_stream_(stream), - type_(type), - dwarf_(dwarf), - shstrtab_(new (zone) StringTable(zone, /*allocate=*/false)), - dynstrtab_(new (zone) StringTable(zone, /*allocate=*/true)), - dynsym_(new (zone) SymbolTable(zone, dynstrtab_, /*dynamic=*/true)) { - // Separate debugging information should always have a Dwarf object. - ASSERT(type_ == Type::Snapshot || dwarf_ != nullptr); - // Assumed by various offset logic in this file. - ASSERT_EQUAL(unwrapped_stream_->Position(), 0); - // The first section in the section header table is always a reserved - // entry containing only 0 values. - sections_.Add(new (zone_) ReservedSection()); +void Elf::CreateBSS() { + uint8_t* bytes = nullptr; + if (type_ == Type::Snapshot) { + // Ideally the BSS segment would take no space in the object, but Android's + // "strip" utility truncates the memory-size of our segments to their + // file-size. + // + // Therefore we must insert zero-filled data for the BSS. + bytes = zone_->Alloc(kBssSize); + memset(bytes, 0, kBssSize); + } // For the BSS section, we add two local symbols to the static symbol table, // one for each isolate. We use local symbols because these addresses are only // used for relocation. (This matches the behavior in the assembly output, @@ -971,151 +1021,22 @@ Elf::Elf(Zone* zone, BaseWriteStream* stream, Type type, Dwarf* dwarf) bss_symbols->Add({kVmSnapshotBssAsmSymbol, elf::STT_SECTION, 0, kBssVmSize}); bss_symbols->Add({kIsolateSnapshotBssAsmSymbol, elf::STT_SECTION, kBssVmSize, kBssIsolateSize}); - CreateBSS(kBssSize, bss_symbols); - // We always allocate static string and symbol tables for use in relocation - // calculations, but we do not always add them to the final ELF file. - strtab_ = new (zone_) StringTable(zone_, /* allocate= */ false); - symtab_ = new (zone_) SymbolTable(zone, strtab_, /*dynamic=*/false); - // We add an initial segment to represent reserved space for the program - // header, and so we can always assume there's at least one segment in the - // segments_ array. We later remove this and replace it with appropriately - // calculated segments in Elf::FinalizeProgramTable(). - auto const start_segment = - new (zone_) ProgramTableLoadSegment(zone_, kProgramTableSegmentSize); - segments_.Add(start_segment); - // We allocate an initial build ID of all zeroes, since we need the build ID - // memory offset for the InstructionsSection (see BlobImageWriter::WriteText). - // We replace it with the real build ID during finalization. (We add this - // prior to BSS because we make the BuildID section writable also, so they are - // placed in the same segment before any non-writable ones, and if we add it - // after, then in separate debugging information, it'll go into a separate - // segment because the BSS section for debugging info is NOBITS.) - { - uint32_t zeroes[kBuildIdSegmentNamesLength] = {0}; - build_id_ = CreateBuildIdNote(&zeroes, sizeof(zeroes)); - AddSection(build_id_, kBuildIdNoteName, kSnapshotBuildIdAsmSymbol); - } - // Note that the BSS segment must be in the first user-defined segment because - // it cannot be placed in between any two non-writable segments, due to a bug - // in Jelly Bean's ELF loader. (For this reason, the program table segments - // generated during finalization are marked as writable.) See also - // Elf::WriteProgramTable(). - // - // We add it in all cases, even to the separate debugging information ELF, - // to ensure that relocated addresses are consistent between ELF snapshots - // and ELF separate debugging information. + bss_ = new (zone_) BitsContainer( + type_, /*executable=*/false, /*writable=*/true, kBssSize, bytes, + /*relocations=*/nullptr, bss_symbols, ImageWriter::kBssAlignment); AddSection(bss_, ".bss"); } -intptr_t Elf::AddSection(Section* section, - const char* name, - const char* symbol_name) { - ASSERT(section_table_file_size_ < 0); - ASSERT(!shstrtab_->HasBeenFinalized()); - section->set_name(shstrtab_->AddString(name)); - section->set_index(sections_.length()); - if (symbol_name != nullptr) { - section->symbol_name = symbol_name; - // While elf::STT_SECTION might seem more appropriate, section symbols are - // usually local and dlsym won't return them. - ASSERT(!dynsym_->HasBeenFinalized()); - dynsym_->AddSymbol(symbol_name, elf::STB_GLOBAL, elf::STT_FUNC, - section->index(), section->MemorySize()); - // Some tools assume the static symbol table is a superset of the dynamic - // symbol table when it exists (see dartbug.com/41783). - ASSERT(!symtab_->HasBeenFinalized()); - symtab_->AddSymbol(symbol_name, elf::STB_GLOBAL, elf::STT_FUNC, - section->index(), section->FileSize()); - } - if (auto const container = section->AsBitsContainer()) { - if (container->symbols() != nullptr) { - for (const auto& symbol_data : *container->symbols()) { - ASSERT(!symtab_->HasBeenFinalized()); - symtab_->AddSymbol(symbol_data.name, elf::STB_LOCAL, symbol_data.type, - section->index(), symbol_data.size); - } - } - } - sections_.Add(section); - - // No memory offset, so just return -1. - if (!section->IsAllocated()) return -1; - - ASSERT(program_table_file_size_ < 0); - auto const last_load = LastLoadSegment(); - if (!last_load->Add(section)) { - // We can't add this section to the last load segment, so create a new one. - // The new segment starts at the next aligned address. - auto const type = elf::ProgramHeaderType::PT_LOAD; - intptr_t alignment = - Utils::Maximum(section->alignment, Segment::Alignment(type)); - auto const start_address = - Utils::RoundUp(last_load->MemoryEnd(), alignment); - section->set_memory_offset(start_address); - auto const segment = new (zone_) Segment(zone_, section, type); - segments_.Add(segment); - } - return section->memory_offset(); -} - -void Elf::ReplaceSection(Section* old_section, Section* new_section) { - ASSERT(section_table_file_size_ < 0); - ASSERT(old_section->index_is_set()); - ASSERT(!new_section->index_is_set()); - ASSERT_EQUAL(new_section->IsAllocated(), old_section->IsAllocated()); - new_section->set_name(old_section->name()); - new_section->set_index(old_section->index()); - new_section->symbol_name = old_section->symbol_name; - sections_[old_section->index()] = new_section; - - if (!old_section->IsAllocated()) { - return; - } - - ASSERT(program_table_file_size_ < 0); - ASSERT(old_section->load_segment != nullptr); - old_section->load_segment->Replace(old_section, new_section); -} - -intptr_t Elf::AddText(const char* name, - const uint8_t* bytes, - intptr_t size, - const ZoneGrowableArray* relocations, - const ZoneGrowableArray* symbols) { - auto const image = - new (zone_) BitsContainer(type_, /*executable=*/true, - /*writable=*/false, size, bytes, relocations, - symbols, ImageWriter::kTextAlignment); - return AddSection(image, ".text", name); -} - -void Elf::CreateBSS(intptr_t size, +void Elf::AddROData(const char* name, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, const ZoneGrowableArray* symbols) { - uint8_t* bytes = nullptr; - if (type_ == Type::Snapshot) { - // Ideally the BSS segment would take no space in the object, but Android's - // "strip" utility truncates the memory-size of our segments to their - // file-size. - // - // Therefore we must insert zero-filled pages for the BSS. - bytes = zone_->Alloc(size); - memset(bytes, 0, size); - } - bss_ = new (zone_) BitsContainer( - type_, /*executable=*/false, /*writable=*/true, kBssSize, bytes, - /*relocations=*/nullptr, symbols, ImageWriter::kBssAlignment); -} - -intptr_t Elf::AddROData(const char* name, - const uint8_t* bytes, - intptr_t size, - const ZoneGrowableArray* relocations, - const ZoneGrowableArray* symbols) { auto const image = new (zone_) BitsContainer(type_, /*executable=*/false, /*writable=*/false, size, bytes, relocations, symbols, ImageWriter::kRODataAlignment); - return AddSection(image, ".rodata", name); + AddSection(image, ".rodata", name); } #if defined(DART_PRECOMPILER) @@ -1215,36 +1136,34 @@ class DwarfElfStream : public DwarfWriteStream { static constexpr intptr_t kInitialDwarfBufferSize = 64 * KB; #endif -Segment* Elf::LastLoadSegment() const { - for (intptr_t i = segments_.length() - 1; i >= 0; i--) { - auto const segment = segments_.At(i); - if (segment->type == elf::ProgramHeaderType::PT_LOAD) { - return segment; - } - } - // There should always be a load segment, since one is added in construction. - UNREACHABLE(); -} - const Section* Elf::FindSectionBySymbolName(const char* name) const { const Symbol* const symbol = symtab_->Find(name); if (symbol == nullptr) return nullptr; + // Should not be run between OrderSectionsAndCreateSegments (when section + // indices may change) and FinalizeSymbols() (sets the final section index). + ASSERT(segments_.length() == 0 || symbol->HasBeenFinalized()); const Section* const section = sections_[symbol->section_index]; ASSERT_EQUAL(strcmp(section->symbol_name, name), 0); return section; } void Elf::FinalizeSymbols() { + // Must be run after OrderSectionsAndCreateSegments and ComputeOffsets. + ASSERT(segments_.length() > 0); + ASSERT(section_table_file_offset_ > 0); for (const auto& section : sections_) { if (section->symbol_name != nullptr) { - dynsym_->FinalizeSymbol(section->symbol_name, section->memory_offset()); - symtab_->FinalizeSymbol(section->symbol_name, section->memory_offset()); + dynsym_->FinalizeSymbol(section->symbol_name, section->index(), + section->memory_offset()); + symtab_->FinalizeSymbol(section->symbol_name, section->index(), + section->memory_offset()); } if (auto const container = section->AsBitsContainer()) { if (container->symbols() != nullptr) { for (const auto& symbol_data : *container->symbols()) { symtab_->FinalizeSymbol( - symbol_data.name, section->memory_offset() + symbol_data.offset); + symbol_data.name, section->index(), + section->memory_offset() + symbol_data.offset); } } } @@ -1367,65 +1286,138 @@ void Elf::FinalizeDwarfSections() { #endif } -void Elf::Finalize() { - // Must be done prior to adding the dynamic and static symbol tables, and - // we also use the symbols to look up sections in GenerateFinalBuildId(). - FinalizeSymbols(); +void Elf::OrderSectionsAndCreateSegments() { + GrowableArray reordered_sections; + // The first section in the section header table is always a reserved + // entry containing only 0 values. + reordered_sections.Add(new (zone_) ReservedSection()); - if (auto const new_build_id = GenerateFinalBuildId()) { - ReplaceSection(build_id_, new_build_id); + Segment* current_segment = nullptr; + auto add_to_reordered_sections = [&](Section* section) { + section->set_index(reordered_sections.length()); + reordered_sections.Add(section); + if (!section->IsAllocated()) return; + const bool was_added = + current_segment == nullptr ? false : current_segment->Add(section); + if (!was_added) { + // There is no current segment or it is incompatible for merging, so + // following compatible segments will be merged into this one if possible. + current_segment = + new (zone_) Segment(zone_, section, elf::ProgramHeaderType::PT_LOAD); + section->load_segment = current_segment; + segments_.Add(current_segment); + } + }; - // Add a PT_NOTE segment for the build ID. - segments_.Add(new (zone_) NoteSegment(zone_, new_build_id)); + // Add writable, non-executable sections first, due to a bug in Jelly Bean's + // ELF loader when a writable segment is placed between two non-writable + // segments. See also Elf::WriteProgramTable(), which double-checks this. + for (auto* const section : sections_) { + if (section->IsAllocated() && section->IsWritable() && + !section->IsExecutable()) { + add_to_reordered_sections(section); + } } + // Now add the non-writable, non-executable allocated sections in a new + // segment, starting with the data sections. + for (auto* const section : sections_) { + if (section->IsAllocated() && !section->IsWritable() && + !section->IsExecutable()) { + add_to_reordered_sections(section); + } + } + + // Now add the non-writable, executable sections in a new segment. + for (auto* const section : sections_) { + if (section->IsAllocated() && !section->IsWritable() && + section->IsExecutable()) { + add_to_reordered_sections(section); + } + } + + // We put all unallocated sections last because otherwise, they would + // affect the file offset but not the memory offset of any following allocated + // sections. Doing it in this order makes it easier to keep file and memory + // offsets page-aligned with respect to each other, which is required for + // some loaders. + for (auto* const section : sections_) { + if (!section->IsAllocated()) { + add_to_reordered_sections(section); + } + } + + // Now replace sections_. + sections_.Clear(); + sections_.AddArray(reordered_sections); +} + +void Elf::Finalize() { + ASSERT(program_table_file_size_ < 0); + + // Generate the build ID now that we have all user-provided sections. + // Generating it at this point also means it'll be the first writable + // non-executable section added to sections_ and thus end up right after the + // program table after reordering. This limits how much of the ELF file needs + // to be read to get the build ID (header + program table + note segment). + GenerateBuildId(); + + // We add BSS in all cases, even to the separate debugging information ELF, + // to ensure that relocated addresses are consistent between ELF snapshots + // and ELF separate debugging information. + CreateBSS(); + // Adding the dynamic symbol table and associated sections. AddSection(dynstrtab_, ".dynstr"); AddSection(dynsym_, ".dynsym"); - dynsym_->link = dynstrtab_->index(); auto const hash = new (zone_) SymbolHashTable(zone_, dynstrtab_, dynsym_); AddSection(hash, ".hash"); - // Must come before .dynamic, because .dynamic is writable and - // .eh_frame is not. See restriction in Elf::WriteProgramTable. - FinalizeEhFrame(); - - auto const dynamic = - new (zone_) DynamicTable(zone_, dynstrtab_, dynsym_, hash); + auto const dynamic = new (zone_) DynamicTable(zone_); AddSection(dynamic, ".dynamic"); + if (!IsStripped()) { + AddSection(strtab_, ".strtab"); + AddSection(symtab_, ".symtab"); + } + AddSection(shstrtab_, ".shstrtab"); + FinalizeEhFrame(); + FinalizeDwarfSections(); + + OrderSectionsAndCreateSegments(); + + // Now that the sections have indices, set up links between them as needed. + dynsym_->link = dynstrtab_->index(); + hash->link = dynsym_->index(); + dynamic->link = dynstrtab_->index(); + if (!IsStripped()) { + symtab_->link = strtab_->index(); + } + + // Now add any special non-load segments. + + if (build_id_ != nullptr) { + // Add a PT_NOTE segment for the build ID. + segments_.Add(new (zone_) NoteSegment(zone_, build_id_)); + } + // Add a PT_DYNAMIC segment for the dynamic symbol table. segments_.Add(new (zone_) DynamicSegment(zone_, dynamic)); - // Currently, we add all (non-reserved) unallocated sections after all - // allocated sections. If we put unallocated sections between allocated - // sections, they would affect the file offset but not the memory offset - // of the later allocated sections. - // - // However, memory offsets must be page-aligned to the file offset for the - // ELF file to be successfully loaded. This means we'd either have to add - // extra padding _or_ determine file offsets before memory offsets. The - // latter would require us to handle BSS relocations during ELF finalization, - // instead of while writing the .text section content. - if (!IsStripped()) { - AddSection(strtab_, ".strtab"); - AddSection(symtab_, ".symtab"); - symtab_->link = strtab_->index(); - } - AddSection(shstrtab_, ".shstrtab"); - FinalizeDwarfSections(); - - // At this point, all non-programmatically calculated sections and segments - // have been added. Add any programatically calculated sections and segments - // and then calculate file offsets. + // At this point, all sections have been added and ordered and all sections + // appropriately grouped into segments. Add the program table and then + // calculate file and memory offsets. FinalizeProgramTable(); - ComputeFileOffsets(); + ComputeOffsets(); - // Must be done prior to writing the symbol tables and any sections with - // relocations. This doesn't change the size of the symbol tables, so can - // (and should) be done after file and memory offsets have been calculated. + // Now that we have reordered the sections and set memory offsets, we can + // update the symbol tables to add index and address information. This must + // be done prior to writing the symbol tables and any sections with + // relocations. FinalizeSymbols(); + // Also update the entries in the dynamic table. + dynamic->FinalizeEntries(dynstrtab_, dynsym_, hash); // Finally, write the ELF file contents. ElfWriteStream wrapped(unwrapped_stream_, *this); @@ -1435,42 +1427,50 @@ void Elf::Finalize() { WriteSectionTable(&wrapped); } -Section* Elf::GenerateFinalBuildId() { +// For the build ID, we generate a 128-bit hash, where each 32 bits is a hash of +// the contents of the following segments in order: +// +// .text(VM) | .text(Isolate) | .rodata(VM) | .rodata(Isolate) +static constexpr const char* kBuildIdSegmentNames[]{ + kVmSnapshotInstructionsAsmSymbol, + kIsolateSnapshotInstructionsAsmSymbol, + kVmSnapshotDataAsmSymbol, + kIsolateSnapshotDataAsmSymbol, +}; +static constexpr intptr_t kBuildIdSegmentNamesLength = + ARRAY_SIZE(kBuildIdSegmentNames); +// Includes the note name, but not the description. +static constexpr intptr_t kBuildIdHeaderSize = + sizeof(elf::Note) + sizeof(elf::ELF_NOTE_GNU); + +void Elf::GenerateBuildId() { uint32_t hashes[kBuildIdSegmentNamesLength]; for (intptr_t i = 0; i < kBuildIdSegmentNamesLength; i++) { auto const name = kBuildIdSegmentNames[i]; auto const section = FindSectionBySymbolName(name); - if (section == nullptr) { - // If we're missing a section, then we don't generate a final build ID. - return nullptr; - } + // If we're missing a section, then we don't generate a final build ID. + if (section == nullptr) return; auto const bits = section->AsBitsContainer(); if (bits == nullptr) { FATAL1("Section for symbol %s is not a BitsContainer", name); } - if (bits->bytes() == nullptr) { - // For now, if we don't have section contents (because we're generating - // assembly), don't generate a final build ID, as we'll have different - // build IDs in the snapshot and the separate debugging information. - // - // TODO(dartbug.com/43274): Change once we generate consistent build IDs - // between assembly snapshots and their debugging information. - return nullptr; - } + // For now, if we don't have section contents (because we're generating + // assembly), don't generate a final build ID, as we'll have different + // build IDs in the snapshot and the separate debugging information. + // + // TODO(dartbug.com/43274): Change once we generate consistent build IDs + // between assembly snapshots and their debugging information. + if (bits->bytes() == nullptr) return; hashes[i] = bits->Hash(); } + auto const description_bytes = reinterpret_cast(hashes); + const size_t description_length = sizeof(hashes); // To ensure we can quickly check for a final build ID, we ensure the first // byte contains a non-zero value. - auto const bytes = reinterpret_cast(hashes); - if (bytes[0] == 0) { - bytes[0] = 1; + if (description_bytes[0] == 0) { + description_bytes[0] = 1; } - return CreateBuildIdNote(&hashes, sizeof(hashes)); -} - -Section* Elf::CreateBuildIdNote(const void* description_bytes, - intptr_t description_length) { - ASSERT(description_length == 0 || description_bytes != nullptr); + // Now that we have the description field contents, create the section. ZoneWriteStream stream(zone(), kBuildIdHeaderSize + description_length); stream.WriteFixed(sizeof(elf::ELF_NOTE_GNU)); stream.WriteFixed(description_length); @@ -1479,15 +1479,15 @@ Section* Elf::CreateBuildIdNote(const void* description_bytes, stream.WriteBytes(elf::ELF_NOTE_GNU, sizeof(elf::ELF_NOTE_GNU)); ASSERT_EQUAL(stream.bytes_written(), kBuildIdHeaderSize); stream.WriteBytes(description_bytes, description_length); - // While the build ID section does not need to be writable, it and the - // BSS section are allocated segments at the same time. Having the same flags - // ensures they will be combined in the same segment and not unnecessarily - // aligned into a new page. - return new (zone_) BitsContainer( + // While the build ID section does not need to be writable, the first segment + // in our ELF files is writable (see Elf::WriteProgramTable) and so this + // ensures we can put it right after the program table without padding. + build_id_ = new (zone_) BitsContainer( elf::SectionHeaderType::SHT_NOTE, /*allocate=*/true, /*executable=*/false, /*writable=*/true, stream.bytes_written(), stream.buffer(), /*relocations=*/nullptr, /*symbols=*/nullptr, kNoteAlignment); + AddSection(build_id_, kBuildIdNoteName, kSnapshotBuildIdAsmSymbol); } void Elf::FinalizeProgramTable() { @@ -1495,68 +1495,76 @@ void Elf::FinalizeProgramTable() { program_table_file_offset_ = sizeof(elf::ElfHeader); - // There are two segments we need the size of the program table to create, so - // calculate it as if those two segments were already in place. + // There is one additional segment we need the size of the program table to + // create, so calculate it as if that segment were already in place. program_table_file_size_ = - (2 + segments_.length()) * sizeof(elf::ProgramHeader); + (1 + segments_.length()) * sizeof(elf::ProgramHeader); - // We pre-allocated the virtual memory space for the program table itself. - // Check that we didn't generate too many segments. Currently we generate a - // fixed num of segments based on the four pieces of a snapshot, but if we - // use more in the future we'll likely need to do something more compilated - // to generate DWARF without knowing a piece's virtual address in advance. auto const program_table_segment_size = program_table_file_offset_ + program_table_file_size_; - RELEASE_ASSERT(program_table_segment_size < kProgramTableSegmentSize); - // Remove the original stand-in segment we added in the constructor. - segments_.EraseAt(0); + // Segment for loading the initial part of the ELF file, including the + // program header table. Required by Android but not by Linux. + Segment* const initial_load = + new (zone_) ProgramTableLoadSegment(zone_, program_table_segment_size); + // Merge the initial writable segment into this one and replace it (so it + // doesn't change the number of segments). + const bool was_merged = initial_load->Merge(segments_[0]); + ASSERT(was_merged); + segments_[0] = initial_load; // Self-reference to program header table. Required by Android but not by // Linux. Must appear before any PT_LOAD entries. segments_.InsertAt( 0, new (zone_) ProgramTableSelfSegment(zone_, program_table_file_offset_, program_table_file_size_)); - - // Segment for loading the initial part of the ELF file, including the - // program header table. Required by Android but not by Linux. - segments_.InsertAt(1, new (zone_) ProgramTableLoadSegment( - zone_, program_table_segment_size)); } static const intptr_t kElfSectionTableAlignment = compiler::target::kWordSize; -void Elf::ComputeFileOffsets() { +void Elf::ComputeOffsets() { // We calculate the size and offset of the program header table during // finalization. ASSERT(program_table_file_offset_ > 0 && program_table_file_size_ > 0); intptr_t file_offset = program_table_file_offset_ + program_table_file_size_; - // When calculating file offsets for sections, we'll need to know if we've - // changed segments. Start with the one for the program table. + // Program table memory size is same as file size. + intptr_t memory_offset = file_offset; + + // When calculating memory and file offsets for sections, we'll need to know + // if we've changed segments. Start with the one for the program table. + ASSERT(segments_[0]->type != elf::ProgramHeaderType::PT_LOAD); const auto* current_segment = segments_[1]; + ASSERT(current_segment->type == elf::ProgramHeaderType::PT_LOAD); // The non-reserved sections are output to the file in order after the program // header table. If we're entering a new segment, then we need to align // according to the PT_LOAD segment alignment as well to keep the file offsets // aligned with the memory addresses. - auto const load_align = Segment::Alignment(elf::ProgramHeaderType::PT_LOAD); for (intptr_t i = 1; i < sections_.length(); i++) { auto const section = sections_[i]; file_offset = Utils::RoundUp(file_offset, section->alignment); + memory_offset = Utils::RoundUp(memory_offset, section->alignment); if (section->IsAllocated() && section->load_segment != current_segment) { - file_offset = Utils::RoundUp(file_offset, load_align); current_segment = section->load_segment; + ASSERT(current_segment->type == elf::ProgramHeaderType::PT_LOAD); + const intptr_t load_align = Segment::Alignment(current_segment->type); + file_offset = Utils::RoundUp(file_offset, load_align); + memory_offset = Utils::RoundUp(memory_offset, load_align); } section->set_file_offset(file_offset); -#if defined(DEBUG) if (section->IsAllocated()) { - // For files that will be dynamically loaded, make sure the file offsets - // of allocated sections are page aligned to the memory offsets. - ASSERT_EQUAL(section->file_offset() % load_align, - section->memory_offset() % load_align); - } + section->set_memory_offset(memory_offset); +#if defined(DEBUG) + if (type_ == Type::Snapshot) { + // For files that will be dynamically loaded, make sure the file offsets + // of allocated sections are page aligned to the memory offsets. + ASSERT_EQUAL(section->file_offset() % Elf::kPageSize, + section->memory_offset() % Elf::kPageSize); + } #endif + } file_offset += section->FileSize(); + memory_offset += section->MemorySize(); } file_offset = Utils::RoundUp(file_offset, kElfSectionTableAlignment); @@ -1674,21 +1682,29 @@ void Elf::WriteSectionTable(ElfWriteStream* stream) { void Elf::WriteSections(ElfWriteStream* stream) { ASSERT(section_table_file_size_ >= 0); // Check for finalization. - + // Should be writing the first section immediately after the program table. + ASSERT_EQUAL(stream->Position(), + program_table_file_offset_ + program_table_file_size_); // Skip the reserved first section, as its alignment is 0 (which will cause // stream->Align() to fail) and it never contains file contents anyway. ASSERT_EQUAL(static_cast(sections_[0]->type), static_cast(elf::SectionHeaderType::SHT_NULL)); ASSERT_EQUAL(sections_[0]->alignment, 0); - auto const load_align = Segment::Alignment(elf::ProgramHeaderType::PT_LOAD); + // The program table is considered part of the first load segment (the + // second segment in segments_), so other sections in the same segment should + // not have extra segment alignment added. + ASSERT(segments_[0]->type != elf::ProgramHeaderType::PT_LOAD); const Segment* current_segment = segments_[1]; + ASSERT(current_segment->type == elf::ProgramHeaderType::PT_LOAD); for (intptr_t i = 1; i < sections_.length(); i++) { Section* section = sections_[i]; stream->Align(section->alignment); if (section->IsAllocated() && section->load_segment != current_segment) { // Changing segments, so align accordingly. - stream->Align(load_align); current_segment = section->load_segment; + ASSERT(current_segment->type == elf::ProgramHeaderType::PT_LOAD); + const intptr_t load_align = Segment::Alignment(current_segment->type); + stream->Align(load_align); } ASSERT_EQUAL(stream->Position(), section->file_offset()); section->Write(stream); diff --git a/runtime/vm/elf.h b/runtime/vm/elf.h index 52156ae08ac..035b487d703 100644 --- a/runtime/vm/elf.h +++ b/runtime/vm/elf.h @@ -65,50 +65,43 @@ class Elf : public ZoneAllocated { size_t size; }; - intptr_t AddText(const char* name, - const uint8_t* bytes, - intptr_t size, - const ZoneGrowableArray* relocations, - const ZoneGrowableArray* symbol); - intptr_t AddROData(const char* name, - const uint8_t* bytes, - intptr_t size, - const ZoneGrowableArray* relocations, - const ZoneGrowableArray* symbols); + void AddText(const char* name, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbol); + void AddROData(const char* name, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbols); void Finalize(); private: static constexpr const char* kBuildIdNoteName = ".note.gnu.build-id"; - void CreateBSS(intptr_t size, const ZoneGrowableArray* symbols); - // Adds the section and also creates a PT_LOAD segment for the section if it // is an allocated section. // - // For allocated sections, if symbol_name is provided, a symbol for the + // For allocated sections, if a symbol_name is provided, a symbol for the // section will be added to the dynamic table (if allocated) and static // table (if not stripped) during finalization. - // - // Returns the memory offset if the section is allocated. - intptr_t AddSection(Section* section, - const char* name, - const char* symbol_name = nullptr); - // Replaces [old_section] with [new_section] in all appropriate places. If the - // section is allocated, the memory size of the section must be the same as - // the original to ensure any already-calculated memory offsets are unchanged. - void ReplaceSection(Section* old_section, Section* new_section); + void AddSection(Section* section, + const char* name, + const char* symbol_name = nullptr); - Segment* LastLoadSegment() const; const Section* FindSectionBySymbolName(const char* symbol_name) const; - Section* CreateBuildIdNote(const void* description_bytes, - intptr_t description_length); - Section* GenerateFinalBuildId(); + + void CreateBSS(); + void GenerateBuildId(); + + void OrderSectionsAndCreateSegments(); void FinalizeSymbols(); void FinalizeDwarfSections(); void FinalizeProgramTable(); - void ComputeFileOffsets(); + void ComputeOffsets(); void FinalizeEhFrame(); @@ -131,22 +124,22 @@ class Elf : public ZoneAllocated { StringTable* const dynstrtab_; SymbolTable* const dynsym_; - // We always create a BSS section for all Elf files, though it may be NOBITS - // if this is separate debugging information. + // The static tables are always created for use in relocation calculations, + // even though they may not end up in the final ELF file. + StringTable* const strtab_; + SymbolTable* const symtab_; + + // We always create a BSS section for all Elf files to keep memory offsets + // consistent, though it is NOBITS for separate debugging information. Section* bss_ = nullptr; - // The static tables are lazily created when static symbols are added. - StringTable* strtab_ = nullptr; - SymbolTable* symtab_ = nullptr; - - // We always create a GNU build ID for all Elf files. In order to create - // the appropriate offset to it in an InstructionsSection object, we create an - // initial build ID section as a placeholder and then replace that section - // during finalization once we have the information to calculate the real one. - Section* build_id_; + // We currently create a GNU build ID for all ELF snapshots and associated + // debugging information. + Section* build_id_ = nullptr; GrowableArray sections_; GrowableArray segments_; + intptr_t memory_offset_; intptr_t section_table_file_offset_ = -1; intptr_t section_table_file_size_ = -1; diff --git a/runtime/vm/image_snapshot.h b/runtime/vm/image_snapshot.h index 976da10dbdf..3b2ed095c6f 100644 --- a/runtime/vm/image_snapshot.h +++ b/runtime/vm/image_snapshot.h @@ -25,6 +25,7 @@ namespace dart { // Forward declarations. +class BitsContainer; class Code; class Dwarf; class Elf; @@ -137,6 +138,7 @@ class Image : ValueObject { // For access to private constants. friend class AssemblyImageWriter; + friend class BitsContainer; friend class BlobImageWriter; friend class ImageWriter;