[vm/aot] Add a GNU build ID to direct-to-ELF snapshots.

For proper crashpad integration, we need to generate a build ID, as the
build ID generated by crashpad if there is not one will be a simple XOR
of the first text page, which rarely changes for Dart snapshots.
Assembly snapshots already have a build ID included by the assembler, so
we currently only do this for ELF snapshots.

Currently the build ID is a 128-bit hash value that is four separate
32-bit hash values concatenated together. Those hash values come from
the contents of the VM and isolate .text and .rodata sections.

This change also contains work to separate out the concepts of sections
and segments in the ELF builder. Now, consecutive allocated sections
with the same write and execute flags are combined into a single PT_LOAD
segment when possible, which reduces the padding needed to ensure that
segments start on page boundaries in ELF snapshots.

Bug: https://github.com/dart-lang/sdk/issues/42020
Change-Id: I42a837dae665a3902d881b8d151b49ede87d6c67
Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-linux-release-x64-try,vm-kernel-precomp-linux-product-x64-try,vm-kernel-precomp-linux-debug-x64-try,vm-kernel-precomp-linux-release-simarm_x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/150625
Commit-Queue: Tess Strickland <sstrickl@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

pkg/native_stack_traces/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.3.7
+
+- Added buildId accessor for retrieving GNU build IDs from DWARF files that
+  include them.
+
 ## 0.3.6
 
 - Adjusts RegExp for stack trace header line to be more flexible in what it

pkg/native_stack_traces/lib/src/constants.dart

@@ -2,6 +2,13 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// The section name in which the build ID is stored as a note.
+const String buildIdSectionName = ".note.gnu.build-id";
+// The type of a build ID note.
+const int buildIdNoteType = 3;
+// The name of a build ID note.
+const String buildIdNoteName = "GNU";
+
 // The dynamic symbol name for the VM instructions section.
 const String vmSymbolName = "_kDartVmSnapshotInstructions";
 

pkg/native_stack_traces/lib/src/dwarf.dart

@@ -1214,6 +1214,18 @@ class Dwarf {
         vmStartAddress, isolateStartAddress);
   }
 
+  /// The build ID for the debugging information.
+  ///
+  /// Returns null if there is no build ID information recorded.
+  String get buildId {
+    final sections = _elf.namedSections(constants.buildIdSectionName);
+    if (sections.isEmpty) return null;
+    final Note note = sections.single;
+    if (note.type != constants.buildIdNoteType) return null;
+    if (note.name != constants.buildIdNoteName) return null;
+    return note.description.map((i) => i.toRadixString(16)).join();
+  }
+
   /// The call information for the given virtual address. There may be
   /// multiple [CallInfo] objects returned for a single virtual address when
   /// code has been inlined.

pkg/native_stack_traces/lib/src/elf.dart

@@ -322,7 +322,7 @@ class ProgramHeaderEntry {
       ..writeln(paddedHex(paddr, wordSize))
       ..write('Size in file: ')
       ..writeln(filesz)
-      ..write('Size in memory')
+      ..write('Size in memory: ')
       ..writeln(memsz)
       ..write('Alignment: 0x')
       ..write(paddedHex(align, wordSize));
@@ -421,6 +421,7 @@ class SectionHeaderEntry {
   static const _SHT_STRTAB = 3;
   static const _SHT_HASH = 5;
   static const _SHT_DYNAMIC = 6;
+  static const _SHT_NOTE = 7;
   static const _SHT_NOBITS = 8;
   static const _SHT_DYNSYM = 11;
 
@@ -437,6 +438,7 @@ class SectionHeaderEntry {
     _SHT_STRTAB: "SHT_STRTAB",
     _SHT_HASH: "SHT_HASH",
     _SHT_DYNAMIC: "SHT_DYNAMIC",
+    _SHT_NOTE: "SHT_NOTE",
     _SHT_NOBITS: "SHT_NOBITS",
     _SHT_DYNSYM: "SHT_DYNSYM",
   };
@@ -545,6 +547,8 @@ class Section {
         return SymbolTable.fromReader(reader, entry);
       case SectionHeaderEntry._SHT_DYNSYM:
         return SymbolTable.fromReader(reader, entry);
+      case SectionHeaderEntry._SHT_NOTE:
+        return Note.fromReader(reader, entry);
       default:
         return Section._(entry);
     }
@@ -574,6 +578,55 @@ class Section {
   }
 }
 
+/// A section that contains a single note.
+class Note extends Section {
+  final int type;
+  final String name;
+  final Uint8List description;
+
+  Note._(entry, this.type, this.name, this.description) : super._(entry);
+
+  static Note fromReader(Reader originalReader, SectionHeaderEntry entry) {
+    final reader = originalReader.refocusedCopy(entry.offset, entry.size);
+    final nameLength = reader.readBytes(4);
+    final descriptionLength = reader.readBytes(4);
+    final type = reader.readBytes(4);
+    final nameEnd = reader.offset + nameLength;
+    final name = reader.readNullTerminatedString();
+    assert(reader.offset == nameEnd);
+    assert(reader.length - reader.offset == descriptionLength);
+    final descriptionStart = reader.offset;
+    final descriptionEnd = descriptionStart + descriptionLength;
+    final description =
+        Uint8List.sublistView(reader.bdata, descriptionStart, descriptionEnd);
+    return Note._(entry, type, name, description);
+  }
+
+  void writeToStringBuffer(StringBuffer buffer) {
+    buffer
+      ..write('Section "')
+      ..write(headerEntry.name)
+      ..writeln('" is a note:');
+    buffer
+      ..write('  Type: ')
+      ..writeln(type);
+    buffer
+      ..write('  Name: "')
+      ..write(name)
+      ..writeln('"');
+    buffer
+      ..write('  Description: ')
+      ..writeln(description);
+  }
+
+  @override
+  String toString() {
+    final buffer = StringBuffer();
+    writeToStringBuffer(buffer);
+    return buffer.toString();
+  }
+}
+
 /// A map from table offsets to strings, used to store names of ELF objects.
 class StringTable extends Section {
   final _entries;

pkg/native_stack_traces/pubspec.yaml

@@ -1,11 +1,11 @@
 name: native_stack_traces
 description: Utilities for working with non-symbolic stack traces.
-version: 0.3.6
+version: 0.3.7
 
 homepage: https://github.com/dart-lang/sdk/tree/master/pkg/native_stack_traces
 
 environment:
-  sdk: '>=2.7.1 <3.0.0'
+  sdk: '>=2.8 <3.0.0'
 
 executables:
   decode:

runtime/include/dart_api.h

@@ -3524,17 +3524,20 @@ typedef void (*Dart_StreamingWriteCallback)(void* callback_data,
 // Use the '...CSymbol' definitions for resolving through 'dlsym'. The actual
 // symbol names in the objects are given by the '...AsmSymbol' definitions.
 #if defined(__APPLE__)
+#define kSnapshotBuildIdCSymbol "kDartSnapshotBuildId"
 #define kVmSnapshotDataCSymbol "kDartVmSnapshotData"
 #define kVmSnapshotInstructionsCSymbol "kDartVmSnapshotInstructions"
 #define kIsolateSnapshotDataCSymbol "kDartIsolateSnapshotData"
 #define kIsolateSnapshotInstructionsCSymbol "kDartIsolateSnapshotInstructions"
 #else
+#define kSnapshotBuildIdCSymbol "_kDartSnapshotBuildId"
 #define kVmSnapshotDataCSymbol "_kDartVmSnapshotData"
 #define kVmSnapshotInstructionsCSymbol "_kDartVmSnapshotInstructions"
 #define kIsolateSnapshotDataCSymbol "_kDartIsolateSnapshotData"
 #define kIsolateSnapshotInstructionsCSymbol "_kDartIsolateSnapshotInstructions"
 #endif
 
+#define kSnapshotBuildIdAsmSymbol "_kDartSnapshotBuildId"
 #define kVmSnapshotDataAsmSymbol "_kDartVmSnapshotData"
 #define kVmSnapshotInstructionsAsmSymbol "_kDartVmSnapshotInstructions"
 #define kIsolateSnapshotDataAsmSymbol "_kDartIsolateSnapshotData"

runtime/platform/elf.h

@@ -36,8 +36,10 @@ struct ElfHeader {
 };
 
 enum class ProgramHeaderType : uint32_t {
+  PT_NULL = 0,
   PT_LOAD = 1,
   PT_DYNAMIC = 2,
+  PT_NOTE = 4,
   PT_PHDR = 6,
 };
 
@@ -63,10 +65,22 @@ struct ProgramHeader {
 #endif
 };
 
+enum class SectionHeaderType : uint32_t {
+  SHT_NULL = 0,
+  SHT_PROGBITS = 1,
+  SHT_SYMTAB = 2,
+  SHT_STRTAB = 3,
+  SHT_HASH = 5,
+  SHT_NOTE = 7,
+  SHT_NOBITS = 8,
+  SHT_DYNAMIC = 6,
+  SHT_DYNSYM = 11,
+};
+
 struct SectionHeader {
 #if defined(TARGET_ARCH_IS_32_BIT)
   uint32_t name;
-  uint32_t type;
+  SectionHeaderType type;
   uint32_t flags;
   uint32_t memory_offset;
   uint32_t file_offset;
@@ -77,7 +91,7 @@ struct SectionHeader {
   uint32_t entry_size;
 #else
   uint32_t name;
-  uint32_t type;
+  SectionHeaderType type;
   uint64_t flags;
   uint64_t memory_offset;
   uint64_t file_offset;
@@ -107,6 +121,36 @@ struct Symbol {
 #endif
 };
 
+enum class DynamicEntryType : uint32_t {
+  DT_NULL = 0,
+  DT_HASH = 4,
+  DT_STRTAB = 5,
+  DT_SYMTAB = 6,
+  DT_STRSZ = 10,
+  DT_SYMENT = 11,
+};
+
+struct DynamicEntry {
+#if defined(TARGET_ARCH_IS_32_BIT)
+  uint32_t tag;
+  uint32_t value;
+#else
+  uint64_t tag;
+  uint64_t value;
+#endif
+};
+
+enum class NoteType : uint32_t {
+  NT_GNU_BUILD_ID = 3,
+};
+
+struct Note {
+  uint32_t name_size;
+  uint32_t description_size;
+  NoteType type;
+  uint8_t data[];
+};
+
 #pragma pack(pop)
 
 static constexpr intptr_t ELFCLASS32 = 1;
@@ -134,15 +178,6 @@ static const intptr_t PF_X = 1;
 static const intptr_t PF_W = 2;
 static const intptr_t PF_R = 4;
 
-static const intptr_t SHT_NULL = 0;
-static const intptr_t SHT_PROGBITS = 1;
-static const intptr_t SHT_SYMTAB = 2;
-static const intptr_t SHT_STRTAB = 3;
-static const intptr_t SHT_HASH = 5;
-static const intptr_t SHT_NOBITS = 8;
-static const intptr_t SHT_DYNAMIC = 6;
-static const intptr_t SHT_DYNSYM = 11;
-
 static const intptr_t SHF_WRITE = 0x1;
 static const intptr_t SHF_ALLOC = 0x2;
 static const intptr_t SHF_EXECINSTR = 0x4;
@@ -151,11 +186,6 @@ static const intptr_t SHN_UNDEF = 0;
 
 static const intptr_t STN_UNDEF = 0;
 
-static const intptr_t PT_NULL = 0;
-static const intptr_t PT_LOAD = 1;
-static const intptr_t PT_DYNAMIC = 2;
-static const intptr_t PT_PHDR = 6;
-
 static const intptr_t STB_LOCAL = 0;
 static const intptr_t STB_GLOBAL = 1;
 
@@ -163,12 +193,7 @@ static const intptr_t STT_OBJECT = 1;  // I.e., data.
 static const intptr_t STT_FUNC = 2;
 static const intptr_t STT_SECTION = 3;
 
-static const intptr_t DT_NULL = 0;
-static const intptr_t DT_HASH = 4;
-static const intptr_t DT_STRTAB = 5;
-static const intptr_t DT_SYMTAB = 6;
-static const intptr_t DT_STRSZ = 10;
-static const intptr_t DT_SYMENT = 11;
+static constexpr const char* ELF_NOTE_GNU = "GNU";
 
 }  // namespace elf
 }  // namespace dart

runtime/vm/elf.h

@@ -14,12 +14,10 @@
 namespace dart {
 
 class Dwarf;
-class DynamicSegment;
-class DynamicTable;
 class ElfWriteStream;
 class Section;
+class Segment;
 class StringTable;
-class Symbol;
 class SymbolTable;
 
 class Elf : public ZoneAllocated {
@@ -39,34 +37,40 @@ class Elf : public ZoneAllocated {
 
   static const intptr_t kPageSize = 4096;
 
+  bool IsStripped() const { return dwarf_ == nullptr; }
+
   Zone* zone() { return zone_; }
   const Dwarf* dwarf() const { return dwarf_; }
   Dwarf* dwarf() { return dwarf_; }
 
   uword BssStart(bool vm) const;
 
-  intptr_t NextMemoryOffset() const { return memory_offset_; }
-  intptr_t NextSectionIndex() const { return sections_.length(); }
+  // What the next memory offset for a kPageSize-aligned section would be.
+  //
+  // Only used by BlobImageWriter::WriteText() to determine the memory offset
+  // for the text section before it is added.
+  intptr_t NextMemoryOffset() const;
+  intptr_t AddNoBits(const char* name, const uint8_t* bytes, intptr_t size);
   intptr_t AddText(const char* name, const uint8_t* bytes, intptr_t size);
   intptr_t AddROData(const char* name, const uint8_t* bytes, intptr_t size);
   void AddDebug(const char* name, const uint8_t* bytes, intptr_t size);
 
-  // Returns whether the symbol was found. If found, sets the contents of
-  // offset and size appropriately if either or both are not nullptr.
-  bool FindDynamicSymbol(const char* name,
-                         intptr_t* offset,
-                         intptr_t* size) const;
-  // Returns whether the symbol was found. If found, sets the contents of
-  // offset and size appropriately if either or both are not nullptr.
-  bool FindStaticSymbol(const char* name,
-                        intptr_t* offset,
-                        intptr_t* size) const;
-
   void Finalize();
 
  private:
-  void AddSection(Section* section, const char* name);
-  intptr_t AddSegmentSymbol(const Section* section, const char* name);
+  static Section* CreateBSS(Zone* zone, Type type, intptr_t size);
+
+  // Adds the section and also creates a PT_LOAD segment for the section if it
+  // is an allocated section.
+  //
+  // For allocated sections, if symbol_name is provided, a symbol for the
+  // section will be added to the dynamic table (if allocated) and static
+  // table (if not stripped) during finalization.
+  //
+  // Returns the memory offset if the section is allocated.
+  intptr_t AddSection(Section* section,
+                      const char* name,
+                      const char* symbol_name = nullptr);
   void AddStaticSymbol(const char* name,
                        intptr_t info,
                        intptr_t section_index,
@@ -78,10 +82,11 @@ class Elf : public ZoneAllocated {
                         intptr_t address,
                         intptr_t size);
 
-  static Section* CreateBSS(Zone* zone, Type type, intptr_t size);
-
-  const Section* FindSegmentForAddress(intptr_t address) const;
+  Segment* LastLoadSegment() const;
+  const Section* FindSectionForAddress(intptr_t address) const;
+  Section* GenerateBuildId();
 
+  void AddSectionSymbols();
   void FinalizeDwarfSections();
   void FinalizeProgramTable();
   void ComputeFileOffsets();
@@ -94,6 +99,7 @@ class Elf : public ZoneAllocated {
   Zone* const zone_;
   StreamingWriteStream* const unwrapped_stream_;
   const Type type_;
+
   // If nullptr, then the ELF file should be stripped of static information like
   // the static symbol table (and its corresponding string table).
   Dwarf* const dwarf_;
@@ -108,16 +114,12 @@ class Elf : public ZoneAllocated {
   StringTable* const dynstrtab_;
   SymbolTable* const dynsym_;
 
-  // Can only be created once the dynamic symbol table is complete.
-  DynamicTable* dynamic_ = nullptr;
-  DynamicSegment* dynamic_segment_ = nullptr;
-
   // The static tables are lazily created when static symbols are added.
   StringTable* strtab_ = nullptr;
   SymbolTable* symtab_ = nullptr;
 
   GrowableArray<Section*> sections_;
-  GrowableArray<Section*> segments_;
+  GrowableArray<Segment*> segments_;
   intptr_t memory_offset_;
   intptr_t section_table_file_offset_ = -1;
   intptr_t section_table_file_size_ = -1;

runtime/vm/image_snapshot.cc

@@ -412,6 +412,8 @@ void ImageWriter::Write(WriteStream* clustered_stream, bool vm) {
   }
 
   // Append the direct-mapped RO data objects after the clustered snapshot.
+  // We need to do this before WriteText because WriteText currently adds the
+  // finalized contents of the clustered_stream as data sections.
   offset_space_ = vm ? V8SnapshotProfileWriter::kVmData
                      : V8SnapshotProfileWriter::kIsolateData;
   WriteROData(clustered_stream);
@@ -1076,9 +1078,15 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) {
   assembly_stream_.Print(".globl %s\n", data_symbol);
   Align(kMaxObjectAlignment);
   assembly_stream_.Print("%s:\n", data_symbol);
-  uword buffer = reinterpret_cast<uword>(clustered_stream->buffer());
-  intptr_t length = clustered_stream->bytes_written();
+  const uword buffer = reinterpret_cast<uword>(clustered_stream->buffer());
+  const intptr_t length = clustered_stream->bytes_written();
   WriteByteSequence(buffer, buffer + length);
+#if defined(DART_PRECOMPILER)
+  if (debug_elf_ != nullptr) {
+    // Add a NoBits section for the ROData as well.
+    debug_elf_->AddROData(data_symbol, clustered_stream->buffer(), length);
+  }
+#endif  // defined(DART_PRECOMPILER)
 #endif  // !defined(DART_PRECOMPILED_RUNTIME)
 }
 
@@ -1486,18 +1494,20 @@ void BlobImageWriter::WriteText(WriteStream* clustered_stream, bool vm) {
     auto const segment_base2 =
         elf_->AddText(instructions_symbol, instructions_blob_stream_.buffer(),
                       instructions_blob_stream_.bytes_written());
-    ASSERT(segment_base == segment_base2);
+    ASSERT_EQUAL(segment_base2, segment_base);
     // Write the .rodata section here like the AssemblyImageWriter.
     elf_->AddROData(data_symbol, clustered_stream->buffer(),
                     clustered_stream->bytes_written());
   }
   if (debug_elf_ != nullptr) {
-    // To keep memory addresses consistent, we need to add corresponding
-    // sections (though these will be NOBITS sections).
+    // To keep memory addresses consistent, we create elf::SHT_NOBITS sections
+    // in the debugging information. We still pass along the buffers because
+    // we'll need the buffer bytes at generation time to calculate the build ID
+    // so it'll match the one in the snapshot.
     auto const debug_segment_base2 = debug_elf_->AddText(
         instructions_symbol, instructions_blob_stream_.buffer(),
         instructions_blob_stream_.bytes_written());
-    ASSERT(debug_segment_base == debug_segment_base2);
+    ASSERT_EQUAL(debug_segment_base2, debug_segment_base);
     debug_elf_->AddROData(data_symbol, clustered_stream->buffer(),
                           clustered_stream->bytes_written());
   }

tests/standalone_2/dwarf_stack_trace_test.dart

@@ -76,6 +76,8 @@ Future<void> checkStackTrace(String rawStack, Dwarf dwarf,
   // means we can't depend on virtual addresses in the snapshot lining up with
   // those in the separate debugging information.
   if (explicits.isNotEmpty) {
+    // Direct-to-ELF snapshots should have a build ID.
+    Expect.isNotNull(dwarf.buildId);
     Expect.deepEquals(relocatedAddresses, virtualAddresses);
     Expect.deepEquals(explicits, virtualAddresses);
   }