From 6b53073eae118baec4fd5b9c9b6e8cf743df1587 Mon Sep 17 00:00:00 2001 From: Tess Strickland Date: Tue, 27 May 2025 06:30:20 -0700 Subject: [PATCH] Reland "[vm, gen_snapshot] Add app-aot-macho-dylib option for AOT snapshots." This is a reland of commit 38ef28a0589c0ac3009856a636b353e419e25c46 Fixes: * Fix comparisons in ASSERT_EQUAL statements on 32-bit architectures. * Take simulated architectures into account when deciding whether to use dlopen() for native shared object formats. * Fix struct/field name collision for GCC. * Use CPU_TYPE_ANY/CPU_SUBTYPE_ANY for architectures that do not have more specific cpu_type_t/cpu_subtype_t constants defined. Original change's description: > [vm, gen_snapshot] Add app-aot-macho-dylib option for AOT snapshots. > > This is the initial framework for creating snapshots as Mach-O dynamic > libraries. Note that this framework is not 100% feature complete > compared to generating Mach-O snapshots via assembly. In particular, > the directly-compiled Mach-O dylib does not yet contain compact > unwinding information. > > Other changes: > > * Adds UuidCommand to the native_stack_traces package's Mach-O reader, > which now appropriately returns the UUID as the build ID for Mach-O > shared objects. > > * Adds Utils::Basename(path) for portably retrieving the basename > from a path. (Returns nullptr for all arguments where it is not > currently implemented on Fuchsia or Windows.) > > * Adjusts vm/timeline.h to avoid pulling in on MacOS, > as that interferes with uses of the namespaced Mach-O definitions > in platform/mach_o.h. > > * Only attempt to dlopen() a snapshot if ELF is the native format > for the host platform or the snapshot is not an ELF shared object. > If dlopen() is used, report the error message if it fails rather > than attempting to manually load the snapshot as an ELF shared object. > > * Fix the magic number stored in DylibAppSnapshot for loaded non-ELF > dynamic libraries. > > * Remove the detection of reverse-endian Mach-O magic numbers in > DartUtils::SniffForMagicNumber(), since all our Mach-O related code > assumes host-endian Mach-O files and so there's no point other than > to give a slightly better error message when failing. > > TEST=vm/dart/exported_symbols_test > vm/dart/unobfuscated_static_symbols_test > vm/dart/use_dwarf_stack_traces_flag_test > vm/cc/CanDetectMachOFiles > > Issue: https://github.com/dart-lang/sdk/issues/60307 > Change-Id: Idf5b49d6c6d035ab033509613212b95520d65965 > Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-mac-release-arm64-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try,vm-fuchsia-release-x64-try,vm-fuchsia-release-arm64-try > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/415020 > Reviewed-by: Slava Egorov > Commit-Queue: Tess Strickland TEST=vm/dart/exported_symbols_test vm/dart/unobfuscated_static_symbols_test vm/dart/use_dwarf_stack_traces_flag_test vm/cc/CanDetectMachOFiles ci on trybots that failed on the original CL Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-mac-release-arm64-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try,vm-fuchsia-release-x64-try,vm-fuchsia-release-arm64-try,vm-linux-debug-ia32-try,vm-aot-linux-debug-simarm_x64-try,vm-aot-linux-debug-simriscv32-try,vm-aot-linux-debug-simriscv64-try,vm-aot-linux-release-simarm_x64-try,vm-gcc-linux-try,vm-ubsan-linux-release-arm64-try Change-Id: Iaffea0ddc6173100c8b5b2a9fe46c45f4f611a2e Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/431240 Reviewed-by: Slava Egorov Commit-Queue: Tess Strickland --- pkg/native_stack_traces/CHANGELOG.md | 3 + pkg/native_stack_traces/lib/src/macho.dart | 30 +- pkg/native_stack_traces/pubspec.yaml | 4 +- runtime/bin/dart_api_win.c | 24 + runtime/bin/dartutils.cc | 32 +- runtime/bin/dartutils.h | 22 +- runtime/bin/gen_snapshot.cc | 180 +- runtime/bin/snapshot_utils.cc | 275 +- runtime/bin/snapshot_utils.h | 5 +- runtime/bin/snapshot_utils_test.cc | 13 +- runtime/configs.gni | 12 + runtime/include/dart_api.h | 50 +- runtime/platform/mach_o.h | 483 +++- runtime/platform/utils.cc | 18 + runtime/platform/utils.h | 13 + .../vm/dart/analyze_snapshot_binary_test.dart | 4 +- .../tests/vm/dart/disassemble_aot_test.dart | 4 +- .../tests/vm/dart/exported_symbols_test.dart | 1 + .../unobfuscated_static_symbols_test.dart | 249 +- ...dwarf_stack_traces_flag_deferred_test.dart | 21 +- .../use_dwarf_stack_traces_flag_helper.dart | 156 +- .../use_dwarf_stack_traces_flag_test.dart | 128 +- .../tests/vm/dart/use_flag_test_helper.dart | 30 +- runtime/vm/BUILD.gn | 4 + runtime/vm/dart_api_impl.cc | 91 +- runtime/vm/dwarf.cc | 19 +- runtime/vm/dwarf.h | 8 +- runtime/vm/elf.h | 2 +- runtime/vm/image_snapshot.cc | 26 +- runtime/vm/image_snapshot.h | 4 + runtime/vm/mach_o.cc | 2527 +++++++++++++++++ runtime/vm/mach_o.h | 79 + runtime/vm/os_macos.cc | 4 +- runtime/vm/timeline.cc | 4 + runtime/vm/timeline.h | 11 +- runtime/vm/timeline_macos.cc | 2 + runtime/vm/vm_sources.gni | 2 + 37 files changed, 4071 insertions(+), 469 deletions(-) create mode 100644 runtime/vm/mach_o.cc create mode 100644 runtime/vm/mach_o.h diff --git a/pkg/native_stack_traces/CHANGELOG.md b/pkg/native_stack_traces/CHANGELOG.md index 1ad73eb2c12..c339ca52dd5 100644 --- a/pkg/native_stack_traces/CHANGELOG.md +++ b/pkg/native_stack_traces/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.6.1 +- Add handling for Mach-O UUID load commands. + ## 0.6.1-wip - Update SDK constraint to `^3.5.0`. diff --git a/pkg/native_stack_traces/lib/src/macho.dart b/pkg/native_stack_traces/lib/src/macho.dart index c8d7e301deb..15b65f49590 100644 --- a/pkg/native_stack_traces/lib/src/macho.dart +++ b/pkg/native_stack_traces/lib/src/macho.dart @@ -124,6 +124,7 @@ class LoadCommand { static const LC_SEGMENT = 0x1; static const LC_SYMTAB = 0x2; static const LC_SEGMENT_64 = 0x19; + static const LC_UUID = 0x1b; static LoadCommand fromReader(Reader reader) { final start = reader.offset; // cmdsize includes size of cmd and cmdsize. @@ -139,6 +140,9 @@ class LoadCommand { case LC_SYMTAB: command = SymbolTableCommand.fromReader(reader, cmd, cmdsize); break; + case LC_UUID: + command = UuidCommand.fromReader(reader, cmd, cmdsize); + break; default: break; } @@ -324,6 +328,29 @@ class SymbolTableCommand extends LoadCommand { } } +class UuidCommand extends LoadCommand { + Uint8List uuid; + + static const kUuidSize = 16; + + UuidCommand._(super.cmd, super.cmdsize, this.uuid) : super._(); + + static UuidCommand fromReader(Reader reader, int cmd, int cmdsize) { + final uuid = Uint8List.sublistView( + reader.bytes, reader.offset, reader.offset + kUuidSize); + return UuidCommand._(cmd, cmdsize, uuid); + } + + String get uuidString => uuid.map((i) => paddedHex(i, 1)).join(); + + @override + void writeToStringBuffer(StringBuffer buffer) { + buffer + ..write('UUID: ') + ..write(uuidString); + } +} + class MachOHeader { final int magic; final int cputype; @@ -523,7 +550,8 @@ class MachO extends DwarfContainer { _symbolTable[constants.isolateSymbolName]?.value; @override - String? get buildId => null; + String? get buildId => + _commands.whereType().firstOrNull?.uuidString; @override DwarfContainerStringTable? get debugStringTable => _debugStringTable; diff --git a/pkg/native_stack_traces/pubspec.yaml b/pkg/native_stack_traces/pubspec.yaml index 466fad1abf6..d6b09a55a10 100644 --- a/pkg/native_stack_traces/pubspec.yaml +++ b/pkg/native_stack_traces/pubspec.yaml @@ -1,5 +1,5 @@ name: native_stack_traces -version: 0.6.1-wip +version: 0.6.1 description: Utilities for working with non-symbolic stack traces. repository: https://github.com/dart-lang/sdk/tree/main/pkg/native_stack_traces @@ -13,7 +13,7 @@ executables: dependencies: args: ^2.0.0 - path: ^1.8.0 + path: ^1.9.0 # We use 'any' version constraints here as we get our package versions from # the dart-lang/sdk repo's DEPS file. Note that this is a special case; the diff --git a/runtime/bin/dart_api_win.c b/runtime/bin/dart_api_win.c index c8924a21904..626b3dc51fd 100644 --- a/runtime/bin/dart_api_win.c +++ b/runtime/bin/dart_api_win.c @@ -424,6 +424,13 @@ typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsElfsType)( bool, Dart_StreamingWriteCallback, Dart_StreamingCloseCallback); +typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsBinaryType)( + Dart_AotBinaryFormat, + Dart_StreamingWriteCallback, + void*, + bool, + void*, + const char*); typedef Dart_Handle (*Dart_CreateVMAOTSnapshotAsAssemblyType)( Dart_StreamingWriteCallback, void*); @@ -724,6 +731,8 @@ static Dart_CreateAppAOTSnapshotAsElfType Dart_CreateAppAOTSnapshotAsElfFn = NULL; static Dart_CreateAppAOTSnapshotAsElfsType Dart_CreateAppAOTSnapshotAsElfsFn = NULL; +static Dart_CreateAppAOTSnapshotAsBinaryType + Dart_CreateAppAOTSnapshotAsBinaryFn = NULL; static Dart_CreateVMAOTSnapshotAsAssemblyType Dart_CreateVMAOTSnapshotAsAssemblyFn = NULL; static Dart_SortClassesType Dart_SortClassesFn = NULL; @@ -1293,6 +1302,9 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { Dart_CreateAppAOTSnapshotAsElfsFn = (Dart_CreateAppAOTSnapshotAsElfsType)GetProcAddress( process, "Dart_CreateAppAOTSnapshotAsElfs"); + Dart_CreateAppAOTSnapshotAsBinaryFn = + (Dart_CreateAppAOTSnapshotAsBinaryType)GetProcAddress( + process, "Dart_CreateAppAOTSnapshotAsBinary"); Dart_CreateVMAOTSnapshotAsAssemblyFn = (Dart_CreateVMAOTSnapshotAsAssemblyType)GetProcAddress( process, "Dart_CreateVMAOTSnapshotAsAssembly"); @@ -2551,6 +2563,18 @@ Dart_Handle Dart_CreateAppAOTSnapshotAsElfs( close_callback); } +Dart_Handle Dart_CreateAppAOTSnapshotAsBinary( + Dart_AotBinaryFormat format, + Dart_StreamingWriteCallback callback, + void* callback_data, + bool stripped, + void* debug_callback_data, + const char* identifier) { + return Dart_CreateAppAOTSnapshotAsBinaryFn(format, callback, callback_data, + stripped, debug_callback_data, + identifier); +} + Dart_Handle Dart_CreateVMAOTSnapshotAsAssembly( Dart_StreamingWriteCallback callback, void* callback_data) { diff --git a/runtime/bin/dartutils.cc b/runtime/bin/dartutils.cc index 78e2c3109ea..0c5251cb764 100644 --- a/runtime/bin/dartutils.cc +++ b/runtime/bin/dartutils.cc @@ -14,6 +14,7 @@ #include "include/dart_native_api.h" #include "platform/assert.h" #include "platform/globals.h" +#include "platform/mach_o.h" #include "platform/utils.h" // Return the error from the containing function if handle is in error handle. @@ -34,9 +35,6 @@ dart::SimpleHashMap* DartUtils::environment_ = nullptr; MagicNumberData appjit_magic_number = {8, {0xdc, 0xdc, 0xf6, 0xf6, 0, 0, 0, 0}}; MagicNumberData aotelf_magic_number = {4, {0x7F, 0x45, 0x4C, 0x46, 0x0}}; -MagicNumberData aotmacho32_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCE}}; -MagicNumberData aotmacho64_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCF}}; -MagicNumberData aotmacho64_arm64_magic_number = {4, {0xCF, 0xFA, 0xED, 0xFE}}; MagicNumberData aotcoff_arm32_magic_number = {2, {0x01, 0xC0}}; MagicNumberData aotcoff_arm64_magic_number = {2, {0xAA, 0x64}}; MagicNumberData aotcoff_riscv32_magic_number = {2, {0x50, 0x32}}; @@ -404,9 +402,8 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const char* filename) { MagicNumber magic_number = DartUtils::kUnknownMagicNumber; ASSERT(kMaxMagicNumberSize == appjit_magic_number.length); ASSERT(aotelf_magic_number.length <= appjit_magic_number.length); - ASSERT(aotmacho32_magic_number.length <= appjit_magic_number.length); - ASSERT(aotmacho64_magic_number.length <= appjit_magic_number.length); - ASSERT(aotmacho64_arm64_magic_number.length <= appjit_magic_number.length); + ASSERT(static_cast(sizeof(mach_o::mach_header::magic)) <= + appjit_magic_number.length); ASSERT(aotcoff_arm32_magic_number.length <= appjit_magic_number.length); ASSERT(aotcoff_arm64_magic_number.length <= appjit_magic_number.length); ASSERT(aotcoff_riscv32_magic_number.length <= appjit_magic_number.length); @@ -453,16 +450,19 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer, return kAotELFMagicNumber; } - if (CheckMagicNumber(buffer, buffer_length, aotmacho32_magic_number)) { - return kAotMachO32MagicNumber; - } - - if (CheckMagicNumber(buffer, buffer_length, aotmacho64_magic_number)) { - return kAotMachO64MagicNumber; - } - - if (CheckMagicNumber(buffer, buffer_length, aotmacho64_arm64_magic_number)) { - return kAotMachO64Arm64MagicNumber; + // Mach-O magic numbers are reported by whether the endianness of the file + // matches the endianness of the system. Here, we only bother looking for + // host-endian magic numbers, as our Mach-O parsing code won't handle the + // reverse endian case. + if (static_cast(sizeof(mach_o::mach_header::magic)) <= + buffer_length) { + const uint32_t magic = + reinterpret_cast(buffer)->magic; + if (magic == mach_o::MH_MAGIC) { + return kAotMachO32MagicNumber; + } else if (magic == mach_o::MH_MAGIC_64) { + return kAotMachO64MagicNumber; + } } if (CheckMagicNumber(buffer, buffer_length, aotcoff_arm32_magic_number)) { diff --git a/runtime/bin/dartutils.h b/runtime/bin/dartutils.h index 6ec605b58e0..2d0aa818f15 100644 --- a/runtime/bin/dartutils.h +++ b/runtime/bin/dartutils.h @@ -260,9 +260,10 @@ class DartUtils { kKernelListMagicNumber, kGzipMagicNumber, kAotELFMagicNumber, + // Only the host-endian magic numbers are recognized, not the reverse-endian + // ("cigam") ones, as we can't load a reverse-endian snapshot anyway. kAotMachO32MagicNumber, kAotMachO64MagicNumber, - kAotMachO64Arm64MagicNumber, kAotCoffARM32MagicNumber, kAotCoffARM64MagicNumber, kAotCoffRISCV32MagicNumber, @@ -278,7 +279,24 @@ class DartUtils { (number <= DartUtils::kAotCoffRISCV64MagicNumber); } - // Checks if the buffer is a script snapshot, kernel file, or gzip file. + // Returns the bitsize corresponding to the magic number if the bitsize + // is specified by the magic number, otherwise returns -1. + static intptr_t MagicNumberBitSize(MagicNumber number) { + if (number == DartUtils::kAotMachO32MagicNumber || + number == DartUtils::kAotCoffARM32MagicNumber || + number == DartUtils::kAotCoffRISCV32MagicNumber) { + return 32; + } + if (number == DartUtils::kAotMachO64MagicNumber || + number == DartUtils::kAotCoffARM64MagicNumber || + number == DartUtils::kAotCoffRISCV64MagicNumber) { + return 64; + } + return -1; + } + + // Checks if the file is a script snapshot, kernel file, or gzip file + // by reading the first kMaxMagicNumberSize bytes of the file. static MagicNumber SniffForMagicNumber(const char* filename); // Checks if the buffer is a script snapshot, kernel file, or gzip file. diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc index c5e7b2eff02..dcdd49feab4 100644 --- a/runtime/bin/gen_snapshot.cc +++ b/runtime/bin/gen_snapshot.cc @@ -78,6 +78,7 @@ enum SnapshotKind { kAppJIT, kAppAOTAssembly, kAppAOTElf, + kAppAOTMachODylib, kVMAOTAssembly, }; static SnapshotKind snapshot_kind = kCore; @@ -90,6 +91,7 @@ static const char* const kSnapshotKindNames[] = { "app-jit", "app-aot-assembly", "app-aot-elf", + "app-aot-macho-dylib", "vm-aot-assembly", nullptr, // clang-format on @@ -108,6 +110,7 @@ static const char* const kSnapshotKindNames[] = { V(blobs_container_filename, blobs_container_filename) \ V(assembly, assembly_filename) \ V(elf, elf_filename) \ + V(macho, macho_filename) \ V(loading_unit_manifest, loading_unit_manifest_filename) \ V(save_debugging_info, debugging_info_filename) \ V(save_obfuscation_map, obfuscation_map_filename) @@ -137,6 +140,7 @@ DEFINE_CB_OPTION(ProcessEnvironmentOption); static bool IsSnapshottingForPrecompilation() { return (snapshot_kind == kAppAOTAssembly) || (snapshot_kind == kAppAOTElf) || + (snapshot_kind == kAppAOTMachODylib) || (snapshot_kind == kVMAOTAssembly); } @@ -176,6 +180,15 @@ static void PrintUsage() { "[--save-obfuscation-map=] \n" " \n" " \n" +"To create an AOT application snapshot as an Mach-O dynamic library (dylib): \n" +"--snapshot_kind=app-aot-macho-dylib \n" +"--macho= \n" +"[--strip] \n" +"[--obfuscate] \n" +"[--save-debugging-info=] \n" +"[--save-obfuscation-map=] \n" +" \n" +" \n" "AOT snapshots can be obfuscated: that is all identifiers will be renamed \n" "during compilation. This mode is enabled with --obfuscate flag. Mapping \n" "between original and obfuscated names can be serialized as a JSON array \n" @@ -267,6 +280,15 @@ static int ParseArguments(int argc, } break; } + case kAppAOTMachODylib: { + if (macho_filename == nullptr) { + Syslog::PrintErr( + "Building an AOT snapshot as a Mach-O dynamic library requires " + " specifying an output file for --macho.\n\n"); + return -1; + } + break; + } case kAppAOTAssembly: case kVMAOTAssembly: { if (assembly_filename == nullptr) { @@ -597,78 +619,99 @@ static void NextElfCallback(void* callback_data, static void CreateAndWritePrecompiledSnapshot() { ASSERT(IsSnapshottingForPrecompilation()); - Dart_Handle result; + + if (snapshot_kind == kVMAOTAssembly) { + File* file = OpenFile(assembly_filename); + RefCntReleaseScope rs(file); + Dart_Handle result = + Dart_CreateVMAOTSnapshotAsAssembly(StreamingWriteCallback, file); + CHECK_RESULT(result); + return; + } + + Dart_AotBinaryFormat format; + const char* kind_str = nullptr; + const char* filename = nullptr; + // Default to the assembly ones just to avoid having to type-specify here. + auto* next_callback = NextAsmCallback; + auto* create_multiple_callback = Dart_CreateAppAOTSnapshotAsAssemblies; + switch (snapshot_kind) { + case kAppAOTAssembly: + kind_str = "assembly code"; + filename = assembly_filename; + format = Dart_AotBinaryFormat_Assembly; + break; + case kAppAOTElf: + kind_str = "ELF library"; + filename = elf_filename; + format = Dart_AotBinaryFormat_Elf; + next_callback = NextElfCallback; + create_multiple_callback = Dart_CreateAppAOTSnapshotAsElfs; + break; + case kAppAOTMachODylib: + kind_str = "MachO dynamic library"; + filename = macho_filename; + format = Dart_AotBinaryFormat_MachO_Dylib; + // Not currently implemented. + next_callback = nullptr; + create_multiple_callback = nullptr; + break; + default: + UNREACHABLE(); + } + ASSERT(kind_str != nullptr); + ASSERT(filename != nullptr); // Precompile with specified embedder entry points - result = Dart_Precompile(); + Dart_Handle result = Dart_Precompile(); CHECK_RESULT(result); + if (strip && (debugging_info_filename == nullptr)) { + Syslog::PrintErr( + "Warning: Generating %s without DWARF debugging" + " information.\n", + kind_str); + } + + char* identifier = Utils::Basename(filename); + // Create a precompiled snapshot. - if (snapshot_kind == kAppAOTAssembly) { - if (strip && (debugging_info_filename == nullptr)) { - Syslog::PrintErr( - "Warning: Generating assembly code without DWARF debugging" - " information.\n"); + if (loading_unit_manifest_filename == nullptr) { + File* file = OpenFile(filename); + RefCntReleaseScope rs(file); + File* debug_file = nullptr; + if (debugging_info_filename != nullptr) { + debug_file = OpenFile(debugging_info_filename); } - if (loading_unit_manifest_filename == nullptr) { - File* file = OpenFile(assembly_filename); - RefCntReleaseScope rs(file); - File* debug_file = nullptr; - if (debugging_info_filename != nullptr) { - debug_file = OpenFile(debugging_info_filename); - } - result = Dart_CreateAppAOTSnapshotAsAssembly(StreamingWriteCallback, file, - strip, debug_file); - if (debug_file != nullptr) debug_file->Release(); - CHECK_RESULT(result); - } else { - File* manifest_file = OpenLoadingUnitManifest(); - result = Dart_CreateAppAOTSnapshotAsAssemblies( - NextAsmCallback, manifest_file, strip, StreamingWriteCallback, - StreamingCloseCallback); - CHECK_RESULT(result); - CloseLoadingUnitManifest(manifest_file); - } - if (obfuscate && !strip) { - Syslog::PrintErr( - "Warning: The generated assembly code contains unobfuscated DWARF " - "debugging information.\n" - " To avoid this, use --strip to remove it.\n"); - } - } else if (snapshot_kind == kAppAOTElf) { - if (strip && (debugging_info_filename == nullptr)) { - Syslog::PrintErr( - "Warning: Generating ELF library without DWARF debugging" - " information.\n"); - } - if (loading_unit_manifest_filename == nullptr) { - File* file = OpenFile(elf_filename); - RefCntReleaseScope rs(file); - File* debug_file = nullptr; - if (debugging_info_filename != nullptr) { - debug_file = OpenFile(debugging_info_filename); - } - result = Dart_CreateAppAOTSnapshotAsElf(StreamingWriteCallback, file, - strip, debug_file); - if (debug_file != nullptr) debug_file->Release(); - CHECK_RESULT(result); - } else { - File* manifest_file = OpenLoadingUnitManifest(); - result = Dart_CreateAppAOTSnapshotAsElfs(NextElfCallback, manifest_file, - strip, StreamingWriteCallback, - StreamingCloseCallback); - CHECK_RESULT(result); - CloseLoadingUnitManifest(manifest_file); - } - if (obfuscate && !strip) { - Syslog::PrintErr( - "Warning: The generated ELF library contains unobfuscated DWARF " - "debugging information.\n" - " To avoid this, use --strip to remove it and " - "--save-debugging-info=<...> to save it to a separate file.\n"); + result = Dart_CreateAppAOTSnapshotAsBinary( + format, StreamingWriteCallback, file, strip, debug_file, identifier); + if (debug_file != nullptr) debug_file->Release(); + if (identifier != nullptr) { + free(identifier); + identifier = nullptr; } + CHECK_RESULT(result); } else { - UNREACHABLE(); + ASSERT(create_multiple_callback != nullptr); + ASSERT(next_callback != nullptr); + File* manifest_file = OpenLoadingUnitManifest(); + result = create_multiple_callback(next_callback, manifest_file, strip, + StreamingWriteCallback, + StreamingCloseCallback); + if (identifier != nullptr) { + free(identifier); + identifier = nullptr; + } + CHECK_RESULT(result); + CloseLoadingUnitManifest(manifest_file); + } + + if (obfuscate && !strip) { + Syslog::PrintErr( + "Warning: The generated %s contains unobfuscated DWARF " + "debugging information.\n" + " To avoid this, use --strip to remove it.\n", + kind_str); } // Serialize obfuscation map if requested. @@ -769,15 +812,10 @@ static int CreateIsolateAndSnapshot(const CommandLineOptions& inputs) { break; case kAppAOTAssembly: case kAppAOTElf: + case kAppAOTMachODylib: + case kVMAOTAssembly: CreateAndWritePrecompiledSnapshot(); break; - case kVMAOTAssembly: { - File* file = OpenFile(assembly_filename); - RefCntReleaseScope rs(file); - result = Dart_CreateVMAOTSnapshotAsAssembly(StreamingWriteCallback, file); - CHECK_RESULT(result); - break; - } default: UNREACHABLE(); } diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc index 460832a4a70..c56a385c3b3 100644 --- a/runtime/bin/snapshot_utils.cc +++ b/runtime/bin/snapshot_utils.cc @@ -2,10 +2,11 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -#include - #include "bin/snapshot_utils.h" +#include +#include + #include "bin/dartutils.h" #include "bin/dfe.h" #include "bin/elf_loader.h" @@ -23,6 +24,15 @@ #define LOG_SECTION_BOUNDARIES false +#if !defined(USING_SIMULATOR) +#if defined(DART_HOST_OS_LINUX) || defined(DART_HOST_OS_ANDROID) || \ + defined(DART_HOST_OS_FUCHSIA) +#define NATIVE_SHARED_OBJECT_FORMAT_ELF 1 +#elif defined(DART_HOST_OS_MACOS) +#define NATIVE_SHARED_OBJECT_FORMAT_MACHO 1 +#endif +#endif // !defined(USING_SIMULATOR) + namespace dart { namespace bin { @@ -145,6 +155,110 @@ static AppSnapshot* TryReadAppSnapshotBlobs(const char* script_name, #endif // !defined(DART_PRECOMPILED_RUNTIME) #if defined(DART_PRECOMPILED_RUNTIME) +class DylibAppSnapshot : public AppSnapshot { + public: + DylibAppSnapshot(DartUtils::MagicNumber magic_number, + void* library, + const uint8_t* vm_snapshot_data, + const uint8_t* vm_snapshot_instructions, + const uint8_t* isolate_snapshot_data, + const uint8_t* isolate_snapshot_instructions) + : AppSnapshot(magic_number), + library_(library), + vm_snapshot_data_(vm_snapshot_data), + vm_snapshot_instructions_(vm_snapshot_instructions), + isolate_snapshot_data_(isolate_snapshot_data), + isolate_snapshot_instructions_(isolate_snapshot_instructions) {} + + ~DylibAppSnapshot() { Utils::UnloadDynamicLibrary(library_); } + + void SetBuffers(const uint8_t** vm_data_buffer, + const uint8_t** vm_instructions_buffer, + const uint8_t** isolate_data_buffer, + const uint8_t** isolate_instructions_buffer) { + *vm_data_buffer = vm_snapshot_data_; + *vm_instructions_buffer = vm_snapshot_instructions_; + *isolate_data_buffer = isolate_snapshot_data_; + *isolate_instructions_buffer = isolate_snapshot_instructions_; + } + + private: + void* library_; + const uint8_t* vm_snapshot_data_; + const uint8_t* vm_snapshot_instructions_; + const uint8_t* isolate_snapshot_data_; + const uint8_t* isolate_snapshot_instructions_; +}; + +static AppSnapshot* TryReadAppSnapshotDynamicLibrary( + DartUtils::MagicNumber magic_number, + const char* script_name, + const char** error) { +#if defined(USING_SIMULATOR) + *error = "running on a simulated architecture"; + return nullptr; +#else +#if defined(DART_TARGET_OS_LINUX) || defined(DART_TARGET_OS_MACOS) + // On Linux and OSX, resolve the script path before passing into dlopen() + // since dlopen will not search the filesystem for paths like 'libtest.so'. + CStringUniquePtr absolute_path(realpath(script_name, nullptr)); + script_name = absolute_path.get(); + if (script_name == nullptr) { + const intptr_t err = errno; + const int kBufferSize = 1024; + char error_buf[kBufferSize]; + Utils::StrError(err, error_buf, kBufferSize); + *error = Utils::SCreate("could not resolve path: %s", error_buf); + return nullptr; + } +#endif + void* library = Utils::LoadDynamicLibrary(script_name, error); + if (library == nullptr) { +#if defined(NATIVE_SHARED_OBJECT_FORMAT_ELF) + if (*error == nullptr && magic_number != DartUtils::kAotELFMagicNumber) { + *error = "not an ELF shared object"; + } +#elif defined(NATIVE_SHARED_OBJECT_FORMAT_MACHO) + if (*error == nullptr && + magic_number != DartUtils::kAotMachO32MagicNumber && + magic_number != DartUtils::kAotMachO64MagicNumber) { + *error = "not a Mach-O shared object"; + } +#endif + if (*error == nullptr) { + *error = "unknown failure loading dynamic library (wrong format?)"; + } + return nullptr; + } + + const uint8_t* vm_data_buffer = reinterpret_cast( + Utils::ResolveSymbolInDynamicLibrary(library, kVmSnapshotDataCSymbol)); + + const uint8_t* vm_instructions_buffer = + reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( + library, kVmSnapshotInstructionsCSymbol)); + + const uint8_t* isolate_data_buffer = + reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( + library, kIsolateSnapshotDataCSymbol)); + if (isolate_data_buffer == nullptr) { + FATAL("Failed to resolve symbol '%s'\n", kIsolateSnapshotDataCSymbol); + } + + const uint8_t* isolate_instructions_buffer = + reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( + library, kIsolateSnapshotInstructionsCSymbol)); + if (isolate_instructions_buffer == nullptr) { + FATAL("Failed to resolve symbol '%s'\n", + kIsolateSnapshotInstructionsCSymbol); + } + + return new DylibAppSnapshot(magic_number, library, vm_data_buffer, + vm_instructions_buffer, isolate_data_buffer, + isolate_instructions_buffer); +#endif // defined(USING_SIMULATOR) +} + class ElfAppSnapshot : public AppSnapshot { public: ElfAppSnapshot(Dart_LoadedElf* elf, @@ -184,6 +298,18 @@ static AppSnapshot* TryReadAppSnapshotElf( uint64_t file_offset, bool force_load_elf_from_memory = false) { const char* error = nullptr; +#if defined(NATIVE_SHARED_OBJECT_FORMAT_ELF) + if (file_offset == 0 && !force_load_elf_from_memory) { + // The load as a dynamic library should succeed, since this is a platform + // that natively understands ELF. + if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( + DartUtils::kAotELFMagicNumber, script_name, &error)) { + return snapshot; + } + Syslog::PrintErr("Loading dynamic library failed: %s\n", error); + return nullptr; + } +#endif const uint8_t *vm_data_buffer = nullptr, *vm_instructions_buffer = nullptr, *isolate_data_buffer = nullptr, *isolate_instructions_buffer = nullptr; @@ -220,7 +346,8 @@ static AppSnapshot* TryReadAppSnapshotElf( AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElfFromMachO( const char* container_path) { // Ensure file is actually MachO-formatted. - if (!IsMachOFormattedBinary(container_path)) { + DartUtils::MagicNumber magic_number; + if (!IsMachOFormattedBinary(container_path, &magic_number)) { Syslog::PrintErr("Expected a Mach-O binary.\n"); return nullptr; } @@ -235,17 +362,19 @@ AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElfFromMachO( // as the 32-bit header, just with an extra field for alignment, so we can // safely load a 32-bit header to get all the information we need. mach_o::mach_header header; - file->ReadFully(&header, sizeof(header)); - - if (header.magic == mach_o::MH_CIGAM || header.magic == mach_o::MH_CIGAM_64) { - Syslog::PrintErr( - "Expected a host endian header but found a byte-swapped header.\n"); + if (!file->ReadFully(&header, sizeof(header))) { + Syslog::PrintErr("Could not read a complete Mach-O header.\n"); return nullptr; } - if (header.magic == mach_o::MH_MAGIC_64) { - // Set the file position as if we had read a 64-bit header. - file->SetPosition(sizeof(mach_o::mach_header_64)); + auto const bitsize = DartUtils::MagicNumberBitSize(magic_number); + if (bitsize == 64) { + // The load commands start immediately after the full header. + if (!file->SetPosition(sizeof(mach_o::mach_header_64))) { + Syslog::PrintErr("Could not read a complete Mach-O 64-bit header.\n"); + } + } else { + ASSERT_EQUAL(bitsize, 32); } // Now we search through the load commands to find our snapshot note, which @@ -400,108 +529,29 @@ AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElf( return TryReadAppSnapshotElf(container_path, appended_offset); } - -class DylibAppSnapshot : public AppSnapshot { - public: - DylibAppSnapshot(void* library, - const uint8_t* vm_snapshot_data, - const uint8_t* vm_snapshot_instructions, - const uint8_t* isolate_snapshot_data, - const uint8_t* isolate_snapshot_instructions) - : AppSnapshot(DartUtils::kAotELFMagicNumber), - library_(library), - vm_snapshot_data_(vm_snapshot_data), - vm_snapshot_instructions_(vm_snapshot_instructions), - isolate_snapshot_data_(isolate_snapshot_data), - isolate_snapshot_instructions_(isolate_snapshot_instructions) {} - - ~DylibAppSnapshot() { Utils::UnloadDynamicLibrary(library_); } - - void SetBuffers(const uint8_t** vm_data_buffer, - const uint8_t** vm_instructions_buffer, - const uint8_t** isolate_data_buffer, - const uint8_t** isolate_instructions_buffer) { - *vm_data_buffer = vm_snapshot_data_; - *vm_instructions_buffer = vm_snapshot_instructions_; - *isolate_data_buffer = isolate_snapshot_data_; - *isolate_instructions_buffer = isolate_snapshot_instructions_; - } - - private: - void* library_; - const uint8_t* vm_snapshot_data_; - const uint8_t* vm_snapshot_instructions_; - const uint8_t* isolate_snapshot_data_; - const uint8_t* isolate_snapshot_instructions_; -}; - -static AppSnapshot* TryReadAppSnapshotDynamicLibrary(const char* script_name) { - void* library = Utils::LoadDynamicLibrary(script_name); - if (library == nullptr) { - return nullptr; - } - - const uint8_t* vm_data_buffer = reinterpret_cast( - Utils::ResolveSymbolInDynamicLibrary(library, kVmSnapshotDataCSymbol)); - - const uint8_t* vm_instructions_buffer = - reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( - library, kVmSnapshotInstructionsCSymbol)); - - const uint8_t* isolate_data_buffer = - reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( - library, kIsolateSnapshotDataCSymbol)); - if (isolate_data_buffer == nullptr) { - FATAL("Failed to resolve symbol '%s'\n", kIsolateSnapshotDataCSymbol); - } - - const uint8_t* isolate_instructions_buffer = - reinterpret_cast(Utils::ResolveSymbolInDynamicLibrary( - library, kIsolateSnapshotInstructionsCSymbol)); - if (isolate_instructions_buffer == nullptr) { - FATAL("Failed to resolve symbol '%s'\n", - kIsolateSnapshotInstructionsCSymbol); - } - - return new DylibAppSnapshot(library, vm_data_buffer, vm_instructions_buffer, - isolate_data_buffer, isolate_instructions_buffer); -} - #endif // defined(DART_PRECOMPILED_RUNTIME) -#if defined(DART_TARGET_OS_MACOS) -bool Snapshot::IsMachOFormattedBinary(const char* filename) { +bool Snapshot::IsMachOFormattedBinary(const char* filename, + DartUtils::MagicNumber* out) { File* file = File::Open(nullptr, filename, File::kRead); if (file == nullptr) { return false; } RefCntReleaseScope rs(file); - const uint64_t size = file->Length(); - // Parse the first 4 bytes and check the magic numbers. - uint32_t magic; - if (size < sizeof(magic)) { + uint8_t header[DartUtils::kMaxMagicNumberSize]; + if (!file->ReadFully(&header, DartUtils::kMaxMagicNumberSize)) { // The file isn't long enough to contain the magic bytes. return false; } - file->SetPosition(0); - file->ReadFully(&magic, sizeof(magic)); - - // Depending on the magic numbers, check that the size of the file is - // large enough for either a 32-bit or 64-bit header. - switch (magic) { - case mach_o::MH_MAGIC: - case mach_o::MH_CIGAM: - return size >= sizeof(mach_o::mach_header); - case mach_o::MH_MAGIC_64: - case mach_o::MH_CIGAM_64: - return size >= sizeof(mach_o::mach_header_64); - default: - // Not a Mach-O formatted file. - return false; + DartUtils::MagicNumber magic_number = + DartUtils::SniffForMagicNumber(header, sizeof(header)); + if (out != nullptr) { + *out = magic_number; } + return magic_number == DartUtils::kAotMachO32MagicNumber || + magic_number == DartUtils::kAotMachO64MagicNumber; } -#endif // defined(DART_TARGET_OS_MACOS) #if defined(DART_TARGET_OS_WINDOWS) bool Snapshot::IsPEFormattedBinary(const char* filename) { @@ -597,23 +647,20 @@ AppSnapshot* Snapshot::TryReadAppSnapshot(const char* script_uri, // For testing AOT with the standalone embedder, we also support loading // from a dynamic library to simulate what happens on iOS. - -#if defined(DART_TARGET_OS_LINUX) || defined(DART_TARGET_OS_MACOS) - // On Linux and OSX, resolve the script path before passing into dlopen() - // since dlopen will not search the filesystem for paths like 'libtest.so'. - CStringUniquePtr absolute_path(realpath(script_name, nullptr)); - script_name = absolute_path.get(); -#endif - - AppSnapshot* snapshot = nullptr; - if (!force_load_elf_from_memory) { - snapshot = TryReadAppSnapshotDynamicLibrary(script_name); - if (snapshot != nullptr) { + const intptr_t file_offset = 0; + if (magic_number == DartUtils::kAotELFMagicNumber) { + return TryReadAppSnapshotElf(script_name, file_offset, + force_load_elf_from_memory); + } else { + // This is not a format for which we have a non-native loader, so + // attempt to load it as a native dynamic library. + const char* error = nullptr; + if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( + magic_number, script_name, &error)) { return snapshot; } + Syslog::PrintErr("Loading dynamic library failed: %s\n", error); } - return TryReadAppSnapshotElf(script_name, /*file_offset=*/0, - force_load_elf_from_memory); #else if (magic_number == DartUtils::kAppJITMagicNumber) { // Return the JIT snapshot. diff --git a/runtime/bin/snapshot_utils.h b/runtime/bin/snapshot_utils.h index f77035b587b..2b25de94242 100644 --- a/runtime/bin/snapshot_utils.h +++ b/runtime/bin/snapshot_utils.h @@ -46,9 +46,8 @@ class Snapshot { static void GenerateAppJIT(const char* snapshot_filename); static void GenerateAppAOTAsAssembly(const char* snapshot_filename); -#if defined(DART_TARGET_OS_MACOS) - static bool IsMachOFormattedBinary(const char* container_path); -#endif + static bool IsMachOFormattedBinary(const char* container_path, + DartUtils::MagicNumber* out = nullptr); #if defined(DART_TARGET_OS_WINDOWS) static bool IsPEFormattedBinary(const char* container_path); #endif diff --git a/runtime/bin/snapshot_utils_test.cc b/runtime/bin/snapshot_utils_test.cc index 93317badfcd..153d68c64e0 100644 --- a/runtime/bin/snapshot_utils_test.cc +++ b/runtime/bin/snapshot_utils_test.cc @@ -8,12 +8,11 @@ #include "bin/test_utils.h" #include "platform/assert.h" #include "platform/globals.h" +#include "platform/mach_o.h" #include "vm/unit_test.h" namespace dart { -#if defined(DART_TARGET_OS_MACOS) - static const unsigned char kMachO32BitLittleEndianHeader[] = { 0xce, 0xfa, 0xed, 0xfe, 0x07, 0x00, 0x00, 0x01, 0x03, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -56,12 +55,19 @@ static const struct { TEST_CASE(CanDetectMachOFiles) { for (uintptr_t i = 0; i < ARRAY_SIZE(kTestcases); i++) { const auto& testcase = kTestcases[i]; + auto const magic = + reinterpret_cast(testcase.contents)->magic; + const bool host_endian = + magic == mach_o::MH_MAGIC || magic == mach_o::MH_MAGIC_64; + auto* const file = bin::DartUtils::OpenFile(testcase.filename, /*write=*/true); bin::DartUtils::WriteFile(testcase.contents, testcase.contents_size, file); bin::DartUtils::CloseFile(file); - EXPECT(bin::Snapshot::IsMachOFormattedBinary(testcase.filename)); + // Only host-endian MachO files are recognized. + EXPECT_EQ(host_endian, + bin::Snapshot::IsMachOFormattedBinary(testcase.filename)); EXPECT(bin::File::Delete(nullptr, testcase.filename)); } @@ -70,6 +76,5 @@ TEST_CASE(CanDetectMachOFiles) { bin::test::GetFileName("runtime/bin/snapshot_utils_test.cc"); EXPECT(!bin::Snapshot::IsMachOFormattedBinary(kFilename)); } -#endif } // namespace dart diff --git a/runtime/configs.gni b/runtime/configs.gni index ea79bcb264a..e55a8764811 100644 --- a/runtime/configs.gni +++ b/runtime/configs.gni @@ -203,6 +203,10 @@ template("library_for_all_configs") { if (defined(invoker.extra_nonproduct_deps)) { extra_nonproduct_deps += invoker.extra_nonproduct_deps } + extra_precompiler_deps = [] + if (defined(invoker.extra_precompiler_deps)) { + extra_precompiler_deps += invoker.extra_precompiler_deps + } foreach(conf, _all_configs) { target(invoker.target_type, "${target_name}${conf.suffix}") { forward_variables_from(invoker, @@ -241,6 +245,9 @@ template("library_for_all_configs") { sources += snapshot_sources } } else { + if (conf.compiler) { + deps += extra_precompiler_deps + } if (defined(snapshot_sources)) { not_needed([ "snapshot_sources" ]) } @@ -271,6 +278,10 @@ template("library_for_all_configs_with_compiler") { if (defined(invoker.extra_nonproduct_deps)) { extra_nonproduct_deps += invoker.extra_nonproduct_deps } + extra_precompiler_deps = [] + if (defined(invoker.extra_precompiler_deps)) { + extra_precompiler_deps += invoker.extra_precompiler_deps + } foreach(conf, _all_configs) { if (conf.compiler) { target(invoker.target_type, "${target_name}${conf.suffix}") { @@ -303,6 +314,7 @@ template("library_for_all_configs_with_compiler") { sources += snapshot_sources } } else { + deps += extra_precompiler_deps if (defined(snapshot_sources)) { not_needed([ "snapshot_sources" ]) } diff --git a/runtime/include/dart_api.h b/runtime/include/dart_api.h index 7394525e0e5..675c98bfde9 100644 --- a/runtime/include/dart_api.h +++ b/runtime/include/dart_api.h @@ -3987,7 +3987,7 @@ DART_EXPORT Dart_Handle Dart_LoadingUnitLibraryUris(intptr_t loading_unit_id); * * The assembly should be compiled as a static or shared library and linked or * loaded by the embedder. Running this snapshot requires a VM compiled with - * DART_PRECOMPILED_SNAPSHOT. The kDartVmSnapshotData and + * DART_PRECOMPILED_RUNTIME. The kDartVmSnapshotData and * kDartVmSnapshotInstructions should be passed to Dart_Initialize. The * kDartIsolateSnapshotData and kDartIsolateSnapshotInstructions should be * passed to Dart_CreateIsolateGroup. @@ -4027,7 +4027,7 @@ Dart_CreateAppAOTSnapshotAsAssemblies( * - _kDartIsolateSnapshotInstructions * * The shared library should be dynamically loaded by the embedder. - * Running this snapshot requires a VM compiled with DART_PRECOMPILED_SNAPSHOT. + * Running this snapshot requires a VM compiled with DART_PRECOMPILED_RUNTIME. * The kDartVmSnapshotData and kDartVmSnapshotInstructions should be passed to * Dart_Initialize. The kDartIsolateSnapshotData and * kDartIsolateSnapshotInstructions should be passed to Dart_CreateIsolate. @@ -4054,6 +4054,52 @@ Dart_CreateAppAOTSnapshotAsElfs(Dart_CreateLoadingUnitCallback next_callback, Dart_StreamingWriteCallback write_callback, Dart_StreamingCloseCallback close_callback); +typedef enum { + Dart_AotBinaryFormat_Elf = 0, + Dart_AotBinaryFormat_Assembly = 1, + Dart_AotBinaryFormat_MachO_Dylib = 2, +} Dart_AotBinaryFormat; + +/** + * Creates a precompiled snapshot. + * - A root library must have been loaded. + * - Dart_Precompile must have been called. + * + * Outputs a snapshot in the specified binary format defining the symbols + * - _kDartVmSnapshotData + * - _kDartVmSnapshotInstructions + * - _kDartIsolateSnapshotData + * - _kDartIsolateSnapshotInstructions + * + * The shared library should be dynamically loaded by the embedder. + * Running this snapshot requires a VM compiled with DART_PRECOMPILED_RUNTIME. + * The kDartVmSnapshotData and kDartVmSnapshotInstructions should be passed to + * Dart_Initialize. The kDartIsolateSnapshotData and + * kDartIsolateSnapshotInstructions should be passed to Dart_CreateIsolate. + * + * The callback will be invoked one or more times to provide the binary output. + * + * If stripped is true, then the binary output will not include DWARF + * debugging sections. + * + * If debug_callback_data is provided, debug_callback_data will be used with + * the callback to provide separate debugging information. + * + * The identifier should be an appropriate string for identifying the resulting + * dynamic library. For example, the identifier is used in ID_DYLIB and + * CODE_SIGNATURE load commands for Mach-O dynamic libraries and for DW_AT_name + * in the Dart progam's root DWARF compilation unit. + * + * \return A valid handle if no error occurs during the operation. + */ +DART_EXPORT DART_API_WARN_UNUSED_RESULT Dart_Handle +Dart_CreateAppAOTSnapshotAsBinary(Dart_AotBinaryFormat format, + Dart_StreamingWriteCallback callback, + void* callback_data, + bool stripped, + void* debug_callback_data, + const char* identifier); + /** * Like Dart_CreateAppAOTSnapshotAsAssembly, but only includes * kDartVmSnapshotData and kDartVmSnapshotInstructions. It also does diff --git a/runtime/platform/mach_o.h b/runtime/platform/mach_o.h index b4375822d3a..789c8704497 100644 --- a/runtime/platform/mach_o.h +++ b/runtime/platform/mach_o.h @@ -15,8 +15,48 @@ namespace mach_o { typedef int cpu_type_t; typedef int cpu_subtype_t; + +// Mask for architecture variant bits. +static constexpr cpu_type_t CPU_ARCH_MASK = 0xff000000; +// CPU with a 64-bit ABI. +static constexpr cpu_type_t CPU_ARCH_ABI64 = 0x01000000; + +// Fallback for architectures without more specific constants (e.g., +// architectures like RISCV that MacOS doesn't run on natively). +static constexpr cpu_type_t CPU_TYPE_ANY = -1; +static constexpr cpu_subtype_t CPU_SUBTYPE_ANY = -1; + +// x86-family CPUs. +static constexpr cpu_type_t CPU_TYPE_X86 = 7; +static constexpr cpu_type_t CPU_TYPE_I386 = CPU_TYPE_X86; +static constexpr cpu_type_t CPU_TYPE_X86_64 = CPU_TYPE_X86 | CPU_ARCH_ABI64; + +// x86-family CPU subtypes. +constexpr cpu_subtype_t CPU_SUBTYPE_INTEL(uint8_t f, cpu_subtype_t m) { + return f + (m << 4); +} +static constexpr cpu_subtype_t CPU_SUBTYPE_I386_ALL = CPU_SUBTYPE_INTEL(3, 0); +static constexpr cpu_subtype_t CPU_SUBTYPE_X86_ALL = CPU_SUBTYPE_I386_ALL; +static constexpr cpu_subtype_t CPU_SUBTYPE_X86_64_ALL = CPU_SUBTYPE_I386_ALL; + +// ARM-family CPUs. +static constexpr cpu_type_t CPU_TYPE_ARM = 12; +static constexpr cpu_type_t CPU_TYPE_ARM64 = CPU_TYPE_ARM | CPU_ARCH_ABI64; + +// ARM-family CPU subtypes. +static constexpr cpu_type_t CPU_SUBTYPE_ARM_ALL = 0; +static constexpr cpu_type_t CPU_SUBTYPE_ARM64_ALL = CPU_SUBTYPE_ARM_ALL; + typedef int vm_prot_t; +static constexpr vm_prot_t VM_PROT_NONE = 0x00; +static constexpr vm_prot_t VM_PROT_READ = 0x01; +static constexpr vm_prot_t VM_PROT_WRITE = 0x02; +static constexpr vm_prot_t VM_PROT_EXECUTE = 0x04; +static constexpr vm_prot_t VM_PROT_DEFAULT = (VM_PROT_READ | VM_PROT_WRITE); +static constexpr vm_prot_t VM_PROT_ALL = + (VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE); + struct mach_header { uint32_t magic; cpu_type_t cputype; @@ -44,18 +84,453 @@ struct mach_header_64 { static constexpr uint32_t MH_MAGIC_64 = 0xfeedfacf; static constexpr uint32_t MH_CIGAM_64 = 0xcffaedfe; +// Filetypes for the Mach-O header. + +// A relocatable object file (e.g., an executable). +static constexpr uint32_t MH_OBJECT = 0x1; +// A dynamically bound shared library. +static constexpr uint32_t MH_DYLIB = 0x6; +// An object file that only contains debugging information. +static constexpr uint32_t MH_DSYM = 0xa; + +// Flag values for the Mach-O header. + +// The object file has no undefined references. +static constexpr uint32_t MH_NOUNDEFS = 0x1; +// The object file is an appropriate input for the dynamic linker +// and cannot be statically link edited again. +static constexpr uint32_t MH_DYLDLINK = 0x4; +// The object file does not re-export any of its input dynamic +// libraries. +static constexpr uint32_t MH_NO_REEXPORTED_DYLIBS = 0x100000; + struct load_command { + // The tag that specifies the load command for the following + // bytes. One of the LC_* constants below. uint32_t cmd; + // The total size of the load command, including cmd and cmdsize. uint32_t cmdsize; }; +// The description of the LC_* constants are followed by the name of +// the specific C structure describing their contents in parentheses. + +// A portion of the file that is mapped into memory when the +// object file is loaded. (segment_command) +static constexpr uint32_t LC_SEGMENT = 0x1; +// The static symbol table. (symtab_command) +static constexpr uint32_t LC_SYMTAB = 0x2; +// The dynamic symbol table. (dysymtab_command) +static constexpr uint32_t LC_DYSYMTAB = 0xb; +// A dynamic library that must be loaded to use this object file. +// (dylib_command) +static constexpr uint32_t LC_LOAD_DYLIB = 0xc; +// The identifier for this dynamic library (for MH_DYLIB files). +// (dylib_command) +static constexpr uint32_t LC_ID_DYLIB = 0xd; +// A 64-bit segment. (segment_command_64) +static constexpr uint32_t LC_SEGMENT_64 = 0x19; +// The UUID, used as a build identifier. (uuid_command) +static constexpr uint32_t LC_UUID = 0x1b; +// The code signature which protects the preceding portion of the object file. +// Must be the last contents in the object file. (linkedit_data_command) +static constexpr uint32_t LC_CODE_SIGNATURE = 0x1d; +// An arbitrary piece of data not specified by the Mach-O format. (note_command) static constexpr uint32_t LC_NOTE = 0x31; -struct note_command { - uint32_t cmd; +// The target platform and minimum and target OS versions for this object file. +// (build_version_command) +static constexpr uint32_t LC_BUILD_VERSION = 0x32; + +struct segment_command { + uint32_t cmd; // LC_SEGMENT uint32_t cmdsize; - char data_owner[16]; - uint64_t offset; + // The name of the segment. Must be unique within a given object file. + char segname[16]; + // The starting virtual address and the size of the segment in memory. + uint32_t vmaddr; + uint32_t vmsize; + // The starting file offset and size of the segment in the object file. + // The file size and memory size of the segment may be different, for + // example, if the segment contains zerofill sections. + uint32_t fileoff; + uint32_t filesize; + // The maximum memory protection possible for this segment. + vm_prot_t maxprot; + // The initial memory protection for this segment once loaded. + vm_prot_t initprot; + // The number of sections in the variable-length payload of this load command. + uint32_t nsects; + // + uint32_t flags; + // section_command[] +}; + +// Contains the same fields as segment_command, but the starting memory +// address and size and the file offset and size are 64-bit fields. +struct segment_command_64 { + uint32_t cmd; // LC_SEGMENT_64 + uint32_t cmdsize; + char segname[16]; + uint64_t vmaddr; + uint64_t vmsize; + uint64_t fileoff; + uint64_t filesize; + vm_prot_t maxprot; + vm_prot_t initprot; + uint32_t nsects; + uint32_t flags; + // section_command_64[] +}; + +struct section { + char sectname[16]; + char segname[16]; + uint32_t addr; + uint32_t size; + uint32_t offset; + uint32_t align; + uint32_t reloff; + uint32_t nreloc; + uint32_t flags; + uint32_t reserved1; + uint32_t reserved2; +}; + +struct section_64 { + char sectname[16]; + char segname[16]; + uint64_t addr; uint64_t size; + uint32_t offset; + uint32_t align; + uint32_t reloff; + uint32_t nreloc; + uint32_t flags; + uint32_t reserved1; + uint32_t reserved2; + uint32_t reserved3; +}; + +static constexpr uint32_t SECTION_TYPE = 0x000000ff; +static constexpr uint32_t SECTION_ATTRIBUTES = 0xffffff00; + +// Creates section flags from the type and attributes. +constexpr uint32_t SectionFlags(intptr_t type, intptr_t attributes) { + // Note that the S_* attribute values below do not need shifting. + return (attributes & SECTION_ATTRIBUTES) | (type & SECTION_TYPE); +} + +// Section types. + +static constexpr uint32_t S_REGULAR = 0x0; +static constexpr uint32_t S_ZEROFILL = 0x1; +static constexpr uint32_t S_GB_ZEROFILL = 0xc; + +// Section attributes. Note that these values do not need shifting when +// combining with a type and so the type bits are always 0. + +static constexpr uint32_t S_NO_ATTRIBUTES = 0; +// The section only contains instructions. +static constexpr uint32_t S_ATTR_PURE_INSTRUCTIONS = 0x80000000; +// The section only contains information needed for debugging. +// No symbols should refer to this section and it must have type S_REGULAR. +static constexpr uint32_t S_ATTR_DEBUG = 0x02000000; +// The section contains some instructions. Should be set if +// S_ATTR_PURE_INSTRUCTIONS is also set. +static constexpr uint32_t S_ATTR_SOME_INSTRUCTIONS = 0x00000400; + +// Special segment and section names used by Mach-O files. Only the +// ones used in our Mach-O writer are listed. + +// Segment and section names for the text segment, which also contains +// constant data. +static constexpr char SEG_TEXT[] = "__TEXT"; +static constexpr char SECT_TEXT[] = "__text"; +static constexpr char SECT_CONST[] = "__const"; + +// Segment and section names for the data segment, which contains +// non-constant data (like the BSS section). +static constexpr char SEG_DATA[] = "__DATA"; +static constexpr char SECT_BSS[] = "__bss"; + +// Segment and section names for the DWARF segment. +static constexpr char SEG_DWARF[] = "__DWARF"; +static constexpr char SECT_DEBUG_LINE[] = "__debug_line"; +static constexpr char SECT_DEBUG_INFO[] = "__debug_info"; +static constexpr char SECT_DEBUG_ABBREV[] = "__debug_abbrev"; + +// Segment name for the linkedit segment. Does not contain sections but rather +// the non-header contents for other non-segment link commands like the symbol +// table and code signature. +static constexpr char SEG_LINKEDIT[] = "__LINKEDIT"; + +struct symtab_command { + uint32_t cmd; // LC_SYMTAB + uint32_t cmdsize; + uint32_t symoff; // The offset of the symbol table data in the object file. + uint32_t nsyms; // The number of symbols in the symbol table data. + uint32_t stroff; // The offset of the string table for the symbol table. + uint32_t strsize; // The size of the string table in bytes. +}; + +// The structure used for symbols in the symbol table. +struct nlist { + uint32_t n_idx; // The index of the symbol name in the string table. + uint8_t n_type; // The type of the syble (see below). + uint8_t n_sect; // For section symbols, the section that owns this symbol. + uint16_t n_desc; // Interpreted based on the type of the symbol. + // This is normally defined as a uword, but it must match the target + // architecture's bitsize, not the host. +#if defined(TARGET_ARCH_IS_32_BIT) + uint32_t n_value; +#else + uint64_t n_value; +#endif +}; + +// The "section" for symbols not belonging to a specific section. +static constexpr uint8_t NO_SECT = 0; + +// Masks for n_type. + +// If any bits in (n_type & N_STAB) are set, then the symbol is +// a symbolic debugging symbol and so n_type is a specific constant. +static constexpr uint8_t N_STAB = 0xe0; + +// Otherwise, n_type is a bitfield described by the following masks: + +// The private external symbol bit. +static constexpr uint8_t N_PEXT = 0x10; +// A mask for the actual type of the symbol. +static constexpr uint8_t N_TYPE = 0xe; +// The external symbol bit. +static constexpr uint8_t N_EXT = 0x1; + +// Values for the N_TYPE bits when no bits in N_STAB are set. + +// An undefined symbol. (n_sect == NO_SECT) +static constexpr uint8_t N_UNDEF = 0x0; +// A symbol to an absolute offset in the Mach-O file. (n_sect == NO_SECT) +static constexpr uint8_t N_ABS = 0x2; +// A symbol defined in a specific section (load command index in n_sect). +static constexpr uint8_t N_SECT = 0xe; + +// Values for the N_TYPE bits that set bits in N_STAB. + +// A global symbol. (n_sect == NO_SECT, value = 0). +static constexpr uint8_t N_GSYM = 0x20; +// A function defined in a specific section. +static constexpr uint8_t N_FUN = 0x24; +// A static (object) symbol defined in a specific section. +static constexpr uint8_t N_STSYM = 0x26; +// The start of a function symbol in a specific section. +static constexpr uint8_t N_BNSYM = 0x2e; +// The end of a function symbol in a specific section. +static constexpr uint8_t N_ENSYM = 0x4e; + +// Values for n_desc. + +// Indicates an alternate symbol definition for a symbol value that +// is already defined elsewhere. +static constexpr uint16_t N_ALT_ENTRY = 0x0200; + +struct dysymtab_command { + uint32_t cmd; // LC_DYSYMTAB + uint32_t cmdsize; + + // The initial fields pairs are offsets into the symbol table information + // in the linkedit segment. The first field is the symbol table index of + // the first corresponding symbol (not file offset) and the second field + // is the number of symbols starting at that index. + + // The local symbols in the symbol table. + uint32_t ilocalsym; + uint32_t nlocalsym; + // The defined external symbols in the symbol table. + uint32_t iextdefsym; + uint32_t nextdefsym; + // The undefined external symbols in the symbol table. + uint32_t iundefsym; + uint32_t nundefsym; + + // The remaining fields pairs are offsets into the linkedit segment. + // The first field is the file offset and the second field is the number + // of objects to read starting at that index. + // + // The Mach-O writer in the VM does not use these fields, so there's + // no need for further documentation (they are populated with 0 values). + + uint32_t tocoff; + uint32_t ntoc; + uint32_t modtaboff; + uint32_t nmodtab; + uint32_t extrefsymoff; + uint32_t nextrefsyms; + uint32_t indirectsymoff; + uint32_t nindirectsyms; + uint32_t extreloff; + uint32_t nextrel; + uint32_t locreloff; + uint32_t nlocrel; +}; + +struct note_command { + uint32_t cmd; // LC_NOTE + uint32_t cmdsize; + // An identifier used to determine the owner of this note (e.g., to + // determine how to interpret the contents of the note.) + char data_owner[16]; + // The file offset of the note contents. + uint64_t offset; + // The size of the note contents in bytes. + uint64_t size; +}; + +struct uuid_command { + uint32_t cmd; // LC_UUID + uint32_t cmdsize; + uint8_t uuid[16]; // The 128-bit UUID of this object file. +}; + +struct build_version_command { + uint32_t cmd; // LC_BUILD_VERSION + uint32_t cmdsize; + uint32_t platform; // See PLATFORM_* constants. + // minos and sdk are X.Y.Z versions encoded as a bitfield: + // From most to least significant: + // X : 16 + // Y : 8 + // Z : 8 + uint32_t minos; // Minimum OS version. + uint32_t sdk; // Target OS version. + // The number of build_tool_version structs in the variable-length + // payload of this load command. For our purposes, always 0 and + // so there is no definition of the build_tool_version struct here. + uint32_t ntools; +}; + +// Values for platform. + +static constexpr uint32_t PLATFORM_UNKNOWN = 0x0; +static constexpr uint32_t PLATFORM_ANY = 0xffffffff; + +static constexpr uint32_t PLATFORM_MACOS = 0x1; +static constexpr uint32_t PLATFORM_IOS = 0x2; + +union lc_str { + // The offset of the string in the load command contents. + uint32_t offset; + // We don't include the in-memory pointer alternative here. +}; + +struct dylib_info { + lc_str name; + // The timestamp the library was built and copied into user. + uint32_t timestamp; + // Version format is same as in build_version_command. + uint32_t current_version; + uint32_t compatibility_version; +}; + +struct dylib_command { + uint32_t cmd; // LC_LOAD_DYLIB and LC_ID_DYLIB among others + uint32_t cmdsize; + dylib_info dylib; +}; + +struct linkedit_data_command { + uint32_t cmd; // LC_CODE_SIGNATURE among others + uint32_t cmdsize; + // The file offset of the corresponding contents. (Note that this is + // _not_ the offset into the linkedit segment.) + uint32_t dataoff; + // The size of the contents in bytes. + uint32_t datasize; +}; + +// Magic numbers for code signature blobs. + +static constexpr uint32_t CSMAGIC_CODEDIRECTORY = 0xfade0c02; +static constexpr uint32_t CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0; + +// Types for code signature blobs. + +static constexpr uint32_t CSSLOT_CODEDIRECTORY = 0; + +// Code signature code directory flags. + +static constexpr uint32_t CS_ADHOC = 0x00000002; +static constexpr uint32_t CS_LINKER_SIGNED = 0x00020000; + +// Code signature hash types. + +static constexpr uint8_t CS_HASHTYPE_SHA256 = 0x2; + +// Code signature version numbers. + +// The earliest version that can appear in a code signature. +static constexpr uint32_t CS_SUPPORTSNONE = 0x20001; +static constexpr uint32_t CS_SUPPORTSSCATTER = 0x20100; +static constexpr uint32_t CS_SUPPORTSTEAMID = 0x20200; +static constexpr uint32_t CS_SUPPORTSCODELIMIT64 = 0x20300; +static constexpr uint32_t CS_SUPPORTSEXECSEG = 0x20400; + +struct cs_blob_index { + uint32_t type; // e.g., CSSLOT_CODEDIRECTORY + // the offset of the nested blob within the superblob + uint32_t offset; +}; + +struct cs_superblob { + uint32_t magic; // CSMAGIC_EMBEDDED_SIGNATURE + // The length of the superblob, which includes any nested blobs. + uint32_t length; + // The number of nested blobs in this blob. + uint32_t count; + // The blob indices for the nested blobs. + cs_blob_index index[]; + // The variable length payload also contains the contents of the nested blobs + // after the blob indices. The blob indices are not aligned, and the data for + // each nested blob is 8-byte aligned. +}; + +struct cs_code_directory { + uint32_t magic; // CSMAGIC_CODEDIRECTORY + // The length of the code directory, including the identifier and hashes. + uint32_t length; + uint32_t version; // For us, CS_SUPPORTSEXECSEG above. + uint32_t flags; // For us, CS_ADHOC | CS_LINKED_SIGNED. + uint32_t hash_offset; // The file offset of the hashes. + uint32_t ident_offset; // The file offset of the identifier. + uint32_t num_special_slots; // Unused by us, so 0. + // The number of hashes (one for each page up to the code limit, + // including one for the final incomplete page if any). + uint32_t num_code_slots; + // The end of the file covered by this code directory (for us, the file + // offset of the superblob). + uint32_t code_limit; + // The size of each hash in the special and code slots. + uint8_t hash_size; + // The type of each hash in the special and code slots. + uint8_t hash_type; + uint8_t platform; // Unused by us, so 0. + uint8_t page_size; // log2(page size) + uint32_t spare2; // always 0. + uint32_t scatter_offset; // Unused by us, so 0. + uint32_t teamid_offset; // Unused by us, so 0. + uint32_t spare3; // always 0. + uint64_t code_limit_64; // Code limit if larger than 32 bits. + uint64_t exec_seg_base; // file offset of the executable segment + uint64_t exec_seg_limit; // file size of the executable segment + uint64_t exec_seg_flags; // For our purposes, always 0. + + // Technically there can be more with later code signature versions, + // but the Mach-O writer doesn't output those in the ad-hoc linker + // signed signature. + + // The variable length payload contains the identifier followed by + // the hashes in the special and code slots. The identifier data is + // 8-byte aligned (like blobs) and the hash data is 16-byte aligned. }; #pragma pack(pop) diff --git a/runtime/platform/utils.cc b/runtime/platform/utils.cc index 074c5398871..3d1038aa603 100644 --- a/runtime/platform/utils.cc +++ b/runtime/platform/utils.cc @@ -10,6 +10,7 @@ #if defined(DART_HOST_OS_LINUX) || defined(DART_HOST_OS_MACOS) || \ defined(DART_HOST_OS_ANDROID) #include +#include #elif defined(DART_HOST_OS_FUCHSIA) #include #include @@ -398,4 +399,21 @@ void Utils::UnloadDynamicLibrary(void* library_handle, char** error) { } } +char* Utils::Basename(const char* path) { +#if defined(DART_HOST_OS_FUCHSIA) || defined(DART_HOST_OS_WINDOWS) + // Not handled for these operating systems. + return nullptr; +#else + if (path == nullptr) return nullptr; + char* const path_copy = Utils::StrDup(path); + char* result = basename(path_copy); + // The result may be in statically allocated memory, so copy. + result = Utils::StrDup(result); + // The result may point to a portion of the passed in string, so + // only free the copy after duplicating the result. + free(path_copy); + return result; +#endif +} + } // namespace dart diff --git a/runtime/platform/utils.h b/runtime/platform/utils.h index dd632129962..2eed8295506 100644 --- a/runtime/platform/utils.h +++ b/runtime/platform/utils.h @@ -663,6 +663,11 @@ class Utils { static void* LoadDynamicLibrary(const char* library_path, bool search_dll_load_dir = false, char** error = nullptr); + static void* LoadDynamicLibrary(const char* library_path, + char** error = nullptr) { + return LoadDynamicLibrary(library_path, /*search_dll_load_dir=*/false, + error); + } // Resolve the given |symbol| within the library referenced by the // given |library_handle|. @@ -682,6 +687,14 @@ class Utils { static void UnloadDynamicLibrary(void* library_handle, char** error = nullptr); + // Returns the basename of the given path. The returned string is malloced + // and must be freed by the caller once no longer needed. + // + // If path is nullptr, returns nullptr. + // + // Returns nullptr if the operating system does not support this operation. + static char* Basename(const char* path); + #if defined(DART_HOST_OS_LINUX) static bool IsWindowsSubsystemForLinux(); #endif diff --git a/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart b/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart index ad49f6f2676..408e7dc6fef 100644 --- a/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart +++ b/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart @@ -84,7 +84,7 @@ Future testAOT( if (useAsm) { final assemblyPath = path.join(tempDir, 'test.S'); - await run(genSnapshot, [ + await (disassemble ? runSilent : run)(genSnapshot, [ '--snapshot-kind=app-aot-assembly', '--assembly=$assemblyPath', ...commonSnapshotArgs, @@ -92,7 +92,7 @@ Future testAOT( await assembleSnapshot(assemblyPath, snapshotPath); } else { - await run(genSnapshot, [ + await (disassemble ? runSilent : run)(genSnapshot, [ '--snapshot-kind=app-aot-elf', '--elf=$snapshotPath', ...commonSnapshotArgs, diff --git a/runtime/tests/vm/dart/disassemble_aot_test.dart b/runtime/tests/vm/dart/disassemble_aot_test.dart index e3e3c4649e6..4aeecf74d3e 100644 --- a/runtime/tests/vm/dart/disassemble_aot_test.dart +++ b/runtime/tests/vm/dart/disassemble_aot_test.dart @@ -53,7 +53,7 @@ Future main(List args) async { // Run the AOT compiler with the disassemble flags set. final elfFile = path.join(tempDir, 'aot.snapshot'); - await run(genSnapshot, [ + await runSilent(genSnapshot, [ '--disassemble', '--disassemble_stubs', '--always_generate_trampolines_for_testing', @@ -63,7 +63,7 @@ Future main(List args) async { ]); // Run the AOT runtime with the disassemble flags set. - await run(dartPrecompiledRuntime, [ + await runSilent(dartPrecompiledRuntime, [ '--disassemble', '--disassemble_stubs', elfFile, diff --git a/runtime/tests/vm/dart/exported_symbols_test.dart b/runtime/tests/vm/dart/exported_symbols_test.dart index fd497107f44..eed39790490 100644 --- a/runtime/tests/vm/dart/exported_symbols_test.dart +++ b/runtime/tests/vm/dart/exported_symbols_test.dart @@ -71,6 +71,7 @@ main() { "Dart_CreateAppAOTSnapshotAsAssembly", "Dart_CreateAppAOTSnapshotAsElf", "Dart_CreateAppAOTSnapshotAsElfs", + "Dart_CreateAppAOTSnapshotAsBinary", "Dart_CreateAppJITSnapshotAsBlobs", "Dart_CreateIsolateGroup", "Dart_CreateIsolateGroupFromKernel", diff --git a/runtime/tests/vm/dart/unobfuscated_static_symbols_test.dart b/runtime/tests/vm/dart/unobfuscated_static_symbols_test.dart index 384ddf442fd..4ba1a547d9f 100644 --- a/runtime/tests/vm/dart/unobfuscated_static_symbols_test.dart +++ b/runtime/tests/vm/dart/unobfuscated_static_symbols_test.dart @@ -57,6 +57,7 @@ Future main(List args) async { ]); await checkElf(tempDir, scriptDill); + await checkMachO(tempDir, scriptDill); await checkAssembly(tempDir, scriptDill); }); } @@ -67,145 +68,165 @@ const commonGenSnapshotArgs = [ '--deterministic', ]; -Future checkElf(String tempDir, String scriptDill) async { +enum SnapshotType { + elf, + machoDylib, + assembly; + + String get kindString { + switch (this) { + case elf: + return 'app-aot-elf'; + case machoDylib: + return 'app-aot-macho-dylib'; + case assembly: + return 'app-aot-assembly'; + } + } + + String get fileArgumentName { + switch (this) { + case elf: + return 'elf'; + case machoDylib: + return 'macho'; + case assembly: + return 'assembly'; + } + } + + DwarfContainer? fromFile(String filename) { + switch (this) { + case elf: + return Elf.fromFile(filename); + case machoDylib: + return MachO.fromFile(filename); + case assembly: + return Elf.fromFile(filename) ?? MachO.fromFile(filename); + } + } + + @override + String toString() => name; +} + +Future createSnapshot( + String scriptDill, + SnapshotType snapshotType, + String finalPath, [ + List extraArgs = const [], +]) async { + String output = finalPath; + if (snapshotType == SnapshotType.assembly) { + output = path.withoutExtension(finalPath) + '.S'; + } + await run(genSnapshot, [ + ...commonGenSnapshotArgs, + ...extraArgs, + '--snapshot-kind=${snapshotType.kindString}', + '--${snapshotType.fileArgumentName}=$output', + scriptDill, + ]); + if (snapshotType == SnapshotType.assembly) { + await assembleSnapshot(output, finalPath); + } +} + +Future checkSnapshotType( + String tempDir, + String scriptDill, + SnapshotType snapshotType, +) async { // Run the AOT compiler without Dwarf stack trace, once without obfuscation, // once with obfuscation, and once with obfuscation and saving debugging // information. - final scriptUnobfuscatedSnapshot = path.join(tempDir, 'unobfuscated-elf.so'); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, - '--snapshot-kind=app-aot-elf', - '--elf=$scriptUnobfuscatedSnapshot', - scriptDill, - ]); + final scriptUnobfuscatedSnapshot = path.join( + tempDir, + 'unobfuscated-$snapshotType.so', + ); + await createSnapshot(scriptDill, snapshotType, scriptUnobfuscatedSnapshot); final unobfuscatedCase = TestCase( scriptUnobfuscatedSnapshot, - Elf.fromFile(scriptUnobfuscatedSnapshot)!, + snapshotType.fromFile(scriptUnobfuscatedSnapshot)!, ); final scriptObfuscatedOnlySnapshot = path.join( tempDir, - 'obfuscated-only-elf.so', + 'obfuscated-only-$snapshotType.so', ); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, + await createSnapshot(scriptDill, snapshotType, scriptObfuscatedOnlySnapshot, [ '--obfuscate', - '--snapshot-kind=app-aot-elf', - '--elf=$scriptObfuscatedOnlySnapshot', - scriptDill, ]); final obfuscatedOnlyCase = TestCase( scriptObfuscatedOnlySnapshot, - Elf.fromFile(scriptObfuscatedOnlySnapshot)!, + snapshotType.fromFile(scriptObfuscatedOnlySnapshot)!, ); - final scriptObfuscatedSnapshot = path.join(tempDir, 'obfuscated-elf.so'); - final scriptDebuggingInfo = path.join(tempDir, 'obfuscated-debug-elf.so'); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, - '--obfuscate', - '--snapshot-kind=app-aot-elf', - '--elf=$scriptObfuscatedSnapshot', - '--save-debugging-info=$scriptDebuggingInfo', - scriptDill, - ]); - final obfuscatedCase = TestCase( - scriptObfuscatedSnapshot, - Elf.fromFile(scriptObfuscatedSnapshot)!, - Elf.fromFile(scriptDebuggingInfo)!, - ); + // Don't compare to separate debugging information for assembled snapshots + // because the assembled code introduces a lot of local static symbols for + // relocations and so the two won't contain similar amounts of static symbols. + TestCase? obfuscatedCase; + TestCase? strippedCase; + if (snapshotType != SnapshotType.assembly) { + final scriptObfuscatedSnapshot = path.join( + tempDir, + 'obfuscated-$snapshotType.so', + ); + final scriptDebuggingInfo = path.join( + tempDir, + 'obfuscated-debug-$snapshotType.so', + ); + await createSnapshot(scriptDill, snapshotType, scriptObfuscatedSnapshot, [ + '--obfuscate', + '--save-debugging-info=$scriptDebuggingInfo', + ]); + obfuscatedCase = TestCase( + scriptObfuscatedSnapshot, + snapshotType.fromFile(scriptObfuscatedSnapshot)!, + snapshotType.fromFile(scriptDebuggingInfo)!, + ); - final scriptStrippedSnapshot = path.join( - tempDir, - 'obfuscated-stripped-elf.so', - ); - final scriptSeparateDebuggingInfo = path.join( - tempDir, - 'obfuscated-separate-debug-elf.so', - ); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, - '--strip', - '--obfuscate', - '--snapshot-kind=app-aot-elf', - '--elf=$scriptStrippedSnapshot', - '--save-debugging-info=$scriptSeparateDebuggingInfo', - scriptDill, - ]); - final strippedCase = TestCase( - scriptStrippedSnapshot, - /*container=*/ null, // No static symbols in stripped snapshot. - Elf.fromFile(scriptSeparateDebuggingInfo)!, - ); + final scriptStrippedSnapshot = path.join( + tempDir, + 'obfuscated-stripped-$snapshotType.so', + ); + final scriptSeparateDebuggingInfo = path.join( + tempDir, + 'obfuscated-separate-debug-$snapshotType.so', + ); + await createSnapshot(scriptDill, snapshotType, scriptStrippedSnapshot, [ + '--strip', + '--obfuscate', + '--save-debugging-info=$scriptSeparateDebuggingInfo', + ]); + strippedCase = TestCase( + scriptStrippedSnapshot, + /*container=*/ null, // No static symbols in stripped snapshot. + snapshotType.fromFile(scriptSeparateDebuggingInfo)!, + ); + } await checkCases(unobfuscatedCase, [ obfuscatedOnlyCase, - obfuscatedCase, - strippedCase, + if (obfuscatedCase != null) obfuscatedCase, + if (strippedCase != null) strippedCase, ]); } +Future checkElf(String tempDir, String scriptDill) async { + await checkSnapshotType(tempDir, scriptDill, SnapshotType.elf); +} + +Future checkMachO(String tempDir, String scriptDill) async { + await checkSnapshotType(tempDir, scriptDill, SnapshotType.machoDylib); +} + Future checkAssembly(String tempDir, String scriptDill) async { // Currently there are no appropriate buildtools on the simulator trybots as // normally they compile to ELF and don't need them for compiling assembly // snapshots. if (isSimulator || (!Platform.isLinux && !Platform.isMacOS)) return; - - // Run the AOT compiler without Dwarf stack trace, once without obfuscation, - // once with obfuscation, and once with obfuscation and saving debugging - // information. - final scriptUnobfuscatedAssembly = path.join( - tempDir, - 'unobfuscated-assembly.S', - ); - final scriptUnobfuscatedSnapshot = path.join( - tempDir, - 'unobfuscated-assembly.so', - ); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, - '--snapshot-kind=app-aot-assembly', - '--assembly=$scriptUnobfuscatedAssembly', - scriptDill, - ]); - await assembleSnapshot( - scriptUnobfuscatedAssembly, - scriptUnobfuscatedSnapshot, - ); - final unobfuscatedCase = TestCase( - scriptUnobfuscatedSnapshot, - Platform.isMacOS - ? MachO.fromFile(scriptUnobfuscatedSnapshot)! - : Elf.fromFile(scriptUnobfuscatedSnapshot)!, - ); - - final scriptObfuscatedOnlyAssembly = path.join( - tempDir, - 'obfuscated-only-assembly.S', - ); - final scriptObfuscatedOnlySnapshot = path.join( - tempDir, - 'obfuscated-only-assembly.so', - ); - await run(genSnapshot, [ - ...commonGenSnapshotArgs, - '--obfuscate', - '--snapshot-kind=app-aot-assembly', - '--assembly=$scriptObfuscatedOnlyAssembly', - scriptDill, - ]); - await assembleSnapshot( - scriptObfuscatedOnlyAssembly, - scriptObfuscatedOnlySnapshot, - ); - final obfuscatedOnlyCase = TestCase( - scriptObfuscatedOnlySnapshot, - Platform.isMacOS - ? MachO.fromFile(scriptObfuscatedOnlySnapshot)! - : Elf.fromFile(scriptObfuscatedOnlySnapshot)!, - ); - - await checkCases(unobfuscatedCase, [obfuscatedOnlyCase]); + await checkSnapshotType(tempDir, scriptDill, SnapshotType.assembly); } class TestCase { @@ -221,6 +242,12 @@ Future checkCases( List obfuscateds, ) async { checkStaticSymbolTables(unobfuscated, obfuscateds); + if (!Platform.isMacOS && unobfuscated.container is! Elf) { + assert(unobfuscated.container is MachO); + // Don't try and run Mach-O snapshots on systems where it is not the native + // format because there is no MachOLoader in the runtime. + return; + } await checkTraces(unobfuscated, obfuscateds); } diff --git a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_deferred_test.dart b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_deferred_test.dart index 0b6f60f9696..8804ba09d61 100644 --- a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_deferred_test.dart +++ b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_deferred_test.dart @@ -35,8 +35,7 @@ Future main() async { 'use_dwarf_stack_traces_flag_deferred_program.dart', ), runNonDwarf, - runElf, - runAssembly, + [runElf, runAssembly], ); } @@ -104,10 +103,10 @@ typedef DwarfMap = Map; class DeferredElfState extends ElfState { DeferredElfState( - super.snapshot, - super.debugInfo, super.output, super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, ); @override @@ -163,19 +162,19 @@ Future runElf(String tempDir, String scriptDill) async { final snapshotDwarfMap = useSnapshotForDwarfPath(pathManifest).dwarfMap; return DeferredElfState( - snapshotDwarfMap, - debugInfoDwarfMap, output, outputWithOppositeFlag, + snapshotDwarfMap, + debugInfoDwarfMap, ); } class DeferredAssemblyState extends AssemblyState { DeferredAssemblyState( - super.snapshot, - super.debugInfo, super.output, - super.outputWithOppositeFlag, [ + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ super.singleArch, super.multiArch, ]); @@ -298,10 +297,10 @@ Future runAssembly( } return DeferredAssemblyState( - snapshotDwarfMap, - debugInfoDwarfMap, output, outputWithOppositeFlag, + snapshotDwarfMap, + debugInfoDwarfMap, singleArchSnapshotDwarfMap, multiArchSnapshotDwarfMap, ); diff --git a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_helper.dart b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_helper.dart index aaccbd2a55b..9c94865cfa7 100644 --- a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_helper.dart +++ b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_helper.dart @@ -46,67 +46,128 @@ class DwarfTestOutput { DwarfTestOutput(this.trace, this.allocateObjectStart, this.allocateObjectEnd); } -class NonDwarfState { +abstract class State { final DwarfTestOutput output; final DwarfTestOutput outputWithOppositeFlag; - NonDwarfState(this.output, this.outputWithOppositeFlag); + State(this.output, this.outputWithOppositeFlag); +} + +class NonDwarfState extends State { + NonDwarfState(super.output, super.outputWithOppositeFlag); void check() => expect(outputWithOppositeFlag.trace, equals(output.trace)); } -abstract class ElfState { +abstract class DwarfState extends State { final T snapshot; final T debugInfo; - final DwarfTestOutput output; - final DwarfTestOutput outputWithOppositeFlag; - - ElfState( + DwarfState( + super.output, + super.outputWithOppositeFlag, this.snapshot, this.debugInfo, - this.output, - this.outputWithOppositeFlag, ); + String get description; + Future check(Trace trace, T t); + + Future makeTests(Trace nonDwarfTrace) async { + test( + 'Testing $description traces with separate debugging info', + () async => await check(nonDwarfTrace, debugInfo), + ); + + test( + 'Testing $description traces with original snapshot', + () async => await check(nonDwarfTrace, snapshot), + ); + } } -abstract class AssemblyState { - final T snapshot; - final T debugInfo; - final DwarfTestOutput output; - final DwarfTestOutput outputWithOppositeFlag; +abstract class ElfState extends DwarfState { + ElfState( + super.output, + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, + ); + + @override + String get description => 'ELF'; +} + +abstract class MultiArchDwarfState extends DwarfState { final T? singleArch; final T? multiArch; - AssemblyState( - this.snapshot, - this.debugInfo, - this.output, - this.outputWithOppositeFlag, [ + MultiArchDwarfState( + super.output, + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ this.singleArch, this.multiArch, ]); - Future check(Trace trace, T t); + @override + Future makeTests(Trace nonDwarfTrace) async { + await super.makeTests(nonDwarfTrace); + + test( + 'Testing $description single-architecture universal binary', + () async { + expect(singleArch, isNotNull); + await check(nonDwarfTrace, singleArch!); + }, + skip: skipUniversalBinary, + ); + + test( + 'Testing $description multi-architecture universal binary', + () async { + expect(multiArch, isNotNull); + await check(nonDwarfTrace, multiArch!); + }, + skip: skipUniversalBinary, + ); + } } -abstract class UniversalBinaryState { - final T singleArch; - final T multiArch; +abstract class AssemblyState extends MultiArchDwarfState { + AssemblyState( + super.output, + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ + super.singleArch, + super.multiArch, + ]); - UniversalBinaryState(this.singleArch, this.multiArch); + @override + String get description => 'assembly'; +} - Future checkSingleArch(Trace trace, AssemblyState assemblyState); - Future checkMultiArch(Trace trace, AssemblyState assemblyState); +abstract class MachOState extends MultiArchDwarfState { + MachOState( + super.output, + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ + super.singleArch, + super.multiArch, + ]); + + @override + String get description => 'Mach-O'; } Future runTests( String tempPrefix, String scriptPath, Future Function(String, String) runNonDwarf, - Future> Function(String, String) runElf, - Future?> Function(String, String) runAssembly, + Iterable Function(String, String)> runDwarfs, ) async { if (!isAOTRuntime) { return; // Running in JIT: AOT binaries not available. @@ -143,44 +204,17 @@ Future runTests( ]); final nonDwarfState = await runNonDwarf(tempDir, scriptDill); - final elfState = await runElf(tempDir, scriptDill); - final assemblyState = await runAssembly(tempDir, scriptDill); + final dwarfStates = [ + for (final f in runDwarfs) await f(tempDir, scriptDill), + ]; test('Testing symbolic traces', nonDwarfState.check); final nonDwarfTrace = nonDwarfState.output.trace; - test( - 'Testing ELF traces with separate debugging info', - () async => await elfState.check(nonDwarfTrace, elfState.debugInfo), - ); - - test( - 'Testing ELF traces with original snapshot', - () async => await elfState.check(nonDwarfTrace, elfState.snapshot), - ); - - test('Testing assembly traces with separate debugging info', () async { - expect(assemblyState, isNotNull); - await assemblyState!.check(nonDwarfTrace, assemblyState.debugInfo); - }, skip: skipAssembly); - - test('Testing assembly traces with debug snapshot ', () async { - expect(assemblyState, isNotNull); - await assemblyState!.check(nonDwarfTrace, assemblyState.snapshot); - }, skip: skipAssembly); - - test('Testing single-architecture universal binary', () async { - expect(assemblyState, isNotNull); - expect(assemblyState!.singleArch, isNotNull); - await assemblyState.check(nonDwarfTrace, assemblyState.singleArch!); - }, skip: skipUniversalBinary); - - test('Testing multi-architecture universal binary', () async { - expect(assemblyState, isNotNull); - expect(assemblyState!.multiArch, isNotNull); - await assemblyState.check(nonDwarfTrace, assemblyState.multiArch!); - }, skip: skipUniversalBinary); + for (final dwarfState in dwarfStates.whereType()) { + dwarfState.makeTests(nonDwarfTrace); + } }); } diff --git a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_test.dart b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_test.dart index 466df022223..dccb920efe6 100644 --- a/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_test.dart +++ b/runtime/tests/vm/dart/use_dwarf_stack_traces_flag_test.dart @@ -29,8 +29,13 @@ Future main() async { 'use_dwarf_stack_traces_flag_program.dart', ), runNonDwarf, - runElf, - runAssembly, + [ + runElf, + // Only generate Mach-O on MacOS, since there is no MachOLoader + // to run the binary on platforms where that isn't the native format. + if (Platform.isMacOS) runMachODylib, + runAssembly, + ], ); } @@ -64,10 +69,10 @@ Future runNonDwarf(String tempDir, String scriptDill) async { class DwarfElfState extends ElfState { DwarfElfState( - super.snapshot, - super.debugInfo, super.output, super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, ); @override @@ -76,8 +81,9 @@ class DwarfElfState extends ElfState { } Future runElf(String tempDir, String scriptDill) async { - final snapshotPath = path.join(tempDir, 'dwarf.so'); - final debugInfoPath = path.join(tempDir, 'debug_info.so'); + print("Generating ELF snapshots"); + final snapshotPath = path.join(tempDir, 'dwarf_elf.so'); + final debugInfoPath = path.join(tempDir, 'debug_info_elf.so'); await run(genSnapshot, [ '--dwarf-stack-traces-mode', '--save-debugging-info=$debugInfoPath', @@ -90,7 +96,7 @@ Future runElf(String tempDir, String scriptDill) async { final debugInfo = Dwarf.fromFile(debugInfoPath)!; // Run the resulting Dwarf-AOT compiled script. - + print("Generating ELF snapshot outputs"); final output = await runTestProgram(dartPrecompiledRuntime, [ '--dwarf-stack-traces-mode', snapshotPath, @@ -101,15 +107,15 @@ Future runElf(String tempDir, String scriptDill) async { ['--no-dwarf-stack-traces-mode', snapshotPath, scriptDill], ); - return DwarfElfState(snapshot, debugInfo, output, outputWithOppositeFlag); + return DwarfElfState(output, outputWithOppositeFlag, snapshot, debugInfo); } class DwarfAssemblyState extends AssemblyState { DwarfAssemblyState( - super.snapshot, - super.debugInfo, super.output, - super.outputWithOppositeFlag, [ + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ super.singleArch, super.multiArch, ]); @@ -136,6 +142,7 @@ Future runAssembly( // We get a separate .dSYM bundle on MacOS. var debugSnapshotPath = snapshotPath + (Platform.isMacOS ? '.dSYM' : ''); + print("Generating assembly snapshots"); await run(genSnapshot, [ // We test --dwarf-stack-traces-mode, not --dwarf-stack-traces, because // the latter is a handler that sets the former and also may change @@ -152,6 +159,7 @@ Future runAssembly( await assembleSnapshot(asmPath, snapshotPath, debug: true); + print("Generating assembly snapshot outputs"); // Run the resulting Dwarf-AOT compiled script. final output = await runTestProgram(dartPrecompiledRuntime, [ '--dwarf-stack-traces-mode', @@ -182,6 +190,7 @@ Future runAssembly( emptyFiles[arch] = emptyPath; } + print("Generating multi-arch assembly debugging information"); final singleArchSnapshotPath = path.join(tempDir, "ub-single"); await run(lipo, [ debugSnapshotPath, @@ -203,10 +212,98 @@ Future runAssembly( } return DwarfAssemblyState( - snapshot, - debugInfo, output, outputWithOppositeFlag, + snapshot, + debugInfo, + singleArchSnapshot, + multiArchSnapshot, + ); +} + +class DwarfMachOState extends MachOState { + DwarfMachOState( + super.output, + super.outputWithOppositeFlag, + super.snapshot, + super.debugInfo, [ + super.singleArch, + super.multiArch, + ]); + + @override + Future check(Trace trace, Dwarf dwarf) => + compareTraces(trace, output, outputWithOppositeFlag, dwarf); +} + +Future runMachODylib(String tempDir, String scriptDill) async { + print("Generating Mach-O snapshots"); + final snapshotPath = path.join(tempDir, 'dwarf_macho_dylib.so'); + final debugInfoPath = path.join(tempDir, 'debug_info_macho_dylib.so'); + await run(genSnapshot, [ + '--dwarf-stack-traces-mode', + '--save-debugging-info=$debugInfoPath', + '--snapshot-kind=app-aot-macho-dylib', + '--macho=$snapshotPath', + scriptDill, + ]); + + final snapshot = Dwarf.fromFile(snapshotPath)!; + final debugInfo = Dwarf.fromFile(debugInfoPath)!; + + // Run the resulting Dwarf-AOT compiled script. + print("Generating Mach-O snapshot outputs"); + final output = await runTestProgram(dartPrecompiledRuntime, [ + '--dwarf-stack-traces-mode', + snapshotPath, + scriptDill, + ]); + final outputWithOppositeFlag = await runTestProgram( + dartPrecompiledRuntime, + ['--no-dwarf-stack-traces-mode', snapshotPath, scriptDill], + ); + + Dwarf? singleArchSnapshot; + Dwarf? multiArchSnapshot; + if (skipUniversalBinary == false) { + // Create empty MachO files (just a header) for each of the possible + // architectures. + final emptyFiles = {}; + for (final arch in machOArchNames.values) { + // Don't create an empty file for the current architecture. + if (arch == dartNameForCurrentArchitecture) continue; + final contents = emptyMachOForArchitecture(arch)!; + final emptyPath = path.join(tempDir, "empty_${arch}.so"); + await File(emptyPath).writeAsBytes(contents, flush: true); + emptyFiles[arch] = emptyPath; + } + + print("Generating multi-arch Mach-O debugging information"); + final singleArchSnapshotPath = path.join(tempDir, "ub-single"); + await run(lipo, [ + debugInfoPath, + '-create', + '-output', + singleArchSnapshotPath, + ]); + singleArchSnapshot = Dwarf.fromFile(singleArchSnapshotPath)!; + + final multiArchSnapshotPath = path.join(tempDir, "ub-multiple"); + await run(lipo, [ + ...emptyFiles.values, + debugInfoPath, + '-create', + '-output', + multiArchSnapshotPath, + ]); + multiArchSnapshot = Dwarf.fromFile(multiArchSnapshotPath)!; + } + + return DwarfMachOState( + output, + outputWithOppositeFlag, + snapshot, + debugInfo, singleArchSnapshot, multiArchSnapshot, ); @@ -248,8 +345,9 @@ Future compareTraces( checkTranslatedTrace(nonDwarfTrace, translatedDwarfTrace1); - // Since we compiled directly to ELF, there should be a DSO base address - // in the stack trace header and 'virt' markers in the stack frames. + // Since we compiled directly to a shared object, there should be a + // DSO base address in the stack trace header and 'virt' markers in + // the stack frames. // The offsets of absolute addresses from their respective DSO base // should be the same for both traces. diff --git a/runtime/tests/vm/dart/use_flag_test_helper.dart b/runtime/tests/vm/dart/use_flag_test_helper.dart index a4bba1464d0..07195e8163c 100644 --- a/runtime/tests/vm/dart/use_flag_test_helper.dart +++ b/runtime/tests/vm/dart/use_flag_test_helper.dart @@ -173,20 +173,23 @@ Future stripSnapshot( await run(strip, ['-o', strippedPath, snapshotPath]); } -Future runHelper(String executable, List args) async { +Future runHelper( + String executable, + List args, { + bool printStdout = true, + bool printStderr = true, +}) async { print('Running $executable ${args.join(' ')}'); final result = await Process.run(executable, args); print('Subcommand terminated with exit code ${result.exitCode}.'); - if (result.stdout.isNotEmpty) { + if (printStdout && result.stdout.isNotEmpty) { print('Subcommand stdout:'); print(result.stdout); } - if (result.exitCode != 0) { - if (result.stderr.isNotEmpty) { - print('Subcommand stderr:'); - print(result.stderr); - } + if (printStderr && result.stderr.isNotEmpty) { + print('Subcommand stderr:'); + print(result.stderr); } return result; @@ -210,6 +213,19 @@ Future run(String executable, List args) async { } } +Future runSilent(String executable, List args) async { + final result = await runHelper( + executable, + args, + printStdout: false, + printStderr: false, + ); + + if (result.exitCode != 0) { + throw 'Command failed with unexpected exit code (was ${result.exitCode})'; + } +} + Future> runOutput(String executable, List args) async { final result = await runHelper(executable, args); diff --git a/runtime/vm/BUILD.gn b/runtime/vm/BUILD.gn index 19cfec7d866..a28bd8d7ef9 100644 --- a/runtime/vm/BUILD.gn +++ b/runtime/vm/BUILD.gn @@ -180,6 +180,10 @@ library_for_all_configs("libdart_vm") { "//third_party/icu:icui18n", "//third_party/icu:icuuc", ] + extra_precompiler_deps = [ + # The Mach-O writer uses BoringSSL's SHA256 function for code signatures. + "//third_party/boringssl", + ] if (is_fuchsia) { extra_deps += [ "$fuchsia_sdk/fidl/fuchsia.intl", diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc index 8bdc0e8057f..48608ccfca5 100644 --- a/runtime/vm/dart_api_impl.cc +++ b/runtime/vm/dart_api_impl.cc @@ -31,6 +31,7 @@ #include "vm/isolate_reload.h" #include "vm/kernel_isolate.h" #include "vm/lockers.h" +#include "vm/mach_o.h" #include "vm/message.h" #include "vm/message_handler.h" #include "vm/message_snapshot.h" @@ -6429,20 +6430,16 @@ static constexpr intptr_t kAssemblyInitialSize = 512 * KB; static constexpr intptr_t kInitialSize = 2 * MB; static constexpr intptr_t kInitialDebugSize = 1 * MB; -enum class AOTSnapshotType { - kAssembly, - kElf, -}; - static void CreateAppAOTSnapshot( Dart_StreamingWriteCallback callback, void* callback_data, bool strip, - AOTSnapshotType type, + Dart_AotBinaryFormat format, void* debug_callback_data, GrowableArray* units, LoadingUnitSerializationData* unit, - uint32_t program_hash) { + uint32_t program_hash, + const char* identifier) { Thread* T = Thread::Current(); NOT_IN_PRODUCT(TimelineBeginEndScope tbes2(T, Timeline::GetIsolateStream(), @@ -6465,14 +6462,21 @@ static void CreateAppAOTSnapshot( Dwarf* debug_dwarf = nullptr; SharedObjectWriter* debug_so = nullptr; if (generate_debug) { - debug_dwarf = new (Z) Dwarf(Z, deobfuscation_trie); - debug_so = new (Z) ElfWriter( - Z, &debug_stream, SharedObjectWriter::Type::DebugInfo, debug_dwarf); + debug_dwarf = new (Z) Dwarf(Z, deobfuscation_trie, identifier); + if (format == Dart_AotBinaryFormat_MachO_Dylib) { + debug_so = new (Z) + MachOWriter(Z, &debug_stream, SharedObjectWriter::Type::DebugInfo, + identifier, debug_dwarf); + } else { + debug_so = new (Z) ElfWriter( + Z, &debug_stream, SharedObjectWriter::Type::DebugInfo, debug_dwarf); + } } - StreamingWriteStream output_stream( - type == AOTSnapshotType::kAssembly ? kAssemblyInitialSize : kInitialSize, - callback, callback_data); + StreamingWriteStream output_stream(format == Dart_AotBinaryFormat_Assembly + ? kAssemblyInitialSize + : kInitialSize, + callback, callback_data); auto const use_output_writer = [&](ImageWriter* image_writer) { FullSnapshotWriter writer(Snapshot::kFullAOT, &vm_snapshot_data, @@ -6487,16 +6491,21 @@ static void CreateAppAOTSnapshot( image_writer->Finalize(); }; - Dwarf* const dwarf = (type == AOTSnapshotType::kAssembly || strip) ? nullptr - : generate_debug ? debug_dwarf - : new (Z) Dwarf(Z, deobfuscation_trie); + Dwarf* const dwarf = + (format == Dart_AotBinaryFormat_Assembly || strip) ? nullptr + : generate_debug ? debug_dwarf + : new (Z) Dwarf(Z, deobfuscation_trie, identifier); SharedObjectWriter* so = nullptr; - if (type == AOTSnapshotType::kElf) { + if (format == Dart_AotBinaryFormat_Elf) { so = new (Z) ElfWriter(Z, &output_stream, SharedObjectWriter::Type::Snapshot, dwarf); + } else if (format == Dart_AotBinaryFormat_MachO_Dylib) { + so = new (Z) + MachOWriter(Z, &output_stream, SharedObjectWriter::Type::Snapshot, + identifier, dwarf); } - if (type == AOTSnapshotType::kAssembly) { + if (format == Dart_AotBinaryFormat_Assembly) { ASSERT(so == nullptr); AssemblyImageWriter assembly_writer(T, &output_stream, deobfuscation_trie, strip, debug_so); @@ -6512,7 +6521,7 @@ static void CreateAppAOTSnapshot( static void Split(Dart_CreateLoadingUnitCallback next_callback, void* next_callback_data, bool strip, - AOTSnapshotType type, + Dart_AotBinaryFormat format, Dart_StreamingWriteCallback write_callback, Dart_StreamingCloseCallback close_callback) { Thread* T = Thread::Current(); @@ -6544,9 +6553,9 @@ static void Split(Dart_CreateLoadingUnitCallback next_callback, next_callback(next_callback_data, id, &write_callback_data, &write_debug_callback_data); } - CreateAppAOTSnapshot(write_callback, write_callback_data, strip, type, + CreateAppAOTSnapshot(write_callback, write_callback_data, strip, format, write_debug_callback_data, &data, data[id], - program_hash); + program_hash, /*identifier=*/nullptr); { TransitionVMToNative transition(T); close_callback(write_callback_data); @@ -6579,8 +6588,8 @@ Dart_CreateAppAOTSnapshotAsAssembly(Dart_StreamingWriteCallback callback, T->isolate_group()->object_store()->set_loading_units(Object::null_array()); CreateAppAOTSnapshot(callback, callback_data, strip, - AOTSnapshotType::kAssembly, debug_callback_data, nullptr, - nullptr, 0); + Dart_AotBinaryFormat_Assembly, debug_callback_data, + nullptr, nullptr, 0, /*identifier=*/nullptr); return Api::Success(); #endif @@ -6606,7 +6615,7 @@ DART_EXPORT Dart_Handle Dart_CreateAppAOTSnapshotAsAssemblies( CHECK_NULL(write_callback); CHECK_NULL(close_callback); - Split(next_callback, next_callback_data, strip, AOTSnapshotType::kAssembly, + Split(next_callback, next_callback_data, strip, Dart_AotBinaryFormat_Assembly, write_callback, close_callback); return Api::Success(); @@ -6660,8 +6669,9 @@ Dart_CreateAppAOTSnapshotAsElf(Dart_StreamingWriteCallback callback, // Mark as not split. T->isolate_group()->object_store()->set_loading_units(Object::null_array()); - CreateAppAOTSnapshot(callback, callback_data, strip, AOTSnapshotType::kElf, - debug_callback_data, nullptr, nullptr, 0); + CreateAppAOTSnapshot(callback, callback_data, strip, Dart_AotBinaryFormat_Elf, + debug_callback_data, nullptr, nullptr, 0, + /*identifier=*/nullptr); return Api::Success(); #endif @@ -6685,13 +6695,40 @@ Dart_CreateAppAOTSnapshotAsElfs(Dart_CreateLoadingUnitCallback next_callback, CHECK_NULL(write_callback); CHECK_NULL(close_callback); - Split(next_callback, next_callback_data, strip, AOTSnapshotType::kElf, + Split(next_callback, next_callback_data, strip, Dart_AotBinaryFormat_Elf, write_callback, close_callback); return Api::Success(); #endif } +DART_EXPORT Dart_Handle +Dart_CreateAppAOTSnapshotAsBinary(Dart_AotBinaryFormat format, + Dart_StreamingWriteCallback callback, + void* callback_data, + bool strip, + void* debug_callback_data, + const char* identifier) { +#if defined(TARGET_ARCH_IA32) + return Api::NewError("AOT compilation is not supported on IA32."); +#elif !defined(DART_PRECOMPILER) + return Api::NewError( + "This VM was built without support for AOT compilation."); +#else + DARTSCOPE(Thread::Current()); + API_TIMELINE_DURATION(T); + CHECK_NULL(callback); + + // Mark as not split. + T->isolate_group()->object_store()->set_loading_units(Object::null_array()); + + CreateAppAOTSnapshot(callback, callback_data, strip, format, + debug_callback_data, nullptr, nullptr, 0, identifier); + + return Api::Success(); +#endif +} + DART_EXPORT Dart_Handle Dart_LoadingUnitLibraryUris(intptr_t loading_unit_id) { #if defined(TARGET_ARCH_IA32) return Api::NewError("AOT compilation is not supported on IA32."); diff --git a/runtime/vm/dwarf.cc b/runtime/vm/dwarf.cc index 0aa736fcd18..26c0ac0a0a8 100644 --- a/runtime/vm/dwarf.cc +++ b/runtime/vm/dwarf.cc @@ -92,8 +92,20 @@ class InliningNode : public ZoneAllocated { InliningNode* children_next; }; -Dwarf::Dwarf(Zone* zone, const Trie* deobfuscation_trie) +static const char* GetRootLibraryName(Zone* zone) { + const auto& root_library = Library::Handle( + zone, IsolateGroup::Current()->object_store()->root_library()); + const auto& root_uri = String::Handle(zone, root_library.url()); + return root_uri.ToCString(); +} + +Dwarf::Dwarf(Zone* zone, + const Trie* deobfuscation_trie, + const char* compilation_unit_name) : zone_(zone), + compilation_unit_name_(compilation_unit_name != nullptr + ? compilation_unit_name + : GetRootLibraryName(zone)), deobfuscation_trie_(deobfuscation_trie), codes_(zone, 1024), code_to_label_(zone), @@ -274,10 +286,7 @@ void Dwarf::WriteDebugInfo(DwarfWriteStream* stream) { // compilation unit. Note we write attributes in the same order we declared // them in our abbreviation above in WriteAbbreviations. stream->uleb128(kCompilationUnit); - const Library& root_library = Library::Handle( - zone_, IsolateGroup::Current()->object_store()->root_library()); - const String& root_uri = String::Handle(zone_, root_library.url()); - stream->string(root_uri.ToCString()); // DW_AT_name + stream->string(compilation_unit_name_); // DW_AT_name const char* producer = zone_->PrintToString("Dart %s\n", Version::String()); stream->string(producer); // DW_AT_producer stream->string(""); // DW_AT_comp_dir diff --git a/runtime/vm/dwarf.h b/runtime/vm/dwarf.h index da658108234..04d335d061a 100644 --- a/runtime/vm/dwarf.h +++ b/runtime/vm/dwarf.h @@ -155,7 +155,12 @@ class DwarfWriteStream : public ValueObject { class Dwarf : public ZoneAllocated { public: - explicit Dwarf(Zone* zone, const Trie* deobfuscation_trie); + // The compilation unit name is used as the DW_AT_name for the + // Dart program's compilation unit. If nullptr, then the name of + // the root library is used instead. + Dwarf(Zone* zone, + const Trie* deobfuscation_trie, + const char* compilation_unit_name = nullptr); const ZoneGrowableArray& codes() const { return codes_; } @@ -245,6 +250,7 @@ class Dwarf : public ZoneAllocated { LineNumberProgramWriter* writer); Zone* const zone_; + const char* const compilation_unit_name_; const Trie* const deobfuscation_trie_; ZoneGrowableArray codes_; DwarfCodeMap code_to_label_; diff --git a/runtime/vm/elf.h b/runtime/vm/elf.h index ddba3573a53..64d3d8930a8 100644 --- a/runtime/vm/elf.h +++ b/runtime/vm/elf.h @@ -20,7 +20,7 @@ namespace dart { // The max page size on all supported architectures. Used to determine // the alignment of load segments, so that they are guaranteed page-aligned, -// and no ELF section or segment should have a larger alignment. +// and no shared object section or segment should have a larger alignment. #if defined(DART_TARGET_OS_LINUX) && defined(TARGET_ARCH_ARM64) // Some Linux distributions on ARM64 select 64 KB page size. // Follow LLVM (https://reviews.llvm.org/D25079) and set maximum page size diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc index 2171ca029bb..66d9c3b2536 100644 --- a/runtime/vm/image_snapshot.cc +++ b/runtime/vm/image_snapshot.cc @@ -7,6 +7,7 @@ #include "include/dart_api.h" #include "platform/assert.h" #include "platform/elf.h" +#include "platform/mach_o.h" #include "vm/bss_relocs.h" #include "vm/class_id.h" #include "vm/compiler/runtime_api.h" @@ -106,6 +107,8 @@ const uint8_t* Image::build_id() const { if (compiled_to_elf()) { auto* const note = reinterpret_cast(start); return note->data + note->name_size; + } else if (compiled_to_macho()) { + return reinterpret_cast(start)->uuid; } #endif return nullptr; @@ -119,6 +122,8 @@ intptr_t Image::build_id_length() const { if (compiled_to_elf()) { auto const note = reinterpret_cast(start); return note->description_size; + } else if (compiled_to_macho()) { + return sizeof(mach_o::uuid_command::uuid); } #endif return 0; @@ -148,10 +153,10 @@ const uint8_t* Image::shared_object_start() const { bool Image::compiled_to_elf() const { #if defined(DART_PRECOMPILED_RUNTIME) if (!compiled_to_shared_object()) return false; - constexpr intptr_t len = ARRAY_SIZE(elf::ELFMAG); - const uint8_t* so_start = shared_object_start(); - for (intptr_t i = 0; i < len; ++i) { - if (so_start[i] != elf::ELFMAG[i]) return false; + auto* const ident = + reinterpret_cast(shared_object_start())->ident; + for (size_t i = 0; i < ARRAY_SIZE(elf::ELFMAG); ++i) { + if (ident[i] != elf::ELFMAG[i]) return false; } return true; #else @@ -159,6 +164,19 @@ bool Image::compiled_to_elf() const { #endif } +bool Image::compiled_to_macho() const { +#if defined(DART_PRECOMPILED_RUNTIME) + if (!compiled_to_shared_object()) return false; + auto const magic = + reinterpret_cast(shared_object_start()) + ->magic; + return magic == mach_o::MH_MAGIC || magic == mach_o::MH_CIGAM || + magic == mach_o::MH_MAGIC_64 || magic == mach_o::MH_CIGAM_64; +#else + return false; +#endif +} + uword ObjectOffsetTrait::Hash(Key key) { ObjectPtr obj = key; ASSERT(!obj->IsSmi()); diff --git a/runtime/vm/image_snapshot.h b/runtime/vm/image_snapshot.h index 7b7fa0d60bc..baeb115a8b4 100644 --- a/runtime/vm/image_snapshot.h +++ b/runtime/vm/image_snapshot.h @@ -91,6 +91,10 @@ class Image : ValueObject { // Only valid for instructions images from precompiled snapshots. bool compiled_to_elf() const; + // Returns whether this instructions section was directly compiled to MachO. + // Only valid for instructions images from precompiled snapshots. + bool compiled_to_macho() const; + private: // For snapshots directly compiled to a shared object, returns a pointer to // the beginning of the build id container. Otherwise returns nullptr; diff --git a/runtime/vm/mach_o.cc b/runtime/vm/mach_o.cc new file mode 100644 index 00000000000..003236f0ca8 --- /dev/null +++ b/runtime/vm/mach_o.cc @@ -0,0 +1,2527 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "vm/mach_o.h" + +#if defined(DART_PRECOMPILER) + +#include + +#include "openssl/sha.h" +#include "platform/mach_o.h" +#include "vm/dwarf.h" +#include "vm/dwarf_so_writer.h" +#include "vm/hash_map.h" +#include "vm/os.h" +#include "vm/zone_text_buffer.h" + +namespace dart { + +static constexpr intptr_t kLinearInitValue = -1; + +#define DEFINE_LINEAR_FIELD_METHODS(name) \ + intptr_t name() const { \ + ASSERT(name##_ != kLinearInitValue); \ + return name##_; \ + } \ + bool name##_is_set() const { \ + return name##_ != kLinearInitValue; \ + } \ + void set_##name(intptr_t value) { \ + ASSERT(value != kLinearInitValue); \ + ASSERT_EQUAL(name##_, kLinearInitValue); \ + name##_ = value; \ + } + +#define DEFINE_LINEAR_FIELD(name) intptr_t name##_ = kLinearInitValue; + +// Only subclasses of MachOContents that need to be distinguished dynamically +// via Is/As checks are listed here. +#define FOR_EACH_CHECKABLE_MACHO_CONTENTS_TYPE(V) \ + V(MachOCommand) \ + V(MachOSegment) \ + V(MachOSection) \ + V(MachOHeader) + +#define DEFINE_TYPE_CHECK_FOR(Type) \ + bool Is##Type() const override { \ + return true; \ + } + +// All concrete subclasses of MachOContents should go here: +#define FOR_EACH_CONCRETE_MACHO_CONTENTS_TYPE(V) \ + V(MachOHeader) \ + V(MachOSegment) \ + V(MachOSection) \ + V(MachOSymbolTable) \ + V(MachODynamicSymbolTable) \ + V(MachOUuid) \ + V(MachOBuildVersion) \ + V(MachOIdDylib) \ + V(MachOLoadDylib) \ + V(MachOCodeSignature) + +#define DECLARE_CONTENTS_TYPE_CLASS(Type) class Type; +FOR_EACH_CHECKABLE_MACHO_CONTENTS_TYPE(DECLARE_CONTENTS_TYPE_CLASS) +FOR_EACH_CONCRETE_MACHO_CONTENTS_TYPE(DECLARE_CONTENTS_TYPE_CLASS) +#undef DECLARE_CONTENTS_TYPE_CLASS + +// The interface for a SharedObjectWriter::WriteStream with MachO-specific +// utility methods. +// +// If HasHashes() is true, the stream calculates and store hashes of +// written content up to the point that FinalizeHashedContent() is called. +class MachOWriteStream : public SharedObjectWriter::WriteStream { + template + using only_if_unsigned = typename std::enable_if_t, S>; + + public: + explicit MachOWriteStream(const MachOWriter& macho) + : SharedObjectWriter::WriteStream(), macho_(macho) {} + + const MachOSegment& TextSegment() const; + + // Write methods that write values of a certain size out to disk. + // The disk are written in host endian format, which matches the + // header's magic value (since it is also written with this). + void Write16(uword value) { WriteBytes(&value, sizeof(uint16_t)); } + void Write32(uint32_t value) { WriteBytes(&value, sizeof(uint32_t)); } + void Write64(uint64_t value) { WriteBytes(&value, sizeof(uint64_t)); } + void WriteWord(compiler::target::uword value) { + WriteBytes(&value, sizeof(compiler::target::uword)); + } + + // Write methods that force big endian output. Used in the code signature. + void WriteBE16(uint16_t value) { Write16(Utils::HostToBigEndian16(value)); } + void WriteBE32(uint32_t value) { Write32(Utils::HostToBigEndian32(value)); } + void WriteBE64(uint64_t value) { Write64(Utils::HostToBigEndian64(value)); } + + // Many load commands have adjacent uint32_t fields that correspond to an + // offset into the file and a number of bytes or objects to read starting + // from that offset, so abstract that out to make such writes stand out. + void WriteOffsetCount(uintptr_t offset, uintptr_t count) { + ASSERT(Utils::IsUint(32, offset)); + Write32(offset); + ASSERT(Utils::IsUint(32, count)); + Write32(count); + } + + void WriteNullTerminatedCString(const char* str) { + WriteBytes(str, strlen(str) + 1); + } + + // Writes the first n bytes of the given string. If the string is shorter + // than n bytes, then the remainder of the space is padded with '\0'. + void WriteFixedLengthCString(const char* str, intptr_t n) { + const intptr_t len = strlen(str); + WriteBytes(str, n - len <= 0 ? n : len); + for (intptr_t i = n - len; i > 0; --i) { + WriteByte('\0'); + } + } + + bool HasValueForLabel(intptr_t label, intptr_t* value) const override; + + // The maximum size of a chunk of hashed content. + static constexpr intptr_t kChunkSize = 1 << 12; + static_assert(Utils::IsPowerOfTwo(kChunkSize)); + + // Used for cs_code_directory::hash_type. + static constexpr uint8_t kHashType = mach_o::CS_HASHTYPE_SHA256; + // used for cs_code_directory::hash_size. + static constexpr uint8_t kHashSize = SHA256_DIGEST_LENGTH; + + // Whether or not this MachOWriter supports hashing content. + virtual bool HasHashes() const = 0; + // The number of hashes calculated from the hashed content. + // Assumes the hashed content has already been finalized. + virtual intptr_t num_hashes() const = 0; + // Writes the calculated hashes to the stream. + // Assumes the hashed content has already been finalized. + virtual void WriteHashes() = 0; + // Call once all content that should be hashed has been written to the stream. + virtual void FinalizeHashedContent() = 0; + + protected: + const MachOWriter& macho_; + + private: + DISALLOW_COPY_AND_ASSIGN(MachOWriteStream); +}; + +// A MachOWriteStream that strictly delegates to the provided BaseWriteStream +// without any internal caching. +class NonHashingMachOWriteStream + : public SharedObjectWriter::DelegatingWriteStream, + public MachOWriteStream { + public: + explicit NonHashingMachOWriteStream(BaseWriteStream* stream, + const MachOWriter& macho) + : SharedObjectWriter::DelegatingWriteStream(stream, macho), + MachOWriteStream(macho) {} + + intptr_t Position() const override { + return SharedObjectWriter::DelegatingWriteStream::Position(); + } + void WriteByte(const uint8_t value) override { + SharedObjectWriter::DelegatingWriteStream::WriteByte(value); + } + void WriteBytes(const void* bytes, intptr_t len) override { + SharedObjectWriter::DelegatingWriteStream::WriteBytes(bytes, len); + } + intptr_t Align(intptr_t alignment, intptr_t offset = 0) override { + return SharedObjectWriter::DelegatingWriteStream::Align(alignment, offset); + } + bool HasValueForLabel(intptr_t label, intptr_t* value) const override { + return MachOWriteStream::HasValueForLabel(label, value); + } + + bool HasHashes() const override { return false; } + intptr_t num_hashes() const override { UNREACHABLE(); } + void WriteHashes() override { UNREACHABLE(); } + void FinalizeHashedContent() override { UNREACHABLE(); } + + private: + DISALLOW_COPY_AND_ASSIGN(NonHashingMachOWriteStream); +}; + +// A wrapper around an BaseWriteStream that calculates hashes for kChunkSize +// chunks being flushed. +// +// FinalizeHashedContent() is called after the last write of content that +// should be hashed; further writes skip the hashing process. +// (E.g., FinalizeHashes() is called before writing the code signature in +// a Mach-O file.) +class HashingMachOWriteStream : public BaseWriteStream, + public MachOWriteStream { + public: + HashingMachOWriteStream(Zone* zone, + BaseWriteStream* stream, + const MachOWriter& macho) + : BaseWriteStream(stream->initial_size()), + MachOWriteStream(macho), + zone_(zone), + wrapped_stream_(stream), + hashes_(zone, SHA256_DIGEST_LENGTH) { + // So that we can use the underlying stream's Align, as all alignments + // will be less than or equal to this alignment. + ASSERT(Utils::IsAligned(stream->Position(), macho_.page_size())); + } + + ~HashingMachOWriteStream() { + // Hashed content should always been finalized earlier so the + // hashes can be retrieved before destruction. + ASSERT(!hashing_); + Flush(/*chunks_only=*/false); // Flush all bytes. + ASSERT_EQUAL(BaseWriteStream::Position(), 0); + } + + intptr_t Position() const override { + return flushed_size_ + BaseWriteStream::Position(); + } + void WriteByte(const uint8_t value) override { + BaseWriteStream::WriteByte(value); + } + void WriteBytes(const void* bytes, intptr_t len) override { + BaseWriteStream::WriteBytes(bytes, len); + } + intptr_t Align(intptr_t alignment, intptr_t offset = 0) override { + ASSERT(Utils::IsPowerOfTwo(alignment)); + ASSERT(alignment <= macho_.page_size()); + return BaseWriteStream::Align(alignment, offset); + } + + bool HasHashes() const override { return true; } + intptr_t num_hashes() const override { + ASSERT(!hashing_); // Don't allow uses until hashes are finalized. + return num_hashes_; + } + void WriteHashes() override { + ASSERT(!hashing_); // Don't allow uses until hashes are finalized. + WriteBytes(hashes_.buffer(), num_hashes_ * kHashSize); + } + + // First hashes and then flushes all data in the internal buffer. Afterwards, + // the internal buffer is empty and future Flush() calls no longer perform + // hashing before flushing to the wrapped stream. + // + // Changes current_ and flushed_size_ accordingly. + void FinalizeHashedContent() override { + Flush(/*chunks_only=*/false); + hashing_ = false; // End of the hashed content. + // The only content in the hashes buffer should be the hashes themselves. + ASSERT_EQUAL(num_hashes_ * kHashSize, hashes_.Position()); + } + + private: + // Hashes [count] bytes of [buffer_] in [kChunkSize]-sized chunks and + // returns the number of bytes hashed. + intptr_t Hash(intptr_t count) { + ASSERT(count >= 0); + if (count > 0) { + ASSERT(count <= BaseWriteStream::Position()); + for (intptr_t offset = 0; offset < count; offset += kChunkSize) { + const intptr_t len = Utils::Minimum(count - offset, kChunkSize); + SHA256(buffer_ + offset, len, digest_); + hashes_.WriteBytes(digest_, kHashSize); + num_hashes_ += 1; + } + } + return count; + } + + // If hashing, then hash all complete chunks and, if [chunks_only] is false, + // a final incomplete one, then flush all hashed bytes to the wrapped stream. + // The internal buffer is then reset to contain only unhashed bytes (if any). + // + // If not hashing, then all cached content is flushed immediately. + // + // Changes current_ and flushed_size_ accordingly. + void Flush(bool chunks_only) { + intptr_t size_to_flush = BaseWriteStream::Position(); + if (hashing_) { + intptr_t size_to_hash = size_to_flush; + if (chunks_only) { + size_to_hash -= size_to_hash % kChunkSize; + } + size_to_flush = Hash(size_to_hash); + } + FlushBytes(size_to_flush); + } + + // Flushes the initial [count] bytes of [buffer_] to the wrapped stream. + // + // Changes current_ and flushed_size_ accordingly. + void FlushBytes(intptr_t count) { + ASSERT(count >= 0); + if (count == 0) return; + const intptr_t remaining = BaseWriteStream::Position() - count; + ASSERT(remaining >= 0); + wrapped_stream_->WriteBytes(buffer_, count); + flushed_size_ += count; + if (remaining > 0) { + memmove(buffer_, buffer_ + count, remaining); + } + current_ = buffer_ + remaining; + } + + void Realloc(intptr_t new_size) override { + Flush(/*chunks_only=*/true); + // Check whether there's enough space after flushing. + if (new_size <= Remaining()) return; + // There isn't, so realloc the buffer. + const intptr_t old_offset = BaseWriteStream::Position(); + buffer_ = zone_->Realloc(buffer_, capacity_, new_size); + capacity_ = buffer_ != nullptr ? new_size : 0; + current_ = buffer_ != nullptr ? buffer_ + old_offset : nullptr; + } + + void SetPosition(intptr_t value) override { + // Make sure we're not trying to set the position to already-flushed data. + ASSERT(value >= flushed_size_); + BaseWriteStream::SetPosition(value - flushed_size_); + } + + Zone* const zone_; + BaseWriteStream* const wrapped_stream_; + ZoneWriteStream hashes_; + bool hashing_ = true; + intptr_t flushed_size_ = 0; + intptr_t num_hashes_ = 0; + uint8_t digest_[kHashSize]; // Used for SHA256(). + + DISALLOW_COPY_AND_ASSIGN(HashingMachOWriteStream); +}; + +// A superclass for all objects that represent some content in the MachO output. +class MachOContents : public ZoneAllocated { + public: + explicit MachOContents(bool needs_offset = true, bool in_segment = true) + // Set the file offset and/or (relative) memory address to 0 if unneeded. + : file_offset_(needs_offset ? kLinearInitValue : 0), + memory_address_(in_segment ? kLinearInitValue : 0) {} + virtual ~MachOContents() {} + + struct Visitor : public ValueObject { + public: + Visitor() {} + virtual ~Visitor() {} + + virtual void Default(MachOContents* c) {} + +#define DEFINE_VISIT_METHOD(Type) \ + virtual void Visit##Type(Type* m) { \ + Default(reinterpret_cast(m)); \ + } + FOR_EACH_CONCRETE_MACHO_CONTENTS_TYPE(DEFINE_VISIT_METHOD) +#undef DEFINE_VISIT_METHOD + + private: + DISALLOW_COPY_AND_ASSIGN(Visitor); + }; + + virtual void Accept(Visitor* visitor) = 0; + virtual void VisitChildren(Visitor* visitor) {} + + // Content methods. + + // Whether WriteSelf() for this object or any nested object writes content + // to the file. For most objects, the file offset is set to 0 at construction + // if no content is written by it or nested objects. + // + // Overwrite this if the computed file offset can be 0 (e.g., the header). + virtual bool HasContents() const { return file_offset_ != 0; } + + // Returns the size written to disk by WriteSelf(). + // + // Only needs to be overwritten for unallocated objects or objects where + // the number of bytes written by WriteSelf() does not match SelfMemorySize(). + virtual intptr_t SelfFileSize() const { + if (!HasContents()) return 0; + return SelfMemorySize(); + } + + // Writes the file contents for this object to the stream. + // + // Note that this does not write the load command for a command, as that + // is handled separately by MachOCommand::WriteLoadCommand(). + // + // Only needs to be overwritten for objects with non-zero SelfFileSize(). + virtual void WriteSelf(MachOWriteStream* stream) const { + ASSERT_EQUAL(SelfFileSize(), 0); + return; + } + + // Returns whether the contents of an object is a segment or contained within + // a segment and thus has an assigned relative memory address. If it has none, + // then the memory offset is set to 0 at construction. + // + // Note: While technically load commands are in a segment due to being in the + // header, this returns false for commands that only generate load commands. + // + // Should be overwritten if a segment or segment-contained object has a + // computed relative memory address of 0 (e.g., the header). + virtual bool IsAllocated() const { return memory_address_ != 0; } + + // Returns the size allocated in the output's memory space for this object + // without including any allocation for nested objects. + // + // Should be overridden for allocated objects. + virtual intptr_t SelfMemorySize() const { + if (!IsAllocated()) return 0; + UNREACHABLE(); + } + + // Utility/miscellaneous methods. + +#define DEFINE_BASE_TYPE_CHECKS(Type) \ + Type* As##Type() { \ + return Is##Type() ? reinterpret_cast(this) : nullptr; \ + } \ + const Type* As##Type() const { \ + return const_cast(const_cast(this)->As##Type()); \ + } \ + virtual bool Is##Type() const { return false; } + + FOR_EACH_CHECKABLE_MACHO_CONTENTS_TYPE(DEFINE_BASE_TYPE_CHECKS) +#undef DEFINE_BASE_TYPE_CHECKS + + // Returns the alignment needed for the non-header contents. + virtual intptr_t Alignment() const { + // No need to override for non-allocated commands with no contents. + ASSERT(!IsAllocated() && !HasContents()); + UNREACHABLE(); + } + + // The size of the contents written to disk by WriteSelf() for this + // object and any nested subobjects. + // + // Should be overwritten for objects that can have different + // file and memory sizes. + virtual intptr_t FileSize() const { + if (!HasContents()) return 0; + ASSERT(IsAllocated()); + return MemorySize(); + } + + // The size of this object and any subobjects combined in the output's memory + // space. Note that objects may have a different MemorySize() than FileSize() + // (e.g., a segment that contains zerofill sections). + // + // Should be overridden when the object contains nested objects. + virtual intptr_t MemorySize() const { return SelfMemorySize(); } + +#define FOR_EACH_CONTENTS_LINEAR_FIELD(M) \ + M(file_offset) \ + M(memory_address) + + FOR_EACH_CONTENTS_LINEAR_FIELD(DEFINE_LINEAR_FIELD_METHODS); + + private: + FOR_EACH_CONTENTS_LINEAR_FIELD(DEFINE_LINEAR_FIELD); + +#undef FOR_EACH_CONTENTS_LINEAR_FIELD + + DISALLOW_COPY_AND_ASSIGN(MachOContents); +}; + +// Each MachO command corresponds to two parts in the file contents: +// the load command in the header that describes how to load the command +// contents and the command contents somewhere after the header. +// +// The load command is written via WriteLoadCommand() while WriteSelf() +// handles writing the command contents. +// +// Each concrete subclass of MachOCommand should define +// static constexpr uint32_t kCommandCode = ... +// with the appropriate mach_o::LC_* constant. +class MachOCommand : public MachOContents { + public: + explicit MachOCommand(intptr_t cmd, + bool needs_offset = true, + bool in_segment = true) + : MachOContents(needs_offset, in_segment), cmd_(cmd) { + ASSERT(Utils::IsUint(32, cmd)); + } + + DEFINE_TYPE_CHECK_FOR(MachOCommand) + + // Load command fields and methods. + + // The value identifying the type of section the load command represents. + // Should be one of the LC_* constants in platform/mach_o.h. + uint32_t cmd() const { return cmd_; } + + // The alignment expected for load commands. + static constexpr intptr_t kLoadCommandAlignment = compiler::target::kWordSize; + + // The size of the load command representing this command in the header. + // + // Note that all load commands must have a size that is a multiple of + // kLoadCommandAlignment, so padding may be required. + virtual uint32_t cmdsize() const = 0; + + // Each load command has a common prefix, which is written by the + // class's WriteLoadCommand. Call the base class's implementation + // prior to writing the rest of the load command for the subclass. + virtual void WriteLoadCommand(MachOWriteStream* stream) const { + stream->Write32(cmd()); + stream->Write32(cmdsize()); + } + + // Only the offset within the header is defined since the file offset + // and memory address for the load command can be derived from the + // header's file offset and memory address using this offset. +#define FOR_EACH_COMMAND_LINEAR_FIELD(M) M(header_offset) + + FOR_EACH_COMMAND_LINEAR_FIELD(DEFINE_LINEAR_FIELD_METHODS); + + private: + FOR_EACH_COMMAND_LINEAR_FIELD(DEFINE_LINEAR_FIELD); + +#undef FOR_EACH_COMMAND_LINEAR_FIELD + + private: + uint32_t cmd_; + + DISALLOW_COPY_AND_ASSIGN(MachOCommand); +}; + +class MachOSection : public MachOContents { +#if defined(TARGET_ARCH_IS_32_BIT) + using SectionType = mach_o::section; +#else + using SectionType = mach_o::section_64; +#endif + + public: + MachOSection(Zone* zone, + const char* name, + intptr_t type = mach_o::S_REGULAR, + intptr_t attributes = mach_o::S_NO_ATTRIBUTES, + bool has_contents = true, + intptr_t alignment = MachOWriter::kPageSize) + : MachOContents(/*needs_offset=*/has_contents, + /*in_segment=*/true), + name_(name), + flags_(mach_o::SectionFlags(type, attributes)), + alignment_(alignment), + portions_(zone, 0) { + ASSERT(strlen(name) <= sizeof(SectionType::sectname)); + ASSERT(Utils::IsPowerOfTwo(alignment)); + ASSERT_EQUAL(type & mach_o::SECTION_TYPE, static_cast(type)); + ASSERT_EQUAL(attributes & mach_o::SECTION_ATTRIBUTES, + static_cast(attributes)); + if (type == mach_o::S_ZEROFILL && type == mach_o::S_GB_ZEROFILL) { + ASSERT(!has_contents); + } + } + + DEFINE_TYPE_CHECK_FOR(MachOSection) + + intptr_t Alignment() const override { return alignment_; } + + const char* name() const { return name_; } + + bool HasName(const char* name) const { return strcmp(name_, name) == 0; } + + struct Portion { + void Write(MachOWriteStream* stream, intptr_t section_start) const { + ASSERT(bytes != nullptr); + if (relocations != nullptr) { + const intptr_t address = section_start + offset; + stream->WriteBytesWithRelocations(bytes, size, address, *relocations); + } else { + stream->WriteBytes(bytes, size); + } + } + + bool ContainsSymbols() const { + return symbol_name != nullptr || + (symbols != nullptr && !symbols->is_empty()); + } + + intptr_t offset; + const char* symbol_name; + intptr_t label; + const uint8_t* bytes; + intptr_t size; + const SharedObjectWriter::RelocationArray* relocations; + const SharedObjectWriter::SymbolDataArray* symbols; + + private: + DISALLOW_ALLOCATION(); + }; + + const GrowableArray& portions() const { return portions_; } + + void AddPortion(const uint8_t* bytes, + intptr_t size, + const SharedObjectWriter::RelocationArray* relocations, + const SharedObjectWriter::SymbolDataArray* symbols = nullptr, + const char* symbol_name = nullptr, + intptr_t label = 0) { + // Any named portion should also have a valid symbol label. + ASSERT(symbol_name == nullptr || label > 0); + ASSERT(!HasContents() || bytes != nullptr); + ASSERT(bytes != nullptr || relocations == nullptr); + // Make sure all portions are consistent in containing bytes. + ASSERT(portions_.is_empty() || + (portions_[0].bytes != nullptr) == (bytes != nullptr)); + intptr_t offset = 0; + if (!portions_.is_empty()) { + const auto& last = portions_.Last(); + offset = last.offset + last.size; + } + // Each portion is aligned within the section. + offset = Utils::RoundUp(offset, Alignment()); + portions_.Add( + {offset, symbol_name, label, bytes, size, relocations, symbols}); + } + + intptr_t SelfMemorySize() const override { + const auto& last = portions_.Last(); + return last.offset + last.size; + } + + void WriteSelf(MachOWriteStream* stream) const override { + if (!HasContents()) return; + for (const auto& portion : portions_) { + // Each portion is aligned within the section. + stream->Align(Alignment()); + ASSERT_EQUAL(stream->Position(), file_offset() + portion.offset); + portion.Write(stream, memory_address()); + } + } + + const Portion* FindPortion(const char* symbol_name) const { + for (const auto& portion : portions_) { + if (strcmp(symbol_name, portion.symbol_name) == 0) { + return &portion; + } + } + return nullptr; + } + + bool ContainsSymbols() const { + for (const auto& p : portions_) { + if (p.ContainsSymbols()) return true; + } + return false; + } + + void Accept(Visitor* visitor) override { visitor->VisitMachOSection(this); } + + private: + uint32_t HeaderInfoSize() const { return sizeof(SectionType); } + + // Called during MachOSegment::WriteLoadCommand. + void WriteHeaderInfo(MachOWriteStream* stream, const char* segname) const { + auto const start = stream->Position(); + stream->WriteFixedLengthCString(name_, sizeof(SectionType::sectname)); + stream->WriteFixedLengthCString(segname, sizeof(SectionType::segname)); + // While + stream->WriteWord(memory_address()); + stream->WriteWord(MemorySize()); + stream->Write32(file_offset()); + stream->Write32(Utils::ShiftForPowerOfTwo(Alignment())); + stream->WriteOffsetCount(0, 0); // No relocation entries. + stream->Write32(flags_); + // All reserved fields are 0 for our purposes. + stream->Write32(0); // reserved1 + stream->Write32(0); // reserved2 +#if defined(TARGET_ARCH_IS_64_BIT) + stream->Write32(0); // reserved3 +#endif + ASSERT_EQUAL(stream->Position(), + static_cast(start + HeaderInfoSize())); + } + + const char* const name_; + const decltype(SectionType::flags) flags_ = 0; + const intptr_t alignment_; + GrowableArray portions_; + + friend class MachOSegment; + + DISALLOW_COPY_AND_ASSIGN(MachOSection); +}; + +class MachOSegment : public MachOCommand { +#if defined(TARGET_ARCH_IS_32_BIT) + using SegmentCommandType = mach_o::segment_command; +#else + using SegmentCommandType = mach_o::segment_command_64; +#endif + + public: +#if defined(TARGET_ARCH_IS_32_BIT) + static constexpr uint32_t kCommandCode = mach_o::LC_SEGMENT; +#else + static constexpr uint32_t kCommandCode = mach_o::LC_SEGMENT_64; +#endif + + MachOSegment(Zone* zone, + const char* name, + intptr_t initial_vm_protection = mach_o::VM_PROT_READ, + intptr_t max_vm_protection = mach_o::VM_PROT_READ) + // We don't know if a segment has a file offset until we + // know what it contains, so set it to 0 in ComputeOffsets() + // if there are no contents. + : MachOCommand(kCommandCode), + name_(name), + initial_vm_protection_(initial_vm_protection), + max_vm_protection_(max_vm_protection), + contents_(zone, 0) { + ASSERT(Utils::IsInt(32, initial_vm_protection)); + ASSERT(Utils::IsInt(32, max_vm_protection)); + ASSERT(strlen(name) <= sizeof(SegmentCommandType::segname)); + } + + DEFINE_TYPE_CHECK_FOR(MachOSegment) + + const char* name() const { return name_; } + const GrowableArray& contents() const { return contents_; } + + bool IsReadable() const { + return (initial_vm_protection_ & mach_o::VM_PROT_READ) != 0; + } + bool IsWritable() const { + return (initial_vm_protection_ & mach_o::VM_PROT_WRITE) != 0; + } + bool IsExecutable() const { + return (initial_vm_protection_ & mach_o::VM_PROT_EXECUTE) != 0; + } + + intptr_t Alignment() const override { return MachOWriter::kPageSize; } + + // The text segment has a file and memory offset of 0, so the superclass's + // implementations give false negatives after ComputeOffsets. + bool HasContents() const override { return next_contents_index_ > 0; } + bool IsAllocated() const override { return true; } + + uint32_t cmdsize() const override { + uword size = sizeof(SegmentCommandType); + // The header information for sections is nested within the + // segment load command. + for (auto* const c : contents_) { + if (auto* const s = c->AsMachOSection()) { + size += s->HeaderInfoSize(); + } + } + ASSERT(Utils::IsUint(32, size)); + return size; + } + + bool PadFileSizeToAlignment() const { + // The linkedit segment should _not_ be padded to alignment, because + // that means the code signature isn't the last contents of the file + // when applicable. + return !HasName(mach_o::SEG_LINKEDIT); + } + + // Segments do not contain any header information, just nested content. + intptr_t SelfMemorySize() const override { return 0; } + + intptr_t FileSize() const override { + intptr_t file_size = SelfFileSize(); + for (auto* const c : contents_) { + if (!c->HasContents()) continue; + file_size = Utils::RoundUp(file_size, c->Alignment()); + file_size += c->FileSize(); + } + if (PadFileSizeToAlignment()) { + file_size = Utils::RoundUp(file_size, Alignment()); + } + return file_size; + } + + intptr_t MemorySize() const override { + intptr_t memory_size = SelfMemorySize(); + for (auto* const c : contents_) { + ASSERT(c->IsAllocated()); // Segments never contain unallocated contents. + memory_size = Utils::RoundUp(memory_size, c->Alignment()); + memory_size += c->MemorySize(); + } + return Utils::RoundUp(memory_size, Alignment()); + } + + // The initial segment of the Mach-O file always includes the header + // as its first contents. + bool IsInitial() const { return header() != nullptr; } + + // Returns the header if this is the initial segment (which contains it), + // otherwise nullptr. + const MachOHeader* header() const { + return contents_.is_empty() ? nullptr : contents_[0]->AsMachOHeader(); + } + + bool HasName(const char* name) const { return strcmp(name_, name) == 0; } + + bool ContainsSymbols() const { + for (auto* const c : contents_) { + if (auto* const s = c->AsMachOSection()) { + if (s->ContainsSymbols()) { + return true; + } + } + } + return false; + } + + void AddContents(MachOContents* c); + + bool IsDebugOnly() const { + // Currently, the dwarf segment is the only debug-only info we add. + return HasName(mach_o::SEG_DWARF); + } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->WriteFixedLengthCString(name_, sizeof(SegmentCommandType::segname)); + stream->WriteWord(memory_address()); + stream->WriteWord(MemorySize()); + stream->WriteWord(file_offset()); + // Only report the actual file size if there is non-header content. + if (IsInitial() && next_contents_index_ == 1) { + stream->WriteWord(0); + } else { + stream->WriteWord(FileSize()); + } + stream->Write32(max_vm_protection_); + stream->Write32(initial_vm_protection_); + stream->Write32(NumSections()); + // The writer never uses segment flags. + stream->Write32(0); + // The load command for a segment also contains descriptions for its + // sections instead of these being in separate load commands. + for (auto* const c : contents_) { + if (!c->IsMachOSection()) continue; + c->AsMachOSection()->WriteHeaderInfo(stream, name_); + } + } + + MachOSection* FindSection(const char* name) const { + for (auto* const c : contents_) { + if (auto* const s = c->AsMachOSection()) { + if (s->HasName(name)) return s; + } + } + return nullptr; + } + + intptr_t NumSections() const { + intptr_t count = 0; + for (auto* const c : contents_) { + if (c->IsMachOSection()) { + count += 1; + } + } + return count; + } + + void Accept(Visitor* visitor) override { visitor->VisitMachOSegment(this); } + void VisitChildren(Visitor* visitor) override { + for (auto* const c : contents_) { + c->Accept(visitor); + } + } + + private: + const char* const name_; + bool has_contents_ = false; + intptr_t next_contents_index_ = 0; + mach_o::vm_prot_t initial_vm_protection_; + mach_o::vm_prot_t max_vm_protection_; + GrowableArray contents_; + + DISALLOW_COPY_AND_ASSIGN(MachOSegment); +}; + +class MachOUuid : public MachOCommand { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_UUID; + + explicit MachOUuid(const void* bytes, intptr_t len) + : MachOCommand(kCommandCode, + /*needs_offset=*/false, + /*in_segment=*/false), + bytes_() { + // Make sure the length of the byte buffer matches the UUID length, so + // that the provided UUID isn't unexpectedly truncated or extended. + ASSERT_EQUAL(len, sizeof(bytes_)); + memmove(bytes_, bytes, sizeof(bytes_)); + } + + uint32_t cmdsize() const override { return sizeof(mach_o::uuid_command); } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->WriteBytes(bytes_, sizeof(bytes_)); + } + + void Accept(Visitor* visitor) override { visitor->VisitMachOUuid(this); } + + private: + uint8_t bytes_[sizeof(mach_o::uuid_command::uuid)]; + DISALLOW_COPY_AND_ASSIGN(MachOUuid); +}; + +#define MACHO_XYZ_VERSION_ENCODING(x, y, z) \ + static_cast(((x) << 16) | ((y) << 8) | (z)) + +class MachOBuildVersion : public MachOCommand { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_BUILD_VERSION; + + MachOBuildVersion() + : MachOCommand(kCommandCode, + /*needs_offset=*/false, + /*in_segment=*/false) {} + + uint32_t cmdsize() const override { + return sizeof(mach_o::build_version_command); + } + + uint32_t platform() const { +#if defined(DART_TARGET_OS_MACOS_IOS) + return mach_o::PLATFORM_IOS; +#elif defined(DART_TARGET_OS_MACOS) + return mach_o::PLATFORM_MACOS; +#else + return mach_o::PLATFORM_UNKNOWN; +#endif + } + + uint32_t minos() const { +#if defined(DART_TARGET_OS_MACOS_IOS) + // TODO(sstrickl): No minimum version for iOS currently defined. + UNIMPLEMENTED(); +#elif defined(DART_TARGET_OS_MACOS) + return kMinMacOSVersion; +#else + return 0; // No version for the unknown platform. +#endif + } + + uint32_t sdk() const { +#if defined(DART_TARGET_OS_MACOS_IOS) + // TODO(sstrickl): No SDK version for iOS currently defined. + UNIMPLEMENTED(); +#elif defined(DART_TARGET_OS_MACOS) + return kMacOSSdkVersion; +#else + return 0; // No version for the unknown platform. +#endif + } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->Write32(platform()); + stream->Write32(minos()); + stream->Write32(sdk()); + stream->Write32(0); // No tool versions. + } + + void Accept(Visitor* visitor) override { + visitor->VisitMachOBuildVersion(this); + } + + private: + static constexpr auto kMinMacOSVersion = MACHO_XYZ_VERSION_ENCODING(15, 0, 0); + static constexpr auto kMacOSSdkVersion = MACHO_XYZ_VERSION_ENCODING(15, 4, 0); + + DISALLOW_COPY_AND_ASSIGN(MachOBuildVersion); +}; + +class MachODylib : public MachOCommand { + public: + uint32_t cmdsize() const override { + intptr_t size = NameOffset() + strlen(name_) + 1; + return Utils::RoundUp(size, kLoadCommandAlignment); + } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->Write32(NameOffset()); + stream->Write32(timestamp_); + stream->Write32(current_version_); + stream->Write32(compatibility_version_); + stream->WriteNullTerminatedCString(name_); + stream->Align(kLoadCommandAlignment); + } + + static constexpr auto kNoVersion = MACHO_XYZ_VERSION_ENCODING(0, 0, 0); + + protected: + // This is really an abstract class, with concrete subclasses providing + // the command code. + MachODylib(intptr_t cmd, + const char* name, + intptr_t timestamp, + intptr_t current_version = kNoVersion, + intptr_t compatibility_version = kNoVersion) + : MachOCommand(cmd, + /*needs_offset=*/false, + /*in_segment=*/false), + name_(ASSERT_NOTNULL(name)), + timestamp_(timestamp), + current_version_(current_version), + compatibility_version_(compatibility_version) { + ASSERT(Utils::IsUint(32, timestamp)); + ASSERT(Utils::IsUint(32, current_version)); + ASSERT(Utils::IsUint(32, compatibility_version)); + } + + private: + uint32_t NameOffset() const { return sizeof(mach_o::dylib_command); } + + const char* const name_; + const uint32_t timestamp_; + const uint32_t current_version_; + const uint32_t compatibility_version_; + + DISALLOW_COPY_AND_ASSIGN(MachODylib); +}; + +class MachOIdDylib : public MachODylib { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_ID_DYLIB; + + explicit MachOIdDylib(const char* name = kDefaultSnapshotName, + intptr_t current_version = kNoVersion, + intptr_t compatibility_version = kNoVersion) + : MachODylib(kCommandCode, + name, + 0, // Snapshots aren't copied into user. + current_version, + compatibility_version) {} + + void Accept(Visitor* visitor) override { visitor->VisitMachOIdDylib(this); } + + private: + static constexpr char kDefaultSnapshotName[] = "aot.snapshot"; + DISALLOW_COPY_AND_ASSIGN(MachOIdDylib); +}; + +class MachOLoadDylib : public MachODylib { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_LOAD_DYLIB; + + static MachOLoadDylib* CreateLoadSystemDylib(Zone* zone) { + return new (zone) MachOLoadDylib(kSystemDylibName, 0, kSystemCurrentVersion, + kSystemCompatVersion); + } + + void Accept(Visitor* visitor) override { visitor->VisitMachOLoadDylib(this); } + + private: + MachOLoadDylib(const char* name, + intptr_t timestamp, + intptr_t current_version, + intptr_t compatibility_version) + : MachODylib(kCommandCode, + name, + timestamp, + current_version, + compatibility_version) {} + + static constexpr char kSystemDylibName[] = "/usr/lib/libSystem.B.dylib"; + static constexpr auto kSystemCurrentVersion = + MACHO_XYZ_VERSION_ENCODING(1351, 0, 0); + static constexpr auto kSystemCompatVersion = + MACHO_XYZ_VERSION_ENCODING(1, 0, 0); + + DISALLOW_COPY_AND_ASSIGN(MachOLoadDylib); +}; + +#undef MACHO_XYZ_VERSION_ENCODING + +class MachOSymbolTable : public MachOCommand { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_SYMTAB; + + explicit MachOSymbolTable(Zone* zone) + : MachOCommand(kCommandCode), + zone_(zone), + strings_(zone), + symbols_(zone, 0), + by_label_index_(zone) {} + + class StringTable : public ValueObject { + public: + explicit StringTable(Zone* zone) : text_(zone), text_indices_(zone) { + // Ensure the string containing a single space is always at index 0. + const intptr_t index = Add(" "); + ASSERT_EQUAL(index, 0); + // Assign the empty string the index of the null byte in the + // string added above. + text_indices_.Insert({"", index + 1}); + } + + intptr_t Add(const char* str) { + ASSERT(str != nullptr); + if (auto const kv = text_indices_.Lookup(str)) { + return kv->value; + } + intptr_t offset = text_.length(); + text_.AddString(str); + text_.AddChar('\0'); + text_indices_.Insert({str, offset}); + return offset; + } + + const char* At(intptr_t index) const { + if (index >= text_.length()) return nullptr; + return text_.buffer() + index; + } + + intptr_t FileSize() const { return text_.length(); } + + void Write(MachOWriteStream* stream) const { + stream->WriteBytes(text_.buffer(), text_.length()); + } + + private: + ZoneTextBuffer text_; + CStringIntMap text_indices_; + DISALLOW_COPY_AND_ASSIGN(StringTable); + }; + + struct Symbol { + Symbol(intptr_t n_idx, + intptr_t n_type, + intptr_t n_sect, + intptr_t n_desc, + uword n_value) + : name_index(n_idx), + type(n_type), + section_index(n_sect), + description(n_desc), + value(n_value) { + ASSERT(Utils::IsUint(32, n_idx)); + ASSERT(Utils::IsUint(8, n_type)); + ASSERT(Utils::IsUint(8, n_sect)); + ASSERT(Utils::IsUint(16, n_desc)); + ASSERT(Utils::IsUint(sizeof(compiler::target::uword) * kBitsPerByte, + n_value)); + } + + void Write(MachOWriteStream* stream) const { + const intptr_t start = stream->Position(); + stream->Write32(name_index); + stream->WriteByte(type); + stream->WriteByte(section_index); + stream->Write16(description); + stream->WriteWord(value); + ASSERT_EQUAL(stream->Position() - start, sizeof(mach_o::nlist)); + } + + // The index of the name in the symbol table's string table. + uint32_t name_index; + // See the mach_o::N_* constants for the encoding of this field. + uint8_t type; + // The section to which this symbol belongs if not equal to mach_o::NO_SECT. + // The sections are indexed by their appearance in the load commands + // (e.g., the first section of the first segment command that contains + // sections has index 1, and the first section of the second segment command + // that contains sections has index [k + 1] if the first segment contains + // [k] sections). + uint8_t section_index; + // See the mach_o::N_* constants for the encoding of this field. + uint16_t description; + // For symbols where section_index != macho_o::NO_SECT, this is the section + // offset until finalization, when it is converted to the offset into the + // snapshot. + compiler::target::uword value; + + DISALLOW_ALLOCATION(); + }; + + const StringTable& strings() const { return strings_; } + const GrowableArray& symbols() const { return symbols_; } + DEBUG_ONLY(intptr_t max_label() const { return max_label_; }) + + void AddSymbol(const char* name, + intptr_t type, + intptr_t section_index, + intptr_t description, + uword value, + intptr_t label = -1) { + // Section symbols should always have labels, and other symbols + // (including symbolic debugging symbols) do not. + if ((type & mach_o::N_STAB) != 0) { + ASSERT(label <= 0); + } else { + ASSERT_EQUAL((type & mach_o::N_TYPE) == mach_o::N_SECT, label > 0); + } + ASSERT(!file_offset_is_set()); // Can grow until offsets computed. + auto const name_index = strings_.Add(name); + ASSERT(*name == '\0' || name_index != 0); + const intptr_t new_index = num_symbols(); + symbols_.Add({name_index, type, section_index, description, value}); + if (label > 0) { + DEBUG_ONLY(max_label_ = max_label_ > label ? max_label_ : label); + // Store an 1-based index since 0 is kNoValue for IntMap. + by_label_index_.Insert(label, new_index + 1); + } + } + + const Symbol* FindLabel(intptr_t label) const { + ASSERT(label > 0); + // The stored index is 1-based. + const intptr_t symbols_index = by_label_index_.Lookup(label) - 1; + if (symbols_index < 0) return nullptr; // Not found. + return &symbols_[symbols_index]; + } + + void Initialize(const GrowableArray& sections, + bool is_stripped); + + void UpdateSectionIndices(const GrowableArray& index_map) { + const intptr_t map_size = index_map.length(); +#if defined(DEBUG) + for (intptr_t i = 0; i < map_size; i++) { + const intptr_t new_index = index_map[i]; + ASSERT(Utils::IsUint(8, new_index)); + ASSERT(new_index < map_size); + if (i == mach_o::NO_SECT) { + ASSERT_EQUAL(new_index, mach_o::NO_SECT); + } else { + ASSERT(new_index != mach_o::NO_SECT); + } + } +#endif + for (auto& symbol : symbols_) { + const uint8_t old_index = symbol.section_index; + ASSERT(old_index < map_size); + symbol.section_index = index_map[old_index]; + } + } + + void Finalize(const GrowableArray& address_map) { + const intptr_t map_size = address_map.length(); +#if defined(DEBUG) + for (intptr_t i = 0; i < map_size; i++) { + if (i == mach_o::NO_SECT) { + // The entry for NO_SECT must be 0 so that symbols with that index, + // like global symbols, are unchanged. + ASSERT_EQUAL(address_map[mach_o::NO_SECT], 0); + } else { + // No valid section begins at the start of the snapshot. + ASSERT(address_map[i] > 0); + } + } +#endif + for (auto& symbol : symbols_) { + ASSERT(symbol.section_index < map_size); + symbol.value += address_map[symbol.section_index]; + } + } + + uint32_t cmdsize() const override { return sizeof(mach_o::symtab_command); } + + intptr_t SelfMemorySize() const override { + return SymbolsSize() + strings_.FileSize(); + } + + intptr_t Alignment() const override { return compiler::target::kWordSize; } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->WriteOffsetCount(file_offset(), num_symbols()); + stream->WriteOffsetCount(file_offset() + SymbolsSize(), + strings_.FileSize()); + } + + void WriteSelf(MachOWriteStream* stream) const override { + for (const auto& symbol : symbols_) { + symbol.Write(stream); + } + strings_.Write(stream); + } + + intptr_t num_symbols() const { return symbols_.length(); } + + void Accept(Visitor* visitor) override { + visitor->VisitMachOSymbolTable(this); + } + +#define FOR_EACH_SYMBOL_TABLE_LINEAR_FIELD(M) \ + M(num_local_symbols) \ + M(num_external_symbols) + + FOR_EACH_SYMBOL_TABLE_LINEAR_FIELD(DEFINE_LINEAR_FIELD_METHODS); + + private: + intptr_t SymbolsSize() const { return num_symbols() * sizeof(mach_o::nlist); } + + Zone* const zone_; + StringTable strings_; + GrowableArray symbols_; + // Maps symbol labels (positive integers) to indexes in symbols_. + IntMap by_label_index_; + DEBUG_ONLY(intptr_t max_label_ = 0;) // For consistency checks. + + FOR_EACH_SYMBOL_TABLE_LINEAR_FIELD(DEFINE_LINEAR_FIELD); +#undef FOR_EACH_SYMBOL_TABLE_LINEAR_FIELD + + DISALLOW_COPY_AND_ASSIGN(MachOSymbolTable); +}; + +class MachODynamicSymbolTable : public MachOCommand { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_DYSYMTAB; + + explicit MachODynamicSymbolTable(const MachOSymbolTable& table) + : MachOCommand(kCommandCode), table_(table) {} + + uint32_t cmdsize() const override { return sizeof(mach_o::dysymtab_command); } + + intptr_t Alignment() const override { return compiler::target::kWordSize; } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + // The symbol table contains local symbols and then external symbols. + intptr_t index = 0; + stream->WriteOffsetCount(index, table_.num_local_symbols()); + index += table_.num_local_symbols(); + stream->WriteOffsetCount(index, table_.num_external_symbols()); + index += table_.num_external_symbols(); + // No undefined symbols. + stream->WriteOffsetCount(index, 0); + // The rest of the fields are 0-filled. + for (intptr_t i = 0; i < kUnusedOffsetCountPairs; ++i) { + stream->WriteOffsetCount(0, 0); + } + } + + // Currently no contents are written to the linkedit segment, as the + // only non-zero fields are indexes/counts into the symbol table. + intptr_t SelfMemorySize() const override { return 0; } + + void Accept(Visitor* visitor) override { + visitor->VisitMachODynamicSymbolTable(this); + } + + private: + static constexpr intptr_t kUnusedOffsetCountPairs = 6; + + const MachOSymbolTable& table_; + DISALLOW_COPY_AND_ASSIGN(MachODynamicSymbolTable); +}; + +class MachOLinkEditData : public MachOCommand { + public: + uint32_t cmdsize() const override { + return sizeof(mach_o::linkedit_data_command); + } + + void WriteLoadCommand(MachOWriteStream* stream) const override { + MachOCommand::WriteLoadCommand(stream); + stream->WriteOffsetCount(file_offset(), FileSize()); + } + + protected: + // This is really an abstract class, with concrete subclasses providing + // the command code. + explicit MachOLinkEditData(intptr_t cmd) + : MachOCommand(cmd, /*needs_offset=*/true, /*in_segment=*/true) {} + + private: + DISALLOW_COPY_AND_ASSIGN(MachOLinkEditData); +}; + +class MachOCodeSignature : public MachOLinkEditData { + public: + static constexpr uint32_t kCommandCode = mach_o::LC_CODE_SIGNATURE; + + explicit MachOCodeSignature(const char* identifier) + : MachOLinkEditData(kCommandCode), identifier_(identifier) {} + + static constexpr intptr_t kHeaderAlignment = 8; + static constexpr intptr_t kHashAlignment = 16; + + intptr_t Alignment() const override { return kHashAlignment; } + + intptr_t SelfMemorySize() const override { + return DirectoryOffset() + DirectoryLength(); + } + + void WriteSelf(MachOWriteStream* stream) const override { + // The code signature marks the end of the hashed content, as + // it contains the hashes that ensure the previous content has + // not been modified (modulo hash collisions). + stream->FinalizeHashedContent(); + ASSERT_EQUAL(stream->num_hashes(), ExpectedNumHashes()); + const intptr_t start = stream->Position(); + // The superblob header, which includes a single blob index. + stream->WriteBE32(mach_o::CSMAGIC_EMBEDDED_SIGNATURE); // magic + stream->WriteBE32(FileSize()); // length + stream->WriteBE32(1); // count + // Blob index for the code directory. + stream->WriteBE32(mach_o::CSSLOT_CODEDIRECTORY); // type + stream->WriteBE32(DirectoryOffset()); // offset + stream->Align(kHeaderAlignment); + // Now the header for the code directory. + ASSERT_EQUAL(stream->Position() - start, DirectoryOffset()); + const intptr_t directory_start = stream->Position(); + stream->WriteBE32(mach_o::CSMAGIC_CODEDIRECTORY); // magic + stream->WriteBE32(DirectoryLength()); // length + stream->WriteBE32(mach_o::CS_SUPPORTSEXECSEG); // version + stream->WriteBE32(mach_o::CS_ADHOC | mach_o::CS_LINKER_SIGNED); // flags + stream->WriteBE32(HashOffset()); + stream->WriteBE32(IdentOffset()); + stream->WriteBE32(0); // num special slots (hashes) + stream->WriteBE32(stream->num_hashes()); // num code slots (hashes) + stream->WriteBE32(file_offset()); // code limit + stream->WriteByte(MachOWriteStream::kHashSize); + stream->WriteByte(MachOWriteStream::kHashType); + stream->WriteByte(0); // platform + // The page size is represented by its base 2 logarithm. + stream->WriteByte(Utils::ShiftForPowerOfTwo(MachOWriteStream::kChunkSize)); + stream->WriteBE32(0); // spare2 (always 0) + // version >= 0x20100 (CS_SUPPORTSSCATTER) + stream->WriteBE32(0); // scatter offset + // version >= 0x20200 (CS_SUPPORTSTEAMID) + stream->WriteBE32(0); // teamid offset + // version >= 0x20300 (CS_SUPPORTSCODELIMIT64) + stream->WriteBE32(0); // spare3 (always 0) + stream->WriteBE64(0); // code limit (64-bit) + // version >= 0x20400 (CS_SUPPORTSEXECSEG) + stream->WriteBE64(stream->TextSegment().file_offset()); // offset + stream->WriteBE64(stream->TextSegment().FileSize()); // limit + stream->WriteBE64(0); // flags + stream->Align(kHeaderAlignment); + ASSERT_EQUAL(stream->Position() - directory_start, IdentOffset()); + stream->WriteFixedLengthCString(identifier_, strlen(identifier_) + 1); + stream->Align(kHashAlignment); + ASSERT_EQUAL(stream->Position() - directory_start, HashOffset()); + stream->WriteHashes(); + ASSERT_EQUAL(stream->Position() - directory_start, DirectoryLength()); + } + + void Accept(Visitor* visitor) override { + visitor->VisitMachOCodeSignature(this); + } + + private: + // The offset of the code directory in the code signature. + intptr_t DirectoryOffset() const { + // A single blob index for the code directory. + const intptr_t offset = + sizeof(mach_o::cs_superblob) + sizeof(mach_o::cs_blob_index); + return Utils::RoundUp(offset, kHeaderAlignment); + } + + intptr_t DirectoryLength() const { + return HashOffset() + ExpectedNumHashes() * MachOWriteStream::kHashSize; + } + + // The offset of the identifier within the code directory. + intptr_t IdentOffset() const { + // Include the directory offset to ensure proper alignment, but the + // returned value is relative to the code directory start. + intptr_t signature_offset = + DirectoryOffset() + sizeof(mach_o::cs_code_directory); + return Utils::RoundUp(signature_offset, kHeaderAlignment) - + DirectoryOffset(); + } + + // The offset of the list of hashes within the code directory. + intptr_t HashOffset() const { + // Include the directory offset to ensure proper alignment, but the + // returned value is relative to the code directory start. + const intptr_t signature_offset = + DirectoryOffset() + IdentOffset() + strlen(identifier_) + 1; + return Utils::RoundUp(signature_offset, kHashAlignment) - DirectoryOffset(); + } + + intptr_t ExpectedNumHashes() const { + // The actual hashes are stored in the stream, which isn't available yet. + // However, if the file offsets of the code signature has been computed, the + // number of hashes that should be contained in the stream can be computed. + const intptr_t chunk_size = MachOWriteStream::kChunkSize; + return (file_offset() + chunk_size - 1) / chunk_size; + } + + const char* const identifier_; + + DISALLOW_COPY_AND_ASSIGN(MachOCodeSignature); +}; + +// A representation of the header of the Mach-O file. This contains +// any commands that have load commands within the header. +class MachOHeader : public MachOContents { +#if defined(TARGET_ARCH_IS_32_BIT) + using HeaderType = mach_o::mach_header; +#else + using HeaderType = mach_o::mach_header_64; +#endif + + using SnapshotType = SharedObjectWriter::Type; + + public: + MachOHeader(Zone* zone, + SnapshotType type, + bool is_stripped, + const char* identifier, + Dwarf* dwarf) + : MachOContents(), + zone_(zone), + type_(type), + is_stripped_(is_stripped), + identifier_(identifier != nullptr ? identifier : ""), + dwarf_(dwarf), + commands_(zone, 0), + full_symtab_(zone) { +#if defined(DART_TARGET_OS_MACOS) + // A non-nullptr identifier must be provided for MacOS targets. + ASSERT(identifier != nullptr); +#endif + // Unstripped content must have DWARF information available. + ASSERT(dwarf != nullptr || is_stripped_); + // Only snapshots should be stripped. + ASSERT(!is_stripped_ || type == SnapshotType::Snapshot); + } + + DEFINE_TYPE_CHECK_FOR(MachOHeader) + + Zone* zone() const { return zone_; } + const GrowableArray& commands() const { return commands_; } + const MachOSymbolTable& relocation_symbol_table() const { + return full_symtab_; + } + const MachOSegment& text_segment() const { + ASSERT(text_segment_ != nullptr); + return *text_segment_; + } + + intptr_t NumSections() const { + intptr_t num_sections = 0; + for (auto* const command : commands()) { + if (auto* const s = command->AsMachOSegment()) { + num_sections += s->NumSections(); + } + } + return num_sections; + } + + // The contents of the header is always at offset/address 0, so the + // superclass's check returns a false negative here after ComputeOffsets. + bool HasContents() const override { return true; } + bool IsAllocated() const override { return true; } + intptr_t Alignment() const override { return compiler::target::kWordSize; } + + // The header uses the default MemorySize() implementation, because + // VisitChildren() doesn't visit the load commands and so the header is + // not considered to contain nested content. + // + // This should be used if the size of the header without the load commands + // is desired. + intptr_t SizeWithoutLoadCommands() const { + const intptr_t size = sizeof(HeaderType); + ASSERT(Utils::IsAligned(size, MachOCommand::kLoadCommandAlignment)); + return size; + } + + intptr_t SelfMemorySize() const override { + intptr_t size = SizeWithoutLoadCommands(); + for (auto* const command : commands_) { + size += command->cmdsize(); + } + return size; + } + + uint32_t filetype() const { + if (type_ == SnapshotType::Snapshot) { + return mach_o::MH_DYLIB; + } + ASSERT(type_ == SnapshotType::DebugInfo); + return mach_o::MH_DSYM; + } + + uint32_t flags() const { + if (type_ == SnapshotType::Snapshot) { + return mach_o::MH_NOUNDEFS | mach_o::MH_DYLDLINK | + mach_o::MH_NO_REEXPORTED_DYLIBS; + } + ASSERT(type_ == SnapshotType::DebugInfo); + return 0; + } + + mach_o::cpu_type_t cpu_type() const { +#if defined(TARGET_ARCH_X64) + return mach_o::CPU_TYPE_X86_64; +#elif defined(TARGET_ARCH_ARM64) + return mach_o::CPU_TYPE_ARM64; +#elif defined(TARGET_ARCH_IA32) + return mach_o::CPU_TYPE_I386; +#elif defined(TARGET_ARCH_ARM) + return mach_o::CPU_TYPE_ARM; +#else + // This architecture doesn't have specific constants defined in + // , so just mark it as ANY since the snapshot + // header check also catches architecture mismatches. + return mach_o::CPU_TYPE_ANY; +#endif + } + + mach_o::cpu_subtype_t cpu_subtype() const { +#if defined(TARGET_ARCH_X64) + return mach_o::CPU_SUBTYPE_X86_64_ALL; +#elif defined(TARGET_ARCH_ARM64) + return mach_o::CPU_SUBTYPE_ARM64_ALL; +#elif defined(TARGET_ARCH_IA32) + return mach_o::CPU_SUBTYPE_I386_ALL; +#elif defined(TARGET_ARCH_ARM) + return mach_o::CPU_SUBTYPE_ARM_ALL; +#else + // This architecture doesn't have specific constants defined in + // , so just mark it as ANY since the snapshot + // header check also catches architecture mismatches. + return mach_o::CPU_SUBTYPE_ANY; +#endif + } + + void WriteSelf(MachOWriteStream* stream) const override { + intptr_t start = stream->Position(); + ASSERT_EQUAL(start, 0); +#if defined(TARGET_ARCH_IS_32_BIT) + stream->Write32(mach_o::MH_MAGIC); +#else + stream->Write32(mach_o::MH_MAGIC_64); +#endif + stream->Write32(cpu_type()); + stream->Write32(cpu_subtype()); + stream->Write32(filetype()); + stream->Write32(commands_.length()); + uint32_t sizeofcmds = 0; + for (auto* const command : commands_) { + sizeofcmds += command->cmdsize(); + } + stream->Write32(sizeofcmds); + stream->Write32(flags()); +#if !defined(TARGET_ARCH_IS_32_BIT) + stream->Write32(0); // Reserved field. +#endif + ASSERT_EQUAL(stream->Position() - start, sizeof(HeaderType)); + for (auto* const command : commands_) { + const intptr_t load_start = stream->Position(); + ASSERT_EQUAL(load_start, start + command->header_offset()); + command->WriteLoadCommand(stream); + ASSERT_EQUAL(stream->Position() - load_start, + static_cast(command->cmdsize())); + } + } + + // Returns the command with the given concrete subclass of MachOCommand + // (that is, a subclass that defines a kCommandCode constant). Should only + // be used for commands that appear at most once (e.g., not segments). + template + T* FindCommand() const { + return reinterpret_cast(FindCommand(T::kCommandCode)); + } + + // Returns the command with the given command code. Should only be used + // for commands that appear at most once (e.g., not segments). + MachOCommand* FindCommand(uint32_t cmd) const { + MachOCommand* result = nullptr; + for (auto* const command : commands_) { + if (command->cmd() == cmd) { + ASSERT(result == nullptr); + result = command; +#if !defined(DEBUG) + break; // No checking, so don't continue iterating. +#endif + } + } + return result; + } + + // Returns whether there is a command has the given command code. + bool HasCommand(uint32_t cmd) const { + for (auto* const command : commands_) { + if (command->cmd() == cmd) return true; + } + return false; + } + + // Returns the segment with name [name] or nullptr if there is none. + MachOSegment* FindSegment(const char* name) const { + for (auto* const command : commands_) { + if (auto* const s = command->AsMachOSegment()) { + if (s->HasName(name)) return s; + } + } + return nullptr; + } + + // Returns the section with name [sectname] in segment [segname] + // or nullptr if there is none. + MachOSection* FindSection(const char* segname, const char* sectname) const { + auto* const s = FindSegment(segname); + if (s == nullptr) return nullptr; + return s->FindSection(sectname); + } + + MachOSegment* EnsureTextSegment() { + if (text_segment_ == nullptr) { + // Make sure it didn't get added outside this method. + ASSERT(FindSegment(mach_o::SEG_TEXT) == nullptr); + auto const vm_protection = mach_o::VM_PROT_READ | mach_o::VM_PROT_EXECUTE; + text_segment_ = new (zone()) + MachOSegment(zone(), mach_o::SEG_TEXT, vm_protection, vm_protection); + commands_.Add(text_segment_); + } + return text_segment_; + } + + void Finalize(); + + void Accept(Visitor* visitor) override { visitor->VisitMachOHeader(this); } + + // Since the header is in the initial segment, visiting the load commands + // here and also visiting the header in MachOSegment::VisitChildren() would + // cause a cycle if, say, Default() is overridden to be recursive. + // Thus, the default VisitChildren implementation here does no recursion, + void VisitChildren(Visitor* visitor) override {} + void VisitSegments(Visitor* visitor) { + for (auto* const c : commands_) { + if (!c->IsMachOSegment()) continue; + c->Accept(visitor); + } + } + + private: + void GenerateUuid(); + void CreateBSS(); + void GenerateMiscellaneousCommands(); + void InitializeSymbolTables(); + void FinalizeDwarfSections(); + void FinalizeCommands(); + void ComputeOffsets(); + + // Returns the symbol table that is included in the output, which + // may or may not be the full symbol table. + // + // Returns nullptr if called before symbol table initialization. + MachOSymbolTable* IncludedSymbolTable() { + // True when the symbol tables haven't been initialized. + if (full_symtab_.symbols().is_empty()) return nullptr; + // The full symbol table is reused for unstripped contents. + if (!is_stripped_) return &full_symtab_; + return FindCommand(); + } + + Zone* const zone_; + const SnapshotType type_; + // Used to determine whether to include non-global symbols in the + // symbol table written to disk. + bool const is_stripped_; + // The identifier, used in the LC_ID_DYLIB command and the code signature. + const char* const identifier_; + Dwarf* const dwarf_; + GrowableArray commands_; + // Contains all symbols for relocation calculations. + MachOSymbolTable full_symtab_; + MachOSegment* text_segment_ = nullptr; + + DISALLOW_COPY_AND_ASSIGN(MachOHeader); +}; + +void MachOSegment::AddContents(MachOContents* c) { + ASSERT(c != nullptr); + // Segment contents are always allocated. + ASSERT(c->IsAllocated()); + // The order of segment contents is as follows: + // 1) The header (if this is the initial segment). + // 2) Content-containing sections and commands (in the linkedit segment). + // 3) Sections without contents like zerofill sections. + if (c->IsMachOHeader()) { + ASSERT(c->AsMachOHeader()->commands()[0] == this); + contents_.InsertAt(0, c); + next_contents_index_ += 1; + } else if (c->HasContents()) { + ASSERT_EQUAL(c->IsMachOCommand(), HasName(mach_o::SEG_LINKEDIT)); + contents_.InsertAt(next_contents_index_, c); + next_contents_index_ += 1; + } else { + ASSERT(c->IsMachOSection()); + contents_.Add(c); + } +} + +bool MachOWriteStream::HasValueForLabel(intptr_t label, intptr_t* value) const { + ASSERT(value != nullptr); + const auto& header = macho_.header(); + if (label == SharedObjectWriter::kBuildIdLabel) { + // Unlike ELF, the uuid is not in a MachO section and so can't have a symbol + // assigned. Instead, we look up its load command offset in the header. + auto* const uuid = header.FindCommand(); + if (uuid == nullptr) return false; + *value = header.file_offset() + uuid->header_offset(); + return true; + } + const auto& symtab = header.relocation_symbol_table(); + auto* const symbol = symtab.FindLabel(label); + if (symbol == nullptr) return false; + *value = symbol->value; + return true; +} + +const MachOSegment& MachOWriteStream::TextSegment() const { + return macho_.header().text_segment(); +} + +MachOWriter::MachOWriter(Zone* zone, + BaseWriteStream* stream, + Type type, + const char* id, + Dwarf* dwarf) + : SharedObjectWriter(zone, stream, type, dwarf), + header_(*new (zone) + MachOHeader(zone, type, IsStripped(dwarf), id, dwarf)) {} + +void MachOWriter::AddText(const char* name, + intptr_t label, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbols) { + auto* const text_segment = header_.EnsureTextSegment(); + auto* text_section = text_segment->FindSection(mach_o::SECT_TEXT); + if (text_section == nullptr) { + const bool has_contents = type_ == Type::Snapshot; + const intptr_t attributes = + mach_o::S_ATTR_PURE_INSTRUCTIONS | mach_o::S_ATTR_SOME_INSTRUCTIONS; + text_section = new (zone()) MachOSection( + zone(), mach_o::SECT_TEXT, mach_o::S_REGULAR, attributes, has_contents); + text_segment->AddContents(text_section); + } + text_section->AddPortion(bytes, size, relocations, symbols, name, label); +} + +void MachOWriter::AddROData(const char* name, + intptr_t label, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbols) { + // Const data goes in the text segment, not the data one. + auto* const text_segment = header_.EnsureTextSegment(); + auto* const_section = text_segment->FindSection(mach_o::SECT_CONST); + if (const_section == nullptr) { + const bool has_contents = type_ == Type::Snapshot; + const_section = + new (zone()) MachOSection(zone(), mach_o::SECT_CONST, mach_o::S_REGULAR, + mach_o::S_NO_ATTRIBUTES, has_contents); + text_segment->AddContents(const_section); + } + const_section->AddPortion(bytes, size, relocations, symbols, name, label); +} + +class WriteVisitor : public MachOContents::Visitor { + public: + explicit WriteVisitor(MachOWriteStream* stream) : stream_(stream) {} + + void Default(MachOContents* contents) override { + if (!contents->HasContents()) return; + stream_->Align(contents->Alignment()); + const intptr_t start = stream_->Position(); + ASSERT_EQUAL(start, contents->file_offset()); + contents->WriteSelf(stream_); + ASSERT_EQUAL(stream_->Position() - start, contents->SelfFileSize()); + contents->VisitChildren(this); + // Segments include post-nested content alignment. + if (auto* const s = contents->AsMachOSegment()) { + if (s->PadFileSizeToAlignment()) { + stream_->Align(contents->Alignment()); + } + } + ASSERT_EQUAL(stream_->Position() - start, contents->FileSize()); + } + + private: + MachOWriteStream* stream_; + DISALLOW_COPY_AND_ASSIGN(WriteVisitor); +}; + +void MachOWriter::Finalize() { + header_.Finalize(); + if (header_.HasCommand(MachOCodeSignature::kCommandCode)) { + HashingMachOWriteStream wrapped(zone_, unwrapped_stream_, *this); + WriteVisitor visitor(&wrapped); + header_.VisitSegments(&visitor); + } else { + NonHashingMachOWriteStream wrapped(unwrapped_stream_, *this); + WriteVisitor visitor(&wrapped); + header_.VisitSegments(&visitor); + } +} + +void MachOHeader::Finalize() { + // Generate the UUID now that we have all user-provided sections. + GenerateUuid(); + + // We add a BSS section for all Mach-O output with text sections, even in + // the separate debugging information, to ensure that relocated addresses + // are consistent between snapshots and the corresponding separate + // debugging information. + CreateBSS(); + + FinalizeDwarfSections(); + + // Create and initialize the dynamic and static symbol tables. + InitializeSymbolTables(); + + // Generate miscellenous load commands needed for the final output. + GenerateMiscellaneousCommands(); + + // Reorders the added commands as well as adding segments and commands + // that must appear at the end of the file. + FinalizeCommands(); + + // Calculate file and memory offsets, and finalizes symbol values in any + // symbol tables. + ComputeOffsets(); +} + +void MachOWriter::AssertConsistency(const MachOWriter* snapshot, + const MachOWriter* debug_info) { +#if defined(DEBUG) + // For now, just check that the symbol information for both match + // in that all labelled symbols used for relocation have the same + // value. + const auto& snapshot_symtab = snapshot->header().relocation_symbol_table(); + const auto& debug_info_symtab = + debug_info->header().relocation_symbol_table(); + + intptr_t max_label = snapshot_symtab.max_label(); + ASSERT_EQUAL(max_label, debug_info_symtab.max_label()); + for (intptr_t i = 1; i < max_label; ++i) { + if (auto* const snapshot_symbol = snapshot_symtab.FindLabel(i)) { + auto* const debug_info_symbol = debug_info_symtab.FindLabel(i); + ASSERT(debug_info_symbol != nullptr); + if (snapshot_symbol->value != debug_info_symbol->value) { + FATAL("Snapshot: %s -> %" Px64 ", %s -> %" Px64 "", + snapshot_symtab.strings().At(snapshot_symbol->name_index), + static_cast(snapshot_symbol->value), + debug_info_symtab.strings().At(debug_info_symbol->name_index), + static_cast(debug_info_symbol->value)); + } + } else { + ASSERT(debug_info_symtab.FindLabel(i) == nullptr); + } + } +#endif +} + +static uint32_t HashPortion(const MachOSection::Portion& portion) { + if (portion.bytes == nullptr) return 0; + const uint32_t hash = Utils::StringHash(portion.bytes, portion.size); + // Ensure a non-zero return. + return hash == 0 ? 1 : hash; +} + +// For the UUID, we generate a 128-bit hash, where each 32 bits is a +// hash of the contents of the following segments in order: +// +// .text(VM) | .text(Isolate) | .rodata(VM) | .rodata(Isolate) +// +// Any component of the build ID which does not have an associated section +// in the output is kept as 0. +void MachOHeader::GenerateUuid() { + // Not idempotent. + ASSERT(!HasCommand(MachOUuid::kCommandCode)); + // Currently, we construct the UUID out of data from two different + // sections in the text segment: the text section and the const section. + auto* const text_segment = FindSegment(mach_o::SEG_TEXT); + if (text_segment == nullptr) return; + + auto* const text_section = text_segment->FindSection(mach_o::SECT_TEXT); + // If there is no text section, then a UUID is not needed, as it is only + // used to symbolicize non-symbolic stack traces. + if (text_section == nullptr) return; + + auto* const vm_instructions = + text_section->FindPortion(kVmSnapshotInstructionsAsmSymbol); + auto* const isolate_instructions = + text_section->FindPortion(kIsolateSnapshotInstructionsAsmSymbol); + // All MachO snapshots have at least one of the two instruction sections. + ASSERT(vm_instructions != nullptr || isolate_instructions != nullptr); + + auto* const data_section = text_segment->FindSection(mach_o::SECT_CONST); + auto* const vm_data = + data_section == nullptr + ? nullptr + : data_section->FindPortion(kVmSnapshotDataAsmSymbol); + auto* const isolate_data = + data_section == nullptr + ? nullptr + : data_section->FindPortion(kIsolateSnapshotDataAsmSymbol); + + uint32_t hashes[4]; + hashes[0] = vm_instructions == nullptr ? 0 : HashPortion(*vm_instructions); + hashes[1] = + isolate_instructions == nullptr ? 0 : HashPortion(*isolate_instructions); + hashes[2] = vm_data == nullptr ? 0 : HashPortion(*vm_data); + hashes[3] = isolate_data == nullptr ? 0 : HashPortion(*isolate_data); + + auto* const uuid_command = new (zone()) MachOUuid(hashes, sizeof(hashes)); + commands_.Add(uuid_command); +} + +void MachOHeader::CreateBSS() { + // No text section means no BSS section. + auto* const text_section = FindSection(mach_o::SEG_TEXT, mach_o::SECT_TEXT); + ASSERT(text_section != nullptr); + + // Not idempotent. Currently the data segment only contains BSS data, so it + // shouldn't already exist. + ASSERT(FindSegment(mach_o::SEG_DATA) == nullptr); + auto const vm_protection = mach_o::VM_PROT_READ | mach_o::VM_PROT_WRITE; + auto* const data_segment = new (zone()) + MachOSegment(zone(), mach_o::SEG_DATA, vm_protection, vm_protection); + commands_.Add(data_segment); + + auto* const bss_section = + new (zone()) MachOSection(zone(), mach_o::SECT_BSS, mach_o::S_ZEROFILL, + mach_o::S_NO_ATTRIBUTES, /*has_contents=*/false, + /*alignment=*/compiler::target::kWordSize); + data_segment->AddContents(bss_section); + + for (const auto& portion : text_section->portions()) { + size_t size; + const char* symbol_name; + intptr_t label; + // First determine whether this is the VM's text portion or the isolate's. + if (strcmp(portion.symbol_name, kVmSnapshotInstructionsAsmSymbol) == 0) { + size = BSS::kVmEntryCount * compiler::target::kWordSize; + symbol_name = kVmSnapshotBssAsmSymbol; + label = SharedObjectWriter::kVmBssLabel; + } else if (strcmp(portion.symbol_name, + kIsolateSnapshotInstructionsAsmSymbol) == 0) { + size = BSS::kIsolateGroupEntryCount * compiler::target::kWordSize; + symbol_name = kIsolateSnapshotBssAsmSymbol; + label = SharedObjectWriter::kIsolateBssLabel; + } else { + // Not VM or isolate text. + UNREACHABLE(); + } + + // For the BSS section, we add the section symbols as local symbols in the + // static symbol table, as these addresses are only used for relocation. + // (This matches the behavior in the assembly output.) + auto* symbols = new (zone_) SharedObjectWriter::SymbolDataArray(zone_, 1); + symbols->Add({symbol_name, SharedObjectWriter::SymbolData::Type::Section, 0, + size, label}); + bss_section->AddPortion(/*bytes=*/nullptr, size, /*relocations=*/nullptr, + symbols); + } +} + +void MachOHeader::GenerateMiscellaneousCommands() { + // Not idempotent; + ASSERT(!HasCommand(MachOBuildVersion::kCommandCode)); + ASSERT(!HasCommand(MachOIdDylib::kCommandCode)); + ASSERT(!HasCommand(MachOLoadDylib::kCommandCode)); + + commands_.Add(new (zone_) MachOBuildVersion()); + if (type_ == SnapshotType::Snapshot) { + commands_.Add(new (zone_) MachOIdDylib(identifier_)); +#if defined(DART_TARGET_OS_MACOS) || defined(DART_TARGET_OS_MACOS_IOS) + commands_.Add(MachOLoadDylib::CreateLoadSystemDylib(zone_)); +#endif + } +} + +void MachOHeader::InitializeSymbolTables() { + // Not idempotent. + ASSERT_EQUAL(full_symtab_.num_symbols(), 0); + ASSERT(!HasCommand(MachOSymbolTable::kCommandCode)); + + // Grab all the sections in order. + GrowableArray sections(zone_, 0); + for (auto* const command : commands_) { + // Should be run before ComputeOffsets. + ASSERT(!command->HasContents() || !command->file_offset_is_set()); + if (auto* const s = command->AsMachOSegment()) { + for (auto* const c : s->contents()) { + if (auto* const section = c->AsMachOSection()) { + sections.Add(section); + } + } + } + } + + // This symbol table is for the MachOWriter's internal use. All symbols + // should be added to it so the writer can resolve relocations. + full_symtab_.Initialize(sections, /*is_stripped=*/false); + auto* table = &full_symtab_; + if (is_stripped_) { + // Create a separate symbol table that is actually written to the output. + // This one will only contain what's needed for the dynamic symbol table. + auto* const table = new (zone()) MachOSymbolTable(zone()); + table->Initialize(sections, is_stripped_); + } + commands_.Add(table); + + // For snapshots, include a dynamic symbol table as well. + if (type_ == SnapshotType::Snapshot) { + auto* const dynamic_symtab = new (zone()) MachODynamicSymbolTable(*table); + commands_.Add(dynamic_symtab); + } +} + +void MachOHeader::FinalizeDwarfSections() { + if (dwarf_ == nullptr) return; + + // Currently we only output DWARF information involving code. +#if defined(DEBUG) + auto* const text_segment = FindSegment(mach_o::SEG_TEXT); + ASSERT(text_segment != nullptr); + ASSERT(text_segment->FindSection(mach_o::SECT_TEXT) != nullptr); +#endif + + // Create the DWARF segment, which should not already exist. + ASSERT(FindSegment(mach_o::SEG_DWARF) == nullptr); + auto const init_vm_protection = mach_o::VM_PROT_READ | mach_o::VM_PROT_WRITE; + auto const max_vm_protection = init_vm_protection | mach_o::VM_PROT_EXECUTE; + auto* const dwarf_segment = new (zone()) MachOSegment( + zone(), mach_o::SEG_DWARF, init_vm_protection, max_vm_protection); + commands_.Add(dwarf_segment); + + const intptr_t alignment = 1; // No extra padding. + auto add_debug = [&](const char* name, + const DwarfSharedObjectStream& stream) { + auto* const section = new (zone()) + MachOSection(zone(), name, mach_o::S_REGULAR, mach_o::S_ATTR_DEBUG, + /*has_contents=*/true, alignment); + section->AddPortion(stream.buffer(), stream.bytes_written(), + stream.relocations()); + dwarf_segment->AddContents(section); + }; + + { + ZoneWriteStream stream(zone(), DwarfSharedObjectStream::kInitialBufferSize); + DwarfSharedObjectStream dwarf_stream(zone_, &stream); + dwarf_->WriteAbbreviations(&dwarf_stream); + add_debug(mach_o::SECT_DEBUG_ABBREV, dwarf_stream); + } + + { + ZoneWriteStream stream(zone(), DwarfSharedObjectStream::kInitialBufferSize); + DwarfSharedObjectStream dwarf_stream(zone_, &stream); + dwarf_->WriteDebugInfo(&dwarf_stream); + add_debug(mach_o::SECT_DEBUG_INFO, dwarf_stream); + } + + { + ZoneWriteStream stream(zone(), DwarfSharedObjectStream::kInitialBufferSize); + DwarfSharedObjectStream dwarf_stream(zone_, &stream); + dwarf_->WriteLineNumberProgram(&dwarf_stream); + add_debug(mach_o::SECT_DEBUG_LINE, dwarf_stream); + } +} + +void MachOHeader::FinalizeCommands() { + // Not idempotent. + ASSERT(FindSegment(mach_o::SEG_LINKEDIT) == nullptr); + ASSERT(!HasCommand(MachOCodeSignature::kCommandCode)); + + intptr_t num_commands = commands_.length(); + // We shouldn't be writing empty Mach-O snapshots. + ASSERT(num_commands != 0); + GrowableArray reordered_commands(zone_, num_commands); + + // Now do a single pass over the commands, sorting them into bins based on + // the desired final ordering and also calculating a map from old section + // indices in the old order to new section indices in the new order. + + // First, any commands that are only part of the header. + GrowableArray header_only_commands(zone_, 0); + + // Ensure the text segment is the initial segment. This means the + // text segment contains the header in its file contents/memory space. + MachOSegment* text_segment = text_segment_; + // We should be writing instructions and/or const data. + ASSERT(text_segment != nullptr); + + // Then all segments that have defined symbols. These segments + // are present in both snapshots and separate debugging information, + // and the symbols defined in these sections should have consistent + // relocated memory addresses in both. + GrowableArray symbol_segments(zone_, 0); + + // Then all other segments added prior to calling this function. + // These need to be before the linkedit segment, which is created + // below, so that they are also protected by the code signature + // (if there is one). + GrowableArray other_segments(zone_, 0); + + // Next comes any non-segment load commands that have allocated content + // outside of the header like the symbol table. A linkedit segment + // is created later to contain the non-header contents of these commands. + GrowableArray linkedit_commands(zone_, 0); + + // Maps segments to the section count and old initial section index for + // that segment. (Sections are not reordered during this, so this is + // all that's needed to calculate new section indices.) + using SegmentMapTrait = + RawPointerKeyValueTrait>; + DirectChainedHashMap section_info(zone_, num_commands); + intptr_t num_sections = 0; + for (auto* const command : commands_) { + // Check that we're not reordering after offsets have been computed. + ASSERT(!command->HasContents() || !command->file_offset_is_set()); + if (auto* const s = command->AsMachOSegment()) { + const intptr_t count = s->NumSections(); + if (count != 0) { + // Section indices start from 1. + section_info.Insert({s, {count, num_sections + 1}}); + num_sections += count; + } + if (s->HasName(mach_o::SEG_TEXT)) { + ASSERT(text_segment == s); + } else if (s->ContainsSymbols()) { + symbol_segments.Add(s); + } else { + other_segments.Add(s); + } + } else if (!command->HasContents()) { + header_only_commands.Add(command); + } else { + linkedit_commands.Add(command); + } + } + + // We should always have a symbol table, even in stripped files where + // it only contains global exported symbols, which means there should + // be a linkedit segment. + ASSERT(!linkedit_commands.is_empty()); + auto* const linkedit_segment = + new (zone_) MachOSegment(zone_, mach_o::SEG_LINKEDIT); + num_commands += 1; + for (auto* const c : linkedit_commands) { + linkedit_segment->AddContents(c); + } + if (type_ == SnapshotType::Snapshot) { + // Also include an embedded ad-hoc linker signed code signature as the + // last contents of the linkedit segment (which is the last segment). + auto* const signature = new (zone_) MachOCodeSignature(identifier_); + linkedit_segment->AddContents(signature); + linkedit_commands.Add(signature); + num_commands += 1; + } + + GrowableArray segments( + zone_, symbol_segments.length() + other_segments.length() + 2); + // Put the text, data, and linkedit segments in the expected ordering. + segments.Add(text_segment); + segments.AddArray(symbol_segments); + segments.AddArray(other_segments); + segments.Add(linkedit_segment); + + // The initial segment in the file should have the header as its initial + // contents. Since the header is not a section, this won't change the + // section numbering. + segments[0]->AddContents(this); + + // Now populate reordered_commands. + reordered_commands.AddArray(header_only_commands); + + // While adding segments, also map old section indices to new ones. Include + // a map of mach_o::NO_SECT to mach_o::NO_SECT so that changing the section + // index on a non-section symbol is a no-op. + GrowableArray index_map(zone_, num_sections + 1); + index_map.FillWith(mach_o::NO_SECT, 0, num_sections + 1); + // Section indices start from 1. + intptr_t current_section_index = 1; + for (auto* const s : segments) { + reordered_commands.Add(s); + auto* const kv = section_info.Lookup(s); + if (kv != nullptr) { + const auto& [num_sections, old_start] = SegmentMapTrait::ValueOf(*kv); + ASSERT(num_sections > 0); // Otherwise it's not in the map. + ASSERT(old_start != mach_o::NO_SECT); + for (intptr_t i = 0; i < num_sections; ++i) { + ASSERT(current_section_index != mach_o::NO_SECT); + index_map[old_start + i] = current_section_index++; + } + } + } + reordered_commands.AddArray(linkedit_commands); + + // All sections should have been accounted for in the loops above as well as + // the new linkedit segment (and, if applicable, the code signature). + ASSERT_EQUAL(reordered_commands.length(), num_commands); + // Replace the content of commands_ with the reordered commands. + commands_.Clear(); + commands_.AddArray(reordered_commands); + + // This must be true for uses of the map to be correct. + ASSERT_EQUAL(index_map[mach_o::NO_SECT], mach_o::NO_SECT); +#if defined(DEBUG) + for (intptr_t i = 1; i < num_sections; ++i) { + ASSERT(index_map[i] != mach_o::NO_SECT); + } +#endif + + // Update the section indices of any section-owned symbols. + full_symtab_.UpdateSectionIndices(index_map); + auto* const table = IncludedSymbolTable(); + if (table != &full_symtab_) { + ASSERT(is_stripped_); + table->UpdateSectionIndices(index_map); + } +} + +struct ContentOffsetsVisitor : public MachOContents::Visitor { + explicit ContentOffsetsVisitor(Zone* zone) : address_map(zone, 1) { + // Add NO_SECT -> 0 mapping. + address_map.Add(0); + } + + void Default(MachOContents* contents) { + ASSERT_EQUAL(contents->IsMachOHeader(), file_offset == 0); + ASSERT_EQUAL(contents->IsMachOHeader(), memory_address == 0); + // Increment the file and memory offsets by the appropriate amounts. + if (contents->HasContents()) { + file_offset = Utils::RoundUp(file_offset, contents->Alignment()); + contents->set_file_offset(file_offset); + file_offset += contents->SelfFileSize(); + } + if (contents->IsAllocated()) { + memory_address = Utils::RoundUp(memory_address, contents->Alignment()); + contents->set_memory_address(memory_address); + memory_address += contents->SelfMemorySize(); + } + contents->VisitChildren(this); + if (contents->HasContents()) { + ASSERT_EQUAL(file_offset, contents->file_offset() + contents->FileSize()); + } + if (contents->IsAllocated()) { + ASSERT_EQUAL(memory_address, + contents->memory_address() + contents->MemorySize()); + } + } + + void VisitMachOSegment(MachOSegment* segment) { + ASSERT_EQUAL(segment->IsInitial(), file_offset == 0); + ASSERT_EQUAL(segment->IsInitial(), memory_address == 0); + // Segments are always allocated and we set the file offset even + // when the segment doesn't actually write any contents. + file_offset = Utils::RoundUp(file_offset, segment->Alignment()); + segment->set_file_offset(file_offset); + file_offset += segment->SelfFileSize(); + memory_address = Utils::RoundUp(memory_address, segment->Alignment()); + segment->set_memory_address(memory_address); + memory_address += segment->SelfMemorySize(); + segment->VisitChildren(this); + if (segment->PadFileSizeToAlignment()) { + file_offset = Utils::RoundUp(file_offset, segment->Alignment()); + } + memory_address = Utils::RoundUp(memory_address, segment->Alignment()); + ASSERT_EQUAL(file_offset, segment->file_offset() + segment->FileSize()); + ASSERT_EQUAL(memory_address, + segment->memory_address() + segment->MemorySize()); + } + + void VisitMachOSection(MachOSection* section) { + // Sections do not contain other sections, so the visitor can use the + // default behavior without worrying about adding to the address map in + // the wrong order. + Visitor::VisitMachOSection(section); + address_map.Add(section->memory_address()); + } + + // Maps indices of allocated sections in the section table to memory offsets. + // Note that sections are 1-indexed, with 0 (NO_SECT) mapping to 0. + GrowableArray address_map; + intptr_t file_offset = 0; + intptr_t memory_address = 0; + + private: + DISALLOW_COPY_AND_ASSIGN(ContentOffsetsVisitor); +}; + +void MachOHeader::ComputeOffsets() { + intptr_t header_offset = SizeWithoutLoadCommands(); + for (auto* const c : commands_) { + ASSERT( + Utils::IsAligned(header_offset, MachOCommand::kLoadCommandAlignment)); + c->set_header_offset(header_offset); + header_offset += c->cmdsize(); + } + + ContentOffsetsVisitor visitor(zone()); + // All commands with non-header content should be part of a segment. + // In addition, the header is visited during the initial segment. + VisitSegments(&visitor); + + // Finalize the dynamic symbol table, now that the file offset for the + // symbol table has been calculated. + + // Entry for NO_SECT + 1-indexed entries for sections. + ASSERT_EQUAL(visitor.address_map.length(), NumSections() + 1); + + // Adjust addresses in symbol tables as we now have section memory offsets. + full_symtab_.Finalize(visitor.address_map); + auto* const table = IncludedSymbolTable(); + if (table != &full_symtab_) { + ASSERT(is_stripped_); + table->Finalize(visitor.address_map); + } +} + +void MachOSymbolTable::Initialize(const GrowableArray& sections, + bool is_stripped) { + // Not idempotent. + ASSERT(!num_local_symbols_is_set()); + + // If symbolic debugging symbols are emitted, then any section + // symbols are marked as alternate entries in favor of the symbolic + // debugging symbols. + const intptr_t desc = is_stripped ? 0 : mach_o::N_ALT_ENTRY; + + // For unstripped symbol tables, we do two initial passes. In the first + // pass, we add section symbols for local static symbols. + if (!is_stripped) { + for (intptr_t i = 0, n = sections.length(); i < n; ++i) { + auto* const section = sections[i]; + const intptr_t section_index = i + 1; // 1-indexed, as 0 is NO_SECT. + for (const auto& portion : section->portions()) { + if (portion.symbols != nullptr) { + for (const auto& symbol_data : *portion.symbols) { + AddSymbol(symbol_data.name, mach_o::N_SECT, section_index, desc, + portion.offset + symbol_data.offset, symbol_data.label); + } + } + } + } + + // In the second pass, we add appropriate symbolic debugging symbols. + using Type = SharedObjectWriter::SymbolData::Type; + auto add_symbolic_debugging_symbols = + [&](const char* name, Type type, intptr_t section_index, + intptr_t offset, intptr_t size, bool is_global) { + switch (type) { + case Type::Function: { + AddSymbol("", mach_o::N_BNSYM, section_index, /*desc=*/0, offset); + AddSymbol(name, mach_o::N_FUN, section_index, /*desc=*/0, offset); + // The size is output as an unnamed N_FUN symbol with no section + // following the actual N_FUN symbol. + AddSymbol("", mach_o::N_FUN, mach_o::NO_SECT, /*desc=*/0, size); + AddSymbol("", mach_o::N_ENSYM, section_index, /*desc=*/0, + offset + size); + + break; + } + case Type::Section: + case Type::Object: { + if (is_global) { + AddSymbol(name, mach_o::N_GSYM, mach_o::NO_SECT, /*desc=*/0, + /*value=*/0); + } else { + AddSymbol(name, mach_o::N_STSYM, section_index, + /*desc=*/0, offset); + } + break; + } + } + }; + + for (intptr_t i = 0, n = sections.length(); i < n; ++i) { + auto* const section = sections[i]; + const intptr_t section_index = i + 1; // 1-indexed, as 0 is NO_SECT. + // We handle global symbols for text sections slightly differently than + // those for other sections. + const bool is_text_section = section->HasName(mach_o::SECT_TEXT); + for (const auto& portion : section->portions()) { + if (portion.symbol_name != nullptr) { + // Matching the symbolic debugging symbols created for assembled + // snapshots. + auto const type = is_text_section ? Type::Function : Type::Section; + // The "size" of a function symbol created for start of a text portion + // is up to the first function symbol. + auto const size = is_text_section && portion.symbols != nullptr + ? portion.symbols->At(0).offset + : portion.size; + add_symbolic_debugging_symbols(portion.symbol_name, type, + section_index, portion.offset, size, + /*is_global=*/true); + } + if (portion.symbols != nullptr) { + for (const auto& symbol_data : *portion.symbols) { + add_symbolic_debugging_symbols( + symbol_data.name, symbol_data.type, section_index, + portion.offset + symbol_data.offset, symbol_data.size, + /*is_global=*/false); + } + } + } + } + } + set_num_local_symbols(num_symbols()); + + // In the final pass, we add external symbols for section global symbols + // (so added to both stripped and unstripped symbol tables). + for (intptr_t i = 0, n = sections.length(); i < n; ++i) { + auto* const section = sections[i]; + const intptr_t section_index = i + 1; // 1-indexed, as 0 is NO_SECT. + for (const auto& portion : section->portions()) { + if (portion.symbol_name != nullptr) { + AddSymbol(portion.symbol_name, mach_o::N_SECT | mach_o::N_EXT, + section_index, desc, portion.offset, portion.label); + } + } + } + set_num_external_symbols(num_symbols() - num_local_symbols()); +} + +} // namespace dart + +#endif // defined(DART_PRECOMPILER) diff --git a/runtime/vm/mach_o.h b/runtime/vm/mach_o.h new file mode 100644 index 00000000000..f652f97c850 --- /dev/null +++ b/runtime/vm/mach_o.h @@ -0,0 +1,79 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_VM_MACH_O_H_ +#define RUNTIME_VM_MACH_O_H_ + +#include "platform/globals.h" + +#if defined(DART_PRECOMPILER) + +#include "vm/allocation.h" +#include "vm/compiler/runtime_api.h" +#include "vm/datastream.h" +#include "vm/growable_array.h" +#include "vm/so_writer.h" +#include "vm/zone.h" + +namespace dart { + +class MachOHeader; +class MachOSymbolTable; +class MachOWriteStream; + +class MachOWriter : public SharedObjectWriter { + public: + MachOWriter(Zone* zone, + BaseWriteStream* stream, + Type type, + const char* id, + Dwarf* dwarf = nullptr); + +#if defined(TARGET_ARCH_ARM64) + static constexpr intptr_t kPageSize = 16 * KB; +#else + static constexpr intptr_t kPageSize = 4 * KB; +#endif + intptr_t page_size() const override { return kPageSize; } + + Output output() const override { return Output::MachO; } + const MachOHeader& header() const { return header_; } + + void AddText(const char* name, + intptr_t label, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbol) override; + void AddROData(const char* name, + intptr_t label, + const uint8_t* bytes, + intptr_t size, + const ZoneGrowableArray* relocations, + const ZoneGrowableArray* symbols) override; + + void Finalize() override; + + void AssertConsistency(const SharedObjectWriter* debug) const override { + if (auto* const debug_macho = debug->AsMachOWriter()) { + AssertConsistency(this, debug_macho); + } else { + FATAL("Expected both snapshot and debug to be MachO"); + } + } + + const MachOWriter* AsMachOWriter() const override { return this; } + + private: + static void AssertConsistency(const MachOWriter* snapshot, + const MachOWriter* debug_info); + + MachOHeader& header_; +}; + +} // namespace dart + +#endif // DART_PRECOMPILER + +#endif // RUNTIME_VM_MACH_O_H_ diff --git a/runtime/vm/os_macos.cc b/runtime/vm/os_macos.cc index 26ba160f3f6..4a84c7c2650 100644 --- a/runtime/vm/os_macos.cc +++ b/runtime/vm/os_macos.cc @@ -304,7 +304,9 @@ OS::BuildId OS::GetAppBuildId(const uint8_t* snapshot_instructions) { const uint8_t* dso_base = GetAppDSOBase(snapshot_instructions); const auto& macho_header = *reinterpret_cast(dso_base); - // We assume host endianness in the Mach-O file. + // If the Mach-O file is not host endian, then we'd need to adjust the code + // below (and also the snapshot loading code) to load multibyte integers + // as reverse endian. if (macho_header.magic != MH_MAGIC && macho_header.magic != MH_MAGIC_64) { return {0, nullptr}; } diff --git a/runtime/vm/timeline.cc b/runtime/vm/timeline.cc index 8e010f3d6bb..8b499d7a67a 100644 --- a/runtime/vm/timeline.cc +++ b/runtime/vm/timeline.cc @@ -16,6 +16,10 @@ #include #include +#if defined(DART_HOST_OS_MACOS) +#include +#endif + #include "platform/atomic.h" #include "platform/hashmap.h" #include "vm/isolate.h" diff --git a/runtime/vm/timeline.h b/runtime/vm/timeline.h index f22a1890877..f73ecb8303b 100644 --- a/runtime/vm/timeline.h +++ b/runtime/vm/timeline.h @@ -29,10 +29,17 @@ #if defined(FUCHSIA_SDK) || defined(DART_HOST_OS_FUCHSIA) #include #include -#elif defined(DART_HOST_OS_MACOS) -#include #endif // defined(FUCHSIA_SDK) || defined(DART_HOST_OS_FUCHSIA) +#if defined(DART_HOST_OS_MACOS) +// Including in this header leads to an include of +// , which causes files that use this header and the +// definitions in "platform/mach_o.h" to fail to build. To avoid this, +// duplicate the typedef that creates here and only include +// in timeline.cc and timeline_macos.cc. +typedef struct os_log_s* os_log_t; +#endif + namespace dart { #if !defined(SUPPORT_TIMELINE) diff --git a/runtime/vm/timeline_macos.cc b/runtime/vm/timeline_macos.cc index ee77188af0a..7972e4b1aad 100644 --- a/runtime/vm/timeline_macos.cc +++ b/runtime/vm/timeline_macos.cc @@ -5,6 +5,8 @@ #include "vm/globals.h" #if defined(DART_HOST_OS_MACOS) && defined(SUPPORT_TIMELINE) +#include + #include "vm/log.h" #include "vm/timeline.h" diff --git a/runtime/vm/vm_sources.gni b/runtime/vm/vm_sources.gni index dc2209fc725..7e74862bb7c 100644 --- a/runtime/vm/vm_sources.gni +++ b/runtime/vm/vm_sources.gni @@ -166,6 +166,8 @@ vm_sources = [ "log.h", "longjump.cc", "longjump.h", + "mach_o.cc", + "mach_o.h", "megamorphic_cache_table.cc", "megamorphic_cache_table.h", "memory_region.cc",