From 98b286de216b7426fdefa0dabeafed48ebedecc2 Mon Sep 17 00:00:00 2001 From: Samir Jindel Date: Wed, 2 Oct 2019 13:42:39 +0000 Subject: [PATCH] [vm] Use ELF loader in dart2native. Change-Id: I58834e3626cb18830ad9f9b1b95bdfa9d1ebd040 Cq-Include-Trybots:luci.dart.try:vm-kernel-precomp-linux-debug-x64-try,vm-kernel-precomp-linux-release-simarm-try Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119150 Commit-Queue: Samir Jindel Reviewed-by: Clement Skau --- pkg/dart2native/lib/dart2native.dart | 6 +- pkg/dart2native/pubspec.yaml | 1 - runtime/bin/elf_loader.cc | 37 ++++++---- runtime/bin/elf_loader.h | 21 ++++-- runtime/bin/main.cc | 54 +++++++------- runtime/bin/snapshot_utils.cc | 105 +++++++++++++-------------- runtime/bin/snapshot_utils.h | 3 +- 7 files changed, 118 insertions(+), 109 deletions(-) diff --git a/pkg/dart2native/lib/dart2native.dart b/pkg/dart2native/lib/dart2native.dart index 754fec9022c..78068611b91 100644 --- a/pkg/dart2native/lib/dart2native.dart +++ b/pkg/dart2native/lib/dart2native.dart @@ -5,8 +5,6 @@ import 'dart:io'; import 'dart:typed_data'; -import 'package:path/path.dart' as path; - const appSnapshotPageSize = 4096; const appjitMagicNumber = [0xdc, 0xdc, 0xf6, 0xf6, 0, 0, 0, 0]; @@ -64,8 +62,8 @@ Future generateAotKernel( Future generateAotSnapshot(String genSnapshot, String kernelFile, String snapshotFile, bool enableAsserts) { return Process.run(genSnapshot, [ - '--snapshot-kind=app-aot-blobs', - '--blobs_container_filename=${snapshotFile}', + '--snapshot-kind=app-aot-elf', + '--elf=${snapshotFile}', if (enableAsserts) '--enable-asserts', kernelFile ]); diff --git a/pkg/dart2native/pubspec.yaml b/pkg/dart2native/pubspec.yaml index be267fc4206..99e6f947542 100644 --- a/pkg/dart2native/pubspec.yaml +++ b/pkg/dart2native/pubspec.yaml @@ -10,7 +10,6 @@ executables: dependencies: args: ^1.4.0 - path: dev_dependencies: diff --git a/runtime/bin/elf_loader.cc b/runtime/bin/elf_loader.cc index b74fe3a7ca0..30210d4c95e 100644 --- a/runtime/bin/elf_loader.cc +++ b/runtime/bin/elf_loader.cc @@ -20,8 +20,9 @@ namespace elf { /// Dart_CreateAppAOTSnapshotAsElf. class LoadedElf { public: - explicit LoadedElf(const char* filename) - : filename_(strdup(filename), std::free) {} + explicit LoadedElf(const char* filename, uint64_t elf_data_offset) + : filename_(strdup(filename), std::free), + elf_data_offset_(elf_data_offset) {} ~LoadedElf(); /// Loads the ELF object into memory. Returns whether the load was successful. @@ -61,6 +62,7 @@ class LoadedElf { const void** mapping_start); std::unique_ptr filename_; + const uint64_t elf_data_offset_; // Initialized on a successful Load(). File* file_; @@ -119,9 +121,13 @@ bool LoadedElf::Load() { return false; } + CHECK_ERROR(Utils::IsAligned(elf_data_offset_, PageSize()), + "File offset must be page-aligned."); + file_ = File::Open(/*namespc=*/nullptr, filename_.get(), bin::File::FileOpenMode::kRead); CHECK_ERROR(file_ != nullptr, "Cannot open ELF object file."); + CHECK_ERROR(file_->SetPosition(elf_data_offset_), "Invalid file offset."); CHECK(ReadHeader()); CHECK(ReadProgramTable()); @@ -262,7 +268,7 @@ bool LoadedElf::LoadSegments() { void* const memory_start = static_cast(base_->address()) + memory_offset - adjustment; - const uword file_start = file_offset - adjustment; + const uword file_start = elf_data_offset_ + file_offset - adjustment; const uword length = header.memory_size + adjustment; File::MapType map_type = File::kReadOnly; @@ -350,9 +356,11 @@ bool LoadedElf::ResolveSymbols(const uint8_t** vm_data, MappedMemory* LoadedElf::MapFilePiece(uword file_start, uword file_length, const void** mem_start) { - const uword mapping_offset = Utils::RoundDown(file_start, PageSize()); + const uword adjustment = (elf_data_offset_ + file_start) % PageSize(); + const uword mapping_offset = elf_data_offset_ + file_start - adjustment; const uword mapping_length = - Utils::RoundUp(file_length + file_start % PageSize(), PageSize()); + Utils::RoundUp(elf_data_offset_ + file_start + file_length, PageSize()) - + mapping_offset; MappedMemory* const mapping = file_->Map(bin::File::kReadOnly, mapping_offset, mapping_length); @@ -368,14 +376,15 @@ MappedMemory* LoadedElf::MapFilePiece(uword file_start, } // namespace bin } // namespace dart -DART_EXPORT void* Dart_LoadELF(const char* filename, - const char** error, - const uint8_t** vm_snapshot_data, - const uint8_t** vm_snapshot_instrs, - const uint8_t** vm_isolate_data, - const uint8_t** vm_isolate_instrs) { +DART_EXPORT Dart_LoadedElf* Dart_LoadELF(const char* filename, + uint64_t file_offset, + const char** error, + const uint8_t** vm_snapshot_data, + const uint8_t** vm_snapshot_instrs, + const uint8_t** vm_isolate_data, + const uint8_t** vm_isolate_instrs) { std::unique_ptr elf( - new dart::bin::elf::LoadedElf(filename)); + new dart::bin::elf::LoadedElf(filename, file_offset)); if (!elf->Load() || !elf->ResolveSymbols(vm_snapshot_data, vm_snapshot_instrs, @@ -384,9 +393,9 @@ DART_EXPORT void* Dart_LoadELF(const char* filename, return nullptr; } - return elf.release(); + return reinterpret_cast(elf.release()); } -DART_EXPORT void Dart_UnloadELF(void* loaded) { +DART_EXPORT void Dart_UnloadELF(Dart_LoadedElf* loaded) { delete reinterpret_cast(loaded); } diff --git a/runtime/bin/elf_loader.h b/runtime/bin/elf_loader.h index b604ee63d42..c4094f979cd 100644 --- a/runtime/bin/elf_loader.h +++ b/runtime/bin/elf_loader.h @@ -7,7 +7,8 @@ #include -typedef void* LoadedElfLibrary; +typedef struct { +} Dart_LoadedElf; /// Loads an ELF object in 'filename'. /// @@ -15,16 +16,20 @@ typedef void* LoadedElfLibrary; /// in Dart_UnloadELF. On error, returns 'nullptr' and sets 'error'. The error /// string should not be 'free'-d. /// +/// `file_offset` may be non-zero to read an ELF object embedded inside another +/// type of file. +/// /// Looks up the Dart snapshot symbols "_kVmSnapshotData", /// "_kVmSnapshotInstructions", "_kVmIsoalteData" and "_kVmIsolateInstructions" /// into the respectively named out-parameters. -DART_EXPORT LoadedElfLibrary Dart_LoadELF(const char* filename, - const char** error, - const uint8_t** vm_snapshot_data, - const uint8_t** vm_snapshot_instrs, - const uint8_t** vm_isolate_data, - const uint8_t** vm_isolate_instrs); +DART_EXPORT Dart_LoadedElf* Dart_LoadELF(const char* filename, + uint64_t file_offset, + const char** error, + const uint8_t** vm_snapshot_data, + const uint8_t** vm_snapshot_instrs, + const uint8_t** vm_isolate_data, + const uint8_t** vm_isolate_instrs); -DART_EXPORT void Dart_UnloadELF(LoadedElfLibrary loaded); +DART_EXPORT void Dart_UnloadELF(Dart_LoadedElf* loaded); #endif // RUNTIME_BIN_ELF_LOADER_H_ diff --git a/runtime/bin/main.cc b/runtime/bin/main.cc index 483368c8749..fbc69b3f48f 100644 --- a/runtime/bin/main.cc +++ b/runtime/bin/main.cc @@ -1082,12 +1082,13 @@ void main(int argc, char** argv) { } vm_options.AddArgument("--new_gen_growth_factor=4"); + AppSnapshot* app_snapshot = nullptr; +#if defined(DART_PRECOMPILED_RUNTIME) // If the executable binary contains the runtime together with an appended // snapshot, load and run that. // Any arguments passed to such an executable are meant for the actual // application so skip all Dart VM flag parsing. - AppSnapshot* app_snapshot = - Snapshot::TryReadAppendedAppSnapshotBlobs(argv[0]); + app_snapshot = Snapshot::TryReadAppendedAppSnapshotElf(argv[0]); if (app_snapshot != nullptr) { script_name = argv[0]; @@ -1098,32 +1099,33 @@ void main(int argc, char** argv) { for (int i = 1; i < argc; i++) { dart_options.AddArgument(argv[i]); } - } else { - // Parse command line arguments. - if (Options::ParseArguments(argc, argv, vm_run_app_snapshot, &vm_options, - &script_name, &dart_options, &print_flags_seen, - &verbose_debug_seen) < 0) { - if (Options::help_option()) { - Options::PrintUsage(); - Platform::Exit(0); - } else if (Options::version_option()) { - Options::PrintVersion(); - Platform::Exit(0); - } else if (print_flags_seen) { - // Will set the VM flags, print them out and then we exit as no - // script was specified on the command line. - char* error = - Dart_SetVMFlags(vm_options.count(), vm_options.arguments()); - if (error != NULL) { - Syslog::PrintErr("Setting VM flags failed: %s\n", error); - free(error); - Platform::Exit(kErrorExitCode); - } - Platform::Exit(0); - } else { - Options::PrintUsage(); + } +#endif + + // Parse command line arguments. + if (app_snapshot == nullptr && + Options::ParseArguments(argc, argv, vm_run_app_snapshot, &vm_options, + &script_name, &dart_options, &print_flags_seen, + &verbose_debug_seen) < 0) { + if (Options::help_option()) { + Options::PrintUsage(); + Platform::Exit(0); + } else if (Options::version_option()) { + Options::PrintVersion(); + Platform::Exit(0); + } else if (print_flags_seen) { + // Will set the VM flags, print them out and then we exit as no + // script was specified on the command line. + char* error = Dart_SetVMFlags(vm_options.count(), vm_options.arguments()); + if (error != NULL) { + Syslog::PrintErr("Setting VM flags failed: %s\n", error); + free(error); Platform::Exit(kErrorExitCode); } + Platform::Exit(0); + } else { + Options::PrintUsage(); + Platform::Exit(kErrorExitCode); } } DartUtils::SetEnvironment(Options::environment()); diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc index aceba6c9aa1..1d63fe63682 100644 --- a/runtime/bin/snapshot_utils.cc +++ b/runtime/bin/snapshot_utils.cc @@ -158,41 +158,10 @@ static AppSnapshot* TryReadAppSnapshotBlobs(const char* script_name) { return TryReadAppSnapshotBlobs(script_name, file); } -AppSnapshot* Snapshot::TryReadAppendedAppSnapshotBlobs( - const char* container_path) { - File* file = File::Open(NULL, container_path, File::kRead); - if (file == nullptr) { - return nullptr; - } - RefCntReleaseScope rs(file); - - // Check for payload appended at the end of the container file. - // If header is found, jump to payload offset. - int64_t appended_header[2]; - if (!file->SetPosition(file->Length() - sizeof(appended_header))) { - return nullptr; - } - if (!file->ReadFully(&appended_header, sizeof(appended_header))) { - return nullptr; - } - // Length is always encoded as Little Endian. - const uint64_t appended_length = - Utils::LittleEndianToHost64(appended_header[0]); - if (memcmp(&appended_header[1], appjit_magic_number.bytes, - appjit_magic_number.length) != 0 || - appended_length <= 0 || !file->SetPosition(appended_length)) { - return nullptr; - } - - return TryReadAppSnapshotBlobs(container_path, file); -} - #if defined(DART_PRECOMPILED_RUNTIME) - -#if defined(TARGET_OS_WINDOWS) || defined(USING_SIMULATOR) class ElfAppSnapshot : public AppSnapshot { public: - ElfAppSnapshot(LoadedElfLibrary elf, + ElfAppSnapshot(Dart_LoadedElf* elf, const uint8_t* vm_snapshot_data, const uint8_t* vm_snapshot_instructions, const uint8_t* isolate_snapshot_data, @@ -216,13 +185,60 @@ class ElfAppSnapshot : public AppSnapshot { } private: - LoadedElfLibrary elf_; + Dart_LoadedElf* elf_; const uint8_t* vm_snapshot_data_; const uint8_t* vm_snapshot_instructions_; const uint8_t* isolate_snapshot_data_; const uint8_t* isolate_snapshot_instructions_; }; -#endif // defined(TARGET_OS_WINDOWS) || defined(USING_SIMULATOR) + +static AppSnapshot* TryReadAppSnapshotElf(const char* script_name, + uint64_t file_offset) { + const char* error = nullptr; + const uint8_t *vm_data_buffer = nullptr, *vm_instructions_buffer = nullptr, + *isolate_data_buffer = nullptr, + *isolate_instructions_buffer = nullptr; + Dart_LoadedElf* handle = + Dart_LoadELF(script_name, file_offset, &error, &vm_data_buffer, + &vm_instructions_buffer, &isolate_data_buffer, + &isolate_instructions_buffer); + if (handle == nullptr) { + Syslog::PrintErr("Loading failed: %s\n", error); + return nullptr; + } + return new ElfAppSnapshot(handle, vm_data_buffer, vm_instructions_buffer, + isolate_data_buffer, isolate_instructions_buffer); + return nullptr; +} + +AppSnapshot* Snapshot::TryReadAppendedAppSnapshotElf( + const char* container_path) { + File* file = File::Open(NULL, container_path, File::kRead); + if (file == nullptr) { + return nullptr; + } + RefCntReleaseScope rs(file); + + // Check for payload appended at the end of the container file. + // If header is found, jump to payload offset. + int64_t appended_header[2]; + if (!file->SetPosition(file->Length() - sizeof(appended_header))) { + return nullptr; + } + if (!file->ReadFully(&appended_header, sizeof(appended_header))) { + return nullptr; + } + // Length is always encoded as Little Endian. + const uint64_t appended_offset = + Utils::LittleEndianToHost64(appended_header[0]); + if (memcmp(&appended_header[1], appjit_magic_number.bytes, + appjit_magic_number.length) != 0 || + appended_offset <= 0) { + return nullptr; + } + + return TryReadAppSnapshotElf(container_path, appended_offset); +} class DylibAppSnapshot : public AppSnapshot { public: @@ -294,25 +310,6 @@ static AppSnapshot* TryReadAppSnapshotDynamicLibrary(const char* script_name) { isolate_data_buffer, isolate_instructions_buffer); } -static AppSnapshot* TryReadAppSnapshotElf(const char* script_name) { -#if defined(TARGET_OS_WINDOWS) || defined(USING_SIMULATOR) - const char* error = nullptr; - const uint8_t *vm_data_buffer = nullptr, *vm_instructions_buffer = nullptr, - *isolate_data_buffer = nullptr, - *isolate_instructions_buffer = nullptr; - void* handle = Dart_LoadELF(script_name, &error, &vm_data_buffer, - &vm_instructions_buffer, &isolate_data_buffer, - &isolate_instructions_buffer); - if (handle == nullptr) { - Syslog::PrintErr("Loading failed: %s\n", error); - return nullptr; - } - return new ElfAppSnapshot(handle, vm_data_buffer, vm_instructions_buffer, - isolate_data_buffer, isolate_instructions_buffer); -#else - return nullptr; -#endif -} #endif // defined(DART_PRECOMPILED_RUNTIME) AppSnapshot* Snapshot::TryReadAppSnapshot(const char* script_name) { @@ -343,7 +340,7 @@ AppSnapshot* Snapshot::TryReadAppSnapshot(const char* script_name) { return snapshot; } - snapshot = TryReadAppSnapshotElf(script_name); + snapshot = TryReadAppSnapshotElf(script_name, /*file_offset=*/0); if (snapshot != nullptr) { return snapshot; } diff --git a/runtime/bin/snapshot_utils.h b/runtime/bin/snapshot_utils.h index 5b7cf829f06..4fa221c66e5 100644 --- a/runtime/bin/snapshot_utils.h +++ b/runtime/bin/snapshot_utils.h @@ -37,8 +37,7 @@ class Snapshot { const uint8_t* shared_instructions); static void GenerateAppAOTAsAssembly(const char* snapshot_filename); - static AppSnapshot* TryReadAppendedAppSnapshotBlobs( - const char* container_path); + static AppSnapshot* TryReadAppendedAppSnapshotElf(const char* container_path); static AppSnapshot* TryReadAppSnapshot(const char* script_name); static void WriteAppSnapshot(const char* filename, uint8_t* vm_data_buffer,