diff --git a/runtime/bin/dart_api_win.c b/runtime/bin/dart_api_win.c index 58fc4544d40..ef9709577a0 100644 --- a/runtime/bin/dart_api_win.c +++ b/runtime/bin/dart_api_win.c @@ -359,6 +359,8 @@ typedef Dart_Handle (*Dart_DeferredLoadCompleteErrorType)(intptr_t, typedef Dart_Handle (*Dart_LoadScriptFromKernelType)(const uint8_t*, intptr_t); typedef Dart_Handle (*Dart_LoadScriptFromBytecodeType)(const uint8_t*, intptr_t); +typedef Dart_Handle (*Dart_LoadModuleSnapshotType)(const uint8_t*, + const uint8_t*); typedef Dart_Handle (*Dart_RootLibraryType)(); typedef Dart_Handle (*Dart_SetRootLibraryType)(Dart_Handle); typedef Dart_Handle (*Dart_GetTypeType)(Dart_Handle, @@ -722,6 +724,7 @@ static Dart_DeferredLoadCompleteErrorType Dart_DeferredLoadCompleteErrorFn = NULL; static Dart_LoadScriptFromKernelType Dart_LoadScriptFromKernelFn = NULL; static Dart_LoadScriptFromBytecodeType Dart_LoadScriptFromBytecodeFn = NULL; +static Dart_LoadModuleSnapshotType Dart_LoadModuleSnapshotFn = NULL; static Dart_RootLibraryType Dart_RootLibraryFn = NULL; static Dart_SetRootLibraryType Dart_SetRootLibraryFn = NULL; static Dart_GetTypeType Dart_GetTypeFn = NULL; @@ -1273,6 +1276,8 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { Dart_LoadScriptFromBytecodeFn = (Dart_LoadScriptFromBytecodeType)GetProcAddress( process, "Dart_LoadScriptFromBytecode"); + Dart_LoadModuleSnapshotFn = (Dart_LoadModuleSnapshotType)GetProcAddress( + process, "Dart_LoadModuleSnapshot"); Dart_RootLibraryFn = (Dart_RootLibraryType)GetProcAddress(process, "Dart_RootLibrary"); Dart_SetRootLibraryFn = @@ -2459,6 +2464,11 @@ Dart_Handle Dart_LoadScriptFromBytecode(const uint8_t* kernel_buffer, return Dart_LoadScriptFromBytecodeFn(kernel_buffer, kernel_size); } +Dart_Handle Dart_LoadModuleSnapshot(const uint8_t* snapshot_data, + const uint8_t* snapshot_instructions) { + return Dart_LoadModuleSnapshotFn(snapshot_data, snapshot_instructions); +} + Dart_Handle Dart_RootLibrary() { return Dart_RootLibraryFn(); } diff --git a/runtime/bin/isolate_data.cc b/runtime/bin/isolate_data.cc index af117e04635..3097039ae79 100644 --- a/runtime/bin/isolate_data.cc +++ b/runtime/bin/isolate_data.cc @@ -29,8 +29,8 @@ IsolateGroupData::IsolateGroupData(const char* url, } IsolateGroupData::~IsolateGroupData() { - for (intptr_t i = 0; i < loading_units_.length(); i++) { - delete loading_units_[i]; + for (intptr_t i = 0; i < loaded_snapshots_.length(); i++) { + delete loaded_snapshots_[i]; } free(script_url); free(asset_resolution_base); diff --git a/runtime/bin/isolate_data.h b/runtime/bin/isolate_data.h index b2de0a1a553..0c5e856bdc3 100644 --- a/runtime/bin/isolate_data.h +++ b/runtime/bin/isolate_data.h @@ -95,15 +95,17 @@ class IsolateGroupData { return app_snapshot_ != nullptr || isolate_run_app_snapshot_; } - void AddLoadingUnit(AppSnapshot* loading_unit) { - loading_units_.Add(loading_unit); + // Take ownership of the loaded snapshot. + // Snapshot will be freed when isolate group shuts down. + void AddLoadedSnapshot(AppSnapshot* snapshot) { + loaded_snapshots_.Add(snapshot); } private: friend class IsolateData; // For packages_file_ std::unique_ptr app_snapshot_; - MallocGrowableArray loading_units_; + MallocGrowableArray loaded_snapshots_; char* resolved_packages_config_; std::shared_ptr kernel_buffer_; intptr_t kernel_buffer_size_; diff --git a/runtime/bin/loader.cc b/runtime/bin/loader.cc index dbd60699bd9..9f9ec9c8237 100644 --- a/runtime/bin/loader.cc +++ b/runtime/bin/loader.cc @@ -146,7 +146,7 @@ Dart_Handle Loader::DeferredLoadHandler(intptr_t loading_unit_id) { AppSnapshot* loading_unit_snapshot = Snapshot::TryReadAppSnapshot(unit_url); Dart_Handle result; if (loading_unit_snapshot != nullptr) { - isolate_group_data->AddLoadingUnit(loading_unit_snapshot); + isolate_group_data->AddLoadedSnapshot(loading_unit_snapshot); const uint8_t* isolate_snapshot_data = nullptr; const uint8_t* isolate_snapshot_instructions = nullptr; const uint8_t* ignore_vm_snapshot_data; diff --git a/runtime/bin/main_impl.cc b/runtime/bin/main_impl.cc index e984475ebdf..966f5270fd1 100644 --- a/runtime/bin/main_impl.cc +++ b/runtime/bin/main_impl.cc @@ -280,6 +280,7 @@ static Dart_Isolate IsolateSetupHelper(Dart_Isolate isolate, CHECK_RESULT(result); auto isolate_data = reinterpret_cast(Dart_IsolateData(isolate)); + auto isolate_group_data = isolate_data->isolate_group_data(); const char* resolved_packages_config = nullptr; result = @@ -289,7 +290,6 @@ static Dart_Isolate IsolateSetupHelper(Dart_Isolate isolate, CHECK_RESULT(result); #if !defined(DART_PRECOMPILED_RUNTIME) - auto isolate_group_data = isolate_data->isolate_group_data(); const uint8_t* kernel_buffer = isolate_group_data->kernel_buffer().get(); intptr_t kernel_buffer_size = isolate_group_data->kernel_buffer_size(); if (!isolate_run_app_snapshot && kernel_buffer == nullptr && @@ -385,6 +385,28 @@ static Dart_Isolate IsolateSetupHelper(Dart_Isolate isolate, #endif // !defined(DART_PRECOMPILED_RUNTIME) } + if (Options::load_module_snapshot() != nullptr) { + auto snapshot = + Snapshot::TryReadAppSnapshot(Options::load_module_snapshot()); + if (snapshot == nullptr) { + Syslog::PrintErr("Unable to load module snapshot %s.\n", + Options::load_module_snapshot()); + Dart_ExitScope(); + Dart_ShutdownIsolate(); + return nullptr; + } + isolate_group_data->AddLoadedSnapshot(snapshot); + const uint8_t* ignore_vm_snapshot_data; + const uint8_t* ignore_vm_snapshot_instructions; + const uint8_t* snapshot_data = nullptr; + const uint8_t* snapshot_instructions = nullptr; + snapshot->SetBuffers(&ignore_vm_snapshot_data, + &ignore_vm_snapshot_instructions, &snapshot_data, + &snapshot_instructions); + result = Dart_LoadModuleSnapshot(snapshot_data, snapshot_instructions); + CHECK_RESULT(result); + } + if ((Options::gen_snapshot_kind() == kAppJIT) && is_main_isolate) { result = Dart_SortClasses(); CHECK_RESULT(result); diff --git a/runtime/bin/main_options.h b/runtime/bin/main_options.h index fb901e7b826..72105506059 100644 --- a/runtime/bin/main_options.h +++ b/runtime/bin/main_options.h @@ -30,6 +30,7 @@ namespace bin { V(write_service_info, vm_write_service_info_filename) \ V(executable_name, executable_name) \ V(resolved_executable_name, resolved_executable_name) \ + V(load_module_snapshot, load_module_snapshot) \ /* The purpose of these flags is documented in */ \ /* pkg/dartdev/lib/src/commands/compilation_server.dart. */ \ V(resident_server_info_file, resident_server_info_file_path) \ diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc index 79543bf19e8..2b21b53e4b1 100644 --- a/runtime/bin/snapshot_utils.cc +++ b/runtime/bin/snapshot_utils.cc @@ -175,7 +175,6 @@ static DartUtils::MagicNumber ReadMagicNumberAt(File& file, int64_t offset) { return DartUtils::SniffForMagicNumber(header, read_size); } -#if defined(DART_PRECOMPILED_RUNTIME) class DylibAppSnapshot : public AppSnapshot { public: DylibAppSnapshot(DartUtils::MagicNumber magic_number, @@ -214,9 +213,9 @@ class DylibAppSnapshot : public AppSnapshot { static AppSnapshot* TryReadAppSnapshotDynamicLibrary( DartUtils::MagicNumber magic_number, const char* script_name, - const char** error) { + char** error) { #if defined(DART_INCLUDE_SIMULATOR) - *error = "running on a simulated architecture"; + *error = Utils::StrDup("running on a simulated architecture"); return nullptr; #else #if defined(DART_TARGET_OS_LINUX) || defined(DART_TARGET_OS_MACOS) @@ -237,17 +236,18 @@ static AppSnapshot* TryReadAppSnapshotDynamicLibrary( if (library == nullptr) { #if defined(NATIVE_SHARED_OBJECT_FORMAT_ELF) if (*error == nullptr && magic_number != DartUtils::kAotELFMagicNumber) { - *error = "not an ELF shared object"; + *error = Utils::StrDup("not an ELF shared object"); } #elif defined(NATIVE_SHARED_OBJECT_FORMAT_MACHO) if (*error == nullptr && magic_number != DartUtils::kAotMachO32MagicNumber && magic_number != DartUtils::kAotMachO64MagicNumber) { - *error = "not a Mach-O shared object"; + *error = Utils::StrDup("not a Mach-O shared object"); } #endif if (*error == nullptr) { - *error = "unknown failure loading dynamic library (wrong format?)"; + *error = Utils::StrDup( + "unknown failure loading dynamic library (wrong format?)"); } return nullptr; } @@ -280,6 +280,7 @@ static AppSnapshot* TryReadAppSnapshotDynamicLibrary( #endif // defined(DART_INCLUDE_SIMULATOR) } +#if defined(DART_PRECOMPILED_RUNTIME) class ElfAppSnapshot : public AppSnapshot { public: ElfAppSnapshot(Dart_LoadedElf* elf, @@ -317,16 +318,17 @@ class ElfAppSnapshot : public AppSnapshot { static AppSnapshot* TryReadAppSnapshotElf(const char* script_name, uint64_t file_offset, bool force_load_from_memory) { - const char* error = nullptr; #if defined(NATIVE_SHARED_OBJECT_FORMAT_ELF) if (file_offset == 0 && !force_load_from_memory) { // The load as a dynamic library should succeed, since this is a platform // that natively understands ELF. + char* error = nullptr; if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( DartUtils::kAotELFMagicNumber, script_name, &error)) { return snapshot; } Syslog::PrintErr("Loading dynamic library failed: %s\n", error); + free(error); return nullptr; } #endif @@ -334,6 +336,7 @@ static AppSnapshot* TryReadAppSnapshotElf(const char* script_name, *isolate_data_buffer = nullptr, *isolate_instructions_buffer = nullptr; Dart_LoadedElf* handle = nullptr; + const char* error = nullptr; if (force_load_from_memory) { File* const file = File::Open(/*namespc=*/nullptr, script_name, File::kRead); @@ -402,16 +405,17 @@ static AppSnapshot* TryReadAppSnapshotMachODylib( const char* script_name, uint64_t file_offset, bool force_load_from_memory) { - const char* error = nullptr; #if defined(NATIVE_SHARED_OBJECT_FORMAT_MACHO) if (file_offset == 0 && !force_load_from_memory) { // The load as a dynamic library should succeed, since this is a platform // that natively understands Mach-O. + char* error = nullptr; if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( magic_number, script_name, &error)) { return snapshot; } Syslog::PrintErr("Loading dynamic library failed: %s\n", error); + free(error); return nullptr; } #endif @@ -419,6 +423,7 @@ static AppSnapshot* TryReadAppSnapshotMachODylib( *isolate_data_buffer = nullptr, *isolate_instructions_buffer = nullptr; Dart_LoadedMachODylib* handle = nullptr; + const char* error = nullptr; if (force_load_from_memory) { File* const file = File::Open(/*namespc=*/nullptr, script_name, File::kRead); @@ -471,12 +476,13 @@ static AppSnapshot* TryReadAppSnapshotAt(const char* script_name, if (file_offset == 0) { // This is a non-appended snapshot which is not handled by any of the // non-native loaders, so attempt to load it as a native dynamic library. - const char* error = nullptr; + char* error = nullptr; if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( magic_number, script_name, &error)) { return snapshot; } Syslog::PrintErr("Loading dynamic library failed: %s\n", error); + free(error); } return nullptr; @@ -801,6 +807,17 @@ AppSnapshot* Snapshot::TryReadAppSnapshot(const char* script_uri, // Return the JIT snapshot. return TryReadAppSnapshotBlobs(script_name, file); } + if (magic_number == DartUtils::kAotMachO64MagicNumber) { + // Read module snapshot. + char* error = nullptr; + if (auto* const snapshot = TryReadAppSnapshotDynamicLibrary( + magic_number, script_name, &error)) { + return snapshot; + } + Syslog::PrintErr("Loading dynamic library failed: %s\n", error); + free(error); + return nullptr; + } // We create a dummy snapshot object just to remember the type which // has already been identified by sniffing the magic number. return new DummySnapshot(magic_number); diff --git a/runtime/include/dart_api.h b/runtime/include/dart_api.h index dad30ce21cd..b146d32bad1 100644 --- a/runtime/include/dart_api.h +++ b/runtime/include/dart_api.h @@ -3606,6 +3606,18 @@ Dart_LoadScriptFromKernel(const uint8_t* kernel_buffer, intptr_t kernel_size); DART_EXPORT DART_API_WARN_UNUSED_RESULT Dart_Handle Dart_LoadScriptFromBytecode(const uint8_t* kernel_buffer, intptr_t kernel_size); +/** + * Loads a module snapshot. + * + * \param snapshot_data Buffer containing the module snapshot data. + * Must remain valid until isolate group shutdown. + * \param snapshot_instructions Buffer containing the module snapshot + * instructions. Must remain valid until isolate group shutdown. + */ +DART_EXPORT DART_API_WARN_UNUSED_RESULT Dart_Handle +Dart_LoadModuleSnapshot(const uint8_t* snapshot_data, + const uint8_t* snapshot_instructions); + /** * Gets the library for the root script for the current isolate. * diff --git a/runtime/tests/vm/dart/exported_symbols_test.dart b/runtime/tests/vm/dart/exported_symbols_test.dart index b1d8efd7838..34ab7fee8c9 100644 --- a/runtime/tests/vm/dart/exported_symbols_test.dart +++ b/runtime/tests/vm/dart/exported_symbols_test.dart @@ -234,6 +234,7 @@ main() { "Dart_LoadLibrary", "Dart_LoadLibraryFromBytecode", "Dart_LoadLibraryFromKernel", + "Dart_LoadModuleSnapshot", "Dart_LoadScriptFromBytecode", "Dart_LoadScriptFromKernel", "Dart_LookupLibrary", diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc index 03e50aaf973..ead5d819e61 100644 --- a/runtime/vm/dart_api_impl.cc +++ b/runtime/vm/dart_api_impl.cc @@ -36,6 +36,7 @@ #include "vm/message.h" #include "vm/message_handler.h" #include "vm/message_snapshot.h" +#include "vm/module_snapshot.h" #include "vm/native_entry.h" #include "vm/native_symbol.h" #include "vm/object.h" @@ -5635,6 +5636,39 @@ DART_EXPORT Dart_Handle Dart_LoadScriptFromBytecode(const uint8_t* buffer, #endif // defined(DART_DYNAMIC_MODULES) } +DART_EXPORT DART_API_WARN_UNUSED_RESULT Dart_Handle +Dart_LoadModuleSnapshot(const uint8_t* snapshot_data, + const uint8_t* snapshot_instructions) { +#if defined(DART_PRECOMPILED_RUNTIME) + return Api::NewError("%s: Cannot load module snapshots on an AOT runtime.", + CURRENT_FUNC); +#else + DARTSCOPE(Thread::Current()); + API_TIMELINE_DURATION(T); + CHECK_CALLBACK_STATE(T); + +#if defined(SUPPORT_TIMELINE) + TimelineBeginEndScope tbes(T, Timeline::GetIsolateStream(), + "ReadModuleSnapshot"); +#endif // defined(SUPPORT_TIMELINE) + const Snapshot* snapshot = Snapshot::SetupFromBuffer(snapshot_data); + if (snapshot == nullptr) { + return Api::NewError("Invalid snapshot"); + } + if (snapshot->kind() != Snapshot::kModule) { + return Api::NewError("Invalid snapshot kind"); + } + + const Error& error = Error::Handle( + module_snapshot::ReadModuleSnapshot(T, snapshot, snapshot_instructions)); + if (!error.IsNull()) { + return Api::NewHandle(T, error.ptr()); + } + + return Api::Success(); +#endif // defined(DART_PRECOMPILED_RUNTIME) +} + DART_EXPORT Dart_Handle Dart_RootLibrary() { Thread* thread = Thread::Current(); IsolateGroup* isolate_group = thread->isolate_group(); diff --git a/runtime/vm/dart_entry.cc b/runtime/vm/dart_entry.cc index 83087573262..e02f2af1d85 100644 --- a/runtime/vm/dart_entry.cc +++ b/runtime/vm/dart_entry.cc @@ -140,13 +140,11 @@ ObjectPtr DartEntry::InvokeFunction(const Function& function, ASSERT(!function.IsNull()); #if defined(DART_DYNAMIC_MODULES) - if (function.HasBytecode()) { + if (function.IsInterpreted()) { // SuspendLongJumpScope suspend_long_jump_scope(thread); TransitionToGenerated transition(thread); return Interpreter::Current()->Call(function, arguments_descriptor, arguments, thread); - } else { - ASSERT(!function.is_declared_in_bytecode()); } #endif // defined(DART_DYNAMIC_MODULES) diff --git a/runtime/vm/datastream.h b/runtime/vm/datastream.h index 07b53d43ddc..684d9f8f6c9 100644 --- a/runtime/vm/datastream.h +++ b/runtime/vm/datastream.h @@ -47,6 +47,10 @@ struct LEB128Constants : AllStatic { class NonStreamingWriteStream; +namespace module_snapshot { +class Deserializer; +} + // Stream for reading various types from a buffer. class ReadStream : public ValueObject { public: @@ -324,6 +328,7 @@ class ReadStream : public ValueObject { const uint8_t* end_; friend class Deserializer; + friend class module_snapshot::Deserializer; DISALLOW_COPY_AND_ASSIGN(ReadStream); }; diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc index de350190527..ca97e19ee7d 100644 --- a/runtime/vm/interpreter.cc +++ b/runtime/vm/interpreter.cc @@ -695,6 +695,7 @@ DART_FORCE_INLINE bool Interpreter::InvokeBytecode(Thread* thread, ObjectPtr** FP, ObjectPtr** SP) { ASSERT(Function::HasBytecode(function)); + ASSERT(Function::IsInterpreted(function)); #if defined(DEBUG) if (IsTracingExecution()) { THR_Print("%" Pu64 " ", icount_); @@ -729,7 +730,7 @@ DART_FORCE_INLINE bool Interpreter::Invoke(Thread* thread, FunctionPtr function = FrameFunction(callee_fp); for (;;) { - if (Function::HasBytecode(function)) { + if (Function::IsInterpreted(function)) { return InvokeBytecode(thread, function, call_base, call_top, pc, FP, SP); } else if (Function::HasCode(function)) { return InvokeCompiled(thread, function, call_base, call_top, pc, FP, SP); @@ -4164,7 +4165,7 @@ SwitchDispatchNoSingleStep: FunctionPtr function = Function::RawCast(SP[1]); for (;;) { - if (Function::HasBytecode(function)) { + if (Function::IsInterpreted(function)) { ASSERT(function->IsFunction()); BytecodePtr bytecode = Function::GetBytecode(function); ASSERT(bytecode->IsBytecode()); diff --git a/runtime/vm/module_snapshot.cc b/runtime/vm/module_snapshot.cc new file mode 100644 index 00000000000..e3fc29fa17e --- /dev/null +++ b/runtime/vm/module_snapshot.cc @@ -0,0 +1,799 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#if !defined(DART_PRECOMPILED_RUNTIME) + +#include +#include + +#include "vm/module_snapshot.h" + +#include "platform/assert.h" +#include "vm/bootstrap.h" +#include "vm/canonical_tables.h" +#include "vm/class_id.h" +#include "vm/code_observers.h" +#include "vm/compiler/api/print_filter.h" +#include "vm/compiler/assembler/disassembler.h" +#include "vm/dart.h" +#include "vm/dart_entry.h" +#include "vm/dispatch_table.h" +#include "vm/flag_list.h" +#include "vm/growable_array.h" +#include "vm/heap/heap.h" +#include "vm/image_snapshot.h" +#include "vm/native_entry.h" +#include "vm/object.h" +#include "vm/object_store.h" +#include "vm/resolver.h" +#include "vm/stub_code.h" +#include "vm/symbols.h" +#include "vm/timeline.h" +#include "vm/version.h" +#include "vm/zone_text_buffer.h" + +namespace dart { +namespace module_snapshot { + +class ModuleSnapshot : public AllStatic { + public: + // Version of module snapshot format. + // Should match Snapshot.moduleSnapshotFormatVersion + // constant declared in pkg/native_compiler/lib/snapshot/snapshot.dart. + static constexpr intptr_t kFormatVersion = 1; + + // Predefined clusters in the module snapshot. + // Should match PredefinedClusters enum + // declared in pkg/native_compiler/lib/snapshot/snapshot.dart. + enum PredefinedClusters { + kOneByteStrings, + kTwoByteStrings, + kLibraryRefs, + kPrivateNames, + kClassRefs, + kFieldRefs, + kFunctionRefs, + kInts, + kDoubles, + kArrays, + kInterfaceTypes, + kFunctionTypes, + kRecordTypes, + kTypeParameterTypes, + kTypeArguments, + kCodes, + kObjectPools, + kNumPredefinedClusters, + }; +}; + +class Deserializer; + +class DeserializationCluster : public ZoneAllocated { + public: + explicit DeserializationCluster(const char* name) + : name_(name), start_index_(-1), stop_index_(-1) {} + virtual ~DeserializationCluster() {} + + // Read references to base objects. + virtual void PreLoad(Deserializer* deserializer) {} + + // Allocate memory for all objects in the cluster and write their addresses + // into the ref array. Do not touch this memory. + virtual void ReadAlloc(Deserializer* deserializer) {} + + // Initialize the cluster's objects. Do not touch the memory of other objects. + virtual void ReadFill(Deserializer* deserializer) {} + + // Complete any action that requires the full graph to be deserialized, such + // as rehashing. + virtual void PostLoad(Deserializer* deserializer, const Array& refs) {} + + const char* name() const { return name_; } + + protected: + void ReadAllocFixedSize(Deserializer* deserializer, intptr_t instance_size); + + const char* const name_; + // The range of the ref array that belongs to this cluster. + intptr_t start_index_; + intptr_t stop_index_; +}; + +static constexpr intptr_t kFirstReference = 1; + +class Deserializer : public ThreadStackResource { + public: + Deserializer(Thread* thread, + const uint8_t* buffer, + intptr_t size, + const uint8_t* instructions_buffer); + ~Deserializer(); + + ApiErrorPtr VerifyVersionAndFeatures(); + + ObjectPtr Allocate(intptr_t size); + static void InitializeHeader(ObjectPtr raw, + intptr_t cid, + intptr_t size, + bool is_canonical = false) { + InitializeHeader(raw, cid, size, is_canonical, + ShouldHaveImmutabilityBitSetCid(cid)); + } + static void InitializeHeader(ObjectPtr raw, + intptr_t cid, + intptr_t size, + bool is_canonical, + bool is_immutable); + + // Reads raw data (for basic types). + // sizeof(T) must be in {1,2,4,8}. + template + T Read() { + return ReadStream::Raw::Read(&stream_); + } + intptr_t ReadRefId() { return stream_.ReadRefId(); } + intptr_t ReadUnsigned() { return stream_.ReadUnsigned(); } + uint64_t ReadUnsigned64() { return stream_.ReadUnsigned(); } + void ReadBytes(uint8_t* addr, intptr_t len) { stream_.ReadBytes(addr, len); } + + intptr_t position() const { return stream_.Position(); } + void set_position(intptr_t p) { stream_.SetPosition(p); } + const uint8_t* AddressOfCurrentPosition() const { + return stream_.AddressOfCurrentPosition(); + } + void Advance(intptr_t value) { stream_.Advance(value); } + + void AddBaseObject(const Object& object) { AssignRefPreLoad(object); } + + void AssignRefPreLoad(const Object& object) { + refs_array_.SetAt(next_ref_index_, object); + next_ref_index_++; + } + + void AssignRef(ObjectPtr object) { + ASSERT(next_ref_index_ <= num_objects_); + refs_->untag()->data()[next_ref_index_] = object; + next_ref_index_++; + } + + ObjectPtr Ref(intptr_t index) const { + ASSERT(index > 0); + ASSERT(index <= num_objects_); + return refs_array_.At(index); + } + + ObjectPtr ReadRef() { return Ref(ReadRefId()); } + + void Deserialize(); + + DeserializationCluster* ReadCluster(); + + uword instructions() const { + return reinterpret_cast(instructions_buffer_); + } + intptr_t next_index() const { return next_ref_index_; } + Heap* heap() const { return heap_; } + Zone* zone() const { return zone_; } + + // This serves to make the snapshot cursor, ref table and null be locals + // during ReadFill, which allows the C compiler to see they are not aliased + // and can be kept in registers. + class Local : public ReadStream { + public: + explicit Local(Deserializer* d) + : ReadStream(d->stream_.buffer_, d->stream_.current_, d->stream_.end_), + d_(d), + refs_(d->refs_), + null_(Object::null()) { +#if defined(DEBUG) + // Can't mix use of Deserializer::Read*. + d->stream_.current_ = nullptr; +#endif + } + ~Local() { d_->stream_.current_ = current_; } + + ObjectPtr Ref(intptr_t index) const { + ASSERT(index > 0); + ASSERT(index <= d_->num_objects_); + return refs_->untag()->element(index); + } + + template + T Read() { + return ReadStream::Raw::Read(this); + } + uint64_t ReadUnsigned64() { return ReadUnsigned(); } + + ObjectPtr ReadRef() { return Ref(ReadRefId()); } + + private: + Deserializer* const d_; + const ArrayPtr refs_; + const ObjectPtr null_; + }; + + private: + Heap* heap_; + PageSpace* old_space_; + FreeList* freelist_; + Zone* zone_; + ReadStream stream_; + const uint8_t* instructions_buffer_; + intptr_t num_base_objects_ = 0; + intptr_t num_objects_ = 0; + intptr_t num_clusters_ = 0; + Array& refs_array_; + ArrayPtr refs_; + intptr_t next_ref_index_ = kFirstReference; + DeserializationCluster** clusters_ = nullptr; +}; + +DART_FORCE_INLINE +ObjectPtr Deserializer::Allocate(intptr_t size) { + return UntaggedObject::FromAddr( + old_space_->AllocateSnapshotLocked(freelist_, size)); +} + +void Deserializer::InitializeHeader(ObjectPtr raw, + intptr_t class_id, + intptr_t size, + bool is_canonical, + bool is_immutable) { + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + uword tags = 0; + tags = UntaggedObject::ClassIdTag::update(class_id, tags); + tags = UntaggedObject::SizeTag::update(size, tags); + tags = UntaggedObject::CanonicalBit::update(is_canonical, tags); + tags = UntaggedObject::AlwaysSetBit::update(true, tags); + tags = UntaggedObject::NotMarkedBit::update(true, tags); + tags = UntaggedObject::OldAndNotRememberedBit::update(true, tags); + tags = UntaggedObject::NewOrEvacuationCandidateBit::update(false, tags); + tags = UntaggedObject::ImmutableBit::update(is_immutable, tags); + raw->untag()->tags_ = tags; +} + +DART_NOINLINE +void DeserializationCluster::ReadAllocFixedSize(Deserializer* d, + intptr_t instance_size) { + start_index_ = d->next_index(); + intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + d->AssignRef(d->Allocate(instance_size)); + } + stop_index_ = d->next_index(); +} + +class OneByteStringDeserializationCluster : public DeserializationCluster { + public: + explicit OneByteStringDeserializationCluster(Zone* zone) + : DeserializationCluster("OneByteString"), + string_(String::Handle(zone)) {} + ~OneByteStringDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + const intptr_t len = d->ReadUnsigned(); + string_ = + Symbols::FromLatin1(d->thread(), d->AddressOfCurrentPosition(), len); + d->Advance(len); + d->AssignRefPreLoad(string_); + } + } + + private: + String& string_; +}; + +class TwoByteStringDeserializationCluster : public DeserializationCluster { + public: + explicit TwoByteStringDeserializationCluster(Zone* zone) + : DeserializationCluster("TwoByteString"), + string_(String::Handle(zone)) {} + ~TwoByteStringDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + const intptr_t len = d->ReadUnsigned(); + string_ = Symbols::FromUTF16( + d->thread(), + reinterpret_cast(d->AddressOfCurrentPosition()), + len); + d->Advance(len << 1); + d->AssignRefPreLoad(string_); + } + } + + private: + String& string_; +}; + +class LibraryRefDeserializationCluster : public DeserializationCluster { + public: + explicit LibraryRefDeserializationCluster(Zone* zone) + : DeserializationCluster("LibraryRef"), + uri_(String::Handle(zone)), + library_(Library::Handle(zone)) {} + ~LibraryRefDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + uri_ = static_cast(d->ReadRef()); + library_ = Library::LookupLibrary(d->thread(), uri_); + if (library_.IsNull()) { + FATAL("Unable to find library %s", uri_.ToCString()); + } + d->AssignRefPreLoad(library_); + } + } + + private: + String& uri_; + Library& library_; +}; + +class ClassRefDeserializationCluster : public DeserializationCluster { + public: + explicit ClassRefDeserializationCluster(Zone* zone) + : DeserializationCluster("ClassRef"), + library_(Library::Handle(zone)), + class_name_(String::Handle(zone)), + class_(Class::Handle(zone)) {} + ~ClassRefDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + library_ = static_cast(d->ReadRef()); + class_name_ = static_cast(d->ReadRef()); + class_ = library_.LookupClass(class_name_); + if (class_.IsNull()) { + FATAL("Unable to find class %s in %s", class_name_.ToCString(), + library_.ToCString()); + } + d->AssignRefPreLoad(class_); + } + } + + private: + Library& library_; + String& class_name_; + Class& class_; +}; + +class PrivateNameDeserializationCluster : public DeserializationCluster { + public: + explicit PrivateNameDeserializationCluster(Zone* zone) + : DeserializationCluster("PrivateName"), + library_(Library::Handle(zone)), + name_(String::Handle(zone)) {} + ~PrivateNameDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + library_ = static_cast(d->ReadRef()); + name_ = static_cast(d->ReadRef()); + name_ = library_.PrivateName(name_); + d->AssignRefPreLoad(name_); + } + } + + private: + Library& library_; + String& name_; +}; + +class FieldRefDeserializationCluster : public DeserializationCluster { + public: + explicit FieldRefDeserializationCluster(Zone* zone) + : DeserializationCluster("FieldRef"), + owner_(Object::Handle(zone)), + field_name_(String::Handle(zone)), + field_(Field::Handle(zone)) {} + ~FieldRefDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + owner_ = d->ReadRef(); + field_name_ = static_cast(d->ReadRef()); + if (owner_.IsLibrary()) { + owner_ = Library::Cast(owner_).toplevel_class(); + } + field_ = Class::Cast(owner_).LookupField(field_name_); + if (field_.IsNull()) { + FATAL("Unable to find field %s in %s", field_name_.ToCString(), + owner_.ToCString()); + } + d->AssignRefPreLoad(field_); + } + } + + private: + Object& owner_; + String& field_name_; + Field& field_; +}; + +class FunctionRefDeserializationCluster : public DeserializationCluster { + public: + explicit FunctionRefDeserializationCluster(Zone* zone) + : DeserializationCluster("FunctionRef"), + zone_(zone), + owner_(Object::Handle(zone)), + function_name_(String::Handle(zone)), + function_(Function::Handle(zone)) {} + ~FunctionRefDeserializationCluster() {} + + void PreLoad(Deserializer* d) override { + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + owner_ = d->ReadRef(); + function_name_ = static_cast(d->ReadRef()); + if (owner_.IsLibrary()) { + owner_ = Library::Cast(owner_).toplevel_class(); + } + // TODO(alexmarkov): support method extractors and closures. + function_ = + Resolver::ResolveFunction(zone_, Class::Cast(owner_), function_name_); + if (function_.IsNull()) { + FATAL("Unable to find function %s in %s", function_name_.ToCString(), + owner_.ToCString()); + } + d->AssignRefPreLoad(function_); + } + } + + private: + Zone* zone_; + Object& owner_; + String& function_name_; + Function& function_; +}; + +class CodeDeserializationCluster : public DeserializationCluster { + public: + CodeDeserializationCluster() : DeserializationCluster("Code") {} + ~CodeDeserializationCluster() {} + + void ReadAlloc(Deserializer* d) override { + ReadAllocFixedSize(d, Code::InstanceSize(0)); + } + + void ReadFill(Deserializer* d) override { + uword instructions = d->instructions(); + for (intptr_t id = start_index_, n = stop_index_; id < n; id++) { + auto const code = static_cast(d->Ref(id)); + + Deserializer::InitializeHeader(code, kCodeCid, Code::InstanceSize(0)); + + const uword entry_point = instructions; + code->untag()->entry_point_ = entry_point; + code->untag()->monomorphic_entry_point_ = entry_point; + code->untag()->unchecked_entry_point_ = entry_point; + code->untag()->monomorphic_unchecked_entry_point_ = entry_point; + code->untag()->object_pool_ = static_cast(d->ReadRef()); + code->untag()->instructions_ = Instructions::null(); + code->untag()->owner_ = d->ReadRef(); + code->untag()->exception_handlers_ = + Object::empty_exception_handlers().ptr(); + code->untag()->pc_descriptors_ = Object::empty_descriptors().ptr(); + code->untag()->catch_entry_ = Object::null(); + code->untag()->compressed_stackmaps_ = CompressedStackMaps::null(); + code->untag()->inlined_id_to_function_ = Array::null(); + code->untag()->code_source_map_ = CodeSourceMap::null(); + code->untag()->active_instructions_ = Instructions::null(); + code->untag()->deopt_info_array_ = Array::null(); + code->untag()->static_calls_target_table_ = Array::null(); + +#if !defined(PRODUCT) + code->untag()->return_address_metadata_ = Object::null(); + code->untag()->var_descriptors_ = LocalVarDescriptors::null(); + code->untag()->comments_ = Array::null(); + code->untag()->compile_timestamp_ = 0; +#endif + + code->untag()->state_bits_ = Code::OptimizedBit::update(true, 0); + code->untag()->unchecked_offset_ = 0; + + const uword instr_size = d->ReadUnsigned(); + instructions += instr_size; + } + } + + void PostLoad(Deserializer* d, const Array& refs) override { + Code& code = Code::Handle(d->zone()); + Object& owner = Object::Handle(d->zone()); + + for (intptr_t id = start_index_, n = stop_index_; id < n; id++) { + code ^= refs.At(id); + owner = code.owner(); + + if (owner.IsFunction()) { + Function::Cast(owner).SetInstructionsSafe(code); + +#if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER) + if ((FLAG_disassemble || + (code.is_optimized() && FLAG_disassemble_optimized)) && + compiler::PrintFilter::ShouldPrint(Function::Cast(owner))) { + Disassembler::DisassembleCode(Function::Cast(owner), code, + code.is_optimized()); + } +#endif // !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER) + } else { + UNREACHABLE(); + } + +#if !defined(PRODUCT) + if (CodeObservers::AreActive()) { + Code::NotifyCodeObservers(code, code.is_optimized()); + } +#endif + } + } +}; + +class ObjectPoolDeserializationCluster : public DeserializationCluster { + public: + ObjectPoolDeserializationCluster() : DeserializationCluster("ObjectPool") {} + ~ObjectPoolDeserializationCluster() {} + + void ReadAlloc(Deserializer* d) override { + start_index_ = d->next_index(); + const intptr_t count = d->ReadUnsigned(); + for (intptr_t i = 0; i < count; i++) { + const intptr_t length = d->ReadUnsigned(); + d->AssignRef(d->Allocate(ObjectPool::InstanceSize(length))); + } + stop_index_ = d->next_index(); + } + + void ReadFill(Deserializer* d_) override { + Deserializer::Local d(d_); + + const uint8_t entry_bits = + ObjectPool::EncodeBits(ObjectPool::EntryType::kTaggedObject, + ObjectPool::Patchability::kPatchable, + ObjectPool::SnapshotBehavior::kSnapshotable); + + for (intptr_t id = start_index_, n = stop_index_; id < n; id++) { + const intptr_t length = d.ReadUnsigned(); + ObjectPoolPtr pool = static_cast(d.Ref(id)); + Deserializer::InitializeHeader(pool, kObjectPoolCid, + ObjectPool::InstanceSize(length)); + pool->untag()->length_ = length; + for (intptr_t j = 0; j < length; j++) { + pool->untag()->entry_bits()[j] = entry_bits; + UntaggedObjectPool::Entry& entry = pool->untag()->data()[j]; + entry.raw_obj_ = d.ReadRef(); + } + } + } +}; + +Deserializer::Deserializer(Thread* thread, + const uint8_t* buffer, + intptr_t size, + const uint8_t* instructions_buffer) + : ThreadStackResource(thread), + heap_(thread->isolate_group()->heap()), + old_space_(heap_->old_space()), + freelist_(old_space_->DataFreeList()), + zone_(thread->zone()), + stream_(buffer, size), + instructions_buffer_(instructions_buffer), + refs_array_(Array::Handle(zone_)), + refs_(Array::null()) {} + +Deserializer::~Deserializer() { + delete[] clusters_; +} + +ApiErrorPtr Deserializer::VerifyVersionAndFeatures() { + stream_.SetPosition(Snapshot::kHeaderSize); + + const intptr_t format_version = stream_.ReadUnsigned(); + if (format_version != ModuleSnapshot::kFormatVersion) { + return ApiError::New(String::Handle(String::NewFormatted( + "Invalid module snapshot format version %" Pd " (expected %" Pd ")", + format_version, ModuleSnapshot::kFormatVersion))); + } + + const char* features = + reinterpret_cast(stream_.AddressOfCurrentPosition()); + const intptr_t features_length = + Utils::StrNLen(features, stream_.PendingBytes()); + if (features_length == stream_.PendingBytes()) { + return ApiError::New( + String::Handle(String::New("The features string in the module snapshot " + "was not zero-terminated."))); + } + stream_.Advance(features_length + 1); + + const char* expected_features = kHostArchitectureName; + if (strcmp(expected_features, features) != 0) { + return ApiError::New(String::Handle(String::NewFormatted( + "Invalid module snapshot configuration '%s' (expected '%s')", features, + expected_features))); + } + return ApiError::null(); +} + +DeserializationCluster* Deserializer::ReadCluster() { + const intptr_t cluster_id = ReadUnsigned(); + Zone* Z = zone_; + if (cluster_id >= ModuleSnapshot::kNumPredefinedClusters) { + // return new (Z) InstanceDeserializationCluster(); + UNIMPLEMENTED(); + } + switch (cluster_id) { + case ModuleSnapshot::kOneByteStrings: + return new (Z) OneByteStringDeserializationCluster(Z); + case ModuleSnapshot::kTwoByteStrings: + return new (Z) TwoByteStringDeserializationCluster(Z); + case ModuleSnapshot::kLibraryRefs: + return new (Z) LibraryRefDeserializationCluster(Z); + case ModuleSnapshot::kPrivateNames: + return new (Z) PrivateNameDeserializationCluster(Z); + case ModuleSnapshot::kClassRefs: + return new (Z) ClassRefDeserializationCluster(Z); + case ModuleSnapshot::kFieldRefs: + return new (Z) FieldRefDeserializationCluster(Z); + case ModuleSnapshot::kFunctionRefs: + return new (Z) FunctionRefDeserializationCluster(Z); + case ModuleSnapshot::kInts: + case ModuleSnapshot::kDoubles: + case ModuleSnapshot::kArrays: + case ModuleSnapshot::kInterfaceTypes: + case ModuleSnapshot::kFunctionTypes: + case ModuleSnapshot::kRecordTypes: + case ModuleSnapshot::kTypeParameterTypes: + case ModuleSnapshot::kTypeArguments: + UNIMPLEMENTED(); + return nullptr; + case ModuleSnapshot::kCodes: + return new (Z) CodeDeserializationCluster(); + case ModuleSnapshot::kObjectPools: + return new (Z) ObjectPoolDeserializationCluster(); + default: + break; + } + FATAL("No cluster defined for cluster id %" Pd, cluster_id); + return nullptr; +} + +class HeapLocker : public StackResource { + public: + HeapLocker(Thread* thread, PageSpace* page_space) + : StackResource(thread), + page_space_(page_space), + freelist_(page_space->DataFreeList()) { + page_space_->AcquireLock(freelist_); + } + ~HeapLocker() { page_space_->ReleaseLock(freelist_); } + + private: + PageSpace* page_space_; + FreeList* freelist_; +}; + +void Deserializer::Deserialize() { + const void* clustered_start = AddressOfCurrentPosition(); + + num_base_objects_ = ReadUnsigned(); + num_objects_ = ReadUnsigned(); + const uword instructions_size = ReadUnsigned(); + num_clusters_ = ReadUnsigned(); + + // TODO(alexmarkov): register image pages + // + // heap_->SetupImagePage(const_cast(instructions_buffer_), + // instructions_size, + // /* is_executable */ true); + USE(instructions_size); + + clusters_ = new DeserializationCluster*[num_clusters_]; + refs_array_ = Array::New(num_objects_ + kFirstReference, Heap::kOld); + + AddBaseObject(Object::null_object()); + AddBaseObject(Bool::True()); + AddBaseObject(Bool::False()); + + if (num_base_objects_ != (next_ref_index_ - kFirstReference)) { + FATAL("Snapshot expects %" Pd + " base objects, but deserializer provided %" Pd, + num_base_objects_, next_ref_index_ - kFirstReference); + } + + { + TIMELINE_DURATION(thread(), Isolate, "PreLoad"); + for (intptr_t i = 0; i < num_clusters_; i++) { + clusters_[i] = ReadCluster(); + clusters_[i]->PreLoad(this); + } + } + + { + // The deserializer initializes objects without using the write barrier, + // partly for speed since we know all the deserialized objects will be + // long-lived and partly because the target objects can be not yet + // initialized at the time of the write. To make this safe, we must ensure + // there are no other threads mutating this heap, and that incremental + // marking is not in progress. This is normally the case anyway for the + // module snapshots being deserialized at isolate load. + HeapIterationScope iter(thread()); + // For bump-pointer allocation in old-space. + HeapLocker hl(thread(), heap_->old_space()); + // Must not perform any other type of allocation, which might trigger GC + // while there are still uninitialized objects. + NoSafepointScope no_safepoint(thread()); + refs_ = refs_array_.ptr(); + + { + TIMELINE_DURATION(thread(), Isolate, "ReadAlloc"); + for (intptr_t i = 0; i < num_clusters_; i++) { + clusters_[i]->ReadAlloc(this); + } + } + + // We should have completely filled the ref array. + ASSERT_EQUAL(next_ref_index_ - kFirstReference, num_objects_); + + { + TIMELINE_DURATION(thread(), Isolate, "ReadFill"); + for (intptr_t i = 0; i < num_clusters_; i++) { + clusters_[i]->ReadFill(this); + } + } + + refs_ = nullptr; + } + + auto isolate_group = thread()->isolate_group(); +#if defined(DEBUG) + isolate_group->heap()->Verify("Deserializer::Deserialize"); +#endif + + { + TIMELINE_DURATION(thread(), Isolate, "PostLoad"); + for (intptr_t i = 0; i < num_clusters_; i++) { + clusters_[i]->PostLoad(this, refs_array_); + } + } + + if (isolate_group->snapshot_is_dontneed_safe()) { + size_t clustered_length = + reinterpret_cast(AddressOfCurrentPosition()) - + reinterpret_cast(clustered_start); + VirtualMemory::DontNeed(const_cast(clustered_start), + clustered_length); + } +} + +ApiErrorPtr ReadModuleSnapshot(Thread* thread, + const Snapshot* snapshot, + const uint8_t* instructions_buffer) { + ASSERT(snapshot->kind() == Snapshot::kModule); + + Deserializer deserializer(thread, snapshot->Addr(), snapshot->length(), + instructions_buffer); + + ApiErrorPtr api_error = deserializer.VerifyVersionAndFeatures(); + if (api_error != ApiError::null()) { + return api_error; + } + + deserializer.Deserialize(); + + return ApiError::null(); +} + +} // namespace module_snapshot +} // namespace dart + +#endif // !defined(DART_PRECOMPILED_RUNTIME) diff --git a/runtime/vm/module_snapshot.h b/runtime/vm/module_snapshot.h new file mode 100644 index 00000000000..214a0c12a13 --- /dev/null +++ b/runtime/vm/module_snapshot.h @@ -0,0 +1,27 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_VM_MODULE_SNAPSHOT_H_ +#define RUNTIME_VM_MODULE_SNAPSHOT_H_ + +#if !defined(DART_PRECOMPILED_RUNTIME) + +#include "platform/assert.h" +#include "vm/allocation.h" +#include "vm/globals.h" +#include "vm/snapshot.h" +#include "vm/thread.h" + +namespace dart { +namespace module_snapshot { + +ApiErrorPtr ReadModuleSnapshot(Thread* thread, + const Snapshot* snapshot, + const uint8_t* instructions_buffer); + +} // namespace module_snapshot +} // namespace dart + +#endif // !defined(DART_PRECOMPILED_RUNTIME) +#endif // RUNTIME_VM_MODULE_SNAPSHOT_H_ diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 367121e3d39..41b26ef6a6b 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -8440,6 +8440,10 @@ void Function::ClearBytecode() const { ClearCode(); } +bool Function::IsInterpreted(FunctionPtr function) { + return function->untag()->code() == StubCode::InterpretCall().ptr(); +} + #endif // defined(DART_DYNAMIC_MODULES) bool Function::HasCode(FunctionPtr function) { diff --git a/runtime/vm/object.h b/runtime/vm/object.h index cfb9216a758..fb598e9b2ee 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -3287,8 +3287,11 @@ class Function : public Object { static inline BytecodePtr GetBytecode(FunctionPtr function); inline bool HasBytecode() const; static inline bool HasBytecode(FunctionPtr function); + static bool IsInterpreted(FunctionPtr function); + inline bool IsInterpreted() const { return IsInterpreted(ptr()); } #else inline bool HasBytecode() const { return false; } + inline bool IsInterpreted() const { return false; } #endif virtual uword Hash() const; @@ -6993,7 +6996,11 @@ class Code : public Object { : 0; return EntryPointOf(code) - entry_offset; #else - return Instructions::PayloadStart(InstructionsOf(code)); + auto instr = InstructionsOf(code); + if (instr == Instructions::null()) { + return code->untag()->entry_point_; + } + return Instructions::PayloadStart(instr); #endif } @@ -7003,7 +7010,11 @@ class Code : public Object { #if defined(DART_PRECOMPILED_RUNTIME) return code->untag()->entry_point_; #else - return Instructions::EntryPoint(InstructionsOf(code)); + auto instr = InstructionsOf(code); + if (instr == Instructions::null()) { + return code->untag()->entry_point_; + } + return Instructions::EntryPoint(instr); #endif } @@ -7024,7 +7035,11 @@ class Code : public Object { #if defined(DART_PRECOMPILED_RUNTIME) return untag()->monomorphic_entry_point_; #else - return Instructions::MonomorphicEntryPoint(instructions()); + auto instr = instructions(); + if (instr == Instructions::null()) { + return untag()->monomorphic_entry_point_; + } + return Instructions::MonomorphicEntryPoint(instr); #endif } // Returns the unchecked monomorphic entry point of [instructions()]. @@ -7043,7 +7058,12 @@ class Code : public Object { if (IsUnknownDartCode(code)) return kUwordMax; return code->untag()->instructions_length_; #else - return Instructions::Size(InstructionsOf(code)); + auto instr = InstructionsOf(code); + if (instr == Instructions::null()) { + // TODO(alexmarkov): keep size in the Code objects. + return 0; + } + return Instructions::Size(instr); #endif } @@ -7531,6 +7551,7 @@ class Code : public Object { friend class CodeKeyValueTrait; // for UncheckedEntryPointOffset friend class InstanceCall; // for StorePointerUnaligned friend class StaticCall; // for StorePointerUnaligned + friend class module_snapshot::CodeDeserializationCluster; friend void DumpStackFrame(intptr_t frame_index, uword pc, uword fp); }; diff --git a/runtime/vm/object_store.h b/runtime/vm/object_store.h index 75948bc3627..15505924c13 100644 --- a/runtime/vm/object_store.h +++ b/runtime/vm/object_store.h @@ -623,6 +623,7 @@ class ObjectStore { case Snapshot::kFullJIT: case Snapshot::kFullAOT: return reinterpret_cast(&slow_tts_stub_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index f3270fc357a..734a26a5315 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -46,6 +46,12 @@ CLASS_LIST(DEFINE_FORWARD_DECLARATION) class CodeStatistics; class StackFrame; +namespace module_snapshot { +class CodeDeserializationCluster; +class Deserializer; +class ObjectPoolDeserializationCluster; +} // namespace module_snapshot + #define DEFINE_CONTAINS_COMPRESSED(type) \ static constexpr bool kContainsCompressedPointers = \ is_compressed_ptr::value; @@ -886,6 +892,7 @@ class UntaggedObject { friend class WriteBarrierUpdateVisitor; // CheckHeapPointerStore friend class OffsetsTable; friend class Object; + friend class module_snapshot::Deserializer; friend uword TagsFromUntaggedObject(UntaggedObject*); // tags_ friend void SetNewSpaceTaggingWord(ObjectPtr, classid_t, uint32_t); // tags_ friend class ObjectCopyBase; // LoadPointer/StorePointer @@ -1249,6 +1256,7 @@ class UntaggedClass : public UntaggedObject { #if !defined(DART_PRECOMPILED_RUNTIME) return reinterpret_cast(&dependent_code_); #endif + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1333,6 +1341,7 @@ class UntaggedPatchClass : public UntaggedObject { UNREACHABLE(); return nullptr; #endif + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1527,6 +1536,7 @@ class UntaggedFunction : public UntaggedObject { case Snapshot::kFullCore: case Snapshot::kFullJIT: return reinterpret_cast(&data_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1692,6 +1702,7 @@ class UntaggedField : public UntaggedObject { case Snapshot::kFullJIT: case Snapshot::kFullAOT: return reinterpret_cast(&initializer_function_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1761,6 +1772,7 @@ class alignas(8) UntaggedScript : public UntaggedObject { case Snapshot::kFullCore: case Snapshot::kFullJIT: return reinterpret_cast(&kernel_program_info_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1838,6 +1850,7 @@ class UntaggedLibrary : public UntaggedObject { UNREACHABLE(); return nullptr; #endif + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -1894,6 +1907,7 @@ class UntaggedNamespace : public UntaggedObject { case Snapshot::kFullCore: case Snapshot::kFullJIT: return reinterpret_cast(&owner_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -2077,6 +2091,7 @@ class UntaggedCode : public UntaggedObject { friend class UnitSerializationRoots; friend class UnitDeserializationRoots; friend class CallSiteResetter; + friend class module_snapshot::CodeDeserializationCluster; }; class UntaggedBytecode : public UntaggedObject { @@ -2142,6 +2157,7 @@ class UntaggedObjectPool : public UntaggedObject { friend class Interpreter; friend class UnitSerializationRoots; friend class UnitDeserializationRoots; + friend class module_snapshot::ObjectPoolDeserializationCluster; }; class UntaggedInstructions : public UntaggedObject { @@ -2713,6 +2729,7 @@ class UntaggedICData : public UntaggedCallSiteData { case Snapshot::kFullCore: case Snapshot::kFullJIT: return to(); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -2851,6 +2868,7 @@ class UntaggedLibraryPrefix : public UntaggedInstance { case Snapshot::kFullCore: case Snapshot::kFullJIT: return reinterpret_cast(&importer_); + case Snapshot::kModule: case Snapshot::kNone: case Snapshot::kInvalid: break; @@ -3387,6 +3405,7 @@ class UntaggedArray : public UntaggedInstance { friend class Page; friend class MarkingVisitor; friend class FastObjectCopy; // For initializing fields. + friend class module_snapshot::Deserializer; friend void UpdateLengthField(intptr_t, ObjectPtr, ObjectPtr); // length_ }; diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc index 0172dcd6afc..9689338f844 100644 --- a/runtime/vm/runtime_entry.cc +++ b/runtime/vm/runtime_entry.cc @@ -4831,6 +4831,7 @@ extern "C" uword /*ObjectPtr*/ InterpretCall(uword /*FunctionPtr*/ function_in, // We stay in "in generated code" execution state when interpreting code. ASSERT(thread->execution_state() == Thread::kThreadInGenerated); ASSERT(Function::HasBytecode(function)); + ASSERT(Function::IsInterpreted(function)); ASSERT(interpreter != nullptr); #endif // Tell MemorySanitizer 'argv' is initialized by generated code. diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index 4e35e66173e..2b743c59870 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h @@ -26,6 +26,7 @@ class Snapshot { kFullCore, // Full snapshot of core libraries. kFullJIT, // Full + JIT code kFullAOT, // Full + AOT code + kModule, // Module snapshot with code. kNone, // gen_snapshot kInvalid }; @@ -65,7 +66,7 @@ class Snapshot { (kind == kFullAOT); } static bool IncludesCode(Kind kind) { - return (kind == kFullJIT) || (kind == kFullAOT); + return (kind == kFullJIT) || (kind == kFullAOT) || (kind == kModule); } static bool IncludesStringsInROData(Kind kind) { diff --git a/runtime/vm/stack_trace.cc b/runtime/vm/stack_trace.cc index 6c699b7d84c..4d982818b7f 100644 --- a/runtime/vm/stack_trace.cc +++ b/runtime/vm/stack_trace.cc @@ -355,7 +355,7 @@ void AsyncAwareStackUnwinder::Unwind( // will handle successful completion. This function is not yet executing // so we have to use artificial marker offset (1). #if defined(DART_DYNAMIC_MODULES) - if (function_.HasBytecode()) { + if (function_.IsInterpreted()) { bytecode_ = function_.GetBytecode(); code_ = Code::null(); pc_offset = StackTraceUtils::kFutureListenerPcOffset; diff --git a/runtime/vm/vm_sources.gni b/runtime/vm/vm_sources.gni index 56a9942aeba..f3b980a8f51 100644 --- a/runtime/vm/vm_sources.gni +++ b/runtime/vm/vm_sources.gni @@ -184,6 +184,8 @@ vm_sources = [ "metrics.h", "microtask_mirror_queues.cc", "microtask_mirror_queues.h", + "module_snapshot.cc", + "module_snapshot.h", "native_arguments.h", "native_entry.cc", "native_entry.h",