From 7dfe62f70d0742c09733fdecd36a2e697c8cb190 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Fri, 11 Aug 2017 12:18:16 -0700 Subject: [PATCH] [standalone] Automatically decompress gzip'd resources, including sources and script snapshots. Closes #30315 R=asiva@google.com, zra@google.com Review-Url: https://codereview.chromium.org/2991393002 . --- runtime/bin/BUILD.gn | 13 +++++- runtime/bin/dartutils.cc | 48 ++++++++++--------- runtime/bin/dartutils.h | 16 +++---- runtime/bin/dfe.cc | 5 +- runtime/bin/gzip.cc | 86 +++++++++++++++++++++++++++++++++++ runtime/bin/gzip.h | 24 ++++++++++ runtime/bin/loader.cc | 43 +++++++++++++++++- runtime/bin/main.cc | 68 +-------------------------- runtime/bin/snapshot_utils.cc | 2 +- 9 files changed, 200 insertions(+), 105 deletions(-) create mode 100644 runtime/bin/gzip.cc create mode 100644 runtime/bin/gzip.h diff --git a/runtime/bin/BUILD.gn b/runtime/bin/BUILD.gn index cd0213a2b6e..1b17b1489c2 100644 --- a/runtime/bin/BUILD.gn +++ b/runtime/bin/BUILD.gn @@ -316,6 +316,8 @@ template("build_gen_snapshot") { "address_sanitizer.cc", "builtin.cc", "builtin.h", + "gzip.cc", + "gzip.h", "loader.cc", "loader.h", @@ -329,7 +331,10 @@ template("build_gen_snapshot") { "vmservice_impl.h", ] - include_dirs = [ ".." ] + include_dirs = [ + "..", + "//third_party", + ] if (dart_use_tcmalloc) { deps += [ "//third_party/tcmalloc" ] @@ -760,6 +765,8 @@ dart_executable("dart") { "dfe.h", "loader.cc", "loader.h", + "gzip.cc", + "gzip.h", "$target_gen_dir/resources_gen.cc", ] if (dart_runtime_mode == "release") { @@ -778,6 +785,8 @@ dart_executable("dart_precompiled_runtime") { "snapshot_empty.cc", "loader.cc", "loader.h", + "gzip.cc", + "gzip.h", ] if (dart_runtime_mode == "release") { extra_sources += [ "observatory_assets_empty.cc" ] @@ -816,6 +825,8 @@ dart_executable("dart_bootstrap") { "dfe.h", "loader.cc", "loader.h", + "gzip.cc", + "gzip.h", "observatory_assets_empty.cc", "snapshot_empty.cc", diff --git a/runtime/bin/dartutils.cc b/runtime/bin/dartutils.cc index bde946503f9..ed2f6759465 100644 --- a/runtime/bin/dartutils.cc +++ b/runtime/bin/dartutils.cc @@ -48,14 +48,15 @@ const char* const DartUtils::kHttpScheme = "http:"; const char* const DartUtils::kVMServiceLibURL = "dart:vmservice"; struct MagicNumberData { - static const intptr_t kLength = 4; + static const intptr_t kMaxLength = 4; - const uint8_t bytes[kLength]; - bool should_skip; + intptr_t length; + const uint8_t bytes[kMaxLength]; }; -MagicNumberData snapshot_magic_number = {{0xf5, 0xf5, 0xdc, 0xdc}, true}; -MagicNumberData kernel_magic_number = {{0x90, 0xab, 0xcd, 0xef}, false}; +MagicNumberData snapshot_magic_number = {4, {0xf5, 0xf5, 0xdc, 0xdc}}; +MagicNumberData kernel_magic_number = {4, {0x90, 0xab, 0xcd, 0xef}}; +MagicNumberData gzip_magic_number = {2, {0x1f, 0x8b, 0, 0}}; static bool IsWindowsHost() { #if defined(HOST_OS_WINDOWS) @@ -339,40 +340,45 @@ Dart_Handle DartUtils::ResolveScript(Dart_Handle url) { kNumArgs, dart_args); } -static bool CheckMagicNumber(const uint8_t** buf, - intptr_t* len, +static bool CheckMagicNumber(const uint8_t* buffer, + intptr_t buffer_length, const MagicNumberData& magic_number) { - if ((*len >= MagicNumberData::kLength) && - (memcmp(*buf, magic_number.bytes, MagicNumberData::kLength) == 0)) { - if (magic_number.should_skip) { - *buf += MagicNumberData::kLength; - *len -= MagicNumberData::kLength; - } - return true; + if ((buffer_length >= magic_number.length)) { + return memcmp(buffer, magic_number.bytes, magic_number.length) == 0; } return false; } -DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t** buf, - intptr_t* len) { - if (CheckMagicNumber(buf, len, snapshot_magic_number)) { +DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer, + intptr_t buffer_length) { + if (CheckMagicNumber(buffer, buffer_length, snapshot_magic_number)) { return kSnapshotMagicNumber; } - if (CheckMagicNumber(buf, len, kernel_magic_number)) { + if (CheckMagicNumber(buffer, buffer_length, kernel_magic_number)) { return kKernelMagicNumber; } + if (CheckMagicNumber(buffer, buffer_length, gzip_magic_number)) { + return kGzipMagicNumber; + } + return kUnknownMagicNumber; } -void DartUtils::WriteMagicNumber(File* file) { +void DartUtils::WriteSnapshotMagicNumber(File* file) { // Write a magic number and version information into the snapshot file. - bool bytes_written = - file->WriteFully(snapshot_magic_number.bytes, MagicNumberData::kLength); + bool bytes_written = file->WriteFully(snapshot_magic_number.bytes, + snapshot_magic_number.length); ASSERT(bytes_written); } +void DartUtils::SkipSnapshotMagicNumber(const uint8_t** buffer, + intptr_t* buffer_length) { + *buffer += snapshot_magic_number.length; + *buffer_length -= snapshot_magic_number.length; +} + void FUNCTION_NAME(Builtin_GetCurrentDirectory)(Dart_NativeArguments args) { const char* current = Directory::Current(); if (current != NULL) { diff --git a/runtime/bin/dartutils.h b/runtime/bin/dartutils.h index 135d12cbc8c..f36aac9bf2a 100644 --- a/runtime/bin/dartutils.h +++ b/runtime/bin/dartutils.h @@ -200,21 +200,17 @@ class DartUtils { enum MagicNumber { kSnapshotMagicNumber, kKernelMagicNumber, + kGzipMagicNumber, kUnknownMagicNumber }; - // static const uint8_t* GetMagicNumber(MagicNumber number); - - // Sniffs the specified text_buffer to see if it contains the magic number - // representing a script snapshot. If the text_buffer is a script snapshot - // the return value is an updated pointer to the text_buffer pointing past - // the magic number value. The 'buffer_len' parameter is also appropriately - // adjusted. - static MagicNumber SniffForMagicNumber(const uint8_t** text_buffer, - intptr_t* buffer_len); + // Checks if the buffer is a script snapshot, kernel file, or gzip file. + static MagicNumber SniffForMagicNumber(const uint8_t* text_buffer, + intptr_t buffer_len); // Write a magic number to indicate a script snapshot file. - static void WriteMagicNumber(File* file); + static void WriteSnapshotMagicNumber(File* file); + static void SkipSnapshotMagicNumber(const uint8_t** buffer, intptr_t* length); // Global state that stores the original working directory.. static const char* original_working_directory; diff --git a/runtime/bin/dfe.cc b/runtime/bin/dfe.cc index bc83f5be15b..54e30494cd9 100644 --- a/runtime/bin/dfe.cc +++ b/runtime/bin/dfe.cc @@ -143,10 +143,7 @@ bool DFE::TryReadKernelFile(const char* script_uri, DartUtils::ReadFile(&buffer, kernel_ir_size, script_file); DartUtils::CloseFile(script_file); if (*kernel_ir_size > 0 && buffer != NULL) { - // We need a temporary variable because SniffForMagicNumber modifies the - // buffer pointer to skip snapshot magic number. - const uint8_t* temp = buffer; - if (DartUtils::SniffForMagicNumber(&temp, kernel_ir_size) != + if (DartUtils::SniffForMagicNumber(buffer, *kernel_ir_size) != DartUtils::kKernelMagicNumber) { free(const_cast(buffer)); *kernel_ir = NULL; diff --git a/runtime/bin/gzip.cc b/runtime/bin/gzip.cc new file mode 100644 index 00000000000..ebeb4082610 --- /dev/null +++ b/runtime/bin/gzip.cc @@ -0,0 +1,86 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "bin/gzip.h" + +#include "platform/assert.h" +#include "platform/globals.h" +#include "zlib/zlib.h" + +namespace dart { +namespace bin { + +void Decompress(const uint8_t* input, + intptr_t input_len, + uint8_t** output, + intptr_t* output_length) { + ASSERT(input != NULL); + ASSERT(input_len > 0); + ASSERT(output != NULL); + ASSERT(output_length != NULL); + + const intptr_t kChunkSize = 256 * 1024; + + // Initialize output. + intptr_t output_capacity = input_len * 2; + if (output_capacity < kChunkSize) { + output_capacity = kChunkSize; + } + *output = reinterpret_cast(malloc(output_capacity)); + + uint8_t chunk_out[kChunkSize]; + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = 0; + int ret = inflateInit2(&strm, 32 + MAX_WBITS); + ASSERT(ret == Z_OK); + + intptr_t input_cursor = 0; + intptr_t output_cursor = 0; + do { + // Setup input. + intptr_t size_in = input_len - input_cursor; + if (size_in > kChunkSize) { + size_in = kChunkSize; + } + strm.avail_in = size_in; + strm.next_in = const_cast(&input[input_cursor]); + + // Inflate until we've exhausted the current input chunk. + do { + // Setup output. + strm.avail_out = kChunkSize; + strm.next_out = &chunk_out[0]; + // Inflate. + ret = inflate(&strm, Z_SYNC_FLUSH); + // We either hit the end of the stream or made forward progress. + ASSERT((ret == Z_STREAM_END) || (ret == Z_OK)); + // Grow output buffer size. + intptr_t size_out = kChunkSize - strm.avail_out; + if (size_out > (output_capacity - output_cursor)) { + output_capacity *= 2; + ASSERT(size_out <= (output_capacity - output_cursor)); + *output = reinterpret_cast(realloc(*output, output_capacity)); + } + // Copy output. + memmove(&((*output)[output_cursor]), &chunk_out[0], size_out); + output_cursor += size_out; + } while (strm.avail_out == 0); + + // We've processed size_in bytes. + input_cursor += size_in; + + // We're finished decompressing when zlib tells us. + } while (ret != Z_STREAM_END); + + inflateEnd(&strm); + + *output_length = output_cursor; +} + +} // namespace bin +} // namespace dart diff --git a/runtime/bin/gzip.h b/runtime/bin/gzip.h new file mode 100644 index 00000000000..c2fcfb96423 --- /dev/null +++ b/runtime/bin/gzip.h @@ -0,0 +1,24 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_BIN_GZIP_H_ +#define RUNTIME_BIN_GZIP_H_ + +#include "platform/globals.h" + +namespace dart { +namespace bin { + +// |input| is assumed to be a gzipped stream. +// This function allocates the output buffer in the C heap and the caller +// is responsible for freeing it. +void Decompress(const uint8_t* input, + intptr_t input_len, + uint8_t** output, + intptr_t* output_length); + +} // namespace bin +} // namespace dart + +#endif // RUNTIME_BIN_GZIP_H_ diff --git a/runtime/bin/loader.cc b/runtime/bin/loader.cc index a8ebdc8c484..28a59159be4 100644 --- a/runtime/bin/loader.cc +++ b/runtime/bin/loader.cc @@ -9,6 +9,7 @@ #include "bin/dfe.h" #include "bin/extensions.h" #include "bin/file.h" +#include "bin/gzip.h" #include "bin/lockers.h" #include "bin/utils.h" #include "include/dart_tools_api.h" @@ -318,6 +319,39 @@ void Loader::ResolveDependenciesAsFilePaths() { } } +class ScopedDecompress : public ValueObject { + public: + ScopedDecompress(const uint8_t** payload, intptr_t* payload_length) + : payload_(payload), + payload_length_(payload_length), + decompressed_(NULL) { + DartUtils::MagicNumber payload_type = + DartUtils::SniffForMagicNumber(*payload, *payload_length); + if (payload_type == DartUtils::kGzipMagicNumber) { + int64_t start = Dart_TimelineGetMicros(); + intptr_t decompressed_length = 0; + Decompress(*payload, *payload_length, &decompressed_, + &decompressed_length); + int64_t end = Dart_TimelineGetMicros(); + Dart_TimelineEvent("Decompress", start, end, Dart_Timeline_Event_Duration, + 0, NULL, NULL); + *payload_ = decompressed_; + *payload_length_ = decompressed_length; + } + } + + ~ScopedDecompress() { + if (decompressed_ != NULL) { + free(decompressed_); + } + } + + private: + const uint8_t** payload_; + intptr_t* payload_length_; + uint8_t* decompressed_; +}; + bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) { // We have to copy everything we care about out of |result| because after // dropping the lock below |result| may no longer valid. @@ -388,11 +422,15 @@ bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) { // Check for payload and load accordingly. const uint8_t* payload = result->payload; intptr_t payload_length = result->payload_length; + + // Decompress if gzip'd. + ScopedDecompress decompress(&payload, &payload_length); + const DartUtils::MagicNumber payload_type = - DartUtils::SniffForMagicNumber(&payload, &payload_length); + DartUtils::SniffForMagicNumber(payload, payload_length); Dart_Handle source = Dart_Null(); if (payload_type == DartUtils::kUnknownMagicNumber) { - source = Dart_NewStringFromUTF8(result->payload, result->payload_length); + source = Dart_NewStringFromUTF8(payload, payload_length); if (Dart_IsError(source)) { loader->error_ = DartUtils::NewError("%s is not a valid UTF-8 script", @@ -424,6 +462,7 @@ bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) { } break; case Dart_kScriptTag: if (payload_type == DartUtils::kSnapshotMagicNumber) { + DartUtils::SkipSnapshotMagicNumber(&payload, &payload_length); dart_result = Dart_LoadScriptFromSnapshot(payload, payload_length); reload_extensions = true; } else if (payload_type == DartUtils::kKernelMagicNumber) { diff --git a/runtime/bin/main.cc b/runtime/bin/main.cc index 9022af0d7ae..c8be4326adc 100644 --- a/runtime/bin/main.cc +++ b/runtime/bin/main.cc @@ -32,7 +32,7 @@ #include "platform/hashmap.h" #include "platform/text_buffer.h" #if !defined(DART_PRECOMPILER) -#include "zlib/zlib.h" +#include "bin/gzip.h" #endif #include "vm/kernel.h" @@ -1658,74 +1658,10 @@ bool RunMainIsolate(const char* script_name, CommandLineOptions* dart_options) { extern unsigned int observatory_assets_archive_len; extern const uint8_t* observatory_assets_archive; -// |input| is assumed to be a gzipped stream. -// This function allocates the output buffer in the C heap and the caller -// is responsible for freeing it. -void Decompress(const uint8_t* input, - unsigned int input_len, - uint8_t** output, - unsigned int* output_length) { - ASSERT(input != NULL); - ASSERT(input_len > 0); - ASSERT(output != NULL); - ASSERT(output_length != NULL); - - // Initialize output. - *output = NULL; - *output_length = 0; - - const unsigned int kChunkSize = 256 * 1024; - uint8_t chunk_out[kChunkSize]; - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = 0; - int ret = inflateInit2(&strm, 32 + MAX_WBITS); - ASSERT(ret == Z_OK); - - unsigned int input_cursor = 0; - unsigned int output_cursor = 0; - do { - // Setup input. - unsigned int size_in = input_len - input_cursor; - if (size_in > kChunkSize) { - size_in = kChunkSize; - } - strm.avail_in = size_in; - strm.next_in = const_cast(&input[input_cursor]); - - // Inflate until we've exhausted the current input chunk. - do { - // Setup output. - strm.avail_out = kChunkSize; - strm.next_out = &chunk_out[0]; - // Inflate. - ret = inflate(&strm, Z_SYNC_FLUSH); - // We either hit the end of the stream or made forward progress. - ASSERT((ret == Z_STREAM_END) || (ret == Z_OK)); - // Grow output buffer size. - unsigned int size_out = kChunkSize - strm.avail_out; - *output_length += size_out; - *output = reinterpret_cast(realloc(*output, *output_length)); - // Copy output. - memmove(&((*output)[output_cursor]), &chunk_out[0], size_out); - output_cursor += size_out; - } while (strm.avail_out == 0); - - // We've processed size_in bytes. - input_cursor += size_in; - - // We're finished decompressing when zlib tells us. - } while (ret != Z_STREAM_END); - - inflateEnd(&strm); -} Dart_Handle GetVMServiceAssetsArchiveCallback() { uint8_t* decompressed = NULL; - unsigned int decompressed_len = 0; + intptr_t decompressed_len = 0; Decompress(observatory_assets_archive, observatory_assets_archive_len, &decompressed, &decompressed_len); Dart_Handle tar_file = diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc index e6ea9c7ceb9..6c58a30a8c6 100644 --- a/runtime/bin/snapshot_utils.cc +++ b/runtime/bin/snapshot_utils.cc @@ -259,7 +259,7 @@ static void WriteSnapshotFile(const char* filename, if (write_magic_number) { // Write the magic number to indicate file is a script snapshot. - DartUtils::WriteMagicNumber(file); + DartUtils::WriteSnapshotMagicNumber(file); } if (!file->WriteFully(buffer, size)) {