dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[standalone] Automatically decompress gzip'd resources, including sources and script snapshots.

Browse Source 
Closes #30315

R=asiva@google.com, zra@google.com

Review-Url: https://codereview.chromium.org/2991393002 .
This commit is contained in:
Ryan Macnak
2017-08-11 12:18:16 -07:00
parent ada4db3fb6
commit 7dfe62f70d
9 changed files with 200 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/bin/BUILD.gn
+12 -1
View File
@@ -316,6 +316,8 @@ template("build_gen_snapshot") {
"address_sanitizer.cc",
"builtin.cc",
"builtin.h",
"gzip.cc",
"gzip.h",
"loader.cc",
"loader.h",
@@ -329,7 +331,10 @@ template("build_gen_snapshot") {
"vmservice_impl.h",
]
include_dirs = [ ".." ]
include_dirs = [
"..",
"//third_party",
]
if (dart_use_tcmalloc) {
deps += [ "//third_party/tcmalloc" ]
@@ -760,6 +765,8 @@ dart_executable("dart") {
"dfe.h",
"loader.cc",
"loader.h",
"gzip.cc",
"gzip.h",
"$target_gen_dir/resources_gen.cc",
]
if (dart_runtime_mode == "release") {
@@ -778,6 +785,8 @@ dart_executable("dart_precompiled_runtime") {
"snapshot_empty.cc",
"loader.cc",
"loader.h",
"gzip.cc",
"gzip.h",
]
if (dart_runtime_mode == "release") {
extra_sources += [ "observatory_assets_empty.cc" ]
@@ -816,6 +825,8 @@ dart_executable("dart_bootstrap") {
"dfe.h",
"loader.cc",
"loader.h",
"gzip.cc",
"gzip.h",
"observatory_assets_empty.cc",
"snapshot_empty.cc",
runtime/bin/dartutils.cc
+27 -21
View File
@@ -48,14 +48,15 @@ const char* const DartUtils::kHttpScheme = "http:";
const char* const DartUtils::kVMServiceLibURL = "dart:vmservice";
struct MagicNumberData {
static const intptr_t kLength = 4;
static const intptr_t kMaxLength = 4;
const uint8_t bytes[kLength];
bool should_skip;
intptr_t length;
const uint8_t bytes[kMaxLength];
};
MagicNumberData snapshot_magic_number = {{0xf5, 0xf5, 0xdc, 0xdc}, true};
MagicNumberData kernel_magic_number = {{0x90, 0xab, 0xcd, 0xef}, false};
MagicNumberData snapshot_magic_number = {4, {0xf5, 0xf5, 0xdc, 0xdc}};
MagicNumberData kernel_magic_number = {4, {0x90, 0xab, 0xcd, 0xef}};
MagicNumberData gzip_magic_number = {2, {0x1f, 0x8b, 0, 0}};
static bool IsWindowsHost() {
#if defined(HOST_OS_WINDOWS)
@@ -339,40 +340,45 @@ Dart_Handle DartUtils::ResolveScript(Dart_Handle url) {
kNumArgs, dart_args);
}
static bool CheckMagicNumber(const uint8_t** buf,
intptr_t* len,
static bool CheckMagicNumber(const uint8_t* buffer,
intptr_t buffer_length,
const MagicNumberData& magic_number) {
if ((*len >= MagicNumberData::kLength) &&
(memcmp(*buf, magic_number.bytes, MagicNumberData::kLength) == 0)) {
if (magic_number.should_skip) {
*buf += MagicNumberData::kLength;
*len -= MagicNumberData::kLength;
}
return true;
if ((buffer_length >= magic_number.length)) {
return memcmp(buffer, magic_number.bytes, magic_number.length) == 0;
}
return false;
}
DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t** buf,
intptr_t* len) {
if (CheckMagicNumber(buf, len, snapshot_magic_number)) {
DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer,
intptr_t buffer_length) {
if (CheckMagicNumber(buffer, buffer_length, snapshot_magic_number)) {
return kSnapshotMagicNumber;
}
if (CheckMagicNumber(buf, len, kernel_magic_number)) {
if (CheckMagicNumber(buffer, buffer_length, kernel_magic_number)) {
return kKernelMagicNumber;
}
if (CheckMagicNumber(buffer, buffer_length, gzip_magic_number)) {
return kGzipMagicNumber;
}
return kUnknownMagicNumber;
}
void DartUtils::WriteMagicNumber(File* file) {
void DartUtils::WriteSnapshotMagicNumber(File* file) {
// Write a magic number and version information into the snapshot file.
bool bytes_written =
file->WriteFully(snapshot_magic_number.bytes, MagicNumberData::kLength);
bool bytes_written = file->WriteFully(snapshot_magic_number.bytes,
snapshot_magic_number.length);
ASSERT(bytes_written);
}
void DartUtils::SkipSnapshotMagicNumber(const uint8_t** buffer,
intptr_t* buffer_length) {
*buffer += snapshot_magic_number.length;
*buffer_length -= snapshot_magic_number.length;
}
void FUNCTION_NAME(Builtin_GetCurrentDirectory)(Dart_NativeArguments args) {
const char* current = Directory::Current();
if (current != NULL) {
runtime/bin/dartutils.h
+6 -10
View File
@@ -200,21 +200,17 @@ class DartUtils {
enum MagicNumber {
kSnapshotMagicNumber,
kKernelMagicNumber,
kGzipMagicNumber,
kUnknownMagicNumber
};
// static const uint8_t* GetMagicNumber(MagicNumber number);
// Sniffs the specified text_buffer to see if it contains the magic number
// representing a script snapshot. If the text_buffer is a script snapshot
// the return value is an updated pointer to the text_buffer pointing past
// the magic number value. The 'buffer_len' parameter is also appropriately
// adjusted.
static MagicNumber SniffForMagicNumber(const uint8_t** text_buffer,
intptr_t* buffer_len);
// Checks if the buffer is a script snapshot, kernel file, or gzip file.
static MagicNumber SniffForMagicNumber(const uint8_t* text_buffer,
intptr_t buffer_len);
// Write a magic number to indicate a script snapshot file.
static void WriteMagicNumber(File* file);
static void WriteSnapshotMagicNumber(File* file);
static void SkipSnapshotMagicNumber(const uint8_t** buffer, intptr_t* length);
// Global state that stores the original working directory..
static const char* original_working_directory;
runtime/bin/dfe.cc
+1 -4
View File
@@ -143,10 +143,7 @@ bool DFE::TryReadKernelFile(const char* script_uri,
DartUtils::ReadFile(&buffer, kernel_ir_size, script_file);
DartUtils::CloseFile(script_file);
if (*kernel_ir_size > 0 && buffer != NULL) {
// We need a temporary variable because SniffForMagicNumber modifies the
// buffer pointer to skip snapshot magic number.
const uint8_t* temp = buffer;
if (DartUtils::SniffForMagicNumber(&temp, kernel_ir_size) !=
if (DartUtils::SniffForMagicNumber(buffer, *kernel_ir_size) !=
DartUtils::kKernelMagicNumber) {
free(const_cast<uint8_t*>(buffer));
*kernel_ir = NULL;
runtime/bin/gzip.cc
+86
View File
@@ -0,0 +1,86 @@
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "bin/gzip.h"
#include "platform/assert.h"
#include "platform/globals.h"
#include "zlib/zlib.h"
namespace dart {
namespace bin {
void Decompress(const uint8_t* input,
intptr_t input_len,
uint8_t** output,
intptr_t* output_length) {
ASSERT(input != NULL);
ASSERT(input_len > 0);
ASSERT(output != NULL);
ASSERT(output_length != NULL);
const intptr_t kChunkSize = 256 * 1024;
// Initialize output.
intptr_t output_capacity = input_len * 2;
if (output_capacity < kChunkSize) {
output_capacity = kChunkSize;
}
*output = reinterpret_cast<uint8_t*>(malloc(output_capacity));
uint8_t chunk_out[kChunkSize];
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = 0;
int ret = inflateInit2(&strm, 32 + MAX_WBITS);
ASSERT(ret == Z_OK);
intptr_t input_cursor = 0;
intptr_t output_cursor = 0;
do {
// Setup input.
intptr_t size_in = input_len - input_cursor;
if (size_in > kChunkSize) {
size_in = kChunkSize;
}
strm.avail_in = size_in;
strm.next_in = const_cast<uint8_t*>(&input[input_cursor]);
// Inflate until we've exhausted the current input chunk.
do {
// Setup output.
strm.avail_out = kChunkSize;
strm.next_out = &chunk_out[0];
// Inflate.
ret = inflate(&strm, Z_SYNC_FLUSH);
// We either hit the end of the stream or made forward progress.
ASSERT((ret == Z_STREAM_END) || (ret == Z_OK));
// Grow output buffer size.
intptr_t size_out = kChunkSize - strm.avail_out;
if (size_out > (output_capacity - output_cursor)) {
output_capacity *= 2;
ASSERT(size_out <= (output_capacity - output_cursor));
*output = reinterpret_cast<uint8_t*>(realloc(*output, output_capacity));
}
// Copy output.
memmove(&((*output)[output_cursor]), &chunk_out[0], size_out);
output_cursor += size_out;
} while (strm.avail_out == 0);
// We've processed size_in bytes.
input_cursor += size_in;
// We're finished decompressing when zlib tells us.
} while (ret != Z_STREAM_END);
inflateEnd(&strm);
*output_length = output_cursor;
}
} // namespace bin
} // namespace dart
runtime/bin/gzip.h
+24
View File
@@ -0,0 +1,24 @@
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_BIN_GZIP_H_
#define RUNTIME_BIN_GZIP_H_
#include "platform/globals.h"
namespace dart {
namespace bin {
// |input| is assumed to be a gzipped stream.
// This function allocates the output buffer in the C heap and the caller
// is responsible for freeing it.
void Decompress(const uint8_t* input,
intptr_t input_len,
uint8_t** output,
intptr_t* output_length);
} // namespace bin
} // namespace dart
#endif // RUNTIME_BIN_GZIP_H_
runtime/bin/loader.cc
+41 -2
View File
@@ -9,6 +9,7 @@
#include "bin/dfe.h"
#include "bin/extensions.h"
#include "bin/file.h"
#include "bin/gzip.h"
#include "bin/lockers.h"
#include "bin/utils.h"
#include "include/dart_tools_api.h"
@@ -318,6 +319,39 @@ void Loader::ResolveDependenciesAsFilePaths() {
}
}
class ScopedDecompress : public ValueObject {
public:
ScopedDecompress(const uint8_t** payload, intptr_t* payload_length)
: payload_(payload),
payload_length_(payload_length),
decompressed_(NULL) {
DartUtils::MagicNumber payload_type =
DartUtils::SniffForMagicNumber(*payload, *payload_length);
if (payload_type == DartUtils::kGzipMagicNumber) {
int64_t start = Dart_TimelineGetMicros();
intptr_t decompressed_length = 0;
Decompress(*payload, *payload_length, &decompressed_,
&decompressed_length);
int64_t end = Dart_TimelineGetMicros();
Dart_TimelineEvent("Decompress", start, end, Dart_Timeline_Event_Duration,
0, NULL, NULL);
*payload_ = decompressed_;
*payload_length_ = decompressed_length;
}
}
~ScopedDecompress() {
if (decompressed_ != NULL) {
free(decompressed_);
}
}
private:
const uint8_t** payload_;
intptr_t* payload_length_;
uint8_t* decompressed_;
};
bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) {
// We have to copy everything we care about out of |result| because after
// dropping the lock below |result| may no longer valid.
@@ -388,11 +422,15 @@ bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) {
// Check for payload and load accordingly.
const uint8_t* payload = result->payload;
intptr_t payload_length = result->payload_length;
// Decompress if gzip'd.
ScopedDecompress decompress(&payload, &payload_length);
const DartUtils::MagicNumber payload_type =
DartUtils::SniffForMagicNumber(&payload, &payload_length);
DartUtils::SniffForMagicNumber(payload, payload_length);
Dart_Handle source = Dart_Null();
if (payload_type == DartUtils::kUnknownMagicNumber) {
source = Dart_NewStringFromUTF8(result->payload, result->payload_length);
source = Dart_NewStringFromUTF8(payload, payload_length);
if (Dart_IsError(source)) {
loader->error_ =
DartUtils::NewError("%s is not a valid UTF-8 script",
@@ -424,6 +462,7 @@ bool Loader::ProcessResultLocked(Loader* loader, Loader::IOResult* result) {
} break;
case Dart_kScriptTag:
if (payload_type == DartUtils::kSnapshotMagicNumber) {
DartUtils::SkipSnapshotMagicNumber(&payload, &payload_length);
dart_result = Dart_LoadScriptFromSnapshot(payload, payload_length);
reload_extensions = true;
} else if (payload_type == DartUtils::kKernelMagicNumber) {
runtime/bin/main.cc
+2 -66
View File
@@ -32,7 +32,7 @@
#include "platform/hashmap.h"
#include "platform/text_buffer.h"
#if !defined(DART_PRECOMPILER)
#include "zlib/zlib.h"
#include "bin/gzip.h"
#endif
#include "vm/kernel.h"
@@ -1658,74 +1658,10 @@ bool RunMainIsolate(const char* script_name, CommandLineOptions* dart_options) {
extern unsigned int observatory_assets_archive_len;
extern const uint8_t* observatory_assets_archive;
// |input| is assumed to be a gzipped stream.
// This function allocates the output buffer in the C heap and the caller
// is responsible for freeing it.
void Decompress(const uint8_t* input,
unsigned int input_len,
uint8_t** output,
unsigned int* output_length) {
ASSERT(input != NULL);
ASSERT(input_len > 0);
ASSERT(output != NULL);
ASSERT(output_length != NULL);
// Initialize output.
*output = NULL;
*output_length = 0;
const unsigned int kChunkSize = 256 * 1024;
uint8_t chunk_out[kChunkSize];
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = 0;
int ret = inflateInit2(&strm, 32 + MAX_WBITS);
ASSERT(ret == Z_OK);
unsigned int input_cursor = 0;
unsigned int output_cursor = 0;
do {
// Setup input.
unsigned int size_in = input_len - input_cursor;
if (size_in > kChunkSize) {
size_in = kChunkSize;
}
strm.avail_in = size_in;
strm.next_in = const_cast<uint8_t*>(&input[input_cursor]);
// Inflate until we've exhausted the current input chunk.
do {
// Setup output.
strm.avail_out = kChunkSize;
strm.next_out = &chunk_out[0];
// Inflate.
ret = inflate(&strm, Z_SYNC_FLUSH);
// We either hit the end of the stream or made forward progress.
ASSERT((ret == Z_STREAM_END) || (ret == Z_OK));
// Grow output buffer size.
unsigned int size_out = kChunkSize - strm.avail_out;
*output_length += size_out;
*output = reinterpret_cast<uint8_t*>(realloc(*output, *output_length));
// Copy output.
memmove(&((*output)[output_cursor]), &chunk_out[0], size_out);
output_cursor += size_out;
} while (strm.avail_out == 0);
// We've processed size_in bytes.
input_cursor += size_in;
// We're finished decompressing when zlib tells us.
} while (ret != Z_STREAM_END);
inflateEnd(&strm);
}
Dart_Handle GetVMServiceAssetsArchiveCallback() {
uint8_t* decompressed = NULL;
unsigned int decompressed_len = 0;
intptr_t decompressed_len = 0;
Decompress(observatory_assets_archive, observatory_assets_archive_len,
&decompressed, &decompressed_len);
Dart_Handle tar_file =
runtime/bin/snapshot_utils.cc
+1 -1
View File
@@ -259,7 +259,7 @@ static void WriteSnapshotFile(const char* filename,
if (write_magic_number) {
// Write the magic number to indicate file is a script snapshot.
DartUtils::WriteMagicNumber(file);
DartUtils::WriteSnapshotMagicNumber(file);
}
if (!file->WriteFully(buffer, size)) {