Files
sdk/runtime/vm/kernel_binary.h
T
Jens Johansen 3e599229b9 [kernel] Don't scan strings up front when serializing.
Currently serializing the ast for kernel is done in two passe:

1) Scan the program to find and index all strings. These are then
   sorted based on frequency and assigned an id. All string-
   references are refering to that id. As small numbers use less
   space in the binary than big numbers, sorting the numbers by
   frequency saves a certain amount of space.
   In addition the string indexing is "hijacked" for the
   "LimitedBinaryPrinter" to also perform some CanonicalName
   re-indexing.

2) We then serialize the entire thing.

This CL gets rid of a pass by not indexing the strings up-front.
Whenever it is asked to serialize a string it adds it to the index
(if not already there). The serialization is otherwise the same.
This means that:

1) Strings are not sorted by frequency, i.e. the binary output size
   can by bigger (numbers below).

2) The stringindex and canonical names are moved to the end of the
   binary instead of the front. As we still need it up front for
   deserialization some additional data is added to the
   ProgramIndex.

3) The "hijacking" done in "LimitedBinaryPrinter" is replaced by
   an alternative.

4) We don't spend time on walking the tree twice.

The cost is the binary size. Compiling helloworld with fasta,
as well as looking at outline.dill, platform.dill and
vmservice_io.dill reveals these numbers:

* helloworld.dill is 0.657248732% bigger (26573 bytes)
* outline.dill is 1.686911399% bigger (9395 bytes)
* platform.dill is 0.657062238% bigger (26565 bytes)
* vmservice_io.dill is 0.44991899% bigger (19147 bytes)

The cost does thus not appear to be very big.

The gain is the serialization time.

From 20 runs of an instrumented VM/serialization, running numbers
through calculations stolens from ministat
(https://www.freebsd.org/cgi/man.cgi?query=ministat) reveals the
following:

* Serialization time: -21.69% +/- 1.44%

* Total time spend in relevant parts of bootstrap_nocore.cc,
dart_api_impl.cc (Dart_LoadKernel), bootstrap_nocore.cc,
dart_api_impl.cc (LoadKernelProgram) as well as serialization:
-14.01% +/- 1.58%

From 5 runs of
"time python tools/test.py -m release -cdartk language -j6"
(again run through ministat calculations):

* real: -4.18% +/- 0.5%
* user: -4.2% +/- 0.29%
* sys: No difference at 95%
* user+sys: -3.3% +/- 0.36%

Change-Id: I1c220eac083496994f0a9f1e2a2445b3707c9a93
Reviewed-on: https://dart-review.googlesource.com/2880
Reviewed-by: Samir Jindel <sjindel@google.com>
2017-09-06 11:55:16 +00:00

350 lines
9.4 KiB
C++

// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_VM_KERNEL_BINARY_H_
#define RUNTIME_VM_KERNEL_BINARY_H_
#if !defined(DART_PRECOMPILED_RUNTIME)
#include <map>
#include "vm/compiler/frontend/kernel_to_il.h"
#include "vm/kernel.h"
#include "vm/object.h"
namespace dart {
namespace kernel {
static const uint32_t kMagicProgramFile = 0x90ABCDEFu;
// Keep in sync with package:dynamo/lib/binary/tag.dart
enum Tag {
kNothing = 0,
kSomething = 1,
kClass = 2,
kFunctionNode = 3,
kField = 4,
kConstructor = 5,
kProcedure = 6,
kInvalidInitializer = 7,
kFieldInitializer = 8,
kSuperInitializer = 9,
kRedirectingInitializer = 10,
kLocalInitializer = 11,
kDirectPropertyGet = 15,
kDirectPropertySet = 16,
kDirectMethodInvocation = 17,
kConstStaticInvocation = 18,
kInvalidExpression = 19,
kVariableGet = 20,
kVariableSet = 21,
kPropertyGet = 22,
kPropertySet = 23,
kSuperPropertyGet = 24,
kSuperPropertySet = 25,
kStaticGet = 26,
kStaticSet = 27,
kMethodInvocation = 28,
kSuperMethodInvocation = 29,
kStaticInvocation = 30,
kConstructorInvocation = 31,
kConstConstructorInvocation = 32,
kNot = 33,
kLogicalExpression = 34,
kConditionalExpression = 35,
kStringConcatenation = 36,
kIsExpression = 37,
kAsExpression = 38,
kStringLiteral = 39,
kDoubleLiteral = 40,
kTrueLiteral = 41,
kFalseLiteral = 42,
kNullLiteral = 43,
kSymbolLiteral = 44,
kTypeLiteral = 45,
kThisExpression = 46,
kRethrow = 47,
kThrow = 48,
kListLiteral = 49,
kMapLiteral = 50,
kAwaitExpression = 51,
kFunctionExpression = 52,
kLet = 53,
kPositiveIntLiteral = 55,
kNegativeIntLiteral = 56,
kBigIntLiteral = 57,
kConstListLiteral = 58,
kConstMapLiteral = 59,
kInvalidStatement = 60,
kExpressionStatement = 61,
kBlock = 62,
kEmptyStatement = 63,
kAssertStatement = 64,
kLabeledStatement = 65,
kBreakStatement = 66,
kWhileStatement = 67,
kDoStatement = 68,
kForStatement = 69,
kForInStatement = 70,
kSwitchStatement = 71,
kContinueSwitchStatement = 72,
kIfStatement = 73,
kReturnStatement = 74,
kTryCatch = 75,
kTryFinally = 76,
kYieldStatement = 77,
kVariableDeclaration = 78,
kFunctionDeclaration = 79,
kAsyncForInStatement = 80,
kTypedefType = 87,
kVectorType = 88,
kBottomType = 89,
kInvalidType = 90,
kDynamicType = 91,
kVoidType = 92,
kInterfaceType = 93,
kFunctionType = 94,
kTypeParameterType = 95,
kSimpleInterfaceType = 96,
kSimpleFunctionType = 97,
kVectorCreation = 102,
kVectorGet = 103,
kVectorSet = 104,
kVectorCopy = 105,
kClosureCreation = 106,
kSpecializedTagHighBit = 0x80, // 10000000
kSpecializedTagMask = 0xF8, // 11111000
kSpecializedPayloadMask = 0x7, // 00000111
kSpecializedVariableGet = 128,
kSpecializedVariableSet = 136,
kSpecialIntLiteral = 144,
};
static const int SpecializedIntLiteralBias = 3;
static const int LibraryCountFieldCountFromEnd = 2;
static const int SourceTableFieldCountFromFirstLibraryOffset = 4;
class Reader {
public:
Reader(const uint8_t* buffer, intptr_t size)
: raw_buffer_(buffer), typed_data_(NULL), size_(size), offset_(0) {}
explicit Reader(const TypedData& typed_data)
: raw_buffer_(NULL),
typed_data_(&typed_data),
size_(typed_data.IsNull() ? 0 : typed_data.Length()),
offset_(0) {}
uint32_t ReadUInt32() {
ASSERT(offset_ + 4 <= size_);
const uint8_t* buffer = this->buffer();
uint32_t value = (buffer[offset_ + 0] << 24) | (buffer[offset_ + 1] << 16) |
(buffer[offset_ + 2] << 8) | (buffer[offset_ + 3] << 0);
offset_ += 4;
return value;
}
uint32_t ReadUInt() {
ASSERT(offset_ + 1 <= size_);
const uint8_t* buffer = this->buffer();
uint8_t byte0 = buffer[offset_];
if ((byte0 & 0x80) == 0) {
// 0...
offset_++;
return byte0;
} else if ((byte0 & 0xc0) == 0x80) {
// 10...
ASSERT(offset_ + 2 <= size_);
uint32_t value = ((byte0 & ~0x80) << 8) | (buffer[offset_ + 1]);
offset_ += 2;
return value;
} else {
// 11...
ASSERT(offset_ + 4 <= size_);
uint32_t value = ((byte0 & ~0xc0) << 24) | (buffer[offset_ + 1] << 16) |
(buffer[offset_ + 2] << 8) | (buffer[offset_ + 3] << 0);
offset_ += 4;
return value;
}
}
/**
* Read and return a TokenPosition from this reader.
*/
TokenPosition ReadPosition() {
// Position is saved as unsigned,
// but actually ranges from -1 and up (thus the -1)
intptr_t value = ReadUInt() - 1;
TokenPosition result = TokenPosition(value);
max_position_ = Utils::Maximum(max_position_, result);
if (min_position_.IsNoSource()) {
min_position_ = result;
} else if (result.IsReal()) {
min_position_ = Utils::Minimum(min_position_, result);
}
return result;
}
intptr_t ReadListLength() { return ReadUInt(); }
uint8_t ReadByte() { return buffer()[offset_++]; }
uint8_t PeekByte() { return buffer()[offset_]; }
bool ReadBool() { return (ReadByte() & 1) == 1; }
uint8_t ReadFlags() { return ReadByte(); }
Tag ReadTag(uint8_t* payload = NULL) {
uint8_t byte = ReadByte();
bool has_payload = (byte & kSpecializedTagHighBit) != 0;
if (has_payload) {
if (payload != NULL) {
*payload = byte & kSpecializedPayloadMask;
}
return static_cast<Tag>(byte & kSpecializedTagMask);
} else {
return static_cast<Tag>(byte);
}
}
Tag PeekTag(uint8_t* payload = NULL) {
uint8_t byte = PeekByte();
bool has_payload = (byte & kSpecializedTagHighBit) != 0;
if (has_payload) {
if (payload != NULL) {
*payload = byte & kSpecializedPayloadMask;
}
return static_cast<Tag>(byte & kSpecializedTagMask);
} else {
return static_cast<Tag>(byte);
}
}
void EnsureEnd() {
if (offset_ != size_) {
FATAL2(
"Reading Kernel file: Expected to be at EOF "
"(offset: %" Pd ", size: %" Pd ")",
offset_, size_);
}
}
// The largest position read yet (since last reset).
// This is automatically updated when calling ReadPosition,
// but can be overwritten (e.g. via the PositionScope class).
TokenPosition max_position() { return max_position_; }
// The smallest position read yet (since last reset).
// This is automatically updated when calling ReadPosition,
// but can be overwritten (e.g. via the PositionScope class).
TokenPosition min_position() { return min_position_; }
// A canonical name reference of -1 indicates none (for optional names), not
// the root name as in the canonical name table.
NameIndex ReadCanonicalNameReference() { return NameIndex(ReadUInt() - 1); }
intptr_t offset() { return offset_; }
void set_offset(intptr_t offset) { offset_ = offset; }
intptr_t size() { return size_; }
void set_size(intptr_t size) { size_ = size; }
const TypedData* typed_data() { return typed_data_; }
void set_typed_data(const TypedData* typed_data) { typed_data_ = typed_data; }
const uint8_t* raw_buffer() { return raw_buffer_; }
void set_raw_buffer(const uint8_t* raw_buffer) { raw_buffer_ = raw_buffer; }
TypedData& CopyDataToVMHeap(Zone* zone,
intptr_t from_byte,
intptr_t to_byte) {
intptr_t size = to_byte - from_byte;
TypedData& data = TypedData::Handle(
zone, TypedData::New(kTypedDataUint8ArrayCid, size, Heap::kOld));
{
NoSafepointScope no_safepoint;
memmove(data.DataAddr(0), buffer() + from_byte, size);
}
return data;
}
uint8_t* CopyDataIntoZone(Zone* zone, intptr_t offset, intptr_t length) {
uint8_t* buffer_ = zone->Alloc<uint8_t>(length);
{
NoSafepointScope no_safepoint;
memmove(buffer_, buffer() + offset, length);
}
return buffer_;
}
private:
const uint8_t* buffer() {
if (raw_buffer_ != NULL) {
return raw_buffer_;
}
NoSafepointScope no_safepoint;
return reinterpret_cast<uint8_t*>(typed_data_->DataAddr(0));
}
const uint8_t* raw_buffer_;
const TypedData* typed_data_;
intptr_t size_;
intptr_t offset_;
TokenPosition max_position_;
TokenPosition min_position_;
intptr_t current_script_id_;
friend class PositionScope;
friend class Program;
};
// A helper class that resets the readers min and max positions both upon
// initialization and upon destruction, i.e. when created the min an max
// positions will be reset to "noSource", when destructing the min and max will
// be reset to have they value they would have had, if they hadn't been reset in
// the first place.
class PositionScope {
public:
explicit PositionScope(Reader* reader)
: reader_(reader),
min_(reader->min_position_),
max_(reader->max_position_) {
reader->min_position_ = reader->max_position_ = TokenPosition::kNoSource;
}
~PositionScope() {
if (reader_->min_position_.IsNoSource()) {
reader_->min_position_ = min_;
} else if (min_.IsReal()) {
reader_->min_position_ = Utils::Minimum(reader_->min_position_, min_);
}
reader_->max_position_ = Utils::Maximum(reader_->max_position_, max_);
}
private:
Reader* reader_;
TokenPosition min_;
TokenPosition max_;
};
} // namespace kernel
} // namespace dart
#endif // !defined(DART_PRECOMPILED_RUNTIME)
#endif // RUNTIME_VM_KERNEL_BINARY_H_