dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, bytecode] Populate bytecode cache when initializing an incremental compiler from a kernel file.

Browse Source 
AST
"without_change_elapsed_time_ms": 2863,
"implementation_change_elapsed_time_ms": 6009,
"interface_change_elapsed_time_ms": 5888,
"with_coverage_time_ms": 2884

BYTECODE BEFORE
"without_change_elapsed_time_ms": 5216,
"implementation_change_elapsed_time_ms": 8517,
"interface_change_elapsed_time_ms": 8895,
"with_coverage_time_ms": 5462

BYTECODE AFTER
"without_change_elapsed_time_ms": 3384,
"implementation_change_elapsed_time_ms": 8139,
"interface_change_elapsed_time_ms": 8073,
"with_coverage_time_ms": 3473

Change-Id: Ic8e1c183070cb6019ea48f16c04ea5363df41a97
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119620
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
This commit is contained in:
Ryan Macnak
2019-10-02 23:53:42 +00:00
committed by commit-bot@chromium.org
parent 22479a17ff
commit d65a60db00
6 changed files with 143 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/front_end/lib/src/fasta/incremental_compiler.dart
+4 -5
View File
@@ -9,11 +9,9 @@ import 'dart:async' show Future;
import 'package:front_end/src/fasta/dill/dill_class_builder.dart'
show DillClassBuilder;
import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder;
import 'package:kernel/binary/ast_from_binary.dart'
show
BinaryBuilder,
BinaryBuilderWithMetadata,
CanonicalNameError,
CanonicalNameSdkError,
InvalidKernelVersionError;
@@ -730,7 +728,7 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
if (summaryBytes != null) {
ticker.logMs("Read ${c.options.sdkSummary}");
data.component = c.options.target.configureComponent(new Component());
new BinaryBuilder(summaryBytes,
new BinaryBuilderWithMetadata(summaryBytes,
disableLazyReading: false, disableLazyClassReading: true)
.readComponent(data.component);
ticker.logMs("Deserialized ${c.options.sdkSummary}");
@@ -755,7 +753,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
// We're going to output all we read here so lazy loading it
// doesn't make sense.
new BinaryBuilder(initializationBytes, disableLazyReading: true)
new BinaryBuilderWithMetadata(initializationBytes,
disableLazyReading: true)
.readComponent(data.component, checkCanonicalNames: true);
// Check the any package-urls still point to the same file
pkg/kernel/lib/binary/ast_from_binary.dart
+10 -2
View File
@@ -2269,8 +2269,16 @@ class BinaryBuilderWithMetadata extends BinaryBuilder implements BinarySource {
/// and are awaiting to be parsed and attached to nodes.
List<_MetadataSubsection> _subsections;
BinaryBuilderWithMetadata(bytes, [filename])
: super(bytes, filename: filename);
BinaryBuilderWithMetadata(List<int> bytes,
{String filename,
bool disableLazyReading = false,
bool disableLazyClassReading = false,
bool alwaysCreateNewNamedNodes})
: super(bytes,
filename: filename,
disableLazyReading: disableLazyReading,
disableLazyClassReading: disableLazyClassReading,
alwaysCreateNewNamedNodes: alwaysCreateNewNamedNodes);
@override
void _readMetadataMappings(
pkg/vm/lib/frontend_server.dart
+61 -29
View File
@@ -25,6 +25,9 @@ import 'package:kernel/kernel.dart'
import 'package:path/path.dart' as path;
import 'package:usage/uuid/uuid.dart';
import 'package:vm/metadata/binary_cache.dart'
show BinaryCacheMetadataRepository;
import 'package:vm/bytecode/gen_bytecode.dart'
show generateBytecode, createFreshComponentWithBytecode;
import 'package:vm/bytecode/options.dart' show BytecodeOptions;
@@ -364,6 +367,15 @@ class FrontendCompiler implements CompilerInterface {
];
}
if (compilerOptions.bytecode && _initializeFromDill != null) {
// If we are generating bytecode, put bytecode only (not AST) in
// [_kernelBinaryFilename], which the user of this tool will eventually
// feed to Flutter engine or flutter_tester. Use a separate file to cache
// the AST result to initialize the incremental compiler for the next
// invocation of this tool.
_initializeFromDill += ".ast";
}
_compilerOptions = compilerOptions;
_bytecodeOptions = bytecodeOptions;
@@ -470,20 +482,53 @@ class FrontendCompiler implements CompilerInterface {
writeDillFile(Component component, String filename,
{bool filterExternal: false}) async {
final IOSink sink = new File(filename).openWrite();
// Remove the cache that came either from this function or from
// initializing from a kernel file.
component.metadata.remove(BinaryCacheMetadataRepository.repositoryTag);
if (_compilerOptions.bytecode) {
await runWithFrontEndCompilerContext(
_mainSource, _compilerOptions, component, () async {
if (_options['incremental']) {
await forEachPackage(component,
(String package, List<Library> libraries) async {
_writePackage(component, package, libraries, sink);
});
} else {
_writePackage(component, "main", component.libraries, sink);
{
// Generate bytecode as the output proper.
final IOSink sink = new File(filename).openWrite();
await runWithFrontEndCompilerContext(
_mainSource, _compilerOptions, component, () async {
if (_options['incremental']) {
await forEachPackage(component,
(String package, List<Library> libraries) async {
_writePackage(component, package, libraries, sink);
});
} else {
_writePackage(component, 'main', component.libraries, sink);
}
});
await sink.close();
}
{
// Generate AST as a cache.
final repository = new BinaryCacheMetadataRepository();
component.addMetadataRepository(repository);
for (var lib in component.libraries) {
var bytes = BinaryCacheMetadataRepository.lookup(lib);
if (bytes != null) {
repository.mapping[lib] = bytes;
}
}
});
final IOSink sink = new File(filename + ".ast").openWrite();
final BinaryPrinter printer = filterExternal
? new LimitedBinaryPrinter(
sink, (lib) => !lib.isExternal, true /* excludeUriToSource */)
: printerFactory.newBinaryPrinter(sink);
sortComponent(component);
printer.writeComponentFile(component);
await sink.close();
}
} else {
// Generate AST as the output proper.
final IOSink sink = new File(filename).openWrite();
final BinaryPrinter printer = filterExternal
? new LimitedBinaryPrinter(
sink, (lib) => !lib.isExternal, true /* excludeUriToSource */)
@@ -496,8 +541,8 @@ class FrontendCompiler implements CompilerInterface {
}
printer.writeComponentFile(component);
await sink.close();
}
await sink.close();
}
Future<Null> invalidateIfInitializingFromDill() async {
@@ -557,17 +602,6 @@ class FrontendCompiler implements CompilerInterface {
}
}
bool _elementsIdentical(List a, List b) {
if (a.length != b.length) return false;
for (int i = 0; i < a.length; i++) {
if (!identical(a[i], b[i])) return false;
}
return true;
}
final _packageLibraries = new Expando<List<Library>>();
final _packageBytes = new Expando<List<int>>();
void _writePackage(Component component, String package,
List<Library> libraries, IOSink sink) {
final canCache = libraries.isNotEmpty &&
@@ -576,10 +610,9 @@ class FrontendCompiler implements CompilerInterface {
package != "main";
if (canCache) {
var cachedLibraries = _packageLibraries[libraries.first];
if ((cachedLibraries != null) &&
_elementsIdentical(cachedLibraries, libraries)) {
sink.add(_packageBytes[libraries.first]);
var cachedBytes = BinaryCacheMetadataRepository.lookup(libraries.first);
if (cachedBytes != null) {
sink.add(cachedBytes);
return;
}
}
@@ -605,8 +638,7 @@ class FrontendCompiler implements CompilerInterface {
final bytes = byteSink.builder.takeBytes();
sink.add(bytes);
if (canCache) {
_packageLibraries[libraries.first] = libraries;
_packageBytes[libraries.first] = bytes;
BinaryCacheMetadataRepository.insert(libraries.first, bytes);
}
}
pkg/vm/lib/metadata/binary_cache.dart
+37
View File
@@ -0,0 +1,37 @@
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
library vm.metadata.binary_cache;
import 'package:kernel/ast.dart'
show BinarySink, BinarySource, MetadataRepository, Node, TreeNode;
class BinaryCacheMetadataRepository extends MetadataRepository<List<int>> {
static const repositoryTag = 'vm.binary_cache';
@override
String get tag => repositoryTag;
@override
final Map<TreeNode, List<int>> mapping = <TreeNode, List<int>>{};
@override
void writeToBinary(List<int> metadata, Node node, BinarySink sink) {
sink.writeByteList(metadata);
}
@override
List<int> readFromBinary(Node node, BinarySource source) {
List<int> result = source.readByteList();
_weakMap[node] = result;
return result;
}
static List<int> lookup(Node node) => _weakMap[node];
static void insert(Node node, List<int> metadata) {
_weakMap[node] = metadata;
}
static final _weakMap = new Expando<List<int>>();
}
pkg/vm/lib/metadata/bytecode.dart
+29
View File
@@ -68,3 +68,32 @@ class BytecodeMetadataRepository extends MetadataRepository<BytecodeMetadata> {
return new BytecodeMetadata(bytecodeComponent);
}
}
class BinaryCacheMetadataRepository extends MetadataRepository<List<int>> {
static const repositoryTag = 'vm.bytecode.cache';
@override
String get tag => repositoryTag;
@override
final Map<TreeNode, List<int>> mapping = <TreeNode, List<int>>{};
@override
void writeToBinary(List<int> metadata, Node node, BinarySink sink) {
sink.writeByteList(metadata);
}
@override
List<int> readFromBinary(Node node, BinarySource source) {
List<int> result = source.readByteList();
_weakMap[node] = result;
return result;
}
static List<int> lookup(Node node) => _weakMap[node];
static void insert(Node node, List<int> metadata) {
_weakMap[node] = metadata;
}
static final _weakMap = new Expando<List<int>>();
}
pkg/vm/lib/target/vm.dart
+2
View File
@@ -17,6 +17,7 @@ import 'package:kernel/transformations/continuation.dart' as transformAsync
import 'package:kernel/vm/constants_native_effects.dart'
show VmConstantsBackend;
import '../metadata/binary_cache.dart' show BinaryCacheMetadataRepository;
import '../transformations/call_site_annotator.dart' as callSiteAnnotator;
import '../transformations/list_factory_specializer.dart'
as listFactorySpecializer;
@@ -369,6 +370,7 @@ class VmTarget extends Target {
@override
Component configureComponent(Component component) {
callSiteAnnotator.addRepositoryTo(component);
component.addMetadataRepository(new BinaryCacheMetadataRepository());
return super.configureComponent(component);
}