diff --git a/pkg/front_end/lib/src/fasta/incremental_compiler.dart b/pkg/front_end/lib/src/fasta/incremental_compiler.dart index 43af7e58d2f..61e208b45b8 100644 --- a/pkg/front_end/lib/src/fasta/incremental_compiler.dart +++ b/pkg/front_end/lib/src/fasta/incremental_compiler.dart @@ -9,11 +9,9 @@ import 'dart:async' show Future; import 'package:front_end/src/fasta/dill/dill_class_builder.dart' show DillClassBuilder; -import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder; - import 'package:kernel/binary/ast_from_binary.dart' show - BinaryBuilder, + BinaryBuilderWithMetadata, CanonicalNameError, CanonicalNameSdkError, InvalidKernelVersionError; @@ -730,7 +728,7 @@ class IncrementalCompiler implements IncrementalKernelGenerator { if (summaryBytes != null) { ticker.logMs("Read ${c.options.sdkSummary}"); data.component = c.options.target.configureComponent(new Component()); - new BinaryBuilder(summaryBytes, + new BinaryBuilderWithMetadata(summaryBytes, disableLazyReading: false, disableLazyClassReading: true) .readComponent(data.component); ticker.logMs("Deserialized ${c.options.sdkSummary}"); @@ -755,7 +753,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator { // We're going to output all we read here so lazy loading it // doesn't make sense. - new BinaryBuilder(initializationBytes, disableLazyReading: true) + new BinaryBuilderWithMetadata(initializationBytes, + disableLazyReading: true) .readComponent(data.component, checkCanonicalNames: true); // Check the any package-urls still point to the same file diff --git a/pkg/kernel/lib/binary/ast_from_binary.dart b/pkg/kernel/lib/binary/ast_from_binary.dart index 952e3a545c6..4ef7288098d 100644 --- a/pkg/kernel/lib/binary/ast_from_binary.dart +++ b/pkg/kernel/lib/binary/ast_from_binary.dart @@ -2269,8 +2269,16 @@ class BinaryBuilderWithMetadata extends BinaryBuilder implements BinarySource { /// and are awaiting to be parsed and attached to nodes. List<_MetadataSubsection> _subsections; - BinaryBuilderWithMetadata(bytes, [filename]) - : super(bytes, filename: filename); + BinaryBuilderWithMetadata(List bytes, + {String filename, + bool disableLazyReading = false, + bool disableLazyClassReading = false, + bool alwaysCreateNewNamedNodes}) + : super(bytes, + filename: filename, + disableLazyReading: disableLazyReading, + disableLazyClassReading: disableLazyClassReading, + alwaysCreateNewNamedNodes: alwaysCreateNewNamedNodes); @override void _readMetadataMappings( diff --git a/pkg/vm/lib/frontend_server.dart b/pkg/vm/lib/frontend_server.dart index 2b12020ca39..4329c60e1c3 100644 --- a/pkg/vm/lib/frontend_server.dart +++ b/pkg/vm/lib/frontend_server.dart @@ -25,6 +25,9 @@ import 'package:kernel/kernel.dart' import 'package:path/path.dart' as path; import 'package:usage/uuid/uuid.dart'; +import 'package:vm/metadata/binary_cache.dart' + show BinaryCacheMetadataRepository; + import 'package:vm/bytecode/gen_bytecode.dart' show generateBytecode, createFreshComponentWithBytecode; import 'package:vm/bytecode/options.dart' show BytecodeOptions; @@ -364,6 +367,15 @@ class FrontendCompiler implements CompilerInterface { ]; } + if (compilerOptions.bytecode && _initializeFromDill != null) { + // If we are generating bytecode, put bytecode only (not AST) in + // [_kernelBinaryFilename], which the user of this tool will eventually + // feed to Flutter engine or flutter_tester. Use a separate file to cache + // the AST result to initialize the incremental compiler for the next + // invocation of this tool. + _initializeFromDill += ".ast"; + } + _compilerOptions = compilerOptions; _bytecodeOptions = bytecodeOptions; @@ -470,20 +482,53 @@ class FrontendCompiler implements CompilerInterface { writeDillFile(Component component, String filename, {bool filterExternal: false}) async { - final IOSink sink = new File(filename).openWrite(); + // Remove the cache that came either from this function or from + // initializing from a kernel file. + component.metadata.remove(BinaryCacheMetadataRepository.repositoryTag); + if (_compilerOptions.bytecode) { - await runWithFrontEndCompilerContext( - _mainSource, _compilerOptions, component, () async { - if (_options['incremental']) { - await forEachPackage(component, - (String package, List libraries) async { - _writePackage(component, package, libraries, sink); - }); - } else { - _writePackage(component, "main", component.libraries, sink); + { + // Generate bytecode as the output proper. + final IOSink sink = new File(filename).openWrite(); + await runWithFrontEndCompilerContext( + _mainSource, _compilerOptions, component, () async { + if (_options['incremental']) { + await forEachPackage(component, + (String package, List libraries) async { + _writePackage(component, package, libraries, sink); + }); + } else { + _writePackage(component, 'main', component.libraries, sink); + } + }); + await sink.close(); + } + + { + // Generate AST as a cache. + final repository = new BinaryCacheMetadataRepository(); + component.addMetadataRepository(repository); + for (var lib in component.libraries) { + var bytes = BinaryCacheMetadataRepository.lookup(lib); + if (bytes != null) { + repository.mapping[lib] = bytes; + } } - }); + + final IOSink sink = new File(filename + ".ast").openWrite(); + final BinaryPrinter printer = filterExternal + ? new LimitedBinaryPrinter( + sink, (lib) => !lib.isExternal, true /* excludeUriToSource */) + : printerFactory.newBinaryPrinter(sink); + + sortComponent(component); + + printer.writeComponentFile(component); + await sink.close(); + } } else { + // Generate AST as the output proper. + final IOSink sink = new File(filename).openWrite(); final BinaryPrinter printer = filterExternal ? new LimitedBinaryPrinter( sink, (lib) => !lib.isExternal, true /* excludeUriToSource */) @@ -496,8 +541,8 @@ class FrontendCompiler implements CompilerInterface { } printer.writeComponentFile(component); + await sink.close(); } - await sink.close(); } Future invalidateIfInitializingFromDill() async { @@ -557,17 +602,6 @@ class FrontendCompiler implements CompilerInterface { } } - bool _elementsIdentical(List a, List b) { - if (a.length != b.length) return false; - for (int i = 0; i < a.length; i++) { - if (!identical(a[i], b[i])) return false; - } - return true; - } - - final _packageLibraries = new Expando>(); - final _packageBytes = new Expando>(); - void _writePackage(Component component, String package, List libraries, IOSink sink) { final canCache = libraries.isNotEmpty && @@ -576,10 +610,9 @@ class FrontendCompiler implements CompilerInterface { package != "main"; if (canCache) { - var cachedLibraries = _packageLibraries[libraries.first]; - if ((cachedLibraries != null) && - _elementsIdentical(cachedLibraries, libraries)) { - sink.add(_packageBytes[libraries.first]); + var cachedBytes = BinaryCacheMetadataRepository.lookup(libraries.first); + if (cachedBytes != null) { + sink.add(cachedBytes); return; } } @@ -605,8 +638,7 @@ class FrontendCompiler implements CompilerInterface { final bytes = byteSink.builder.takeBytes(); sink.add(bytes); if (canCache) { - _packageLibraries[libraries.first] = libraries; - _packageBytes[libraries.first] = bytes; + BinaryCacheMetadataRepository.insert(libraries.first, bytes); } } diff --git a/pkg/vm/lib/metadata/binary_cache.dart b/pkg/vm/lib/metadata/binary_cache.dart new file mode 100644 index 00000000000..e2151ef12d1 --- /dev/null +++ b/pkg/vm/lib/metadata/binary_cache.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library vm.metadata.binary_cache; + +import 'package:kernel/ast.dart' + show BinarySink, BinarySource, MetadataRepository, Node, TreeNode; + +class BinaryCacheMetadataRepository extends MetadataRepository> { + static const repositoryTag = 'vm.binary_cache'; + + @override + String get tag => repositoryTag; + + @override + final Map> mapping = >{}; + + @override + void writeToBinary(List metadata, Node node, BinarySink sink) { + sink.writeByteList(metadata); + } + + @override + List readFromBinary(Node node, BinarySource source) { + List result = source.readByteList(); + _weakMap[node] = result; + return result; + } + + static List lookup(Node node) => _weakMap[node]; + static void insert(Node node, List metadata) { + _weakMap[node] = metadata; + } + + static final _weakMap = new Expando>(); +} diff --git a/pkg/vm/lib/metadata/bytecode.dart b/pkg/vm/lib/metadata/bytecode.dart index 83e4a8effd4..15781bdd3cd 100644 --- a/pkg/vm/lib/metadata/bytecode.dart +++ b/pkg/vm/lib/metadata/bytecode.dart @@ -68,3 +68,32 @@ class BytecodeMetadataRepository extends MetadataRepository { return new BytecodeMetadata(bytecodeComponent); } } + +class BinaryCacheMetadataRepository extends MetadataRepository> { + static const repositoryTag = 'vm.bytecode.cache'; + + @override + String get tag => repositoryTag; + + @override + final Map> mapping = >{}; + + @override + void writeToBinary(List metadata, Node node, BinarySink sink) { + sink.writeByteList(metadata); + } + + @override + List readFromBinary(Node node, BinarySource source) { + List result = source.readByteList(); + _weakMap[node] = result; + return result; + } + + static List lookup(Node node) => _weakMap[node]; + static void insert(Node node, List metadata) { + _weakMap[node] = metadata; + } + + static final _weakMap = new Expando>(); +} diff --git a/pkg/vm/lib/target/vm.dart b/pkg/vm/lib/target/vm.dart index 377a2894257..a647eb806ec 100644 --- a/pkg/vm/lib/target/vm.dart +++ b/pkg/vm/lib/target/vm.dart @@ -17,6 +17,7 @@ import 'package:kernel/transformations/continuation.dart' as transformAsync import 'package:kernel/vm/constants_native_effects.dart' show VmConstantsBackend; +import '../metadata/binary_cache.dart' show BinaryCacheMetadataRepository; import '../transformations/call_site_annotator.dart' as callSiteAnnotator; import '../transformations/list_factory_specializer.dart' as listFactorySpecializer; @@ -369,6 +370,7 @@ class VmTarget extends Target { @override Component configureComponent(Component component) { callSiteAnnotator.addRepositoryTo(component); + component.addMetadataRepository(new BinaryCacheMetadataRepository()); return super.configureComponent(component); }