diff --git a/pkg/compiler/lib/src/deferred_load.dart b/pkg/compiler/lib/src/deferred_load.dart index dedd5025ca0..974e8a07535 100644 --- a/pkg/compiler/lib/src/deferred_load.dart +++ b/pkg/compiler/lib/src/deferred_load.dart @@ -14,8 +14,7 @@ import '../compiler_new.dart' show OutputType; import 'common/metrics.dart' show Metric, Metrics, CountMetric, DurationMetric; import 'common/tasks.dart' show CompilerTask; import 'common.dart'; -import 'common_elements.dart' - show CommonElements, ElementEnvironment, KElementEnvironment; +import 'common_elements.dart' show CommonElements, KElementEnvironment; import 'compiler.dart' show Compiler; import 'constants/values.dart' show @@ -89,13 +88,13 @@ class _DeferredLoadTaskMetrics implements Metrics { String get namespace => 'deferred_load'; DurationMetric time = DurationMetric('time'); - CountMetric hunkListElements = CountMetric('hunkListElements'); + CountMetric outputUnitElements = CountMetric('outputUnitElements'); @override Iterable get primary => [time]; @override - Iterable get secondary => [hunkListElements]; + Iterable get secondary => [outputUnitElements]; } /// For each deferred import, find elements and constants to be loaded when that @@ -119,7 +118,7 @@ class DeferredLoadTask extends CompilerTask { /// A cache of the result of calling `computeImportDeferName` on the keys of /// this map. - final Map _importDeferName = {}; + final Map importDeferName = {}; /// A mapping from classes to their import set. Map _classToSet = {}; @@ -655,6 +654,7 @@ class DeferredLoadTask extends CompilerTask { counter++; importSet.unit = unit; _allOutputUnits.add(unit); + metrics.outputUnitElements.add(1); } // Generate an output unit for all import sets that are associated with an @@ -669,8 +669,7 @@ class DeferredLoadTask extends CompilerTask { _allOutputUnits.sort(); } - Map> _setupHunksToLoad() { - Map> hunksToLoad = {}; + void _setupImportNames() { Set usedImportNames = {}; for (ImportEntity import in _allDeferredImports) { @@ -679,38 +678,8 @@ class DeferredLoadTask extends CompilerTask { // Note: tools that process the json file to build multi-part initial load // bundles depend on the fact that makeUnique appends only digits, or a // period followed by digits. - _importDeferName[import] = makeUnique(result, usedImportNames, '.'); + importDeferName[import] = makeUnique(result, usedImportNames, '.'); } - - // Sort the output units in descending order of the number of imports they - // include. - - // The loading of the output units must be ordered because a superclass - // needs to be initialized before its subclass. - // But a class can only depend on another class in an output unit shared by - // a strict superset of the imports: - // By contradiction: Assume a class C in output unit shared by imports in - // the set S1 = (lib1,.., lib_n) depends on a class D in an output unit - // shared by S2 such that S2 not a superset of S1. Let lib_s be a library in - // S1 not in S2. lib_s must depend on C, and then in turn on D. Therefore D - // is not in the right output unit. - List sortedOutputUnits = _allOutputUnits.reversed.toList(); - - // For each deferred import we find out which outputUnits to load. - for (ImportEntity import in _allDeferredImports) { - // We expect to find an entry for any call to `loadLibrary`, even if - // there is no code to load. In that case, the entry will be an empty - // list. - hunksToLoad[_importDeferName[import]] = []; - for (OutputUnit outputUnit in sortedOutputUnits) { - if (outputUnit == _mainOutputUnit) continue; - if (outputUnit.imports.contains(import)) { - hunksToLoad[_importDeferName[import]].add(outputUnit); - metrics.hunkListElements.add(1); - } - } - } - return hunksToLoad; } /// Returns a name for a deferred import. @@ -880,7 +849,7 @@ class DeferredLoadTask extends CompilerTask { OutputUnitData _buildResult() { _createOutputUnits(); - Map> hunksToLoad = _setupHunksToLoad(); + _setupImportNames(); if (compiler.options.deferredGraphUri != null) { _dumpDeferredGraph(); } @@ -911,8 +880,7 @@ class DeferredLoadTask extends CompilerTask { localFunctionMap, constantMap, _allOutputUnits, - _importDeferName, - hunksToLoad, + importDeferName, _deferredImportDescriptions); } @@ -1031,10 +999,10 @@ class ImportDescription { /// The prefix this import is imported as. final String prefix; - final LibraryEntity _importingLibrary; + final LibraryEntity importingLibrary; ImportDescription.internal( - this.importingUri, this.prefix, this._importingLibrary); + this.importingUri, this.prefix, this.importingLibrary); ImportDescription( ImportEntity import, LibraryEntity importingLibrary, Uri mainLibraryUri) @@ -1412,20 +1380,11 @@ class OutputUnitData { final Map _localFunctionToUnit; final Map _constantToUnit; final List outputUnits; - final Map _importDeferName; - - /// A mapping from the name of a defer import to all the output units it - /// depends on in a list of lists to be loaded in the order they appear. - /// - /// For example {"lib1": [[lib1_lib2_lib3], [lib1_lib2, lib1_lib3], - /// [lib1]]} would mean that in order to load "lib1" first the hunk - /// lib1_lib2_lib2 should be loaded, then the hunks lib1_lib2 and lib1_lib3 - /// can be loaded in parallel. And finally lib1 can be loaded. - final Map> hunksToLoad; + final Map importDeferName; /// Because the token-stream is forgotten later in the program, we cache a /// description of each deferred import. - final Map _deferredImportDescriptions; + final Map deferredImportDescriptions; OutputUnitData( this.isProgramSplit, @@ -1436,9 +1395,8 @@ class OutputUnitData { this._localFunctionToUnit, this._constantToUnit, this.outputUnits, - this._importDeferName, - this.hunksToLoad, - this._deferredImportDescriptions); + this.importDeferName, + this.deferredImportDescriptions); // Creates J-world data from the K-world data. factory OutputUnitData.from( @@ -1461,12 +1419,12 @@ class OutputUnitData { Map constantToUnit = convertConstantMap(other._constantToUnit); Map deferredImportDescriptions = {}; - other._deferredImportDescriptions + other.deferredImportDescriptions .forEach((ImportEntity import, ImportDescription description) { deferredImportDescriptions[import] = ImportDescription.internal( description.importingUri, description.prefix, - convertLibrary(description._importingLibrary)); + convertLibrary(description.importingLibrary)); }); return OutputUnitData( @@ -1479,8 +1437,7 @@ class OutputUnitData { const {}, constantToUnit, other.outputUnits, - other._importDeferName, - other.hunksToLoad, + other.importDeferName, deferredImportDescriptions); } @@ -1511,11 +1468,6 @@ class OutputUnitData { }); Map importDeferName = source.readImportMap(source.readString); - Map> hunksToLoad = source.readStringMap(() { - return source.readList(() { - return outputUnits[source.readInt()]; - }); - }); Map deferredImportDescriptions = source.readImportMap(() { String importingUri = source.readString(); @@ -1535,7 +1487,6 @@ class OutputUnitData { constantToUnit, outputUnits, importDeferName, - hunksToLoad, deferredImportDescriptions); } @@ -1564,18 +1515,12 @@ class OutputUnitData { sink.writeConstantMap(_constantToUnit, (OutputUnit outputUnit) { sink.writeInt(outputUnitIndices[outputUnit]); }); - sink.writeImportMap(_importDeferName, sink.writeString); - sink.writeStringMap(hunksToLoad, (List outputUnits) { - sink.writeList( - outputUnits, - (OutputUnit outputUnit) => - sink.writeInt(outputUnitIndices[outputUnit])); - }); - sink.writeImportMap(_deferredImportDescriptions, + sink.writeImportMap(importDeferName, sink.writeString); + sink.writeImportMap(deferredImportDescriptions, (ImportDescription importDescription) { sink.writeString(importDescription.importingUri); sink.writeString(importDescription.prefix); - sink.writeLibrary(importDescription._importingLibrary); + sink.writeLibrary(importDescription.importingLibrary); }); sink.end(tag); } @@ -1690,7 +1635,7 @@ class OutputUnitData { /// Returns the unique name for the given deferred [import]. String getImportDeferName(Spannable node, ImportEntity import) { - String name = _importDeferName[import]; + String name = importDeferName[import]; if (name == null) { throw SpannableAssertionFailure(node, "No deferred name for $import."); } @@ -1699,48 +1644,7 @@ class OutputUnitData { /// Returns the names associated with each deferred import in [unit]. Iterable getImportNames(OutputUnit unit) { - return unit.imports.map((i) => _importDeferName[i]); - } - - /// Returns a json-style map for describing what files that are loaded by a - /// given deferred import. - /// The mapping is structured as: - /// library uri -> {"name": library name, "files": (prefix -> list of files)} - /// Where - /// - /// - is the relative uri of the library making a deferred - /// import. - /// - is the name of the library, or "" if it is - /// unnamed. - /// - is the `as` prefix used for a given deferred import. - /// - is a list of the filenames the must be loaded when that - /// import is loaded. - Map> computeDeferredMap( - CompilerOptions options, ElementEnvironment elementEnvironment, - {Set omittedUnits}) { - omittedUnits ??= Set(); - Map> mapping = {}; - - _deferredImportDescriptions.keys.forEach((ImportEntity import) { - List outputUnits = hunksToLoad[_importDeferName[import]]; - ImportDescription description = _deferredImportDescriptions[import]; - String getName(LibraryEntity library) { - var name = elementEnvironment.getLibraryName(library); - return name == '' ? '' : name; - } - - Map libraryMap = mapping.putIfAbsent( - description.importingUri, - () => - {"name": getName(description._importingLibrary), "imports": {}}); - - List partFileNames = outputUnits - .where((outputUnit) => !omittedUnits.contains(outputUnit)) - .map((outputUnit) => deferredPartFileName(options, outputUnit.name)) - .toList(); - libraryMap["imports"][_importDeferName[import]] = partFileNames; - }); - return mapping; + return unit.imports.map((i) => importDeferName[i]); } } diff --git a/pkg/compiler/lib/src/dump_info.dart b/pkg/compiler/lib/src/dump_info.dart index 882895ccd80..761427f5f96 100644 --- a/pkg/compiler/lib/src/dump_info.dart +++ b/pkg/compiler/lib/src/dump_info.dart @@ -651,8 +651,11 @@ class DumpInfoTask extends CompilerTask implements InfoReporter { } } - result.deferredFiles = closedWorld.outputUnitData - .computeDeferredMap(compiler.options, closedWorld.elementEnvironment); + var fragmentsToLoad = + compiler.backendStrategy.emitterTask.emitter.fragmentsToLoad; + var fragmentMerger = + compiler.backendStrategy.emitterTask.emitter.fragmentMerger; + result.deferredFiles = fragmentMerger.computeDeferredMap(fragmentsToLoad); stopwatch.stop(); result.program = new ProgramInfo( diff --git a/pkg/compiler/lib/src/js_emitter/code_emitter_task.dart b/pkg/compiler/lib/src/js_emitter/code_emitter_task.dart index fb728255d59..4815c6dafeb 100644 --- a/pkg/compiler/lib/src/js_emitter/code_emitter_task.dart +++ b/pkg/compiler/lib/src/js_emitter/code_emitter_task.dart @@ -5,6 +5,7 @@ library dart2js.js_emitter.code_emitter_task; import '../common.dart'; +import '../common/metrics.dart' show Metric, Metrics, CountMetric; import '../common/tasks.dart' show CompilerTask; import '../compiler.dart' show Compiler; import '../constants/values.dart'; @@ -50,6 +51,9 @@ class CodeEmitterTask extends CompilerTask { /// Contains a list of all classes that are emitted. Set neededClasses; + @override + final _EmitterMetrics metrics = _EmitterMetrics(); + CodeEmitterTask(this._compiler, this._generateSourceMap) : super(_compiler.measurer); @@ -209,6 +213,12 @@ abstract class Emitter implements ModularEmitter { List get preDeferredFragmentsForTesting; + /// A map of loadId to list of [FinalizedFragments]. + Map> get fragmentsToLoad; + + /// The [FragmentMerger] itself. + fragment_merger.FragmentMerger get fragmentMerger; + /// Uses the [programBuilder] to generate a model of the program, emits /// the program, and returns the size of the generated output. int emitProgram(ProgramBuilder programBuilder, CodegenWorld codegenWorld); @@ -228,3 +238,16 @@ abstract class Emitter implements ModularEmitter { /// Returns the size of the code generated for a given output [unit]. int generatedSize(OutputUnit unit); } + +class _EmitterMetrics implements Metrics { + @override + String get namespace => 'emitter'; + + CountMetric hunkListElements = CountMetric('hunkListElements'); + + @override + Iterable get primary => []; + + @override + Iterable get secondary => [hunkListElements]; +} diff --git a/pkg/compiler/lib/src/js_emitter/model.dart b/pkg/compiler/lib/src/js_emitter/model.dart index 2cdfb739280..dd195cf1af0 100644 --- a/pkg/compiler/lib/src/js_emitter/model.dart +++ b/pkg/compiler/lib/src/js_emitter/model.dart @@ -21,9 +21,6 @@ class Program { final bool needsNativeSupport; final bool hasSoftDeferredClasses; - /// A map from load id to the list of fragments that need to be loaded. - final Map> loadMap; - // If this field is not `null` then its value must be emitted in the embedded // global `TYPE_TO_INTERCEPTOR_MAP`. The map references constants and classes. final js.Expression typeToInterceptorMap; @@ -33,7 +30,7 @@ class Program { final MetadataCollector _metadataCollector; final Iterable finalizers; - Program(this.fragments, this.holders, this.loadMap, this.typeToInterceptorMap, + Program(this.fragments, this.holders, this.typeToInterceptorMap, this._metadataCollector, this.finalizers, {this.needsNativeSupport, this.outputContainsConstantList, diff --git a/pkg/compiler/lib/src/js_emitter/program_builder/program_builder.dart b/pkg/compiler/lib/src/js_emitter/program_builder/program_builder.dart index 0cd7059ef85..947929fd20c 100644 --- a/pkg/compiler/lib/src/js_emitter/program_builder/program_builder.dart +++ b/pkg/compiler/lib/src/js_emitter/program_builder/program_builder.dart @@ -273,8 +273,8 @@ class ProgramBuilder { finalizers.add(namingFinalizer as js.TokenFinalizer); } - return new Program(fragments, holders, _buildLoadMap(), - _buildTypeToInterceptorMap(), _task.metadataCollector, finalizers, + return new Program(fragments, holders, _buildTypeToInterceptorMap(), + _task.metadataCollector, finalizers, needsNativeSupport: needsNativeSupport, outputContainsConstantList: collector.outputContainsConstantList, hasSoftDeferredClasses: _notSoftDeferred != null); @@ -348,18 +348,6 @@ class ProgramBuilder { } } - /// Builds a map from loadId to outputs-to-load. - Map> _buildLoadMap() { - Map> loadMap = >{}; - _closedWorld.outputUnitData.hunksToLoad - .forEach((String loadId, List outputUnits) { - loadMap[loadId] = outputUnits - .map((OutputUnit unit) => _outputs[unit]) - .toList(growable: false); - }); - return loadMap; - } - js.Expression _buildTypeToInterceptorMap() { InterceptorStubGenerator stubGenerator = new InterceptorStubGenerator( _commonElements, diff --git a/pkg/compiler/lib/src/js_emitter/startup_emitter/emitter.dart b/pkg/compiler/lib/src/js_emitter/startup_emitter/emitter.dart index ca28cdb72aa..5a878fbdb42 100644 --- a/pkg/compiler/lib/src/js_emitter/startup_emitter/emitter.dart +++ b/pkg/compiler/lib/src/js_emitter/startup_emitter/emitter.dart @@ -7,7 +7,6 @@ library dart2js.js_emitter.startup_emitter; import '../../../compiler_new.dart'; import '../../common.dart'; import '../../common/codegen.dart'; -import '../../common/tasks.dart'; import '../../constants/values.dart'; import '../../deferred_load.dart' show OutputUnit; import '../../dump_info.dart'; @@ -20,7 +19,7 @@ import '../../js_backend/runtime_types_new.dart' show RecipeEncoder; import '../../options.dart'; import '../../universe/codegen_world_builder.dart' show CodegenWorld; import '../../world.dart' show JClosedWorld; -import '../js_emitter.dart' show Emitter, ModularEmitter; +import '../js_emitter.dart' show CodeEmitterTask, Emitter, ModularEmitter; import '../model.dart'; import '../native_emitter.dart'; import '../program_builder/program_builder.dart' show ProgramBuilder; @@ -150,7 +149,7 @@ class EmitterImpl extends ModularEmitterBase implements Emitter { final DiagnosticReporter _reporter; final JClosedWorld _closedWorld; final RecipeEncoder _rtiRecipeEncoder; - final CompilerTask _task; + final CodeEmitterTask _task; ModelEmitter _emitter; final NativeEmitter _nativeEmitter; @@ -160,6 +159,12 @@ class EmitterImpl extends ModularEmitterBase implements Emitter { @override List preDeferredFragmentsForTesting; + @override + Map> fragmentsToLoad; + + @override + FragmentMerger fragmentMerger; + EmitterImpl( CompilerOptions options, this._reporter, @@ -201,6 +206,11 @@ class EmitterImpl extends ModularEmitterBase implements Emitter { } return _task.measureSubtask('emit program', () { var size = _emitter.emitProgram(program, codegenWorld); + fragmentsToLoad = _emitter.fragmentsToLoad; + fragmentMerger = _emitter.fragmentMerger; + fragmentsToLoad.values.forEach((fragments) { + _task.metrics.hunkListElements.add(fragments.length); + }); if (retainDataForTesting) { preDeferredFragmentsForTesting = _emitter.preDeferredFragmentsForTesting; diff --git a/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_emitter.dart b/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_emitter.dart index 939a39f492b..f5f2d9a9604 100644 --- a/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_emitter.dart +++ b/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_emitter.dart @@ -730,7 +730,9 @@ class FragmentEmitter { } js.Statement emitMainFragment( - Program program, DeferredLoadingState deferredLoadingState) { + Program program, + Map> fragmentsToLoad, + DeferredLoadingState deferredLoadingState) { MainFragment fragment = program.fragments.first; Iterable nonStaticStateHolders = @@ -778,8 +780,8 @@ class FragmentEmitter { 'constants': emitConstants(fragment), 'staticNonFinalFields': emitStaticNonFinalFields(fragment), 'lazyStatics': emitLazilyInitializedStatics(fragment), - 'embeddedGlobalsPart1': - emitEmbeddedGlobalsPart1(program, deferredLoadingState), + 'embeddedGlobalsPart1': emitEmbeddedGlobalsPart1( + program, fragmentsToLoad, deferredLoadingState), 'embeddedGlobalsPart2': emitEmbeddedGlobalsPart2(program, deferredLoadingState), 'typeRules': emitTypeRules(fragment), @@ -1874,10 +1876,10 @@ class FragmentEmitter { // array of hashes indexed by part. // [deferredLoadHashes] may have missing entries to indicate empty parts. void finalizeDeferredLoadingData( - Map> loadMap, + Map> fragmentsToLoad, Map deferredLoadHashes, DeferredLoadingState deferredLoadingState) { - if (loadMap.isEmpty) return; + if (fragmentsToLoad.isEmpty) return; Map fragmentIndexes = {}; List fragmentUris = []; @@ -1885,7 +1887,8 @@ class FragmentEmitter { List libraryPartsMapEntries = []; - loadMap.forEach((String loadId, List fragmentList) { + fragmentsToLoad + .forEach((String loadId, List fragmentList) { List indexes = []; for (FinalizedFragment fragment in fragmentList) { String fragmentHash = deferredLoadHashes[fragment]; @@ -1971,10 +1974,12 @@ class FragmentEmitter { /// Emits all embedded globals. js.Statement emitEmbeddedGlobalsPart1( - Program program, DeferredLoadingState deferredLoadingState) { + Program program, + Map> fragmentsToLoad, + DeferredLoadingState deferredLoadingState) { List globals = []; - if (program.loadMap.isNotEmpty) { + if (fragmentsToLoad.isNotEmpty) { globals .addAll(emitEmbeddedGlobalsForDeferredLoading(deferredLoadingState)); } diff --git a/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart b/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart index 122a716bc3c..2a8011306c1 100644 --- a/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart +++ b/pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart @@ -3,6 +3,10 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:collection'; +import '../../common_elements.dart' show ElementEnvironment; +import '../../deferred_load.dart' + show ImportDescription, OutputUnit, OutputUnitData, deferredPartFileName; +import '../../elements/entities.dart'; import '../../deferred_load.dart' show OutputUnit; import '../../js/js.dart' as js; import '../../js/size_estimator.dart'; @@ -24,6 +28,7 @@ class PreFragment { final List nativeSupport = []; final Set successors = {}; final Set predecessors = {}; + FinalizedFragment finalizedFragment; int size = 0; PreFragment( @@ -103,8 +108,8 @@ class PreFragment { } FinalizedFragment finalize( - Program program, Map fragmentMap) { - FinalizedFragment finalizedFragment; + Program program, Map outputUnitMap) { + assert(finalizedFragment == null); var seedFragment = fragments.first; var seedOutputUnit = seedFragment.outputUnit; @@ -128,7 +133,7 @@ class PreFragment { lazyInitializers.first, nativeSupport.first, program.metadataTypesForOutputUnit(seedOutputUnit)); - fragmentMap[seedFragment] = finalizedFragment; + outputUnitMap[seedOutputUnit] = finalizedFragment; } else { List outputUnits = [seedOutputUnit]; List libraries = []; @@ -156,8 +161,8 @@ class PreFragment { js.Block(lazyInitializers), js.Block(nativeSupport), program.metadataTypesForOutputUnit(seedOutputUnit)); - for (var fragment in fragments) { - fragmentMap[fragment] = finalizedFragment; + for (var outputUnit in outputUnits) { + outputUnitMap[outputUnit] = finalizedFragment; } } return finalizedFragment; @@ -294,28 +299,32 @@ class _Partition { class FragmentMerger { final CompilerOptions _options; + final ElementEnvironment _elementEnvironment; + final OutputUnitData outputUnitData; int totalSize = 0; - FragmentMerger(this._options); + FragmentMerger(this._options, this._elementEnvironment, this.outputUnitData); - // Converts a map of (loadId, List) to a map of + // Converts a map of (loadId, List) to a map of // (loadId, List). - static Map> processLoadMap( - Map> programLoadMap, - Map fragmentMap) { - Map> loadMap = {}; - programLoadMap.forEach((loadId, fragments) { + Map> computeFragmentsToLoad( + Map> outputUnitsToLoad, + Map outputUnitMap, + Set omittedOutputUnits) { + Map> fragmentsToLoad = {}; + outputUnitsToLoad.forEach((loadId, outputUnits) { Set unique = {}; List finalizedFragments = []; - loadMap[loadId] = finalizedFragments; - for (var fragment in fragments) { - var finalizedFragment = fragmentMap[fragment]; + fragmentsToLoad[loadId] = finalizedFragments; + for (var outputUnit in outputUnits) { + if (omittedOutputUnits.contains(outputUnit)) continue; + var finalizedFragment = outputUnitMap[outputUnit]; if (unique.add(finalizedFragment)) { finalizedFragments.add(finalizedFragment); } } }); - return loadMap; + return fragmentsToLoad; } /// Given a list of OutputUnits sorted by their import entites, @@ -353,22 +362,22 @@ class FragmentMerger { } /// Attachs predecessors and successors to each PreFragment. - void attachDependencies(Map fragmentMap, + /// Expects outputUnits to be sorted. + void attachDependencies( + List outputUnits, + Map fragmentMap, List preDeferredFragments) { // Create a map of OutputUnit to Fragment. Map outputUnitMap = {}; - List allOutputUnits = []; for (var preFragment in preDeferredFragments) { var fragment = preFragment.fragments.single; var outputUnit = fragment.outputUnit; outputUnitMap[outputUnit] = fragment; - allOutputUnits.add(outputUnit); totalSize += preFragment.size; } - allOutputUnits.sort(); // Get a list of direct edges and then attach them to PreFragments. - var allEdges = createDirectEdges(allOutputUnits); + var allEdges = createDirectEdges(outputUnits); allEdges.forEach((outputUnit, edges) { var predecessor = fragmentMap[outputUnitMap[outputUnit]]; for (var edge in edges) { @@ -449,4 +458,79 @@ class FragmentMerger { } return merged; } + + /// Computes load lists using a list of sorted OutputUnits. + Map> computeOutputUnitsToLoad( + List outputUnits) { + // Sort the output units in descending order of the number of imports they + // include. + + // The loading of the output units must be ordered because a superclass + // needs to be initialized before its subclass. + // But a class can only depend on another class in an output unit shared by + // a strict superset of the imports: + // By contradiction: Assume a class C in output unit shared by imports in + // the set S1 = (lib1,.., lib_n) depends on a class D in an output unit + // shared by S2 such that S2 not a superset of S1. Let lib_s be a library in + // S1 not in S2. lib_s must depend on C, and then in turn on D. Therefore D + // is not in the right output unit. + List sortedOutputUnits = outputUnits.reversed.toList(); + + Map> outputUnitsToLoad = {}; + for (var import in outputUnitData.deferredImportDescriptions.keys) { + var loadId = outputUnitData.importDeferName[import]; + List loadList = []; + for (var outputUnit in sortedOutputUnits) { + assert(!outputUnit.isMainOutput); + if (outputUnit.imports.contains(import)) { + loadList.add(outputUnit); + } + } + outputUnitsToLoad[loadId] = loadList; + } + return outputUnitsToLoad; + } + + /// Returns a json-style map for describing what files that are loaded by a + /// given deferred import. + /// The mapping is structured as: + /// library uri -> {"name": library name, "files": (prefix -> list of files)} + /// Where + /// + /// - is the import uri of the library making a deferred + /// import. + /// - is the name of the library, or "" if it is + /// unnamed. + /// - is the `as` prefix used for a given deferred import. + /// - is a list of the filenames the must be loaded when that + /// import is loaded. + /// TODO(joshualitt): the library name is unused and should be removed. This + /// will be a breaking change. + Map> computeDeferredMap( + Map> fragmentsToLoad) { + Map> mapping = {}; + + outputUnitData.deferredImportDescriptions.keys + .forEach((ImportEntity import) { + var importDeferName = outputUnitData.importDeferName[import]; + List fragments = fragmentsToLoad[importDeferName]; + ImportDescription description = + outputUnitData.deferredImportDescriptions[import]; + String getName(LibraryEntity library) { + var name = _elementEnvironment.getLibraryName(library); + return name == '' ? '' : name; + } + + Map libraryMap = mapping.putIfAbsent( + description.importingUri, + () => {"name": getName(description.importingLibrary), "imports": {}}); + + List partFileNames = fragments + .map((fragment) => + deferredPartFileName(_options, fragment.canonicalOutputUnit.name)) + .toList(); + libraryMap["imports"][importDeferName] = partFileNames; + }); + return mapping; + } } diff --git a/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart b/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart index 943f80e78f7..a7c42adcdcc 100644 --- a/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart +++ b/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart @@ -107,6 +107,15 @@ class ModelEmitter { List preDeferredFragmentsForTesting; + /// A mapping from the name of a defer import to all the fragments it + /// depends on in a list of lists to be loaded in the order they appear. + /// + /// For example {"lib1": [[lib1_lib2_lib3], [lib1_lib2, lib1_lib3], + /// [lib1]]} would mean that in order to load "lib1" first the hunk + /// lib1_lib2_lib2 should be loaded, then the hunks lib1_lib2 and lib1_lib3 + /// can be loaded in parallel. And fially lib1 can be loaded. + Map> fragmentsToLoad; + /// For deferred loading we communicate the initializers via this global var. static const String deferredInitializersGlobal = r"$__dart_deferred_initializers__"; @@ -130,7 +139,8 @@ class ModelEmitter { RecipeEncoder rtiRecipeEncoder, this._shouldGenerateSourceMap) : _constantOrdering = new ConstantOrdering(_closedWorld.sorter), - fragmentMerger = FragmentMerger(_options) { + fragmentMerger = FragmentMerger(_options, + _closedWorld.elementEnvironment, _closedWorld.outputUnitData) { this.constantEmitter = new ConstantEmitter( _options, _namer, @@ -213,11 +223,8 @@ class ModelEmitter { _closedWorld, codegenWorld); - var deferredLoadingState = new DeferredLoadingState(); - js.Statement mainCode = - fragmentEmitter.emitMainFragment(program, deferredLoadingState); - // In order to get size estimates, we partially emit deferred fragments. + List outputUnits = []; List preDeferredFragments = []; Map preFragmentMap = {}; _task.measureSubtask('emit prefragments', () { @@ -225,15 +232,23 @@ class ModelEmitter { var preFragment = fragmentEmitter.emitPreFragment(fragment, shouldMergeFragments); preFragmentMap[fragment] = preFragment; + outputUnits.add(fragment.outputUnit); preDeferredFragments.add(preFragment); } }); + // Sort output units so they are in a canonical order and generate a map of + // loadId to list of OutputUnits to load. + outputUnits.sort(); + var outputUnitsToLoad = + fragmentMerger.computeOutputUnitsToLoad(outputUnits); + // If we are going to merge, then we attach dependencies to each PreFragment // and merge. if (shouldMergeFragments) { preDeferredFragments = _task.measureSubtask('merge fragments', () { - fragmentMerger.attachDependencies(preFragmentMap, preDeferredFragments); + fragmentMerger.attachDependencies( + outputUnits, preFragmentMap, preDeferredFragments); return fragmentMerger.mergeFragments(preDeferredFragments); }); } @@ -243,11 +258,12 @@ class ModelEmitter { preDeferredFragmentsForTesting = preDeferredFragments; } - Map fragmentMap = {}; + // Finalize and emit fragments. + Map outputUnitMap = {}; Map deferredFragmentsCode = {}; for (var preDeferredFragment in preDeferredFragments) { var finalizedFragment = - preDeferredFragment.finalize(program, fragmentMap); + preDeferredFragment.finalize(program, outputUnitMap); js.Expression fragmentCode = fragmentEmitter.emitDeferredFragment( finalizedFragment, program.holders); if (fragmentCode != null) { @@ -257,6 +273,17 @@ class ModelEmitter { } } + // With all deferred fragments finalized, we can now compute a map of + // loadId to the files(FinalizedFragments) which need to be loaded. + fragmentsToLoad = fragmentMerger.computeFragmentsToLoad( + outputUnitsToLoad, outputUnitMap, omittedOutputUnits); + + // Emit main Fragment. + var deferredLoadingState = new DeferredLoadingState(); + js.Statement mainCode = fragmentEmitter.emitMainFragment( + program, fragmentsToLoad, deferredLoadingState); + + // Count tokens and run finalizers. js.TokenCounter counter = new js.TokenCounter(); deferredFragmentsCode.values.forEach(counter.countTokens); counter.countTokens(mainCode); @@ -274,10 +301,8 @@ class ModelEmitter { // Now that we have written the deferred hunks, we can create the deferred // loading data. - Map> loadMap = - FragmentMerger.processLoadMap(program.loadMap, fragmentMap); fragmentEmitter.finalizeDeferredLoadingData( - loadMap, hunkHashes, deferredLoadingState); + fragmentsToLoad, hunkHashes, deferredLoadingState); _task.measureSubtask('write fragments', () { writeMainFragment(mainFragment, mainCode, @@ -498,9 +523,7 @@ class ModelEmitter { // data. mapping["_comment"] = "This mapping shows which compiled `.js` files are " "needed for a given deferred library import."; - mapping.addAll(_closedWorld.outputUnitData.computeDeferredMap( - _options, _closedWorld.elementEnvironment, - omittedUnits: omittedOutputUnits)); + mapping.addAll(fragmentMerger.computeDeferredMap(fragmentsToLoad)); _outputProvider.createOutputSink( _options.deferredMapUri.path, '', OutputType.deferredMap) ..add(const JsonEncoder.withIndent(" ").convert(mapping)) diff --git a/pkg/compiler/test/analyses/dart2js_allowed.json b/pkg/compiler/test/analyses/dart2js_allowed.json index 10ea0694960..3fb066868c2 100644 --- a/pkg/compiler/test/analyses/dart2js_allowed.json +++ b/pkg/compiler/test/analyses/dart2js_allowed.json @@ -45,8 +45,7 @@ }, "pkg/compiler/lib/src/deferred_load.dart": { "Dynamic access of 'memberContext'.": 1, - "Dynamic access of 'name'.": 1, - "Dynamic invocation of '[]='.": 1 + "Dynamic access of 'name'.": 1 }, "pkg/compiler/lib/src/inferrer/typemasks/container_type_mask.dart": { "Dynamic access of 'isNullable'.": 2, @@ -197,6 +196,9 @@ "Dynamic invocation of '[]='.": 1, "Dynamic invocation of 'add'.": 1 }, + "pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart": { + "Dynamic invocation of '[]='.": 1 + }, "pkg/js_ast/lib/src/builder.dart": { "Dynamic invocation of 'call'.": 2 }, diff --git a/pkg/compiler/test/deferred/load_graph_segmentation_test.dart b/pkg/compiler/test/deferred/load_graph_segmentation_test.dart index 3dc7942daba..660e27ba635 100644 --- a/pkg/compiler/test/deferred/load_graph_segmentation_test.dart +++ b/pkg/compiler/test/deferred/load_graph_segmentation_test.dart @@ -11,9 +11,18 @@ import 'package:async_helper/async_helper.dart'; import 'package:compiler/src/compiler.dart'; import 'package:compiler/src/deferred_load.dart'; +import 'package:compiler/src/js_emitter/startup_emitter/fragment_merger.dart'; import 'package:expect/expect.dart'; import '../helpers/memory_compiler.dart'; +List collectOutputUnits(List fragments) { + List outputUnits = []; + for (var fragment in fragments) { + outputUnits.addAll(fragment.outputUnits); + } + return outputUnits; +} + void main() { asyncTest(() async { CompilationResult result = @@ -59,12 +68,11 @@ void main() { // InputElement is native, so it should be in the mainOutputUnit. Expect.equals(mainOutputUnit, outputUnitForClass(inputElement)); - var hunksToLoad = closedWorld.outputUnitData.hunksToLoad; - - var hunksLib1 = hunksToLoad["lib1"]; - var hunksLib2 = hunksToLoad["lib2"]; - var hunksLib4_1 = hunksToLoad["lib4_1"]; - var hunksLib4_2 = hunksToLoad["lib4_2"]; + var hunksToLoad = backendStrategy.emitterTask.emitter.fragmentsToLoad; + var hunksLib1 = collectOutputUnits(hunksToLoad["lib1"]); + var hunksLib2 = collectOutputUnits(hunksToLoad["lib2"]); + var hunksLib4_1 = collectOutputUnits(hunksToLoad["lib4_1"]); + var hunksLib4_2 = collectOutputUnits(hunksToLoad["lib4_2"]); Expect.listEquals([ou_lib1_lib2, ou_lib1], hunksLib1); Expect.listEquals([ou_lib1_lib2, ou_lib2], hunksLib2); Expect.listEquals([ou_lib4_1], hunksLib4_1); diff --git a/pkg/compiler/test/deferred/load_mapping_test.dart b/pkg/compiler/test/deferred/load_mapping_test.dart index b6a68213acc..ffb7b841286 100644 --- a/pkg/compiler/test/deferred/load_mapping_test.dart +++ b/pkg/compiler/test/deferred/load_mapping_test.dart @@ -4,27 +4,24 @@ // @dart = 2.7 +import 'dart:convert'; import 'package:expect/expect.dart'; import 'package:async_helper/async_helper.dart'; import 'package:compiler/compiler_new.dart'; -import 'package:compiler/src/world.dart'; -import '../helpers/memory_source_file_helper.dart'; import '../helpers/memory_compiler.dart'; void testLoadMap() async { var collector = new OutputCollector(); - CompilationResult result = await runCompiler( + await runCompiler( memorySourceFiles: MEMORY_SOURCE_FILES, options: ['--deferred-map=deferred_map.json'], outputProvider: collector); - CompilerImpl compiler = result.compiler; - JClosedWorld closedWorld = compiler.backendClosedWorldForTesting; // Ensure a mapping file is output. - Expect.isNotNull( - collector.getOutput("deferred_map.json", OutputType.deferredMap)); + var deferredMap = + collector.getOutput("deferred_map.json", OutputType.deferredMap); + Expect.isNotNull(deferredMap); + var mapping = jsonDecode(deferredMap); - Map mapping = closedWorld.outputUnitData - .computeDeferredMap(compiler.options, closedWorld.elementEnvironment); // Test structure of mapping. Expect.equals("", mapping["main.dart"]["name"]); Expect.equals(2, mapping["main.dart"]["imports"]["lib1"].length); diff --git a/pkg/compiler/test/deferred_loading/data/deferred_overlapping/main.dart b/pkg/compiler/test/deferred_loading/data/deferred_overlapping/main.dart index 58108e925c8..154ad5910f8 100644 --- a/pkg/compiler/test/deferred_loading/data/deferred_overlapping/main.dart +++ b/pkg/compiler/test/deferred_loading/data/deferred_overlapping/main.dart @@ -31,9 +31,9 @@ import 'lib2.dart' deferred as lib2; /*member: main:member_unit=main{}*/ void main() { lib1.loadLibrary().then(/*closure_unit=main{}*/ (_) { - new lib1.C1(); + print(new lib1.C1()); lib2.loadLibrary().then(/*closure_unit=main{}*/ (_) { - new lib2.C2(); + print(new lib2.C2()); }); }); } diff --git a/pkg/compiler/test/deferred_loading/data/lazy_types/lib.dart b/pkg/compiler/test/deferred_loading/data/lazy_types/lib.dart index 4cd9f153710..22064c88531 100644 --- a/pkg/compiler/test/deferred_loading/data/lazy_types/lib.dart +++ b/pkg/compiler/test/deferred_loading/data/lazy_types/lib.dart @@ -16,15 +16,18 @@ class Foo { x = DateTime.now().millisecond; } /*member: Foo.method:member_unit=1{libB}*/ + @pragma('dart2js:noInline') int method() => x; } /*member: isFoo:member_unit=3{libA, libB, libC}*/ +@pragma('dart2js:noInline') bool isFoo(o) { return o is Foo; } /*member: callFooMethod:member_unit=1{libB}*/ +@pragma('dart2js:noInline') int callFooMethod() { return Foo().method(); } @@ -33,6 +36,7 @@ typedef int FunFoo(Foo a); typedef int FunFunFoo(FunFoo b, int c); /*member: isFunFunFoo:member_unit=3{libA, libB, libC}*/ +@pragma('dart2js:noInline') bool isFunFunFoo(o) { return o is FunFunFoo; } @@ -64,6 +68,7 @@ class Coo {} class Doo extends Coo with Boo {} /*member: createDooFunFunFoo:member_unit=2{libC}*/ +@pragma('dart2js:noInline') createDooFunFunFoo() => Doo(); /*class: B: @@ -120,6 +125,7 @@ class D2 {} class D3 = D2 with D1; /*member: isMega:member_unit=6{libA}*/ +@pragma('dart2js:noInline') bool isMega(o) { return o is B2 || o is C3 || o is D3; } diff --git a/pkg/compiler/test/deferred_loading/data/lazy_types/main.dart b/pkg/compiler/test/deferred_loading/data/lazy_types/main.dart index 15e802220e9..ce78d8ae7fa 100644 --- a/pkg/compiler/test/deferred_loading/data/lazy_types/main.dart +++ b/pkg/compiler/test/deferred_loading/data/lazy_types/main.dart @@ -5,15 +5,13 @@ /*spec.library: output_units=[ f1: {units: [3{libA, libB, libC}], usedBy: [], needs: []}, - f2: {units: [4{libA, libC}], usedBy: [], needs: []}, - f3: {units: [6{libA}], usedBy: [], needs: []}, - f4: {units: [5{libB, libC}], usedBy: [], needs: []}, - f5: {units: [1{libB}], usedBy: [], needs: []}, - f6: {units: [2{libC}], usedBy: [], needs: []}], + f2: {units: [6{libA}], usedBy: [], needs: []}, + f3: {units: [1{libB}], usedBy: [], needs: []}, + f4: {units: [2{libC}], usedBy: [], needs: []}], steps=[ - libA=(f1, f2, f3), - libB=(f1, f4, f5), - libC=(f1, f4, f2, f6)] + libA=(f1, f2), + libB=(f1, f3), + libC=(f1, f4)] */ /*two-frag.library: diff --git a/pkg/compiler/test/deferred_loading/data/type_arguments/main.dart b/pkg/compiler/test/deferred_loading/data/type_arguments/main.dart index 13f5f038ebb..9433a91705e 100644 --- a/pkg/compiler/test/deferred_loading/data/type_arguments/main.dart +++ b/pkg/compiler/test/deferred_loading/data/type_arguments/main.dart @@ -2,24 +2,13 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -/*spec.library: +/*library: output_units=[ - f1: {units: [3{lib1, lib3}], usedBy: [], needs: []}, - f2: {units: [1{lib1}], usedBy: [], needs: []}, - f3: {units: [2{lib3}], usedBy: [], needs: []}], + f1: {units: [1{lib1}], usedBy: [], needs: []}, + f2: {units: [2{lib3}], usedBy: [], needs: []}], steps=[ - lib1=(f1, f2), - lib3=(f1, f3)] -*/ - -/*two-frag|three-frag.library: - output_units=[ - f1: {units: [3{lib1, lib3}], usedBy: [], needs: [2, 3]}, - f2: {units: [1{lib1}], usedBy: [1], needs: []}, - f3: {units: [2{lib3}], usedBy: [1], needs: []}], - steps=[ - lib1=(f1, f2), - lib3=(f1, f3)] + lib1=(f1), + lib3=(f2)] */ // @dart = 2.7 diff --git a/pkg/compiler/test/deferred_loading/deferred_loading_test.dart b/pkg/compiler/test/deferred_loading/deferred_loading_test.dart index e64d30529a3..2e749341757 100644 --- a/pkg/compiler/test/deferred_loading/deferred_loading_test.dart +++ b/pkg/compiler/test/deferred_loading/deferred_loading_test.dart @@ -15,7 +15,6 @@ import 'package:compiler/src/elements/entities.dart'; import 'package:compiler/src/ir/util.dart'; import 'package:compiler/src/js_model/element_map.dart'; import 'package:compiler/src/js_model/js_world.dart'; -import 'package:compiler/src/js_emitter/model.dart'; import 'package:compiler/src/js_emitter/startup_emitter/fragment_merger.dart'; import 'package:compiler/src/kernel/kernel_strategy.dart'; import 'package:expect/expect.dart'; @@ -87,23 +86,18 @@ String outputUnitString(OutputUnit unit) { } Map> buildPreFragmentMap( - Map> loadMap, + Map> fragmentsToLoad, List preDeferredFragments) { - Map fragmentMap = {}; + Map fragmentMap = {}; for (var preFragment in preDeferredFragments) { - for (var fragment in preFragment.fragments) { - assert(!fragmentMap.containsKey(fragment)); - fragmentMap[fragment] = preFragment; - } + fragmentMap[preFragment.finalizedFragment] = preFragment; } - Map> preFragmentMap = {}; - loadMap.forEach((loadId, fragments) { - Set preFragments = {}; + fragmentsToLoad.forEach((loadId, fragments) { + List preFragments = []; for (var fragment in fragments) { preFragments.add(fragmentMap[fragment]); } - assert(!preFragmentMap.containsKey(loadId)); preFragmentMap[loadId] = preFragments.toList(); }); return preFragmentMap; @@ -160,10 +154,10 @@ class OutputUnitDataComputer extends DataComputer { ir.Library node = frontendStrategy.elementMap.getLibraryNode(library); List preDeferredFragments = compiler .backendStrategy.emitterTask.emitter.preDeferredFragmentsForTesting; - Program program = - compiler.backendStrategy.emitterTask.emitter.programForTesting; + Map> fragmentsToLoad = + compiler.backendStrategy.emitterTask.emitter.fragmentsToLoad; Map> preFragmentMap = - buildPreFragmentMap(program.loadMap, preDeferredFragments); + buildPreFragmentMap(fragmentsToLoad, preDeferredFragments); PreFragmentsIrComputer(compiler.reporter, actualMap, preFragmentMap) .computeForLibrary(node); } @@ -208,13 +202,15 @@ class PreFragmentsIrComputer extends IrDataExtractor { List supplied = []; List usedBy = []; for (var dependent in preFragment.successors) { - assert(preFragmentIndices.containsKey(dependent)); - usedBy.add(preFragmentIndices[dependent]); + if (preFragmentIndices.containsKey(dependent)) { + usedBy.add(preFragmentIndices[dependent]); + } } for (var dependency in preFragment.predecessors) { - assert(preFragmentIndices.containsKey(dependency)); - needs.add(preFragmentIndices[dependency]); + if (preFragmentIndices.containsKey(dependency)) { + needs.add(preFragmentIndices[dependency]); + } } for (var fragment in preFragment.fragments) {