From 3d22a350b3665e9db4e2eb2b3fa7296dfe8a4d6e Mon Sep 17 00:00:00 2001 From: "johnniwinther@google.com" Date: Tue, 3 Sep 2013 08:04:38 +0000 Subject: [PATCH] First pass at asynchronous input loading in dart2js. R=ahe@google.com, johnniwinther@google.com Review URL: https://codereview.chromium.org//17759007 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@27028 260f80e4-7a28-3924-810f-c04153c831b5 --- pkg/async_helper/lib/async_helper.dart | 5 + sdk/lib/_internal/compiler/compiler.dart | 26 +- .../compiler/implementation/apiimpl.dart | 58 +- .../compiler/implementation/compiler.dart | 142 +-- .../compiler/implementation/dart2js.dart | 30 +- .../implementation/dart_backend/backend.dart | 18 +- .../dart_backend/dart_backend.dart | 1 + .../implementation/deferred_load.dart | 2 - .../implementation/js_backend/backend.dart | 3 +- .../implementation/js_backend/js_backend.dart | 1 + .../implementation/library_loader.dart | 265 ++--- .../mirrors/dart2js_mirror.dart | 15 +- .../compiler/implementation/patch_parser.dart | 45 +- .../implementation/source_file_provider.dart | 5 +- .../compiler/implementation/util/util.dart | 2 +- .../compiler/samples/leap/leap_leg.dart | 8 + tests/compiler/dart2js/analyze_all_test.dart | 44 +- tests/compiler/dart2js/analyze_api_test.dart | 3 +- .../dart2js/analyze_dart2js_test.dart | 3 +- tests/compiler/dart2js/analyze_helper.dart | 8 +- tests/compiler/dart2js/analyze_only_test.dart | 4 +- .../async_compiler_input_provider_test.dart | 62 ++ tests/compiler/dart2js/bad_loop_test.dart | 14 +- .../call_site_simple_type_inferer_test.dart | 11 +- .../compiler/dart2js/class_codegen2_test.dart | 18 +- .../compiler/dart2js/class_codegen_test.dart | 23 +- tests/compiler/dart2js/class_order_test.dart | 11 +- .../dart2js/closure_codegen_test.dart | 10 +- tests/compiler/dart2js/codegen_helper.dart | 27 +- tests/compiler/dart2js/compiler_helper.dart | 33 +- .../dart2js/concrete_type_inference_test.dart | 23 +- .../compiler/dart2js/cpa_inference_test.dart | 907 ++++++++++-------- tests/compiler/dart2js/dart_backend_test.dart | 5 +- .../dart2js/dead_bailout_target_test.dart | 28 +- tests/compiler/dart2js/dead_code_test.dart | 6 +- ...deferred_load_graph_segmentation_test.dart | 35 +- .../dart2js/diagnose_ambiguous_test.dart | 50 +- .../dart2js/emit_const_fields_test.dart | 8 +- .../compiler/dart2js/field_codegen_test.dart | 12 +- .../field_type_simple_inferer_test.dart | 13 +- .../dart2js/gvn_dynamic_field_get_test.dart | 26 +- tests/compiler/dart2js/gvn_test.dart | 23 +- tests/compiler/dart2js/import_test.dart | 15 +- .../dart2js/inferrer_factory_test.dart | 10 +- tests/compiler/dart2js/library_load_test.dart | 34 - tests/compiler/dart2js/list_tracer2_test.dart | 18 +- .../dart2js/list_tracer_length_test.dart | 6 +- .../dart2js/list_tracer_node_type_test.dart | 45 +- tests/compiler/dart2js/list_tracer_test.dart | 72 +- .../compiler/dart2js/lookup_member_test.dart | 154 +-- tests/compiler/dart2js/memory_compiler.dart | 8 +- .../dart2js/memory_source_file_helper.dart | 9 +- .../compiler/dart2js/message_kind_helper.dart | 28 +- tests/compiler/dart2js/message_kind_test.dart | 17 +- tests/compiler/dart2js/metadata_test.dart | 89 +- .../dart2js/mirror_helper_rename_test.dart | 81 +- .../compiler/dart2js/mirror_helper_test.dart | 55 +- ...irror_helper_unique_minification_test.dart | 59 +- .../dart2js/mirror_tree_shaking_test.dart | 17 +- .../compiler/dart2js/mirrors_lookup_test.dart | 5 +- .../dart2js/mirrors_metadata_test.dart | 43 +- tests/compiler/dart2js/mirrors_test.dart | 4 +- tests/compiler/dart2js/mirrors_used_test.dart | 148 +-- tests/compiler/dart2js/missing_file_test.dart | 78 ++ tests/compiler/dart2js/mock_compiler.dart | 30 +- .../dart2js/no_constructor_body_test.dart | 7 +- .../no_duplicate_constructor_body2_test.dart | 10 +- .../no_duplicate_constructor_body_test.dart | 10 +- .../dart2js/no_duplicate_stub_test.dart | 10 +- tests/compiler/dart2js/package_root_test.dart | 17 +- tests/compiler/dart2js/part_of_test.dart | 22 +- tests/compiler/dart2js/patch_test.dart | 763 ++++++++------- tests/compiler/dart2js/private_test.dart | 6 +- .../dart2js/reexport_handled_test.dart | 27 +- .../compiler/dart2js/regress_10231_test.dart | 11 +- tests/compiler/dart2js/resolution_test.dart | 6 +- tests/compiler/dart2js/resolver_test.dart | 46 +- .../dart2js/rewrite_better_user_test.dart | 14 +- .../dart2js/simple_inferrer_and_or_test.dart | 32 +- .../dart2js/simple_inferrer_closure_test.dart | 32 +- .../simple_inferrer_final_field2_test.dart | 20 +- .../simple_inferrer_final_field3_test.dart | 22 +- .../simple_inferrer_final_field_test.dart | 30 +- .../simple_inferrer_no_such_method_test.dart | 56 +- .../simple_inferrer_postfix_prefix_test.dart | 56 +- .../simple_inferrer_relations_test.dart | 12 +- .../dart2js/simple_inferrer_test.dart | 225 ++--- .../simple_inferrer_try_catch_test.dart | 49 +- .../simple_inferrer_unregister_call_test.dart | 19 +- tests/compiler/dart2js/size_test.dart | 13 +- .../compiler/dart2js/source_mapping_test.dart | 48 +- .../compiler/dart2js/static_closure_test.dart | 20 +- .../dart2js/string_interpolation_test.dart | 14 +- tests/compiler/dart2js/subtype_test.dart | 769 ++++++++------- tests/compiler/dart2js/tag_mapping_test.dart | 5 +- tests/compiler/dart2js/tree_shaking_test.dart | 10 +- tests/compiler/dart2js/type_equals_test.dart | 51 +- .../dart2js/type_representation_test.dart | 213 ++-- .../dart2js/type_substitution_test.dart | 187 ++-- tests/compiler/dart2js/type_test_helper.dart | 8 +- .../dart2js/type_variable_bound_test.dart | 121 +-- .../type_variable_occurrence_test.dart | 135 +-- .../types_of_captured_variables_test.dart | 16 +- .../dart2js/unneeded_part_js_test.dart | 9 +- tests/utils/dummy_compiler_test.dart | 4 +- tests/utils/recursive_import_test.dart | 6 +- 106 files changed, 3341 insertions(+), 2848 deletions(-) create mode 100644 tests/compiler/dart2js/async_compiler_input_provider_test.dart delete mode 100644 tests/compiler/dart2js/library_load_test.dart create mode 100644 tests/compiler/dart2js/missing_file_test.dart diff --git a/pkg/async_helper/lib/async_helper.dart b/pkg/async_helper/lib/async_helper.dart index f1a84b369f7..72a391f03a0 100644 --- a/pkg/async_helper/lib/async_helper.dart +++ b/pkg/async_helper/lib/async_helper.dart @@ -61,3 +61,8 @@ void asyncEnd() { print('unittest-suite-success'); } } + +void asyncTest(Future f()) { + asyncStart(); + f().whenComplete(() => asyncEnd()); +} \ No newline at end of file diff --git a/sdk/lib/_internal/compiler/compiler.dart b/sdk/lib/_internal/compiler/compiler.dart index abeb36c4594..215f248749e 100644 --- a/sdk/lib/_internal/compiler/compiler.dart +++ b/sdk/lib/_internal/compiler/compiler.dart @@ -92,19 +92,21 @@ Future compile(Uri script, libraryRoot, packageRoot, options); - compiler.run(script); - String code = compiler.assembledCode; - if (code != null && outputProvider != null) { - String outputType = 'js'; - if (options.contains('--output-type=dart')) { - outputType = 'dart'; + // TODO(ahe): Use the value of the future (which signals success or failure). + return compiler.run(script).then((_) { + String code = compiler.assembledCode; + if (code != null && outputProvider != null) { + String outputType = 'js'; + if (options.contains('--output-type=dart')) { + outputType = 'dart'; + } + outputProvider('', outputType) + ..add(code) + ..close(); + code = ''; // Non-null signals success. } - outputProvider('', outputType) - ..add(code) - ..close(); - code = ''; // Non-null signals success. - } - return new Future.value(code); + return code; + }); } /** diff --git a/sdk/lib/_internal/compiler/implementation/apiimpl.dart b/sdk/lib/_internal/compiler/implementation/apiimpl.dart index b15631a731f..d211e9415c0 100644 --- a/sdk/lib/_internal/compiler/implementation/apiimpl.dart +++ b/sdk/lib/_internal/compiler/implementation/apiimpl.dart @@ -133,12 +133,10 @@ class Compiler extends leg.Compiler { return "lib/$path"; } - elements.LibraryElement scanBuiltinLibrary(String path) { + Future scanBuiltinLibrary(String path) { Uri uri = libraryRoot.resolve(lookupLibraryPath(path)); Uri canonicalUri = new Uri(scheme: "dart", path: path); - elements.LibraryElement library = - libraryLoader.loadLibrary(uri, null, canonicalUri); - return library; + return libraryLoader.loadLibrary(uri, null, canonicalUri); } void log(message) { @@ -157,30 +155,37 @@ class Compiler extends leg.Compiler { /** * Reads the script designated by [readableUri]. */ - leg.Script readScript(Uri readableUri, [tree.Node node]) { + Future readScript(Uri readableUri, + [elements.Element element, tree.Node node]) { if (!readableUri.isAbsolute) { internalError('Relative uri $readableUri provided to readScript(Uri)', node: node); } - return fileReadingTask.measure(() { - Uri resourceUri = translateUri(readableUri, node); - String text = ""; - try { - // TODO(ahe): We expect the future to be complete and call value - // directly. In effect, we don't support truly asynchronous API. - text = deprecatedFutureValue(provider(resourceUri)); - } catch (exception) { + + // TODO(johnniwinther): Add [:report(..., {Element element}):] to + // report methods in Compiler. + void reportReadError(String exception) { + withCurrentElement(element, () { reportError(node, leg.MessageKind.READ_SCRIPT_ERROR, {'uri': readableUri, 'exception': exception}); - return null; - } + }); + } + + Uri resourceUri = translateUri(readableUri, node); + // TODO(johnniwinther): Wrap the result from [provider] in a specialized + // [Future] to ensure that we never execute an asynchronous action without setting + // up the current element of the compiler. + return new Future.sync(() => provider(resourceUri)).then((String text) { SourceFile sourceFile = new SourceFile(resourceUri.toString(), text); // We use [readableUri] as the URI for the script since need to preserve // the scheme in the script because [Script.uri] is used for resolving // relative URIs mentioned in the script. See the comment on // [LibraryLoader] for more details. return new leg.Script(readableUri, sourceFile); + }).catchError((error) { + reportReadError(error); + return null; }); } @@ -254,18 +259,19 @@ class Compiler extends leg.Compiler { return packageRoot.resolve(uri.path); } - bool run(Uri uri) { + Future run(Uri uri) { log('Allowed library categories: $allowedLibraryCategories'); - bool success = super.run(uri); - int cumulated = 0; - for (final task in tasks) { - cumulated += task.timing; - log('${task.name} took ${task.timing}msec'); - } - int total = totalCompileTime.elapsedMilliseconds; - log('Total compile-time ${total}msec;' - ' unaccounted ${total - cumulated}msec'); - return success; + return super.run(uri).then((bool success) { + int cumulated = 0; + for (final task in tasks) { + cumulated += task.timing; + log('${task.name} took ${task.timing}msec'); + } + int total = totalCompileTime.elapsedMilliseconds; + log('Total compile-time ${total}msec;' + ' unaccounted ${total - cumulated}msec'); + return success; + }); } void reportDiagnostic(leg.SourceSpan span, String message, diff --git a/sdk/lib/_internal/compiler/implementation/compiler.dart b/sdk/lib/_internal/compiler/implementation/compiler.dart index 7e543d0ca65..158bb57edb1 100644 --- a/sdk/lib/_internal/compiler/implementation/compiler.dart +++ b/sdk/lib/_internal/compiler/implementation/compiler.dart @@ -36,7 +36,6 @@ abstract class WorkItem { assert(invariant(element, element.isDeclaration)); } - void run(Compiler compiler, Enqueuer world); } @@ -81,11 +80,6 @@ class PostProcessTask { PostProcessTask(this.element, this.action); } -class ReadingFilesTask extends CompilerTask { - ReadingFilesTask(Compiler compiler) : super(compiler); - String get name => 'Reading input files'; -} - abstract class Backend { final Compiler compiler; final ConstantSystem constantSystem; @@ -253,7 +247,9 @@ abstract class Backend { void registerStaticUse(Element element, Enqueuer enqueuer) {} - void onLibraryLoaded(LibraryElement library, Uri uri) {} + Future onLibraryLoaded(LibraryElement library, Uri uri) { + return new Future.value(); + } void registerMetadataInstantiatedType(DartType type, TreeElements elements) {} void registerMetadataStaticUse(Element element) {} @@ -512,7 +508,6 @@ abstract class Compiler implements DiagnosticListener { ConstantHandler constantHandler; ConstantHandler metadataHandler; EnqueueTask enqueuer; - CompilerTask fileReadingTask; DeferredLoadTask deferredLoadTask; MirrorUsageAnalyzerTask mirrorUsageAnalyzerTask; ContainerTracer containerTracer; @@ -614,7 +609,6 @@ abstract class Compiler implements DiagnosticListener { validator = new TreeValidatorTask(this); tasks = [ - fileReadingTask = new ReadingFilesTask(this), libraryLoader = new LibraryLoaderTask(this), scanner = new ScannerTask(this), dietParser = new DietParserTask(this), @@ -731,14 +725,15 @@ abstract class Compiler implements DiagnosticListener { reportDiagnostic(null, message, api.Diagnostic.VERBOSE_INFO); } - bool run(Uri uri) { + Future run(Uri uri) { totalCompileTime.start(); - try { - runCompiler(uri); - } on CompilerCancelledException catch (exception) { - log('Error: $exception'); - return false; - } catch (exception) { + + return new Future.sync(() => runCompiler(uri)).catchError((error) { + if (error is CompilerCancelledException) { + log('Error: $error'); + return false; + } + try { if (!hasCrashed) { hasCrashed = true; @@ -750,12 +745,13 @@ abstract class Compiler implements DiagnosticListener { } catch (doubleFault) { // Ignoring exceptions in exception handling. } - rethrow; - } finally { + throw error; + }).whenComplete(() { tracer.close(); totalCompileTime.stop(); - } - return !compilationFailed; + }).then((_) { + return !compilationFailed; + }); } bool hasIsolateSupport() => isolateLibrary != null; @@ -764,7 +760,7 @@ abstract class Compiler implements DiagnosticListener { * This method is called before [library] import and export scopes have been * set up. */ - void onLibraryLoaded(LibraryElement library, Uri uri) { + Future onLibraryLoaded(LibraryElement library, Uri uri) { if (dynamicClass != null) { // When loading the built-in libraries, dynamicClass is null. We // take advantage of this as core imports js_helper and sees [dynamic] @@ -791,12 +787,12 @@ abstract class Compiler implements DiagnosticListener { findRequiredElement(library, const SourceString('DeferredLibrary')); } else if (isolateHelperLibrary == null && (uri == new Uri(scheme: 'dart', path: '_isolate_helper'))) { - isolateHelperLibrary = scanBuiltinLibrary('_isolate_helper'); + isolateHelperLibrary = library; } else if (foreignLibrary == null && (uri == new Uri(scheme: 'dart', path: '_foreign_helper'))) { - foreignLibrary = scanBuiltinLibrary('_foreign_helper'); + foreignLibrary = library; } - backend.onLibraryLoaded(library, uri); + return backend.onLibraryLoaded(library, uri); } Element findRequiredElement(LibraryElement library, SourceString name) { @@ -824,7 +820,7 @@ abstract class Compiler implements DiagnosticListener { } } - LibraryElement scanBuiltinLibrary(String filename); + Future scanBuiltinLibrary(String filename); void initializeSpecialClasses() { final List missingCoreClasses = []; @@ -902,26 +898,32 @@ abstract class Compiler implements DiagnosticListener { listClass.lookupConstructor(callConstructor); } - void scanBuiltinLibraries() { - jsHelperLibrary = scanBuiltinLibrary('_js_helper'); - interceptorsLibrary = scanBuiltinLibrary('_interceptors'); - assertMethod = jsHelperLibrary.find(const SourceString('assertHelper')); - identicalFunction = coreLibrary.find(const SourceString('identical')); + Future scanBuiltinLibraries() { + return scanBuiltinLibrary('_js_helper').then((LibraryElement library) { + jsHelperLibrary = library; + return scanBuiltinLibrary('_interceptors'); + }).then((LibraryElement library) { + interceptorsLibrary = library; - initializeSpecialClasses(); + assertMethod = jsHelperLibrary.find(const SourceString('assertHelper')); + identicalFunction = coreLibrary.find(const SourceString('identical')); - functionClass.ensureResolved(this); - functionApplyMethod = - functionClass.lookupLocalMember(const SourceString('apply')); - jsInvocationMirrorClass.ensureResolved(this); - invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON); + initializeSpecialClasses(); - if (preserveComments) { - var uri = new Uri(scheme: 'dart', path: 'mirrors'); - LibraryElement libraryElement = - libraryLoader.loadLibrary(uri, null, uri); - documentClass = libraryElement.find(const SourceString('Comment')); - } + functionClass.ensureResolved(this); + functionApplyMethod = + functionClass.lookupLocalMember(const SourceString('apply')); + jsInvocationMirrorClass.ensureResolved(this); + invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON); + + if (preserveComments) { + var uri = new Uri(scheme: 'dart', path: 'mirrors'); + return libraryLoader.loadLibrary(uri, null, uri).then( + (LibraryElement libraryElement) { + documentClass = libraryElement.find(const SourceString('Comment')); + }); + } + }); } void importHelperLibrary(LibraryElement library) { @@ -936,28 +938,39 @@ abstract class Compiler implements DiagnosticListener { */ Uri resolvePatchUri(String dartLibraryPath); - void runCompiler(Uri uri) { + Future runCompiler(Uri uri) { // TODO(ahe): This prevents memory leaks when invoking the compiler // multiple times. Implement a better mechanism where StringWrapper // instances are shared on a per library basis. SourceString.canonicalizedValues.clear(); assert(uri != null || analyzeOnly); - scanBuiltinLibraries(); - if (librariesToAnalyzeWhenRun != null) { - for (Uri libraryUri in librariesToAnalyzeWhenRun) { - log('analyzing $libraryUri ($buildId)'); - libraryLoader.loadLibrary(libraryUri, null, libraryUri); + return scanBuiltinLibraries().then((_) { + if (librariesToAnalyzeWhenRun != null) { + return Future.forEach(librariesToAnalyzeWhenRun, (libraryUri) { + log('analyzing $libraryUri ($buildId)'); + return libraryLoader.loadLibrary(libraryUri, null, libraryUri); + }); } - } - if (uri != null) { - if (analyzeOnly) { - log('analyzing $uri ($buildId)'); - } else { - log('compiling $uri ($buildId)'); + }).then((_) { + if (uri != null) { + if (analyzeOnly) { + log('analyzing $uri ($buildId)'); + } else { + log('compiling $uri ($buildId)'); + } + return libraryLoader.loadLibrary(uri, null, uri) + .then((LibraryElement library) { + mainApp = library; + }); } - mainApp = libraryLoader.loadLibrary(uri, null, uri); - } + }).then((_) { + compileLoadedLibraries(); + }); + } + + /// Performs the compilation when all libraries have been loaded. + void compileLoadedLibraries() { Element main = null; if (mainApp != null) { main = mainApp.find(MAIN); @@ -973,8 +986,8 @@ abstract class Compiler implements DiagnosticListener { mainApp, MessageKind.GENERIC, {'text': 'Error: Could not find "${MAIN.slowToString()}". ' - 'No source will be analyzed. ' - 'Use "--analyze-all" to analyze all code in the library.'}); + 'No source will be analyzed. ' + 'Use "--analyze-all" to analyze all code in the library.'}); } } else { if (!main.isFunction()) { @@ -990,7 +1003,7 @@ abstract class Compiler implements DiagnosticListener { parameter, MessageKind.GENERIC, {'text': - 'Error: "${MAIN.slowToString()}" cannot have parameters.'}); + 'Error: "${MAIN.slowToString()}" cannot have parameters.'}); }); } @@ -1046,7 +1059,8 @@ abstract class Compiler implements DiagnosticListener { backend.enableNoSuchMethod(enqueuer.codegen); } if (compileAll) { - libraries.forEach((_, lib) => fullyEnqueueLibrary(lib, enqueuer.codegen)); + libraries.forEach((_, lib) => fullyEnqueueLibrary(lib, + enqueuer.codegen)); } processQueue(enqueuer.codegen, main); enqueuer.codegen.logSummary(log); @@ -1373,7 +1387,7 @@ abstract class Compiler implements DiagnosticListener { * * See [LibraryLoader] for terminology on URIs. */ - Script readScript(Uri readableUri, [Node node]) { + Future