From f5e26456bfa6feed59136fee66c93d3096710094 Mon Sep 17 00:00:00 2001 From: Devon Carew Date: Mon, 16 Nov 2020 03:31:18 +0000 Subject: [PATCH] Remove the --completion-model flag from the analysis server. Change-Id: I6bedbb0b5c4b8bedba18d2e3b788db0a9686f6df Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/172026 Commit-Queue: Devon Carew Reviewed-by: Brian Wilkerson --- .dart_tool/package_config.json | 6 - .packages | 1 - DEPS | 3 - .../lib/src/analysis_server.dart | 5 - .../lib/src/server/driver.dart | 82 +---- .../lib/src/server/sdk_configuration.dart | 3 - .../completion/dart/completion_manager.dart | 36 +- .../completion/dart/completion_ranking.dart | 309 ------------------ .../completion/dart/language_model.dart | 155 --------- .../lib/src/status/diagnostics.dart | 96 ------ pkg/analysis_server/pubspec.yaml | 1 - .../dart/completion_ranking_test.dart | 35 -- .../completion/dart/language_model_test.dart | 86 ----- .../services/completion/dart/test_all.dart | 10 - .../src/server/sdk_configuration_test.dart | 6 +- sdk/BUILD.gn | 42 --- 16 files changed, 6 insertions(+), 870 deletions(-) delete mode 100644 pkg/analysis_server/lib/src/services/completion/dart/completion_ranking.dart delete mode 100644 pkg/analysis_server/lib/src/services/completion/dart/language_model.dart delete mode 100644 pkg/analysis_server/test/services/completion/dart/completion_ranking_test.dart delete mode 100644 pkg/analysis_server/test/services/completion/dart/language_model_test.dart diff --git a/.dart_tool/package_config.json b/.dart_tool/package_config.json index 7e58d34a423..c75c95d3816 100644 --- a/.dart_tool/package_config.json +++ b/.dart_tool/package_config.json @@ -688,12 +688,6 @@ "packageUri": "lib/", "languageVersion": "2.0" }, - { - "name": "tflite_native", - "rootUri": "../third_party/pkg/tflite_native", - "packageUri": "lib/", - "languageVersion": "2.6" - }, { "name": "typed_data", "rootUri": "../third_party/pkg/typed_data", diff --git a/.packages b/.packages index f264745b12b..d31bc9b829f 100644 --- a/.packages +++ b/.packages @@ -107,7 +107,6 @@ test_process:third_party/pkg/test_process/lib test_reflective_loader:third_party/pkg/test_reflective_loader/lib test_runner:pkg/test_runner/lib testing:pkg/testing/lib -tflite_native:third_party/pkg/tflite_native/lib typed_data:third_party/pkg/typed_data/lib usage:third_party/pkg/usage/lib vector_math:third_party/pkg/vector_math/lib diff --git a/DEPS b/DEPS index 400341461e1..f53c12edd6c 100644 --- a/DEPS +++ b/DEPS @@ -157,7 +157,6 @@ vars = { "term_glyph_rev": "6a0f9b6fb645ba75e7a00a4e20072678327a0347", "test_reflective_loader_tag": "0.1.9", "test_rev": "e37a93bbeae23b215972d1659ac865d71287ff6a", - "tflite_native_rev": "0.4.0+1", "typed_data_tag": "f94fc57b8e8c0e4fe4ff6cfd8290b94af52d3719", "usage_tag": "16fbfd90c58f16e016a295a880bc722d2547d2c9", "vector_math_rev": "0c9f5d68c047813a6dcdeb88ba7a42daddf25025", @@ -431,8 +430,6 @@ deps = { Var("dart_git") + "term_glyph.git" + "@" + Var("term_glyph_rev"), Var("dart_root") + "/third_party/pkg/test": Var("dart_git") + "test.git" + "@" + Var("test_rev"), - Var("dart_root") + "/third_party/pkg/tflite_native": - Var("dart_git") + "tflite_native.git" + "@" + Var("tflite_native_rev"), Var("dart_root") + "/third_party/pkg/test_descriptor": Var("dart_git") + "test_descriptor.git" + "@" + Var("test_descriptor_tag"), Var("dart_root") + "/third_party/pkg/test_process": diff --git a/pkg/analysis_server/lib/src/analysis_server.dart b/pkg/analysis_server/lib/src/analysis_server.dart index a6a5a4e4143..4599f0a9d9b 100644 --- a/pkg/analysis_server/lib/src/analysis_server.dart +++ b/pkg/analysis_server/lib/src/analysis_server.dart @@ -625,11 +625,6 @@ class AnalysisServerOptions { /// Whether to use the Language Server Protocol. bool useLanguageServerProtocol = false; - /// Base path to locate trained completion language model files. - /// - /// ML completion is enabled if this is non-null. - String completionModelFolder; - /// The set of enabled features. FeatureSet featureSet = FeatureSet(); } diff --git a/pkg/analysis_server/lib/src/server/driver.dart b/pkg/analysis_server/lib/src/server/driver.dart index 642f8958ec2..734177de7c4 100644 --- a/pkg/analysis_server/lib/src/server/driver.dart +++ b/pkg/analysis_server/lib/src/server/driver.dart @@ -3,7 +3,6 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:ffi' as ffi; import 'dart:io'; import 'dart:isolate'; import 'dart:math'; @@ -24,11 +23,9 @@ import 'package:analysis_server/src/server/isolate_analysis_server.dart'; import 'package:analysis_server/src/server/lsp_stdio_server.dart'; import 'package:analysis_server/src/server/sdk_configuration.dart'; import 'package:analysis_server/src/server/stdio_server.dart'; -import 'package:analysis_server/src/services/completion/dart/completion_ranking.dart'; import 'package:analysis_server/src/socket_server.dart'; import 'package:analysis_server/src/utilities/request_statistics.dart'; import 'package:analysis_server/starter.dart'; -import 'package:analyzer/exception/exception.dart'; import 'package:analyzer/file_system/physical_file_system.dart'; import 'package:analyzer/instrumentation/file_instrumentation.dart'; import 'package:analyzer/instrumentation/instrumentation.dart'; @@ -38,7 +35,6 @@ import 'package:analyzer/src/generated/sdk.dart'; import 'package:args/args.dart'; import 'package:cli_util/cli_util.dart'; import 'package:linter/src/rules.dart' as linter; -import 'package:path/path.dart' as path; import 'package:telemetry/crash_reporting.dart'; import 'package:telemetry/telemetry.dart' as telemetry; @@ -265,13 +261,6 @@ class Driver implements ServerStarter { /// The name of the flag to use the Language Server Protocol (LSP). static const String USE_LSP = 'lsp'; - /// Whether or not to enable ML ranking for code completion. - static const String ENABLE_COMPLETION_MODEL = 'enable-completion-model'; - - /// The path on disk to a directory containing language model files for smart - /// code completion. - static const String COMPLETION_MODEL_FOLDER = 'completion-model'; - /// A directory to analyze in order to train an analysis server snapshot. static const String TRAIN_USING = 'train-using'; @@ -318,35 +307,6 @@ class Driver implements ServerStarter { var sdkConfig = SdkConfiguration.readFromSdk(); analysisServerOptions.configurationOverrides = sdkConfig; - // ML model configuration. - // TODO(brianwilkerson) Uncomment the line below and delete the second line - // when there is a new completion model to query. Until then we ignore the - // flag to enable the model so that we can't try to read from a file that - // doesn't exist. -// final bool enableCompletionModel = results[ENABLE_COMPLETION_MODEL]; - final enableCompletionModel = false; - analysisServerOptions.completionModelFolder = - results[COMPLETION_MODEL_FOLDER]; - if (results.wasParsed(ENABLE_COMPLETION_MODEL) && !enableCompletionModel) { - // This is the case where the user has explicitly turned off model-based - // code completion. - analysisServerOptions.completionModelFolder = null; - } - // TODO(devoncarew): Simplify this logic and use the value from sdkConfig. - if (enableCompletionModel && - analysisServerOptions.completionModelFolder == null) { - // The user has enabled ML code completion without explicitly setting a - // model for us to choose, so use the default one. We need to walk over - // from $SDK/bin/snapshots/analysis_server.dart.snapshot to - // $SDK/bin/model/lexeme. - analysisServerOptions.completionModelFolder = path.join( - File.fromUri(Platform.script).parent.path, - '..', - 'model', - 'lexeme', - ); - } - // Analytics bool disableAnalyticsForSession = results[SUPPRESS_ANALYTICS_FLAG]; if (results.wasParsed(TRAIN_USING)) { @@ -601,7 +561,6 @@ class Driver implements ServerStarter { socketServer.analysisServer.shutdown(); if (sendPort == null) exit(0); }); - startCompletionRanking(socketServer, null, analysisServerOptions); }, print: results[INTERNAL_PRINT_TO_CONSOLE] ? null @@ -609,36 +568,6 @@ class Driver implements ServerStarter { } } - /// This will be invoked after createAnalysisServer has been called on the - /// socket server. At that point, we'll be able to send a server.error - /// notification in case model startup fails. - void startCompletionRanking( - SocketServer socketServer, - LspSocketServer lspSocketServer, - AnalysisServerOptions analysisServerOptions) { - // If ML completion is not enabled, or we're on a 32-bit machine, don't try - // and start the completion model. - if (analysisServerOptions.completionModelFolder == null || - ffi.sizeOf() == 4) { - return; - } - - // Start completion model isolate if this is a 64 bit system and analysis - // server was configured to load a language model on disk. - CompletionRanking.instance = - CompletionRanking(analysisServerOptions.completionModelFolder); - CompletionRanking.instance.start().catchError((exception, stackTrace) { - // Disable smart ranking if model startup fails. - analysisServerOptions.completionModelFolder = null; - // TODO(brianwilkerson) Shutdown the isolates that have already been - // started. - CompletionRanking.instance = null; - AnalysisEngine.instance.instrumentationService.logException( - CaughtException.withMessage( - 'Failed to start ranking model isolate', exception, stackTrace)); - }); - } - void startLspServer( ArgResults args, AnalysisServerOptions analysisServerOptions, @@ -681,7 +610,6 @@ class Driver implements ServerStarter { exit(0); } }); - startCompletionRanking(null, socketServer, analysisServerOptions); }); } @@ -772,18 +700,16 @@ class Driver implements ServerStarter { help: 'Pass in a directory to analyze for purposes of training an ' 'analysis server snapshot.'); - parser.addFlag(ENABLE_COMPLETION_MODEL, - help: 'Whether or not to turn on ML ranking for code completion.'); - parser.addOption(COMPLETION_MODEL_FOLDER, - valueHelp: 'path', - help: 'Path to the location of a code completion model.'); - // // Deprecated options - no longer read from. // + // Removed 11/15/2020. + parser.addOption('completion-model', hide: true); // Removed 11/8/2020. parser.addFlag('dartpad', hide: true); + // Removed 11/15/2020. + parser.addFlag('enable-completion-model', hide: true); // Removed 10/30/2020. parser.addMultiOption('enable-experiment', hide: true); // Removed 9/23/2020. diff --git a/pkg/analysis_server/lib/src/server/sdk_configuration.dart b/pkg/analysis_server/lib/src/server/sdk_configuration.dart index d952a219873..08105f8dd0d 100644 --- a/pkg/analysis_server/lib/src/server/sdk_configuration.dart +++ b/pkg/analysis_server/lib/src/server/sdk_configuration.dart @@ -61,9 +61,6 @@ class SdkConfiguration { /// Returns whether this SDK configuration has any configured values. bool get hasAnyOverrides => _values.isNotEmpty; - /// Return an override value for the analysis server's ML model file path. - String get mlModelPath => _values['server.ml.model.path']; - @override String toString() => displayString; diff --git a/pkg/analysis_server/lib/src/services/completion/dart/completion_manager.dart b/pkg/analysis_server/lib/src/services/completion/dart/completion_manager.dart index 67150b830ee..0b05d08d421 100644 --- a/pkg/analysis_server/lib/src/services/completion/dart/completion_manager.dart +++ b/pkg/analysis_server/lib/src/services/completion/dart/completion_manager.dart @@ -10,7 +10,6 @@ import 'package:analysis_server/src/services/completion/completion_core.dart'; import 'package:analysis_server/src/services/completion/completion_performance.dart'; import 'package:analysis_server/src/services/completion/dart/arglist_contributor.dart'; import 'package:analysis_server/src/services/completion/dart/combinator_contributor.dart'; -import 'package:analysis_server/src/services/completion/dart/completion_ranking.dart'; import 'package:analysis_server/src/services/completion/dart/extension_member_contributor.dart'; import 'package:analysis_server/src/services/completion/dart/feature_computer.dart'; import 'package:analysis_server/src/services/completion/dart/field_formal_contributor.dart'; @@ -36,7 +35,6 @@ import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/ast/token.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:analyzer/dart/element/type.dart'; -import 'package:analyzer/exception/exception.dart'; import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart'; import 'package:analyzer/src/dart/ast/ast.dart'; @@ -116,10 +114,6 @@ class DartCompletionManager { request.checkAborted(); - final ranking = CompletionRanking.instance; - var probabilityFuture = - ranking != null ? ranking.predict(dartRequest) : Future.value(null); - var range = dartRequest.target.computeReplacementRange(dartRequest.offset); (request as CompletionRequestImpl) ..replacementOffset = range.offset @@ -169,35 +163,7 @@ class DartCompletionManager { throw AbortCompletion(); } - // Adjust suggestion relevance before returning - var suggestions = builder.suggestions.toList(); - const SORT_TAG = 'DartCompletionManager - sort'; - await performance.runAsync(SORT_TAG, (_) async { - if (ranking != null) { - request.checkAborted(); - try { - suggestions = await ranking.rerank( - probabilityFuture, - suggestions, - includedElementNames, - includedSuggestionRelevanceTags, - dartRequest, - request.result.unit.featureSet); - } catch (exception, stackTrace) { - // TODO(brianwilkerson) Shutdown the isolates that have already been - // started. - // Disable smart ranking if prediction fails. - CompletionRanking.instance = null; - AnalysisEngine.instance.instrumentationService.logException( - CaughtException.withMessage( - 'Failed to rerank completion suggestions', - exception, - stackTrace)); - } - } - }); - request.checkAborted(); - return suggestions; + return builder.suggestions.toList(); } void _addIncludedElementKinds(DartCompletionRequestImpl request) { diff --git a/pkg/analysis_server/lib/src/services/completion/dart/completion_ranking.dart b/pkg/analysis_server/lib/src/services/completion/dart/completion_ranking.dart deleted file mode 100644 index 021aaf88166..00000000000 --- a/pkg/analysis_server/lib/src/services/completion/dart/completion_ranking.dart +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:collection'; -import 'dart:isolate'; - -import 'package:analysis_server/src/protocol_server.dart'; -import 'package:analysis_server/src/provisional/completion/dart/completion_dart.dart'; -import 'package:analysis_server/src/services/completion/completion_performance.dart'; -import 'package:analysis_server/src/services/completion/dart/completion_ranking_internal.dart'; -import 'package:analysis_server/src/services/completion/dart/language_model.dart'; -import 'package:analyzer/dart/analysis/features.dart'; - -/// Number of code completion isolates. -// TODO(devoncarew): We need to explore the memory costs of running multiple ML -// isolates. -const int _ISOLATE_COUNT = 2; - -/// Number of lookback tokens. -const int _LOOKBACK = 100; - -/// Minimum probability to prioritize model-only suggestion. -const double _MODEL_RELEVANCE_CUTOFF = 0.5; - -/// Prediction service run by the model isolate. -void entrypoint(SendPort sendPort) { - LanguageModel model; - final port = ReceivePort(); - sendPort.send(port.sendPort); - port.listen((message) { - var response = >{}; - switch (message['method']) { - case 'load': - model = LanguageModel.load(message['args'][0]); - break; - case 'predict': - response['data'] = model.predictWithScores(message['args']); - break; - } - - message['port'].send(response); - }); -} - -class CompletionRanking { - /// Singleton instance. - static CompletionRanking instance; - - /// Filesystem location of model files. - final String _directory; - - /// Ports to communicate from main to model isolates. - List _writes; - - /// Pointer for round robin load balancing over isolates. - int _index; - - /// General performance metrics around ML completion. - final PerformanceMetrics performanceMetrics = PerformanceMetrics._(); - - CompletionRanking(this._directory); - - /// Send an RPC to the isolate worker requesting that it load the model and - /// wait for it to respond. - Future>> makeLoadRequest( - SendPort sendPort, List args) async { - final receivePort = ReceivePort(); - sendPort.send({ - 'method': 'load', - 'args': args, - 'port': receivePort.sendPort, - }); - return await receivePort.first; - } - - /// Send an RPC to the isolate worker requesting that it make a prediction and - /// wait for it to respond. - Future>> makePredictRequest( - List args) async { - final receivePort = ReceivePort(); - _writes[_index].send({ - 'method': 'predict', - 'args': args, - 'port': receivePort.sendPort, - }); - _index = (_index + 1) % _writes.length; - return await receivePort.first; - } - - /// Return a next-token prediction starting at the completion request cursor - /// and walking back to find previous input tokens, or `null` if the - /// prediction isolates are not running. - Future> predict(DartCompletionRequest request) async { - if (_writes == null || _writes.isEmpty) { - // The field `_writes` is initialized in `start`, but the code that - // invokes `start` doesn't wait for it complete. That means that it's - // possible for this method to be invoked before `_writes` is initialized. - // In those cases we return `null` - return null; - } - final query = constructQuery(request, _LOOKBACK); - if (query == null) { - return Future.value(); - } - - performanceMetrics._incrementPredictionRequestCount(); - - var timer = Stopwatch()..start(); - var response = await makePredictRequest(query); - timer.stop(); - - var result = response['data']; - - performanceMetrics._addPredictionResult(PredictionResult( - result, - timer.elapsed, - request.source.fullName, - computeCompletionSnippet(request.sourceContents, request.offset), - )); - - return result; - } - - /// Transforms [CompletionSuggestion] relevances and - /// [IncludedSuggestionRelevanceTag] relevanceBoosts based on language model - /// predicted next-token probability distribution. - Future> rerank( - Future> probabilityFuture, - List suggestions, - Set includedElementNames, - List includedSuggestionRelevanceTags, - DartCompletionRequest request, - FeatureSet featureSet) async { - assert((includedElementNames != null && - includedSuggestionRelevanceTags != null) || - (includedElementNames == null && - includedSuggestionRelevanceTags == null)); - final probability = await probabilityFuture - .timeout(const Duration(seconds: 1), onTimeout: () => null); - if (probability == null || probability.isEmpty) { - // Failed to compute probability distribution, don't rerank. - return suggestions; - } - - // Discard the type-based relevance boosts. - if (includedSuggestionRelevanceTags != null) { - includedSuggestionRelevanceTags.forEach((tag) { - tag.relevanceBoost = 0; - }); - } - - // Intersection between static analysis and model suggestions. - var middle = DART_RELEVANCE_HIGH + probability.length; - // Up to one suggestion from model with very high confidence. - var high = middle + probability.length; - // Lower relevance, model-only suggestions (perhaps literals). - var low = DART_RELEVANCE_LOW - 1; - - List entries = probability.entries.toList() - ..sort((a, b) => b.value.compareTo(a.value)); - - if (testInsideQuotes(request)) { - // If completion is requested inside of quotes, remove any suggestions - // which are not string literal. - entries = selectStringLiterals(entries); - } else if (request.opType.includeVarNameSuggestions && - suggestions.every((CompletionSuggestion suggestion) => - suggestion.kind == CompletionSuggestionKind.IDENTIFIER)) { - // If analysis server thinks this is a declaration context, - // remove all of the model-suggested literals. - // TODO(lambdabaa): Ask Brian for help leveraging - // SimpleIdentifier#inDeclarationContext. - entries.retainWhere((MapEntry entry) => !isLiteral(entry.key)); - } - - var allowModelOnlySuggestions = - !testNamedArgument(suggestions) && !testFollowingDot(request); - for (var entry in entries) { - // There may be multiple like - // CompletionSuggestion and CompletionSuggestion(). - final completionSuggestions = suggestions.where((suggestion) => - areCompletionsEquivalent(suggestion.completion, entry.key)); - List includedSuggestions; - final isIncludedElementName = includedElementNames != null && - includedElementNames.contains(entry.key); - if (includedSuggestionRelevanceTags != null) { - includedSuggestions = includedSuggestionRelevanceTags - .where((tag) => areCompletionsEquivalent( - elementNameFromRelevanceTag(tag.tag), entry.key)) - .toList(); - } else { - includedSuggestions = []; - } - if (allowModelOnlySuggestions && entry.value > _MODEL_RELEVANCE_CUTOFF) { - final relevance = high--; - if (completionSuggestions.isNotEmpty || - includedSuggestions.isNotEmpty) { - completionSuggestions.forEach((completionSuggestion) { - completionSuggestion.relevance = relevance; - }); - includedSuggestions.forEach((includedSuggestion) { - includedSuggestion.relevanceBoost = relevance; - }); - } else if (isIncludedElementName) { - if (includedSuggestionRelevanceTags != null) { - includedSuggestionRelevanceTags - .add(IncludedSuggestionRelevanceTag(entry.key, relevance)); - } - } else { - suggestions - .add(createCompletionSuggestion(entry.key, featureSet, high--)); - } - } else if (completionSuggestions.isNotEmpty || - includedSuggestions.isNotEmpty || - isIncludedElementName) { - final relevance = middle--; - completionSuggestions.forEach((completionSuggestion) { - completionSuggestion.relevance = relevance; - }); - if (includedSuggestions.isNotEmpty) { - includedSuggestions.forEach((includedSuggestion) { - includedSuggestion.relevanceBoost = relevance; - }); - } else if (includedSuggestionRelevanceTags != null) { - includedSuggestionRelevanceTags - .add(IncludedSuggestionRelevanceTag(entry.key, relevance)); - } - } else if (allowModelOnlySuggestions) { - final relevance = low--; - suggestions - .add(createCompletionSuggestion(entry.key, featureSet, relevance)); - if (includedSuggestionRelevanceTags != null) { - includedSuggestionRelevanceTags - .add(IncludedSuggestionRelevanceTag(entry.key, relevance)); - } - } - } - return suggestions; - } - - /// Spin up the model isolates and load the tflite model. - Future start() async { - _writes = []; - _index = 0; - final initializations = >[]; - - // Start the first isolate. - await _startIsolate(); - - // Start the 2nd and later isolates. - for (var i = 1; i < _ISOLATE_COUNT; i++) { - initializations.add(_startIsolate()); - } - - return Future.wait(initializations); - } - - Future _startIsolate() async { - var timer = Stopwatch()..start(); - var port = ReceivePort(); - await Isolate.spawn(entrypoint, port.sendPort); - SendPort sendPort = await port.first; - return makeLoadRequest(sendPort, [_directory]).whenComplete(() { - timer.stop(); - performanceMetrics._isolateInitTimes.add(timer.elapsed); - _writes.add(sendPort); - }); - } -} - -class PerformanceMetrics { - static const int _maxResultBuffer = 50; - - final Queue _predictionResults = Queue(); - int _predictionRequestCount = 0; - final List _isolateInitTimes = []; - - PerformanceMetrics._(); - - List get isolateInitTimes => _isolateInitTimes; - - /// The total prediction requests to ML Complete. - int get predictionRequestCount => _predictionRequestCount; - - /// An iterable of the last `n` prediction results; - Iterable get predictionResults => _predictionResults; - - void _addPredictionResult(PredictionResult request) { - _predictionResults.addFirst(request); - if (_predictionResults.length > _maxResultBuffer) { - _predictionResults.removeLast(); - } - } - - void _incrementPredictionRequestCount() { - _predictionRequestCount++; - } -} - -class PredictionResult { - final Map results; - final Duration elapsedTime; - final String sourcePath; - final String snippet; - - PredictionResult( - this.results, this.elapsedTime, this.sourcePath, this.snippet); -} diff --git a/pkg/analysis_server/lib/src/services/completion/dart/language_model.dart b/pkg/analysis_server/lib/src/services/completion/dart/language_model.dart deleted file mode 100644 index 8a2e044849e..00000000000 --- a/pkg/analysis_server/lib/src/services/completion/dart/language_model.dart +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:path/path.dart' as path; -import 'package:tflite_native/tflite.dart' as tfl; - -/// Interface to TensorFlow-based Dart language model for next-token prediction. -class LanguageModel { - static const _probabilityThreshold = 0.0001; - static final _numeric = RegExp(r'^\d+(\.\d+)?$'); - static final _alphanumeric = RegExp(r"^['\w]+$"); - static final _doubleQuote = '"'.codeUnitAt(0); - - final tfl.Interpreter _interpreter; - final Map _word2idx; - final Map _idx2word; - final int _lookback; - - /// Load model from directory. - factory LanguageModel.load(String directory) { - // Load model. - final interpreter = - tfl.Interpreter.fromFile(path.join(directory, 'model.tflite')); - interpreter.allocateTensors(); - - // Load word2idx mapping for input. - final word2idx = json - .decode(File(path.join(directory, 'word2idx.json')).readAsStringSync()) - .cast(); - - // Load idx2word mapping for output. - final idx2word = json - .decode(File(path.join(directory, 'idx2word.json')).readAsStringSync()) - .map((k, v) => MapEntry(int.parse(k), v)); - - // Get lookback size from model input tensor shape. - final tensorShape = interpreter.getInputTensors().single.shape; - if (tensorShape.length != 2 || tensorShape.first != 1) { - throw ArgumentError( - 'tensor shape $tensorShape does not match the expected [1, X]'); - } - final lookback = tensorShape.last; - - return LanguageModel._(interpreter, word2idx, idx2word, lookback); - } - - LanguageModel._( - this._interpreter, this._word2idx, this._idx2word, this._lookback); - - /// Number of previous tokens to look at during predictions. - int get lookback => _lookback; - - /// Tear down the interpreter. - void close() { - _interpreter.delete(); - } - - bool isNumber(String token) { - return _numeric.hasMatch(token) || token.startsWith('0x'); - } - - /// Predicts the next token to follow a list of precedent tokens - /// - /// Returns a list of tokens, sorted by most probable first. - List predict(List tokens) => - predictWithScores(tokens).keys.toList(); - - /// Predicts the next token with confidence scores. - /// - /// Returns an ordered map of tokens to scores, sorted by most probable first. - Map predictWithScores(List tokens) { - final tensorIn = _interpreter.getInputTensors().single; - tensorIn.data = _transformInput(tokens); - _interpreter.invoke(); - final tensorOut = _interpreter.getOutputTensors().single; - return _transformOutput(tensorOut.data, tokens); - } - - bool _isAlphanumeric(String token) { - // Note that _numeric covers integral and decimal values whereas - // _alphanumeric only matches integral values. Check both. - return _alphanumeric.hasMatch(token) || _numeric.hasMatch(token); - } - - bool _isString(String token) { - return token.contains('"') || token.contains("'"); - } - - /// Transforms tokens to data bytes that can be used as interpreter input. - List _transformInput(List tokens) { - // Replace out of vocabulary tokens. - final sanitizedTokens = tokens.map((token) { - if (_word2idx.containsKey(token)) { - return token; - } - if (isNumber(token)) { - return ''; - } - if (_isString(token)) { - return ''; - } - return ''; - }); - // Get indexes (as floats). - final indexes = Float32List(lookback) - ..setAll(0, sanitizedTokens.map((token) => _word2idx[token].toDouble())); - // Get bytes - return Uint8List.view(indexes.buffer); - } - - /// Transforms interpreter output data to map of tokens to scores. - Map _transformOutput( - List databytes, List tokens) { - // Get bytes. - final bytes = Uint8List.fromList(databytes); - - // Get scores (as floats) - final probabilities = Float32List.view(bytes.buffer); - - final scores = {}; - final scoresAboveThreshold = {}; - probabilities.asMap().forEach((k, v) { - // x in 0, 1, ..., |V| - 1 correspond to specific members of the vocabulary. - // x in |V|, |V| + 1, ..., |V| + 49 are pointers to reference positions along the - // network input. - if (k >= _idx2word.length + tokens.length) { - return; - } - // Find the name corresponding to this position along the network output. - final lexeme = - k < _idx2word.length ? _idx2word[k] : tokens[k - _idx2word.length]; - // Normalize double to single quotes. - final sanitized = lexeme.codeUnitAt(0) != _doubleQuote - ? lexeme - : lexeme.replaceAll('"', '\''); - final score = (scores[sanitized] ?? 0.0) + v; - scores[sanitized] = score; - if (score < _probabilityThreshold || - k >= _idx2word.length && !_isAlphanumeric(sanitized)) { - // Discard names below a fixed likelihood, and - // don't assign probability to punctuation by reference. - return; - } - scoresAboveThreshold[sanitized] = score; - }); - - return Map.fromEntries(scoresAboveThreshold.entries.toList() - ..sort((a, b) => b.value.compareTo(a.value))); - } -} diff --git a/pkg/analysis_server/lib/src/status/diagnostics.dart b/pkg/analysis_server/lib/src/status/diagnostics.dart index bd39067516c..8f047474330 100644 --- a/pkg/analysis_server/lib/src/status/diagnostics.dart +++ b/pkg/analysis_server/lib/src/status/diagnostics.dart @@ -16,7 +16,6 @@ import 'package:analysis_server/src/lsp/lsp_analysis_server.dart' import 'package:analysis_server/src/plugin/plugin_manager.dart'; import 'package:analysis_server/src/server/http_server.dart'; import 'package:analysis_server/src/services/completion/completion_performance.dart'; -import 'package:analysis_server/src/services/completion/dart/completion_ranking.dart'; import 'package:analysis_server/src/socket_server.dart'; import 'package:analysis_server/src/status/ast_writer.dart'; import 'package:analysis_server/src/status/element_writer.dart'; @@ -759,7 +758,6 @@ class DiagnosticsSite extends Site implements AbstractGetHandler { // Add server-specific pages. Ordering doesn't matter as the items are // sorted later. var server = socketServer.analysisServer; - pages.add(MLCompletionPage(this, server)); pages.add(PluginsPage(this, server)); if (server is AnalysisServer) { @@ -1074,100 +1072,6 @@ class MemoryAndCpuPage extends DiagnosticPageWithNav { } } -class MLCompletionPage extends DiagnosticPageWithNav { - @override - final AbstractAnalysisServer server; - - MLCompletionPage(DiagnosticsSite site, this.server) - : super(site, 'ml-completion', 'ML Completion', - description: 'Statistics for ML code completion.'); - - path.Context get pathContext => server.resourceProvider.pathContext; - - @override - Future generateContent(Map params) async { - var hasMLComplete = CompletionRanking.instance != null; - if (!hasMLComplete) { - blankslate('''ML code completion is not enabled (see previewing Dart ML completion for how to enable it).'''); - return; - } - - buf.writeln('ML completion enabled.
'); - - var isolateTimes = CompletionRanking - .instance.performanceMetrics.isolateInitTimes - .map((Duration time) { - return '${time.inMilliseconds}ms'; - }).join(', '); - p('ML isolate init times: $isolateTimes'); - - var predictions = CompletionRanking - .instance.performanceMetrics.predictionResults - .toList(); - - if (predictions.isEmpty) { - blankslate('No completions recorded.'); - return; - } - - p('${CompletionRanking.instance.performanceMetrics.predictionRequestCount} ' - 'requests'); - - // draw a chart - buf.writeln( - '
'); - var rowData = StringBuffer(); - for (var prediction in predictions.reversed) { - // [' ', 101.5] - if (rowData.isNotEmpty) { - rowData.write(','); - } - rowData.write("[' ', ${prediction.elapsedTime.inMilliseconds}]"); - } - buf.writeln(''' - -'''); - - String summarize(PredictionResult prediction) { - var entries = prediction.results.entries.toList(); - entries.sort((a, b) => b.value.compareTo(a.value)); - var summary = entries - .take(3) - .map((entry) => '"${entry.key}":${entry.value.toStringAsFixed(3)}') - .join('
'); - return summary; - } - - // emit the data as a table - buf.writeln(''); - buf.writeln( - ''); - for (var prediction in predictions) { - buf.writeln('' - '' - '' - '' - '' - ''); - } - buf.writeln('
TimeResultsSnippetTop suggestions
${printMilliseconds(prediction.elapsedTime.inMilliseconds)}${prediction.results.length}${escape(prediction.snippet)}${summarize(prediction)}
'); - } -} - class NotFoundPage extends DiagnosticPage { @override final String path; diff --git a/pkg/analysis_server/pubspec.yaml b/pkg/analysis_server/pubspec.yaml index 2b6420351b5..9a94f7dc690 100644 --- a/pkg/analysis_server/pubspec.yaml +++ b/pkg/analysis_server/pubspec.yaml @@ -25,7 +25,6 @@ dependencies: path: ../telemetry test: any path: any - tflite_native: any watcher: any yaml: any diff --git a/pkg/analysis_server/test/services/completion/dart/completion_ranking_test.dart b/pkg/analysis_server/test/services/completion/dart/completion_ranking_test.dart deleted file mode 100644 index 841ad47e8f5..00000000000 --- a/pkg/analysis_server/test/services/completion/dart/completion_ranking_test.dart +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:io'; - -import 'package:analysis_server/src/services/completion/dart/completion_ranking.dart'; -import 'package:path/path.dart' as path; -import 'package:test/test.dart'; - -void main() { - CompletionRanking ranking; - - setUp(() async { - ranking = CompletionRanking(directory); - await ranking.start(); - }); - - test('make request to isolate', () async { - final tokens = - tokenize('if (list == null) { return; } for (final i = 0; i < list.'); - final response = await ranking.makePredictRequest(tokens); - expect(response['data']['length'], greaterThan(0.9)); - }, skip: 'https://github.com/dart-lang/sdk/issues/42988'); -} - -final directory = path.join(File.fromUri(Platform.script).parent.path, '..', - '..', '..', '..', 'language_model', 'lexeme'); - -/// Tokenizes the input string. -/// -/// The input is split by word boundaries and trimmed of whitespace. -List tokenize(String input) => - input.split(RegExp(r'\b|\s')).map((t) => t.trim()).toList() - ..removeWhere((t) => t.isEmpty); diff --git a/pkg/analysis_server/test/services/completion/dart/language_model_test.dart b/pkg/analysis_server/test/services/completion/dart/language_model_test.dart deleted file mode 100644 index 7a2434effa7..00000000000 --- a/pkg/analysis_server/test/services/completion/dart/language_model_test.dart +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:ffi'; -import 'dart:io'; - -import 'package:analysis_server/src/services/completion/dart/language_model.dart'; -import 'package:path/path.dart' as path; -import 'package:test/test.dart'; - -void main() { - if (sizeOf() == 4) { - // We don't yet support running tflite on 32-bit systems. - return; - } - - group('LanguageModel', () { - LanguageModel model; - - setUp(() { - model = LanguageModel.load(directory); - }); - - tearDown(() { - model.close(); - }); - - test('calculates lookback', () { - expect(model.lookback, expectedLookback); - }); - - test('predict with defaults', () { - final tokens = - tokenize('if (list == null) { return; } for (final i = 0; i < list.'); - final suggestions = model.predict(tokens); - expect(suggestions.first, 'length'); - }); - - test('predict with confidence scores', () { - final tokens = - tokenize('if (list == null) { return; } for (final i = 0; i < list.'); - final suggestions = model.predictWithScores(tokens); - final best = suggestions.entries.first; - expect(best.key, 'length'); - expect(best.value, greaterThan(0.9)); - }); - - test('predict when no previous tokens', () { - final tokens = []; - final suggestions = model.predict(tokens); - expect(suggestions.first, isNotEmpty); - }); - - test('load fail', () { - try { - LanguageModel.load('doesnotexist'); - fail('Failure to load language model should throw an exception'); - } catch (e) { - expect(e.toString(), - equals('Invalid argument(s): Unable to create model.')); - } - }); - - test('isNumber', () { - expect(model.isNumber('0xCAb005E'), true); - expect(model.isNumber('foo'), false); - expect(model.isNumber('3.1415'), true); - expect(model.isNumber('1337'), true); - expect(model.isNumber('"four score and seven years ago"'), false); - expect(model.isNumber('0.0'), true); - }); - }, skip: 'https://github.com/dart-lang/sdk/issues/42988'); -} - -const expectedLookback = 100; - -final directory = path.join(File.fromUri(Platform.script).parent.path, '..', - '..', '..', '..', 'language_model', 'lexeme'); - -/// Tokenizes the input string. -/// -/// The input is split by word boundaries and trimmed of whitespace. -List tokenize(String input) => - input.split(RegExp(r'\b|\s')).map((t) => t.trim()).toList() - ..removeWhere((t) => t.isEmpty); diff --git a/pkg/analysis_server/test/services/completion/dart/test_all.dart b/pkg/analysis_server/test/services/completion/dart/test_all.dart index 3509cdec61a..bf9d30b2ae2 100644 --- a/pkg/analysis_server/test/services/completion/dart/test_all.dart +++ b/pkg/analysis_server/test/services/completion/dart/test_all.dart @@ -9,15 +9,11 @@ import 'combinator_contributor_test.dart' as combinator_test; import 'completion_manager_test.dart' as completion_manager; import 'completion_ranking_internal_test.dart' as completion_ranking_internal_test; -// ignore: unused_import -import 'completion_ranking_test.dart' as completion_ranking_test; import 'extension_member_contributor_test.dart' as extension_member_contributor; import 'field_formal_contributor_test.dart' as field_formal_contributor_test; import 'imported_reference_contributor_test.dart' as imported_ref_test; import 'keyword_contributor_test.dart' as keyword_test; import 'label_contributor_test.dart' as label_contributor_test; -// ignore: unused_import -import 'language_model_test.dart' as language_model_test; import 'library_member_contributor_test.dart' as library_member_test; import 'library_prefix_contributor_test.dart' as library_prefix_test; import 'local_library_contributor_test.dart' as local_lib_test; @@ -35,18 +31,12 @@ void main() { arglist_test.main(); combinator_test.main(); completion_manager.main(); - // TODO(lambdabaa): Run this test once we figure out how to suppress - // output from the tflite shared library - // completion_ranking_test.main(); completion_ranking_internal_test.main(); extension_member_contributor.main(); field_formal_contributor_test.main(); imported_ref_test.main(); keyword_test.main(); label_contributor_test.main(); - // TODO(brianwilkerson) Run this test when it's been updated to not rely on - // the location of the 'script' being run. - // language_model_test.main(); library_member_test.main(); library_prefix_test.main(); local_lib_test.main(); diff --git a/pkg/analysis_server/test/src/server/sdk_configuration_test.dart b/pkg/analysis_server/test/src/server/sdk_configuration_test.dart index a9613937540..b007d66ee21 100644 --- a/pkg/analysis_server/test/src/server/sdk_configuration_test.dart +++ b/pkg/analysis_server/test/src/server/sdk_configuration_test.dart @@ -43,7 +43,6 @@ void main() { expect(config.analyticsForceEnabled, isNull); expect(config.crashReportingId, isNull); expect(config.crashReportingForceEnabled, isNull); - expect(config.mlModelPath, isNull); }); test('is configured', () { @@ -55,9 +54,7 @@ void main() { "server.analytics.forceEnabled": true, "server.crash.reporting.id": "Test_crash_id", - "server.crash.reporting.forceEnabled": true, - - "server.ml.model.path": "/foo/bar/baz.ml" + "server.crash.reporting.forceEnabled": true } '''); @@ -68,7 +65,6 @@ void main() { expect(config.analyticsForceEnabled, isTrue); expect(config.crashReportingId, 'Test_crash_id'); expect(config.crashReportingForceEnabled, isTrue); - expect(config.mlModelPath, '/foo/bar/baz.ml'); }); }); } diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 39c0c0a8c07..4e616499090 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -315,45 +315,6 @@ if (is_win) { ] } -if (target_cpu == "x64") { - if (is_linux || is_android || is_fuchsia) { - copy_tree_specs += [ - { - target = "copy_libtensorflowlite_c" - visibility = [ ":create_common_sdk" ] - deps = [ ":copy_libraries" ] - source = "../third_party/pkg/tflite_native/lib/src/blobs" - dest = "$root_out_dir/dart-sdk/bin/snapshots" - ignore_patterns = "*.dll,*mac64.so" - }, - ] - } - if (is_mac) { - copy_tree_specs += [ - { - target = "copy_libtensorflowlite_c" - visibility = [ ":create_common_sdk" ] - deps = [ ":copy_libraries" ] - source = "../third_party/pkg/tflite_native/lib/src/blobs" - dest = "$root_out_dir/dart-sdk/bin/snapshots" - ignore_patterns = "*.dll,*linux64.so" - }, - ] - } - if (is_win) { - copy_tree_specs += [ - { - target = "copy_libtensorflowlite_c" - visibility = [ ":create_common_sdk" ] - deps = [ ":copy_libraries" ] - source = "../third_party/pkg/tflite_native/lib/src/blobs" - dest = "$root_out_dir/dart-sdk/bin/snapshots" - ignore_patterns = "*.so" - }, - ] - } -} - # This generates targets for everything in copy_tree_specs. The targets have the # same name as the "target" fields in the scopes of copy_tree_specs. copy_trees("copy_trees") { @@ -918,9 +879,6 @@ group("create_common_sdk") { if (is_win) { public_deps += [ ":copy_7zip" ] } - if (target_cpu == "x64") { - public_deps += [ ":copy_libtensorflowlite_c" ] - } # CIPD only has versions of the Rust compiler for linux and mac x64 hosts. # We also disallow cross-compialtion (it may be possible in future, but it