diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/benchmark_utils.dart similarity index 58% rename from pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart rename to pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/benchmark_utils.dart index a48773a3369..8d0c7149b80 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/benchmark_utils.dart @@ -2,20 +2,20 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:convert'; import 'dart:io'; -import '../language_server_benchmark.dart'; import '../utils.dart'; RunDetails copyData( Uri packageDirUri, Uri outerDirForAdditionalData, int numFiles, - CodeType copyType, + CodeType? copyType, List args, { - required bool includePlugin, + required ({bool includePlugin})? extraInformation, }) { + bool includePlugin = extraInformation?.includePlugin ?? false; + if (copyType == null) throw 'Unsupported.'; Uri filesUri = Platform.script.resolve('files/'); Uri libDirUri = packageDirUri.resolve('lib/'); Directory.fromUri(libDirUri).createSync(); @@ -201,45 +201,10 @@ analyzer: ); } -String formatDuration(Duration duration) { - int seconds = duration.inSeconds; - int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond; - return '$seconds.${ms.toString().padLeft(6, '0')}'; -} - -String formatKb(int kb) { - if (kb > 1024) { - return '${kb ~/ 1024} MB'; - } else { - return '$kb KB'; - } -} - -String getFilenameFor(int i) { - return "file${i.toString().padLeft(5, '0')}.dart"; -} - -Future runHelper( - List args, - DartLanguageServerBenchmark Function( - List args, - Uri rootUri, - Uri cacheFolder, - RunDetails runDetails, - ) - benchmarkCreator, { - required bool runAsLsp, - List numberOfFileOptions = const [16, 32, 64, 128, 256, 512, 1024], - List codeTypes = CodeType.values, - bool includePlugin = false, -}) async { - int verbosity = 0; - bool jsonOutput = false; +List getExtraIterations(List args) { + var codeTypes = CodeType.values; for (String arg in args) { - if (arg.startsWith('--files=')) { - numberOfFileOptions = - arg.substring('--files='.length).split(',').map(int.parse).toList(); - } else if (arg.startsWith('--types=')) { + if (arg.startsWith('--types=')) { codeTypes = []; for (var type in arg.substring('--types='.length).split(',')) { type = type.toLowerCase(); @@ -249,118 +214,13 @@ Future runHelper( } } } - } else if (arg.startsWith('--verbosity=')) { - verbosity = int.parse(arg.substring('--verbosity='.length)); - } else if (arg == '--json') { - jsonOutput = true; } } - if (jsonOutput) { - verbosity = -1; - } - StringBuffer sb = StringBuffer(); - Map jsonData = {}; - for (CodeType codeType in codeTypes) { - for (int numFiles in numberOfFileOptions) { - try { - Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark'); - try { - Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/')) - ..createSync(recursive: true); - Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/')) - ..createSync(recursive: true); - var runDetails = copyData( - dartDir.uri, - tmpDir.uri, - numFiles, - codeType, - args, - includePlugin: includePlugin, - ); - var benchmark = benchmarkCreator( - args, - dartDir.uri, - cacheDir.uri, - runDetails, - ); - try { - benchmark.verbosity = verbosity; - await benchmark.run(); - } finally { - benchmark.exit(); - } + return codeTypes; +} - var caption = '$numFiles files / $codeType'; - if (jsonOutput) { - for (var durationInfo in benchmark.durationInfo) { - var key = '$caption: ${durationInfo.name} (ms)'; - if (jsonData.containsKey(key)) { - throw 'Already contains data for $key'; - } - jsonData[key] = durationInfo.duration.inMilliseconds; - } - for (var memoryInfo in benchmark.memoryInfo) { - jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb; - } - } else { - if (verbosity >= 0) print('===================='); - if (verbosity >= 0) print('$caption:'); - sb.writeln('$caption:'); - for (var durationInfo in benchmark.durationInfo) { - if (verbosity >= 0) { - print( - '${durationInfo.name}: ' - '${formatDuration(durationInfo.duration)}', - ); - } - sb.writeln( - '${durationInfo.name}: ' - '${formatDuration(durationInfo.duration)}', - ); - } - for (var memoryInfo in benchmark.memoryInfo) { - if (verbosity >= 0) { - print( - '${memoryInfo.name}: ' - '${formatKb(memoryInfo.kb)}', - ); - } - sb.writeln( - '${memoryInfo.name}: ' - '${formatKb(memoryInfo.kb)}', - ); - } - if (verbosity >= 0) print('===================='); - sb.writeln(); - } - } finally { - try { - tmpDir.deleteSync(recursive: true); - } catch (e) { - // Wait a little and retry. - sleep(const Duration(milliseconds: 42)); - try { - tmpDir.deleteSync(recursive: true); - } catch (e) { - if (verbosity >= 0) print('Warning: $e'); - } - } - } - } catch (e) { - stderr.writeln( - 'Error while processing $numFiles files / $codeType: $e', - ); - } - } - } - - if (jsonOutput) { - print(json.encode(jsonData)); - } else { - print('=================================='); - print(sb.toString().trim()); - print('=================================='); - } +String getFilenameFor(int i) { + return "file${i.toString().padLeft(5, '0')}.dart"; } enum CodeType { diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart index 2e6e7d63e5f..bf373e15a91 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../legacy_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// At least until "https://github.com/flutter/flutter-intellij/issues/7980" is /// fixed, when a user in IntelliJ in a Flutter project opens a file it's added @@ -26,6 +27,8 @@ Future main(List args) async { await runHelper( args, LegacyManyFilesInFlutterSetSubscriptionsBenchmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: false, ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart index cea4086db76..d907c4acbcf 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../legacy_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// In reports of the analyzer being slow we've seen `edit.getFixes` causing a /// long queue because they take longer to execute than the wait before the next @@ -19,9 +20,11 @@ Future main(List args) async { await runHelper( args, LegacyManyGetFixesAndGetAssisstRequestsBenchmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: false, // The number of files doesn't seem to be important on this one. - numberOfFileOptions: [4], + sizeOptions: [4], ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart index d60d5ab3bf7..24b6b7230f6 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../legacy_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// Hovering over an import with ctrl down in IntelliJ sends getHover requests /// for the import uri at position 0 every ~8 ms and never cancels old requests. @@ -15,9 +16,11 @@ Future main(List args) async { await runHelper( args, LegacyManyHoverRequestsBenchmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: false, // The number of files doesn't seem to be important on this one. - numberOfFileOptions: [4], + sizeOptions: [4], ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart index 441d98d1c05..182cfdf781e 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../legacy_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// In IntelliJ, typing `if (something) {` only automatically inserts the /// matching end brace `}` when hitting enter. This makes it "not unlikely" @@ -20,6 +21,8 @@ Future main(List args) async { await runHelper( args, LegacyTypingTemporaryMissingEndBraceBenchmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: false, ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart index 478ae015333..4af5e0dd17b 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../legacy_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// We've observed that sometimes the plugin that users has installed times out /// (takes > 500 ms to answer). This benchmark simulates that and shows the @@ -13,10 +14,12 @@ Future main(List args) async { await runHelper( args, LegacyWithPluginThatTimesOutBencmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: false, - includePlugin: true, + extraInformation: (includePlugin: true), // The number of files isn't important here. - numberOfFileOptions: [10], + sizeOptions: [10], ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart index 457daa6a79e..2d7b1a56947 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart @@ -4,13 +4,20 @@ import '../language_server_benchmark.dart'; import '../lsp_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// Change a file in a big project and reports how long it takes before the /// analysis server is responsive again (measured by when it responds to a /// completion request) and when it's actually done analysing. Future main(List args) async { - await runHelper(args, LspCompletionAfterChange.new, runAsLsp: true); + await runHelper( + args, + LspCompletionAfterChange.new, + copyData, + extraIterations: getExtraIterations, + runAsLsp: true, + ); } class LspCompletionAfterChange extends DartLanguageServerBenchmark { diff --git a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart index d1721802b04..28bbf51a07e 100644 --- a/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart +++ b/pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart @@ -4,7 +4,8 @@ import '../language_server_benchmark.dart'; import '../lsp_messages.dart'; -import 'utils.dart'; +import '../run_utils.dart'; +import 'benchmark_utils.dart'; /// We've observed that sometimes the plugin that users has installed times out /// (takes > 500 ms to answer). This benchmark simulates that and shows the @@ -13,10 +14,12 @@ Future main(List args) async { await runHelper( args, LspWithPluginThatTimesOutBencmark.new, + copyData, + extraIterations: getExtraIterations, runAsLsp: true, - includePlugin: true, + extraInformation: (includePlugin: true), // The number of files isn't important here. - numberOfFileOptions: [10], + sizeOptions: [10], ); } diff --git a/pkg/analysis_server/tool/benchmark_tools/run_utils.dart b/pkg/analysis_server/tool/benchmark_tools/run_utils.dart new file mode 100644 index 00000000000..6e85958009d --- /dev/null +++ b/pkg/analysis_server/tool/benchmark_tools/run_utils.dart @@ -0,0 +1,166 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'language_server_benchmark.dart'; + +String formatDuration(Duration duration) { + int seconds = duration.inSeconds; + int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond; + return '$seconds.${ms.toString().padLeft(6, '0')}'; +} + +String formatKb(int kb) { + if (kb > 1024) { + return '${kb ~/ 1024} MB'; + } else { + return '$kb KB'; + } +} + +Future runHelper( + List args, + DartLanguageServerBenchmark Function( + List args, + Uri rootUri, + Uri cacheFolder, + E runDetails, + ) + benchmarkCreator, + E Function( + Uri packageDirUri, + Uri outerDirForAdditionalData, + int size, + F? extraIterationData, + List args, { + required G? extraInformation, + }) + createDataAndCreateRunDetails, { + required bool runAsLsp, + List sizeOptions = const [16, 32, 64, 128, 256, 512, 1024], + required List Function(List args) extraIterations, + G? extraInformation, +}) async { + int verbosity = 0; + bool jsonOutput = false; + for (String arg in args) { + if (arg.startsWith('--sizes=')) { + sizeOptions = + arg.substring('--sizes='.length).split(',').map(int.parse).toList(); + } else if (arg.startsWith('--verbosity=')) { + verbosity = int.parse(arg.substring('--verbosity='.length)); + } else if (arg == '--json') { + jsonOutput = true; + } + } + if (jsonOutput) { + verbosity = -1; + } + StringBuffer sb = StringBuffer(); + Map jsonData = {}; + for (F? extraIteration in extraIterations(args)) { + for (int size in sizeOptions) { + var caption = 'size $size / $extraIteration'; + if (extraIteration == null) { + caption = 'size $size'; + } + try { + Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark'); + try { + Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/')) + ..createSync(recursive: true); + Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/')) + ..createSync(recursive: true); + var runDetails = createDataAndCreateRunDetails( + dartDir.uri, + tmpDir.uri, + size, + extraIteration, + args, + extraInformation: extraInformation, + ); + var benchmark = benchmarkCreator( + args, + dartDir.uri, + cacheDir.uri, + runDetails, + ); + try { + benchmark.verbosity = verbosity; + await benchmark.run(); + } finally { + benchmark.exit(); + } + + if (jsonOutput) { + for (var durationInfo in benchmark.durationInfo) { + var key = '$caption: ${durationInfo.name} (ms)'; + if (jsonData.containsKey(key)) { + throw 'Already contains data for $key'; + } + jsonData[key] = durationInfo.duration.inMilliseconds; + } + for (var memoryInfo in benchmark.memoryInfo) { + jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb; + } + } else { + if (verbosity >= 0) print('===================='); + if (verbosity >= 0) print('$caption:'); + sb.writeln('$caption:'); + for (var durationInfo in benchmark.durationInfo) { + if (verbosity >= 0) { + print( + '${durationInfo.name}: ' + '${formatDuration(durationInfo.duration)}', + ); + } + sb.writeln( + '${durationInfo.name}: ' + '${formatDuration(durationInfo.duration)}', + ); + } + for (var memoryInfo in benchmark.memoryInfo) { + if (verbosity >= 0) { + print( + '${memoryInfo.name}: ' + '${formatKb(memoryInfo.kb)}', + ); + } + sb.writeln( + '${memoryInfo.name}: ' + '${formatKb(memoryInfo.kb)}', + ); + } + if (verbosity >= 0) print('===================='); + sb.writeln(); + } + } finally { + try { + tmpDir.deleteSync(recursive: true); + } catch (e) { + // Wait a little and retry. + sleep(const Duration(milliseconds: 42)); + try { + tmpDir.deleteSync(recursive: true); + } catch (e) { + if (verbosity >= 0) print('Warning: $e'); + } + } + } + } catch (e) { + stderr.writeln('Error while processing $caption: $e'); + } + } + } + + if (jsonOutput) { + print(json.encode(jsonData)); + } else { + print('=================================='); + print(sb.toString().trim()); + print('=================================='); + } +}