dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[analyzer] Refactor benchmarks to make it easier to add new benchmarks based on different sources

Browse Source 
Change-Id: I7e6f70c62faad55619de6677100608f406091a5e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/425502
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
Jens Johansen
2025-04-30 23:27:59 -07:00
committed by Commit Queue
parent f38d092f3a
commit 7d82418140
9 changed files with 216 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/utils.dart → pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/benchmark_utils.dart
+11 -151
View File
@@ -2,20 +2,20 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import '../language_server_benchmark.dart';
import '../utils.dart';
RunDetails copyData(
Uri packageDirUri,
Uri outerDirForAdditionalData,
int numFiles,
CodeType copyType,
CodeType? copyType,
List<String> args, {
required bool includePlugin,
required ({bool includePlugin})? extraInformation,
}) {
bool includePlugin = extraInformation?.includePlugin ?? false;
if (copyType == null) throw 'Unsupported.';
Uri filesUri = Platform.script.resolve('files/');
Uri libDirUri = packageDirUri.resolve('lib/');
Directory.fromUri(libDirUri).createSync();
@@ -201,45 +201,10 @@ analyzer:
);
}
String formatDuration(Duration duration) {
int seconds = duration.inSeconds;
int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond;
return '$seconds.${ms.toString().padLeft(6, '0')}';
}
String formatKb(int kb) {
if (kb > 1024) {
return '${kb ~/ 1024} MB';
} else {
return '$kb KB';
}
}
String getFilenameFor(int i) {
return "file${i.toString().padLeft(5, '0')}.dart";
}
Future<void> runHelper(
List<String> args,
DartLanguageServerBenchmark Function(
List<String> args,
Uri rootUri,
Uri cacheFolder,
RunDetails runDetails,
)
benchmarkCreator, {
required bool runAsLsp,
List<int> numberOfFileOptions = const [16, 32, 64, 128, 256, 512, 1024],
List<CodeType> codeTypes = CodeType.values,
bool includePlugin = false,
}) async {
int verbosity = 0;
bool jsonOutput = false;
List<CodeType?> getExtraIterations(List<String> args) {
var codeTypes = CodeType.values;
for (String arg in args) {
if (arg.startsWith('--files=')) {
numberOfFileOptions =
arg.substring('--files='.length).split(',').map(int.parse).toList();
} else if (arg.startsWith('--types=')) {
if (arg.startsWith('--types=')) {
codeTypes = [];
for (var type in arg.substring('--types='.length).split(',')) {
type = type.toLowerCase();
@@ -249,118 +214,13 @@ Future<void> runHelper(
}
}
}
} else if (arg.startsWith('--verbosity=')) {
verbosity = int.parse(arg.substring('--verbosity='.length));
} else if (arg == '--json') {
jsonOutput = true;
}
}
if (jsonOutput) {
verbosity = -1;
}
StringBuffer sb = StringBuffer();
Map<String, int> jsonData = {};
for (CodeType codeType in codeTypes) {
for (int numFiles in numberOfFileOptions) {
try {
Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
try {
Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
..createSync(recursive: true);
Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
..createSync(recursive: true);
var runDetails = copyData(
dartDir.uri,
tmpDir.uri,
numFiles,
codeType,
args,
includePlugin: includePlugin,
);
var benchmark = benchmarkCreator(
args,
dartDir.uri,
cacheDir.uri,
runDetails,
);
try {
benchmark.verbosity = verbosity;
await benchmark.run();
} finally {
benchmark.exit();
}
return codeTypes;
}
var caption = '$numFiles files / $codeType';
if (jsonOutput) {
for (var durationInfo in benchmark.durationInfo) {
var key = '$caption: ${durationInfo.name} (ms)';
if (jsonData.containsKey(key)) {
throw 'Already contains data for $key';
}
jsonData[key] = durationInfo.duration.inMilliseconds;
}
for (var memoryInfo in benchmark.memoryInfo) {
jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb;
}
} else {
if (verbosity >= 0) print('====================');
if (verbosity >= 0) print('$caption:');
sb.writeln('$caption:');
for (var durationInfo in benchmark.durationInfo) {
if (verbosity >= 0) {
print(
'${durationInfo.name}: '
'${formatDuration(durationInfo.duration)}',
);
}
sb.writeln(
'${durationInfo.name}: '
'${formatDuration(durationInfo.duration)}',
);
}
for (var memoryInfo in benchmark.memoryInfo) {
if (verbosity >= 0) {
print(
'${memoryInfo.name}: '
'${formatKb(memoryInfo.kb)}',
);
}
sb.writeln(
'${memoryInfo.name}: '
'${formatKb(memoryInfo.kb)}',
);
}
if (verbosity >= 0) print('====================');
sb.writeln();
}
} finally {
try {
tmpDir.deleteSync(recursive: true);
} catch (e) {
// Wait a little and retry.
sleep(const Duration(milliseconds: 42));
try {
tmpDir.deleteSync(recursive: true);
} catch (e) {
if (verbosity >= 0) print('Warning: $e');
}
}
}
} catch (e) {
stderr.writeln(
'Error while processing $numFiles files / $codeType: $e',
);
}
}
}
if (jsonOutput) {
print(json.encode(jsonData));
} else {
print('==================================');
print(sb.toString().trim());
print('==================================');
}
String getFilenameFor(int i) {
return "file${i.toString().padLeft(5, '0')}.dart";
}
enum CodeType {
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_files_in_flutter_set_subscriptions.dart
+4 -1
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../legacy_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// At least until "https://github.com/flutter/flutter-intellij/issues/7980" is
/// fixed, when a user in IntelliJ in a Flutter project opens a file it's added
@@ -26,6 +27,8 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LegacyManyFilesInFlutterSetSubscriptionsBenchmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: false,
);
}
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_get_fixes_and_get_assists_requests.dart
+5 -2
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../legacy_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// In reports of the analyzer being slow we've seen `edit.getFixes` causing a
/// long queue because they take longer to execute than the wait before the next
@@ -19,9 +20,11 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LegacyManyGetFixesAndGetAssisstRequestsBenchmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: false,
// The number of files doesn't seem to be important on this one.
numberOfFileOptions: [4],
sizeOptions: [4],
);
}
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_many_hover_requests.dart
+5 -2
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../legacy_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// Hovering over an import with ctrl down in IntelliJ sends getHover requests
/// for the import uri at position 0 every ~8 ms and never cancels old requests.
@@ -15,9 +16,11 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LegacyManyHoverRequestsBenchmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: false,
// The number of files doesn't seem to be important on this one.
numberOfFileOptions: [4],
sizeOptions: [4],
);
}
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_typing_temporary_missing_end_brace.dart
+4 -1
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../legacy_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// In IntelliJ, typing `if (something) {` only automatically inserts the
/// matching end brace `}` when hitting enter. This makes it "not unlikely"
@@ -20,6 +21,8 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LegacyTypingTemporaryMissingEndBraceBenchmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: false,
);
}
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/legacy_with_plugin_that_times_out.dart
+6 -3
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../legacy_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// We've observed that sometimes the plugin that users has installed times out
/// (takes > 500 ms to answer). This benchmark simulates that and shows the
@@ -13,10 +14,12 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LegacyWithPluginThatTimesOutBencmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: false,
includePlugin: true,
extraInformation: (includePlugin: true),
// The number of files isn't important here.
numberOfFileOptions: [10],
sizeOptions: [10],
);
}
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_completion_after_change.dart
+9 -2
View File
@@ -4,13 +4,20 @@
import '../language_server_benchmark.dart';
import '../lsp_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// Change a file in a big project and reports how long it takes before the
/// analysis server is responsive again (measured by when it responds to a
/// completion request) and when it's actually done analysing.
Future<void> main(List<String> args) async {
await runHelper(args, LspCompletionAfterChange.new, runAsLsp: true);
await runHelper(
args,
LspCompletionAfterChange.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: true,
);
}
class LspCompletionAfterChange extends DartLanguageServerBenchmark {
pkg/analysis_server/tool/benchmark_tools/big_chain_benchmark/lsp_with_plugin_that_times_out.dart
+6 -3
View File
@@ -4,7 +4,8 @@
import '../language_server_benchmark.dart';
import '../lsp_messages.dart';
import 'utils.dart';
import '../run_utils.dart';
import 'benchmark_utils.dart';
/// We've observed that sometimes the plugin that users has installed times out
/// (takes > 500 ms to answer). This benchmark simulates that and shows the
@@ -13,10 +14,12 @@ Future<void> main(List<String> args) async {
await runHelper(
args,
LspWithPluginThatTimesOutBencmark.new,
copyData,
extraIterations: getExtraIterations,
runAsLsp: true,
includePlugin: true,
extraInformation: (includePlugin: true),
// The number of files isn't important here.
numberOfFileOptions: [10],
sizeOptions: [10],
);
}
pkg/analysis_server/tool/benchmark_tools/run_utils.dart
+166
View File
@@ -0,0 +1,166 @@
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'language_server_benchmark.dart';
String formatDuration(Duration duration) {
int seconds = duration.inSeconds;
int ms = duration.inMicroseconds - seconds * Duration.microsecondsPerSecond;
return '$seconds.${ms.toString().padLeft(6, '0')}';
}
String formatKb(int kb) {
if (kb > 1024) {
return '${kb ~/ 1024} MB';
} else {
return '$kb KB';
}
}
Future<void> runHelper<E, F, G>(
List<String> args,
DartLanguageServerBenchmark Function(
List<String> args,
Uri rootUri,
Uri cacheFolder,
E runDetails,
)
benchmarkCreator,
E Function(
Uri packageDirUri,
Uri outerDirForAdditionalData,
int size,
F? extraIterationData,
List<String> args, {
required G? extraInformation,
})
createDataAndCreateRunDetails, {
required bool runAsLsp,
List<int> sizeOptions = const [16, 32, 64, 128, 256, 512, 1024],
required List<F?> Function(List<String> args) extraIterations,
G? extraInformation,
}) async {
int verbosity = 0;
bool jsonOutput = false;
for (String arg in args) {
if (arg.startsWith('--sizes=')) {
sizeOptions =
arg.substring('--sizes='.length).split(',').map(int.parse).toList();
} else if (arg.startsWith('--verbosity=')) {
verbosity = int.parse(arg.substring('--verbosity='.length));
} else if (arg == '--json') {
jsonOutput = true;
}
}
if (jsonOutput) {
verbosity = -1;
}
StringBuffer sb = StringBuffer();
Map<String, int> jsonData = {};
for (F? extraIteration in extraIterations(args)) {
for (int size in sizeOptions) {
var caption = 'size $size / $extraIteration';
if (extraIteration == null) {
caption = 'size $size';
}
try {
Directory tmpDir = Directory.systemTemp.createTempSync('lsp_benchmark');
try {
Directory cacheDir = Directory.fromUri(tmpDir.uri.resolve('cache/'))
..createSync(recursive: true);
Directory dartDir = Directory.fromUri(tmpDir.uri.resolve('dart/'))
..createSync(recursive: true);
var runDetails = createDataAndCreateRunDetails(
dartDir.uri,
tmpDir.uri,
size,
extraIteration,
args,
extraInformation: extraInformation,
);
var benchmark = benchmarkCreator(
args,
dartDir.uri,
cacheDir.uri,
runDetails,
);
try {
benchmark.verbosity = verbosity;
await benchmark.run();
} finally {
benchmark.exit();
}
if (jsonOutput) {
for (var durationInfo in benchmark.durationInfo) {
var key = '$caption: ${durationInfo.name} (ms)';
if (jsonData.containsKey(key)) {
throw 'Already contains data for $key';
}
jsonData[key] = durationInfo.duration.inMilliseconds;
}
for (var memoryInfo in benchmark.memoryInfo) {
jsonData['$caption: ${memoryInfo.name} (kb)'] = memoryInfo.kb;
}
} else {
if (verbosity >= 0) print('====================');
if (verbosity >= 0) print('$caption:');
sb.writeln('$caption:');
for (var durationInfo in benchmark.durationInfo) {
if (verbosity >= 0) {
print(
'${durationInfo.name}: '
'${formatDuration(durationInfo.duration)}',
);
}
sb.writeln(
'${durationInfo.name}: '
'${formatDuration(durationInfo.duration)}',
);
}
for (var memoryInfo in benchmark.memoryInfo) {
if (verbosity >= 0) {
print(
'${memoryInfo.name}: '
'${formatKb(memoryInfo.kb)}',
);
}
sb.writeln(
'${memoryInfo.name}: '
'${formatKb(memoryInfo.kb)}',
);
}
if (verbosity >= 0) print('====================');
sb.writeln();
}
} finally {
try {
tmpDir.deleteSync(recursive: true);
} catch (e) {
// Wait a little and retry.
sleep(const Duration(milliseconds: 42));
try {
tmpDir.deleteSync(recursive: true);
} catch (e) {
if (verbosity >= 0) print('Warning: $e');
}
}
}
} catch (e) {
stderr.writeln('Error while processing $caption: $e');
}
}
}
if (jsonOutput) {
print(json.encode(jsonData));
} else {
print('==================================');
print(sb.toString().trim());
print('==================================');
}
}