// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer/file_system/physical_file_system.dart'; import 'package:args/command_runner.dart'; import 'package:intl/intl.dart'; import 'package:path/path.dart' as path; import 'perf/benchmarks_impl.dart'; import 'perf/flutter_analyze_benchmark.dart'; Future main(List args) async { var benchmarks = [ ColdAnalysisBenchmark(ServerBenchmark.das), ColdAnalysisBenchmark(ServerBenchmark.lsp), AnalysisBenchmark(ServerBenchmark.das), AnalysisBenchmark(ServerBenchmark.lsp), FlutterAnalyzeBenchmark(), ]; var runner = CommandRunner('benchmark', 'A benchmark runner for the analysis server.'); runner.addCommand(ListCommand(benchmarks)); runner.addCommand(RunCommand(benchmarks)); runner.run(args); } String get analysisServerSrcPath { var script = Platform.script.toFilePath(windows: Platform.isWindows); var pkgPath = path.normalize(path.join(path.dirname(script), '..', '..')); return path.join(pkgPath, 'analysis_server'); } void deleteServerCache() { // ~/.dartServer/.analysis-driver/ ResourceProvider resourceProvider = PhysicalResourceProvider.INSTANCE; var stateLocation = resourceProvider.getStateLocation('.analysis-driver'); try { if (stateLocation != null && stateLocation.exists) { stateLocation.delete(); } } catch (e) { // ignore any exception } } List getProjectRoots({bool quick = false}) { var script = Platform.script.toFilePath(windows: Platform.isWindows); var pkgPath = path.normalize(path.join(path.dirname(script), '..', '..')); return [path.join(pkgPath, quick ? 'meta' : 'analysis_server')]; } abstract class Benchmark { final String id; final String description; final bool enabled; /// One of 'memory', 'cpu', or 'group'. final String kind; Benchmark(this.id, this.description, {this.enabled = true, this.kind = 'cpu'}); int get maxIterations => 0; bool get needsSetup => false; Future oneTimeCleanup() => Future.value(); Future oneTimeSetup() => Future.value(); Future run({ bool quick = false, bool verbose = false, }); Map toJson() => {'id': id, 'description': description, 'enabled': enabled, 'kind': kind}; @override String toString() => '$id: $description'; } class BenchMarkResult { static final NumberFormat nf = NumberFormat.decimalPattern(); /// One of 'bytes', 'micros', or 'compound'. final String kindName; final int value; BenchMarkResult(this.kindName, this.value); BenchMarkResult combine(BenchMarkResult other) { return BenchMarkResult(kindName, math.min(value, other.value)); } Map toJson() => {kindName: value}; @override String toString() => '$kindName: ${nf.format(value)}'; } class CompoundBenchMarkResult extends BenchMarkResult { final String name; Map results = {}; CompoundBenchMarkResult(this.name) : super('compound', 0); void add(String name, BenchMarkResult result) { results[name] = result; } @override BenchMarkResult combine(BenchMarkResult other) { BenchMarkResult _combine(BenchMarkResult? a, BenchMarkResult? b) { if (a == null) return b!; if (b == null) return a; return a.combine(b); } var o = other as CompoundBenchMarkResult; var combined = CompoundBenchMarkResult(name); var keys = ({}..addAll(results.keys)..addAll(o.results.keys)).toList(); for (var key in keys) { combined.add(key, _combine(results[key], o.results[key])); } return combined; } @override Map toJson() { var m = {}; for (var entry in results.entries) { m['$name-${entry.key}'] = entry.value.toJson(); } return m; } @override String toString() => '${toJson()}'; } class ListCommand extends Command { final List benchmarks; ListCommand(this.benchmarks) { argParser.addFlag('machine', negatable: false, help: 'Emit the list of benchmarks as json.'); } @override String get description => 'List available benchmarks.'; @override String get invocation => '${runner!.executableName} $name'; @override String get name => 'list'; @override void run() { if (argResults!['machine'] as bool) { var map = { 'benchmarks': benchmarks.map((b) => b.toJson()).toList() }; print(JsonEncoder.withIndent(' ').convert(map)); } else { for (var benchmark in benchmarks) { print('${benchmark.id}: ${benchmark.description}'); } } } } class RunCommand extends Command { final List benchmarks; RunCommand(this.benchmarks) { argParser.addFlag('quick', negatable: false, help: 'Run a quick version of the benchmark. This is not useful for ' 'gathering accurate times,\nbut can be used to validate that the ' 'benchmark works.'); argParser.addOption('repeat', defaultsTo: '4', help: 'The number of times to repeat the benchmark.'); argParser.addFlag('verbose', negatable: false, help: 'Print all communication to and from the analysis server.'); } @override String get description => 'Run a given benchmark.'; @override String get invocation => '${runner!.executableName} $name '; @override String get name => 'run'; @override Future run() async { if (argResults!.rest.isEmpty) { printUsage(); exit(1); } var benchmarkId = argResults!.rest.first; var repeatCount = int.parse(argResults!['repeat'] as String); var quick = argResults!['quick']; var verbose = argResults!['verbose']; var benchmark = benchmarks.firstWhere((b) => b.id == benchmarkId, orElse: () { print("Benchmark '$benchmarkId' not found."); exit(1); }); var actualIterations = repeatCount; if (benchmark.maxIterations > 0) { actualIterations = math.min(benchmark.maxIterations, repeatCount); } if (benchmark.needsSetup) { print('Setting up $benchmarkId...'); await benchmark.oneTimeSetup(); } try { BenchMarkResult? result; var time = Stopwatch()..start(); print('Running $benchmarkId $actualIterations times...'); for (var iteration = 0; iteration < actualIterations; iteration++) { var newResult = await benchmark.run( quick: quick, verbose: verbose, ); print(' $newResult'); result = result == null ? newResult : result.combine(newResult); } time.stop(); print('Finished in ${time.elapsed.inSeconds} seconds.\n'); var m = { 'benchmark': benchmarkId, 'result': result!.toJson() }; print(json.encode(m)); await benchmark.oneTimeCleanup(); } catch (error, st) { print('$benchmarkId threw exception: $error'); print(st); exit(1); } } }