// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. // This files contains methods for benchmarking Kernel binary serialization // and deserialization routines. import 'package:kernel/ast.dart'; import 'package:kernel/binary/ast_from_binary.dart'; import 'package:kernel/binary/ast_to_binary.dart'; import 'dart:io'; import 'dart:math'; import 'dart:typed_data'; import 'package:kernel/src/printer.dart'; final String usage = ''' Usage: binary_bench2.dart [--golem|--raw] {--metadata|--onlyCold} Benchmark can be one of: ${benchmarks.keys.join(', ')} '''; typedef void Benchmark(Uint8List bytes); final Map benchmarks = { 'AstFromBinaryEager': (Uint8List bytes) { return _benchmarkAstFromBinary(bytes, eager: true); }, 'AstFromBinaryLazy': (Uint8List bytes) { return _benchmarkAstFromBinary(bytes, eager: false); }, 'AstToBinary': (Uint8List bytes) { return _benchmarkAstToBinary(bytes); }, }; Benchmark? benchmark; late File sourceDill; bool forGolem = false; bool forRaw = false; bool metadataAware = false; bool onlyCold = false; void main(List args) async { if (!_parseArgs(args)) { print(usage); exit(-1); } final Uint8List bytes = sourceDill.readAsBytesSync(); benchmark!(bytes); } int warmupIterations = 100; int benchmarkIterations = 50; void _benchmarkAstFromBinary(Uint8List bytes, {bool eager = true}) { final String nameSuffix = eager ? 'Eager' : 'Lazy'; final Stopwatch sw = new Stopwatch()..start(); _fromBinary(bytes, eager: eager); final int coldRunUs = sw.elapsedMicroseconds; sw.reset(); if (onlyCold) { new BenchmarkResult('AstFromBinary${nameSuffix}', coldRunUs, coldRunUs.toDouble(), [coldRunUs]).report(); return; } for (int i = 0; i < warmupIterations; i++) { _fromBinary(bytes, eager: eager); } final double warmupUs = sw.elapsedMicroseconds / warmupIterations; final List runsUs = new List.filled(benchmarkIterations, /* dummy value = */ 0); for (int i = 0; i < benchmarkIterations; i++) { sw.reset(); _fromBinary(bytes, eager: eager, verbose: i == benchmarkIterations - 1); runsUs[i] = sw.elapsedMicroseconds; } new BenchmarkResult('AstFromBinary${nameSuffix}', coldRunUs, warmupUs, runsUs) .report(); } void _benchmarkAstToBinary(Uint8List bytes) { final Component p = _fromBinary(bytes, eager: true); final Stopwatch sw = new Stopwatch()..start(); _toBinary(p); final int coldRunUs = sw.elapsedMicroseconds; sw.reset(); for (int i = 0; i < warmupIterations; i++) { _toBinary(p); } final double warmupUs = sw.elapsedMicroseconds / warmupIterations; final List runsUs = new List.filled(benchmarkIterations, /* dummy value = */ 0); for (int i = 0; i < benchmarkIterations; i++) { sw.reset(); _toBinary(p); runsUs[i] = sw.elapsedMicroseconds; } new BenchmarkResult('AstToBinary', coldRunUs, warmupUs, runsUs).report(); } class BenchmarkResult { final String name; final int coldRunUs; final double warmupUs; final List runsUs; BenchmarkResult(this.name, this.coldRunUs, this.warmupUs, this.runsUs); static T add(T x, T y) => x + y as T; void report() { runsUs.sort(); int P(int p) => runsUs[((runsUs.length - 1) * (p / 100)).ceil()]; final int sum = runsUs.reduce(add); final double avg = sum / runsUs.length; final int min = runsUs.first; final int max = runsUs.last; final double std = sqrt(runsUs.map((v) => pow(v - avg, 2)).reduce(add) / runsUs.length); if (forGolem) { print('${name}(RunTimeRaw): ${avg} us.'); print('${name}P50(RunTimeRaw): ${P(50)} us.'); print('${name}P90(RunTimeRaw): ${P(90)} us.'); } else if (forRaw) { runsUs.forEach(print); } else { print('${name}Cold: ${coldRunUs} us'); print('${name}Warmup: ${warmupUs} us'); print('${name}: ${avg} us.'); final String prefix = '-' * name.length; print('${prefix}> Range: ${min}...${max} us.'); print('${prefix}> Std Dev: ${std.toStringAsFixed(2)}'); print('${prefix}> 50th percentile: ${P(50)} us.'); print('${prefix}> 90th percentile: ${P(90)} us.'); } } } bool _parseArgs(List argsOrg) { List trimmedArgs = []; for (String arg in argsOrg) { if (arg == "--golem") { forGolem = true; } else if (arg == "--raw") { forRaw = true; } else if (arg == "--metadata") { metadataAware = true; } else if (arg == "--onlyCold") { onlyCold = true; } else if (arg.startsWith("--warmups=")) { warmupIterations = int.parse(arg.substring("--warmups=".length)); } else if (arg.startsWith("--iterations=")) { benchmarkIterations = int.parse(arg.substring("--iterations=".length)); } else { trimmedArgs.add(arg); } } if (trimmedArgs.length != 2) { return false; } if (forGolem && forRaw) { return false; } benchmark = benchmarks[trimmedArgs[0]]; if (benchmark == null) { return false; } sourceDill = new File(trimmedArgs[1]); if (!sourceDill.existsSync()) { return false; } return true; } Component _fromBinary(List bytes, {required bool eager, bool verbose = false}) { Component component = new Component(); if (metadataAware) { // This is currently (October 2024) what VmTarget.configureComponent does. component.metadata.putIfAbsent( CallSiteAttributesMetadataRepository.repositoryTag, () => new CallSiteAttributesMetadataRepository()); BinaryBuilderWithMetadata builder = new BinaryBuilderWithMetadata(bytes, filename: 'filename', disableLazyReading: eager); builder.readComponent(component); if (verbose) { // No current verbose output. } } else { new BinaryBuilder(bytes, filename: 'filename', disableLazyReading: eager) .readComponent(component); } return component; } class SimpleSink implements Sink> { final List> chunks = >[]; @override void add(List chunk) { chunks.add(chunk); } @override void close() {} } void _toBinary(Component p) { new BinaryPrinter(new SimpleSink()).writeComponentFile(p); } // The below is copied from package:vm so to test metadata properly without // depending on package:vm. /// Metadata for annotating call sites with various attributes. class CallSiteAttributesMetadata { final DartType receiverType; const CallSiteAttributesMetadata({required this.receiverType}); @override String toString() => "receiverType:${receiverType.toText(astTextStrategyForTesting)}"; } /// Repository for [CallSiteAttributesMetadata]. class CallSiteAttributesMetadataRepository extends MetadataRepository { static final repositoryTag = 'vm.call-site-attributes.metadata'; @override final String tag = repositoryTag; @override final Map mapping = {}; @override void writeToBinary( CallSiteAttributesMetadata metadata, Node node, BinarySink sink) { sink.writeDartType(metadata.receiverType); } @override CallSiteAttributesMetadata readFromBinary(Node node, BinarySource source) { final type = source.readDartType(); return new CallSiteAttributesMetadata(receiverType: type); } }