// @dart = 2.9 import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; import 'package:front_end/src/fasta/kernel/utils.dart'; import 'package:kernel/kernel.dart'; import 'package:kernel/binary/ast_from_binary.dart'; import "simple_stats.dart"; /// This pass all parameters after "--" to the compiler (via `compileEntryPoint` /// via the helper `compile_benchmark_helper.dart`). It is meant to "benchmark"/ /// instrument the compiler to give insights into specific runs, i.e. specific /// compiles (compiling program `a` and program `b` might have different /// characteristics). Said another way, it will always instrument and give /// insights on *the compiler*, not the program it's asked to compile (not /// directly anyway -- if you want to know if that program successfully /// exercises a specific part of the compiler it _does_ do that). final Uri benchmarkHelper = Platform.script.resolve("compile_benchmark_helper.dart"); main(List args) { List arguments; bool tryToAnnotate = false; bool tryToSlowDown = false; bool timeInsteadOfCount = false; for (int i = 0; i < args.length; i++) { if (args[i] == "--tryToAnnotate") { tryToAnnotate = true; } else if (args[i] == "--tryToSlowDown") { tryToSlowDown = true; } else if (args[i] == "--timeInsteadOfCount") { timeInsteadOfCount = true; } else if (args[i] == "--") { arguments = args.sublist(i + 1); break; } else { throw "Unknown argument '${args[i]}'"; } } if (arguments == null || arguments.isEmpty) { throw "No arguments given to compiler.\n" "Give arguments as `dart compile_benchmark.dart -- " "argument1 argument2 (etc)`"; } Directory tmp = Directory.systemTemp.createTempSync("benchmark"); try { // Compile the helper to get a dill we can compile with. final Uri helperDill = tmp.uri.resolve("compile.dill"); print("Compiling $benchmarkHelper into $helperDill"); runXTimes(1, [ benchmarkHelper.toString(), benchmarkHelper.toString(), "-o", helperDill.toString(), ]); File f = new File.fromUri(helperDill); if (!f.existsSync()) throw "$f doesn't exist!"; List dillData = new File.fromUri(helperDill).readAsBytesSync(); doWork( tmp, dillData, arguments, tryToAnnotate: tryToAnnotate, tryToSlowDown: tryToSlowDown, timeInsteadOfCount: timeInsteadOfCount, ); } finally { tmp.deleteSync(recursive: true); } } /// Perform asked operations on the dill data provided. /// /// Will save files into the provided tmp directory, do timing or counting /// instructions and run that with the arguments passed, and then do comparative /// runs of transformed/non-transformed versions of the dill. The possible /// transformations are: /// * [tryToAnnotate] which will add a `@pragma("vm:prefer-inline")` annotation /// to each procedure one at a time. /// * [tryToSlowDown] which will add a busywait for approximately 0.002 ms to /// each procedure one at a time. (Unsurprisingly this makes it slower /// proportional to the number of times the procedure is called, so the /// counting annotation is likely at least as useful). /// void doWork(Directory tmp, List dillData, List arguments, {bool tryToAnnotate: false, bool tryToSlowDown: false, bool timeInsteadOfCount: false}) { File f = new File.fromUri(tmp.uri.resolve("a.dill")); f.writeAsBytesSync(dillData); Uri dillOrgInTmp = f.uri; print("Wrote to $f"); List sortedProcedures; if (timeInsteadOfCount) { sortedProcedures = doTimingInstrumentation(dillData, tmp, arguments); } else { sortedProcedures = doCountingInstrumentation(dillData, tmp, arguments); } print("\n\n"); bool didSomething = false; if (tryToAnnotate) { didSomething = true; for (Procedure p in sortedProcedures) { print("Prefer inline $p (${p.location})"); Uri preferredInlined = preferInlineProcedure( dillData, tmp.uri, (lib) => lib.importUri == p.enclosingLibrary.importUri, p.enclosingClass?.name, p.name.text); print("\nOriginal runs:"); List runtimesA = runXTimes(5, [dillOrgInTmp.toString(), ...arguments]); print("\nModified runs:"); List runtimesB = runXTimes(5, [preferredInlined.toString(), ...arguments]); print(SimpleTTestStat.ttest(runtimesB, runtimesA)); print("\n------------\n"); } } if (tryToSlowDown) { didSomething = true; for (Procedure p in sortedProcedures) { Uri busyWaiting = busyWaitProcedure( dillData, tmp.uri, (lib) => lib.importUri == p.enclosingLibrary.importUri, p.enclosingClass?.name, p.name.text); if (busyWaiting == null) continue; print("Slow down $p (${p.location})"); print("\nOriginal runs:"); List runtimesA = runXTimes(2, [dillOrgInTmp.toString(), ...arguments]); print("\nModified runs:"); List runtimesB = runXTimes(2, [busyWaiting.toString(), ...arguments]); print(SimpleTTestStat.ttest(runtimesB, runtimesA)); print("\n------------\n"); } } if (!didSomething) { runXTimes(10, [dillOrgInTmp.toString(), ...arguments]); } } /// Instrument the [dillData] so that each procedure-call can be registered /// (in package:front_end) and we find out how many times each procedure is /// called for a specific run, then run it, print the result and return the /// procedures in sorted order (most calls first). List doCountingInstrumentation( List dillData, Directory tmp, List arguments) { Instrumented instrumented = instrumentCallsCount(dillData, tmp.uri); List stdout = []; runXTimes(1, [instrumented.dill.toString(), ...arguments], stdout); List procedureCountsTmp = new List.from(jsonDecode(stdout.single)); List procedureCounts = []; for (int i = 0; i < procedureCountsTmp.length; i += 2) { procedureCounts .add(new IntPair(procedureCountsTmp[i], procedureCountsTmp[i + 1])); } // Sort highest call-count first. procedureCounts.sort((a, b) => b.value - a.value); List sortedProcedures = []; for (IntPair p in procedureCounts) { if (p.value > 1000) { Procedure procedure = instrumented.procedures[p.key]; String location = procedure.location.toString(); if (location.length > 50) { location = location.substring(location.length - 50); } print("Called $procedure ${p.value} times ($location)"); sortedProcedures.add(procedure); } } return sortedProcedures; } /// Instrument the [dillData] so that each (sync) procedure-call can be timed /// (time on stack, i.e. not only the procedure itself, but also the /// procedure-calls it makes) (in package:front_end) and we find out how long /// each procedure is on the stack for a specific run, then run it, print the /// result and return the procedures in sorted order (most time on stack first). List doTimingInstrumentation( List dillData, Directory tmp, List arguments) { Instrumented instrumented = instrumentCallsTiming(dillData, tmp.uri); List stdout = []; runXTimes(1, [instrumented.dill.toString(), ...arguments], stdout); List procedureTimeTmp = new List.from(jsonDecode(stdout.single)); List procedureTime = []; for (int i = 0; i < procedureTimeTmp.length; i += 2) { procedureTime .add(new IntPair(procedureTimeTmp[i], procedureTimeTmp[i + 1])); } // Sort highest time-on-stack first. procedureTime.sort((a, b) => b.value - a.value); List sortedProcedures = []; for (IntPair p in procedureTime) { if (p.value > 1000) { Procedure procedure = instrumented.procedures[p.key]; String location = procedure.location.toString(); if (location.length > 50) { location = location.substring(location.length - 50); } print("$procedure was on stack for ${p.value} microseconds ($location)"); sortedProcedures.add(procedure); } } return sortedProcedures; } class IntPair { final int key; final int value; IntPair(this.key, this.value); String toString() { return "IntPair[$key: $value]"; } } /// Adds the annotation `@pragma("vm:prefer-inline")` to the specified procedure /// and serialize the resulting dill into `b.dill` (return uri). /// /// The annotation is copied from the [preferInlineMe] method in the helper. Uri preferInlineProcedure(List dillData, Uri tmp, bool libraryMatcher(Library lib), String className, String procedureName) { Component component = new Component(); new BinaryBuilder(dillData, disableLazyReading: true) .readComponent(component); Procedure preferInlineMeProcedure = getProcedure(component, (lib) => lib.fileUri == benchmarkHelper, null, "preferInlineMe"); ConstantExpression annotation = preferInlineMeProcedure.annotations.single; Procedure markProcedure = getProcedure(component, libraryMatcher, className, procedureName); markProcedure.addAnnotation( new ConstantExpression(annotation.constant, annotation.type)); Uint8List newDillData = serializeComponent(component); File f = new File.fromUri(tmp.resolve("b.dill")); f.writeAsBytesSync(newDillData); return f.uri; } /// Makes the procedure specified call [busyWait] from the helper and serialize /// the resulting dill into `c.dill` (return uri). /// /// This will make the procedure busy-wait approximately 0.002 ms for each /// invocation (+ whatever overhead and imprecision). Uri busyWaitProcedure(List dillData, Uri tmp, bool libraryMatcher(Library lib), String className, String procedureName) { Component component = new Component(); new BinaryBuilder(dillData, disableLazyReading: true) .readComponent(component); Procedure busyWaitProcedure = getProcedure( component, (lib) => lib.fileUri == benchmarkHelper, null, "busyWait"); Procedure markProcedure = getProcedure(component, libraryMatcher, className, procedureName); if (markProcedure.function.body == null) return null; Statement orgBody = markProcedure.function.body; markProcedure.function.body = new Block([ new ExpressionStatement(new StaticInvocation( busyWaitProcedure, new Arguments([new IntLiteral(2 /* 0.002 ms */)]))), orgBody ]) ..parent = markProcedure.function; Uint8List newDillData = serializeComponent(component); File f = new File.fromUri(tmp.resolve("c.dill")); f.writeAsBytesSync(newDillData); return f.uri; } /// Instrument the [dillData] so that each procedure-call can be registered /// (in package:front_end) and we find out how many times each procedure is /// called for a specific run. /// /// Uses the [registerCall] in the helper. /// Numbers each procedure, saves the instrumented dill and returns both the /// dill and the list of procedures so that procedure i in the list will be /// annotated with a call to `registerCall(i)`. Instrumented instrumentCallsCount(List dillData, Uri tmp) { Component component = new Component(); new BinaryBuilder(dillData, disableLazyReading: true) .readComponent(component); Procedure registerCallProcedure = getProcedure( component, (lib) => lib.fileUri == benchmarkHelper, null, "registerCall"); RegisterCallTransformer registerCallTransformer = new RegisterCallTransformer(registerCallProcedure); component.accept(registerCallTransformer); Uint8List newDillData = serializeComponent(component); File f = new File.fromUri(tmp.resolve("counting.dill")); f.writeAsBytesSync(newDillData); return new Instrumented(f.uri, registerCallTransformer.procedures); } /// Instrument the [dillData] so that each (sync) procedure-call can be timed /// (time on stack, i.e. not only the procedure itself, but also the /// procedure-calls it makes) (in package:front_end) and we find out how long /// each procedure is on the stack for a specific run. /// /// Uses [registerCallStart] and [registerCallEnd] from the helper. /// Numbers each sync procedure, saves the instrumented dill and returns both /// the dill and the list of procedures so that procedure i in the list will be /// annotated with a call-pair to `registerCallStart(i)` and /// `registerCallEnd(i)`. Instrumented instrumentCallsTiming(List dillData, Uri tmp) { Component component = new Component(); new BinaryBuilder(dillData, disableLazyReading: true) .readComponent(component); Procedure registerCallStartProcedure = getProcedure(component, (lib) => lib.fileUri == benchmarkHelper, null, "registerCallStart"); Procedure registerCallEndProcedure = getProcedure(component, (lib) => lib.fileUri == benchmarkHelper, null, "registerCallEnd"); RegisterTimeTransformer registerTimeTransformer = new RegisterTimeTransformer( registerCallStartProcedure, registerCallEndProcedure); component.accept(registerTimeTransformer); Uint8List newDillData = serializeComponent(component); File f = new File.fromUri(tmp.resolve("timing.dill")); f.writeAsBytesSync(newDillData); return new Instrumented(f.uri, registerTimeTransformer.procedures); } /// Class holding both the uri of a saved dill file and a list of procedures (in /// order) that has some reference to the annotation added to the dill file. class Instrumented { final Uri dill; final List procedures; Instrumented(this.dill, this.procedures); } class RegisterCallTransformer extends RecursiveVisitor { final Procedure registerCallProcedure; RegisterCallTransformer(this.registerCallProcedure); List procedures = []; visitLibrary(Library node) { if (node.importUri.scheme == "package" && node.importUri.pathSegments.first == "front_end") { super.visitLibrary(node); } } visitProcedure(Procedure node) { if (node.function.body == null) return; int procedureNum = procedures.length; procedures.add(node); Statement orgBody = node.function.body; node.function.body = new Block([ new ExpressionStatement(new StaticInvocation(registerCallProcedure, new Arguments([new IntLiteral(procedureNum)]))), orgBody ]); node.function.body.parent = node.function; } } class RegisterTimeTransformer extends RecursiveVisitor { final Procedure registerCallStartProcedure; final Procedure registerCallEndProcedure; RegisterTimeTransformer( this.registerCallStartProcedure, this.registerCallEndProcedure); List procedures = []; visitLibrary(Library node) { if (node.importUri.scheme == "package" && node.importUri.pathSegments.first == "front_end") { super.visitLibrary(node); } } visitProcedure(Procedure node) { if (node.function.body == null) return; if (node.function.dartAsyncMarker != AsyncMarker.Sync) return; int procedureNum = procedures.length; procedures.add(node); Statement orgBody = node.function.body; // Rewrite as // { // registerCallStartProcedure(x); // try { // originalBody // } finally { // registerCallEndProcedure(x); // } // } Block block = new Block([ new ExpressionStatement(new StaticInvocation(registerCallStartProcedure, new Arguments([new IntLiteral(procedureNum)]))), new TryFinally( orgBody, new ExpressionStatement(new StaticInvocation(registerCallEndProcedure, new Arguments([new IntLiteral(procedureNum)]))), ) ]); node.function.body = block; node.function.body.parent = node.function; } } Procedure getProcedure(Component component, bool libraryMatcher(Library lib), String className, String procedureName) { Library lib = component.libraries.where(libraryMatcher).single; List procedures = lib.procedures; if (className != null) { Class cls = lib.classes.where((c) => c.name == className).single; procedures = cls.procedures; } // TODO: This will fail for getter/setter pairs. Fix that. return procedures.where((p) => p.name.text == procedureName).single; } List runXTimes(int x, List arguments, [List stdout]) { List result = []; Stopwatch stopwatch = new Stopwatch()..start(); for (int i = 0; i < x; i++) { stopwatch.reset(); ProcessResult run = Process.runSync(Platform.resolvedExecutable, arguments, runInShell: true); int ms = stopwatch.elapsedMilliseconds; result.add(ms); print(ms); if (run.exitCode != 0) throw "Got exit code ${run.exitCode}"; if (stdout != null) { stdout.add(run.stdout); } } return result; }