[tool] Bisection tool

A basic bisection script.

Currently only supports substring matching for detecting the error.
This was enough for three use cases today:

* https://github.com/dart-lang/sdk/issues/52910
* https://github.com/dart-lang/sdk/issues/52911
* https://github.com/dart-lang/sdk/issues/52912

Produces a concise output on standard out, and a very detailed log
with all process invocation results in `.dart_tool/bisect_dart`.

Usage: tools/bisect.dart -Dstart=23f41452 -Dend=2c97bd78 -Dtest_command="tools/test.py --build -n dartk-linux-debug-x64 lib_2/isolate/package_resolve_test" -Dfailure_string="Error: The argument type 'String' can't be assigned to the parameter type 'Uri'." -Dsdk_path=/usr/local/google/home/dacoharkes/dart-sdk/sdk/ -Dname=20230712_package_resolve_test

This script starts a bisection in the provided SDK path.

It will write logs to .dart_tool/bisect_dart/.

start          : The commit has at the start of the commit range.
end            : The commit has at the end of the commit range.
test_command   : The invocation of test.py.
                 This should include `--build`.
                 This should be within quotes when passed in terminal because of spaces.
failure_string : A string from the failing output.
                 Regexes are not yet supported.
                 This should be within quotes when passed in terminal when containing spaces.
sdk_path       : The SDK path is optional.
                 The SDK path defaults to the current working directory.
name           : The name is optional.
                 The name defaults to the current date and the recognized test name.
                 The name is used for distinguishing logs.

Change-Id: Ib071a5305d4992cf189e35eb3dcc50c83101503e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/313384
Commit-Queue: Daco Harkes <dacoharkes@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

.gitignore

@@ -107,3 +107,4 @@ tools/xcodebuild
 /pkg/front_end/testcases/old_dills/
 logs/logs.json
 logs/results.json
+.dart_tool/bisect_dart/

pkg/bisect_dart/.gitignore

@@ -0,0 +1 @@
+.dart_tool

pkg/bisect_dart/OWNERS

@@ -0,0 +1 @@
+file:/tools/OWNERS_ENG

pkg/bisect_dart/bin/bisect_dart.dart

@@ -0,0 +1,9 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:bisect_dart/src/run_bisection.dart';
+
+Future<void> main(List<String> args) async {
+  await runMain(args);
+}

pkg/bisect_dart/lib/src/bisection_config.dart

@@ -0,0 +1,154 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:cli_config/cli_config.dart';
+import 'package:intl/intl.dart';
+
+class BisectionConfig {
+  /// A way to identify this bisection.
+  ///
+  /// Used for log names etc.
+  final String name;
+
+  static const String _nameKey = 'name';
+
+  /// Hash of the first commit.
+  final String start;
+
+  static const String _startKey = 'start';
+
+  /// Hash of the last commit.
+  final String end;
+
+  static const String _endKey = 'end';
+
+  /// The command to run.
+  ///
+  /// Should include a `--build`.
+  final String testCommand;
+
+  static const String _testCommandKey = 'test_command';
+
+  /// The pattern to recognize in the stdout of [testCommand].
+  final String failureString;
+
+  static const String _failureStringKey = 'failure_string';
+
+  // This will likely be extended later to support regexes.
+  Pattern get failurePattern => failureString.toPattern();
+
+  /// The SDK checkout to use for bisecting.
+  ///
+  /// This will modify the SDK checkout!
+  ///
+  /// Will be created if it doens't exist.
+  final Uri sdkPath;
+
+  static const _sdkPathKey = 'sdk_path';
+
+  BisectionConfig({
+    required this.name,
+    required this.start,
+    required this.end,
+    required this.testCommand,
+    required this.sdkPath,
+    required this.failureString,
+  });
+
+  factory BisectionConfig.fromConfig(Config config) {
+    final testCommand = config.string(_testCommandKey);
+    final name = config.optionalString(_nameKey) ??
+        '${DateFormat('yyyyMMdd').format(DateTime.now())}_'
+            '${testCommand.split(' ').last.split('/').last}';
+    final sdkPath = config.optionalPath(_sdkPathKey, mustExist: true) ??
+        Directory.current.uri;
+    return BisectionConfig(
+      name: name,
+      start: config.string(_startKey),
+      end: config.string(_endKey),
+      testCommand: testCommand,
+      sdkPath: sdkPath,
+      failureString: config.string(_failureStringKey),
+    );
+  }
+
+  Map<String, String> asMap() => {
+        _startKey: start,
+        _endKey: end,
+        _testCommandKey: testCommand,
+        _failureStringKey: failureString,
+        _sdkPathKey: sdkPath.toFilePath(),
+        _nameKey: name,
+      };
+
+  @override
+  String toString() {
+    return 'BisectionConfig(${asMap()})';
+  }
+
+  static final BisectionConfig _example = BisectionConfig(
+    name: '20230712_package_resolve_test',
+    start: '23f41452',
+    end: '2c97bd78',
+    testCommand:
+        'tools/test.py --build -n dartk-linux-debug-x64 lib_2/isolate/package_resolve_test',
+    sdkPath: Directory.current.uri,
+    failureString:
+        "Error: The argument type 'String' can't be assigned to the parameter type 'Uri'.",
+  );
+
+  static const _argumentDescriptions = {
+    _startKey: 'The commit has at the start of the commit range.',
+    _endKey: 'The commit has at the end of the commit range.',
+    _testCommandKey: '''The invocation of test.py.
+This should include `--build`.
+This should be within quotes when passed in terminal because of spaces.
+''',
+    _failureStringKey: '''A string from the failing output.
+Regexes are not yet supported.
+This should be within quotes when passed in terminal when containing spaces.
+''',
+    _sdkPathKey: '''The SDK path is optional.
+The SDK path defaults to the current working directory.
+''',
+    _nameKey: '''The name is optional.
+The name defaults to the current date and the recognized test name.
+The name is used for distinguishing logs.
+''',
+  };
+
+  static String helpMessage() {
+    final exampleArguments = _example.asMap().entries.map((e) {
+      var value = e.value;
+      if (value.contains(' ')) {
+        value = '"$value"';
+      }
+      return '-D${e.key}=$value';
+    }).join(' ');
+    const padding = _failureStringKey.length;
+    final descriptions = _argumentDescriptions.entries.map((e) {
+      final value = e.value
+          .split('\n')
+          .map((l) => '${' ' * (padding + 3)}$l')
+          .join('\n')
+          .trim();
+      return '${e.key.padRight(padding)} : $value';
+    }).join('\n');
+    return '''
+Usage: tools/bisect.dart $exampleArguments
+
+This script starts a bisection in the provided SDK path.
+
+It will write logs to .dart_tool/bisect_dart/.
+
+$descriptions
+''';
+  }
+}
+
+extension on String {
+  toPattern() => RegExp(RegExp.escape(this));
+}

pkg/bisect_dart/lib/src/run_bisection.dart

@@ -0,0 +1,220 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:cli_config/cli_config.dart';
+import 'package:logging/logging.dart';
+
+import 'bisection_config.dart';
+import 'run_process.dart';
+
+Future<void> runMain(List<String> args) async {
+  if (args.contains('--help')) {
+    print(BisectionConfig.helpMessage());
+    return;
+  }
+  final config = BisectionConfig.fromConfig(await Config.fromArgs(args: args));
+  await runBisection(config);
+}
+
+Future<void> runBisection(BisectionConfig config) async {
+  final name = config.name;
+  final startHash = config.start;
+  final endHash = config.end;
+  final testCommand = config.testCommand;
+  final failurePattern = config.failurePattern;
+  final sdkCheckout = config.sdkPath;
+
+  final logsDir =
+      Directory.current.uri.resolve('.dart_tool/bisect_dart/$name/logs/');
+  await Directory.fromUri(logsDir).create(recursive: true);
+  final logFileUri = logsDir.resolve('full.txt');
+  final logFile = File.fromUri(logFileUri);
+  if (await logFile.exists()) {
+    await logFile.delete();
+  }
+  final logger = _mainLogger('', logFileUri);
+  logger.info('Writing detailed log to ${logFileUri.toFilePath()}.');
+  logger.config('Bisection configuration: $config.');
+
+  await _ensureSdkRepo(sdkCheckout, logger);
+
+  logger.info('Ensuring failure reproduces on $startHash.');
+  final shouldFail = await _checkCommit(
+      startHash, testCommand, failurePattern, sdkCheckout, logger);
+  if (!shouldFail) {
+    throw Exception('$startHash failed to reproduce the error.');
+  }
+
+  final hashBeforeRange = await _commitHashBefore(endHash, sdkCheckout, logger);
+  logger.info('Ensuring failure does not reproduce on $hashBeforeRange.');
+  final shouldSucceed = await _checkCommit(
+      hashBeforeRange, testCommand, failurePattern, sdkCheckout, logger);
+  if (shouldSucceed) {
+    throw Exception('$startHash failed to reproduced the error.');
+  }
+
+  final commitHashes =
+      await _commitHashesInRange(startHash, endHash, sdkCheckout, logger);
+  final regressionCommit = await _bisect(
+      commitHashes, testCommand, failurePattern, sdkCheckout, logger);
+  logger.info('Bisected to $regressionCommit.');
+}
+
+Future<String> _bisect(
+  List<String> commitHashes,
+  String testCommand,
+  Pattern failurePattern,
+  Uri sdkCheckout,
+  Logger logger,
+) async {
+  if (commitHashes.length == 1) {
+    return commitHashes.single;
+  }
+  final numCommits = commitHashes.length;
+  final pivotIndex = numCommits ~/ 2;
+  final pivot = commitHashes[pivotIndex];
+  logger.info(
+      'Bisecting ${commitHashes.first}...${commitHashes.last} ($numCommits commits). Trying $pivot.');
+  final commitResult = await _checkCommit(
+    pivot,
+    testCommand,
+    failurePattern,
+    sdkCheckout,
+    logger,
+  );
+  List<String> remainingCommits;
+  if (commitResult) {
+    // Reproduces on pivot, so it must be in the older half of commits.
+    remainingCommits = commitHashes.skip(pivotIndex).toList();
+  } else {
+    remainingCommits = commitHashes.take(pivotIndex).toList();
+  }
+  return await _bisect(
+      remainingCommits, testCommand, failurePattern, sdkCheckout, logger);
+}
+
+/// Returns true if the commit has the [failurePattern].
+Future<bool> _checkCommit(String hash, String testCommand,
+    Pattern failurePattern, Uri sdkCheckout, Logger logger) async {
+  logger.config('Testing $hash.');
+  await _gitCheckout(hash, sdkCheckout, logger);
+  await _gclientSync(sdkCheckout, logger);
+  final testOutput = await _runTest(testCommand, sdkCheckout, logger);
+  final matches = failurePattern.allMatches(testOutput).toList();
+  final foundFailure = matches.isNotEmpty;
+  if (foundFailure) {
+    logger.info('Commit $hash, reproduces failure.');
+  } else {
+    logger.info('Commit $hash, does not reproduce failure.');
+  }
+  return foundFailure;
+}
+
+Future<void> _ensureSdkRepo(Uri sdkCheckout, Logger logger) async {
+  logger.info('Ensuring SDK repo in ${sdkCheckout.toFilePath()}.');
+  final workDir = Directory.fromUri(sdkCheckout).parent;
+  if (!await workDir.exists()) {
+    await workDir.create(recursive: true);
+    await runProcess(
+      executable: Uri.file('fetch'),
+      arguments: ['dart'],
+      logger: logger,
+      workingDirectory: workDir.uri,
+    );
+  } else {
+    await runProcess(
+      executable: Uri.file('git'),
+      arguments: ['stash', '--include-untracked'],
+      logger: logger,
+      workingDirectory: sdkCheckout,
+    );
+    await runProcess(
+      executable: Uri.file('git'),
+      arguments: ['fetch'],
+      logger: logger,
+      workingDirectory: sdkCheckout,
+    );
+  }
+}
+
+Future<void> _gitCheckout(String hash, Uri sdkCheckout, Logger logger) {
+  return runProcess(
+    executable: Uri.file('git'),
+    arguments: ['checkout', hash],
+    logger: logger,
+    workingDirectory: sdkCheckout,
+  );
+}
+
+Future<void> _gclientSync(Uri sdkCheckout, Logger logger) {
+  return runProcess(
+    executable: Uri.file('gclient'),
+    arguments: ['sync', '-D'],
+    logger: logger,
+    workingDirectory: sdkCheckout,
+  );
+}
+
+Future<String> _runTest(
+    String testCommand, Uri sdkCheckout, Logger logger) async {
+  final arguments = testCommand.split(' ');
+  final result = await runProcess(
+    executable: Uri.file('python3'),
+    arguments: arguments,
+    logger: logger,
+    workingDirectory: sdkCheckout,
+    captureOutput: true,
+  );
+  return result.stdout;
+}
+
+/// Ordered from now to old.
+Future<List<String>> _commitHashesInRange(String commitHashStart,
+    String commitHashEnd, Uri sdkCheckout, Logger logger) async {
+  final result = await runProcess(
+    executable: Uri.file('git'),
+    arguments: [
+      'log',
+      '--pretty=format:"%h"',
+      '$commitHashStart...$commitHashEnd',
+    ],
+    captureOutput: true,
+    logger: logger,
+    workingDirectory: sdkCheckout,
+  );
+  return result.stdout.trim().replaceAll('"', '').split('\n');
+}
+
+Future<String> _commitHashBefore(
+    String commitHash, Uri sdkCheckout, Logger logger) async {
+  final result = await runProcess(
+    executable: Uri.file('git'),
+    arguments: [
+      'log',
+      '--pretty=format:"%h"',
+      '$commitHash~1...$commitHash~2',
+    ],
+    captureOutput: true,
+    logger: logger,
+    workingDirectory: sdkCheckout,
+  );
+  return result.stdout.trim().replaceAll('"', '');
+}
+
+Logger _mainLogger(String name, Uri filePath) {
+  final file = File.fromUri(filePath);
+  return Logger('')
+    ..level = Level.ALL
+    ..onRecord.listen((record) {
+      if (record.level >= Level.INFO) {
+        print(record.message);
+      }
+      file.writeAsStringSync(
+        '${record.message}\n',
+        mode: FileMode.append,
+      );
+    });
+}

pkg/bisect_dart/lib/src/run_process.dart

@@ -0,0 +1,128 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:logging/logging.dart';
+
+/// Runs a [Process].
+///
+/// If [logger] is provided, stream stdout and stderr to it.
+///
+/// If [captureOutput], captures stdout and stderr.
+Future<RunProcessResult> runProcess({
+  required Uri executable,
+  List<String> arguments = const [],
+  Uri? workingDirectory,
+  Map<String, String>? environment,
+  bool includeParentEnvironment = true,
+  required Logger? logger,
+  bool captureOutput = true,
+  int expectedExitCode = 0,
+  bool throwOnUnexpectedExitCode = false,
+}) async {
+  if (Platform.isWindows && !includeParentEnvironment) {
+    const winEnvKeys = [
+      'SYSTEMROOT',
+    ];
+    environment = {
+      for (final winEnvKey in winEnvKeys)
+        winEnvKey: Platform.environment[winEnvKey]!,
+      ...?environment,
+    };
+  }
+
+  final printWorkingDir =
+      workingDirectory != null && workingDirectory != Directory.current.uri;
+  final commandString = [
+    if (printWorkingDir) '(cd ${workingDirectory.toFilePath()};',
+    ...?environment?.entries.map((entry) => '${entry.key}=${entry.value}'),
+    executable.toFilePath(),
+    ...arguments.map((a) => a.contains(' ') ? "'$a'" : a),
+    if (printWorkingDir) ')',
+  ].join(' ');
+  logger?.config('Running `$commandString`.');
+
+  final stdoutBuffer = StringBuffer();
+  final stderrBuffer = StringBuffer();
+  final process = await Process.start(
+    executable.toFilePath(),
+    arguments,
+    workingDirectory: workingDirectory?.toFilePath(),
+    environment: environment,
+    includeParentEnvironment: includeParentEnvironment,
+    runInShell: Platform.isWindows && !includeParentEnvironment,
+  );
+
+  final stdoutSub = process.stdout
+      .transform(utf8.decoder)
+      .transform(const LineSplitter())
+      .listen(captureOutput
+          ? (s) {
+              logger?.fine(s);
+              stdoutBuffer.writeln(s);
+            }
+          : logger?.fine);
+  final stderrSub = process.stderr
+      .transform(utf8.decoder)
+      .transform(const LineSplitter())
+      .listen(captureOutput
+          ? (s) {
+              logger?.config(s);
+              stderrBuffer.writeln(s);
+            }
+          : logger?.config);
+
+  final (exitCode, _, _) = await (
+    process.exitCode,
+    stdoutSub.asFuture<void>(),
+    stderrSub.asFuture<void>()
+  ).wait;
+  final result = RunProcessResult(
+    pid: process.pid,
+    command: commandString,
+    exitCode: exitCode,
+    stdout: stdoutBuffer.toString(),
+    stderr: stderrBuffer.toString(),
+  );
+  if (throwOnUnexpectedExitCode && expectedExitCode != exitCode) {
+    throw ProcessException(
+      executable.toFilePath(),
+      arguments,
+      "Full command string: '$commandString'.\n"
+      "Exit code: '$exitCode'.\n"
+      'For the output of the process check the logger output.',
+    );
+  }
+  return result;
+}
+
+/// Drop in replacement of [ProcessResult].
+class RunProcessResult {
+  final int pid;
+
+  final String command;
+
+  final int exitCode;
+
+  final String stderr;
+
+  final String stdout;
+
+  RunProcessResult({
+    required this.pid,
+    required this.command,
+    required this.exitCode,
+    required this.stderr,
+    required this.stdout,
+  });
+
+  @override
+  String toString() => '''command: $command
+exitCode: $exitCode
+stdout: $stdout
+stderr: $stderr''';
+}

pkg/bisect_dart/pubspec.yaml

@@ -0,0 +1,11 @@
+name: bisect_dart
+# This package is not intended for consumption on pub.dev. DO NOT publish.
+publish_to: none
+
+environment:
+  sdk: '>=3.0.0 <4.0.0'
+
+dependencies:
+  cli_config: any
+  intl: any
+  logging: any

tools/bisect.dart

@@ -0,0 +1,10 @@
+#!/usr/bin/env dart
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:bisect_dart/src/run_bisection.dart';
+
+Future<void> main(List<String> args) async {
+  await runMain(args);
+}