diff --git a/.gitignore b/.gitignore index b38fe503f10..229fe115665 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,4 @@ tools/xcodebuild /pkg/front_end/testcases/old_dills/ logs/logs.json logs/results.json +.dart_tool/bisect_dart/ diff --git a/pkg/bisect_dart/.gitignore b/pkg/bisect_dart/.gitignore new file mode 100644 index 00000000000..c7f77dcf76e --- /dev/null +++ b/pkg/bisect_dart/.gitignore @@ -0,0 +1 @@ +.dart_tool \ No newline at end of file diff --git a/pkg/bisect_dart/OWNERS b/pkg/bisect_dart/OWNERS new file mode 100644 index 00000000000..2b67506d84c --- /dev/null +++ b/pkg/bisect_dart/OWNERS @@ -0,0 +1 @@ +file:/tools/OWNERS_ENG diff --git a/pkg/bisect_dart/bin/bisect_dart.dart b/pkg/bisect_dart/bin/bisect_dart.dart new file mode 100644 index 00000000000..99ccb306340 --- /dev/null +++ b/pkg/bisect_dart/bin/bisect_dart.dart @@ -0,0 +1,9 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:bisect_dart/src/run_bisection.dart'; + +Future main(List args) async { + await runMain(args); +} diff --git a/pkg/bisect_dart/lib/src/bisection_config.dart b/pkg/bisect_dart/lib/src/bisection_config.dart new file mode 100644 index 00000000000..10ef8fcb58a --- /dev/null +++ b/pkg/bisect_dart/lib/src/bisection_config.dart @@ -0,0 +1,154 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:cli_config/cli_config.dart'; +import 'package:intl/intl.dart'; + +class BisectionConfig { + /// A way to identify this bisection. + /// + /// Used for log names etc. + final String name; + + static const String _nameKey = 'name'; + + /// Hash of the first commit. + final String start; + + static const String _startKey = 'start'; + + /// Hash of the last commit. + final String end; + + static const String _endKey = 'end'; + + /// The command to run. + /// + /// Should include a `--build`. + final String testCommand; + + static const String _testCommandKey = 'test_command'; + + /// The pattern to recognize in the stdout of [testCommand]. + final String failureString; + + static const String _failureStringKey = 'failure_string'; + + // This will likely be extended later to support regexes. + Pattern get failurePattern => failureString.toPattern(); + + /// The SDK checkout to use for bisecting. + /// + /// This will modify the SDK checkout! + /// + /// Will be created if it doens't exist. + final Uri sdkPath; + + static const _sdkPathKey = 'sdk_path'; + + BisectionConfig({ + required this.name, + required this.start, + required this.end, + required this.testCommand, + required this.sdkPath, + required this.failureString, + }); + + factory BisectionConfig.fromConfig(Config config) { + final testCommand = config.string(_testCommandKey); + final name = config.optionalString(_nameKey) ?? + '${DateFormat('yyyyMMdd').format(DateTime.now())}_' + '${testCommand.split(' ').last.split('/').last}'; + final sdkPath = config.optionalPath(_sdkPathKey, mustExist: true) ?? + Directory.current.uri; + return BisectionConfig( + name: name, + start: config.string(_startKey), + end: config.string(_endKey), + testCommand: testCommand, + sdkPath: sdkPath, + failureString: config.string(_failureStringKey), + ); + } + + Map asMap() => { + _startKey: start, + _endKey: end, + _testCommandKey: testCommand, + _failureStringKey: failureString, + _sdkPathKey: sdkPath.toFilePath(), + _nameKey: name, + }; + + @override + String toString() { + return 'BisectionConfig(${asMap()})'; + } + + static final BisectionConfig _example = BisectionConfig( + name: '20230712_package_resolve_test', + start: '23f41452', + end: '2c97bd78', + testCommand: + 'tools/test.py --build -n dartk-linux-debug-x64 lib_2/isolate/package_resolve_test', + sdkPath: Directory.current.uri, + failureString: + "Error: The argument type 'String' can't be assigned to the parameter type 'Uri'.", + ); + + static const _argumentDescriptions = { + _startKey: 'The commit has at the start of the commit range.', + _endKey: 'The commit has at the end of the commit range.', + _testCommandKey: '''The invocation of test.py. +This should include `--build`. +This should be within quotes when passed in terminal because of spaces. +''', + _failureStringKey: '''A string from the failing output. +Regexes are not yet supported. +This should be within quotes when passed in terminal when containing spaces. +''', + _sdkPathKey: '''The SDK path is optional. +The SDK path defaults to the current working directory. +''', + _nameKey: '''The name is optional. +The name defaults to the current date and the recognized test name. +The name is used for distinguishing logs. +''', + }; + + static String helpMessage() { + final exampleArguments = _example.asMap().entries.map((e) { + var value = e.value; + if (value.contains(' ')) { + value = '"$value"'; + } + return '-D${e.key}=$value'; + }).join(' '); + const padding = _failureStringKey.length; + final descriptions = _argumentDescriptions.entries.map((e) { + final value = e.value + .split('\n') + .map((l) => '${' ' * (padding + 3)}$l') + .join('\n') + .trim(); + return '${e.key.padRight(padding)} : $value'; + }).join('\n'); + return ''' +Usage: tools/bisect.dart $exampleArguments + +This script starts a bisection in the provided SDK path. + +It will write logs to .dart_tool/bisect_dart/. + +$descriptions +'''; + } +} + +extension on String { + toPattern() => RegExp(RegExp.escape(this)); +} diff --git a/pkg/bisect_dart/lib/src/run_bisection.dart b/pkg/bisect_dart/lib/src/run_bisection.dart new file mode 100644 index 00000000000..310d8c25683 --- /dev/null +++ b/pkg/bisect_dart/lib/src/run_bisection.dart @@ -0,0 +1,220 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:cli_config/cli_config.dart'; +import 'package:logging/logging.dart'; + +import 'bisection_config.dart'; +import 'run_process.dart'; + +Future runMain(List args) async { + if (args.contains('--help')) { + print(BisectionConfig.helpMessage()); + return; + } + final config = BisectionConfig.fromConfig(await Config.fromArgs(args: args)); + await runBisection(config); +} + +Future runBisection(BisectionConfig config) async { + final name = config.name; + final startHash = config.start; + final endHash = config.end; + final testCommand = config.testCommand; + final failurePattern = config.failurePattern; + final sdkCheckout = config.sdkPath; + + final logsDir = + Directory.current.uri.resolve('.dart_tool/bisect_dart/$name/logs/'); + await Directory.fromUri(logsDir).create(recursive: true); + final logFileUri = logsDir.resolve('full.txt'); + final logFile = File.fromUri(logFileUri); + if (await logFile.exists()) { + await logFile.delete(); + } + final logger = _mainLogger('', logFileUri); + logger.info('Writing detailed log to ${logFileUri.toFilePath()}.'); + logger.config('Bisection configuration: $config.'); + + await _ensureSdkRepo(sdkCheckout, logger); + + logger.info('Ensuring failure reproduces on $startHash.'); + final shouldFail = await _checkCommit( + startHash, testCommand, failurePattern, sdkCheckout, logger); + if (!shouldFail) { + throw Exception('$startHash failed to reproduce the error.'); + } + + final hashBeforeRange = await _commitHashBefore(endHash, sdkCheckout, logger); + logger.info('Ensuring failure does not reproduce on $hashBeforeRange.'); + final shouldSucceed = await _checkCommit( + hashBeforeRange, testCommand, failurePattern, sdkCheckout, logger); + if (shouldSucceed) { + throw Exception('$startHash failed to reproduced the error.'); + } + + final commitHashes = + await _commitHashesInRange(startHash, endHash, sdkCheckout, logger); + final regressionCommit = await _bisect( + commitHashes, testCommand, failurePattern, sdkCheckout, logger); + logger.info('Bisected to $regressionCommit.'); +} + +Future _bisect( + List commitHashes, + String testCommand, + Pattern failurePattern, + Uri sdkCheckout, + Logger logger, +) async { + if (commitHashes.length == 1) { + return commitHashes.single; + } + final numCommits = commitHashes.length; + final pivotIndex = numCommits ~/ 2; + final pivot = commitHashes[pivotIndex]; + logger.info( + 'Bisecting ${commitHashes.first}...${commitHashes.last} ($numCommits commits). Trying $pivot.'); + final commitResult = await _checkCommit( + pivot, + testCommand, + failurePattern, + sdkCheckout, + logger, + ); + List remainingCommits; + if (commitResult) { + // Reproduces on pivot, so it must be in the older half of commits. + remainingCommits = commitHashes.skip(pivotIndex).toList(); + } else { + remainingCommits = commitHashes.take(pivotIndex).toList(); + } + return await _bisect( + remainingCommits, testCommand, failurePattern, sdkCheckout, logger); +} + +/// Returns true if the commit has the [failurePattern]. +Future _checkCommit(String hash, String testCommand, + Pattern failurePattern, Uri sdkCheckout, Logger logger) async { + logger.config('Testing $hash.'); + await _gitCheckout(hash, sdkCheckout, logger); + await _gclientSync(sdkCheckout, logger); + final testOutput = await _runTest(testCommand, sdkCheckout, logger); + final matches = failurePattern.allMatches(testOutput).toList(); + final foundFailure = matches.isNotEmpty; + if (foundFailure) { + logger.info('Commit $hash, reproduces failure.'); + } else { + logger.info('Commit $hash, does not reproduce failure.'); + } + return foundFailure; +} + +Future _ensureSdkRepo(Uri sdkCheckout, Logger logger) async { + logger.info('Ensuring SDK repo in ${sdkCheckout.toFilePath()}.'); + final workDir = Directory.fromUri(sdkCheckout).parent; + if (!await workDir.exists()) { + await workDir.create(recursive: true); + await runProcess( + executable: Uri.file('fetch'), + arguments: ['dart'], + logger: logger, + workingDirectory: workDir.uri, + ); + } else { + await runProcess( + executable: Uri.file('git'), + arguments: ['stash', '--include-untracked'], + logger: logger, + workingDirectory: sdkCheckout, + ); + await runProcess( + executable: Uri.file('git'), + arguments: ['fetch'], + logger: logger, + workingDirectory: sdkCheckout, + ); + } +} + +Future _gitCheckout(String hash, Uri sdkCheckout, Logger logger) { + return runProcess( + executable: Uri.file('git'), + arguments: ['checkout', hash], + logger: logger, + workingDirectory: sdkCheckout, + ); +} + +Future _gclientSync(Uri sdkCheckout, Logger logger) { + return runProcess( + executable: Uri.file('gclient'), + arguments: ['sync', '-D'], + logger: logger, + workingDirectory: sdkCheckout, + ); +} + +Future _runTest( + String testCommand, Uri sdkCheckout, Logger logger) async { + final arguments = testCommand.split(' '); + final result = await runProcess( + executable: Uri.file('python3'), + arguments: arguments, + logger: logger, + workingDirectory: sdkCheckout, + captureOutput: true, + ); + return result.stdout; +} + +/// Ordered from now to old. +Future> _commitHashesInRange(String commitHashStart, + String commitHashEnd, Uri sdkCheckout, Logger logger) async { + final result = await runProcess( + executable: Uri.file('git'), + arguments: [ + 'log', + '--pretty=format:"%h"', + '$commitHashStart...$commitHashEnd', + ], + captureOutput: true, + logger: logger, + workingDirectory: sdkCheckout, + ); + return result.stdout.trim().replaceAll('"', '').split('\n'); +} + +Future _commitHashBefore( + String commitHash, Uri sdkCheckout, Logger logger) async { + final result = await runProcess( + executable: Uri.file('git'), + arguments: [ + 'log', + '--pretty=format:"%h"', + '$commitHash~1...$commitHash~2', + ], + captureOutput: true, + logger: logger, + workingDirectory: sdkCheckout, + ); + return result.stdout.trim().replaceAll('"', ''); +} + +Logger _mainLogger(String name, Uri filePath) { + final file = File.fromUri(filePath); + return Logger('') + ..level = Level.ALL + ..onRecord.listen((record) { + if (record.level >= Level.INFO) { + print(record.message); + } + file.writeAsStringSync( + '${record.message}\n', + mode: FileMode.append, + ); + }); +} diff --git a/pkg/bisect_dart/lib/src/run_process.dart b/pkg/bisect_dart/lib/src/run_process.dart new file mode 100644 index 00000000000..2cdbaa719d2 --- /dev/null +++ b/pkg/bisect_dart/lib/src/run_process.dart @@ -0,0 +1,128 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:logging/logging.dart'; + +/// Runs a [Process]. +/// +/// If [logger] is provided, stream stdout and stderr to it. +/// +/// If [captureOutput], captures stdout and stderr. +Future runProcess({ + required Uri executable, + List arguments = const [], + Uri? workingDirectory, + Map? environment, + bool includeParentEnvironment = true, + required Logger? logger, + bool captureOutput = true, + int expectedExitCode = 0, + bool throwOnUnexpectedExitCode = false, +}) async { + if (Platform.isWindows && !includeParentEnvironment) { + const winEnvKeys = [ + 'SYSTEMROOT', + ]; + environment = { + for (final winEnvKey in winEnvKeys) + winEnvKey: Platform.environment[winEnvKey]!, + ...?environment, + }; + } + + final printWorkingDir = + workingDirectory != null && workingDirectory != Directory.current.uri; + final commandString = [ + if (printWorkingDir) '(cd ${workingDirectory.toFilePath()};', + ...?environment?.entries.map((entry) => '${entry.key}=${entry.value}'), + executable.toFilePath(), + ...arguments.map((a) => a.contains(' ') ? "'$a'" : a), + if (printWorkingDir) ')', + ].join(' '); + logger?.config('Running `$commandString`.'); + + final stdoutBuffer = StringBuffer(); + final stderrBuffer = StringBuffer(); + final process = await Process.start( + executable.toFilePath(), + arguments, + workingDirectory: workingDirectory?.toFilePath(), + environment: environment, + includeParentEnvironment: includeParentEnvironment, + runInShell: Platform.isWindows && !includeParentEnvironment, + ); + + final stdoutSub = process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen(captureOutput + ? (s) { + logger?.fine(s); + stdoutBuffer.writeln(s); + } + : logger?.fine); + final stderrSub = process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen(captureOutput + ? (s) { + logger?.config(s); + stderrBuffer.writeln(s); + } + : logger?.config); + + final (exitCode, _, _) = await ( + process.exitCode, + stdoutSub.asFuture(), + stderrSub.asFuture() + ).wait; + final result = RunProcessResult( + pid: process.pid, + command: commandString, + exitCode: exitCode, + stdout: stdoutBuffer.toString(), + stderr: stderrBuffer.toString(), + ); + if (throwOnUnexpectedExitCode && expectedExitCode != exitCode) { + throw ProcessException( + executable.toFilePath(), + arguments, + "Full command string: '$commandString'.\n" + "Exit code: '$exitCode'.\n" + 'For the output of the process check the logger output.', + ); + } + return result; +} + +/// Drop in replacement of [ProcessResult]. +class RunProcessResult { + final int pid; + + final String command; + + final int exitCode; + + final String stderr; + + final String stdout; + + RunProcessResult({ + required this.pid, + required this.command, + required this.exitCode, + required this.stderr, + required this.stdout, + }); + + @override + String toString() => '''command: $command +exitCode: $exitCode +stdout: $stdout +stderr: $stderr'''; +} diff --git a/pkg/bisect_dart/pubspec.yaml b/pkg/bisect_dart/pubspec.yaml new file mode 100644 index 00000000000..9cc20391f77 --- /dev/null +++ b/pkg/bisect_dart/pubspec.yaml @@ -0,0 +1,11 @@ +name: bisect_dart +# This package is not intended for consumption on pub.dev. DO NOT publish. +publish_to: none + +environment: + sdk: '>=3.0.0 <4.0.0' + +dependencies: + cli_config: any + intl: any + logging: any diff --git a/tools/bisect.dart b/tools/bisect.dart new file mode 100755 index 00000000000..9cbd869eedd --- /dev/null +++ b/tools/bisect.dart @@ -0,0 +1,10 @@ +#!/usr/bin/env dart +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:bisect_dart/src/run_bisection.dart'; + +Future main(List args) async { + await runMain(args); +}