// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. // results.json and flaky.json parses. import 'dart:async'; import 'dart:convert'; import 'dart:io'; /// The path to the gsutil script. String gsutilPy; /// Cloud storage location containing results. const testResultsStoragePath = "gs://dart-test-results/builders"; /// Cloud storage location containing approved results. const approvedResultsStoragePath = "gs://dart-test-results-approved-results/builders"; /// Runs gsutil with the provided [arguments] and returns the standard output. /// Returns null if the requested URL didn't exist. Future runGsutil(List arguments) async { final processResult = await Process.run( "python", [gsutilPy]..addAll(arguments), runInShell: Platform.isWindows); if (processResult.exitCode != 0) { if (processResult.exitCode == 1 && processResult.stderr.contains("No URLs matched") || processResult.stderr.contains("One or more URLs matched no objects")) { return null; } String error = "Failed to run: python $gsutilPy $arguments\n" "exitCode: ${processResult.exitCode}\n" "stdout:\n${processResult.stdout}\n" "stderr:\n${processResult.stderr}"; if (processResult.exitCode == 1 && processResult.stderr.contains("401 Anonymous caller")) { error = "\n\nYou need to authenticate by running:\npython $gsutilPy config\n"; } throw new Exception(error); } return processResult.stdout; } /// Returns the contents of the provided cloud storage [path], or null if it /// didn't exist. Future catGsutil(String path) => runGsutil(["cat", path]); /// Returns the files and directories in the provided cloud storage [directory], /// or null if it didn't exist. Future> lsGsutil(String directory) async { final contents = await runGsutil(["ls", directory]); if (contents == null) { return null; } return LineSplitter.split(contents).map((String path) { final elements = path.split("/"); if (elements[elements.length - 1].isEmpty) { return elements[elements.length - 2]; } else { return elements[elements.length - 1]; } }); } /// Copies a file to or from cloud storage. Future cpGsutil(String source, String destination) => runGsutil(["cp", source, destination]); /// Copies a directory recursively to or from cloud strorage. Future cpRecursiveGsutil(String source, String destination) => runGsutil(["-m", "cp", "-r", "-Z", source, destination]); /// Lists the bots in cloud storage. Future> listBots() => lsGsutil("$testResultsStoragePath"); /// Returns the cloud storage path for the [bot]. String botCloudPath(String bot) => "$testResultsStoragePath/$bot"; /// Returns the cloud storage path to the [build] on the [bot]. String buildCloudPath(String bot, String build) => "${botCloudPath(bot)}/$build"; /// Returns the cloud storage path to the [file] inside the [bot]'s directory. String fileCloudPath(String bot, String file) => "${botCloudPath(bot)}/$file"; /// Reads the contents of the [file] inside the [bot]'s cloud storage. Future readFile(String bot, String file) => catGsutil(fileCloudPath(bot, file)); /// Returns the cloud storage path to the [file] inside the [build] on the /// [bot]. String buildFileCloudPath(String bot, String build, String file) => "${buildCloudPath(bot, build)}/$file"; /// Reads the contents of the [file] inside the [build] in the [bot]'s cloud /// storage. Future readBuildFile(String bot, String build, String file) => catGsutil(buildFileCloudPath(bot, build, file)); List> parseResults(String contents) { return LineSplitter.split(contents) .map(jsonDecode) .toList() .cast>(); } Future>> loadResults(String path) async { final results = >[]; final lines = new File(path) .openRead() .transform(utf8.decoder) .transform(new LineSplitter()); await for (final line in lines) { final Map map = jsonDecode(line); results.add(map); } return results; } Map> createResultsMap( List> results) => new Map>.fromIterable(results, key: (dynamic result) => "${result["configuration"]}:${result["name"]}"); Map> parseResultsMap(String contents) => createResultsMap(parseResults(contents)); Future>> loadResultsMap(String path) async => createResultsMap(await loadResults(path));