dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Delete ABI infrastructure

Browse Source 
Change-Id: I26d455d96c87010293a6dcb9f6c0eecfcaf3c816
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/157102
Reviewed-by: Alexander Thomas <athom@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Liam Appelbe <liama@google.com>
This commit is contained in:
Liam Appelbe
2020-08-06 19:55:12 +00:00
committed by commit-bot@chromium.org
parent 507ffcfdaf
commit 79cec0e864
23 changed files with 51 additions and 652 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DEPS
-6
View File
@@ -677,12 +677,6 @@ hooks = [
'pattern': '.',
'action': ['python', 'sdk/build/vs_toolchain.py', 'update'],
},
{
# Download dill files for all supported ABI versions, if necessary.
'name': 'abiversions',
'pattern': '.',
'action': ['python', 'sdk/tools/download_abi_dills.py'],
},
]
hooks_os = {
pkg/test_runner/lib/src/command.dart
+3 -11
View File
@@ -340,24 +340,21 @@ class FastaCompilationCommand extends CompilationCommand {
}
class VMKernelCompilationCommand extends CompilationCommand {
final List<String> batchArgs;
VMKernelCompilationCommand(
String outputFile,
List<Uri> bootstrapDependencies,
String executable,
List<String> arguments,
Map<String, String> environmentOverrides,
this.batchArgs,
{bool alwaysCompile,
int index = 0})
: super('vm_compile_to_kernel $batchArgs', outputFile,
bootstrapDependencies, executable, arguments, environmentOverrides,
: super('vm_compile_to_kernel', outputFile, bootstrapDependencies,
executable, arguments, environmentOverrides,
alwaysCompile: alwaysCompile, index: index);
VMKernelCompilationCommand indexedCopy(int index) =>
VMKernelCompilationCommand(outputFile, _bootstrapDependencies, executable,
arguments, environmentOverrides, batchArgs,
arguments, environmentOverrides,
alwaysCompile: _alwaysCompile, index: index);
VMKernelCompilationCommandOutput createOutput(
@@ -372,11 +369,6 @@ class VMKernelCompilationCommand extends CompilationCommand {
this, exitCode, timedOut, stdout, stderr, time, compilationSkipped);
int get maxNumRetries => 1;
@override
List<String> get batchArguments {
return batchArgs;
}
}
/// This is just a Pair(String, Map) class with hashCode and operator ==
pkg/test_runner/lib/src/compiler_configuration.dart
+2 -11
View File
@@ -1080,15 +1080,8 @@ abstract class VMKernelCompilerMixin {
var pkgVmDir = Platform.script.resolve('../../../pkg/vm').toFilePath();
var genKernel = '$pkgVmDir/tool/gen_kernel$shellScriptExtension';
var useAbiVersion = arguments.firstWhere(
(arg) => arg.startsWith('--use-abi-version='),
orElse: () => null);
var kernelBinariesFolder = '${_configuration.buildDirectory}';
if (useAbiVersion != null) {
var version = useAbiVersion.split('=')[1];
kernelBinariesFolder += '/dart-sdk/lib/_internal/abiversions/$version';
} else if (_useSdk) {
if (_useSdk) {
kernelBinariesFolder += '/dart-sdk/lib/_internal';
}
@@ -1119,10 +1112,8 @@ abstract class VMKernelCompilerMixin {
..._configuration.genKernelOptions,
];
var batchArgs = [if (useAbiVersion != null) useAbiVersion];
return VMKernelCompilationCommand(dillFile, bootstrapDependencies(),
genKernel, args, environmentOverrides, batchArgs,
genKernel, args, environmentOverrides,
alwaysCompile: true);
}
}
pkg/vm/tool/gen_kernel
+1 -34
View File
@@ -8,25 +8,6 @@
set -e
ABI_VERSION=""
HAS_PLATFORM=""
ARGV=()
for arg in "$@"; do
case $arg in
--use-abi-version=*)
ABI_VERSION="$(echo "$arg" | sed "s|--use-abi-version=||")"
;;
--platform*)
HAS_PLATFORM="TRUE"
ARGV+=("$arg")
;;
*)
ARGV+=("$arg")
;;
esac
done
function follow_links() {
file="$1"
while [ -h "$file" ]; do
@@ -54,18 +35,4 @@ else
OUT_DIR="$SDK_DIR/out"
fi
export DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64}
BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
if [ $ABI_VERSION ]; then
ABI_DIR="$BIN_DIR/dart-sdk/lib/_internal/abiversions/$ABI_VERSION"
PLATFORM=()
if [ -z $HAS_PLATFORM ]; then
PLATFORM+=("--platform" "$ABI_DIR/vm_platform_strong.dill")
fi
exec "$BIN_DIR/dart" $DART_VM_FLAGS --enable-interpreter \
"$ABI_DIR/gen_kernel_bytecode.dill" "${PLATFORM[@]}" "${ARGV[@]}"
else
exec "$DART" $DART_VM_FLAGS \
"${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" "${ARGV[@]}"
fi
exec "$DART" $DART_VM_FLAGS "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@
runtime/bin/BUILD.gn
+1 -25
View File
@@ -160,7 +160,7 @@ template("build_gen_snapshot") {
"..:dart_config",
"..:dart_precompiler_config",
] + extra_configs
deps = [ ":generate_abi_version_cc_file" ] + extra_deps
deps = extra_deps
defines = [ "EXCLUDE_CFE_AND_KERNEL_PLATFORM" ]
@@ -178,7 +178,6 @@ template("build_gen_snapshot") {
"snapshot_utils.h",
# Very limited native resolver provided.
"$target_gen_dir/abi_version.cc",
"builtin_gen_snapshot.cc",
"dfe.cc",
"dfe.h",
@@ -728,25 +727,6 @@ source_set("dart_kernel_platform_cc") {
get_target_outputs(":platform_strong_dill_linkable")
}
action("generate_abi_version_cc_file") {
inputs = [
"../../tools/utils.py",
"../../tools/VERSION",
"abi_version_in.cc",
]
output = "$target_gen_dir/abi_version.cc"
outputs = [ output ]
script = "../../tools/make_version.py"
args = [
"--quiet",
"--output",
rebase_path(output, root_build_dir),
"--input",
rebase_path("abi_version_in.cc", root_build_dir),
]
}
template("dart_executable") {
use_product_mode = dart_runtime_mode == "release"
if (defined(invoker.use_product_mode)) {
@@ -803,7 +783,6 @@ template("dart_executable") {
deps = [
":crashpad",
":generate_abi_version_cc_file",
"//third_party/boringssl",
"//third_party/zlib",
]
@@ -840,7 +819,6 @@ template("dart_executable") {
"snapshot_utils.h",
"vmservice_impl.cc",
"vmservice_impl.h",
"$target_gen_dir/abi_version.cc",
] + extra_sources
if (is_win) {
@@ -1006,7 +984,6 @@ executable("run_vm_tests") {
":dart_kernel_platform_cc",
":dart_snapshot_cc",
":gen_kernel_bytecode_dill",
":generate_abi_version_cc_file",
":standalone_dart_io",
"..:libdart_precompiler",
"//third_party/zlib",
@@ -1043,7 +1020,6 @@ executable("run_vm_tests") {
heap_tests = rebase_path(heap_sources_tests, ".", "../vm/heap")
sources = [
"$target_gen_dir/abi_version.cc",
"builtin.cc",
"dfe.cc",
"dfe.h",
runtime/bin/abi_version.h
-18
View File
@@ -1,18 +0,0 @@
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_BIN_ABI_VERSION_H_
#define RUNTIME_BIN_ABI_VERSION_H_
namespace dart {
class AbiVersion {
public:
static int GetCurrent();
static int GetOldestSupported();
};
} // namespace dart
#endif // RUNTIME_BIN_ABI_VERSION_H_
runtime/bin/abi_version_in.cc
-17
View File
@@ -1,17 +0,0 @@
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "bin/abi_version.h"
namespace dart {
int AbiVersion::GetCurrent() {
return {{ABI_VERSION}};
}
int AbiVersion::GetOldestSupported() {
return {{OLDEST_SUPPORTED_ABI_VERSION}};
}
} // namespace dart
runtime/bin/builtin_impl_sources.gni
-1
View File
@@ -7,7 +7,6 @@
# io_impl_sources.gypi.
builtin_impl_sources = [
"abi_version.h",
"crypto.cc",
"crypto.h",
"crypto_android.cc",
runtime/bin/dfe.cc
+39 -89
View File
@@ -4,13 +4,11 @@
#include "bin/dfe.h"
#include "bin/abi_version.h"
#include "bin/dartutils.h"
#include "bin/directory.h"
#include "bin/error_exit.h"
#include "bin/exe_utils.h"
#include "bin/file.h"
#include "bin/main_options.h"
#include "bin/platform.h"
#include "bin/utils.h"
#include "include/dart_tools_api.h"
@@ -33,6 +31,28 @@ intptr_t kPlatformStrongDillSize = 0;
namespace dart {
namespace bin {
// The run_vm_tests binary has the DART_PRECOMPILER set in order to allow unit
// tests to exercise JIT and AOT pipeline.
//
// Only on X64 do we have kernel-service.dart.snapshot available otherwise we
// need to fall back to the built-in one (if we have it).
#if defined(EXCLUDE_CFE_AND_KERNEL_PLATFORM) || \
(defined(DART_PRECOMPILER) && defined(TARGET_ARCH_X64))
const uint8_t* kernel_service_dill = nullptr;
const intptr_t kernel_service_dill_size = 0;
#else
const uint8_t* kernel_service_dill = kKernelServiceDill;
const intptr_t kernel_service_dill_size = kKernelServiceDillSize;
#endif
#if defined(EXCLUDE_CFE_AND_KERNEL_PLATFORM)
const uint8_t* platform_strong_dill = nullptr;
const intptr_t platform_strong_dill_size = 0;
#else
const uint8_t* platform_strong_dill = kPlatformStrongDill;
const intptr_t platform_strong_dill_size = kPlatformStrongDillSize;
#endif
#if !defined(DART_PRECOMPILED_RUNTIME)
DFE dfe;
#endif
@@ -46,31 +66,6 @@ DFE::DFE()
frontend_filename_(nullptr),
application_kernel_buffer_(nullptr),
application_kernel_buffer_size_(0) {
// The run_vm_tests binary has the DART_PRECOMPILER set in order to allow unit
// tests to exercise JIT and AOT pipeline.
//
// Only on X64 do we have kernel-service.dart.snapshot available otherwise we
// need to fall back to the built-in one (if we have it).
#if defined(EXCLUDE_CFE_AND_KERNEL_PLATFORM) || \
(defined(DART_PRECOMPILER) && defined(TARGET_ARCH_X64))
kernel_service_dill_ = nullptr;
kernel_service_dill_size_ = 0;
#else
kernel_service_dill_ = kKernelServiceDill;
kernel_service_dill_size_ = kKernelServiceDillSize;
#endif
#if defined(EXCLUDE_CFE_AND_KERNEL_PLATFORM)
platform_strong_dill_for_compilation_ = nullptr;
platform_strong_dill_for_compilation_size_ = 0;
platform_strong_dill_for_execution_ = nullptr;
platform_strong_dill_for_execution_size_ = 0;
#else
platform_strong_dill_for_compilation_ = kPlatformStrongDill;
platform_strong_dill_for_compilation_size_ = kPlatformStrongDillSize;
platform_strong_dill_for_execution_ = kPlatformStrongDill;
platform_strong_dill_for_execution_size_ = kPlatformStrongDillSize;
#endif
}
DFE::~DFE() {
@@ -85,72 +80,28 @@ DFE::~DFE() {
}
void DFE::Init() {
Init(Options::kAbiVersionUnset);
}
void DFE::Init(int target_abi_version) {
if (platform_strong_dill_for_compilation_ == nullptr) {
if (platform_strong_dill == nullptr) {
return;
}
if (!InitKernelServiceAndPlatformDills(target_abi_version)) {
return;
}
Dart_SetDartLibrarySourcesKernel(platform_strong_dill_for_compilation_,
platform_strong_dill_for_compilation_size_);
InitKernelServiceAndPlatformDills();
Dart_SetDartLibrarySourcesKernel(platform_strong_dill,
platform_strong_dill_size);
}
bool DFE::InitKernelServiceAndPlatformDills(int target_abi_version) {
const char kAbiVersionsDir[] = "dart-sdk/lib/_internal/abiversions";
const char kKernelServiceDillFile[] = "kernel_service.dill";
const char kPlatformStrongDillFile[] = "vm_platform_strong.dill";
void DFE::InitKernelServiceAndPlatformDills() {
if (frontend_filename_ != nullptr) {
return true;
return;
}
// |dir_prefix| includes the last path seperator.
auto dir_prefix = EXEUtils::GetDirectoryPrefixFromExeName();
if (target_abi_version != Options::kAbiVersionUnset) {
kernel_service_dill_ = nullptr;
kernel_service_dill_size_ = 0;
platform_strong_dill_for_compilation_ = nullptr;
platform_strong_dill_for_compilation_size_ = 0;
// Look in the old abi version directory.
char* script_uri =
Utils::SCreate("%s%s/%d/%s", dir_prefix.get(), kAbiVersionsDir,
target_abi_version, kPlatformStrongDillFile);
if (!TryReadKernelFile(
script_uri,
const_cast<uint8_t**>(&platform_strong_dill_for_compilation_),
&platform_strong_dill_for_compilation_size_)) {
Syslog::PrintErr("Can't find old ABI dill file: %s\n", script_uri);
free(script_uri);
return false;
}
free(script_uri);
script_uri = Utils::SCreate("%s%s/%d/%s", dir_prefix.get(), kAbiVersionsDir,
target_abi_version, kKernelServiceDillFile);
if (!TryReadKernelFile(script_uri,
const_cast<uint8_t**>(&kernel_service_dill_),
&kernel_service_dill_size_)) {
Syslog::PrintErr("Can't find old ABI dill file: %s\n", script_uri);
free(script_uri);
return false;
} else {
frontend_filename_ = script_uri;
return true;
}
}
// Look for the frontend snapshot next to the executable.
frontend_filename_ =
Utils::SCreate("%s%s", dir_prefix.get(), kKernelServiceSnapshot);
if (File::Exists(nullptr, frontend_filename_)) {
return true;
return;
}
free(frontend_filename_);
frontend_filename_ = nullptr;
@@ -161,31 +112,30 @@ bool DFE::InitKernelServiceAndPlatformDills(int target_abi_version) {
Utils::SCreate("%s%s%s%s", dir_prefix.get(), kSnapshotsDirectory,
File::PathSeparator(), kKernelServiceSnapshot);
if (File::Exists(nullptr, frontend_filename_)) {
return true;
return;
}
free(frontend_filename_);
frontend_filename_ = nullptr;
return true;
}
bool DFE::KernelServiceDillAvailable() const {
return kernel_service_dill_ != nullptr;
return kernel_service_dill != nullptr;
}
void DFE::LoadKernelService(const uint8_t** kernel_service_buffer,
intptr_t* kernel_service_buffer_size) {
*kernel_service_buffer = kernel_service_dill_;
*kernel_service_buffer_size = kernel_service_dill_size_;
*kernel_service_buffer = kernel_service_dill;
*kernel_service_buffer_size = kernel_service_dill_size;
}
void DFE::LoadPlatform(const uint8_t** kernel_buffer,
intptr_t* kernel_buffer_size) {
*kernel_buffer = platform_strong_dill_for_execution_;
*kernel_buffer_size = platform_strong_dill_for_execution_size_;
*kernel_buffer = platform_strong_dill;
*kernel_buffer_size = platform_strong_dill_size;
}
bool DFE::CanUseDartFrontend() const {
return (platform_strong_dill_for_compilation_ != nullptr) &&
return (platform_strong_dill != nullptr) &&
(KernelServiceDillAvailable() || (frontend_filename() != nullptr));
}
@@ -238,9 +188,9 @@ Dart_KernelCompilationResult DFE::CompileScript(const char* script_uri,
PathSanitizer path_sanitizer(script_uri);
const char* sanitized_uri = path_sanitizer.sanitized_uri();
return Dart_CompileToKernel(
sanitized_uri, platform_strong_dill_for_compilation_,
platform_strong_dill_for_compilation_size_, incremental, package_config);
return Dart_CompileToKernel(sanitized_uri, platform_strong_dill,
platform_strong_dill_size, incremental,
package_config);
}
void DFE::CompileAndReadScript(const char* script_uri,
runtime/bin/dfe.h
+1 -8
View File
@@ -24,7 +24,6 @@ class DFE {
// Call Init before Dart_Initialize to prevent races between the
// different isolates.
void Init();
void Init(int target_abi_version);
char* frontend_filename() const { return frontend_filename_; }
@@ -108,18 +107,12 @@ class DFE {
bool use_dfe_;
bool use_incremental_compiler_;
char* frontend_filename_;
const uint8_t* kernel_service_dill_;
intptr_t kernel_service_dill_size_;
const uint8_t* platform_strong_dill_for_compilation_;
intptr_t platform_strong_dill_for_compilation_size_;
const uint8_t* platform_strong_dill_for_execution_;
intptr_t platform_strong_dill_for_execution_size_;
// Kernel binary specified on the cmd line.
uint8_t* application_kernel_buffer_;
intptr_t application_kernel_buffer_size_;
bool InitKernelServiceAndPlatformDills(int target_abi_version);
void InitKernelServiceAndPlatformDills();
DISALLOW_COPY_AND_ASSIGN(DFE);
};
runtime/bin/main.cc
+1 -1
View File
@@ -1163,7 +1163,7 @@ void main(int argc, char** argv) {
// they might affect how the platform is loaded.
#if !defined(DART_PRECOMPILED_RUNTIME)
if (script_name != nullptr) {
dfe.Init(Options::target_abi_version());
dfe.Init();
uint8_t* application_kernel_buffer = NULL;
intptr_t application_kernel_buffer_size = 0;
dfe.ReadScript(script_name, &application_kernel_buffer,
runtime/bin/main_options.cc
-28
View File
@@ -8,7 +8,6 @@
#include <stdlib.h>
#include <string.h>
#include "bin/abi_version.h"
#include "bin/dartdev_isolate.h"
#include "bin/error_exit.h"
#include "bin/options.h"
@@ -373,33 +372,6 @@ bool Options::ProcessVMDebuggingOptions(const char* arg,
return false;
}
int Options::target_abi_version_ = Options::kAbiVersionUnset;
bool Options::ProcessAbiVersionOption(const char* arg,
CommandLineOptions* vm_options) {
const char* value = OptionProcessor::ProcessOption(arg, "--use_abi_version=");
if (value == NULL) {
return false;
}
int ver = 0;
for (int i = 0; value[i] != '\0'; ++i) {
if (value[i] >= '0' && value[i] <= '9') {
ver = (ver * 10) + value[i] - '0';
} else {
Syslog::PrintErr("--use_abi_version must be an int\n");
return false;
}
}
if (ver < AbiVersion::GetOldestSupported() ||
ver > AbiVersion::GetCurrent()) {
Syslog::PrintErr("--use_abi_version must be between %d and %d inclusive\n",
AbiVersion::GetOldestSupported(),
AbiVersion::GetCurrent());
return false;
}
target_abi_version_ = ver;
return true;
}
bool Options::ProcessEnableExperimentOption(const char* arg,
CommandLineOptions* vm_options) {
const char* value =
runtime/bin/main_options.h
-5
View File
@@ -69,7 +69,6 @@ namespace bin {
V(ProcessEnvironmentOption) \
V(ProcessEnableVmServiceOption) \
V(ProcessObserveOption) \
V(ProcessAbiVersionOption) \
V(ProcessEnableExperimentOption) \
V(ProcessVMDebuggingOptions)
@@ -127,9 +126,6 @@ class Options {
static const char* vm_service_server_ip() { return vm_service_server_ip_; }
static int vm_service_server_port() { return vm_service_server_port_; }
static constexpr int kAbiVersionUnset = -1;
static int target_abi_version() { return target_abi_version_; }
#if !defined(DART_PRECOMPILED_RUNTIME)
static DFE* dfe() { return dfe_; }
static void set_dfe(DFE* dfe) { dfe_ = dfe; }
@@ -178,7 +174,6 @@ class Options {
int default_port,
const char* default_ip);
static int target_abi_version_;
static MallocGrowableArray<const char*> enabled_experiments_;
#define OPTION_FRIEND(flag, variable) friend class OptionProcessor_##flag;
runtime/vm/version.h
-2
View File
@@ -14,8 +14,6 @@ class Version : public AllStatic {
static const char* String();
static const char* SnapshotString();
static const char* CommitString();
static int CurrentAbiVersion();
static int OldestSupportedAbiVersion();
static const char* SdkHash();
private:
runtime/vm/version_in.cc
-9
View File
@@ -5,7 +5,6 @@
#include "vm/version.h"
#include "vm/cpu.h"
#include "vm/flags.h"
#include "vm/os.h"
namespace dart {
@@ -30,14 +29,6 @@ const char* Version::CommitString() {
return commit_;
}
int Version::CurrentAbiVersion() {
return {{ABI_VERSION}};
}
int Version::OldestSupportedAbiVersion() {
return {{OLDEST_SUPPORTED_ABI_VERSION}};
}
const char* Version::SdkHash() {
return git_short_hash_;
}
sdk/BUILD.gn
-14
View File
@@ -616,12 +616,6 @@ copy("copy_vm_dill_files") {
outputs = [ "$root_out_dir/dart-sdk/lib/_internal/{{source_file_part}}" ]
}
copy("copy_abi_dill_files") {
visibility = [ ":create_sdk_with_abi_versions" ]
sources = [ "../tools/abiversions" ]
outputs = [ "$root_out_dir/dart-sdk/lib/_internal/abiversions" ]
}
copy("copy_dart2js_dill_files") {
visibility = [ ":create_full_sdk" ]
deps = [
@@ -968,11 +962,3 @@ group("create_sdk") {
public_deps += [ ":create_full_sdk" ]
}
}
# Same as create_sdk, but with abi version files.
group("create_sdk_with_abi_versions") {
public_deps = [
":copy_abi_dill_files",
":create_sdk",
]
}
tools/VERSION
-8
View File
@@ -23,17 +23,9 @@
# * Making cherry-picks to stable channel
# - increase PATCH by 1
#
# * Making a change to the ABI:
# - increase ABI_VERSION by 1
#
# * Deprecating an old ABI version:
# - increase OLDEST_SUPPORTED_ABI_VERSION to the version that is supported.
#
CHANNEL be
MAJOR 2
MINOR 10
PATCH 0
PRERELEASE 0
PRERELEASE_PATCH 0
ABI_VERSION 39
OLDEST_SUPPORTED_ABI_VERSION 39
tools/bots/test_matrix.json
-8
View File
@@ -2695,14 +2695,6 @@
"arguments": [
"api_docs"
]
},
{
"name": "upload abi dills",
"script": "tools/upload_abi_dills.sh",
"arguments": [
"tools/VERSION",
"out/ReleaseX64"
]
}
]
},
tools/download_abi_dills.py
-50
View File
@@ -1,50 +0,0 @@
# Downloads dill files from CIPD for each supported ABI version.
import os
import subprocess
import sys
import utils
def procWait(p):
while p.returncode is None:
p.communicate()
p.poll()
return p.returncode
def findAbiVersion(version):
cmd = ['cipd', 'instances', 'dart/abiversions/%d' % version]
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=utils.IsWindows(),
cwd=utils.DART_DIR)
return procWait(p) == 0
def main():
abi_version = int(utils.GetAbiVersion())
oldest_abi_version = int(utils.GetOldestSupportedAbiVersion())
cmd = ['cipd', 'ensure', '-root', 'tools/abiversions', '-ensure-file', '-']
ensure_file = ''
for i in range(oldest_abi_version, abi_version + 1):
if findAbiVersion(i):
ensure_file += '@Subdir %d\ndart/abiversions/%d latest\n\n' % (i, i)
if not ensure_file:
return 0
p = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=utils.IsWindows(),
cwd=utils.DART_DIR)
p.communicate(ensure_file)
p.stdin.close()
return procWait(p)
if __name__ == '__main__':
sys.exit(main())
tools/make_version.py
-8
View File
@@ -108,14 +108,6 @@ def FormatVersionString(version,
version_time = 'Unknown timestamp'
version = version.replace('{{COMMIT_TIME}}', version_time.decode('utf-8'))
abi_version = utils.GetAbiVersion(version_file)
version = version.replace('{{ABI_VERSION}}', abi_version)
oldest_supported_abi_version = utils.GetOldestSupportedAbiVersion(
version_file)
version = version.replace('{{OLDEST_SUPPORTED_ABI_VERSION}}',
oldest_supported_abi_version)
snapshot_hash = MakeSnapshotHashString()
version = version.replace('{{SNAPSHOT_HASH}}', snapshot_hash)
tools/run_abi_tests.py
-215
View File
@@ -1,215 +0,0 @@
# Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Runs tests with old ABI versions and check if the results differ from the
# current results.
import argparse
import json
import os
import subprocess
import sys
import time
import utils
scriptDir = os.path.dirname(os.path.realpath(__file__))
outDir = os.path.join(scriptDir, '..', 'out', 'ReleaseX64')
abiDir = os.path.join(outDir, 'dart-sdk', 'lib', '_internal', 'abiversions')
# Parse command line args to flags.
def parseArgs():
parser = argparse.ArgumentParser(
'Runs test.py on all supported ABI versions')
parser.add_argument(
'--output-directory',
default=os.path.join(outDir, 'logs'),
metavar='DIR',
dest='logDir',
help='Directory to output results.json and logs.json to.')
parser.add_argument(
'-n',
metavar='NAME',
dest='configuration_name',
help='Name of the configuration to use in the results.')
return parser.parse_args()
# Info about a running test.
class Test:
def __init__(self, cmd, resultFile, logFile, version):
self.cmd = cmd # The test command.
self.resultFile = resultFile # The expected location of the result file.
self.logFile = logFile # The expected location of the log file.
self.version = version # The ABI version, or None.
# Recursively make directories for the path.
def makeDirs(path):
try:
os.makedirs(path)
except OSError:
pass
# Build a Test object for the given version (or None).
def buildTest(version):
testDir = os.path.join(outDir,
'test%s' % ('' if version is None else str(version)))
logDir = os.path.join(testDir, 'logs')
makeDirs(logDir)
vm_options = ['--enable-interpreter']
if version is not None:
vm_options += ['--use-abi-version=%d' % version]
cmd = [
'python',
os.path.join(scriptDir, 'test.py'),
'--compiler=dartkb',
'--mode=release',
'--write-results',
'--write-logs',
'--output_directory=%s' % logDir,
'--vm-options=%s' % ' '.join(vm_options),
'lib_2',
]
resultFile = os.path.join(logDir, 'results.json')
logFile = os.path.join(logDir, 'logs.json')
return Test(cmd, resultFile, logFile, version)
# Returns whether the dill files exist for an ABI version.
def abiVersionExists(version):
return os.path.isdir(os.path.join(abiDir, str(version)))
# Build tests for every supported version, and return a list of Test objects.
def buildAllTests():
abi_version = int(utils.GetAbiVersion())
oldest_abi_version = int(utils.GetOldestSupportedAbiVersion())
tests = [buildTest(None)]
for version in xrange(oldest_abi_version, abi_version + 1):
if abiVersionExists(version):
tests.append(buildTest(version))
return tests
# Run all tests, one by one, and wait for them all to complete.
def runAllTests(tests):
for test in tests:
print('\n\n\n=== Running tests %s ===' % (
('for ABI version %d' % test.version)
if test.version is not None else ('without an ABI version')))
print(subprocess.list2cmdline(test.cmd) + '\n\n')
proc = subprocess.Popen(test.cmd)
while proc.returncode is None:
time.sleep(1)
proc.communicate()
proc.poll()
# Read a test result file or log file and convert JSON lines to a dictionary of
# JSON records indexed by name. Assumes result and log files both use name key.
def readTestFile(fileName):
with open(fileName, 'r') as f:
return {r['name']: r for r in [json.loads(line) for line in f]}
# Read the test result or log files for every version and return a dict like:
# {name: {version: resultJson, ...}, ...}
def readAllTestFiles(tests, nameGetter):
allRecords = {}
for test in tests:
records = readTestFile(nameGetter(test))
for name, result in records.items():
if name not in allRecords:
allRecords[name] = {}
allRecords[name][test.version] = result
return allRecords
# Pick any element of the dictionary, favoring the None key if it exists.
def pickOne(d):
if None in d:
return d[None]
for v in d.values():
return v
return None
# Diff the results of a test for each version and construct a new test result
# that reports whether the test results match for each version.
def diffResults(results, configuration_name):
outResult = pickOne(results)
exp = results[None]['result'] if None in results else None
outResult['configuration'] = configuration_name
outResult['expected'] = exp
outResult['result'] = exp
outResult['matches'] = True
diffs = []
for version, result in results.items():
if version is not None:
act = result['result']
if exp != act:
diffs.append(version)
outResult[
'result'] = act # May be overwritten by other versions.
outResult['matches'] = False
return outResult, diffs
# Create a log entry for a test that has diffs. Concatenate all the log records
# and include which tests failed.
def makeLog(diffs, results, logRecords, configuration_name):
result = pickOne(results)
logs = ["%s: %s" % (str(v), l['log']) for v, l in logRecords.items()]
log = ('This test fails if there is a difference in the test results\n'
'between ABI versions. The expected result is the result on the\n'
'current ABI: %s\n'
'These ABI versions reported a different result: %s\n\n'
'These are the logs of the test runs on different ABI versions.\n'
'There are no logs for versions where the test passed.\n\n%s' %
(result['result'], repr(diffs), '\n\n\n'.join(logs)))
return {
'name': result['name'],
'configuration': configuration_name,
'result': result['result'],
'log': log,
}
# Diff the results of all the tests and create the merged result and log files.
def diffAllResults(tests, flags):
allResults = readAllTestFiles(tests, lambda test: test.resultFile)
allLogs = readAllTestFiles(tests, lambda test: test.logFile)
makeDirs(flags.logDir)
resultFileName = os.path.join(flags.logDir, 'results.json')
logFileName = os.path.join(flags.logDir, 'logs.json')
with open(resultFileName, 'w') as resultFile:
with open(logFileName, 'w') as logFile:
for name, results in allResults.items():
outResult, diffs = diffResults(results,
flags.configuration_name)
resultFile.write(json.dumps(outResult) + '\n')
if diffs:
logRecords = allLogs.get(name, {})
logFile.write(
json.dumps(
makeLog(diffs, results, logRecords, flags.
configuration_name)) + '\n')
print('Log files emitted to %s and %s' % (resultFileName, logFileName))
def main():
flags = parseArgs()
tests = buildAllTests()
runAllTests(tests)
diffAllResults(tests, flags)
return 0
if __name__ == '__main__':
sys.exit(main())
tools/upload_abi_dills.sh
-64
View File
@@ -1,64 +0,0 @@
#!/usr/bin/env bash
# Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Uploads the following dill files to CIPD, indexed by the current ABI version:
# $build_dir/vm_platform_strong.dill
# $build_dir/gen/kernel_service.dill
# $build_dir/gen_kernel_bytecode.dill
# This script is a no-op unless $BUILDBOT_BUILDERNAME is "dart-sdk-linux-be".
# It's also a no-op if dill files were already uploaded today.
#
# If the ABI was modified, the ABI_VERSION in tools/VERSIONS should be manually
# incremented accordingly.
set -e
set -x
if [ -z "$2" ]; then
echo "Usage: upload_abi_dills.sh version_file build_dir"
exit 1
fi
if [ "$BUILDBOT_BUILDERNAME" != "dart-sdk-linux-be" ]; then
echo "This script only works on the dart-sdk-linux-be buildbot"
exit 0
fi
abi_version=$(sed -n "s/^ABI_VERSION \([0-9]*\)$/\1/p" "$1")
git_revision=$(git rev-parse HEAD)
current_date=$(date +%F)
search_results=$(cipd search \
"dart/abiversions/$abi_version" \
-tag "date:$current_date" | grep "Instances:" || echo "")
if [ ! -z "$search_results" ]; then
exit 0
fi
sdk_dir=$(pwd)
tmpdir=$(mktemp -d)
chmod 755 $tmpdir
cleanup() {
rm -rf "$tmpdir"
}
trap cleanup EXIT HUP INT QUIT TERM PIPE
pushd "$tmpdir"
mkdir abiversions
cp "$sdk_dir/$2/vm_platform_strong.dill" "abiversions/vm_platform_strong.dill"
cp "$sdk_dir/$2/gen/kernel_service.dill" "abiversions/kernel_service.dill"
cp "$sdk_dir/$2/gen_kernel_bytecode.dill" "abiversions/gen_kernel_bytecode.dill"
cipd create \
-name dart/abiversions/$abi_version \
-in abiversions \
-install-mode copy \
-tag version:$abi_version \
-tag date:$current_date \
-tag git_revision:$git_revision \
-ref latest \
-ref version_$abi_version
popd
tools/utils.py
+3 -20
View File
@@ -114,15 +114,13 @@ def GetMinidumpUtils(repo_path=DART_DIR):
class Version(object):
def __init__(self, channel, major, minor, patch, prerelease,
prerelease_patch, abi_version, oldest_supported_abi_version):
prerelease_patch):
self.channel = channel
self.major = major
self.minor = minor
self.patch = patch
self.prerelease = prerelease
self.prerelease_patch = prerelease_patch
self.abi_version = abi_version
self.oldest_supported_abi_version = oldest_supported_abi_version
# Try to guess the host operating system.
@@ -410,16 +408,6 @@ def GetUserName():
return os.environ.get(key, '')
def GetAbiVersion(version_file=None):
version = ReadVersionFile(version_file)
return version.abi_version
def GetOldestSupportedAbiVersion(version_file=None):
version = ReadVersionFile(version_file)
return version.oldest_supported_abi_version
def ReadVersionFile(version_file=None):
def match_against(pattern, file_content):
@@ -445,15 +433,10 @@ def ReadVersionFile(version_file=None):
patch = match_against('^PATCH (\d+)$', content)
prerelease = match_against('^PRERELEASE (\d+)$', content)
prerelease_patch = match_against('^PRERELEASE_PATCH (\d+)$', content)
abi_version = match_against('^ABI_VERSION (\d+)$', content)
oldest_supported_abi_version = match_against(
'^OLDEST_SUPPORTED_ABI_VERSION (\d+)$', content)
if (channel and major and minor and prerelease and prerelease_patch and
abi_version and oldest_supported_abi_version):
if (channel and major and minor and prerelease and prerelease_patch):
return Version(channel, major, minor, patch, prerelease,
prerelease_patch, abi_version,
oldest_supported_abi_version)
prerelease_patch)
print('Warning: VERSION file ({}) has wrong format'.format(version_file))
return None