From edde575dcd6eb98bc7735693b2bfd92cd2a552f0 Mon Sep 17 00:00:00 2001 From: Clement Skau Date: Fri, 26 Jun 2020 13:35:42 +0000 Subject: [PATCH] [SDK] Adds an SDK hash to kernels and the VM. Adds a new SDK hash to kernels and the VM which is optionally checked to verify kernels are built for the same SDK as the VM. This helps catch incompatibilities that are currently causing subtle bugs and (not so subtle) crashes. The SDK hash is encoded in kernels as a new field in components. The hash is derived from the 10 byte git short hash. This new check can be disabled via: tools/gn.py ... --no-verify-sdk-hash This CL bumps the min. (and max.) supported kernel format version, making the VM backwards incompatible from this point back. Bug: https://github.com/dart-lang/sdk/issues/41802 Change-Id: I3cbb2d481239ee64dafdaa0e4aac36c80281931b Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/150343 Commit-Queue: Clement Skau Reviewed-by: Jens Johansen Reviewed-by: Martin Kustermann --- DEPS | 2 +- pkg/front_end/test/binary_md_dill_reader.dart | 50 +++-- .../test/spell_checking_list_code.txt | 4 + pkg/kernel/binary.md | 3 +- pkg/kernel/lib/binary/ast_from_binary.dart | 28 ++- pkg/kernel/lib/binary/ast_to_binary.dart | 2 + pkg/kernel/lib/binary/tag.dart | 26 ++- runtime/bin/BUILD.gn | 2 + runtime/tests/vm/dart/sdk_hash_test.dart | 88 ++++++++ .../tests/vm/dart/snapshot_test_helper.dart | 1 + runtime/tests/vm/dart_2/sdk_hash_test.dart | 88 ++++++++ .../tests/vm/dart_2/snapshot_test_helper.dart | 1 + .../frontend/kernel_translation_helper.cc | 13 +- runtime/vm/kernel_binary.cc | 38 ++-- runtime/vm/kernel_binary.h | 10 +- runtime/vm/kernel_loader.cc | 15 +- runtime/vm/version.h | 2 + runtime/vm/version_in.cc | 5 + sdk_args.gni | 18 ++ tools/gn.py | 11 +- tools/make_version.py | 212 +++++++++--------- tools/utils.py | 18 +- utils/application_snapshot.gni | 2 + utils/compile_platform.gni | 3 + utils/dartanalyzer/BUILD.gn | 1 + utils/dartdevc/BUILD.gn | 8 + utils/kernel-service/BUILD.gn | 2 + 27 files changed, 474 insertions(+), 179 deletions(-) create mode 100644 runtime/tests/vm/dart/sdk_hash_test.dart create mode 100644 runtime/tests/vm/dart_2/sdk_hash_test.dart diff --git a/DEPS b/DEPS index 6f5047ec7ec..8e6ad1795ce 100644 --- a/DEPS +++ b/DEPS @@ -512,7 +512,7 @@ deps = { "packages": [ { "package": "dart/cfe/dart2js_dills", - "version": "binary_version:42", + "version": "binary_version:43_2", } ], "dep_type": "cipd", diff --git a/pkg/front_end/test/binary_md_dill_reader.dart b/pkg/front_end/test/binary_md_dill_reader.dart index 01cf4d98d44..c0a4b5b7fbf 100644 --- a/pkg/front_end/test/binary_md_dill_reader.dart +++ b/pkg/front_end/test/binary_md_dill_reader.dart @@ -473,31 +473,33 @@ class BinaryMdDillReader { type = type.substring(0, type.indexOf("[")); type = _lookupGenericType(typeNames, type, types); - int intCount = -1; - if (vars[count] != null && vars[count] is int) { - intCount = vars[count]; - } else if (count.contains(".")) { - List countData = - count.split(regExpSplit).map((s) => s.trim()).toList(); - if (vars[countData[0]] != null) { - dynamic v = vars[countData[0]]; - if (v is Map && - countData[1] == "last" && - v["items"] is List && - v["items"].last is int) { - intCount = v["items"].last; - } else if (v is Map && v[countData[1]] != null) { - v = v[countData[1]]; - if (v is Map && v[countData[2]] != null) { - v = v[countData[2]]; - if (v is int) intCount = v; - } else if (v is int && - countData.length == 4 && - countData[2] == "+") { - intCount = v + int.parse(countData[3]); + int intCount = int.tryParse(count) ?? -1; + if (intCount == -1) { + if (vars[count] != null && vars[count] is int) { + intCount = vars[count]; + } else if (count.contains(".")) { + List countData = + count.split(regExpSplit).map((s) => s.trim()).toList(); + if (vars[countData[0]] != null) { + dynamic v = vars[countData[0]]; + if (v is Map && + countData[1] == "last" && + v["items"] is List && + v["items"].last is int) { + intCount = v["items"].last; + } else if (v is Map && v[countData[1]] != null) { + v = v[countData[1]]; + if (v is Map && v[countData[2]] != null) { + v = v[countData[2]]; + if (v is int) intCount = v; + } else if (v is int && + countData.length == 4 && + countData[2] == "+") { + intCount = v + int.parse(countData[3]); + } + } else { + throw "Unknown dot to int ($count)"; } - } else { - throw "Unknown dot to int ($count)"; } } } diff --git a/pkg/front_end/test/spell_checking_list_code.txt b/pkg/front_end/test/spell_checking_list_code.txt index 190fa6af9b9..d27f9e7a645 100644 --- a/pkg/front_end/test/spell_checking_list_code.txt +++ b/pkg/front_end/test/spell_checking_list_code.txt @@ -313,6 +313,7 @@ disallow disambiguator disjoint dispatched +distribute divided dmitryas doc @@ -325,6 +326,7 @@ downloaded downloading dq dquote +dsdk dst dummy dupdate @@ -435,12 +437,14 @@ futured futureor g gardening +gen generation gets getter1a getter1b getting gft +git github glb glob diff --git a/pkg/kernel/binary.md b/pkg/kernel/binary.md index 2b37b2c55e2..72e2057d80a 100644 --- a/pkg/kernel/binary.md +++ b/pkg/kernel/binary.md @@ -143,7 +143,8 @@ type CanonicalName { type ComponentFile { UInt32 magic = 0x90ABCDEF; - UInt32 formatVersion = 42; + UInt32 formatVersion = 43; + Byte[10] shortSdkHash; List problemsAsJson; // Described in problems.md. Library[] libraries; UriSource sourceMap; diff --git a/pkg/kernel/lib/binary/ast_from_binary.dart b/pkg/kernel/lib/binary/ast_from_binary.dart index 9a32677b497..b5e70b69d07 100644 --- a/pkg/kernel/lib/binary/ast_from_binary.dart +++ b/pkg/kernel/lib/binary/ast_from_binary.dart @@ -5,6 +5,7 @@ library kernel.ast_from_binary; import 'dart:core' hide MapEntry; import 'dart:developer'; +import 'dart:convert'; import 'dart:typed_data'; import '../ast.dart'; @@ -28,11 +29,22 @@ class InvalidKernelVersionError { InvalidKernelVersionError(this.version); String toString() { - return 'Unexpected Kernel version ${version} ' + return 'Unexpected Kernel Format Version ${version} ' '(expected ${Tag.BinaryFormatVersion}).'; } } +class InvalidKernelSdkVersionError { + final String version; + + InvalidKernelSdkVersionError(this.version); + + String toString() { + return 'Unexpected Kernel SDK Version ${version} ' + '(expected ${expectedSdkHash}).'; + } +} + class CompilationModeError { final String message; @@ -484,6 +496,13 @@ class BinaryBuilder { if (_bytes.length == 0) throw new StateError("Empty input given."); } + void _readAndVerifySdkHash() { + final sdkHash = ascii.decode(readBytes(sdkHashLength)); + if (!isValidSdkHash(sdkHash)) { + throw InvalidKernelSdkVersionError(sdkHash); + } + } + /// Deserializes a kernel component and stores it in [component]. /// /// When linking with a non-empty component, canonical names must have been @@ -511,6 +530,9 @@ class BinaryBuilder { if (version != Tag.BinaryFormatVersion) { throw InvalidKernelVersionError(version); } + + _readAndVerifySdkHash(); + _byteOffset = offset; List componentFileSizes = _indexComponents(); if (componentFileSizes.length > 1) { @@ -694,6 +716,8 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } + _readAndVerifySdkHash(); + // Read component index from the end of this ComponentFiles serialized data. _ComponentIndex index = _readComponentIndex(componentFileSize); @@ -718,6 +742,8 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } + _readAndVerifySdkHash(); + List problemsAsJson = readListOfStrings(); if (problemsAsJson != null) { component.problemsAsJson ??= []; diff --git a/pkg/kernel/lib/binary/ast_to_binary.dart b/pkg/kernel/lib/binary/ast_to_binary.dart index 57ccd2fd17e..dc3be1e21b9 100644 --- a/pkg/kernel/lib/binary/ast_to_binary.dart +++ b/pkg/kernel/lib/binary/ast_to_binary.dart @@ -4,6 +4,7 @@ library kernel.ast_to_binary; import 'dart:core' hide MapEntry; +import 'dart:convert'; import 'dart:developer'; import 'dart:io' show BytesBuilder; import 'dart:typed_data'; @@ -537,6 +538,7 @@ class BinaryPrinter implements Visitor, BinarySink { final componentOffset = getBufferOffset(); writeUInt32(Tag.ComponentFile); writeUInt32(Tag.BinaryFormatVersion); + writeBytes(ascii.encode(expectedSdkHash)); writeListOfStrings(component.problemsAsJson); indexLinkTable(component); _collectMetadata(component); diff --git a/pkg/kernel/lib/binary/tag.dart b/pkg/kernel/lib/binary/tag.dart index cec230824aa..b35dde9bc90 100644 --- a/pkg/kernel/lib/binary/tag.dart +++ b/pkg/kernel/lib/binary/tag.dart @@ -149,7 +149,7 @@ class Tag { /// Internal version of kernel binary format. /// Bump it when making incompatible changes in kernel binaries. /// Keep in sync with runtime/vm/kernel_binary.h, pkg/kernel/binary.md. - static const int BinaryFormatVersion = 42; + static const int BinaryFormatVersion = 43; } abstract class ConstantTag { @@ -169,3 +169,27 @@ abstract class ConstantTag { static const int UnevaluatedConstant = 12; // 13 is occupied by [SetConstant] } + +const int sdkHashLength = 10; // Bytes, a Git "short hash". + +const String sdkHashNull = '0000000000'; + +// Will be correct hash for Flutter SDK / Dart SDK we distribute. +// If non-null we will validate when consuming kernel, will use when producing +// kernel. +// If null, local development setting (e.g. run gen_kernel.dart from source), +// we put 0x00..00 into when producing, do not validate when consuming. +String get expectedSdkHash { + final sdkHash = + const String.fromEnvironment('sdk_hash', defaultValue: sdkHashNull); + if (sdkHash.length != 10) { + throw '-Dsdk_hash= must be a 10 byte string!'; + } + return sdkHash; +} + +bool isValidSdkHash(String sdkHash) { + return (sdkHash == sdkHashNull || + expectedSdkHash == sdkHashNull || + sdkHash == expectedSdkHash); +} diff --git a/runtime/bin/BUILD.gn b/runtime/bin/BUILD.gn index 0fd5acd5ab0..f90ddcabbd1 100644 --- a/runtime/bin/BUILD.gn +++ b/runtime/bin/BUILD.gn @@ -964,9 +964,11 @@ prebuilt_dart_action("gen_kernel_bytecode_dill") { abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] args = [ diff --git a/runtime/tests/vm/dart/sdk_hash_test.dart b/runtime/tests/vm/dart/sdk_hash_test.dart new file mode 100644 index 00000000000..f95bf813a48 --- /dev/null +++ b/runtime/tests/vm/dart/sdk_hash_test.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as path; +import 'package:expect/expect.dart'; + +import 'snapshot_test_helper.dart'; + +// Keep in sync with pkg/kernel/lib/binary/tag.dart: +const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; +const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; + +Future main(List args) async { + if (args.length == 1 && args[0] == '--child') { + print('Hello, SDK Hash!'); + return; + } + + final String sourcePath = + path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); + + await withTempDir((String tmp) async { + final String dillPath = path.join(tmp, 'test.dill'); + + { + final result = await Process.run(dart, [ + genKernel, + '--platform', + platformDill, + '-o', + dillPath, + sourcePath, + ]); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('', result.stdout); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!\n', result.stdout); + } + + // Invalidate the SDK hash in the kernel dill: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + // The SDK Hash is located after the ComponentFile and BinaryFormatVersion + // tags (both UInt32). + Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); + Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); + // Flip the first byte in the hash: + bytes[8] ^= bytes[8]; + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals( + 'Can\'t load Kernel binary: Invalid SDK hash.\n', result.stderr); + Expect.equals(253, result.exitCode); + Expect.equals('', result.stdout); + } + + // Zero out the SDK hash in the kernel dill to disable the check: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + bytes.setRange(8, 18, ascii.encode('0000000000')); + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!\n', result.stdout); + } + }); +} diff --git a/runtime/tests/vm/dart/snapshot_test_helper.dart b/runtime/tests/vm/dart/snapshot_test_helper.dart index 853fc3388e2..974050e1806 100644 --- a/runtime/tests/vm/dart/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart/snapshot_test_helper.dart @@ -48,6 +48,7 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); +final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/tests/vm/dart_2/sdk_hash_test.dart b/runtime/tests/vm/dart_2/sdk_hash_test.dart new file mode 100644 index 00000000000..f95bf813a48 --- /dev/null +++ b/runtime/tests/vm/dart_2/sdk_hash_test.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as path; +import 'package:expect/expect.dart'; + +import 'snapshot_test_helper.dart'; + +// Keep in sync with pkg/kernel/lib/binary/tag.dart: +const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; +const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; + +Future main(List args) async { + if (args.length == 1 && args[0] == '--child') { + print('Hello, SDK Hash!'); + return; + } + + final String sourcePath = + path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); + + await withTempDir((String tmp) async { + final String dillPath = path.join(tmp, 'test.dill'); + + { + final result = await Process.run(dart, [ + genKernel, + '--platform', + platformDill, + '-o', + dillPath, + sourcePath, + ]); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('', result.stdout); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!\n', result.stdout); + } + + // Invalidate the SDK hash in the kernel dill: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + // The SDK Hash is located after the ComponentFile and BinaryFormatVersion + // tags (both UInt32). + Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); + Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); + // Flip the first byte in the hash: + bytes[8] ^= bytes[8]; + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals( + 'Can\'t load Kernel binary: Invalid SDK hash.\n', result.stderr); + Expect.equals(253, result.exitCode); + Expect.equals('', result.stdout); + } + + // Zero out the SDK hash in the kernel dill to disable the check: + { + final myFile = File(dillPath); + Uint8List bytes = myFile.readAsBytesSync(); + bytes.setRange(8, 18, ascii.encode('0000000000')); + myFile.writeAsBytesSync(bytes); + } + + { + final result = await Process.run(dart, [dillPath, '--child']); + Expect.equals('', result.stderr); + Expect.equals(0, result.exitCode); + Expect.equals('Hello, SDK Hash!\n', result.stdout); + } + }); +} diff --git a/runtime/tests/vm/dart_2/snapshot_test_helper.dart b/runtime/tests/vm/dart_2/snapshot_test_helper.dart index 9be810a28e8..b2754d0bf9d 100644 --- a/runtime/tests/vm/dart_2/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart_2/snapshot_test_helper.dart @@ -48,6 +48,7 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); +final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/vm/compiler/frontend/kernel_translation_helper.cc b/runtime/vm/compiler/frontend/kernel_translation_helper.cc index 58407684162..93a9b490fca 100644 --- a/runtime/vm/compiler/frontend/kernel_translation_helper.cc +++ b/runtime/vm/compiler/frontend/kernel_translation_helper.cc @@ -2654,18 +2654,7 @@ TokenPosition KernelReaderHelper::ReadPosition() { } intptr_t KernelReaderHelper::SourceTableFieldCountFromFirstLibraryOffset() { - // translation_helper_.info() might not be initialized at this point so we - // can't use translation_helper_.info().kernel_binary_version(). - SetOffset(KernelFormatVersionOffset); - uint32_t formatVersion = reader_.ReadUInt32(); - intptr_t count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffsetPre41; - static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); - if (formatVersion >= 41) { - count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffset41Plus; - } - return count_from_first_library_offset; + return SourceTableFieldCountFromFirstLibraryOffset41Plus; } intptr_t KernelReaderHelper::SourceTableSize() { diff --git a/runtime/vm/kernel_binary.cc b/runtime/vm/kernel_binary.cc index 3720fe23ac7..469f7464799 100644 --- a/runtime/vm/kernel_binary.cc +++ b/runtime/vm/kernel_binary.cc @@ -15,6 +15,7 @@ #include "vm/kernel.h" #include "vm/object.h" #include "vm/os.h" +#include "vm/version.h" namespace dart { @@ -84,12 +85,14 @@ const char* kKernelInvalidBinaryFormatVersion = "Invalid kernel binary format version"; const char* kKernelInvalidSizeIndicated = "Invalid kernel binary: Indicated size is invalid"; +const char* kKernelInvalidSdkHash = "Invalid SDK hash"; std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { - if (reader->size() < 60) { - // A kernel file (v41) currently contains at least the following: + if (reader->size() < 70) { + // A kernel file (v43) currently contains at least the following: // * Magic number (32) // * Kernel version (32) + // * SDK Hash (10 * 8) // * List of problems (8) // * Length of source map (32) // * Length of canonical name table (8) @@ -98,7 +101,7 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // * Length of constant table (8) // * Component index (11 * 32) // - // so is at least 64 bytes. + // so is at least 74 bytes. // (Technically it will also contain an empty entry in both source map and // string table, taking up another 8 bytes.) if (error != nullptr) { @@ -124,6 +127,18 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { return nullptr; } + uint8_t sdkHash[11]; + reader->ReadBytes(sdkHash, 10); + sdkHash[10] = 0; // Null terminate. + if (strcmp(Version::SdkHash(), "0000000000") != 0 && + strcmp((const char*)sdkHash, "0000000000") != 0 && + strcmp((const char*)sdkHash, Version::SdkHash()) != 0) { + if (error != nullptr) { + *error = kKernelInvalidSdkHash; + } + return nullptr; + } + std::unique_ptr program(new Program()); program->binary_version_ = formatVersion; program->typed_data_ = reader->typed_data(); @@ -152,13 +167,8 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // Read backwards at the end. program->library_count_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd, 1, 0); - static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); intptr_t count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffsetPre41; - if (formatVersion >= 41) { - count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffset41Plus; - } + SourceTableFieldCountFromFirstLibraryOffset41Plus; program->source_table_offset_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd + 1 + program->library_count_ + 1 + @@ -171,13 +181,9 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { program->constant_table_offset_ = reader->ReadUInt32(); program->main_method_reference_ = NameIndex(reader->ReadUInt32() - 1); - if (formatVersion >= 41) { - NNBDCompiledMode compilation_mode = - static_cast(reader->ReadUInt32()); - program->compilation_mode_ = compilation_mode; - } else { - program->compilation_mode_ = NNBDCompiledMode::kDisabled; - } + NNBDCompiledMode compilation_mode = + static_cast(reader->ReadUInt32()); + program->compilation_mode_ = compilation_mode; return program; } diff --git a/runtime/vm/kernel_binary.h b/runtime/vm/kernel_binary.h index 11527eba3e4..33dc838a980 100644 --- a/runtime/vm/kernel_binary.h +++ b/runtime/vm/kernel_binary.h @@ -20,8 +20,8 @@ namespace kernel { static const uint32_t kMagicProgramFile = 0x90ABCDEFu; // Both version numbers are inclusive. -static const uint32_t kMinSupportedKernelFormatVersion = 29; -static const uint32_t kMaxSupportedKernelFormatVersion = 42; +static const uint32_t kMinSupportedKernelFormatVersion = 43; +static const uint32_t kMaxSupportedKernelFormatVersion = 43; // Keep in sync with package:kernel/lib/binary/tag.dart #define KERNEL_TAG_LIST(V) \ @@ -326,6 +326,12 @@ class Reader : public ValueObject { uint8_t PeekByte() { return buffer()[offset_]; } + void ReadBytes(uint8_t* buffer, uint8_t size) { + for (int i = 0; i < size; i++) { + buffer[i] = ReadByte(); + } + } + bool ReadBool() { return (ReadByte() & 1) == 1; } uint8_t ReadFlags() { return ReadByte(); } diff --git a/runtime/vm/kernel_loader.cc b/runtime/vm/kernel_loader.cc index 1d0d5437726..88b1c637792 100644 --- a/runtime/vm/kernel_loader.cc +++ b/runtime/vm/kernel_loader.cc @@ -297,7 +297,11 @@ Object& KernelLoader::LoadEntireProgram(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - std::unique_ptr subprogram = Program::ReadFrom(&reader); + const char* error = nullptr; + std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); + if (subprogram == nullptr) { + FATAL1("Failed to load kernel file: %s", error); + } ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), &uri_to_source_table); Object& load_result = Object::Handle(loader.LoadProgram(false)); @@ -849,7 +853,8 @@ void KernelLoader::FindModifiedLibraries(Program* program, intptr_t* p_num_classes, intptr_t* p_num_procedures) { LongJumpScope jump; - Zone* zone = Thread::Current()->zone(); + Thread* thread = Thread::Current(); + Zone* zone = thread->zone(); if (setjmp(*jump.Set()) == 0) { if (force_reload) { // If a reload is being forced we mark all libraries as having @@ -894,7 +899,11 @@ void KernelLoader::FindModifiedLibraries(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - std::unique_ptr subprogram = Program::ReadFrom(&reader); + const char* error = nullptr; + std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); + if (subprogram == nullptr) { + FATAL1("Failed to load kernel file: %s", error); + } ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), /*uri_to_source_table=*/nullptr); loader.walk_incremental_kernel(modified_libs, is_empty_program, diff --git a/runtime/vm/version.h b/runtime/vm/version.h index fdc6bb0f5bc..a3769f6ef44 100644 --- a/runtime/vm/version.h +++ b/runtime/vm/version.h @@ -16,11 +16,13 @@ class Version : public AllStatic { static const char* CommitString(); static int CurrentAbiVersion(); static int OldestSupportedAbiVersion(); + static const char* SdkHash(); private: static const char* str_; static const char* snapshot_hash_; static const char* commit_; + static const char* git_short_hash_; }; } // namespace dart diff --git a/runtime/vm/version_in.cc b/runtime/vm/version_in.cc index b4f1e2d6e11..f63fe3fde8a 100644 --- a/runtime/vm/version_in.cc +++ b/runtime/vm/version_in.cc @@ -38,8 +38,13 @@ int Version::OldestSupportedAbiVersion() { return {{OLDEST_SUPPORTED_ABI_VERSION}}; } +const char* Version::SdkHash() { + return git_short_hash_; +} + const char* Version::snapshot_hash_ = "{{SNAPSHOT_HASH}}"; const char* Version::str_ = "{{VERSION_STR}} ({{CHANNEL}}) ({{COMMIT_TIME}})"; const char* Version::commit_ = "{{VERSION_STR}}"; +const char* Version::git_short_hash_ = "{{GIT_HASH}}"; } // namespace dart diff --git a/sdk_args.gni b/sdk_args.gni index 94e6b7a8eed..06dc3e8540b 100644 --- a/sdk_args.gni +++ b/sdk_args.gni @@ -19,4 +19,22 @@ declare_args() { # to out/ReleaseX64/args.gn. The path above can be extracted from the `.git` # file under the git worktree folder. default_git_folder = "$_dart_root/.git" + + # Whether to enable the SDK hash check that will prevent loading a kernel + # into a VM which was built with a different SDK. + verify_sdk_hash = true +} + +# The SDK hash to build into VM and kernels. +# The value 0000000000 signifies no hash is set, which will disable the check. +if (verify_sdk_hash) { + sdk_hash = exec_script("$_dart_root/tools/make_version.py", + [ "--format={{GIT_HASH}}" ], + "trim string", + [ + "$_dart_root/tools/VERSION", + "$_dart_root/tools/utils.py", + ]) +} else { + sdk_hash = "0000000000" } diff --git a/tools/gn.py b/tools/gn.py index 830c8b3c64b..4bd29b452a5 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -136,7 +136,7 @@ def UseSysroot(args, gn_args): return True -def ToGnArgs(args, mode, arch, target_os, sanitizer): +def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash): gn_args = {} host_os = HostOsForGn(HOST_OS) @@ -258,6 +258,8 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer): gn_args['dart_debug_optimization_level'] = args.debug_opt_level gn_args['debug_optimization_level'] = args.debug_opt_level + gn_args['verify_sdk_hash'] = verify_sdk_hash + return gn_args @@ -466,6 +468,10 @@ def parse_args(args): default=False, dest='use_qemu', action='store_true') + other_group.add_argument('--no-verify-sdk-hash', + help='Disable SDK hash checks.', + default=False, + action='store_true') options = parser.parse_args(args) if not ProcessOptions(options): @@ -506,7 +512,8 @@ def Main(argv): # See dartbug.com/32364 command = [gn, 'gen', out_dir] gn_args = ToCommandLine( - ToGnArgs(args, mode, arch, target_os, sanitizer)) + ToGnArgs(args, mode, arch, target_os, sanitizer, + not args.no_verify_sdk_hash)) gn_args += GetGNArgs(args) if args.verbose: print("gn gen --check in %s" % out_dir) diff --git a/tools/make_version.py b/tools/make_version.py index eb01d354f13..ee21de04eca 100755 --- a/tools/make_version.py +++ b/tools/make_version.py @@ -7,19 +7,13 @@ from __future__ import print_function +import argparse import hashlib import os import sys import time -from optparse import OptionParser import utils - -def debugLog(message): - print(message, file=sys.stderr) - sys.stderr.flush() - - # When these files change, snapshots created by the VM are potentially no longer # backwards-compatible. VM_SNAPSHOT_FILES = [ @@ -45,141 +39,141 @@ VM_SNAPSHOT_FILES = [ ] -def MakeVersionString(quiet, no_git_hash, custom_for_pub=None): - channel = utils.GetChannel() - - if custom_for_pub and channel == 'be': - latest = utils.GetLatestDevTag() - if not latest: - # If grabbing the dev tag fails, then fall back on the VERSION file. - latest = utils.GetSemanticSDKVersion(no_git_hash=True) - if no_git_hash: - version_string = ("%s.%s" % (latest, custom_for_pub)) - else: - git_hash = utils.GetShortGitHash() - version_string = ("%s.%s-%s" % (latest, custom_for_pub, git_hash)) - else: - version_string = utils.GetSemanticSDKVersion(no_git_hash=no_git_hash) - if not quiet: - debugLog("Returning version string: %s " % version_string) - return version_string - - def MakeSnapshotHashString(): vmhash = hashlib.md5() for vmfilename in VM_SNAPSHOT_FILES: vmfilepath = os.path.join(utils.DART_DIR, 'runtime', 'vm', vmfilename) - with open(vmfilepath) as vmfile: - vmhash.update(vmfile.read().encode('utf-8')) + with open(vmfilepath, 'rb') as vmfile: + vmhash.update(vmfile.read()) return vmhash.hexdigest() -def MakeFile(quiet, - output_file, - input_file, - no_git_hash, - custom_for_pub, - version_file=None): - if version_file: - version_string = utils.GetVersion(no_git_hash, version_file) - else: - version_string = MakeVersionString(quiet, no_git_hash, custom_for_pub) +def GetSemanticVersionFormat(no_git_hash, custom_for_pub): + version_format = '{{SEMANTIC_SDK_VERSION}}' + + if custom_for_pub and utils.GetChannel() == 'be': + if no_git_hash: + version_format = '{{LATEST}}.{{PUB_CUSTOM}}' + else: + version_format = '{{LATEST}}.{{PUB_CUSTOM}}-{{GIT_HASH}}' + + return version_format + + +def FormatVersionString(version, + no_git_hash, + custom_for_pub, + version_file=None, + git_revision_file=None): + semantic_sdk_version = utils.GetSemanticSDKVersion(no_git_hash, + version_file, + git_revision_file) + semantic_version_format = GetSemanticVersionFormat(no_git_hash, + custom_for_pub) + version_str = (semantic_sdk_version + if version_file else semantic_version_format) + + version = version.replace('{{VERSION_STR}}', version_str) + + version = version.replace('{{SEMANTIC_SDK_VERSION}}', semantic_sdk_version) + + # If grabbing the dev tag fails, then fall back on the default VERSION file. + latest = utils.GetLatestDevTag() + if not latest: + utils.GetSemanticSDKVersion(no_git_hash=True) + version = version.replace('{{LATEST}}', latest) + + if custom_for_pub: + version = version.replace('{{PUB_CUSTOM}}', custom_for_pub) + + git_hash = utils.GetShortGitHash() + if git_hash is None or len(git_hash) != 10: + git_hash = '0000000000' + version = version.replace('{{GIT_HASH}}', git_hash) - version_cc_text = open(input_file).read() - version_cc_text = version_cc_text.replace("{{VERSION_STR}}", version_string) channel = utils.GetChannel() - version_cc_text = version_cc_text.replace("{{CHANNEL}}", channel) + version = version.replace('{{CHANNEL}}', channel) + version_time = utils.GetGitTimestamp() if no_git_hash or version_time == None: - version_time = "Unknown timestamp" - version_cc_text = version_cc_text.replace("{{COMMIT_TIME}}", version_time.decode("utf-8")) + version_time = 'Unknown timestamp' + version = version.replace('{{COMMIT_TIME}}', version_time.decode('utf-8')) + abi_version = utils.GetAbiVersion(version_file) - version_cc_text = version_cc_text.replace("{{ABI_VERSION}}", abi_version) + version = version.replace('{{ABI_VERSION}}', abi_version) + oldest_supported_abi_version = utils.GetOldestSupportedAbiVersion( version_file) - version_cc_text = version_cc_text.replace( - "{{OLDEST_SUPPORTED_ABI_VERSION}}", oldest_supported_abi_version) + version = version.replace('{{OLDEST_SUPPORTED_ABI_VERSION}}', + oldest_supported_abi_version) + snapshot_hash = MakeSnapshotHashString() - version_cc_text = version_cc_text.replace("{{SNAPSHOT_HASH}}", - snapshot_hash) - open(output_file, 'w').write(version_cc_text) - return True + version = version.replace('{{SNAPSHOT_HASH}}', snapshot_hash) + + return version -def main(args): +def main(): try: # Parse input. - parser = OptionParser() - parser.add_option( - "--custom_for_pub", - action="store", - type="string", - help=("Generates a version string that works with pub that includes" - "the given string. This is silently ignored on channels other" - "than be")) - parser.add_option( - "--input", - action="store", - type="string", - help="input template file.") - parser.add_option( - "--no_git_hash", - action="store_true", - default=False, - help="Don't try to determine svn revision") - parser.add_option( - "--output", action="store", type="string", help="output file name") - parser.add_option( - "-q", - "--quiet", - action="store_true", - default=False, - help="disable console output") - parser.add_option( - "--version-file", - action="store", - type="string", - default=None, - help="Version file") + parser = argparse.ArgumentParser() + parser.add_argument( + '--custom_for_pub', + help=('Generates a version string that works with pub that ' + 'includes the given string. This is silently ignored on ' + 'channels other than be.')) + parser.add_argument('--input', help='Input template file.') + parser.add_argument('--no_git_hash', + action='store_true', + default=False, + help='Don\'t try to determine VCS revision.') + parser.add_argument('--output', help='output file name') + parser.add_argument('-q', + '--quiet', + action='store_true', + default=False, + help='DEPRECATED: Does nothing!') + parser.add_argument('--version-file', help='Path to the VERSION file.') + parser.add_argument('--git-revision-file', + help='Path to the GIT_REVISION file.') + parser.add_argument( + '--format', + default='{{VERSION_STR}}', + help='Version format used if no input template is given.') - (options, args) = parser.parse_args() + args = parser.parse_args() # If there is no input template, then write the bare version string to - # options.output. If there is no options.output, then write the version + # args.output. If there is no args.output, then write the version # string to stdout. - if not options.input: - version_string = MakeVersionString( - options.quiet, options.no_git_hash, options.custom_for_pub) - if options.output: - open(options.output, 'w').write(version_string) - else: - print(version_string) - return 0 - if not options.output: - sys.stderr.write('--output not specified\n') - return -1 - if not options.input: - sys.stderr.write('--input not specified\n') - return -1 + version_template = '' + if args.input: + version_template = open(args.input).read() + elif not args.format is None: + version_template = args.format + else: + raise 'No version template given! Set either --input or --format.' - files = [] - for arg in args: - files.append(arg) + version = FormatVersionString(version_template, args.no_git_hash, + args.custom_for_pub, args.version_file, + args.git_revision_file) - if not MakeFile(options.quiet, options.output, options.input, - options.no_git_hash, options.custom_for_pub, - options.version_file): - return -1 + if args.output: + with open(args.output, 'w') as fh: + fh.write(version) + else: + sys.stdout.write(version) return 0 + except Exception as inst: sys.stderr.write('make_version.py exception\n') sys.stderr.write(str(inst)) sys.stderr.write('\n') + return -1 if __name__ == '__main__': - exit_code = main(sys.argv) - sys.exit(exit_code) + sys.exit(main()) diff --git a/tools/utils.py b/tools/utils.py index 1b543e3c1fb..c6dc813a604 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -350,13 +350,16 @@ def GetShortVersion(): version.prerelease, version.prerelease_patch)) -def GetSemanticSDKVersion(no_git_hash=False, version_file=None): +def GetSemanticSDKVersion(no_git_hash=False, + version_file=None, + git_revision_file=None): version = ReadVersionFile(version_file) if not version: return None if version.channel == 'be': - postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision() + postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision( + git_revision_file) elif version.channel in ('beta', 'dev'): postfix = '-%s.%s.%s' % (version.prerelease, version.prerelease_patch, version.channel) @@ -367,8 +370,8 @@ def GetSemanticSDKVersion(no_git_hash=False, version_file=None): return '%s.%s.%s%s' % (version.major, version.minor, version.patch, postfix) -def GetVersion(no_git_hash=False, version_file=None): - return GetSemanticSDKVersion(no_git_hash, version_file) +def GetVersion(no_git_hash=False, version_file=None, git_revision_file=None): + return GetSemanticSDKVersion(no_git_hash, version_file, git_revision_file) # The editor used to produce the VERSION file put on gcs. We now produce this @@ -467,12 +470,13 @@ def GetArchiveVersion(): return GetSemanticSDKVersion() -def GetGitRevision(): +def GetGitRevision(git_revision_file=None): # When building from tarball use tools/GIT_REVISION - git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION') + if git_revision_file is None: + git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION') try: with open(git_revision_file) as fd: - return fd.read() + return fd.read().strip() except: pass p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], diff --git a/utils/application_snapshot.gni b/utils/application_snapshot.gni index 10743175143..8f3f8822730 100644 --- a/utils/application_snapshot.gni +++ b/utils/application_snapshot.gni @@ -98,9 +98,11 @@ template("_application_snapshot") { depfile = "$output.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script diff --git a/utils/compile_platform.gni b/utils/compile_platform.gni index a2e94b3cd61..447ed644a04 100644 --- a/utils/compile_platform.gni +++ b/utils/compile_platform.gni @@ -3,6 +3,7 @@ # BSD-style license that can be found in the LICENSE file. import("../build/dart/dart_action.gni") +import("../sdk_args.gni") _dart_root = get_path_info("..", "abspath") @@ -46,6 +47,8 @@ template("compile_platform") { outputs = invoker.outputs + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + inputs = [] deps = [] args = [] diff --git a/utils/dartanalyzer/BUILD.gn b/utils/dartanalyzer/BUILD.gn index f1d9b014c7a..48c9dcf8668 100644 --- a/utils/dartanalyzer/BUILD.gn +++ b/utils/dartanalyzer/BUILD.gn @@ -44,6 +44,7 @@ prebuilt_dart_action("generate_summary_strong") { inputs = sdk_lib_files + analyzer_files output = "$root_gen_dir/strong.sum" outputs = [ output ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] args = [ "build", rebase_path(output), diff --git a/utils/dartdevc/BUILD.gn b/utils/dartdevc/BUILD.gn index 15bfe16d828..c9ff6555035 100644 --- a/utils/dartdevc/BUILD.gn +++ b/utils/dartdevc/BUILD.gn @@ -82,6 +82,8 @@ template("dart2js_compile") { packages = "../../.packages" + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "$abs_main", "-m", @@ -118,6 +120,8 @@ prebuilt_dart_action("dartdevc_patch_sdk") { # TODO(rnystrom): List the outputs more precisely? outputs = [ "$patched_sdk_dir/version" ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "--libraries", rebase_path("$sdk_root/lib/libraries.json"), @@ -230,6 +234,8 @@ template("dartdevc_kernel_compile") { "$js_gen_dir/$module.js", ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + args = [ "-k", "--dart-sdk-summary=$sdk_path", @@ -384,6 +390,8 @@ template("dartdevc_sdk_js") { "$js_gen_dir/legacy/dart_sdk.js.map", ] + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + script = "../../pkg/dev_compiler/bin/dartdevc.dart" args = [ diff --git a/utils/kernel-service/BUILD.gn b/utils/kernel-service/BUILD.gn index 062c3ff108b..a01bb1b5411 100644 --- a/utils/kernel-service/BUILD.gn +++ b/utils/kernel-service/BUILD.gn @@ -85,9 +85,11 @@ template("kernel_service_dill") { depfile = "$root_gen_dir/kernel_service" + invoker.target_name + "_dill.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) + vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", + "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script