From bb8d145616cd8671a9c31cdfd2b03808c8efd7ce Mon Sep 17 00:00:00 2001 From: asiva Date: Fri, 26 Jun 2020 18:54:17 +0000 Subject: [PATCH] Revert "[SDK] Adds an SDK hash to kernels and the VM." This reverts commit edde575dcd6eb98bc7735693b2bfd92cd2a552f0. Reason for revert: Breaks the Dart to Flutter roll and golem Original change's description: > [SDK] Adds an SDK hash to kernels and the VM. > > Adds a new SDK hash to kernels and the VM which is optionally checked > to verify kernels are built for the same SDK as the VM. > This helps catch incompatibilities that are currently causing > subtle bugs and (not so subtle) crashes. > > The SDK hash is encoded in kernels as a new field in components. > The hash is derived from the 10 byte git short hash. > > This new check can be disabled via: > tools/gn.py ... --no-verify-sdk-hash > > This CL bumps the min. (and max.) supported kernel format version, > making the VM backwards incompatible from this point back. > > Bug: https://github.com/dart-lang/sdk/issues/41802 > Change-Id: I3cbb2d481239ee64dafdaa0e4aac36c80281931b > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/150343 > Commit-Queue: Clement Skau > Reviewed-by: Jens Johansen > Reviewed-by: Martin Kustermann TBR=kustermann@google.com,jensj@google.com,cskau@google.com Change-Id: I34cc7d378e2babdaaca4d932d19c19d0f35422fc No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: https://github.com/dart-lang/sdk/issues/41802 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/152703 Reviewed-by: Siva Annamalai Commit-Queue: Siva Annamalai --- DEPS | 2 +- pkg/front_end/test/binary_md_dill_reader.dart | 50 ++-- .../test/spell_checking_list_code.txt | 4 - pkg/kernel/binary.md | 3 +- pkg/kernel/lib/binary/ast_from_binary.dart | 28 +-- pkg/kernel/lib/binary/ast_to_binary.dart | 2 - pkg/kernel/lib/binary/tag.dart | 26 +-- runtime/bin/BUILD.gn | 2 - runtime/tests/vm/dart/sdk_hash_test.dart | 88 ------- .../tests/vm/dart/snapshot_test_helper.dart | 1 - runtime/tests/vm/dart_2/sdk_hash_test.dart | 88 ------- .../tests/vm/dart_2/snapshot_test_helper.dart | 1 - .../frontend/kernel_translation_helper.cc | 13 +- runtime/vm/kernel_binary.cc | 38 ++- runtime/vm/kernel_binary.h | 10 +- runtime/vm/kernel_loader.cc | 15 +- runtime/vm/version.h | 2 - runtime/vm/version_in.cc | 5 - sdk_args.gni | 18 -- tools/gn.py | 11 +- tools/make_version.py | 221 +++++++++--------- tools/utils.py | 18 +- utils/application_snapshot.gni | 2 - utils/compile_platform.gni | 3 - utils/dartanalyzer/BUILD.gn | 1 - utils/dartdevc/BUILD.gn | 8 - utils/kernel-service/BUILD.gn | 2 - 27 files changed, 184 insertions(+), 478 deletions(-) delete mode 100644 runtime/tests/vm/dart/sdk_hash_test.dart delete mode 100644 runtime/tests/vm/dart_2/sdk_hash_test.dart diff --git a/DEPS b/DEPS index c059ffe42f5b..7d11ff890eee 100644 --- a/DEPS +++ b/DEPS @@ -512,7 +512,7 @@ deps = { "packages": [ { "package": "dart/cfe/dart2js_dills", - "version": "binary_version:43_2", + "version": "binary_version:42", } ], "dep_type": "cipd", diff --git a/pkg/front_end/test/binary_md_dill_reader.dart b/pkg/front_end/test/binary_md_dill_reader.dart index c0a4b5b7fbf9..01cf4d98d44f 100644 --- a/pkg/front_end/test/binary_md_dill_reader.dart +++ b/pkg/front_end/test/binary_md_dill_reader.dart @@ -473,33 +473,31 @@ class BinaryMdDillReader { type = type.substring(0, type.indexOf("[")); type = _lookupGenericType(typeNames, type, types); - int intCount = int.tryParse(count) ?? -1; - if (intCount == -1) { - if (vars[count] != null && vars[count] is int) { - intCount = vars[count]; - } else if (count.contains(".")) { - List countData = - count.split(regExpSplit).map((s) => s.trim()).toList(); - if (vars[countData[0]] != null) { - dynamic v = vars[countData[0]]; - if (v is Map && - countData[1] == "last" && - v["items"] is List && - v["items"].last is int) { - intCount = v["items"].last; - } else if (v is Map && v[countData[1]] != null) { - v = v[countData[1]]; - if (v is Map && v[countData[2]] != null) { - v = v[countData[2]]; - if (v is int) intCount = v; - } else if (v is int && - countData.length == 4 && - countData[2] == "+") { - intCount = v + int.parse(countData[3]); - } - } else { - throw "Unknown dot to int ($count)"; + int intCount = -1; + if (vars[count] != null && vars[count] is int) { + intCount = vars[count]; + } else if (count.contains(".")) { + List countData = + count.split(regExpSplit).map((s) => s.trim()).toList(); + if (vars[countData[0]] != null) { + dynamic v = vars[countData[0]]; + if (v is Map && + countData[1] == "last" && + v["items"] is List && + v["items"].last is int) { + intCount = v["items"].last; + } else if (v is Map && v[countData[1]] != null) { + v = v[countData[1]]; + if (v is Map && v[countData[2]] != null) { + v = v[countData[2]]; + if (v is int) intCount = v; + } else if (v is int && + countData.length == 4 && + countData[2] == "+") { + intCount = v + int.parse(countData[3]); } + } else { + throw "Unknown dot to int ($count)"; } } } diff --git a/pkg/front_end/test/spell_checking_list_code.txt b/pkg/front_end/test/spell_checking_list_code.txt index d27f9e7a6450..190fa6af9b91 100644 --- a/pkg/front_end/test/spell_checking_list_code.txt +++ b/pkg/front_end/test/spell_checking_list_code.txt @@ -313,7 +313,6 @@ disallow disambiguator disjoint dispatched -distribute divided dmitryas doc @@ -326,7 +325,6 @@ downloaded downloading dq dquote -dsdk dst dummy dupdate @@ -437,14 +435,12 @@ futured futureor g gardening -gen generation gets getter1a getter1b getting gft -git github glb glob diff --git a/pkg/kernel/binary.md b/pkg/kernel/binary.md index 72e2057d80a2..2b37b2c55e20 100644 --- a/pkg/kernel/binary.md +++ b/pkg/kernel/binary.md @@ -143,8 +143,7 @@ type CanonicalName { type ComponentFile { UInt32 magic = 0x90ABCDEF; - UInt32 formatVersion = 43; - Byte[10] shortSdkHash; + UInt32 formatVersion = 42; List problemsAsJson; // Described in problems.md. Library[] libraries; UriSource sourceMap; diff --git a/pkg/kernel/lib/binary/ast_from_binary.dart b/pkg/kernel/lib/binary/ast_from_binary.dart index b5e70b69d071..9a32677b4973 100644 --- a/pkg/kernel/lib/binary/ast_from_binary.dart +++ b/pkg/kernel/lib/binary/ast_from_binary.dart @@ -5,7 +5,6 @@ library kernel.ast_from_binary; import 'dart:core' hide MapEntry; import 'dart:developer'; -import 'dart:convert'; import 'dart:typed_data'; import '../ast.dart'; @@ -29,22 +28,11 @@ class InvalidKernelVersionError { InvalidKernelVersionError(this.version); String toString() { - return 'Unexpected Kernel Format Version ${version} ' + return 'Unexpected Kernel version ${version} ' '(expected ${Tag.BinaryFormatVersion}).'; } } -class InvalidKernelSdkVersionError { - final String version; - - InvalidKernelSdkVersionError(this.version); - - String toString() { - return 'Unexpected Kernel SDK Version ${version} ' - '(expected ${expectedSdkHash}).'; - } -} - class CompilationModeError { final String message; @@ -496,13 +484,6 @@ class BinaryBuilder { if (_bytes.length == 0) throw new StateError("Empty input given."); } - void _readAndVerifySdkHash() { - final sdkHash = ascii.decode(readBytes(sdkHashLength)); - if (!isValidSdkHash(sdkHash)) { - throw InvalidKernelSdkVersionError(sdkHash); - } - } - /// Deserializes a kernel component and stores it in [component]. /// /// When linking with a non-empty component, canonical names must have been @@ -530,9 +511,6 @@ class BinaryBuilder { if (version != Tag.BinaryFormatVersion) { throw InvalidKernelVersionError(version); } - - _readAndVerifySdkHash(); - _byteOffset = offset; List componentFileSizes = _indexComponents(); if (componentFileSizes.length > 1) { @@ -716,8 +694,6 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } - _readAndVerifySdkHash(); - // Read component index from the end of this ComponentFiles serialized data. _ComponentIndex index = _readComponentIndex(componentFileSize); @@ -742,8 +718,6 @@ class BinaryBuilder { throw InvalidKernelVersionError(formatVersion); } - _readAndVerifySdkHash(); - List problemsAsJson = readListOfStrings(); if (problemsAsJson != null) { component.problemsAsJson ??= []; diff --git a/pkg/kernel/lib/binary/ast_to_binary.dart b/pkg/kernel/lib/binary/ast_to_binary.dart index dc3be1e21b92..57ccd2fd17e1 100644 --- a/pkg/kernel/lib/binary/ast_to_binary.dart +++ b/pkg/kernel/lib/binary/ast_to_binary.dart @@ -4,7 +4,6 @@ library kernel.ast_to_binary; import 'dart:core' hide MapEntry; -import 'dart:convert'; import 'dart:developer'; import 'dart:io' show BytesBuilder; import 'dart:typed_data'; @@ -538,7 +537,6 @@ class BinaryPrinter implements Visitor, BinarySink { final componentOffset = getBufferOffset(); writeUInt32(Tag.ComponentFile); writeUInt32(Tag.BinaryFormatVersion); - writeBytes(ascii.encode(expectedSdkHash)); writeListOfStrings(component.problemsAsJson); indexLinkTable(component); _collectMetadata(component); diff --git a/pkg/kernel/lib/binary/tag.dart b/pkg/kernel/lib/binary/tag.dart index b35dde9bc905..cec230824aa2 100644 --- a/pkg/kernel/lib/binary/tag.dart +++ b/pkg/kernel/lib/binary/tag.dart @@ -149,7 +149,7 @@ class Tag { /// Internal version of kernel binary format. /// Bump it when making incompatible changes in kernel binaries. /// Keep in sync with runtime/vm/kernel_binary.h, pkg/kernel/binary.md. - static const int BinaryFormatVersion = 43; + static const int BinaryFormatVersion = 42; } abstract class ConstantTag { @@ -169,27 +169,3 @@ abstract class ConstantTag { static const int UnevaluatedConstant = 12; // 13 is occupied by [SetConstant] } - -const int sdkHashLength = 10; // Bytes, a Git "short hash". - -const String sdkHashNull = '0000000000'; - -// Will be correct hash for Flutter SDK / Dart SDK we distribute. -// If non-null we will validate when consuming kernel, will use when producing -// kernel. -// If null, local development setting (e.g. run gen_kernel.dart from source), -// we put 0x00..00 into when producing, do not validate when consuming. -String get expectedSdkHash { - final sdkHash = - const String.fromEnvironment('sdk_hash', defaultValue: sdkHashNull); - if (sdkHash.length != 10) { - throw '-Dsdk_hash= must be a 10 byte string!'; - } - return sdkHash; -} - -bool isValidSdkHash(String sdkHash) { - return (sdkHash == sdkHashNull || - expectedSdkHash == sdkHashNull || - sdkHash == expectedSdkHash); -} diff --git a/runtime/bin/BUILD.gn b/runtime/bin/BUILD.gn index f90ddcabbd19..0fd5acd5ab02 100644 --- a/runtime/bin/BUILD.gn +++ b/runtime/bin/BUILD.gn @@ -964,11 +964,9 @@ prebuilt_dart_action("gen_kernel_bytecode_dill") { abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) - vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", - "-Dsdk_hash=$sdk_hash", ] args = [ diff --git a/runtime/tests/vm/dart/sdk_hash_test.dart b/runtime/tests/vm/dart/sdk_hash_test.dart deleted file mode 100644 index 6b05e96271f8..000000000000 --- a/runtime/tests/vm/dart/sdk_hash_test.dart +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:path/path.dart' as path; -import 'package:expect/expect.dart'; - -import 'snapshot_test_helper.dart'; - -// Keep in sync with pkg/kernel/lib/binary/tag.dart: -const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; -const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; - -Future main(List args) async { - if (args.length == 1 && args[0] == '--child') { - print('Hello, SDK Hash!'); - return; - } - - final String sourcePath = - path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); - - await withTempDir((String tmp) async { - final String dillPath = path.join(tmp, 'test.dill'); - - { - final result = await Process.run(dart, [ - genKernel, - '--platform', - platformDill, - '-o', - dillPath, - sourcePath, - ]); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('', result.stdout); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('Hello, SDK Hash!', result.stdout.trim()); - } - - // Invalidate the SDK hash in the kernel dill: - { - final myFile = File(dillPath); - Uint8List bytes = myFile.readAsBytesSync(); - // The SDK Hash is located after the ComponentFile and BinaryFormatVersion - // tags (both UInt32). - Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); - Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); - // Flip the first byte in the hash: - bytes[8] ^= bytes[8]; - myFile.writeAsBytesSync(bytes); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals( - 'Can\'t load Kernel binary: Invalid SDK hash.', result.stderr.trim()); - Expect.equals(253, result.exitCode); - Expect.equals('', result.stdout); - } - - // Zero out the SDK hash in the kernel dill to disable the check: - { - final myFile = File(dillPath); - Uint8List bytes = myFile.readAsBytesSync(); - bytes.setRange(8, 18, ascii.encode('0000000000')); - myFile.writeAsBytesSync(bytes); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('Hello, SDK Hash!', result.stdout.trim()); - } - }); -} diff --git a/runtime/tests/vm/dart/snapshot_test_helper.dart b/runtime/tests/vm/dart/snapshot_test_helper.dart index 974050e1806c..853fc3388e2b 100644 --- a/runtime/tests/vm/dart/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart/snapshot_test_helper.dart @@ -48,7 +48,6 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); -final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/tests/vm/dart_2/sdk_hash_test.dart b/runtime/tests/vm/dart_2/sdk_hash_test.dart deleted file mode 100644 index 6b05e96271f8..000000000000 --- a/runtime/tests/vm/dart_2/sdk_hash_test.dart +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:path/path.dart' as path; -import 'package:expect/expect.dart'; - -import 'snapshot_test_helper.dart'; - -// Keep in sync with pkg/kernel/lib/binary/tag.dart: -const tagComponentFile = [0x90, 0xAB, 0xCD, 0xEF]; -const tagBinaryFormatVersion = [0x00, 0x00, 0x00, 43]; - -Future main(List args) async { - if (args.length == 1 && args[0] == '--child') { - print('Hello, SDK Hash!'); - return; - } - - final String sourcePath = - path.join('runtime', 'tests', 'vm', 'dart_2', 'sdk_hash_test.dart'); - - await withTempDir((String tmp) async { - final String dillPath = path.join(tmp, 'test.dill'); - - { - final result = await Process.run(dart, [ - genKernel, - '--platform', - platformDill, - '-o', - dillPath, - sourcePath, - ]); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('', result.stdout); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('Hello, SDK Hash!', result.stdout.trim()); - } - - // Invalidate the SDK hash in the kernel dill: - { - final myFile = File(dillPath); - Uint8List bytes = myFile.readAsBytesSync(); - // The SDK Hash is located after the ComponentFile and BinaryFormatVersion - // tags (both UInt32). - Expect.listEquals(tagComponentFile, bytes.sublist(0, 4)); - Expect.listEquals(tagBinaryFormatVersion, bytes.sublist(4, 8)); - // Flip the first byte in the hash: - bytes[8] ^= bytes[8]; - myFile.writeAsBytesSync(bytes); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals( - 'Can\'t load Kernel binary: Invalid SDK hash.', result.stderr.trim()); - Expect.equals(253, result.exitCode); - Expect.equals('', result.stdout); - } - - // Zero out the SDK hash in the kernel dill to disable the check: - { - final myFile = File(dillPath); - Uint8List bytes = myFile.readAsBytesSync(); - bytes.setRange(8, 18, ascii.encode('0000000000')); - myFile.writeAsBytesSync(bytes); - } - - { - final result = await Process.run(dart, [dillPath, '--child']); - Expect.equals('', result.stderr); - Expect.equals(0, result.exitCode); - Expect.equals('Hello, SDK Hash!', result.stdout.trim()); - } - }); -} diff --git a/runtime/tests/vm/dart_2/snapshot_test_helper.dart b/runtime/tests/vm/dart_2/snapshot_test_helper.dart index b2754d0bf9de..9be810a28e84 100644 --- a/runtime/tests/vm/dart_2/snapshot_test_helper.dart +++ b/runtime/tests/vm/dart_2/snapshot_test_helper.dart @@ -48,7 +48,6 @@ final String executableSuffix = Platform.isWindows ? ".exe" : ""; final String buildDir = p.dirname(Platform.executable); final String platformDill = p.join(buildDir, "vm_platform_strong.dill"); final String genSnapshot = p.join(buildDir, "gen_snapshot${executableSuffix}"); -final String dart = p.join(buildDir, "dart${executableSuffix}"); final String dartPrecompiledRuntime = p.join(buildDir, "dart_precompiled_runtime${executableSuffix}"); final String genKernel = p.join("pkg", "vm", "bin", "gen_kernel.dart"); diff --git a/runtime/vm/compiler/frontend/kernel_translation_helper.cc b/runtime/vm/compiler/frontend/kernel_translation_helper.cc index 93a9b490fcaf..584076841620 100644 --- a/runtime/vm/compiler/frontend/kernel_translation_helper.cc +++ b/runtime/vm/compiler/frontend/kernel_translation_helper.cc @@ -2654,7 +2654,18 @@ TokenPosition KernelReaderHelper::ReadPosition() { } intptr_t KernelReaderHelper::SourceTableFieldCountFromFirstLibraryOffset() { - return SourceTableFieldCountFromFirstLibraryOffset41Plus; + // translation_helper_.info() might not be initialized at this point so we + // can't use translation_helper_.info().kernel_binary_version(). + SetOffset(KernelFormatVersionOffset); + uint32_t formatVersion = reader_.ReadUInt32(); + intptr_t count_from_first_library_offset = + SourceTableFieldCountFromFirstLibraryOffsetPre41; + static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); + if (formatVersion >= 41) { + count_from_first_library_offset = + SourceTableFieldCountFromFirstLibraryOffset41Plus; + } + return count_from_first_library_offset; } intptr_t KernelReaderHelper::SourceTableSize() { diff --git a/runtime/vm/kernel_binary.cc b/runtime/vm/kernel_binary.cc index 469f74647998..3720fe23ac7e 100644 --- a/runtime/vm/kernel_binary.cc +++ b/runtime/vm/kernel_binary.cc @@ -15,7 +15,6 @@ #include "vm/kernel.h" #include "vm/object.h" #include "vm/os.h" -#include "vm/version.h" namespace dart { @@ -85,14 +84,12 @@ const char* kKernelInvalidBinaryFormatVersion = "Invalid kernel binary format version"; const char* kKernelInvalidSizeIndicated = "Invalid kernel binary: Indicated size is invalid"; -const char* kKernelInvalidSdkHash = "Invalid SDK hash"; std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { - if (reader->size() < 70) { - // A kernel file (v43) currently contains at least the following: + if (reader->size() < 60) { + // A kernel file (v41) currently contains at least the following: // * Magic number (32) // * Kernel version (32) - // * SDK Hash (10 * 8) // * List of problems (8) // * Length of source map (32) // * Length of canonical name table (8) @@ -101,7 +98,7 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // * Length of constant table (8) // * Component index (11 * 32) // - // so is at least 74 bytes. + // so is at least 64 bytes. // (Technically it will also contain an empty entry in both source map and // string table, taking up another 8 bytes.) if (error != nullptr) { @@ -127,18 +124,6 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { return nullptr; } - uint8_t sdkHash[11]; - reader->ReadBytes(sdkHash, 10); - sdkHash[10] = 0; // Null terminate. - if (strcmp(Version::SdkHash(), "0000000000") != 0 && - strcmp((const char*)sdkHash, "0000000000") != 0 && - strcmp((const char*)sdkHash, Version::SdkHash()) != 0) { - if (error != nullptr) { - *error = kKernelInvalidSdkHash; - } - return nullptr; - } - std::unique_ptr program(new Program()); program->binary_version_ = formatVersion; program->typed_data_ = reader->typed_data(); @@ -167,8 +152,13 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { // Read backwards at the end. program->library_count_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd, 1, 0); + static_assert(kMinSupportedKernelFormatVersion < 41, "cleanup this code"); intptr_t count_from_first_library_offset = - SourceTableFieldCountFromFirstLibraryOffset41Plus; + SourceTableFieldCountFromFirstLibraryOffsetPre41; + if (formatVersion >= 41) { + count_from_first_library_offset = + SourceTableFieldCountFromFirstLibraryOffset41Plus; + } program->source_table_offset_ = reader->ReadFromIndexNoReset( reader->size_, LibraryCountFieldCountFromEnd + 1 + program->library_count_ + 1 + @@ -181,9 +171,13 @@ std::unique_ptr Program::ReadFrom(Reader* reader, const char** error) { program->constant_table_offset_ = reader->ReadUInt32(); program->main_method_reference_ = NameIndex(reader->ReadUInt32() - 1); - NNBDCompiledMode compilation_mode = - static_cast(reader->ReadUInt32()); - program->compilation_mode_ = compilation_mode; + if (formatVersion >= 41) { + NNBDCompiledMode compilation_mode = + static_cast(reader->ReadUInt32()); + program->compilation_mode_ = compilation_mode; + } else { + program->compilation_mode_ = NNBDCompiledMode::kDisabled; + } return program; } diff --git a/runtime/vm/kernel_binary.h b/runtime/vm/kernel_binary.h index 33dc838a9807..11527eba3e40 100644 --- a/runtime/vm/kernel_binary.h +++ b/runtime/vm/kernel_binary.h @@ -20,8 +20,8 @@ namespace kernel { static const uint32_t kMagicProgramFile = 0x90ABCDEFu; // Both version numbers are inclusive. -static const uint32_t kMinSupportedKernelFormatVersion = 43; -static const uint32_t kMaxSupportedKernelFormatVersion = 43; +static const uint32_t kMinSupportedKernelFormatVersion = 29; +static const uint32_t kMaxSupportedKernelFormatVersion = 42; // Keep in sync with package:kernel/lib/binary/tag.dart #define KERNEL_TAG_LIST(V) \ @@ -326,12 +326,6 @@ class Reader : public ValueObject { uint8_t PeekByte() { return buffer()[offset_]; } - void ReadBytes(uint8_t* buffer, uint8_t size) { - for (int i = 0; i < size; i++) { - buffer[i] = ReadByte(); - } - } - bool ReadBool() { return (ReadByte() & 1) == 1; } uint8_t ReadFlags() { return ReadByte(); } diff --git a/runtime/vm/kernel_loader.cc b/runtime/vm/kernel_loader.cc index 88b1c637792e..1d0d5437726f 100644 --- a/runtime/vm/kernel_loader.cc +++ b/runtime/vm/kernel_loader.cc @@ -297,11 +297,7 @@ Object& KernelLoader::LoadEntireProgram(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - const char* error = nullptr; - std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); - if (subprogram == nullptr) { - FATAL1("Failed to load kernel file: %s", error); - } + std::unique_ptr subprogram = Program::ReadFrom(&reader); ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), &uri_to_source_table); Object& load_result = Object::Handle(loader.LoadProgram(false)); @@ -853,8 +849,7 @@ void KernelLoader::FindModifiedLibraries(Program* program, intptr_t* p_num_classes, intptr_t* p_num_procedures) { LongJumpScope jump; - Thread* thread = Thread::Current(); - Zone* zone = thread->zone(); + Zone* zone = Thread::Current()->zone(); if (setjmp(*jump.Set()) == 0) { if (force_reload) { // If a reload is being forced we mark all libraries as having @@ -899,11 +894,7 @@ void KernelLoader::FindModifiedLibraries(Program* program, reader.set_raw_buffer(program->kernel_data() + subprogram_start); reader.set_size(subprogram_end - subprogram_start); reader.set_offset(0); - const char* error = nullptr; - std::unique_ptr subprogram = Program::ReadFrom(&reader, &error); - if (subprogram == nullptr) { - FATAL1("Failed to load kernel file: %s", error); - } + std::unique_ptr subprogram = Program::ReadFrom(&reader); ASSERT(subprogram->is_single_program()); KernelLoader loader(subprogram.get(), /*uri_to_source_table=*/nullptr); loader.walk_incremental_kernel(modified_libs, is_empty_program, diff --git a/runtime/vm/version.h b/runtime/vm/version.h index a3769f6ef446..fdc6bb0f5bc1 100644 --- a/runtime/vm/version.h +++ b/runtime/vm/version.h @@ -16,13 +16,11 @@ class Version : public AllStatic { static const char* CommitString(); static int CurrentAbiVersion(); static int OldestSupportedAbiVersion(); - static const char* SdkHash(); private: static const char* str_; static const char* snapshot_hash_; static const char* commit_; - static const char* git_short_hash_; }; } // namespace dart diff --git a/runtime/vm/version_in.cc b/runtime/vm/version_in.cc index f63fe3fde8a9..b4f1e2d6e119 100644 --- a/runtime/vm/version_in.cc +++ b/runtime/vm/version_in.cc @@ -38,13 +38,8 @@ int Version::OldestSupportedAbiVersion() { return {{OLDEST_SUPPORTED_ABI_VERSION}}; } -const char* Version::SdkHash() { - return git_short_hash_; -} - const char* Version::snapshot_hash_ = "{{SNAPSHOT_HASH}}"; const char* Version::str_ = "{{VERSION_STR}} ({{CHANNEL}}) ({{COMMIT_TIME}})"; const char* Version::commit_ = "{{VERSION_STR}}"; -const char* Version::git_short_hash_ = "{{GIT_HASH}}"; } // namespace dart diff --git a/sdk_args.gni b/sdk_args.gni index 06dc3e8540bb..94e6b7a8eedf 100644 --- a/sdk_args.gni +++ b/sdk_args.gni @@ -19,22 +19,4 @@ declare_args() { # to out/ReleaseX64/args.gn. The path above can be extracted from the `.git` # file under the git worktree folder. default_git_folder = "$_dart_root/.git" - - # Whether to enable the SDK hash check that will prevent loading a kernel - # into a VM which was built with a different SDK. - verify_sdk_hash = true -} - -# The SDK hash to build into VM and kernels. -# The value 0000000000 signifies no hash is set, which will disable the check. -if (verify_sdk_hash) { - sdk_hash = exec_script("$_dart_root/tools/make_version.py", - [ "--format={{GIT_HASH}}" ], - "trim string", - [ - "$_dart_root/tools/VERSION", - "$_dart_root/tools/utils.py", - ]) -} else { - sdk_hash = "0000000000" } diff --git a/tools/gn.py b/tools/gn.py index 4bd29b452a51..830c8b3c64ba 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -136,7 +136,7 @@ def UseSysroot(args, gn_args): return True -def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash): +def ToGnArgs(args, mode, arch, target_os, sanitizer): gn_args = {} host_os = HostOsForGn(HOST_OS) @@ -258,8 +258,6 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash): gn_args['dart_debug_optimization_level'] = args.debug_opt_level gn_args['debug_optimization_level'] = args.debug_opt_level - gn_args['verify_sdk_hash'] = verify_sdk_hash - return gn_args @@ -468,10 +466,6 @@ def parse_args(args): default=False, dest='use_qemu', action='store_true') - other_group.add_argument('--no-verify-sdk-hash', - help='Disable SDK hash checks.', - default=False, - action='store_true') options = parser.parse_args(args) if not ProcessOptions(options): @@ -512,8 +506,7 @@ def Main(argv): # See dartbug.com/32364 command = [gn, 'gen', out_dir] gn_args = ToCommandLine( - ToGnArgs(args, mode, arch, target_os, sanitizer, - not args.no_verify_sdk_hash)) + ToGnArgs(args, mode, arch, target_os, sanitizer)) gn_args += GetGNArgs(args) if args.verbose: print("gn gen --check in %s" % out_dir) diff --git a/tools/make_version.py b/tools/make_version.py index ee21de04ecad..b0b795539cef 100755 --- a/tools/make_version.py +++ b/tools/make_version.py @@ -7,13 +7,19 @@ from __future__ import print_function -import argparse import hashlib import os import sys import time +from optparse import OptionParser import utils + +def debugLog(message): + print(message, file=sys.stderr) + sys.stderr.flush() + + # When these files change, snapshots created by the VM are potentially no longer # backwards-compatible. VM_SNAPSHOT_FILES = [ @@ -39,141 +45,142 @@ ] -def MakeSnapshotHashString(): - vmhash = hashlib.md5() - for vmfilename in VM_SNAPSHOT_FILES: - vmfilepath = os.path.join(utils.DART_DIR, 'runtime', 'vm', vmfilename) - with open(vmfilepath, 'rb') as vmfile: - vmhash.update(vmfile.read()) - return vmhash.hexdigest() - - -def GetSemanticVersionFormat(no_git_hash, custom_for_pub): - version_format = '{{SEMANTIC_SDK_VERSION}}' +def MakeVersionString(quiet, no_git_hash, custom_for_pub=None): + channel = utils.GetChannel() - if custom_for_pub and utils.GetChannel() == 'be': + if custom_for_pub and channel == 'be': + latest = utils.GetLatestDevTag() + if not latest: + # If grabbing the dev tag fails, then fall back on the VERSION file. + latest = utils.GetSemanticSDKVersion(no_git_hash=True) if no_git_hash: - version_format = '{{LATEST}}.{{PUB_CUSTOM}}' + version_string = ("%s.%s" % (latest, custom_for_pub)) else: - version_format = '{{LATEST}}.{{PUB_CUSTOM}}-{{GIT_HASH}}' - - return version_format + git_hash = utils.GetShortGitHash() + version_string = ("%s.%s-%s" % (latest, custom_for_pub, git_hash)) + else: + version_string = utils.GetSemanticSDKVersion(no_git_hash=no_git_hash) + if not quiet: + debugLog("Returning version string: %s " % version_string) + return version_string -def FormatVersionString(version, - no_git_hash, - custom_for_pub, - version_file=None, - git_revision_file=None): - semantic_sdk_version = utils.GetSemanticSDKVersion(no_git_hash, - version_file, - git_revision_file) - semantic_version_format = GetSemanticVersionFormat(no_git_hash, - custom_for_pub) - version_str = (semantic_sdk_version - if version_file else semantic_version_format) - - version = version.replace('{{VERSION_STR}}', version_str) - - version = version.replace('{{SEMANTIC_SDK_VERSION}}', semantic_sdk_version) - - # If grabbing the dev tag fails, then fall back on the default VERSION file. - latest = utils.GetLatestDevTag() - if not latest: - utils.GetSemanticSDKVersion(no_git_hash=True) - version = version.replace('{{LATEST}}', latest) +def MakeSnapshotHashString(): + vmhash = hashlib.md5() + for vmfilename in VM_SNAPSHOT_FILES: + vmfilepath = os.path.join(utils.DART_DIR, 'runtime', 'vm', vmfilename) + with open(vmfilepath) as vmfile: + vmhash.update(vmfile.read().encode('utf-8')) + return vmhash.hexdigest() - if custom_for_pub: - version = version.replace('{{PUB_CUSTOM}}', custom_for_pub) - git_hash = utils.GetShortGitHash() - if git_hash is None or len(git_hash) != 10: - git_hash = '0000000000' - version = version.replace('{{GIT_HASH}}', git_hash) +def MakeFile(quiet, + output_file, + input_file, + no_git_hash, + custom_for_pub, + version_file=None): + if version_file: + version_string = utils.GetVersion(no_git_hash, version_file) + else: + version_string = MakeVersionString(quiet, no_git_hash, custom_for_pub) + version_cc_text = open(input_file).read() + version_cc_text = version_cc_text.replace("{{VERSION_STR}}", version_string) channel = utils.GetChannel() - version = version.replace('{{CHANNEL}}', channel) - + version_cc_text = version_cc_text.replace("{{CHANNEL}}", channel) version_time = utils.GetGitTimestamp() if no_git_hash or version_time == None: - version_time = 'Unknown timestamp' - version = version.replace('{{COMMIT_TIME}}', version_time.decode('utf-8')) - + version_time = "Unknown timestamp" + version_cc_text = version_cc_text.replace("{{COMMIT_TIME}}", + version_time.decode("utf-8")) abi_version = utils.GetAbiVersion(version_file) - version = version.replace('{{ABI_VERSION}}', abi_version) - + version_cc_text = version_cc_text.replace("{{ABI_VERSION}}", abi_version) oldest_supported_abi_version = utils.GetOldestSupportedAbiVersion( version_file) - version = version.replace('{{OLDEST_SUPPORTED_ABI_VERSION}}', - oldest_supported_abi_version) - + version_cc_text = version_cc_text.replace( + "{{OLDEST_SUPPORTED_ABI_VERSION}}", oldest_supported_abi_version) snapshot_hash = MakeSnapshotHashString() - version = version.replace('{{SNAPSHOT_HASH}}', snapshot_hash) + version_cc_text = version_cc_text.replace("{{SNAPSHOT_HASH}}", + snapshot_hash) + open(output_file, 'w').write(version_cc_text) + return True - return version - -def main(): +def main(args): try: # Parse input. - parser = argparse.ArgumentParser() - parser.add_argument( - '--custom_for_pub', - help=('Generates a version string that works with pub that ' - 'includes the given string. This is silently ignored on ' - 'channels other than be.')) - parser.add_argument('--input', help='Input template file.') - parser.add_argument('--no_git_hash', - action='store_true', - default=False, - help='Don\'t try to determine VCS revision.') - parser.add_argument('--output', help='output file name') - parser.add_argument('-q', - '--quiet', - action='store_true', - default=False, - help='DEPRECATED: Does nothing!') - parser.add_argument('--version-file', help='Path to the VERSION file.') - parser.add_argument('--git-revision-file', - help='Path to the GIT_REVISION file.') - parser.add_argument( - '--format', - default='{{VERSION_STR}}', - help='Version format used if no input template is given.') - - args = parser.parse_args() + parser = OptionParser() + parser.add_option( + "--custom_for_pub", + action="store", + type="string", + help=("Generates a version string that works with pub that includes" + "the given string. This is silently ignored on channels other" + "than be")) + parser.add_option( + "--input", + action="store", + type="string", + help="input template file.") + parser.add_option( + "--no_git_hash", + action="store_true", + default=False, + help="Don't try to determine svn revision") + parser.add_option( + "--output", action="store", type="string", help="output file name") + parser.add_option( + "-q", + "--quiet", + action="store_true", + default=False, + help="disable console output") + parser.add_option( + "--version-file", + action="store", + type="string", + default=None, + help="Version file") + + (options, args) = parser.parse_args() # If there is no input template, then write the bare version string to - # args.output. If there is no args.output, then write the version + # options.output. If there is no options.output, then write the version # string to stdout. - - version_template = '' - if args.input: - version_template = open(args.input).read() - elif not args.format is None: - version_template = args.format - else: - raise 'No version template given! Set either --input or --format.' - - version = FormatVersionString(version_template, args.no_git_hash, - args.custom_for_pub, args.version_file, - args.git_revision_file) - - if args.output: - with open(args.output, 'w') as fh: - fh.write(version) - else: - sys.stdout.write(version) + if not options.input: + version_string = MakeVersionString( + options.quiet, options.no_git_hash, options.custom_for_pub) + if options.output: + open(options.output, 'w').write(version_string) + else: + print(version_string) + return 0 + + if not options.output: + sys.stderr.write('--output not specified\n') + return -1 + if not options.input: + sys.stderr.write('--input not specified\n') + return -1 + + files = [] + for arg in args: + files.append(arg) + + if not MakeFile(options.quiet, options.output, options.input, + options.no_git_hash, options.custom_for_pub, + options.version_file): + return -1 return 0 - except Exception as inst: sys.stderr.write('make_version.py exception\n') sys.stderr.write(str(inst)) sys.stderr.write('\n') - return -1 if __name__ == '__main__': - sys.exit(main()) + exit_code = main(sys.argv) + sys.exit(exit_code) diff --git a/tools/utils.py b/tools/utils.py index c6dc813a604b..1b543e3c1fb8 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -350,16 +350,13 @@ def GetShortVersion(): version.prerelease, version.prerelease_patch)) -def GetSemanticSDKVersion(no_git_hash=False, - version_file=None, - git_revision_file=None): +def GetSemanticSDKVersion(no_git_hash=False, version_file=None): version = ReadVersionFile(version_file) if not version: return None if version.channel == 'be': - postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision( - git_revision_file) + postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision() elif version.channel in ('beta', 'dev'): postfix = '-%s.%s.%s' % (version.prerelease, version.prerelease_patch, version.channel) @@ -370,8 +367,8 @@ def GetSemanticSDKVersion(no_git_hash=False, return '%s.%s.%s%s' % (version.major, version.minor, version.patch, postfix) -def GetVersion(no_git_hash=False, version_file=None, git_revision_file=None): - return GetSemanticSDKVersion(no_git_hash, version_file, git_revision_file) +def GetVersion(no_git_hash=False, version_file=None): + return GetSemanticSDKVersion(no_git_hash, version_file) # The editor used to produce the VERSION file put on gcs. We now produce this @@ -470,13 +467,12 @@ def GetArchiveVersion(): return GetSemanticSDKVersion() -def GetGitRevision(git_revision_file=None): +def GetGitRevision(): # When building from tarball use tools/GIT_REVISION - if git_revision_file is None: - git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION') + git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION') try: with open(git_revision_file) as fd: - return fd.read().strip() + return fd.read() except: pass p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], diff --git a/utils/application_snapshot.gni b/utils/application_snapshot.gni index 8f3f88227306..107431751434 100644 --- a/utils/application_snapshot.gni +++ b/utils/application_snapshot.gni @@ -98,11 +98,9 @@ template("_application_snapshot") { depfile = "$output.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) - vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", - "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script diff --git a/utils/compile_platform.gni b/utils/compile_platform.gni index 447ed644a042..a2e94b3cd616 100644 --- a/utils/compile_platform.gni +++ b/utils/compile_platform.gni @@ -3,7 +3,6 @@ # BSD-style license that can be found in the LICENSE file. import("../build/dart/dart_action.gni") -import("../sdk_args.gni") _dart_root = get_path_info("..", "abspath") @@ -47,8 +46,6 @@ template("compile_platform") { outputs = invoker.outputs - vm_args = [ "-Dsdk_hash=$sdk_hash" ] - inputs = [] deps = [] args = [] diff --git a/utils/dartanalyzer/BUILD.gn b/utils/dartanalyzer/BUILD.gn index 48c9dcf86689..f1d9b014c7ab 100644 --- a/utils/dartanalyzer/BUILD.gn +++ b/utils/dartanalyzer/BUILD.gn @@ -44,7 +44,6 @@ prebuilt_dart_action("generate_summary_strong") { inputs = sdk_lib_files + analyzer_files output = "$root_gen_dir/strong.sum" outputs = [ output ] - vm_args = [ "-Dsdk_hash=$sdk_hash" ] args = [ "build", rebase_path(output), diff --git a/utils/dartdevc/BUILD.gn b/utils/dartdevc/BUILD.gn index c9ff6555035b..15bfe16d828f 100644 --- a/utils/dartdevc/BUILD.gn +++ b/utils/dartdevc/BUILD.gn @@ -82,8 +82,6 @@ template("dart2js_compile") { packages = "../../.packages" - vm_args = [ "-Dsdk_hash=$sdk_hash" ] - args = [ "$abs_main", "-m", @@ -120,8 +118,6 @@ prebuilt_dart_action("dartdevc_patch_sdk") { # TODO(rnystrom): List the outputs more precisely? outputs = [ "$patched_sdk_dir/version" ] - vm_args = [ "-Dsdk_hash=$sdk_hash" ] - args = [ "--libraries", rebase_path("$sdk_root/lib/libraries.json"), @@ -234,8 +230,6 @@ template("dartdevc_kernel_compile") { "$js_gen_dir/$module.js", ] - vm_args = [ "-Dsdk_hash=$sdk_hash" ] - args = [ "-k", "--dart-sdk-summary=$sdk_path", @@ -390,8 +384,6 @@ template("dartdevc_sdk_js") { "$js_gen_dir/legacy/dart_sdk.js.map", ] - vm_args = [ "-Dsdk_hash=$sdk_hash" ] - script = "../../pkg/dev_compiler/bin/dartdevc.dart" args = [ diff --git a/utils/kernel-service/BUILD.gn b/utils/kernel-service/BUILD.gn index a01bb1b5411c..062c3ff108bb 100644 --- a/utils/kernel-service/BUILD.gn +++ b/utils/kernel-service/BUILD.gn @@ -85,11 +85,9 @@ template("kernel_service_dill") { depfile = "$root_gen_dir/kernel_service" + invoker.target_name + "_dill.d" abs_depfile = rebase_path(depfile) rebased_output = rebase_path(output, root_build_dir) - vm_args = [ "--depfile=$abs_depfile", "--depfile_output_filename=$rebased_output", - "-Dsdk_hash=$sdk_hash", ] script = gen_kernel_script