diff --git a/BUILD.gn b/BUILD.gn index 166e8c37410..1603e2b0ec7 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -70,6 +70,8 @@ group("runtime") { if (is_linux || is_android) { deps += [ "runtime/bin:abstract_socket_test" ] + } else if (is_fuchsia) { + deps += [ ":fuchsia_test_package" ] } } @@ -167,41 +169,38 @@ if (is_fuchsia) { import("third_party/fuchsia/sdk/${host_os}/build/component.gni") import("third_party/fuchsia/sdk/${host_os}/build/package.gni") - template("dart_fuchsia_test_package") { - fuchsia_package(target_name) { - package_name = invoker.package_name - if (is_debug) { - package_name += "_debug" - } else if (is_release) { - package_name += "_release" - } else if (is_product) { - package_name += "_product" - } - testonly = true - deps = invoker.deps - } - } + test_binaries = [ + "dart", + "dart_precompiled_runtime", + "run_vm_tests", + ] - template("dart_fuchsia_test_component") { - fuchsia_component(target_name) { + foreach(binary, test_binaries) { + fuchsia_component("${binary}_test_component") { testonly = true - data_deps = [ "runtime/bin:dart" ] - if (defined(invoker.data_deps)) { - data_deps += invoker.data_deps - } - manifest = invoker.manifest - - library_files = [] - if (defined(invoker.library_files)) { - library_files += invoker.library_files - } + manifest = "runtime/bin/${binary}_test_component.cml" + data_deps = [ + "runtime/bin:${binary}", + "runtime/bin:ffi_test_dynamic_library", + "runtime/bin:ffi_test_functions", + ] + library_files = [ + "libffi_test_dynamic_library.so", + "libffi_test_functions.so", + ] resource_files = [ ".dart_tool/package_config.json", "pkg/testing/test/hello_test.dart", "tools/addlatexhash.dart", ] - resource_dirs = invoker.resource_dirs - + resource_dirs = [ + "pkg/async_helper", + "pkg/expect", + "pkg/meta", + "tests/ffi", + "third_party/pkg/ffi", + "third_party/pkg/path", + ] resources = [] foreach(file, library_files) { resources += [ @@ -224,54 +223,18 @@ if (is_fuchsia) { } } - dart_fuchsia_test_component("fuchsia_test_component") { - manifest = "build/fuchsia/fuchsia_test_component.cml" - resource_dirs = [ - "pkg/async_helper", - "pkg/expect", - "pkg/meta", - "pkg/native_stack_traces", - "pkg/smith", - "third_party/pkg/args", - "third_party/pkg/async", - "third_party/pkg/collection", - "third_party/pkg/convert", - "third_party/pkg/crypto", - "third_party/pkg/http", - "third_party/pkg/http_parser", - "third_party/pkg/path", - "third_party/pkg/pool", - "third_party/pkg/stack_trace", - "third_party/pkg/string_scanner", - "third_party/pkg/typed_data", - ] - } - - dart_fuchsia_test_package("fuchsia_test_package") { - package_name = "dart_test" - deps = [ ":fuchsia_test_component" ] - } - - dart_fuchsia_test_component("fuchsia_ffi_test_component") { - manifest = "build/fuchsia/fuchsia_ffi_test_component.cml" - data_deps = [ - "runtime/bin:ffi_test_dynamic_library", - "runtime/bin:ffi_test_functions", - ] - library_files = [ - "libffi_test_dynamic_library.so", - "libffi_test_functions.so", - ] - resource_dirs = [ - "pkg/expect", - "pkg/meta", - "tests/ffi", - "third_party/pkg/ffi", - ] - } - - dart_fuchsia_test_package("fuchsia_ffi_test_package") { - package_name = "dart_ffi_test" - deps = [ ":fuchsia_ffi_test_component" ] + fuchsia_package("fuchsia_test_package") { + testonly = true + if (is_debug) { + package_name = "dart_test_debug" + } else if (is_release) { + package_name = "dart_test_release" + } else if (is_product) { + package_name = "dart_test_product" + } + deps = [] + foreach(binary, test_binaries) { + deps += [ ":${binary}_test_component" ] + } } } diff --git a/build/fuchsia/fuchsia_test_component.cml b/build/fuchsia/fuchsia_test_component.cml deleted file mode 100644 index 6e9d08a4d15..00000000000 --- a/build/fuchsia/fuchsia_test_component.cml +++ /dev/null @@ -1,54 +0,0 @@ -{ - program: { - binary: "exe.stripped/dart", - runner: "elf_test_runner", - }, - capabilities: [ - { protocol: "fuchsia.test.Suite" }, - ], - expose: [ - { - protocol: "fuchsia.test.Suite", - from: "self", - }, - ], - use: [ - { - directory: "config-data", - rights: [ "r*" ], - path: "/config/data", - }, - { - storage: "cache", - path: "/cache", - }, - { - storage: "data", - path: "/data", - }, - { - storage: "tmp", - path: "/tmp", - }, - { - directory: "root-ssl-certificates", - rights: [ "r*" ], - path: "/config/ssl", - }, - { - protocol: [ - "fuchsia.deprecatedtimezone.Timezone", - "fuchsia.device.NameProvider", - "fuchsia.feedback.CrashReporter", - "fuchsia.intl.PropertyProvider", - "fuchsia.kernel.VmexResource", - "fuchsia.logger.LogSink", - "fuchsia.net.name.Lookup", - "fuchsia.posix.socket.Provider", - "fuchsia.sysmem.Allocator", - "fuchsia.timezone.Timezone", - "fuchsia.tracing.provider.Registry", - ], - }, - ] -} diff --git a/pkg/test_runner/lib/src/command_output.dart b/pkg/test_runner/lib/src/command_output.dart index 3b832cc3f99..5b9cfa449d7 100644 --- a/pkg/test_runner/lib/src/command_output.dart +++ b/pkg/test_runner/lib/src/command_output.dart @@ -906,6 +906,15 @@ class VMCommandOutput extends CommandOutput with _UnittestSuiteMessagesMixin { @override Expectation result(TestCase testCase) { + // `ffx test` isn't preserving exit codes. + // TODO(38752): Plumb exit codes through something else? + if (testCase.configuration.system == System.fuchsia) { + if (utf8.decode(stdout).contains("completed with result: PASSED")) { + return Expectation.pass; + } + return Expectation.fail; + } + // Handle crashes and timeouts first. if (exitCode == _dfeErrorExitCode) return Expectation.dartkCrash; if (hasCrashed) return Expectation.crash; @@ -948,6 +957,15 @@ class VMCommandOutput extends CommandOutput with _UnittestSuiteMessagesMixin { /// Delete existing result() function and rename, when status files are gone. @override Expectation realResult(TestCase testCase) { + // `ffx test` isn't preserving exit codes. + // TODO(38752): Plumb exit codes through something else? + if (testCase.configuration.system == System.fuchsia) { + if (utf8.decode(stdout).contains("completed with result: PASSED")) { + return Expectation.pass; + } + return Expectation.fail; + } + // Handle crashes and timeouts first. if (exitCode == _dfeErrorExitCode) return Expectation.dartkCrash; if (hasCrashed) return Expectation.crash; diff --git a/pkg/test_runner/lib/src/fuchsia.dart b/pkg/test_runner/lib/src/fuchsia.dart index f08bfd13572..dd42e521e51 100644 --- a/pkg/test_runner/lib/src/fuchsia.dart +++ b/pkg/test_runner/lib/src/fuchsia.dart @@ -1,27 +1,163 @@ -// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; import 'command.dart'; -import 'fuchsia_cfv2.dart'; -// Sets up and executes commands against a Fuchsia environment. -abstract class FuchsiaEmulator { +// Runs tests on a fuchsia emulator with chromium maintained test-scripts and +// CFv2 targets. +class FuchsiaEmulator { + static const String ffx = "./third_party/fuchsia/sdk/linux/tools/x64/ffx"; + static const String testScriptRoot = + "./third_party/fuchsia/test_scripts/test/"; + static const String withEnv = "./build/fuchsia/with_envs.py"; + static const String tmpRoot = "/tmp/dart_ffi_test/"; + // TODO(#38752): Once we have vm/cc and AOT tests running, this will sometimes + // need to be the component for run_vm_tests or dart_precompiled_runtime. + static const String cmName = "dart_test_component.cm"; + + final Map envs = {}; + Process? daemonProc; + Process? emuProc; + String? emuName; + Process? repoProc; + // Publishes the packages to the Fuchsia environment. - Future publishPackage(String buildDir, String mode, String arch); + Future publishPackage(String buildDir, String mode, String arch) async { + try { + await Directory(tmpRoot).delete(recursive: true); + } catch (_) {} + // The /tmp/ should always be present, recursive creation is not expected. + await Directory(tmpRoot).create(); + assert(daemonProc == null); + daemonProc = await _run("isolate_daemon.py", []); + var isolateDir = await _captureStdout(daemonProc!); + print("+ ffx daemon running on $isolateDir should be ready now."); + envs["FFX_ISOLATE_DIR"] = isolateDir; + assert(emuProc == null); + emuProc = await _run("start_emulator.py", [ + "--disable-graphics", + "--target-id-only", + "--device-spec", + "virtual_device_large" + ]); + emuName = await _captureStdout(emuProc!); + print("+ Targeting emu name $emuName"); + await _assertRun("test_connection.py", [emuName!]); + await _assertRun("publish_package.py", [ + "--packages", + _testPackagePath(buildDir, mode), + "--purge-repo", + "--repo", + _tempDirectoryOf("repo") + ]); + repoProc = await _run("serve_repo.py", [ + "run", + "--serve-repo", + _tempDirectoryOf("repo"), + "--repo-name", + "dart-ffi-test-repo", + "--target-id", + emuName! + ]); + print("+ Fuchsia repo ${await _captureStdout(repoProc!)} is running " + "at ${_tempDirectoryOf('repo')}"); + await _assertRun("pkg_resolve.py", [emuName!, _testPackageName(mode)]); + } + // Tears down the Fuchsia environment. - void stop(); + Future stop() async { + assert(repoProc != null); + repoProc!.kill(); + await repoProc!.exitCode; + assert(emuProc != null); + emuProc!.kill(); + await emuProc!.exitCode; + assert(daemonProc != null); + daemonProc!.kill(); + await daemonProc!.exitCode; + } + // Returns a command to execute a set of tests against the running Fuchsia // environment. VMCommand getTestCommand( - String buildDir, String mode, String arch, List arguments); + String buildDir, String mode, String arch, List arguments) { + return VMCommand( + withEnv, + _runArgs("run_executable_test.py", [ + "--target-id", + emuName!, + "--out-dir", + _tempDirectoryOf("out"), + "--test-name", + "fuchsia-pkg://fuchsia.com/${_testPackageName(mode)}#meta/$cmName", + "--test-realm", + // VmexResource not available in default hermetic realm + // TODO(38752): Setup a Dart test realm. + "/core/testing:system-tests", + "--logs-dir", + _tempDirectoryOf("logs"), + "--package-deps", + _testPackagePath(buildDir, mode), + ...arguments + ]), + envs); + } + + static String _testPackageName(String mode) { + return "dart_test_$mode"; + } + + static String _testPackagePath(String buildDir, String mode) { + var farName = _testPackageName(mode); + return "$buildDir/gen/$farName/$farName.far"; + } + + static String _tempDirectoryOf(String name) { + return tmpRoot + name; + } + + List _runArgs(String script, List args) { + return [testScriptRoot + script, ...args]; + } + + /// Executes a test script inside of third_party/fuchsia/test_scripts/test/ + /// with the required environment setup and the arguments. + Future _run(String script, List args) async { + var newArgs = _runArgs(script, args); + print("+ Start $withEnv with $newArgs with environment $envs."); + return Process.start(withEnv, newArgs, environment: envs); + } + + /// Executes a test script and asserts its return code is 0; see _run and + /// _assert. + Future _assertRun(String script, List args) async { + _assert((await (await _run(script, args)).exitCode) == 0); + } + + /// Captures the first line of output in utf8. + Future _captureStdout(Process proc) async { + // The stderr needs to be fully consumed as well. + proc.stderr.transform(utf8.decoder).forEach((x) => stderr.write(x)); + return (await proc.stdout.transform(utf8.decoder).first).trim(); + } + + /// Unlike assert keyword, always evaluates the input function and throws + /// exception when the evaluated result is false. + void _assert(bool condition) { + if (!condition) { + throw AssertionError(); + } + } static final FuchsiaEmulator _instance = _create(); static FuchsiaEmulator _create() { - return FuchsiaEmulatorCFv2(); + return FuchsiaEmulator(); } static FuchsiaEmulator instance() { diff --git a/pkg/test_runner/lib/src/fuchsia_cfv2.dart b/pkg/test_runner/lib/src/fuchsia_cfv2.dart deleted file mode 100644 index 75608727424..00000000000 --- a/pkg/test_runner/lib/src/fuchsia_cfv2.dart +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'command.dart'; -import 'fuchsia.dart'; - -// Runs tests on a fuchsia emulator with chromium maintained test-scripts and -// CFv2 targets. -// TODO(#38752): Merge back to fuchsia.dart. -class FuchsiaEmulatorCFv2 extends FuchsiaEmulator { - static const String ffx = "./third_party/fuchsia/sdk/linux/tools/x64/ffx"; - static const String testScriptRoot = - "./third_party/fuchsia/test_scripts/test/"; - static const String withEnv = "./build/fuchsia/with_envs.py"; - static const String tmpRoot = "/tmp/dart_ffi_test/"; - static const String cmName = "fuchsia_ffi_test_component.cm"; - - final Map envs = {}; - Process? daemonProc; - Process? emuProc; - String? emuName; - Process? repoProc; - - @override - Future publishPackage(String buildDir, String mode, String arch) async { - try { - Directory(tmpRoot).deleteSync(recursive: true); - } catch (_) {} - // The /tmp/ should always be present, recursive creation is not expected. - Directory(tmpRoot).createSync(); - assert(daemonProc == null); - daemonProc = await _run("isolate_daemon.py", []); - var isolateDir = await _captureStdout(daemonProc!); - print("+ ffx daemon running on $isolateDir should be ready now."); - envs["FFX_ISOLATE_DIR"] = isolateDir; - assert(emuProc == null); - emuProc = await _run("start_emulator.py", [ - "--disable-graphics", - "--target-id-only", - "--device-spec", - "virtual_device_large" - ]); - emuName = await _captureStdout(emuProc!); - print("+ Targeting emu name $emuName"); - await _assertRun("test_connection.py", [emuName!]); - await _assertRun("publish_package.py", [ - "--packages", - _testPackagePath(buildDir, mode), - "--purge-repo", - "--repo", - _tempDirectoryOf("repo") - ]); - repoProc = await _run("serve_repo.py", [ - "run", - "--serve-repo", - _tempDirectoryOf("repo"), - "--repo-name", - "dart-ffi-test-repo", - "--target-id", - emuName! - ]); - print("+ Fuchsia repo ${await _captureStdout(repoProc!)} is running " - "at ${_tempDirectoryOf('repo')}"); - await _assertRun("pkg_resolve.py", [emuName!, cmName]); - } - - @override - Future stop() async { - assert(repoProc != null); - repoProc!.kill(); - await repoProc!.exitCode; - assert(emuProc != null); - emuProc!.kill(); - await emuProc!.exitCode; - assert(daemonProc != null); - daemonProc!.kill(); - await daemonProc!.exitCode; - } - - @override - VMCommand getTestCommand( - String buildDir, String mode, String arch, List arguments) { - return VMCommand( - withEnv, - _runArgs("run_executable_test.py", [ - "--target-id", - emuName!, - "--out-dir", - _tempDirectoryOf("out"), - "--test-name", - "fuchsia-pkg://fuchsia.com/${_testPackageName(mode)}#meta/$cmName", - "--test-realm", - // VmexResource not available in default hermetic realm - // TODO(38752): Setup a Dart test realm. - "/core/testing:system-tests", - "--logs-dir", - _tempDirectoryOf("logs"), - "--package-deps", - _testPackagePath(buildDir, mode), - ...arguments - ]), - envs); - } - - static String _testPackageName(String mode) { - return "dart_ffi_test_$mode"; - } - - static String _testPackagePath(String buildDir, String mode) { - var farName = _testPackageName(mode); - return "$buildDir/gen/$farName/$farName.far"; - } - - static String _tempDirectoryOf(String name) { - return tmpRoot + name; - } - - List _runArgs(String script, List args) { - return [testScriptRoot + script, ...args]; - } - - /// Executes a test script inside of third_party/fuchsia/test_scripts/test/ - /// with the required environment setup and the arguments. - Future _run(String script, List args) async { - var newArgs = _runArgs(script, args); - print("+ Start $withEnv with $newArgs with environment $envs."); - return Process.start(withEnv, newArgs, environment: envs); - } - - /// Executes a test script and asserts its return code is 0; see _run and - /// _assert. - Future _assertRun(String script, List args) async { - _assert((await (await _run(script, args)).exitCode) == 0); - } - - /// Captures the first line of output in utf8. - Future _captureStdout(Process proc) async { - // The stderr needs to be fully consumed as well. - proc.stderr.transform(utf8.decoder).forEach((x) => stderr.write(x)); - return (await proc.stdout.transform(utf8.decoder).first).trim(); - } - - /// Unlike assert keyword, always evaluates the input function and throws - /// exception when the evaluated result is false. - void _assert(bool condition) { - if (!condition) { - throw AssertionError(); - } - } -} diff --git a/pkg/test_runner/lib/src/runtime_configuration.dart b/pkg/test_runner/lib/src/runtime_configuration.dart index 1b8bf8488bc..ce2716aff2a 100644 --- a/pkg/test_runner/lib/src/runtime_configuration.dart +++ b/pkg/test_runner/lib/src/runtime_configuration.dart @@ -484,6 +484,13 @@ class DartkFuchsiaEmulatorRuntimeConfiguration if (isCrashExpected) { arguments.insert(0, '--suppress-core-dump'); } + + // Rewrite paths on the host to paths in the Fuchsia package. + arguments = arguments + .map((argument) => + argument.replaceAll(Directory.current.path, "pkg/data")) + .toList(); + var command = FuchsiaEmulator.instance().getTestCommand( _configuration.buildDirectory, _configuration.mode.name, diff --git a/runtime/bin/dart_precompiled_runtime_test_component.cml b/runtime/bin/dart_precompiled_runtime_test_component.cml new file mode 100644 index 00000000000..dc2db52525f --- /dev/null +++ b/runtime/bin/dart_precompiled_runtime_test_component.cml @@ -0,0 +1,22 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +{ + include: [ + "//runtime/vm.shard.cml", + ], + program: { + binary: "exe.stripped/dart_precompiled_runtime", + runner: "elf_test_runner", + }, + capabilities: [ + { protocol: "fuchsia.test.Suite" }, + ], + expose: [ + { + protocol: "fuchsia.test.Suite", + from: "self", + }, + ], +} diff --git a/runtime/bin/dart_test_component.cml b/runtime/bin/dart_test_component.cml new file mode 100644 index 00000000000..3bff2c933ac --- /dev/null +++ b/runtime/bin/dart_test_component.cml @@ -0,0 +1,22 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +{ + include: [ + "//runtime/vm-jit.shard.cml", + ], + program: { + binary: "exe.stripped/dart", + runner: "elf_test_runner", + }, + capabilities: [ + { protocol: "fuchsia.test.Suite" }, + ], + expose: [ + { + protocol: "fuchsia.test.Suite", + from: "self", + }, + ], +} diff --git a/runtime/bin/run_vm_tests_test_component.cml b/runtime/bin/run_vm_tests_test_component.cml new file mode 100644 index 00000000000..7adedf8a778 --- /dev/null +++ b/runtime/bin/run_vm_tests_test_component.cml @@ -0,0 +1,22 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +{ + include: [ + "//runtime/vm-jit.shard.cml", + ], + program: { + binary: "exe.stripped/run_vm_tests", + runner: "elf_test_runner", + }, + capabilities: [ + { protocol: "fuchsia.test.Suite" }, + ], + expose: [ + { + protocol: "fuchsia.test.Suite", + from: "self", + }, + ], +} diff --git a/runtime/vm-jit.shard.cml b/runtime/vm-jit.shard.cml new file mode 100644 index 00000000000..740cf21fd25 --- /dev/null +++ b/runtime/vm-jit.shard.cml @@ -0,0 +1,16 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +{ + include: [ + "//runtime/vm.shard.cml", + ], + use: [ + { + protocol: [ + "fuchsia.kernel.VmexResource", + ], + }, + ] +} diff --git a/build/fuchsia/fuchsia_ffi_test_component.cml b/runtime/vm.shard.cml similarity index 67% rename from build/fuchsia/fuchsia_ffi_test_component.cml rename to runtime/vm.shard.cml index 6e9d08a4d15..59f4821c72a 100644 --- a/build/fuchsia/fuchsia_ffi_test_component.cml +++ b/runtime/vm.shard.cml @@ -1,17 +1,8 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + { - program: { - binary: "exe.stripped/dart", - runner: "elf_test_runner", - }, - capabilities: [ - { protocol: "fuchsia.test.Suite" }, - ], - expose: [ - { - protocol: "fuchsia.test.Suite", - from: "self", - }, - ], use: [ { directory: "config-data", @@ -38,10 +29,7 @@ { protocol: [ "fuchsia.deprecatedtimezone.Timezone", - "fuchsia.device.NameProvider", - "fuchsia.feedback.CrashReporter", "fuchsia.intl.PropertyProvider", - "fuchsia.kernel.VmexResource", "fuchsia.logger.LogSink", "fuchsia.net.name.Lookup", "fuchsia.posix.socket.Provider", diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index e9f41380b0d..614136aa034 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -1687,10 +1687,7 @@ "script": "tools/build.py", "arguments": [ "--os=fuchsia", - "runtime", - "create_sdk", - "fuchsia_test_package", - "fuchsia_ffi_test_package" + "runtime" ] }, { @@ -1716,10 +1713,7 @@ "script": "tools/build.py", "arguments": [ "--os=fuchsia", - "runtime", - "create_sdk", - "fuchsia_test_package", - "fuchsia_ffi_test_package" + "runtime" ] } ]