dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[infra] Begin RBE Linux x64 support.

Browse Source 
Bug: b/296994239
Change-Id: I0ddf9a1beb2996ac67b59779b8b8113432434786
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/331282
Reviewed-by: William Hesse <whesse@google.com>
Commit-Queue: Jonas Termansen <sortie@google.com>
Reviewed-by: Alexander Thomas <athom@google.com>
This commit is contained in:
Jonas Termansen
2023-10-23 14:42:59 +00:00
committed by Commit Queue
parent c1bdc9bade
commit 9a11fe517c
9 changed files with 225 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DEPS
+13
View File
@@ -82,6 +82,8 @@ vars = {
"clang_version": "git_revision:6d667d4b261e81f325756fdfd5bb43b3b3d2451d",
"gn_version": "git_revision:991530ce394efb58fcd848195469022fa17ae126",
"reclient_version": "git_revision:81e819b39d4743462857cc55430d898b9fcca1af",
# Update from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/gn
"fuchsia_sdk_version": "version:12.20230407.0.1",
"download_fuchsia_deps": False,
@@ -565,6 +567,17 @@ Var("dart_root") + "/third_party/pkg/tar":
"dep_type": "cipd",
},
Var("dart_root") + '/buildtools/reclient': {
'packages': [
{
'package': 'infra/rbe/client/${{platform}}',
'version': Var('reclient_version'),
}
],
'condition': 'host_os == "linux" and host_cpu == "x64"',
'dep_type': 'cipd',
},
Var("dart_root") + "/third_party/webdriver/chrome": {
"packages": [
{
build/toolchain/fuchsia/BUILD.gn
+15 -8
View File
@@ -9,11 +9,17 @@ import("//build/toolchain/goma.gni")
if (use_goma) {
assert(!use_ccache, "Goma and ccache can't be used together.")
assembler_prefix = "$goma_dir/gomacc "
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_ccache) {
assembler_prefix = "ccache "
compiler_prefix = "ccache "
link_prefix = "ccache "
} else {
assembler_prefix = ""
compiler_prefix = ""
link_prefix = ""
}
toolchain("fuchsia") {
@@ -24,10 +30,11 @@ toolchain("fuchsia") {
# We can't do string interpolation ($ in strings) on things with dots in
# them. To allow us to use $cc below, for example, we create copies of
# these values in our scope.
cc = "${toolchain_bin}/clang"
cxx = "${toolchain_bin}/clang++"
cc = "${compiler_prefix}${toolchain_bin}/clang"
cxx = "${compiler_prefix}${toolchain_bin}/clang++"
asm = "${assembler_prefix}${toolchain_bin}/clang"
ar = "${toolchain_bin}/llvm-ar"
ld = "${toolchain_bin}/clang++"
ld = "${link_prefix}${toolchain_bin}/clang++"
readelf = "${toolchain_bin}/llvm-readelf"
nm = "${toolchain_bin}/llvm-nm"
strip = "${toolchain_bin}/llvm-strip"
@@ -51,7 +58,7 @@ toolchain("fuchsia") {
tool("cc") {
depfile = "{{output}}.d"
command = "$compiler_prefix $cc -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
command = "$cc -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
depsformat = "gcc"
description = "CC {{output}}"
outputs =
@@ -60,7 +67,7 @@ toolchain("fuchsia") {
tool("cxx") {
depfile = "{{output}}.d"
command = "$compiler_prefix $cxx -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}"
command = "$cxx -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} -c {{source}} -o {{output}}"
depsformat = "gcc"
description = "CXX {{output}}"
outputs =
@@ -69,7 +76,7 @@ toolchain("fuchsia") {
tool("asm") {
depfile = "{{output}}.d"
command = "$compiler_prefix $cc -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
command = "$asm -MD -MF $depfile $target_triple_flags $sysroot_flags $lto_flags {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} -c {{source}} -o {{output}}"
depsformat = "gcc"
description = "ASM {{output}}"
outputs =
@@ -101,7 +108,7 @@ toolchain("fuchsia") {
# existing .TOC file, overwrite it, otherwise, don't change it.
tocfile = sofile + ".TOC"
temporary_tocname = sofile + ".tmp"
link_command = "$compiler_prefix $ld $target_triple_flags $sysroot_flags $lto_flags -shared {{ldflags}} -o $unstripped_sofile -Wl,--build-id -Wl,-soname=$soname @$rspfile"
link_command = "$ld $target_triple_flags $sysroot_flags $lto_flags -shared {{ldflags}} -o $unstripped_sofile -Wl,--build-id -Wl,-soname=$soname @$rspfile"
toc_command = "{ $readelf -d $unstripped_sofile | grep SONAME ; $nm -gD -f posix $unstripped_sofile | cut -f1-2 -d' '; } > $temporary_tocname"
replace_command = "if ! cmp -s $temporary_tocname $tocfile; then mv $temporary_tocname $tocfile; fi"
strip_command = "$strip -o $sofile $unstripped_sofile"
@@ -145,7 +152,7 @@ toolchain("fuchsia") {
# to point ./dart instead of ./exe.stripped/dart makes the build
# fail because ./dart does not end up in the manifest file.
unstripped_outfile = "{{root_out_dir}}/exe.stripped/$exename"
command = "$compiler_prefix $ld $target_triple_flags $sysroot_flags $lto_flags {{ldflags}} -o $unstripped_outfile -Wl,--build-id -Wl,--start-group @$rspfile {{solibs}} -Wl,--end-group {{libs}} && ${strip} -o $outfile $unstripped_outfile && $symbolizer_script --nm $nm --output $symfile --binary $unstripped_outfile"
command = "$ld $target_triple_flags $sysroot_flags $lto_flags {{ldflags}} -o $unstripped_outfile -Wl,--build-id -Wl,--start-group @$rspfile {{solibs}} -Wl,--end-group {{libs}} && ${strip} -o $outfile $unstripped_outfile && $symbolizer_script --nm $nm --output $symfile --binary $unstripped_outfile"
description = "LINK $outfile"
rspfile_content = "{{inputs}}"
outputs = [
build/toolchain/gcc_toolchain.gni
+3 -1
View File
@@ -45,6 +45,7 @@ concurrent_links = exec_script("get_concurrent_links.py", [], "value")
# when specified.
template("gcc_toolchain") {
toolchain(target_name) {
assert(defined(invoker.asm), "gcc_toolchain() must specify a \"asm\" value")
assert(defined(invoker.cc), "gcc_toolchain() must specify a \"cc\" value")
assert(defined(invoker.cxx), "gcc_toolchain() must specify a \"cxx\" value")
assert(defined(invoker.ar), "gcc_toolchain() must specify a \"ar\" value")
@@ -60,6 +61,7 @@ template("gcc_toolchain") {
# We can't do string interpolation ($ in strings) on things with dots in
# them. To allow us to use $cc below, for example, we create copies of
# these values in our scope.
asm = invoker.asm
cc = invoker.cc
cxx = invoker.cxx
ar = invoker.ar
@@ -117,7 +119,7 @@ template("gcc_toolchain") {
tool("asm") {
# For GCC we can just use the C compiler to compile assembly.
depfile = "{{output}}.d"
command = "$cc -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} -c {{source}} -o {{output}}"
command = "$asm -MMD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} -c {{source}} -o {{output}}"
depsformat = "gcc"
description = "ASM {{output}}"
outputs =
build/toolchain/linux/BUILD.gn
+45 -12
View File
@@ -6,6 +6,7 @@ import("//build/config/sysroot.gni")
import("//build/toolchain/ccache.gni")
import("//build/toolchain/gcc_toolchain.gni")
import("//build/toolchain/goma.gni")
import("//build/toolchain/rbe.gni")
declare_args() {
ia32_toolchain_prefix = ""
@@ -18,11 +19,31 @@ declare_args() {
if (use_goma) {
assert(!use_ccache, "Goma and ccache can't be used together.")
assembler_prefix = "$goma_dir/gomacc "
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_rbe) {
assert(!use_goma)
rewrapper_args = [
"$rbe_dir/rewrapper",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]
assembler_args = []
compiler_args =
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp " ]
link_args = [] # rewrapper_args + [ "--labels=type=link,tool=clang " ]
assembler_prefix = string_join(" ", assembler_args)
compiler_prefix = string_join(" ", compiler_args)
link_prefix = string_join(" ", link_args)
} else if (use_ccache) {
assembler_prefix = "ccache "
compiler_prefix = "ccache "
link_prefix = "ccache "
} else {
assembler_prefix = ""
compiler_prefix = ""
link_prefix = ""
}
# Google's clang does not work for alpine, use alpine's system clang
@@ -44,9 +65,10 @@ gcc_toolchain("arm") {
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
@@ -60,11 +82,12 @@ gcc_toolchain("clang_arm") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "arm"
@@ -80,9 +103,10 @@ gcc_toolchain("arm64") {
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
@@ -96,11 +120,12 @@ gcc_toolchain("clang_arm64") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "arm64"
@@ -112,11 +137,12 @@ gcc_toolchain("clang_x86") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "x86"
@@ -131,11 +157,12 @@ gcc_toolchain("x86") {
}
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
strip = "${prefix}strip"
toolchain_cpu = "x86"
@@ -147,11 +174,12 @@ gcc_toolchain("clang_x64") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "x64"
@@ -166,11 +194,12 @@ gcc_toolchain("x64") {
}
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
strip = "${prefix}strip"
toolchain_cpu = "x64"
@@ -186,9 +215,10 @@ gcc_toolchain("riscv32") {
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
@@ -202,11 +232,12 @@ gcc_toolchain("clang_riscv32") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "riscv32"
@@ -222,9 +253,10 @@ gcc_toolchain("riscv64") {
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"
asm = "${assembler_prefix}${prefix}gcc"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
@@ -238,11 +270,12 @@ gcc_toolchain("clang_riscv64") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"
toolchain_cpu = "riscv64"
build/toolchain/mac/BUILD.gn
+17 -9
View File
@@ -15,9 +15,13 @@ import("//build/config/sysroot.gni")
import("//build/toolchain/goma.gni")
if (use_goma) {
goma_prefix = "$goma_dir/gomacc "
assembler_prefix = "$goma_dir/gomacc "
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else {
goma_prefix = ""
assembler_prefix = ""
compiler_prefix = ""
link_prefix = ""
}
# Goma doesn't support the host-arm64 toolchain, so continue using Rosetta.
@@ -33,6 +37,7 @@ if (host_cpu == "arm64" && !use_goma) {
# build args in this definition.
template("mac_toolchain") {
toolchain(target_name) {
assert(defined(invoker.asm), "mac_toolchain() must specify a \"asm\" value")
assert(defined(invoker.cc), "mac_toolchain() must specify a \"cc\" value")
assert(defined(invoker.cxx), "mac_toolchain() must specify a \"cxx\" value")
assert(defined(invoker.ar), "mac_toolchain() must specify an \"ar\" value")
@@ -45,6 +50,7 @@ template("mac_toolchain") {
# We can't do string interpolation ($ in strings) on things with dots in
# them. To allow us to use $cc below, for example, we create copies of
# these values in our scope.
asm = invoker.asm
cc = invoker.cc
cxx = invoker.cxx
ar = invoker.ar
@@ -87,7 +93,7 @@ template("mac_toolchain") {
tool("asm") {
# For GCC we can just use the C compiler to compile assembly.
depfile = "{{output}}.d"
command = "$cc -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $toolchain_flags {{asmflags}} -c {{source}} -o {{output}}"
command = "$asm -MMD -MF $depfile {{defines}} {{include_dirs}} $sysroot_flags $toolchain_flags {{asmflags}} -c {{source}} -o {{output}}"
depsformat = "gcc"
description = "ASM {{output}}"
outputs =
@@ -236,10 +242,11 @@ mac_toolchain("clang_x64") {
toolchain_cpu = "x64"
toolchain_os = "mac"
prefix = rebased_clang_dir
cc = "${goma_prefix}$prefix/clang"
cxx = "${goma_prefix}$prefix/clang++"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
strip = "${prefix}/llvm-strip"
nm = "${prefix}/llvm-nm"
is_clang = true
@@ -253,10 +260,11 @@ mac_toolchain("clang_arm64") {
toolchain_cpu = "arm64"
toolchain_os = "mac"
prefix = rebased_clang_dir
cc = "${goma_prefix}$prefix/clang"
cxx = "${goma_prefix}$prefix/clang++"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
strip = "${prefix}/llvm-strip"
nm = "${prefix}/llvm-nm"
is_clang = true
build/toolchain/rbe.gni
+16
View File
@@ -0,0 +1,16 @@
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Defines the configuration of RBE.
declare_args() {
# Set to true to enable distributed compilation using RBE.
use_rbe = false
rbe_exec_root = rebase_path("//")
rbe_platform = "container-image=docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304"
rbe_dir = rebase_path("//buildtools/reclient")
}
build/toolchain/win/BUILD.gn
+4 -4
View File
@@ -25,9 +25,9 @@ tool_wrapper_path = rebase_path("tool_wrapper.py", root_build_dir)
ninja_path = rebase_path("//buildtools/ninja/ninja")
if (use_goma) {
goma_prefix = "$goma_dir/gomacc.exe "
compiler_prefix = "$goma_dir/gomacc.exe "
} else {
goma_prefix = ""
compiler_prefix = ""
}
if (current_toolchain == default_toolchain) {
@@ -258,7 +258,7 @@ template("win_toolchains") {
msvc_toolchain(target_name) {
environment = "environment." + toolchain_arch
cl = "${goma_prefix}\"${win_toolchain_data.vc_bin_dir}/cl.exe\""
cl = "${compiler_prefix}\"${win_toolchain_data.vc_bin_dir}/cl.exe\""
toolchain_args = {
if (defined(invoker.toolchain_args)) {
forward_variables_from(invoker.toolchain_args, "*")
@@ -270,7 +270,7 @@ template("win_toolchains") {
msvc_toolchain("clang_" + target_name) {
environment = "environment." + toolchain_arch
prefix = rebase_path("$clang_base_path/bin", root_build_dir)
cl = "${goma_prefix}$prefix/clang-cl.exe"
cl = "${compiler_prefix}$prefix/clang-cl.exe"
toolchain_args = {
if (defined(invoker.toolchain_args)) {
forward_variables_from(invoker.toolchain_args, "*")
tools/build.py
+101 -67
View File
@@ -43,15 +43,20 @@ def BuildOptions():
other_group.add_argument("-j",
type=int,
help='Ninja -j option for Goma builds.',
help='Ninja -j option for Goma/RBE builds.',
default=1000)
other_group.add_argument("-l",
type=int,
help='Ninja -l option for Goma builds.',
help='Ninja -l option for Goma/RBE builds.',
default=64)
other_group.add_argument("--no-start-goma",
help="Don't try to start goma",
default=False,
dest='no_start_rbe',
action='store_true')
other_group.add_argument("--no-start-rbe",
help="Don't try to start rbe",
default=False,
action='store_true')
other_group.add_argument(
"--check-clean",
@@ -124,63 +129,83 @@ def UseGoma(out_dir):
return 'use_goma = true' in open(args_gn, 'r').read()
# Try to start goma, but don't bail out if we can't. Instead print an error
def UseRBE(out_dir):
args_gn = os.path.join(out_dir, 'args.gn')
return 'use_rbe = true' in open(args_gn, 'r').read()
# Try to start RBE, but don't bail out if we can't. Instead print an error
# message, and let the build fail with its own error messages as well.
goma_started = False
rbe_started = None
bootstrap_path = None
def EnsureGomaStarted(out_dir):
global goma_started
if goma_started:
def StartRBE(out_dir, use_goma):
global rbe_started, bootstrap_path
rbe = 'goma' if use_goma else 'rbe'
if rbe_started:
if rbe_started != rbe:
print(f'Cannot mix RBE and Goma')
return False
return True
args_gn_path = os.path.join(out_dir, 'args.gn')
goma_dir = None
rbe_dir = None if use_goma else 'buildtools/reclient'
with open(args_gn_path, 'r') as fp:
for line in fp:
if 'goma_dir' in line:
if ('goma_dir' if use_goma else 'rbe_dir') in line:
words = line.split()
goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma"
if not goma_dir:
print('Could not find goma for ' + out_dir)
rbe_dir = words[2][1:-1] # rbe_dir = "/path/to/rbe"
if not rbe_dir:
print(f'Could not find {rbe} for {out_dir}')
return False
if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir):
print('Could not find goma at ' + goma_dir)
if not os.path.exists(rbe_dir) or not os.path.isdir(rbe_dir):
print(f'Could not find {rbe} at {rbe_dir}')
return False
goma_ctl = os.path.join(goma_dir, 'goma_ctl.py')
goma_ctl_command = [
'python3',
goma_ctl,
'ensure_start',
]
process = subprocess.Popen(goma_ctl_command)
bootstrap = 'goma_ctl.py' if use_goma else 'bootstrap'
bootstrap_path = os.path.join(rbe_dir, bootstrap)
bootstrap_command = [bootstrap_path]
if use_goma:
bootstrap_command = ['python3', goma_ctl, 'ensure_start']
process = subprocess.Popen(bootstrap_command)
process.wait()
if process.returncode != 0:
print(
"Tried to run goma_ctl.py, but it failed. Try running it manually: "
+ "\n\t" + ' '.join(goma_ctl_command))
print(f"Starting {rbe} failed. Try running it manually: " + "\n\t" +
' '.join(goma_ctl_command))
return False
goma_started = True
rbe_started = rbe
return True
# Returns a tuple (build_config, command to run, whether goma is used)
def StopRBE():
global rbe_started, bootstrap_path
if rbe_started != 'rbe':
return
bootstrap_command = [bootstrap_path, '--shutdown']
process = subprocess.Popen(bootstrap_command)
process.wait()
# Returns a tuple (build_config, command to run, whether rbe is used)
def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer):
build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer)
out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
using_goma = False
using_rbe = False
command = ['buildtools/ninja/ninja', '-C', out_dir]
if options.verbose:
command += ['-v']
if UseGoma(out_dir):
if options.no_start_goma or EnsureGomaStarted(out_dir):
using_goma = True
use_rbe = UseRBE(out_dir)
use_goma = UseGoma(out_dir)
if use_rbe or use_goma:
if options.no_start_rbe or StartRBE(out_dir, use_goma):
using_rbe = True
command += [('-j%s' % str(options.j))]
command += [('-l%s' % str(options.l))]
else:
# If we couldn't ensure that goma is started, let the build start, but
# slowly so we can see any helpful error messages that pop out.
# If we couldn't ensure that RBE is started, let the build start,
# but slowly so we can see any helpful error messages that pop out.
command += ['-j1']
command += targets
return (build_config, command, using_goma)
return (build_config, command, using_rbe)
def RunOneBuildCommand(build_config, args, env):
@@ -233,6 +258,43 @@ def SanitizerEnvironmentVariables():
return env
def Build(configs, env, options):
# Build regular configs.
rbe_builds = []
for (build_config, args, rbe) in configs:
if args is None:
return 1
if rbe:
rbe_builds.append([env, args])
elif RunOneBuildCommand(build_config, args, env=env) != 0:
return 1
# Run RBE builds in parallel.
active_rbe_builds = []
for (env, args) in rbe_builds:
print(' '.join(args))
process = subprocess.Popen(args, env=env)
active_rbe_builds.append([args, process])
while active_rbe_builds:
time.sleep(0.1)
for rbe_build in active_rbe_builds:
(args, process) = rbe_build
if process.poll() is not None:
print(' '.join(args) + " done.")
active_rbe_builds.remove(rbe_build)
if process.returncode != 0:
for (_, to_kill) in active_rbe_builds:
to_kill.terminate()
return 1
if options.check_clean:
for (build_config, args, rbe) in configs:
if CheckCleanBuild(build_config, args, env=env) != 0:
return 1
return 0
def Main():
starttime = time.time()
# Parse the options.
@@ -277,42 +339,14 @@ def Main():
BuildOneConfig(options, targets, target_os, mode, arch,
sanitizer))
# Build regular configs.
goma_builds = []
for (build_config, args, goma) in configs:
if args is None:
return 1
if goma:
goma_builds.append([env, args])
elif RunOneBuildCommand(build_config, args, env=env) != 0:
return 1
exit_code = Build(configs, env, options)
# Run goma builds in parallel.
active_goma_builds = []
for (env, args) in goma_builds:
print(' '.join(args))
process = subprocess.Popen(args, env=env)
active_goma_builds.append([args, process])
while active_goma_builds:
time.sleep(0.1)
for goma_build in active_goma_builds:
(args, process) = goma_build
if process.poll() is not None:
print(' '.join(args) + " done.")
active_goma_builds.remove(goma_build)
if process.returncode != 0:
for (_, to_kill) in active_goma_builds:
to_kill.terminate()
return 1
StopRBE()
if options.check_clean:
for (build_config, args, goma) in configs:
if CheckCleanBuild(build_config, args, env=env) != 0:
return 1
endtime = time.time()
print("The build took %.3f seconds" % (endtime - starttime))
return 0
if exit_code == 0:
endtime = time.time()
print("The build took %.3f seconds" % (endtime - starttime))
return exit_code
if __name__ == '__main__':
tools/gn.py
+11 -1
View File
@@ -320,6 +320,9 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash):
if gn_args['target_os'] == 'mac' and gn_args['use_goma']:
gn_args['mac_use_goma_rbe'] = True
gn_args['use_rbe'] = args.rbe
# Code coverage requires -O0 to be set.
if enable_code_coverage:
gn_args['dart_debug_optimization_level'] = 0
@@ -431,7 +434,14 @@ def AddCommonGnOptionArgs(parser):
help='Disable goma',
dest='goma',
action='store_false')
parser.set_defaults(goma=True)
parser.set_defaults(goma=os.environ.get('RBE_cfg') == None)
parser.add_argument('--rbe', help='Use rbe', action='store_true')
parser.add_argument('--no-rbe',
help='Disable rbe',
dest='rbe',
action='store_false')
parser.set_defaults(rbe=os.environ.get('RBE_cfg') != None)
parser.add_argument('--verify-sdk-hash',
help='Enable SDK hash checks (default)',