Linux create_sdk GN build for host

This CL enables GN builds of all targets from dart.gyp
on Linux for the host (i.e. ia32, x64).

R=johnmccutchan@google.com

Review URL: https://codereview.chromium.org/2379803002 .

BUILD.gn

@@ -9,13 +9,149 @@ group("default") {
   ]
 }
 
-group("runtime") {
+group("most") {
   deps = [
-    "//runtime/bin:dart",
-    "//runtime/bin:dart_bootstrap($host_toolchain)",
-    "//runtime/bin:run_vm_tests",
-    "//runtime/bin:process_test",
-    "//runtime/bin:test_extension",
-    "//runtime/bin:sample_extension",
+    ":analysis_server",
+    ":create_sdk",
+    ":dart2js",
+    ":dartanalyzer",
+    ":dartdevc",
+    ":runtime",
+    ":samples",
+  ]
+}
+
+group("runtime") {
+  deps = [
+    "runtime/bin:dart",
+    "runtime/bin:dart_bootstrap($host_toolchain)",
+    "runtime/bin:run_vm_tests",
+    "runtime/bin:process_test",
+    "runtime/bin:test_extension",
+    "runtime/bin:sample_extension",
+  ]
+}
+
+group("runtime_precompiled") {
+  deps = [
+    "runtime/bin:dart_precompiled_runtime",
+    "runtime/bin:dart_bootstrap($host_toolchain)",
+  ]
+}
+
+group("runtime_and_noopt") {
+  deps = [
+    "runtime/bin:dart",
+    "runtime/bin:dart_noopt",
+    "runtime/bin:dart_bootstrap($host_toolchain)",
+    "runtime/bin:run_vm_tests",
+    "runtime/bin:process_test",
+    "runtime/bin:test_extension",
+    "runtime/bin:sample_extension",
+  ]
+}
+
+
+action("create_sdk") {
+  deps = [
+    "runtime/bin:dart",
+    "utils/analysis_server",
+    "utils/compiler:dart2js",
+    "utils/dartanalyzer:generate_dartanalyzer_snapshot",
+    "utils/dartanalyzer:generate_summary_spec",
+    "utils/dartanalyzer:generate_summary_strong",
+    "utils/dartdevc",
+    "utils/dartdoc",
+    "utils/dartfmt",
+    "utils/pub",
+  ]
+
+  sdk_lib_files = exec_script("tools/list_files.py",
+      ["\\.dart\$", rebase_path("sdk/lib")],
+      "list lines")
+
+  preamble_files = exec_script("tools/list_files.py",
+      ["", rebase_path("sdk/lib/_internal/js_runtime/lib/preambles")],
+      "list lines")
+
+  sdk_bin_files = exec_script("tools/list_files.py",
+      ["", rebase_path("sdk/bin")],
+      "list lines")
+
+  inputs = rebase_path(sdk_lib_files, "", "sdk/lib") +
+           rebase_path(preamble_files, "",  "sdk/lib") +
+           rebase_path(sdk_bin_files, "", "sdk/bin") + [
+    "sdk/lib/dart_client.platform",
+    "sdk/lib/dart_server.platform",
+    "sdk/lib/dart_shared.platform",
+    "$root_gen_dir/dart2js.dart.snapshot",
+    "$root_gen_dir/utils_wrapper.dart.snapshot",
+    "$root_gen_dir/pub.dart.snapshot",
+    "$root_gen_dir/dartanalyzer.dart.snapshot",
+    "$root_gen_dir/dartdevc.dart.snapshot",
+    "$root_gen_dir/dartfmt.dart.snapshot",
+    "$root_gen_dir/analysis_server.dart.snapshot",
+    "$root_gen_dir/dartdoc.dart.snapshot",
+    "$root_gen_dir/spec.sum",
+    "$root_gen_dir/strong.sum",
+    "tools/VERSION"
+  ]
+
+  outputs = [
+    "$root_out_dir/dart-sdk/README",
+  ]
+
+  script = "tools/create_sdk.py"
+  args = [
+    "--sdk_output_dir",
+    rebase_path("$root_out_dir/dart-sdk"),
+    "--snapshot_location",
+    rebase_path("$root_gen_dir"),
+  ]
+}
+
+
+group("dart2js") {
+  deps = [
+    "utils/compiler:dart2js"
+  ]
+}
+
+group("dartanalyzer") {
+  deps = [
+    "utils/dartanalyzer"
+  ]
+}
+
+group("dartdevc") {
+  deps = [
+    "utils/dartdevc"
+  ]
+}
+
+group("dartfmt") {
+  deps = [
+    "utils/dartfmt"
+  ]
+}
+
+group("analysis_server") {
+  deps = [
+    "utils/analysis_server"
+  ]
+}
+
+# This is the target that is built on the dart2js build bots.
+# It must depend on anything that is required by the dart2js
+# test suites.
+group("dart2js_bot") {
+  deps = [
+    ":create_sdk"
+  ]
+}
+
+group("samples") {
+  deps = [
+    "runtime/bin:sample_extension"
   ]
 }

build/config/compiler/BUILD.gn

@@ -697,7 +697,6 @@ if (is_win) {
   default_warning_flags += [
     # Enables.
     "-Wendif-labels",  # Weird old-style text after an #endif.
-    "-Werror",  # Warnings as errors.
 
     # Disables.
     "-Wno-missing-field-initializers",  # "struct foo f = {0};"
@@ -763,6 +762,7 @@ config("chromium_code") {
     cflags = [
       "-Wall",
       "-Wextra",
+      "-Werror",
     ]
 
     defines = []

pkg/BUILD.gn

@@ -0,0 +1,71 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../utils/create_timestamp.gni")
+
+template("make_third_party_pkg_files_stamp") {
+  assert(defined(invoker.pattern), "Must define the file pattern")
+  assert(defined(invoker.id), "Must define the stamp file id")
+  path = rebase_path("../third_party/pkg")
+  if (defined(invoker.path)) {
+    path = invoker.path
+  }
+  id = invoker.id
+  create_timestamp_file(target_name) {
+    pattern = invoker.pattern
+    path = path
+    new_base = "//"
+    output = "$target_gen_dir/third_party_pkg_files_$id.stamp"
+  }
+}
+
+make_third_party_pkg_files_stamp("make_third_party_pkg_files_0_stamp") {
+  pattern = "^(?!.*/test/).*(?<!_test)[.]dart\$"
+  path = rebase_path(".")
+  id = "0"
+}
+
+make_third_party_pkg_files_stamp("make_third_party_pkg_files_1_stamp") {
+  pattern = "^(?!.*_test\.dart).*[a-k]\.dart\$"
+  id = "1"
+}
+
+make_third_party_pkg_files_stamp("make_third_party_pkg_files_2_stamp") {
+  pattern = "^(?!.*_test\.dart).*[l-r]\.dart\$"
+  id = "2"
+}
+
+make_third_party_pkg_files_stamp("make_third_party_pkg_files_3_stamp") {
+  pattern = "^(?!.*_test\.dart).*[^a-r]\.dart\$"
+  id = "3"
+}
+
+action("pkg_files_stamp") {
+  deps = [
+    ":make_third_party_pkg_files_0_stamp",
+    ":make_third_party_pkg_files_1_stamp",
+    ":make_third_party_pkg_files_2_stamp",
+    ":make_third_party_pkg_files_3_stamp",
+  ]
+
+  stamp0_outputs = get_target_outputs(":make_third_party_pkg_files_0_stamp")
+  stamp1_outputs = get_target_outputs(":make_third_party_pkg_files_1_stamp")
+  stamp2_outputs = get_target_outputs(":make_third_party_pkg_files_2_stamp")
+  stamp3_outputs = get_target_outputs(":make_third_party_pkg_files_3_stamp")
+
+  inputs = ["../tools/list_files.py"] +
+      stamp0_outputs +
+      stamp1_outputs +
+      stamp2_outputs +
+      stamp3_outputs
+
+  outputs = [
+    "$root_gen_dir/pkg_files.stamp"
+  ]
+
+  script = "../tools/create_timestamp_file.py"
+  args = [
+    rebase_path("$root_gen_dir/pkg_files.stamp"),
+  ]
+}

runtime/BUILD.gn

@@ -237,6 +237,17 @@ libdart_library("libdart") {
   ]
 }
 
+libdart_library("libdart_noopt") {
+  extra_configs = [
+    ":dart_maybe_precompiled_runtime_config",
+    ":dart_precompiler_config",
+  ]
+  extra_deps = [
+    "vm:libdart_lib",
+    "vm:libdart_vm_noopt",
+  ]
+}
+
 libdart_library("libdart_precompiled_runtime") {
   extra_configs = [
     ":dart_precompiled_runtime_config"

runtime/bin/BUILD.gn

@@ -560,6 +560,17 @@ if (!defined(is_fuchsia) || !is_fuchsia) {
     ]
   }
 
+  dart_executable("dart_noopt") {
+    extra_configs = [
+      "..:dart_precompiler_config",
+    ]
+    extra_deps = [
+      "..:libdart_noopt",
+      ":dart_snapshot_cc",
+      "../observatory:standalone_observatory_archive",
+    ]
+  }
+
   dart_executable("dart_precompiled_runtime") {
     extra_configs = [
       "..:dart_precompiled_runtime_config"

runtime/vm/BUILD.gn

@@ -71,6 +71,20 @@ static_library("libdart_vm") {
 }
 
 
+static_library("libdart_vm_noopt") {
+  configs += ["..:dart_config",
+              "..:dart_maybe_product_config",
+              "..:dart_precompiler_config",
+              "..:dart_maybe_precompiled_runtime_config"]
+  public_configs = [":libdart_vm_config"]
+  set_sources_assignment_filter(["*_test.cc", "*_test.h"])
+  sources = vm_sources_list.sources
+  include_dirs = [
+    "..",
+  ]
+}
+
+
 static_library("libdart_vm_precompiled_runtime") {
   configs += ["..:dart_config",
               "..:dart_maybe_product_config",

tools/dart_for_gn.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# Copyright 2016 The Dart project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Usage: dart_for_gn.py path_to_dart_binary [arguments]
+#
+# We have gn set up so that actions can only run python scripts.
+# We use this wrapper script when we need an action to run the Dart VM
+# The first command line argument is mandatory and should be the absolute path
+# to the Dart VM binary.
+
+import subprocess
+import sys
+
+def main(argv):
+  if len(argv) < 2:
+    print "Requires path to Dart VM binary as first argument"
+    return -1
+  command = argv[1:]
+  return subprocess.call(command)
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

tools/list_files.py

@@ -18,8 +18,8 @@ def main(argv):
   pattern = re.compile(argv[1])
   for directory in argv[2:]:
     for root, directories, files in os.walk(directory):
-      if '.svn' in directories:
-        directories.remove('.svn')
+      if '.git' in directories:
+        directories.remove('.git')
       for filename in files:
         fullname = os.path.relpath(os.path.join(root, filename))
         fullname = fullname.replace(os.sep, '/')

utils/analysis_server/BUILD.gn

@@ -0,0 +1,9 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+application_snapshot("analysis_server") {
+  main_dart = "../../pkg/analysis_server/bin/server.dart"
+}

utils/compiler/BUILD.gn

@@ -0,0 +1,50 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../create_timestamp.gni")
+import("../invoke_dart.gni")
+
+create_timestamp_file("dart2js_files_stamp") {
+  pattern = "\\.dart\$"
+  path = rebase_path("../../pkg/compiler/lib")
+  output = "$root_gen_dir/dart2js_files.stamp"
+}
+
+invoke_dart("dart2js") {
+  deps = [
+    ":dart2js_files_stamp",
+  ]
+
+  dart_files = exec_script("../../tools/list_files.py",
+                          ["\\.dart\$",
+                           rebase_path("../../runtime/lib"),
+                           rebase_path("../../sdk/lib/_internal/dartdoc")],
+                          "list lines")
+
+  inputs = dart_files + [
+    "../../tools/list_files.py",
+    "../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart",
+    "create_snapshot.dart",
+    "$root_gen_dir/dart2js_files.stamp",
+    "../../tools/VERSION",
+  ]
+
+  utils_output = "$root_gen_dir/utils_wrapper.dart.snapshot"
+  dart2js_output = "$root_gen_dir/dart2js.dart.snapshot"
+  outputs = [
+    utils_output,
+    dart2js_output,
+  ]
+
+  dot_packages = rebase_path("../../.packages")
+  create_snapshot = rebase_path("create_snapshot.dart")
+  output_dir = rebase_path(root_gen_dir)
+
+  args = [
+    "--packages=$dot_packages",
+    create_snapshot,
+    "--output_dir=$output_dir",
+    "--dart2js_main=pkg/compiler/lib/src/dart2js.dart",
+  ]
+}

utils/create_timestamp.gni

@@ -0,0 +1,31 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+_dart_root = rebase_path("..")
+
+template("create_timestamp_file") {
+  assert(defined(invoker.pattern), "Must specify 'pattern'")
+  assert(defined(invoker.path), "Must specify 'path'")
+  assert(defined(invoker.output), "Must specify 'output'")
+  new_base = "."
+  if (defined(invoker.new_base)) {
+    new_base = invoker.new_base
+  }
+  current_base = "."
+  if (defined(invoker.current_base)) {
+    current_base = invoker.current_base
+  }
+  pattern = invoker.pattern
+  path = invoker.path
+  output = invoker.output
+  action(target_name) {
+    files = exec_script(
+        "$_dart_root/tools/list_files.py", [pattern, path], "list lines")
+    inputs = ["$_dart_root/tools/list_files.py"] +
+             rebase_path(files, new_base, current_base)
+    outputs = [ output ]
+    script = "$_dart_root/tools/create_timestamp_file.py"
+    args = [ rebase_path(output) ]
+  }
+}

utils/dartanalyzer/BUILD.gn

@@ -0,0 +1,68 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+group("dartanalyzer") {
+  deps = [
+    ":generate_dartanalyzer_snapshot",
+    ":generate_summary_spec",
+    ":generate_summary_strong",
+  ]
+}
+
+analyzer_files = exec_script("../../tools/list_files.py",
+                             ["\\.dart\$",
+                             rebase_path("../../pkg/analyzer")],
+                             "list lines")
+
+application_snapshot("generate_dartanalyzer_snapshot") {
+  main_dart = "../../pkg/analyzer_cli/bin/analyzer.dart"
+  name = "dartanalyzer"
+  cli_files = exec_script("../../tools/list_files.py",
+                          ["\\.dart\$",
+                           rebase_path("../../pkg/analyzer_cli")],
+                          "list lines")
+  inputs = cli_files + analyzer_files
+}
+
+sdk_lib_files = exec_script("../../tools/list_files.py",
+                            ["\\.dart\$",
+                             rebase_path("../../sdk/lib")],
+                            "list lines")
+
+template("generate_summary") {
+  assert(defined(invoker.type), "Must specify the summary type")
+  type = invoker.type
+  assert((type == "spec") || (type == "strong"))
+  invoke_dart(target_name) {
+    inputs = sdk_lib_files + analyzer_files
+
+    output = "$root_gen_dir/$type.sum"
+    outputs = [
+      output
+    ]
+
+    dot_packages = rebase_path("../../.packages")
+    build_sdk_summaries =
+        rebase_path("../../pkg/analyzer/tool/summary/build_sdk_summaries.dart")
+    abs_output = rebase_path(output)
+
+    args = [
+      "--packages=$dot_packages",
+      build_sdk_summaries,
+      "build-$type",
+      abs_output,
+      rebase_path("../../sdk"),
+    ]
+  }
+}
+
+generate_summary("generate_summary_spec") {
+  type = "spec"
+}
+
+generate_summary("generate_summary_strong") {
+  type = "strong"
+}

utils/dartdevc/BUILD.gn

@@ -0,0 +1,13 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+application_snapshot("dartdevc") {
+  main_dart = "../../pkg/dev_compiler/bin/dartdevc.dart"
+  inputs = exec_script("../../tools/list_files.py",
+                       ["\\.dart\$",
+                       rebase_path("../../pkg/dev_compiler/bin")],
+                       "list lines")
+}

utils/dartdoc/BUILD.gn

@@ -0,0 +1,13 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+application_snapshot("dartdoc") {
+  main_dart = "../../third_party/pkg/dartdoc/bin/dartdoc.dart"
+  inputs = exec_script("../../tools/list_files.py",
+                       ["\\.dart\$",
+                       rebase_path("../../third_party/pkg/dartdoc")],
+                       "list lines")
+}

utils/dartfmt/BUILD.gn

@@ -0,0 +1,13 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+application_snapshot("dartfmt") {
+  main_dart = "../../third_party/pkg_tested/dart_style/bin/format.dart"
+  inputs = exec_script("../../tools/list_files.py",
+                       ["\\.dart\$",
+                       rebase_path("../../third_party/pkg_tested/dart_style")],
+                       "list lines")
+}

utils/invoke_dart.gni

@@ -0,0 +1,88 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+_dart_root = rebase_path("..")
+
+template("invoke_dart") {
+  assert(defined(invoker.outputs), "invoke_dart must specify outputs")
+  extra_deps = []
+  if (defined(invoker.deps)) {
+    extra_deps += invoker.deps
+  }
+
+  extra_inputs = []
+  if (defined(invoker.inputs)) {
+    extra_inputs += invoker.inputs
+  }
+
+  extra_args = []
+  if (defined(invoker.args)) {
+    extra_args += invoker.args
+  }
+
+  action(target_name) {
+    relative_dart_root = rebase_path(_dart_root, ".")
+    deps = extra_deps + [
+      "$relative_dart_root/runtime/bin:dart($host_toolchain)",
+    ]
+
+    dart_out_dir = get_label_info(
+        "$relative_dart_root/runtime/bin:dart($host_toolchain)", "root_out_dir")
+    # TODO(zra): Add platform specific executable suffix (e.g. .exe on win)
+    dart = rebase_path("$dart_out_dir/dart")
+
+    inputs = [
+      dart,
+    ] + extra_inputs
+
+    outputs = invoker.outputs
+
+    script = "$_dart_root/tools/dart_for_gn.py"
+    args = [
+      dart,
+    ] + extra_args
+  }
+}
+
+template("application_snapshot") {
+  assert(defined(invoker.main_dart), "Must specify 'main_dart'")
+  main_dart = invoker.main_dart
+  name = target_name
+  if (defined(invoker.name)) {
+    name = invoker.name
+  }
+  extra_deps = []
+  if (defined(invoker.deps)) {
+    extra_deps += invoker.deps
+  }
+  extra_inputs = []
+  if (defined(invoker.inputs)) {
+    extra_inputs += invoker.inputs
+  }
+  invoke_dart(target_name) {
+    deps = extra_deps + [
+      "$_dart_root/pkg:pkg_files_stamp"
+    ]
+
+    inputs = extra_inputs + [
+      "$_dart_root/sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart",
+      "$root_gen_dir/pkg_files.stamp",
+    ]
+
+    output = "$root_gen_dir/$name.dart.snapshot"
+    outputs = [
+      output,
+    ]
+
+    dot_packages = rebase_path("$_dart_root/.packages")
+    abs_output = rebase_path(output)
+    main_file = rebase_path(main_dart)
+
+    args = [
+      "--packages=$dot_packages",
+      "--snapshot=$abs_output",
+      main_file
+    ]
+  }
+}

utils/pub/BUILD.gn

@@ -0,0 +1,15 @@
+# Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../invoke_dart.gni")
+
+application_snapshot("pub") {
+  main_dart = "../../third_party/pkg/pub/bin/pub.dart"
+  deps = [
+    "../compiler:dart2js_files_stamp"
+  ]
+  inputs = [
+    "$root_gen_dir/dart2js_files.stamp"
+  ]
+}