Use cargo_gn to build using "cargo build"

.appveyor.yml

@@ -23,6 +23,9 @@ environment:
   AWS_ACCESS_KEY_ID: AKIAIVRN52PLDBP55LBQ
   AWS_SECRET_ACCESS_KEY:
     secure: 8ybpi/y5qE2baChsCBhNHmykng3FitELAtTYOiqZd0mw38i88dzdAX8ETNtBogMV
+  # GN and NINJA are required by https://crates.io/crates/cargo_gn
+  GN: $(APPVEYOR_BUILD_FOLDER)/buildtools/win/gn.exe
+  NINJA: $(APPVEYOR_BUILD_FOLDER)/third_party/depot_tools/ninja.exe
 
 
   # Appveyor uses 7zip to pack cache directories. We use these options:

.gn

@@ -39,7 +39,14 @@ default_args = {
   use_jumbo_build = true
 
   symbol_level = 1
-  treat_warnings_as_errors = true
+
+  # TODO(ry) Change this back to 'true'. This is to workaround windows error
+  # that is only encountered when building with cargo_gn.
+  #   clang-cl: error: argument unused during compilation:
+  #   '-mno-incremental-linker-compatible'
+  #   [-Werror,-Wunused-command-line-argument]
+  treat_warnings_as_errors = false
+
   rust_treat_warnings_as_errors = true
 
   # https://cs.chromium.org/chromium/src/docs/ccache_mac.md

.travis.yml

@@ -114,6 +114,10 @@ jobs:
     - name: "cargo release linux x86_64"
       os: linux
       script:
+      # CARGO_GN_PATH and CARGO_NINJA_PATH allivate the need to build these
+      # tools from scratch. See docs at https://crates.io/crates/cargo_gn
+      - export GN=$TRAVIS_BUILD_DIR/buildtools/linux64/gn
+      - export NINJA=$TRAVIS_BUILD_DIR/third_party/depot_tools/ninja
       - ./tools/lint.py
       - ./tools/test_format.py
       - cargo build --release --locked

cli/Cargo.toml

@@ -61,3 +61,6 @@ fwdansi = "1.0.1"
 
 [target.'cfg(unix)'.dependencies]
 nix = "0.14.1"
+
+[build-dependencies]
+which = "2.0.1"

cli/README.md

@@ -0,0 +1,4 @@
+This is the code for the main deno executable.
+
+The js/ and tests/ symlinks are temporarily added while we transition from
+tools/build.py to cargo build.

cli/js

@@ -0,0 +1 @@
+../js

cli/tests

@@ -0,0 +1 @@
+../tests

core/Cargo.toml

@@ -28,3 +28,7 @@ path = "examples/http_bench.rs"
 # tokio is only used for deno_core_http_bench
 [dev_dependencies]
 tokio = "0.1.18"
+
+[build-dependencies]
+cargo_gn = { git = "https://github.com/denoland/cargo_gn" }
+which = "2.0.1"

core/build.rs

@@ -1,16 +1,17 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 // Run "cargo build -vv" if you want to see gn output.
-mod gn {
-  include!("../tools/gn.rs");
-}
-
+include!("../tools/build_common.rs");
 fn main() {
-  let build = gn::Build::setup();
-
-  println!(
-    "cargo:rustc-link-search=native={}/obj/core/libdeno",
-    build.gn_out_dir
-  );
-
-  build.run("core:deno_core_deps");
+  if !is_rls_build() {
+    let (gn_out_path, ninja_env) = setup();
+    cargo_gn::build("core:deno_core_deps", ninja_env);
+    println!(
+      "cargo:rustc-link-search=native={}",
+      gn_out_path.join("obj/core/libdeno/").to_str().unwrap()
+    );
+  } else {
+    // Enable the 'check-only' feature, which enables some workarounds in the
+    // rust source code to compile successfully without a bundle and snapshot.
+    println!("cargo:rustc-cfg=feature=\"check-only\"");
+  }
 }

tools/benchmark.py

@@ -59,6 +59,12 @@ def import_data_from_gh_pages():
         write_json(all_data_file, [])  # writes empty json data
 
 
+def is_gn_build(build_dir):
+    # TODO(afinch7) maybe find a better check for this?
+    return os.path.exists(
+        os.path.join(build_dir, "cli_test" + executable_suffix))
+
+
 def get_binary_sizes(build_dir):
     # Because cargo's OUT_DIR is not predictable, we have to search the build
     # tree for these files...
@@ -82,6 +88,7 @@ def get_binary_sizes(build_dir):
 
     sizes = {}
     for name, path in path_dict.items():
+        print path
         assert os.path.exists(path)
         sizes[name] = os.path.getsize(path)
     return sizes

tools/binary_downloads.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+# This is called from tools/setup.py in the case of a GN build or independently
+# from tools/build_common.gn in the case of a cargo build.
+import third_party
+from util import (enable_ansi_colors, root_path)
+import os
+import sys
+import prebuilt
+import argparse
+
+
+def binary_downloads():
+    enable_ansi_colors()
+    os.chdir(root_path)
+
+    print "binary download"
+    third_party.download_gn()
+    third_party.download_clang_format()
+    third_party.download_clang()
+    third_party.maybe_download_sysroot()
+    prebuilt.load_sccache()
+
+
+if __name__ == '__main__':
+    sys.exit(binary_downloads())

tools/build_common.rs

@@ -0,0 +1,113 @@
+use cargo_gn;
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+use which::which;
+
+fn binary_downloads() {
+  let cwd = env::current_dir().unwrap();
+  let root = cwd.join("..");
+  let status = Command::new("python")
+    .arg("tools/binary_downloads.py")
+    .current_dir(root)
+    .status()
+    .expect("tools/binary_downloads.py failed");
+  assert!(status.success());
+}
+
+// This is essentially a re-write of the original tools/setup.py
+// but in rust.
+fn setup() -> (PathBuf, Option<cargo_gn::NinjaEnv>) {
+  let is_win = cfg!(target_os = "windows");
+  let is_debug = cargo_gn::is_debug();
+
+  let mut gn_args: cargo_gn::GnArgs = Vec::new();
+  if is_debug && !is_win {
+    gn_args.push("is_debug=true".to_string());
+  } else if is_debug && is_win {
+    // Rust always links with the release flavor of the CRT. Chromium //build
+    // uses the debug version of the CRT when 'is_debug` is set, and there's no
+    // override. Therefore, we cannot build a debug V8 and use Rust to link it.
+    // Rust issue: https://github.com/rust-lang/rust/issues/39016
+    gn_args.push("is_debug=false".to_string());
+  } else {
+    gn_args.push("is_official_build=true".to_string());
+    gn_args.push("symbol_level=0".to_string());
+  }
+
+  if env::var_os("DENO_NO_BINARY_DOWNLOAD").is_none() {
+    binary_downloads();
+  }
+
+  // TODO(ry) Support prebuilt/mac/sccache
+  match which("sccache") {
+    Ok(sccache_path) => {
+      gn_args.push(format!("cc_wrapper={:?}", sccache_path));
+      gn_args.push(format!("rustc_wrapper={:?}", sccache_path));
+    }
+    Err(_) => {}
+  }
+
+  match env::var("DENO_BUILD_ARGS") {
+    Ok(val) => {
+      for arg in val.split_whitespace() {
+        gn_args.push(arg.to_string());
+      }
+    }
+    Err(_) => {}
+  };
+
+  let cwd = env::current_dir().unwrap();
+  let workspace_dir = cwd.parent().unwrap();
+
+  let ninja_env: Option<cargo_gn::NinjaEnv> = if !is_win {
+    None
+  } else {
+    // Windows needs special configuration. This is similar to the function of
+    // python_env() in //tools/util.py.
+    let mut env = Vec::new();
+    let python_path: Vec<String> = vec![
+      "third_party/python_packages",
+      "third_party/python_packages/win32",
+      "third_party/python_packages/win32/lib",
+      "third_party/python_packages/Pythonwin",
+    ]
+    .into_iter()
+    .map(|p| {
+      workspace_dir
+        .join(p)
+        .into_os_string()
+        .into_string()
+        .unwrap()
+    })
+    .collect();
+    let orig_path =
+      String::from(";") + &env::var_os("PATH").unwrap().into_string().unwrap();
+    let path = workspace_dir
+      .join("third_party/python_packages/pywin32_system32")
+      .into_os_string()
+      .into_string()
+      .unwrap();
+    env.push(("PYTHONPATH".to_string(), python_path.join(";")));
+    env.push(("PATH".to_string(), path + &orig_path));
+    env.push(("DEPOT_TOOLS_WIN_TOOLCHAIN".to_string(), "0".to_string()));
+    Some(env)
+  };
+
+  (cargo_gn::maybe_gen("..", gn_args), ninja_env)
+}
+
+/// Detect if we're being invoked by the rust language server (RLS).
+/// When RLS is running "cargo check" to analyze the source code, we're not
+/// trying to build a working executable, rather we're just compiling all
+/// rust code.
+/// Unfortunately we can't detect whether we're being run by `cargo check`.
+fn is_rls_build() -> bool {
+  env::var_os("CARGO")
+    .map(PathBuf::from)
+    .as_ref()
+    .and_then(|p| p.file_stem())
+    .and_then(|f| f.to_str())
+    .map(|s| s.starts_with("rls"))
+    .unwrap_or(false)
+}

tools/setup.py

@@ -9,6 +9,7 @@
 from distutils.spawn import find_executable
 import prebuilt
 import argparse
+from binary_downloads import binary_downloads
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -26,12 +27,7 @@ def main():
     if args.no_binary_download:
         print "no binary download"
     else:
-        print "binary download"
-        third_party.download_gn()
-        third_party.download_clang_format()
-        third_party.download_clang()
-        third_party.maybe_download_sysroot()
-        prebuilt.load_sccache()
+        binary_downloads()
 
     write_lastchange()
 

tools/target_test.py

@@ -22,6 +22,10 @@ def check_exists(filename):
             print "Run ./tools/build.py"
             sys.exit(1)
 
+    def is_gn_build(self):
+        return os.path.exists(
+            os.path.join(self.build_dir, "cli_test" + executable_suffix))
+
     def test_executable_exists(self):
         self.check_exists(self.deno_exe)
 

tools/test_util.py

@@ -74,12 +74,16 @@ def create_test_arg_parser():
 
 TestArgParser = create_test_arg_parser()
 
+TEST_ARGS = None
+
 
 def parse_test_args(argv=None):
+    global TEST_ARGS
     if argv is None:
         argv = sys.argv[1:]
 
     args = TestArgParser.parse_args(argv)
+    TEST_ARGS = args
 
     if args.executable and args.release:
         raise argparse.ArgumentError(