Optimize sandbox performance (dotnet#1045)

* Optimize sandbox performance Link just the files needed to compile, link, or archive, rather than the entire directory tree. This drastically improves build times on macOS, where the sandbox is known to be slow (bazelbuild/bazel#8230). * Linux wants clang to link * all_files not needed? * Linux wants llc * And llvm-ar * Templated build_file_content
radekdoulik · May 31, 2022 · edd205a · edd205a
1 parent fabe920
commit edd205a
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 43 deletions.
diff --git a/bazel/BUILD b/bazel/BUILD
@@ -35,23 +35,34 @@ config_setting(
 filegroup(name = "empty")
 
 alias(
-    name = "binaries",
+    name = "compiler_files",
     actual = select({
-        ":linux": "@emscripten_bin_linux//:all",
-        ":macos": "@emscripten_bin_mac//:all",
-        ":macos_arm64": "@emscripten_bin_mac_arm64//:all",
-        ":windows": "@emscripten_bin_win//:all",
+        ":linux": "@emscripten_bin_linux//:compiler_files",
+        ":macos": "@emscripten_bin_mac//:compiler_files",
+        ":macos_arm64": "@emscripten_bin_mac_arm64//:compiler_files",
+        ":windows": "@emscripten_bin_win//:compiler_files",
         "//conditions:default": ":empty",
     }),
 )
 
 alias(
-    name = "node_modules",
+    name = "linker_files",
     actual = select({
-        ":linux": "@emscripten_npm_linux//:node_modules",
-        ":macos": "@emscripten_npm_mac//:node_modules",
-        ":macos_arm64": "@emscripten_npm_mac//:node_modules",
-        ":windows": "@emscripten_npm_win//:node_modules",
+        ":linux": "@emscripten_bin_linux//:linker_files",
+        ":macos": "@emscripten_bin_mac//:linker_files",
+        ":macos_arm64": "@emscripten_bin_mac_arm64//:linker_files",
+        ":windows": "@emscripten_bin_win//:linker_files",
+        "//conditions:default": ":empty",
+    }),
+)
+
+alias(
+    name = "ar_files",
+    actual = select({
+        ":linux": "@emscripten_bin_linux//:ar_files",
+        ":macos": "@emscripten_bin_mac//:ar_files",
+        ":macos_arm64": "@emscripten_bin_mac_arm64//:ar_files",
+        ":windows": "@emscripten_bin_win//:ar_files",
         "//conditions:default": ":empty",
     }),
 )
diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl
@@ -5,6 +5,53 @@ load(":revisions.bzl", "EMSCRIPTEN_TAGS")
 def _parse_version(v):
     return [int(u) for u in v.split(".")]
 
+BUILD_FILE_CONTENT_TEMPLATE = """
+package(default_visibility = ['//visibility:public'])
+
+filegroup(
+    name = "includes",
+    srcs = glob([
+        "emscripten/cache/sysroot/include/c++/v1/**",
+        "emscripten/cache/sysroot/include/compat/**",
+        "emscripten/cache/sysroot/include/**",
+        "lib/clang/15.0.0/include/**",
+    ]),
+)
+
+filegroup(
+    name = "compiler_files",
+    srcs = [
+        "emscripten/emcc.py",
+        "bin/clang{bin_extension}",
+        "bin/clang++{bin_extension}",
+        ":includes",
+    ],
+)
+
+filegroup(
+    name = "linker_files",
+    srcs = [
+        "emscripten/emcc.py",
+        "bin/clang{bin_extension}",
+        "bin/llc{bin_extension}",
+        "bin/llvm-ar{bin_extension}",
+        "bin/llvm-nm{bin_extension}",
+        "bin/llvm-objcopy{bin_extension}",
+        "bin/wasm-emscripten-finalize{bin_extension}",
+        "bin/wasm-ld{bin_extension}",
+        "bin/wasm-opt{bin_extension}",
+   ],
+)
+
+filegroup(
+    name = "ar_files",
+    srcs = [
+        "emscripten/emar.py",
+        "bin/llvm-ar{bin_extension}",
+   ],
+)
+"""
+
 def emscripten_deps(emscripten_version = "latest"):
     version = emscripten_version
 
@@ -36,7 +83,7 @@ def emscripten_deps(emscripten_version = "latest"):
             strip_prefix = "install",
             url = emscripten_url.format("linux", revision.hash, "", "tbz2"),
             sha256 = revision.sha_linux,
-            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""),
             type = "tar.bz2",
         )
 
@@ -46,7 +93,7 @@ def emscripten_deps(emscripten_version = "latest"):
             strip_prefix = "install",
             url = emscripten_url.format("mac", revision.hash, "", "tbz2"),
             sha256 = revision.sha_mac,
-            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""),
             type = "tar.bz2",
         )
 
@@ -56,7 +103,7 @@ def emscripten_deps(emscripten_version = "latest"):
             strip_prefix = "install",
             url = emscripten_url.format("mac", revision.hash, "-arm64", "tbz2"),
             sha256 = revision.sha_mac_arm64,
-            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""),
             type = "tar.bz2",
         )
 
@@ -66,7 +113,7 @@ def emscripten_deps(emscripten_version = "latest"):
             strip_prefix = "install",
             url = emscripten_url.format("win", revision.hash, "", "zip"),
             sha256 = revision.sha_win,
-            build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD",
+            build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=".exe"),
             type = "zip",
         )
 

diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel
@@ -8,45 +8,43 @@ package(default_visibility = ["//visibility:public"])
 node_files = "@nodejs_host//:node_files" if existing_rule("@nodejs_host//:node_files") else "@nodejs//:node_files"
 
 filegroup(
-    name = "common-script-includes",
+    name = "common_files",
     srcs = [
-        "emar.sh",
-        "emar.bat",
-        "emcc.sh",
-        "emcc.bat",
         "emscripten_config",
         "env.sh",
         "env.bat",
-        "@emsdk//:binaries",
-        "@emsdk//:node_modules",
         node_files,
     ],
 )
 
 filegroup(
-    name = "compile-emscripten",
-    srcs = [":common-script-includes"],
+    name = "compiler_files",
+    srcs = [
+        "emcc.sh",
+        "emcc.bat",
+        "@emsdk//:compiler_files",
+        ":common_files",
+    ],
 )
 
 filegroup(
-    name = "link-emscripten",
+    name = "linker_files",
     srcs = [
         "emcc_link.sh",
         "emcc_link.bat",
         "link_wrapper.py",
-        ":common-script-includes",
-        "@emsdk//:binaries",
-        node_files,
+        "@emsdk//:linker_files",
+        ":common_files",
     ],
 )
 
 filegroup(
-    name = "every-file",
+    name = "ar_files",
     srcs = [
-        ":compile-emscripten",
-        ":link-emscripten",
-        "@emsdk//:binaries",
-        node_files,
+        "emar.sh",
+        "emar.bat",
+        "@emsdk//:ar_files",
+        ":common_files",
     ],
 )
 
@@ -59,7 +57,7 @@ emscripten_cc_toolchain_config_rule(
     name = "wasm",
     cpu = "wasm",
     em_config = "emscripten_config",
-    emscripten_binaries = "@emsdk//:binaries",
+    emscripten_binaries = "@emsdk//:compiler_files",
     script_extension = select({
         "@bazel_tools//src/conditions:host_windows": "bat",
         "//conditions:default": "sh",
@@ -68,12 +66,12 @@ emscripten_cc_toolchain_config_rule(
 
 cc_toolchain(
     name = "cc-compiler-wasm",
-    all_files = ":every-file",
-    ar_files = ":common-script-includes",
+    all_files = ":empty",
+    ar_files = ":ar_files",
     as_files = ":empty",
-    compiler_files = ":compile-emscripten",
+    compiler_files = ":compiler_files",
     dwp_files = ":empty",
-    linker_files = ":link-emscripten",
+    linker_files = ":linker_files",
     objcopy_files = ":empty",
     strip_files = ":empty",
     toolchain_config = "wasm",

diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD