From 58819fd401ff56e18cc82377ab90ee22d51f07de Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Fri, 29 Apr 2022 17:32:49 -0700 Subject: [PATCH 1/6] Optimize sandbox performance Link just the files needed to compile, link, or archive, rather than the entire directory tree. This drastically improves build times on macOS, where the sandbox is known to be slow (https://github.com/bazelbuild/bazel/issues/8230). --- bazel/BUILD | 31 ++++++++---- bazel/emscripten_toolchain/BUILD.bazel | 52 ++++++++++++--------- bazel/emscripten_toolchain/emscripten.BUILD | 39 +++++++++++++++- 3 files changed, 88 insertions(+), 34 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index dfec803d24..87fe18133b 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -35,23 +35,34 @@ config_setting( filegroup(name = "empty") alias( - name = "binaries", + name = "compiler_files", actual = select({ - ":linux": "@emscripten_bin_linux//:all", - ":macos": "@emscripten_bin_mac//:all", - ":macos_arm64": "@emscripten_bin_mac_arm64//:all", - ":windows": "@emscripten_bin_win//:all", + ":linux": "@emscripten_bin_linux//:compiler_files", + ":macos": "@emscripten_bin_mac//:compiler_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:compiler_files", + ":windows": "@emscripten_bin_win//:compiler_files", "//conditions:default": ":empty", }), ) alias( - name = "node_modules", + name = "linker_files", actual = select({ - ":linux": "@emscripten_npm_linux//:node_modules", - ":macos": "@emscripten_npm_mac//:node_modules", - ":macos_arm64": "@emscripten_npm_mac//:node_modules", - ":windows": "@emscripten_npm_win//:node_modules", + ":linux": "@emscripten_bin_linux//:linker_files", + ":macos": "@emscripten_bin_mac//:linker_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:linker_files", + ":windows": "@emscripten_bin_win//:linker_files", + "//conditions:default": ":empty", + }), +) + +alias( + name = "ar_files", + actual = select({ + ":linux": "@emscripten_bin_linux//:ar_files", + ":macos": "@emscripten_bin_mac//:ar_files", + ":macos_arm64": "@emscripten_bin_mac_arm64//:ar_files", + ":windows": "@emscripten_bin_win//:ar_files", "//conditions:default": ":empty", }), ) diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index 4c85acfce2..cf27764209 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -8,44 +8,52 @@ package(default_visibility = ["//visibility:public"]) node_files = "@nodejs_host//:node_files" if existing_rule("@nodejs_host//:node_files") else "@nodejs//:node_files" filegroup( - name = "common-script-includes", + name = "common_files", srcs = [ - "emar.sh", - "emar.bat", - "emcc.sh", - "emcc.bat", "emscripten_config", "env.sh", "env.bat", - "@emsdk//:binaries", - "@emsdk//:node_modules", node_files, ], ) filegroup( - name = "compile-emscripten", - srcs = [":common-script-includes"], + name = "compiler_files", + srcs = [ + "emcc.sh", + "emcc.bat", + "@emsdk//:compiler_files", + ":common_files", + ], ) filegroup( - name = "link-emscripten", + name = "linker_files", srcs = [ "emcc_link.sh", "emcc_link.bat", "link_wrapper.py", - ":common-script-includes", - "@emsdk//:binaries", - node_files, + "@emsdk//:linker_files", + ":common_files", + ], +) + +filegroup( + name = "ar_files", + srcs = [ + "emar.sh", + "emar.bat", + "@emsdk//:ar_files", + ":common_files", ], ) filegroup( - name = "every-file", + name = "all_files", srcs = [ - ":compile-emscripten", - ":link-emscripten", - "@emsdk//:binaries", + ":compiler_files", + ":linker_files", + ":ar_files", node_files, ], ) @@ -59,7 +67,7 @@ emscripten_cc_toolchain_config_rule( name = "wasm", cpu = "wasm", em_config = "emscripten_config", - emscripten_binaries = "@emsdk//:binaries", + emscripten_binaries = "@emsdk//:compiler_files", script_extension = select({ "@bazel_tools//src/conditions:host_windows": "bat", "//conditions:default": "sh", @@ -68,12 +76,12 @@ emscripten_cc_toolchain_config_rule( cc_toolchain( name = "cc-compiler-wasm", - all_files = ":every-file", - ar_files = ":common-script-includes", + all_files = ":all_files", + ar_files = ":ar_files", as_files = ":empty", - compiler_files = ":compile-emscripten", + compiler_files = ":compiler_files", dwp_files = ":empty", - linker_files = ":link-emscripten", + linker_files = ":linker_files", objcopy_files = ":empty", strip_files = ":empty", toolchain_config = "wasm", diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD index 6f11852874..52b1dad278 100644 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ b/bazel/emscripten_toolchain/emscripten.BUILD @@ -1,6 +1,41 @@ package(default_visibility = ['//visibility:public']) filegroup( - name = "all", - srcs = glob(["**"]), + name = "includes", + srcs = glob([ + "emscripten/cache/sysroot/include/c++/v1/**", + "emscripten/cache/sysroot/include/compat/**", + "emscripten/cache/sysroot/include/**", + "lib/clang/15.0.0/include/**", + ]), +) + +filegroup( + name = "compiler_files", + srcs = [ + "emscripten/emcc.py", + "bin/clang", + "bin/clang++", + ":includes", + ], +) + +filegroup( + name = "linker_files", + srcs = [ + "emscripten/emcc.py", + "bin/llvm-nm", + "bin/llvm-objcopy", + "bin/wasm-emscripten-finalize", + "bin/wasm-ld", + "bin/wasm-opt", + ], +) + +filegroup( + name = "ar_files", + srcs = [ + "emscripten/emar.py", + "bin/llvm-ar", + ], ) From a1842805f93b401b38fbecf93bf173d332948fbb Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Mon, 2 May 2022 08:43:09 -0700 Subject: [PATCH 2/6] Linux wants clang to link --- bazel/emscripten_toolchain/emscripten.BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD index 52b1dad278..d17144a3ed 100644 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ b/bazel/emscripten_toolchain/emscripten.BUILD @@ -24,6 +24,7 @@ filegroup( name = "linker_files", srcs = [ "emscripten/emcc.py", + "bin/clang", "bin/llvm-nm", "bin/llvm-objcopy", "bin/wasm-emscripten-finalize", From de465646c0501d712f4490745ec0f09d2775775c Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Mon, 2 May 2022 08:44:32 -0700 Subject: [PATCH 3/6] all_files not needed? --- bazel/emscripten_toolchain/BUILD.bazel | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index cf27764209..d101c67061 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -48,16 +48,6 @@ filegroup( ], ) -filegroup( - name = "all_files", - srcs = [ - ":compiler_files", - ":linker_files", - ":ar_files", - node_files, - ], -) - filegroup(name = "empty") # dlmalloc.bc is implictly added by the emscripten toolchain @@ -76,7 +66,7 @@ emscripten_cc_toolchain_config_rule( cc_toolchain( name = "cc-compiler-wasm", - all_files = ":all_files", + all_files = ":empty", ar_files = ":ar_files", as_files = ":empty", compiler_files = ":compiler_files", From 0eca1473a4293fabf846aae2316baa8b45275352 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Mon, 2 May 2022 08:48:53 -0700 Subject: [PATCH 4/6] Linux wants llc --- bazel/emscripten_toolchain/emscripten.BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD index d17144a3ed..6d4f3fe8f7 100644 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ b/bazel/emscripten_toolchain/emscripten.BUILD @@ -25,6 +25,7 @@ filegroup( srcs = [ "emscripten/emcc.py", "bin/clang", + "bin/llc", "bin/llvm-nm", "bin/llvm-objcopy", "bin/wasm-emscripten-finalize", From 186178b1a8ef87a9076952ab02f59d6207f365e6 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Mon, 2 May 2022 09:22:39 -0700 Subject: [PATCH 5/6] And llvm-ar --- bazel/emscripten_toolchain/emscripten.BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD index 6d4f3fe8f7..330265a7cb 100644 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ b/bazel/emscripten_toolchain/emscripten.BUILD @@ -26,6 +26,7 @@ filegroup( "emscripten/emcc.py", "bin/clang", "bin/llc", + "bin/llvm-ar", "bin/llvm-nm", "bin/llvm-objcopy", "bin/wasm-emscripten-finalize", From b2ac8db3e5ce8acfbb7eb4fa438e768ab8c7bf7d Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Fri, 6 May 2022 19:55:52 -0700 Subject: [PATCH 6/6] Templated build_file_content --- bazel/emscripten_deps.bzl | 55 +++++++++++++++++++-- bazel/emscripten_toolchain/emscripten.BUILD | 44 ----------------- 2 files changed, 51 insertions(+), 48 deletions(-) delete mode 100644 bazel/emscripten_toolchain/emscripten.BUILD diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl index 95801ba758..7e8bc6888a 100644 --- a/bazel/emscripten_deps.bzl +++ b/bazel/emscripten_deps.bzl @@ -5,6 +5,53 @@ load(":revisions.bzl", "EMSCRIPTEN_TAGS") def _parse_version(v): return [int(u) for u in v.split(".")] +BUILD_FILE_CONTENT_TEMPLATE = """ +package(default_visibility = ['//visibility:public']) + +filegroup( + name = "includes", + srcs = glob([ + "emscripten/cache/sysroot/include/c++/v1/**", + "emscripten/cache/sysroot/include/compat/**", + "emscripten/cache/sysroot/include/**", + "lib/clang/15.0.0/include/**", + ]), +) + +filegroup( + name = "compiler_files", + srcs = [ + "emscripten/emcc.py", + "bin/clang{bin_extension}", + "bin/clang++{bin_extension}", + ":includes", + ], +) + +filegroup( + name = "linker_files", + srcs = [ + "emscripten/emcc.py", + "bin/clang{bin_extension}", + "bin/llc{bin_extension}", + "bin/llvm-ar{bin_extension}", + "bin/llvm-nm{bin_extension}", + "bin/llvm-objcopy{bin_extension}", + "bin/wasm-emscripten-finalize{bin_extension}", + "bin/wasm-ld{bin_extension}", + "bin/wasm-opt{bin_extension}", + ], +) + +filegroup( + name = "ar_files", + srcs = [ + "emscripten/emar.py", + "bin/llvm-ar{bin_extension}", + ], +) +""" + def emscripten_deps(emscripten_version = "latest"): version = emscripten_version @@ -36,7 +83,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("linux", revision.hash, "", "tbz2"), sha256 = revision.sha_linux, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""), type = "tar.bz2", ) @@ -46,7 +93,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("mac", revision.hash, "", "tbz2"), sha256 = revision.sha_mac, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""), type = "tar.bz2", ) @@ -56,7 +103,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("mac", revision.hash, "-arm64", "tbz2"), sha256 = revision.sha_mac_arm64, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=""), type = "tar.bz2", ) @@ -66,7 +113,7 @@ def emscripten_deps(emscripten_version = "latest"): strip_prefix = "install", url = emscripten_url.format("win", revision.hash, "", "zip"), sha256 = revision.sha_win, - build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension=".exe"), type = "zip", ) diff --git a/bazel/emscripten_toolchain/emscripten.BUILD b/bazel/emscripten_toolchain/emscripten.BUILD deleted file mode 100644 index 330265a7cb..0000000000 --- a/bazel/emscripten_toolchain/emscripten.BUILD +++ /dev/null @@ -1,44 +0,0 @@ -package(default_visibility = ['//visibility:public']) - -filegroup( - name = "includes", - srcs = glob([ - "emscripten/cache/sysroot/include/c++/v1/**", - "emscripten/cache/sysroot/include/compat/**", - "emscripten/cache/sysroot/include/**", - "lib/clang/15.0.0/include/**", - ]), -) - -filegroup( - name = "compiler_files", - srcs = [ - "emscripten/emcc.py", - "bin/clang", - "bin/clang++", - ":includes", - ], -) - -filegroup( - name = "linker_files", - srcs = [ - "emscripten/emcc.py", - "bin/clang", - "bin/llc", - "bin/llvm-ar", - "bin/llvm-nm", - "bin/llvm-objcopy", - "bin/wasm-emscripten-finalize", - "bin/wasm-ld", - "bin/wasm-opt", - ], -) - -filegroup( - name = "ar_files", - srcs = [ - "emscripten/emar.py", - "bin/llvm-ar", - ], -)