diff --git a/.circleci/config.yml b/.circleci/config.yml index a5d7e20155..bc573c1503 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,46 @@ executors: bionic: docker: - image: buildpack-deps:bionic + mac: + environment: + EMSDK_NOTTY: "1" + # Without this, any `brew install` command will result in self-update of + # brew itself which takes more than 4 minutes. + HOMEBREW_NO_AUTO_UPDATE: "1" + macos: + xcode: "12.5.1" + resource_class: macos.x86.medium.gen2 + mac_arm64: + environment: + EMSDK_NOTTY: "1" + # Without this, any `brew install` command will result in self-update of + # brew itself which takes more than 4 minutes. + HOMEBREW_NO_AUTO_UPDATE: "1" + macos: + xcode: "13.4.1" + resource_class: macos.m1.medium.gen1 + linux_arm64: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.medium + +commands: + setup-macos: + steps: + - checkout + - run: + name: Install CMake + command: brew install cmake + test-macos: + steps: + - run: + name: test.sh + command: test/test.sh + - run: + name: test.py + command: | + source emsdk_env.sh + test/test.py jobs: flake8: @@ -20,9 +60,9 @@ jobs: apt-get install -q -y python-pip python3-pip - run: python2 -m pip install --upgrade pip - run: python3 -m pip install --upgrade pip - - run: python2 -m pip install flake8==3.7.8 - - run: python3 -m pip install flake8==3.7.8 - - run: python2 -m flake8 --show-source --statistics + - run: python2 -m pip install flake8==3.9.2 + - run: python3 -m pip install flake8==3.9.2 + - run: python2 -m flake8 --show-source --statistics --extend-exclude=./scripts - run: python3 -m flake8 --show-source --statistics test-linux: executor: bionic @@ -40,6 +80,7 @@ jobs: - run: name: Install debian packages command: apt-get update -q && apt-get install -q -y cmake build-essential openjdk-8-jre-headless ksh zsh + - run: test/test_node_path.sh - run: test/test.sh - run: test/test_source_env.sh - run: @@ -47,25 +88,24 @@ jobs: command: | source emsdk_env.sh test/test.py - test-mac: - macos: - xcode: "12.5.1" - environment: - EMSDK_NOTTY: "1" - # Without this, any `brew installl` command will result in self-update of - # brew itself which takes more than 4 minutes. - HOMEBREW_NO_AUTO_UPDATE: "1" + test-linux-arm64: + executor: linux_arm64 steps: - checkout - run: - name: Install cmake - command: brew install cmake + name: Install debian packages + command: sudo apt-get update -q && sudo apt-get install -q cmake build-essential openjdk-8-jre-headless - run: test/test.sh - - run: - name: test.py - command: | - source emsdk_env.sh - test/test.py + test-mac: + executor: mac + steps: + - setup-macos + - test-macos + test-mac-arm64: + executor: mac_arm64 + steps: + - setup-macos + - test-macos test-windows: executor: name: win/vs2019 @@ -190,11 +230,7 @@ jobs: - run: test/test_bazel.sh test-bazel-mac: - macos: - xcode: "12.5.1" - environment: - EMSDK_NOTTY: "1" - HOMEBREW_NO_AUTO_UPDATE: "1" + executor: mac steps: - checkout - run: brew install grep @@ -211,6 +247,7 @@ jobs: environment: PYTHONUNBUFFERED: "1" EMSDK_NOTTY: "1" + USE_BAZEL_VERSION: "5.4.0" steps: - checkout - run: @@ -233,9 +270,15 @@ workflows: test-linux: jobs: - test-linux + test-linux-arm64: + jobs: + - test-linux-arm64 test-mac: jobs: - test-mac + test-mac-arm64: + jobs: + - test-mac-arm64 test-windows: jobs: - test-windows diff --git a/.dockerignore b/.dockerignore index 2e4767161f..cf12f58fd6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,8 @@ # Ignore everything * -# Allow to run the test script inside the Docker container +# Allow the entrypoint/test script inside the Docker container +!/docker/entrypoint.sh !/docker/test_dockerimage.sh # Allow license file diff --git a/.flake8 b/.flake8 index b67d9c2198..e49d748821 100644 --- a/.flake8 +++ b/.flake8 @@ -5,19 +5,20 @@ ignore = E501, # Line too long E121, # Continuation line under-indented for hanging indent E722 # bare excepts + E741, # Variable names such as 'l', 'O', or 'I' exclude = - ./llvm - ./gnu - ./upstream - ./fastcomp - ./fastcomp-clang - ./releases - ./clang - ./emscripten - ./binaryen - ./git - ./node - ./python - ./temp - ./zips - ./crunch + ./llvm, + ./gnu, + ./upstream, + ./fastcomp, + ./fastcomp-clang, + ./releases, + ./clang, + ./emscripten, + ./binaryen, + ./git, + ./node, + ./python, + ./temp, + ./downloads, + ./crunch, diff --git a/.gitignore b/.gitignore index 0d794d22d0..d181c30d5a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ __pycache__ /node /python /temp -/zips +/downloads /crunch /java /mingw diff --git a/README.md b/README.md index 33c7c60c4b..3ad91f3efc 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,14 @@ important concepts to help understanding the internals of the SDK: * **SDK**: A set of tools. For example, 'sdk-1.5.6-32bit' is an SDK consisting of the tools `clang-3.2-32bit`, `node-0.10.17-32bit`, `python-2.7.5.1-32bit` and `emscripten-1.5.6`. -* **Active Tool/SDK**: Emscripten stores compiler configuration in a - user-specific file **~/.emscripten**. This file points to paths for - Emscripten, Python, Clang and so on. If the file ~/.emscripten is configured - to point to a Tool in a specific directory, then that tool is denoted as being - **active**. The Emscripten Command Prompt always gives access to the currently - active Tools. This mechanism allows switching between different installed SDK - versions easily. +* **Active Tool/SDK**: Emscripten SDK stores compiler configuration in a file + called `.emscripten` within the emsdk directory. This file points to paths + for Emscripten, Python, Clang and so on. If the configuration file points to a + Tool in a specific directory, then that tool is denoted as being + **active**. This mechanism allows switching between different installed + tools and SDKs. * **emsdk**: This is the name of the manager script that Emscripten SDK is - accessed through. Most operations are of the form `emsdk command`. To access - the emsdk script, launch the Emscripten Command Prompt. + accessed through. Most operations are of the form `emsdk `. ## System Requirements @@ -136,11 +134,11 @@ using git, and compile the package on demand. When you run `emsdk list`, it will group the Tools and SDKs under these two categories. -To obtain and build latest upstream wasm SDK from source, run +To obtain and build latest wasm SDK from source, run ``` -emsdk install sdk-upstream-main-64bit -emsdk activate sdk-upstream-main-64bit +emsdk install sdk-main-64bit +emsdk activate sdk-main-64bit ``` You can use this target for example to bootstrap developing patches to LLVM, @@ -156,7 +154,7 @@ https://emscripten.org/docs/contributing/developers_guide.html?highlight=develop ### When working on git branches compiled from source, how do I update to a newer compiler version? Unlike tags and precompiled versions, a few of the SDK packages are based on -"moving" git branches and compiled from source (e.g. sdk-upstream-main, +"moving" git branches and compiled from source (e.g. sdk-main, sdk-main, emscripten-main, binaryen-main). Because of that, the compiled versions will eventually go out of date as new commits are introduced to the development branches. To update an old compiled installation of one of @@ -179,18 +177,6 @@ activate `. Activating a tool will set up `~/.emscripten` to point to that particular tool. On Windows, you can pass the option `--permanent` to the `activate` command to register the environment permanently for the current user. Use `--system` to do this for all users. -### How do I build multiple projects with different SDK versions in parallel? - -By default, Emscripten locates all configuration files in the home directory of -the user. This may be a problem if you need to simultaneously build with -multiple Emscripten compiler versions, since the user home directory can only be -configured to point to one compiler at a time. This can be overcome by -specifying the '--embedded' option as a parameter to 'emsdk activate', which -will signal emsdk to generate the compiler configuration files inside the emsdk -root directory instead of the user home directory. Use this option also when it -is desirable to run emsdk in a fully portable mode that does not touch any files -outside the emsdk directory. - ### How do I track the latest Emscripten development with the SDK? A common and supported use case of the Emscripten SDK is to enable the workflow @@ -198,11 +184,11 @@ where you directly interact with the github repositories. This allows you to obtain new features and latest fixes immediately as they are pushed to the github repository, without having to wait for release to be tagged. You do not need a github account or a fork of Emscripten to do this. To switch to using the -latest upstream git development branch `main`, run the following: +latest git development branch `main`, run the following: emsdk install git-1.9.4 # Install git. Skip if the system already has it. - emsdk install sdk-upstream-main-64bit # Clone+pull the latest emscripten-core/emscripten/main. - emsdk activate sdk-upstream-main-64bit # Set the main SDK as the currently active one. + emsdk install sdk-main-64bit # Clone+pull the latest emscripten-core/emscripten/main. + emsdk activate sdk-main-64bit # Set the main SDK as the currently active one. ### How do I use my own Emscripten github fork with the SDK? diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..4617728678 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,16 @@ +# Security Policy + +If you have discovered a security vulnerability in this project, please report it +privately. **Do not disclose it as a public issue.** This gives us time to work with you +to fix the issue before public exposure, reducing the chance that the exploit will be +used before a patch is released. + +Please submit the report as a [security bug on the Chromium tracker](https://bugs.chromium.org/p/chromium/issues/entry?template=Security%20Bug). + +Please provide the following information in your report: + +- A description of the vulnerability and its impact +- How to reproduce the issue +- Make it clear that it's an Emscripten SDK bug. + +We ask that you give us 90 days to work on a fix before public exposure. diff --git a/bazel/BUILD b/bazel/BUILD index 7a13c92db7..e67311d074 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -8,6 +8,14 @@ config_setting( ], ) +config_setting( + name = "linux_arm64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:arm64", + ], +) + config_setting( name = "macos", constraint_values = [ @@ -38,6 +46,7 @@ alias( name = "compiler_files", actual = select({ ":linux": "@emscripten_bin_linux//:compiler_files", + ":linux_arm64": "@emscripten_bin_linux_arm64//:compiler_files", ":macos": "@emscripten_bin_mac//:compiler_files", ":macos_arm64": "@emscripten_bin_mac_arm64//:compiler_files", ":windows": "@emscripten_bin_win//:compiler_files", @@ -49,6 +58,7 @@ alias( name = "linker_files", actual = select({ ":linux": "@emscripten_bin_linux//:linker_files", + ":linux_arm64": "@emscripten_bin_linux_arm64//:linker_files", ":macos": "@emscripten_bin_mac//:linker_files", ":macos_arm64": "@emscripten_bin_mac_arm64//:linker_files", ":windows": "@emscripten_bin_win//:linker_files", @@ -60,6 +70,7 @@ alias( name = "ar_files", actual = select({ ":linux": "@emscripten_bin_linux//:ar_files", + ":linux_arm64": "@emscripten_bin_linux_arm64//:ar_files", ":macos": "@emscripten_bin_mac//:ar_files", ":macos_arm64": "@emscripten_bin_mac_arm64//:ar_files", ":windows": "@emscripten_bin_win//:ar_files", @@ -73,3 +84,11 @@ platform( "@platforms//cpu:wasm32", ], ) + +platform( + name = "platform_wasi", + constraint_values = [ + "@platforms//cpu:wasm32", + "@platforms//os:wasi", + ], +) diff --git a/bazel/README.md b/bazel/README.md index 877f2e073d..69a802146b 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -3,7 +3,7 @@ ## Setup Instructions In `WORKSPACE` file, put: -``` +```starlark load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "emsdk", @@ -37,7 +37,7 @@ build --incompatible_enable_cc_toolchain_resolution Then write a new rule wrapping your `cc_binary`. -``` +```starlark load("@rules_cc//cc:defs.bzl", "cc_binary") load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") diff --git a/bazel/deps.bzl b/bazel/deps.bzl index 6632665bca..337f0bafcd 100644 --- a/bazel/deps.bzl +++ b/bazel/deps.bzl @@ -1,11 +1,25 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") def deps(): - excludes = native.existing_rules().keys() - - if "build_bazel_rules_nodejs" not in excludes: - http_archive( - name = "build_bazel_rules_nodejs", - sha256 = "4501158976b9da216295ac65d872b1be51e3eeb805273e68c516d2eb36ae1fbb", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/4.4.1/rules_nodejs-4.4.1.tar.gz"], - ) + maybe( + http_archive, + name = "bazel_skylib", + sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + ], + ) + maybe( + http_archive, + name = "rules_nodejs", + sha256 = "08337d4fffc78f7fe648a93be12ea2fc4e8eb9795a4e6aa48595b66b34555626", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.0/rules_nodejs-core-5.8.0.tar.gz"], + ) + maybe( + http_archive, + name = "build_bazel_rules_nodejs", + sha256 = "dcc55f810142b6cf46a44d0180a5a7fb923c04a5061e2e8d8eb05ccccc60864b", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.8.0/rules_nodejs-5.8.0.tar.gz"], + ) diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl index 5068fe0c46..de156c74b0 100644 --- a/bazel/emscripten_deps.bzl +++ b/bazel/emscripten_deps.bzl @@ -14,7 +14,7 @@ filegroup( "emscripten/cache/sysroot/include/c++/v1/**", "emscripten/cache/sysroot/include/compat/**", "emscripten/cache/sysroot/include/**", - "lib/clang/16/include/**", + "lib/clang/**/include/**", ]), ) @@ -52,14 +52,17 @@ filegroup( name = "linker_files", srcs = [ "bin/clang{bin_extension}", - "bin/llc{bin_extension}", "bin/llvm-ar{bin_extension}", + "bin/llvm-dwarfdump{bin_extension}", "bin/llvm-nm{bin_extension}", "bin/llvm-objcopy{bin_extension}", + "bin/wasm-ctor-eval{bin_extension}", "bin/wasm-emscripten-finalize{bin_extension}", "bin/wasm-ld{bin_extension}", - "bin/wasm-opt{bin_extension}", "bin/wasm-metadce{bin_extension}", + "bin/wasm-opt{bin_extension}", + "bin/wasm-split{bin_extension}", + "bin/wasm2js{bin_extension}", ":emcc_common", ] + glob( include = [ @@ -124,6 +127,17 @@ def emscripten_deps(emscripten_version = "latest"): type = "tar.bz2", ) + if "emscripten_bin_linux_arm64" not in excludes: + http_archive( + name = "emscripten_bin_linux_arm64", + strip_prefix = "install", + url = emscripten_url.format("linux", revision.hash, "-arm64", "tbz2"), + # Not all versions have a linux/arm64 release: https://github.com/emscripten-core/emsdk/issues/547 + sha256 = getattr(revision, "sha_linux_arm64", None), + build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""), + type = "tar.bz2", + ) + if "emscripten_bin_mac" not in excludes: http_archive( name = "emscripten_bin_mac", @@ -161,6 +175,13 @@ def emscripten_deps(emscripten_version = "latest"): package_lock_json = "@emscripten_bin_linux//:emscripten/package-lock.json", ) + if "emscripten_npm_linux_arm64" not in excludes: + npm_install( + name = "emscripten_npm_linux_arm64", + package_json = "@emscripten_bin_linux_arm64//:emscripten/package.json", + package_lock_json = "@emscripten_bin_linux_arm64//:emscripten/package-lock.json", + ) + if "emscripten_npm_mac" not in excludes: npm_install( name = "emscripten_npm_mac", diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index 4486220b56..fb8a6c1b2e 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -2,18 +2,13 @@ load(":toolchain.bzl", "emscripten_cc_toolchain_config_rule") package(default_visibility = ["//visibility:public"]) -# Name depends on rules_nodejs version being used. -# https://github.com/emscripten-core/emsdk/issues/1020 -# https://github.com/bazelbuild/rules_nodejs/issues/3375 -node_files = "@nodejs_host//:node_files" if existing_rule("@nodejs_host//:node_files") else "@nodejs//:node_files" - filegroup( name = "common_files", srcs = [ "emscripten_config", "env.sh", "env.bat", - node_files, + "@nodejs//:node_files", ], ) @@ -48,6 +43,15 @@ filegroup( ], ) +filegroup( + name = "all_files", + srcs = [ + ":ar_files", + ":compiler_files", + ":linker_files", + ], +) + filegroup(name = "empty") # dlmalloc.bc is implictly added by the emscripten toolchain @@ -66,7 +70,7 @@ emscripten_cc_toolchain_config_rule( cc_toolchain( name = "cc-compiler-wasm", - all_files = ":empty", + all_files = ":all_files", ar_files = ":ar_files", as_files = ":empty", compiler_files = ":compiler_files", diff --git a/bazel/emscripten_toolchain/emscripten_config b/bazel/emscripten_toolchain/emscripten_config index 85a98e34b0..648a8fef20 100644 --- a/bazel/emscripten_toolchain/emscripten_config +++ b/bazel/emscripten_toolchain/emscripten_config @@ -9,7 +9,7 @@ FROZEN_CACHE = True system = platform.system() -machine = "arm64" if platform.machine() == "arm64" else "amd64" +machine = "arm64" if platform.machine() in ('arm64', 'aarch64') else "amd64" nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node" NODE_JS = ROOT_DIR + "/external/nodejs_{}_{}/{}".format(system.lower(), machine, nodejs_binary) diff --git a/bazel/emscripten_toolchain/toolchain.bzl b/bazel/emscripten_toolchain/toolchain.bzl index 58d9b6db57..e92a61257f 100644 --- a/bazel/emscripten_toolchain/toolchain.bzl +++ b/bazel/emscripten_toolchain/toolchain.bzl @@ -441,6 +441,9 @@ def _impl(ctx): name = "output_format_js", enabled = True, ), + feature( + name = "wasm_standalone", + ), ] crosstool_default_flag_sets = [ @@ -911,7 +914,7 @@ def _impl(ctx): "-iwithsysroot" + "/include/compat", "-iwithsysroot" + "/include", "-isystem", - emscripten_dir + "/lib/clang/16/include", + emscripten_dir + "/lib/clang/18/include", ], ), # Inputs and outputs @@ -1015,6 +1018,11 @@ def _impl(ctx): flags = ["-Werror"], features = ["wasm_warnings_as_errors"], ), + flag_set( + actions = all_link_actions, + flags = ["-sSTANDALONE_WASM"], + features = ["wasm_standalone"], + ), ] crosstool_default_env_sets = [ @@ -1073,7 +1081,7 @@ def _impl(ctx): emscripten_dir + "/emscripten/cache/sysroot/include/c++/v1", emscripten_dir + "/emscripten/cache/sysroot/include/compat", emscripten_dir + "/emscripten/cache/sysroot/include", - emscripten_dir + "/lib/clang/16/include", + emscripten_dir + "/lib/clang/18/include", ] artifact_name_patterns = [] diff --git a/bazel/emscripten_toolchain/wasm_cc_binary.bzl b/bazel/emscripten_toolchain/wasm_cc_binary.bzl index c4b9c9d9ed..fe77e175b7 100644 --- a/bazel/emscripten_toolchain/wasm_cc_binary.bzl +++ b/bazel/emscripten_toolchain/wasm_cc_binary.bzl @@ -25,6 +25,11 @@ def _wasm_transition_impl(settings, attr): if attr.simd: features.append("wasm_simd") + platform = "@emsdk//:platform_wasm" + if attr.standalone: + platform = "@emsdk//:platform_wasi" + features.append("wasm_standalone") + return { "//command_line_option:compiler": "emscripten", "//command_line_option:crosstool_top": "@emsdk//emscripten_toolchain:everything", @@ -32,7 +37,7 @@ def _wasm_transition_impl(settings, attr): "//command_line_option:features": features, "//command_line_option:dynamic_mode": "off", "//command_line_option:linkopt": linkopts, - "//command_line_option:platforms": ["@emsdk//:platform_wasm"], + "//command_line_option:platforms": [platform], "//command_line_option:custom_malloc": "@emsdk//emscripten_toolchain:malloc", } @@ -86,6 +91,9 @@ _WASM_BINARY_COMMON_ATTRS = { "simd": attr.bool( default = False, ), + "standalone": attr.bool( + default = False, + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), diff --git a/bazel/revisions.bzl b/bazel/revisions.bzl index c67b63bc6f..a8c43e9194 100644 --- a/bazel/revisions.bzl +++ b/bazel/revisions.bzl @@ -1,7 +1,141 @@ -# This file is automatically updated by emsdk/scripts/update_bazel_workspace.sh +# This file is automatically updated by emsdk/scripts/update_bazel_workspace.py # DO NOT MODIFY EMSCRIPTEN_TAGS = { + "3.1.46": struct( + hash = "21644188d5c473e92f1d7df2f9f60c758a78a486", + sha_linux = "75cbf14629b06e417b597d3f897ad7d881c53762380aca2f0dd85f1b15891511", + sha_mac = "06f45608381203d501141be632cab960aa105626c3a0f7a48657b79728103880", + sha_mac_arm64 = "c2a85b509a91663b390f77d51fba775421d42456211466fd3757f9dede7af9e4", + sha_win = "1ed3a3f36dee5d373ebea213fc723b3eeb7d6ba4c43da6a951ea0d76f265f234", + ), + "3.1.45": struct( + hash = "2b7c5fb8ffeac3315deb1f82ab7bf8da544f84a1", + sha_linux = "1c0576765f8b34603eead6f2bd4bc77bf68ea2f0a39ed4c144514103e85bc7d9", + sha_mac = "87f63ebb2f9807435016b238bbf46ccb94c919ec0786b46463cd788634391b0c", + sha_mac_arm64 = "29e698772c0e00c21ce120dd1db1586f5c65507168babff148c2e628add6e72a", + sha_win = "891d49f8828f715ef621d55fe202de4929bbdc89b69101fd33963571458a7f47", + ), + "3.1.44": struct( + hash = "b90507fcf011da61bacfca613569d882f7749552", + sha_linux = "5ffa2bab560a9cda6db6ee041a635d10e1ef26c8fc63675d682917b8d3d53263", + sha_mac = "291b2653f7576f8354f0267047e47a5ddef11223c89d5be399d04618f13b3832", + sha_mac_arm64 = "ad1625821b49ccbbe733596223fdf99fd786470d679f2c9dfabd4a1a7b929282", + sha_win = "8b61f60ef169b1c20207361067c40192c83b96cdbdb2f4cff21dfb20b9ee528d", + ), + "3.1.43": struct( + hash = "bf3c159888633d232c0507f4c76cc156a43c32dc", + sha_linux = "147a67a3454783b8c351780ec0111329d1e6fbb1d2fcdfe1c035e1c0997e0701", + sha_mac = "d84896c6d1ba0fbd9a5e5c5830b3ac4a02da5e683e9d8c7172f4c3ffdfaa0392", + sha_mac_arm64 = "d684f0bfc655f61e76cec29fdaad1668f3d21a229fdd908267f400691468328d", + sha_win = "a335f5f5b070cf354f1ca8e0afb23c06ae5f9ffb2c501124da7fcaea09a7db6d", + ), + "3.1.42": struct( + hash = "9d73bf4bd5b5c9ce6e51be0ed5ce6599fcb28e9e", + sha_linux = "aaa076e64dd511b0d874c348f8dab80a2f9ade0887ba74845fd02c40bbf9e68f", + sha_mac = "4715002394c5d444243c77ca231883eb999cf3313c4869cf0ae288d911f80f89", + sha_mac_arm64 = "84dede714edd81362ed2a2f79b91b1bd9cd544f219f937582e616d73bf0ea7f9", + sha_win = "4c704f4a4927aa537c2815a72915b7591c163ae8f0dbaedc167e810dd2a4a83d", + ), + "3.1.41": struct( + hash = "eb71265ef0ab905620015adbfedacf88c5dbf021", + sha_linux = "493ec8bd3f3ea3d6d616de01d6dac9c2af696978c6c44d453757ab2f8a666656", + sha_mac = "bb088e7b8f83b6bae02a0992eae61351e4e97bd033f8c8937cdaea0cb961ac9e", + sha_mac_arm64 = "aaba2de03a6dcc0db90e61e5e405a52aa47124e5ef21953d052ca015ce5ee773", + sha_win = "c7afbf2dfb6040990bd40bd72c726ada36e3e6f1985c4b62db7296465dd0778f", + ), + "3.1.40": struct( + hash = "c3122846bb040798aab975f61008c37eb19476de", + sha_linux = "5501e750c92f5a54b27ee101f6816e7416f154cb4181b73fd0be3faae947016c", + sha_mac = "052d6236ce49eaf3aa02b3c4d367b5ed4fb78209c1f1e64d48beb79e9c0b7131", + sha_mac_arm64 = "c8af68f904367938bac255f5e64ed271021b289bb135dc77ab3b58b87e1ea5b2", + sha_win = "3ec21ca18b56f7d3953da2e0d468154fcaaf30b5ac663d9ad00c41540923a099", + ), + "3.1.39": struct( + hash = "1b56b171b627af0841cf8d4d8c0160c6cb6d855f", + sha_linux = "7ec6e15a2da2701243f89af7744403ee011211e59e4f0a6fd8ced544e72e917d", + sha_mac = "dad7d270207aaffb8b8ef584cf0579bbad144879ea6f00ec9a8080adf22130dc", + sha_mac_arm64 = "c5e474ca661348d0339c785e25ad81845d49dab19d5e3e84eef2393e623e0bac", + sha_win = "a04f898b9d54dd2dc95fb697a92a1b65d07102a4cc36a02dea44c448fad83472", + ), + "3.1.38": struct( + hash = "03ecb526947f6a3702a0d083083799fe410d3893", + sha_linux = "e2812859fa32b6019f688dd66f2fa48efbfb5594da9a43b876fd4fe4ca474c20", + sha_mac = "a88c4b9eeb5dedb0d9af3b6b84bd45c486de567fbeba1675edb2d7d196e0013b", + sha_mac_arm64 = "0e5d1519ccc1163c13ee93d85f70ef6a520464f59ca3795c47cc7c44ab0f5f49", + sha_win = "bcdea031961a4f3c23008d53e083770d19751dd2a2aa71cacdea8462d09548be", + ), + "3.1.37": struct( + hash = "7c905cfc1ca6699f6ccb288ae174902cfbdcf0a2", + sha_linux = "9ff44ef69d3f389adcacbb9b95331da72cffdf6e9431c8beb6ebf7aedb77499c", + sha_mac = "accfe90322b6449933c3d8e1346024e2e2e3bef7b101942294f995b2c8e1b60f", + sha_mac_arm64 = "a0461e234c08bd7ddd7a86b49b52ccc853ebe4ce0fb5b4314e9de0193c32514a", + sha_win = "e026ea2570e747d0640829c62abddcdc14a4acffe31180110971750b80042d7a", + ), + "3.1.36": struct( + hash = "adedc0750c4a89b65bee866edab24298cb8d6677", + sha_linux = "55d3cc557a83716f7a7fe121a07dbd59ed4b5d425051e22c902570e3e0ea6c4c", + sha_mac = "6643fcef0f928cd730b894f0c2c3343eeef870576e43e56428a7a8247c7bc921", + sha_mac_arm64 = "2e8d9103cd0ba7a2b143927196a630b091b981006c908d7d36995a210a04d73b", + sha_win = "69a197f6fc153d9f98ced539564683cb13ff0ef144d3d4fbddf643e33b5f860c", + ), + "3.1.35": struct( + hash = "671550b5bdceee7bdb21493714f9a815aa5149a9", + sha_linux = "9d4b5dcb719d39e59b646ecf7c409db20c5cb6b9575f5362ffb49a9e66290819", + sha_mac = "01ea06c1f4a6c980bfdc812f9599a8ef424a975c89d5c288c9e6f2fa5e5ef5ad", + sha_mac_arm64 = "f6480ee21c80fe062e0f9d8555f8bdef621601634b9bd1e5ad07b90777ff5e4b", + sha_win = "cd26088365433ce1263a11898406c2f9284e55c2c7f23b26170c2a172c52f0b1", + ), + "3.1.34": struct( + hash = "2fdd6b9e5b67d5b62f84d0501a876513ff118ef1", + sha_linux = "dd3713f077072dcdb811f934d6685187daa47c424039e31cba83633c8d1681b1", + sha_mac = "3824609ee9b7c9919e29b19775d495a16778adb981867901f4bc503fe2f65d7d", + sha_mac_arm64 = "72728637171df46e7cd22f90537dd6faf1d4809ed1befc504ff96768c82f0e0f", + sha_win = "7538d1a1e0d586bd0723f595557551b05d724a5803132949a6fafb8b056af995", + ), + "3.1.33": struct( + hash = "49b960bd03b3a9da478a08541ce6eafe792a58a8", + sha_linux = "eab02b3f4b7c076974452ba602f908a36adf597afa15b16095b441f191ede1bb", + sha_linux_arm64 = "5e15af6affcf37c9ce6c304b4aeccb87a2758e1ef029dbc996f9d77d7444378e", + sha_mac = "b8dad3cddb19c1daf9dae99020bd17b903ae9649cfc58e433ea4951e758804de", + sha_mac_arm64 = "fbf03d06c7503f091191e440b8ea577d65b3261167cdb47359d053f12888974b", + sha_win = "031f951668eaeea39bd9363abb3f514efc3401506374984fa9b1d7ba3130a62f", + ), + "3.1.32": struct( + hash = "29ad1037cd6b99e5d8a1bd75bc188c1e9a6fda8d", + sha_linux = "25fa252e9fc674d1bcef35b3a10dd85024aa93c843b8067f8d917e5151968ffc", + sha_mac = "7881714e7738eb183b5a421bb2b907e96359e791ad0a622be6e7f5690a16b9d6", + sha_mac_arm64 = "04eede7352aca4b6fc1c111a8b31d00e8aa40547c3cd062ff9be4ffe1ed98d95", + sha_win = "22c3429eb1e6051bda46e9c02c14eca1ae3749ba8c411fbd5a3b51e3b9623161", + ), + "3.1.31": struct( + hash = "1eec24930cb2f56f6d9cd10ffcb031e27ea4157a", + sha_linux = "5952523c0c58cfc7c8839c1d3fe42ff34af5d8721231306ee432063dfacf96ca", + sha_mac = "13482cf3cb29f423f2037b9dc2b9e4ff72d0a49fcd471bbaa9b76d9f86f31d82", + sha_mac_arm64 = "654a35af16be5eeb2082e68fb36190fe76de28fa2da75ac0d2197482a203f39a", + sha_win = "493c29f5a505ccd9687036ee4c580d190b1c32b286be0e751a78e68997cec8b2", + ), + "3.1.30": struct( + hash = "dc1fdcfd3f5b9d29cb1ebdf15e6e845bef9b0cc1", + sha_linux = "151d7afdfb728e1e55ed1d100e4d3fbd20925fd65f3c3b9e093061a2c89dcac7", + sha_mac = "f0cdbc676c58bce7a65572418fb1521665ed522d7d05ae90f0764b77801982bb", + sha_mac_arm64 = "fca4eaf8ff528bb9308e5e8d0cf2709713b99fc19d55c6578a6c8f3e66182f55", + sha_win = "3001101622d98b2af3e5209154f60bbe341d32f6178307c6c723e84b5fe08bdc", + ), + "3.1.29": struct( + hash = "d949f1b99a477d4b0b54d95413df3688afa69d0a", + sha_linux = "d3f274446924c27082603170fab60ba78a2fb51360e5578fab4d9b5adab0fa9a", + sha_mac = "ed224c296efd22437f298f0fe0852613b0b1d48810b1b6d87b6b7e6beb589fe2", + sha_mac_arm64 = "af9bb86a7996bbbb36820e93dbc7f537ac23070e8730439b1e49792c4fc008e9", + sha_win = "6203f80273565a2ee6734bd33ad7bc6940ef709cbd593e70d6489e96c02ced25", + ), + "3.1.28": struct( + hash = "30b9e46ddcea66e91530559379089002d8b692cf", + sha_linux = "c23426d8b6d94cea702542c39e3bcef9439425dd4bd03bcc172e291dbbe5ed0d", + sha_mac = "4cfb918fe3233a2b31e5734e85b2a365e634f4e8a83c4390e8595cb98ae6bd8c", + sha_mac_arm64 = "a47f1f09bc7bbd4952cf54445d4fbfae53623ecbfecee0506a637665c7b4ea4c", + sha_win = "4388d230871d5b1e15c2fd0db21a792ab2836f23d860475fe183c03c5db75c8c", + ), "3.1.27": struct( hash = "48ce0b44015d0182fc8c27aa9fbc0a4474b55982", sha_linux = "4dc872260c8f42a8e20c8612b2255adbd466fec54cfbe37b46eca4eb34a2b03f", @@ -47,6 +181,7 @@ EMSCRIPTEN_TAGS = { "3.1.21": struct( hash = "a16a8bca2466eb144f7c93fa899c0272c8815dc3", sha_linux = "7045ddb3b37a2cc63cb1cf976019a6a3b7f8dbdc71254db0eee5b0452f94e9e7", + sha_linux_arm64 = "2852c8b108ec748d52d31dab3f4854bc6022df008019daff1c7e31ac00363b3f", sha_mac = "2a8d3d3ad721fec81ca1a4a581e4183b6e732e9905beb874531851846a05a367", sha_mac_arm64 = "cf788a7bdc38bb40d01f94b2d46acafb0e2f02d8ee3b3d69541c114e467ee37f", sha_win = "81518bba13f41717ffe6990b6d4a5af635d0c9d0f71a8d3bc0980cd0bc8f5f66", diff --git a/docker/Dockerfile b/docker/Dockerfile index 508d7b39b5..4cb32f18c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,8 +46,6 @@ RUN echo "## Aggressive optimization: Remove debug symbols" \ && strip -s `which node` \ # Tests consume ~80MB disc space && rm -fr ${EMSDK}/upstream/emscripten/tests \ - # Fastcomp is not supported - && rm -fr ${EMSDK}/upstream/fastcomp \ # strip out symbols from clang (~extra 50MB disc space) && find ${EMSDK}/upstream/bin -type f -exec strip -s {} + || true \ && echo "## Done" @@ -60,12 +58,12 @@ FROM ubuntu:jammy AS stage_deploy COPY --from=stage_build /emsdk /emsdk -# Fallback in case Emscripten isn't activated. -# This will let use tools offered by this image inside other Docker images -# (sub-stages) or with custom / no entrypoint +# These fallback environment variables are intended for situations where the +# entrypoint is not utilized (as in a derived image) or overridden (e.g. when +# using `--entrypoint /bin/bash` in CLI). +# This corresponds to the env variables set during: `source ./emsdk_env.sh` ENV EMSDK=/emsdk \ - EMSDK_NODE=/emsdk/node/14.18.2_64bit/bin/node \ - PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/upstream/bin:/emsdk/node/14.18.2_64bit/bin:${PATH}" + PATH="/emsdk:/emsdk/upstream/emscripten:/emsdk/node/16.20.0_64bit/bin:${PATH}" # ------------------------------------------------------------------------------ # Create a 'standard` 1000:1000 user @@ -78,8 +76,6 @@ ENV EMSDK=/emsdk \ RUN echo "## Create emscripten user (1000:1000)" \ && groupadd --gid 1000 emscripten \ && useradd --uid 1000 --gid emscripten --shell /bin/bash --create-home emscripten \ - && echo "umask 0000" >> /etc/bash.bashrc \ - && echo ". /emsdk/emsdk_env.sh" >> /etc/bash.bashrc \ && echo "## Done" # ------------------------------------------------------------------------------ @@ -123,6 +119,8 @@ RUN echo "## Update and install packages" \ # Use commonly used /src as working directory WORKDIR /src +ENTRYPOINT ["/emsdk/docker/entrypoint.sh"] + LABEL maintainer="kontakt@trzeci.eu" \ org.label-schema.name="emscripten" \ org.label-schema.description="The official container with Emscripten SDK" \ diff --git a/docker/Makefile b/docker/Makefile index 0c961ca56b..8e84b2e192 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -17,8 +17,8 @@ build: Dockerfile .TEST test: test_dockerimage.sh .TEST # test as non-root - docker run --rm -u `id -u`:`id -g` -w /emsdk/docker --net=host ${image_name}:${version} \ - bash $< + # test fallback env variables by overriding the entrypoint + docker run --rm -u `id -u`:`id -g` -w /emsdk/docker --net=host --entrypoint /bin/bash ${image_name}:${version} $< push: .TEST docker push ${image_name}:${version} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000000..ceadfc438b --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +# Set-up PATH and other environment variables +EMSDK_QUIET=1 source /emsdk/emsdk_env.sh + +exec "$@" diff --git a/docker/test_dockerimage.sh b/docker/test_dockerimage.sh index 4f44806c5f..09648206be 100755 --- a/docker/test_dockerimage.sh +++ b/docker/test_dockerimage.sh @@ -5,7 +5,6 @@ if [ $EUID -eq 0 ]; then sudo -u nobody `which emcc` --version fi -which llvm-ar which emsdk node --version npm --version diff --git a/emscripten-releases-tags.json b/emscripten-releases-tags.json index f970287d94..6d3dccdc09 100644 --- a/emscripten-releases-tags.json +++ b/emscripten-releases-tags.json @@ -1,8 +1,8 @@ { "aliases": { - "latest": "3.1.27", + "latest": "3.1.46", "latest-sdk": "latest", - "latest-arm64-linux": "3.1.21", + "latest-arm64-linux": "3.1.45", "latest-64bit": "latest", "sdk-latest-64bit": "latest", "latest-upstream": "latest", @@ -10,6 +10,44 @@ "latest-releases-upstream": "latest" }, "releases": { + "3.1.46": "21644188d5c473e92f1d7df2f9f60c758a78a486", + "3.1.46-asserts": "3e09b252d0d5a8e045d2ca92c606bfb9874bddf8", + "3.1.45": "2b7c5fb8ffeac3315deb1f82ab7bf8da544f84a1", + "3.1.45-asserts": "2aec03dfd8ce68c95316116dafbe30e273f32a81", + "3.1.44": "b90507fcf011da61bacfca613569d882f7749552", + "3.1.44-asserts": "06d00b0c62e435b743aa37c67b4ab76bc8568c79", + "3.1.43": "bf3c159888633d232c0507f4c76cc156a43c32dc", + "3.1.43-asserts": "3ec53a819a5958665d6bb0ac895c99546921b6ef", + "3.1.42": "9d73bf4bd5b5c9ce6e51be0ed5ce6599fcb28e9e", + "3.1.42-asserts": "7a622e0b66ee095aea9766375a2774b68c8bd8ef", + "3.1.41": "eb71265ef0ab905620015adbfedacf88c5dbf021", + "3.1.41-asserts": "064607aeccaffd93175bb40c072271c0393d71a5", + "3.1.40": "c3122846bb040798aab975f61008c37eb19476de", + "3.1.40-asserts": "5b5daaaabaf7292736454e4f8a73fa79bd455af4", + "3.1.39": "1b56b171b627af0841cf8d4d8c0160c6cb6d855f", + "3.1.39-asserts": "5e237cc9d9f6020af9ba908692361aa71744af1d", + "3.1.38": "03ecb526947f6a3702a0d083083799fe410d3893", + "3.1.38-asserts": "f771c7a0b55bc5d89ecceedfe9521c2c04aa43ae", + "3.1.37": "7c905cfc1ca6699f6ccb288ae174902cfbdcf0a2", + "3.1.37-asserts": "323607efbcdf84886dab55a18d758fe4c7d71d1e", + "3.1.36": "adedc0750c4a89b65bee866edab24298cb8d6677", + "3.1.36-asserts": "fdd2824226a8b001df5697146b88be98920fc215", + "3.1.35": "671550b5bdceee7bdb21493714f9a815aa5149a9", + "3.1.35-asserts": "42481a713ae805eda34f6e3e7170384c426b1d73", + "3.1.34": "2fdd6b9e5b67d5b62f84d0501a876513ff118ef1", + "3.1.34-asserts": "385382932c18a1312fff88000c4f83c2b9d1bb44", + "3.1.33": "49b960bd03b3a9da478a08541ce6eafe792a58a8", + "3.1.33-asserts": "e3ca2c6756b75cf6c6daa40276de0f25218e04a7", + "3.1.32": "29ad1037cd6b99e5d8a1bd75bc188c1e9a6fda8d", + "3.1.32-asserts": "2811c849256ec5b62b4ec32fb8369e5f3c9a54b1", + "3.1.31": "1eec24930cb2f56f6d9cd10ffcb031e27ea4157a", + "3.1.31-asserts": "48488847a38bb9cfb36e7397bea21ab2bb062680", + "3.1.30": "dc1fdcfd3f5b9d29cb1ebdf15e6e845bef9b0cc1", + "3.1.30-asserts": "21cca44e843267533c3d0b258b46c37bd142a2d7", + "3.1.29": "d949f1b99a477d4b0b54d95413df3688afa69d0a", + "3.1.29-asserts": "9d1e32e66e4b5921efc1a45cdc68e8c522c42c32", + "3.1.28": "30b9e46ddcea66e91530559379089002d8b692cf", + "3.1.28-asserts": "19871a9ea4914d63749b8d4d170e27a8854cb565", "3.1.27": "48ce0b44015d0182fc8c27aa9fbc0a4474b55982", "3.1.27-asserts": "630810e5a312f57d17efbe384ed7e4299f796bc1", "3.1.26": "4f68bb2a505c727bcf58195cf4da20592a6e92c8", diff --git a/emsdk.py b/emsdk.py index df57423a4d..c3878f98b1 100644 --- a/emsdk.py +++ b/emsdk.py @@ -38,6 +38,8 @@ from urllib2 import urlopen +skip_node = False + emsdk_packages_url = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds/deps/' emscripten_releases_repo = 'https://chromium.googlesource.com/emscripten-releases' @@ -49,7 +51,7 @@ # `main.zip` perhaps. emsdk_zip_download_url = 'https://github.com/emscripten-core/emsdk/archive/HEAD.zip' -zips_subdir = 'zips/' +download_dir = 'downloads/' extra_release_tag = None @@ -141,15 +143,17 @@ def exit_with_error(msg): elif machine.endswith('86'): ARCH = 'x86' elif machine.startswith('aarch64') or machine.lower().startswith('arm64'): - ARCH = 'aarch64' if WINDOWS: - errlog('No support for Windows on Arm, fallback to x64') - ARCH = 'x86_64' + errlog('No support for Windows on Arm, fallback to x64') + ARCH = 'x86_64' + else: + ARCH = 'arm64' elif machine.startswith('arm'): ARCH = 'arm' else: exit_with_error('unknown machine architecture: ' + machine) + # Don't saturate all cores to not steal the whole system, but be aggressive. CPU_CORES = int(os.getenv('EMSDK_NUM_CORES', max(multiprocessing.cpu_count() - 1, 1))) @@ -256,14 +260,11 @@ def vswhere(version): if not program_files: program_files = os.environ['ProgramFiles'] vswhere_path = os.path.join(program_files, 'Microsoft Visual Studio', 'Installer', 'vswhere.exe') - output = json.loads(subprocess.check_output([vswhere_path, '-latest', '-version', '[%s.0,%s.0)' % (version, version + 1), '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', '-property', 'installationPath', '-format', 'json'])) - # Visual Studio 2017 Express is not included in the above search, and it - # does not have the VC.Tools.x86.x64 tool, so do a catch-all attempt as a - # fallback, to detect Express version. - if not output: - output = json.loads(subprocess.check_output([vswhere_path, '-latest', '-version', '[%s.0,%s.0)' % (version, version + 1), '-products', '*', '-property', 'installationPath', '-format', 'json'])) - if not output: - return '' + # Source: https://learn.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2022 + tools_arch = 'ARM64' if ARCH == 'arm64' else 'x86.x64' + # The "-products *" allows detection of Build Tools, the "-prerelease" allows detection of Preview version + # of Visual Studio and Build Tools. + output = json.loads(subprocess.check_output([vswhere_path, '-latest', '-products', '*', '-prerelease', '-version', '[%s.0,%s.0)' % (version, version + 1), '-requires', 'Microsoft.VisualStudio.Component.VC.Tools.' + tools_arch, '-property', 'installationPath', '-format', 'json'])) return str(output[0]['installationPath']) except Exception: return '' @@ -284,36 +285,30 @@ def vs_filewhere(installation_path, platform, file): # Detect which CMake generator to use when building on Windows if '--mingw' in sys.argv: CMAKE_GENERATOR = 'MinGW Makefiles' - elif '--vs2017' in sys.argv: - CMAKE_GENERATOR = 'Visual Studio 15' + elif '--vs2022' in sys.argv: + CMAKE_GENERATOR = 'Visual Studio 17' elif '--vs2019' in sys.argv: CMAKE_GENERATOR = 'Visual Studio 16' + elif len(vswhere(17)) > 0: + CMAKE_GENERATOR = 'Visual Studio 17' + elif len(vswhere(16)) > 0: + CMAKE_GENERATOR = 'Visual Studio 16' + elif which('mingw32-make') is not None and which('g++') is not None: + CMAKE_GENERATOR = 'MinGW Makefiles' else: - vs2019_exists = len(vswhere(16)) > 0 - vs2017_exists = len(vswhere(15)) > 0 - mingw_exists = which('mingw32-make') is not None and which('g++') is not None - if vs2019_exists: - CMAKE_GENERATOR = 'Visual Studio 16' - elif vs2017_exists: - # VS2017 has an LLVM build issue, see - # https://github.com/kripken/emscripten-fastcomp/issues/185 - CMAKE_GENERATOR = 'Visual Studio 15' - elif mingw_exists: - CMAKE_GENERATOR = 'MinGW Makefiles' - else: - # No detected generator - CMAKE_GENERATOR = '' + # No detected generator + CMAKE_GENERATOR = '' -sys.argv = [a for a in sys.argv if a not in ('--mingw', '--vs2017', '--vs2019')] +sys.argv = [a for a in sys.argv if a not in ('--mingw', '--vs2019', '--vs2022')] # Computes a suitable path prefix to use when building with a given generator. def cmake_generator_prefix(): + if CMAKE_GENERATOR == 'Visual Studio 17': + return '_vs2022' if CMAKE_GENERATOR == 'Visual Studio 16': return '_vs2019' - if CMAKE_GENERATOR == 'Visual Studio 15': - return '_vs2017' elif CMAKE_GENERATOR == 'MinGW Makefiles': return '_mingw' # Unix Makefiles do not specify a path prefix for backwards path compatibility @@ -679,7 +674,8 @@ def get_download_target(url, dstpath, filename_prefix=''): # On success, returns the filename on the disk pointing to the destination file that was produced # On failure, returns None. -def download_file(url, dstpath, download_even_if_exists=False, filename_prefix=''): +def download_file(url, dstpath, download_even_if_exists=False, + filename_prefix='', silent=False): debug_print('download_file(url=' + url + ', dstpath=' + dstpath + ')') file_name = get_download_target(url, dstpath, filename_prefix) @@ -724,9 +720,10 @@ def download_file(url, dstpath, download_even_if_exists=False, filename_prefix=' print(']') sys.stdout.flush() except Exception as e: - errlog("Error: Downloading URL '" + url + "': " + str(e)) - if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e): - errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.") + if not silent: + errlog("Error: Downloading URL '" + url + "': " + str(e)) + if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e): + errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.") rmfile(file_name) return None except KeyboardInterrupt: @@ -861,14 +858,7 @@ def decide_cmake_build_type(tool): # The root directory of the build. def llvm_build_dir(tool): - generator_suffix = '' - if CMAKE_GENERATOR == 'Visual Studio 15': - generator_suffix = '_vs2017' - elif CMAKE_GENERATOR == 'Visual Studio 16': - generator_suffix = '_vs2019' - elif CMAKE_GENERATOR == 'MinGW Makefiles': - generator_suffix = '_mingw' - + generator_suffix = cmake_generator_prefix() bitness_suffix = '_32' if tool.bitness == 32 else '_64' if hasattr(tool, 'git_branch'): @@ -886,7 +876,7 @@ def exe_suffix(filename): # The directory where the binaries are produced. (relative to the installation # root directory of the tool) -def fastcomp_build_bin_dir(tool): +def llvm_build_bin_dir(tool): build_dir = llvm_build_dir(tool) if WINDOWS and 'Visual Studio' in CMAKE_GENERATOR: old_llvm_bin_dir = os.path.join(build_dir, 'bin', decide_cmake_build_type(tool)) @@ -916,112 +906,38 @@ def build_env(generator): # See https://groups.google.com/forum/#!topic/emscripten-discuss/5Or6QIzkqf0 if MACOS: build_env['CXXFLAGS'] = ((build_env['CXXFLAGS'] + ' ') if hasattr(build_env, 'CXXFLAGS') else '') + '-stdlib=libc++' - elif 'Visual Studio 15' in generator or 'Visual Studio 16' in generator: - if 'Visual Studio 16' in generator: - path = vswhere(16) - else: - path = vswhere(15) - - # Configuring CMake for Visual Studio needs and env. var VCTargetsPath to be present. - # How this is supposed to work is unfortunately very undocumented. See - # https://discourse.cmake.org/t/cmake-failed-to-get-the-value-of-vctargetspath-with-vs2019-16-7/1839/16 - # for some conversation. Try a couple of common paths if one of them would work. - # In the future as new versions of VS come out, we likely need to add new paths into this list. - if 'VCTargetsPath' not in build_env: - vctargets_paths = [ - os.path.join(path, 'MSBuild\\Microsoft\\VC\\v160\\'), - os.path.join(path, 'Common7\\IDE\\VC\\VCTargets') - ] - for p in vctargets_paths: - if os.path.isfile(os.path.join(p, 'Microsoft.Cpp.Default.props')): - debug_print('Set env. var VCTargetsPath=' + p + ' for CMake.') - build_env['VCTargetsPath'] = p - break - else: - debug_print('Searched path ' + p + ' as candidate for VCTargetsPath, not working.') - - if 'VCTargetsPath' not in build_env: - errlog('Unable to locate Visual Studio compiler installation for generator "' + generator + '"!') - errlog('Either rerun installation in Visual Studio Command Prompt, or locate directory to Microsoft.Cpp.Default.props manually') - sys.exit(1) - - # CMake and VS2017 cl.exe needs to have mspdb140.dll et al. in its PATH. - vc_bin_paths = [vs_filewhere(path, 'amd64', 'cl.exe'), - vs_filewhere(path, 'x86', 'cl.exe')] - for path in vc_bin_paths: - if os.path.isdir(path): - build_env['PATH'] = build_env['PATH'] + ';' + path - + if WINDOWS: + # MSBuild.exe has an internal mechanism to avoid N^2 oversubscription of threads in its two-tier build model, see + # https://devblogs.microsoft.com/cppblog/improved-parallelism-in-msbuild/ + build_env['UseMultiToolTask'] = 'true' + build_env['EnforceProcessCountAcrossBuilds'] = 'true' return build_env -def get_generator_for_sln_file(sln_file): - contents = open(sln_file, 'r').read() - if '# Visual Studio 16' in contents or '# Visual Studio Version 16' in contents: # VS2019 - return 'Visual Studio 16' - if '# Visual Studio 15' in contents: # VS2017 - return 'Visual Studio 15' - raise Exception('Unknown generator used to build solution file ' + sln_file) - - -def find_msbuild(sln_file): - # The following logic attempts to find a Visual Studio version specific - # MSBuild.exe from a list of known locations. - generator = get_generator_for_sln_file(sln_file) - debug_print('find_msbuild looking for generator ' + str(generator)) - if generator == 'Visual Studio 16': # VS2019 - path = vswhere(16) - search_paths = [os.path.join(path, 'MSBuild/Current/Bin'), - os.path.join(path, 'MSBuild/15.0/Bin/amd64'), - os.path.join(path, 'MSBuild/15.0/Bin')] - elif generator == 'Visual Studio 15': # VS2017 - path = vswhere(15) - search_paths = [os.path.join(path, 'MSBuild/15.0/Bin/amd64'), - os.path.join(path, 'MSBuild/15.0/Bin')] - else: - raise Exception('Unknown generator!') - - for path in search_paths: - p = os.path.join(path, 'MSBuild.exe') - debug_print('Searching for MSBuild.exe: ' + p) - if os.path.isfile(p): - return p - debug_print('MSBuild.exe in PATH? ' + str(which('MSBuild.exe'))) - # Last fallback, try any MSBuild from PATH (might not be compatible, but best effort) - return which('MSBuild.exe') - - -def make_build(build_root, build_type, build_target_platform='x64'): - debug_print('make_build(build_root=' + build_root + ', build_type=' + build_type + ', build_target_platform=' + build_target_platform + ')') +def make_build(build_root, build_type): + debug_print('make_build(build_root=' + build_root + ', build_type=' + build_type + ')') if CPU_CORES > 1: print('Performing a parallel build with ' + str(CPU_CORES) + ' cores.') else: print('Performing a singlethreaded build.') - generator_to_use = CMAKE_GENERATOR - - if WINDOWS: - if 'Visual Studio' in CMAKE_GENERATOR: - solution_name = str(subprocess.check_output(['dir', '/b', '*.sln'], shell=True, cwd=build_root).decode('utf-8').strip()) - generator_to_use = get_generator_for_sln_file(os.path.join(build_root, solution_name)) - # Disabled for now: Don't pass /maxcpucount argument to msbuild, since it - # looks like when building, msbuild already automatically spawns the full - # amount of logical cores the system has, and passing the number of - # logical cores here has been observed to give a quadratic N*N explosion - # on the number of spawned processes (e.g. on a Core i7 5960X with 16 - # logical cores, it would spawn 16*16=256 cl.exe processes, which would - # start crashing when running out of system memory) - # make = [find_msbuild(os.path.join(build_root, solution_name)), '/maxcpucount:' + str(CPU_CORES), '/t:Build', '/p:Configuration=' + build_type, '/nologo', '/verbosity:minimal', solution_name] - make = [find_msbuild(os.path.join(build_root, solution_name)), '/t:Build', '/p:Configuration=' + build_type, '/p:Platform=' + build_target_platform, '/nologo', '/verbosity:minimal', solution_name] - else: - make = ['mingw32-make', '-j' + str(CPU_CORES)] + make = ['cmake', '--build', '.', '--config', build_type] + if 'Visual Studio' in CMAKE_GENERATOR: + # Visual Studio historically has had a two-tier problem in its build system design. A single MSBuild.exe instance only governs + # the build of a single project (.exe/.lib/.dll) in a solution. Passing the -j parameter above will only enable multiple MSBuild.exe + # instances to be spawned to build multiple projects in parallel, but each MSBuild.exe is still singlethreaded. + # To enable each MSBuild.exe instance to also compile several .cpp files in parallel inside a single project, pass the extra + # MSBuild.exe specific "Multi-ToolTask" (MTT) setting /p:CL_MPCount. This enables each MSBuild.exe to parallelize builds wide. + # This requires CMake 3.12 or newer. + make += ['-j', str(CPU_CORES), '--', '/p:CL_MPCount=' + str(CPU_CORES)] else: - make = ['cmake', '--build', '.', '--', '-j' + str(CPU_CORES)] + # Pass -j to native make, CMake might not support -j option. + make += ['--', '-j', str(CPU_CORES)] # Build try: print('Running build: ' + str(make)) - ret = subprocess.check_call(make, cwd=build_root, env=build_env(generator_to_use)) + ret = subprocess.check_call(make, cwd=build_root, env=build_env(CMAKE_GENERATOR)) if ret != 0: errlog('Build failed with exit code ' + ret + '!') errlog('Working directory: ' + build_root) @@ -1108,92 +1024,47 @@ def xcode_sdk_version(): return subprocess.checkplatform.mac_ver()[0].split('.') -def build_fastcomp(tool): - debug_print('build_fastcomp(' + str(tool) + ')') - fastcomp_root = tool.installation_path() - fastcomp_src_root = os.path.join(fastcomp_root, 'src') - # Does this tool want to be git cloned from github? - if hasattr(tool, 'git_branch'): - success = git_clone_checkout_and_pull(tool.download_url(), fastcomp_src_root, tool.git_branch) - if not success: - return False - if hasattr(tool, 'clang_url'): - clang_root = os.path.join(fastcomp_src_root, 'tools/clang') - success = git_clone_checkout_and_pull(tool.clang_url, clang_root, tool.git_branch) - if not success: - return False - if hasattr(tool, 'lld_url'): - lld_root = os.path.join(fastcomp_src_root, 'tools/lld') - success = git_clone_checkout_and_pull(tool.lld_url, lld_root, tool.git_branch) - if not success: - return False +def cmake_target_platform(tool): + # Source: https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html#platform-selection + if hasattr(tool, 'arch'): + if tool.arch == 'arm64': + return 'ARM64' + elif tool.arch == 'x86_64': + return 'x64' + elif tool.arch == 'x86': + return 'Win32' + if ARCH == 'arm64': + return 'ARM64' else: - # Not a git cloned tool, so instead download from git tagged releases - success = download_and_unzip(tool.download_url(), fastcomp_src_root, filename_prefix='llvm-e') - if not success: - return False - success = download_and_unzip(tool.windows_clang_url if WINDOWS else tool.unix_clang_url, os.path.join(fastcomp_src_root, 'tools/clang'), filename_prefix='clang-e') - if not success: - return False + return 'x64' if tool.bitness == 64 else 'Win32' - args = [] +def cmake_host_platform(): + # Source: https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2017%202022.html#toolset-selection + arch_to_cmake_host_platform = { + 'arm64': 'ARM64', + 'arm': 'ARM', + 'x86_64': 'x64', + 'x86': 'x86' + } + return arch_to_cmake_host_platform[ARCH] + + +def get_generator_and_config_args(tool): + args = [] cmake_generator = CMAKE_GENERATOR - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 + if 'Visual Studio 16' in CMAKE_GENERATOR or 'Visual Studio 17' in CMAKE_GENERATOR: # VS2019 or VS2022 # With Visual Studio 16 2019, CMake changed the way they specify target arch. # Instead of appending it into the CMake generator line, it is specified # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] + args += ['-A', cmake_target_platform(tool)] + args += ['-Thost=' + cmake_host_platform()] elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: cmake_generator += ' Win64' - - build_dir = llvm_build_dir(tool) - build_root = os.path.join(fastcomp_root, build_dir) - - build_type = decide_cmake_build_type(tool) - - # Configure - tests_arg = 'ON' if BUILD_FOR_TESTING else 'OFF' - - enable_assertions = ENABLE_LLVM_ASSERTIONS.lower() == 'on' or (ENABLE_LLVM_ASSERTIONS == 'auto' and build_type.lower() != 'release' and build_type.lower() != 'minsizerel') - - only_supports_wasm = hasattr(tool, 'only_supports_wasm') - if ARCH == 'x86' or ARCH == 'x86_64': - targets_to_build = 'X86' - elif ARCH == 'arm': - targets_to_build = 'ARM' - elif ARCH == 'aarch64': - targets_to_build = 'AArch64' - else: - # May have problems with emconfigure - targets_to_build = '' - if not only_supports_wasm: - if targets_to_build != '': - targets_to_build += ';' - targets_to_build += 'JSBackend' - args += ['-DLLVM_TARGETS_TO_BUILD=' + targets_to_build, '-DLLVM_INCLUDE_EXAMPLES=OFF', '-DCLANG_INCLUDE_EXAMPLES=OFF', '-DLLVM_INCLUDE_TESTS=' + tests_arg, '-DCLANG_INCLUDE_TESTS=' + tests_arg, '-DLLVM_ENABLE_ASSERTIONS=' + ('ON' if enable_assertions else 'OFF')] - if os.getenv('LLVM_CMAKE_ARGS'): - extra_args = os.environ['LLVM_CMAKE_ARGS'].split(',') - print('Passing the following extra arguments to LLVM CMake configuration: ' + str(extra_args)) - args += extra_args - - # MacOS < 10.13 workaround for LLVM build bug https://github.com/kripken/emscripten/issues/5418: - # specify HAVE_FUTIMENS=0 in the build if building with target SDK that is older than 10.13. - if MACOS and ('HAVE_FUTIMENS' not in os.getenv('LLVM_CMAKE_ARGS', '')) and xcode_sdk_version() < ['10', '13']: - print('Passing -DHAVE_FUTIMENS=0 to LLVM CMake configure to workaround https://github.com/kripken/emscripten/issues/5418. Please update to macOS 10.13 or newer') - args += ['-DHAVE_FUTIMENS=0'] - - success = cmake_configure(cmake_generator, build_root, fastcomp_src_root, build_type, args) - if not success: - return False - - # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') - return success + args += ['-Thost=x64'] + return (cmake_generator, args) -# LLVM git source tree migrated to a single repository instead of multiple -# ones, build_llvm() builds via that repository structure def build_llvm(tool): debug_print('build_llvm(' + str(tool) + ')') llvm_root = tool.installation_path() @@ -1216,19 +1087,20 @@ def build_llvm(tool): targets_to_build = 'WebAssembly;X86' elif ARCH == 'arm': targets_to_build = 'WebAssembly;ARM' - elif ARCH == 'aarch64': + elif ARCH == 'arm64': targets_to_build = 'WebAssembly;AArch64' else: targets_to_build = 'WebAssembly' - args = ['-DLLVM_TARGETS_TO_BUILD=' + targets_to_build, - '-DLLVM_INCLUDE_EXAMPLES=OFF', - '-DLLVM_INCLUDE_TESTS=' + tests_arg, - '-DCLANG_INCLUDE_TESTS=' + tests_arg, - '-DLLVM_ENABLE_ASSERTIONS=' + ('ON' if enable_assertions else 'OFF'), - # Disable optional LLVM dependencies, these can cause unwanted .so dependencies - # that prevent distributing the generated compiler for end users. - '-DLLVM_ENABLE_LIBXML2=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', '-DLLDB_ENABLE_LIBEDIT=OFF', - '-DLLVM_ENABLE_LIBEDIT=OFF', '-DLLVM_ENABLE_LIBPFM=OFF'] + cmake_generator, args = get_generator_and_config_args(tool) + args += ['-DLLVM_TARGETS_TO_BUILD=' + targets_to_build, + '-DLLVM_INCLUDE_EXAMPLES=OFF', + '-DLLVM_INCLUDE_TESTS=' + tests_arg, + '-DCLANG_INCLUDE_TESTS=' + tests_arg, + '-DLLVM_ENABLE_ASSERTIONS=' + ('ON' if enable_assertions else 'OFF'), + # Disable optional LLVM dependencies, these can cause unwanted .so dependencies + # that prevent distributing the generated compiler for end users. + '-DLLVM_ENABLE_LIBXML2=OFF', '-DLLVM_ENABLE_TERMINFO=OFF', '-DLLDB_ENABLE_LIBEDIT=OFF', + '-DLLVM_ENABLE_LIBEDIT=OFF', '-DLLVM_ENABLE_LIBPFM=OFF'] # LLVM build system bug: compiler-rt does not build on Windows. It insists on performing a CMake install step that writes to C:\Program Files. Attempting # to reroute that to build_root directory then fails on an error # file INSTALL cannot find @@ -1236,16 +1108,6 @@ def build_llvm(tool): # (there instead of $(Configuration), one would need ${CMAKE_BUILD_TYPE} ?) # It looks like compiler-rt is not compatible to build on Windows? args += ['-DLLVM_ENABLE_PROJECTS=clang;lld'] - cmake_generator = CMAKE_GENERATOR - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 - # With Visual Studio 16 2019, CMake changed the way they specify target arch. - # Instead of appending it into the CMake generator line, it is specified - # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] - args += ['-Thost=x64'] - elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: - cmake_generator += ' Win64' - args += ['-Thost=x64'] if os.getenv('LLVM_CMAKE_ARGS'): extra_args = os.environ['LLVM_CMAKE_ARGS'].split(',') @@ -1258,7 +1120,7 @@ def build_llvm(tool): return False # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + success = make_build(build_root, build_type) return success @@ -1276,17 +1138,7 @@ def build_ninja(tool): build_type = decide_cmake_build_type(tool) # Configure - cmake_generator = CMAKE_GENERATOR - args = [] - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 - # With Visual Studio 16 2019, CMake changed the way they specify target arch. - # Instead of appending it into the CMake generator line, it is specified - # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] - args += ['-Thost=x64'] - elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: - cmake_generator += ' Win64' - args += ['-Thost=x64'] + cmake_generator, args = get_generator_and_config_args(tool) cmakelists_dir = os.path.join(src_root) success = cmake_configure(cmake_generator, build_root, cmakelists_dir, build_type, args) @@ -1294,7 +1146,7 @@ def build_ninja(tool): return False # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + success = make_build(build_root, build_type) if success: bin_dir = os.path.join(root, 'bin') @@ -1325,17 +1177,8 @@ def build_ccache(tool): build_type = decide_cmake_build_type(tool) # Configure - cmake_generator = CMAKE_GENERATOR - args = ['-DZSTD_FROM_INTERNET=ON'] - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 - # With Visual Studio 16 2019, CMake changed the way they specify target arch. - # Instead of appending it into the CMake generator line, it is specified - # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] - args += ['-Thost=x64'] - elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: - cmake_generator += ' Win64' - args += ['-Thost=x64'] + cmake_generator, args = get_generator_and_config_args(tool) + args += ['-DZSTD_FROM_INTERNET=ON'] cmakelists_dir = os.path.join(src_root) success = cmake_configure(cmake_generator, build_root, cmakelists_dir, build_type, args) @@ -1343,7 +1186,7 @@ def build_ccache(tool): return False # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + success = make_build(build_root, build_type) if success: bin_dir = os.path.join(root, 'bin') @@ -1398,6 +1241,8 @@ def find_latest_installed_tool(name): # npm install in Emscripten root directory def emscripten_npm_install(tool, directory): + if skip_node: + return True node_tool = find_latest_installed_tool('node') if not node_tool: npm_fallback = which('npm') @@ -1495,24 +1340,15 @@ def emscripten_post_install(tool): build_root = optimizer_build_root(tool) build_type = decide_cmake_build_type(tool) - args = [] - # Configure - cmake_generator = CMAKE_GENERATOR - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 - # With Visual Studio 16 2019, CMake changed the way they specify target arch. - # Instead of appending it into the CMake generator line, it is specified - # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] - elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: - cmake_generator += ' Win64' + cmake_generator, args = get_generator_and_config_args(tool) success = cmake_configure(cmake_generator, build_root, src_root, build_type, args) if not success: return False # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + success = make_build(build_root, build_type) if not success: return False @@ -1550,16 +1386,8 @@ def build_binaryen_tool(tool): build_type = decide_cmake_build_type(tool) # Configure - args = ['-DENABLE_WERROR=0'] # -Werror is not useful for end users - - cmake_generator = CMAKE_GENERATOR - if 'Visual Studio 16' in CMAKE_GENERATOR: # VS2019 - # With Visual Studio 16 2019, CMake changed the way they specify target arch. - # Instead of appending it into the CMake generator line, it is specified - # with a -A arch parameter. - args += ['-A', 'x64' if tool.bitness == 64 else 'x86'] - elif 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64: - cmake_generator += ' Win64' + cmake_generator, args = get_generator_and_config_args(tool) + args += ['-DENABLE_WERROR=0'] # -Werror is not useful for end users if 'Visual Studio' in CMAKE_GENERATOR: if BUILD_FOR_TESTING: @@ -1570,7 +1398,7 @@ def build_binaryen_tool(tool): return False # Make - success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32') + success = make_build(build_root, build_type) # Deploy scripts needed from source repository to build directory remove_tree(os.path.join(build_root, 'scripts')) @@ -1581,23 +1409,41 @@ def build_binaryen_tool(tool): return success -def download_and_unzip(zipfile, dest_dir, filename_prefix='', clobber=True): - debug_print('download_and_unzip(zipfile=' + zipfile + ', dest_dir=' + dest_dir + ')') +def download_and_extract(archive, dest_dir, filename_prefix='', clobber=True): + debug_print('download_and_extract(archive=' + archive + ', dest_dir=' + dest_dir + ')') + + url = urljoin(emsdk_packages_url, archive) + + def try_download(url, silent=False): + return download_file(url, download_dir, not KEEP_DOWNLOADS, + filename_prefix, silent=silent) + + # Special hack for the wasm-binaries we transitioned from `.bzip2` to + # `.xz`, but we can't tell from the version/url which one to use, so + # try one and then fall back to the other. + success = False + if 'wasm-binaries' in archive and os.path.splitext(archive)[1] == '.xz': + success = try_download(url, silent=True) + if not success: + alt_url = url.replace('.tar.xz', '.tbz2') + success = try_download(alt_url, silent=True) + if success: + url = alt_url - url = urljoin(emsdk_packages_url, zipfile) - download_target = get_download_target(url, zips_subdir, filename_prefix) + if not success: + success = try_download(url) - received_download_target = download_file(url, zips_subdir, not KEEP_DOWNLOADS, filename_prefix) - if not received_download_target: + if not success: return False - assert received_download_target == download_target # Remove the old directory, since we have some SDKs that install into the # same directory. If we didn't do this contents of the previous install # could remain. if clobber: remove_tree(dest_dir) - if zipfile.endswith('.zip'): + + download_target = get_download_target(url, download_dir, filename_prefix) + if archive.endswith('.zip'): return unzip(download_target, dest_dir) else: return untargz(download_target, dest_dir) @@ -1617,6 +1463,17 @@ def get_required_path(active_tools): for tool in active_tools: if hasattr(tool, 'activated_path'): path = to_native_path(tool.expand_vars(tool.activated_path)) + # If the tool has an activated_path_skip attribute then we don't add + # the tools path to the users path if a program by that name is found + # in the existing PATH. This allows us to, for example, add our version + # node to the users PATH if, and only if, they don't already have a + # another version of node in thier PATH. + if hasattr(tool, 'activated_path_skip'): + current_path = which(tool.activated_path_skip) + # We found an executable by this name in the current PATH, but we + # ignore our own version for this purpose. + if current_path and os.path.dirname(current_path) != path: + continue path_add.append(path) return path_add @@ -1758,10 +1615,9 @@ def expand_vars(self, str): if '%generator_prefix%' in str: str = str.replace('%generator_prefix%', cmake_generator_prefix()) str = str.replace('%.exe%', '.exe' if WINDOWS else '') - if '%fastcomp_build_dir%' in str: - str = str.replace('%fastcomp_build_dir%', llvm_build_dir(self)) - if '%fastcomp_build_bin_dir%' in str: - str = str.replace('%fastcomp_build_bin_dir%', fastcomp_build_bin_dir(self)) + if '%llvm_build_bin_dir%' in str: + str = str.replace('%llvm_build_bin_dir%', llvm_build_bin_dir(self)) + return str # Return true if this tool requires building from source, and false if this is a precompiled tool. @@ -1874,6 +1730,8 @@ def is_installed(self, skip_version_check=False): # installed for this tool to count as being installed. if hasattr(self, 'uses'): for tool_name in self.uses: + if skip_node and tool_name.startswith('node'): + continue tool = find_tool(tool_name) if tool is None: errlog("Manifest error: No tool by name '" + tool_name + "' found! This may indicate an internal SDK error!") @@ -2007,7 +1865,7 @@ def install_sdk(self): if getattr(self, 'custom_install_script', None) == 'emscripten_npm_install': # upstream tools have hardcoded paths that are not stored in emsdk_manifest.json registry - install_path = 'upstream' if 'releases-upstream' in self.version else 'fastcomp' + install_path = 'upstream' emscripten_dir = os.path.join(EMSDK_PATH, install_path, 'emscripten') # Older versions of the sdk did not include the node_modules directory # and require `npm ci` to be run post-install @@ -2033,9 +1891,7 @@ def install_tool(self): print("Installing tool '" + str(self) + "'..") url = self.download_url() - if hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_fastcomp': - success = build_fastcomp(self) - elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm': + if hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm': success = build_llvm(self) elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_ninja': success = build_ninja(self) @@ -2044,7 +1900,8 @@ def install_tool(self): elif hasattr(self, 'git_branch'): success = git_clone_checkout_and_pull(url, self.installation_path(), self.git_branch) elif url.endswith(ARCHIVE_SUFFIXES): - success = download_and_unzip(url, self.installation_path(), filename_prefix=getattr(self, 'zipfile_prefix', '')) + success = download_and_extract(url, self.installation_path(), + filename_prefix=getattr(self, 'download_prefix', '')) else: assert False, 'unhandled url type: ' + url @@ -2056,8 +1913,8 @@ def install_tool(self): success = emscripten_post_install(self) elif self.custom_install_script == 'emscripten_npm_install': success = emscripten_npm_install(self, self.installation_path()) - elif self.custom_install_script in ('build_fastcomp', 'build_llvm', 'build_ninja', 'build_ccache'): - # 'build_fastcomp' is a special one that does the download on its + elif self.custom_install_script in ('build_llvm', 'build_ninja', 'build_ccache'): + # 'build_llvm' is a special one that does the download on its # own, others do the download manually. pass elif self.custom_install_script == 'build_binaryen': @@ -2094,8 +1951,8 @@ def cleanup_temp_install_files(self): return url = self.download_url() if url.endswith(ARCHIVE_SUFFIXES): - download_target = get_download_target(url, zips_subdir, getattr(self, 'zipfile_prefix', '')) - debug_print("Deleting temporary zip file " + download_target) + download_target = get_download_target(url, download_dir, getattr(self, 'download_prefix', '')) + debug_print("Deleting temporary download: " + download_target) rmfile(download_target) def uninstall(self): @@ -2190,8 +2047,8 @@ def find_latest_hash(): def resolve_sdk_aliases(name, verbose=False): releases_info = load_releases_info() - if name == 'latest' and LINUX and ARCH == 'aarch64': - print("warning: 'latest' on arm64-linux may be slightly behind other architectures") + if name == 'latest' and LINUX and ARCH == 'arm64': + errlog("WARNING: 'latest' on arm64-linux may be slightly behind other architectures") name = 'latest-arm64-linux' while name in releases_info['aliases']: if verbose: @@ -2200,15 +2057,15 @@ def resolve_sdk_aliases(name, verbose=False): return name -def find_latest_sdk(which): - return 'sdk-releases-%s-%s-64bit' % (which, find_latest_hash()) +def find_latest_sdk(): + return 'sdk-releases-%s-64bit' % (find_latest_hash()) def find_tot_sdk(): debug_print('Fetching emscripten-releases repository...') global extra_release_tag extra_release_tag = get_emscripten_releases_tot() - return 'sdk-releases-upstream-%s-64bit' % (extra_release_tag) + return 'sdk-releases-%s-64bit' % (extra_release_tag) def parse_emscripten_version(emscripten_root): @@ -2238,19 +2095,31 @@ def get_emscripten_releases_tot(): # may not be a build for the most recent ones yet; find the last # that does. arch = '' - if ARCH == 'aarch64': + if ARCH == 'arm64': arch = '-arm64' - for release in recent_releases: - url = emscripten_releases_download_url_template % ( + + def make_url(ext): + return emscripten_releases_download_url_template % ( os_name(), release, arch, - 'tbz2' if not WINDOWS else 'zip' + ext, ) + + for release in recent_releases: + make_url('tbz2' if not WINDOWS else 'zip') try: - urlopen(url) + urlopen(make_url('tar.xz' if not WINDOWS else 'zip')) except: - continue + if not WINDOWS: + # Try the old `.tbz2` name + # TODO:remove this once tot builds are all using xz + try: + urlopen(make_url('tbz2')) + except: + continue + else: + continue return release exit_with_error('failed to find build of any recent emsdk revision') @@ -2280,7 +2149,7 @@ def update_emsdk(): if is_emsdk_sourced_from_github(): errlog('You seem to have bootstrapped Emscripten SDK by cloning from GitHub. In this case, use "git pull" instead of "emsdk update" to update emsdk. (Not doing that automatically in case you have local changes)') sys.exit(1) - if not download_and_unzip(emsdk_zip_download_url, EMSDK_PATH, clobber=False): + if not download_and_extract(emsdk_zip_download_url, EMSDK_PATH, clobber=False): sys.exit(1) @@ -2338,31 +2207,28 @@ def get_installed_sdk_version(): return None with open(version_file) as f: version = f.read() - return version.split('-')[2] + return version.split('-')[1] # Get a list of tags for emscripten-releases. def load_releases_tags(): tags = [] - tags_fastcomp = [] info = load_releases_info() for version, sha in sorted(info['releases'].items(), key=lambda x: version_key(x[0])): tags.append(sha) - # Only include versions older than 1.39.0 in fastcomp releases - if version_key(version) < (2, 0, 0): - tags_fastcomp.append(sha) if extra_release_tag: tags.append(extra_release_tag) # Explicitly add the currently installed SDK version. This could be a custom - # version (installed explicitly) so it might not be part of the main list loaded above. + # version (installed explicitly) so it might not be part of the main list + # loaded above. installed = get_installed_sdk_version() if installed and installed not in tags: tags.append(installed) - return tags, tags_fastcomp + return tags def load_releases_versions(): @@ -2390,7 +2256,7 @@ def load_sdk_manifest(): llvm_precompiled_tags_64bit = load_file_index_list('llvm-tags-64bit.txt') llvm_precompiled_tags = llvm_precompiled_tags_32bit + llvm_precompiled_tags_64bit binaryen_tags = load_legacy_binaryen_tags() - releases_tags, releases_tags_fastcomp = load_releases_tags() + releases_tags = load_releases_tags() def dependencies_exist(sdk): for tool_name in sdk.uses: @@ -2474,8 +2340,6 @@ def expand_category_param(param, category_list, t, is_sdk): expand_category_param('%precompiled_tag64%', llvm_precompiled_tags_64bit, t, is_sdk=False) elif '%binaryen_tag%' in t.version: expand_category_param('%binaryen_tag%', binaryen_tags, t, is_sdk=False) - elif '%releases-tag%' in t.version and 'fastcomp' in t.version: - expand_category_param('%releases-tag%', releases_tags_fastcomp, t, is_sdk=False) elif '%releases-tag%' in t.version: expand_category_param('%releases-tag%', releases_tags, t, is_sdk=False) else: @@ -2496,8 +2360,6 @@ def expand_category_param(param, category_list, t, is_sdk): expand_category_param('%precompiled_tag32%', llvm_precompiled_tags_32bit, sdk, is_sdk=True) elif '%precompiled_tag64%' in sdk.version: expand_category_param('%precompiled_tag64%', llvm_precompiled_tags_64bit, sdk, is_sdk=True) - elif '%releases-tag%' in sdk.version and 'fastcomp' in sdk.version: - expand_category_param('%releases-tag%', releases_tags_fastcomp, sdk, is_sdk=True) elif '%releases-tag%' in sdk.version: expand_category_param('%releases-tag%', releases_tags, sdk, is_sdk=True) else: @@ -2521,6 +2383,9 @@ def process_tool_list(tools_to_activate): tools_to_activate = tools_to_activate[:i] + deps + tools_to_activate[i:] i += len(deps) + 1 + if skip_node: + tools_to_activate = [t for t in tools_to_activate if (not t.name.startswith("node-"))] + for tool in tools_to_activate: if not tool.is_installed(): exit_with_error("error: tool is not installed and therefore cannot be activated: '%s'" % tool) @@ -2780,16 +2645,12 @@ def error_on_missing_tool(name): exit_with_error("tool or SDK not found: '%s'" % name) -def exit_with_fastcomp_error(): - exit_with_error('the fastcomp backend is not getting new builds or releases. Please use the upstream llvm backend or use an older version than 2.0.0 (such as 1.40.1).') - - def expand_sdk_name(name, activating): if 'upstream-master' in name: - errlog('upstream-master SDK has been renamed upstream-main') - name = name.replace('upstream-master', 'upstream-main') - if name in ('latest-fastcomp', 'latest-releases-fastcomp', 'tot-fastcomp', 'sdk-nightly-latest'): - exit_with_fastcomp_error() + errlog('upstream-master SDK has been renamed main') + name = name.replace('upstream-master', 'main') + if 'fastcomp' in name: + exit_with_error('the fastcomp backend is no longer supported. Please use an older version of emsdk (for example 3.1.29) if you want to install the old fastcomp-based SDK') if name in ('tot', 'sdk-tot', 'tot-upstream'): if activating: # When we are activating a tot release, assume that the currently @@ -2799,47 +2660,34 @@ def expand_sdk_name(name, activating): installed = get_installed_sdk_version() if installed: debug_print('activating currently installed SDK; not updating tot version') - return 'sdk-releases-upstream-%s-64bit' % installed - return str(find_tot_sdk()) + return 'sdk-releases-%s-64bit' % installed + return find_tot_sdk() + + if '-upstream' in name: + name = name.replace('-upstream', '') name = resolve_sdk_aliases(name, verbose=True) # check if it's a release handled by an emscripten-releases version, # and if so use that by using the right hash. we support a few notations, - # x.y.z[-(upstream|fastcomp_]) - # sdk-x.y.z[-(upstream|fastcomp_])-64bit + # x.y.z + # sdk-x.y.z-64bit # TODO: support short notation for old builds too? - backend = None fullname = name - if '-upstream' in fullname: - fullname = name.replace('-upstream', '') - backend = 'upstream' - elif '-fastcomp' in fullname: - fullname = fullname.replace('-fastcomp', '') - backend = 'fastcomp' version = fullname.replace('sdk-', '').replace('releases-', '').replace('-64bit', '').replace('tag-', '') sdk = 'sdk-' if not name.startswith('releases-') else '' releases_info = load_releases_info()['releases'] release_hash = get_release_hash(version, releases_info) if release_hash: # Known release hash - if backend == 'fastcomp' and version_key(version) >= (2, 0, 0): - exit_with_fastcomp_error() - if backend is None: - if version_key(version) >= (1, 39, 0): - backend = 'upstream' - else: - backend = 'fastcomp' - full_name = '%sreleases-%s-%s-64bit' % (sdk, backend, release_hash) + full_name = '%sreleases-%s-64bit' % (sdk, release_hash) print("Resolving SDK version '%s' to '%s'" % (version, full_name)) return full_name if len(version) == 40: - if backend is None: - backend = 'upstream' global extra_release_tag extra_release_tag = version - return '%sreleases-%s-%s-64bit' % (sdk, backend, version) + return '%sreleases-%s-64bit' % (sdk, version) return name @@ -2907,7 +2755,7 @@ def main(args): purposes. Default: Enabled --disable-assertions: Forces assertions off during the build. - --vs2017/--vs2019: If building from source, overrides to build + --vs2019/--vs2022: If building from source, overrides to build using the specified compiler. When installing precompiled packages, this has no effect. Note: The same compiler specifier must be @@ -2930,7 +2778,7 @@ def main(args): if WINDOWS: print(''' - emsdk activate [--permanent] [--system] [--build=type] [--vs2017/--vs2019] + emsdk activate [--permanent] [--system] [--build=type] [--vs2019/--vs2022] - Activates the given tool or SDK in the environment of the current shell. @@ -2944,8 +2792,8 @@ def main(args): (uses Machine environment variables). - If a custom compiler version was used to override - the compiler to use, pass the same --vs2017/--vs2019 parameter - here to choose which version to activate. + the compiler to use, pass the same --vs2019/--vs2022 + parameter here to choose which version to activate. emcmdprompt.bat - Spawns a new command prompt window with the Emscripten environment active.''') @@ -2987,7 +2835,9 @@ def extract_string_arg(name): value = args[i + 1] del args[i:i + 2] return value - + global skip_node + skip_node = extract_bool_arg('--skip-npm') + # errlog("--skip-npm found") arg_old = extract_bool_arg('--old') arg_uses = extract_bool_arg('--uses') arg_permanent = extract_bool_arg('--permanent') @@ -3070,13 +2920,11 @@ def installed_sdk_text(name): if (LINUX or MACOS or WINDOWS) and (ARCH == 'x86' or ARCH == 'x86_64'): print('The *recommended* precompiled SDK download is %s (%s).' % (find_latest_version(), find_latest_hash())) print() - print('To install/activate it, use one of:') - print(' latest [default (llvm) backend]') - print(' latest-fastcomp [legacy (fastcomp) backend]') + print('To install/activate it use:') + print(' latest') print('') - print('Those are equivalent to installing/activating the following:') - print(' %s %s' % (find_latest_version(), installed_sdk_text(find_latest_sdk('upstream')))) - print(' %s-fastcomp %s' % (find_latest_version(), installed_sdk_text(find_latest_sdk('fastcomp')))) + print('This is equivalent to installing/activating:') + print(' %s %s' % (find_latest_version(), installed_sdk_text(find_latest_sdk()))) print('') else: print('Warning: your platform does not have precompiled SDKs available.') @@ -3084,14 +2932,10 @@ def installed_sdk_text(name): print('') print('All recent (non-legacy) installable versions are:') - releases_versions = sorted( - load_releases_versions(), - key=lambda x: [int(v) if v.isdigit() else -1 for v in x.split('.')], - reverse=True, - ) + releases_versions = sorted(load_releases_versions(), key=version_key, reverse=True) releases_info = load_releases_info()['releases'] for ver in releases_versions: - print(' %s %s' % (ver, installed_sdk_text('sdk-releases-upstream-%s-64bit' % get_release_hash(ver, releases_info)))) + print(' %s %s' % (ver, installed_sdk_text('sdk-releases-%s-64bit' % get_release_hash(ver, releases_info)))) print() # Use array to work around the lack of being able to mutate from enclosing @@ -3255,6 +3099,10 @@ def print_tools(t): errlog("Missing parameter. Type 'emsdk install ' to install a tool or an SDK. Type 'emsdk list' to obtain a list of available tools. Type 'emsdk install latest' to automatically install the newest version of the SDK.") return 1 + if LINUX and ARCH == 'arm64' and args != ['latest']: + errlog('WARNING: arm64-linux binaries are not available for all releases.') + errlog('See https://github.com/emscripten-core/emsdk/issues/547') + for t in args: tool = find_tool(t) if tool is None: diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 345b2f38e7..623cb47d6c 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -9,9 +9,9 @@ "url": "https://github.com/llvm/llvm-project.git", "custom_install_script": "build_llvm", "only_supports_wasm": true, - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=1", + "activated_path": "%installation_dir%/%llvm_build_bin_dir%", + "activated_cfg": "LLVM_ROOT='%installation_dir%/%llvm_build_bin_dir%'", + "activated_env": "LLVM_ROOT=%installation_dir%/%llvm_build_bin_dir%;EMCC_WASM_BACKEND=1", "cmake_build_type": "Release" }, { @@ -23,80 +23,21 @@ "url": "https://github.com/llvm/llvm-project.git", "custom_install_script": "build_llvm", "only_supports_wasm": true, - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=1", - "cmake_build_type": "Release" - }, - - { - "id": "clang", - "version": "tag-e%tag%", - "bitness": 32, - "append_bitness": false, - "windows_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.zip", - "unix_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.tar.gz", - "windows_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.zip", - "unix_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.tar.gz", - "custom_install_script": "build_fastcomp", - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0", - "cmake_build_type": "Release" - }, - { - "id": "fastcomp-clang", - "version": "tag-e%tag%", - "bitness": 64, - "append_bitness": false, - "windows_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.zip", - "unix_url": "https://github.com/emscripten-core/emscripten-fastcomp/archive/%tag%.tar.gz", - "windows_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.zip", - "unix_clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang/archive/%tag%.tar.gz", - "custom_install_script": "build_fastcomp", - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0", - "cmake_build_type": "Release" - }, - { - "id": "fastcomp-clang", - "version": "master", - "bitness": 32, - "install_path": "clang/fastcomp", - "url": "https://github.com/emscripten-core/emscripten-fastcomp.git", - "clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang.git", - "git_branch": "master", - "custom_install_script": "build_fastcomp", - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0", - "cmake_build_type": "Release" - }, - { - "id": "fastcomp-clang", - "version": "master", - "bitness": 64, - "install_path": "clang/fastcomp", - "git_branch": "master", - "url": "https://github.com/emscripten-core/emscripten-fastcomp.git", - "clang_url": "https://github.com/emscripten-core/emscripten-fastcomp-clang.git", - "custom_install_script": "build_fastcomp", - "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", - "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0", + "activated_path": "%installation_dir%/%llvm_build_bin_dir%", + "activated_cfg": "LLVM_ROOT='%installation_dir%/%llvm_build_bin_dir%'", + "activated_env": "LLVM_ROOT=%installation_dir%/%llvm_build_bin_dir%;EMCC_WASM_BACKEND=1", "cmake_build_type": "Release" }, { "id": "releases", - "version": "upstream-%releases-tag%", + "version": "%releases-tag%", "bitness": 64, "arch": "x86_64", - "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2", - "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", + "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tar.xz", + "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tar.xz", "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/%releases-tag%/wasm-binaries.zip", - "zipfile_prefix": "%releases-tag%-", + "download_prefix": "%releases-tag%-", "install_path": "upstream", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten'", @@ -104,68 +45,18 @@ }, { "id": "releases", - "version": "upstream-%releases-tag%", + "version": "%releases-tag%", "bitness": 64, - "arch": "aarch64", - "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries-arm64.tbz2", - "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries-arm64.tbz2", - "zipfile_prefix": "%releases-tag%-", + "arch": "arm64", + "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries-arm64.tar.xz", + "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries-arm64.tar.xz", + "download_prefix": "%releases-tag%-", "install_path": "upstream", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten'", "emscripten_releases_hash": "%releases-tag%" }, - { - "id": "releases", - "version": "fastcomp-%releases-tag%", - "bitness": 64, - "arch": "x86_64", - "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2", - "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", - "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/win/%releases-tag%/wasm-binaries.zip", - "zipfile_prefix": "%releases-tag%-", - "install_path": "fastcomp", - "activated_path": "%installation_dir%/emscripten", - "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/bin/optimizer%.exe%'", - "emscripten_releases_hash": "%releases-tag%" - }, - { - "id": "releases", - "version": "fastcomp-%releases-tag%", - "bitness": 64, - "arch": "aarch64", - "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2", - "zipfile_prefix": "%releases-tag%-", - "install_path": "fastcomp", - "activated_path": "%installation_dir%/emscripten", - "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%';EMSCRIPTEN_ROOT='%installation_dir%/emscripten'", - "emscripten_releases_hash": "%releases-tag%" - }, - { - "id": "clang", - "version": "e%precompiled_tag32%", - "bitness": 32, - "arch": "x86", - "windows_url": "llvm/tag/win_64bit/emscripten-llvm-e%precompiled_tag32%.zip", - "macos_url": "llvm/tag/osx_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz", - "linux_url": "llvm/tag/linux_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz", - "activated_path": "%installation_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'", - "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen;EMCC_WASM_BACKEND=0" - }, - { - "id": "fastcomp-clang", - "version": "e%precompiled_tag64%", - "bitness": 64, - "arch": "x86_64", - "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/win/emscripten-llvm-e%precompiled_tag64%.zip", - "macos_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/mac/emscripten-llvm-e%precompiled_tag64%.tar.gz", - "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/linux/emscripten-llvm-e%precompiled_tag64%.tar.gz", - "activated_path": "%installation_dir%", - "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'", - "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen;EMCC_WASM_BACKEND=0" - }, { "id": "node", "version": "8.9.1", @@ -205,7 +96,7 @@ { "id": "node", "version": "8.9.1", - "arch": "aarch64", + "arch": "arm64", "bitness": 64, "linux_url": "node-v8.9.1-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", @@ -251,7 +142,7 @@ { "id": "node", "version": "12.18.1", - "arch": "aarch64", + "arch": "arm64", "bitness": 64, "linux_url": "node-v12.18.1-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", @@ -268,6 +159,7 @@ "arch": "x86", "windows_url": "node-v14.18.2-win-x86.zip", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -278,6 +170,7 @@ "bitness": 32, "linux_url": "node-v14.18.2-linux-armv7l.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -290,17 +183,19 @@ "windows_url": "node-v14.18.2-win-x64.zip", "linux_url": "node-v14.18.2-linux-x64.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, { "id": "node", "version": "14.18.2", - "arch": "aarch64", + "arch": "arm64", "bitness": 64, "macos_url": "node-v14.18.2-darwin-x64.tar.gz", "linux_url": "node-v14.18.2-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -313,6 +208,7 @@ "arch": "x86", "windows_url": "node-v14.15.5-win-x86.zip", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -323,6 +219,7 @@ "bitness": 32, "linux_url": "node-v14.15.5-linux-armv7l.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -335,17 +232,116 @@ "windows_url": "node-v14.15.5-win-x64.zip", "linux_url": "node-v14.15.5-linux-x64.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, { "id": "node", "version": "14.15.5", - "arch": "aarch64", + "arch": "arm64", "bitness": 64, "macos_url": "node-v14.15.5-darwin-x64.tar.gz", "linux_url": "node-v14.15.5-linux-arm64.tar.xz", "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + + + { + "id": "node", + "version": "15.14.0", + "bitness": 32, + "arch": "x86", + "windows_url": "node-v15.14.0-win-x86.zip", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "15.14.0", + "arch": "arm", + "bitness": 32, + "linux_url": "node-v15.14.0-linux-armv7l.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "15.14.0", + "bitness": 64, + "arch": "x86_64", + "macos_url": "node-v15.14.0-darwin-x64.tar.gz", + "windows_url": "node-v15.14.0-win-x64.zip", + "linux_url": "node-v15.14.0-linux-x64.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "15.14.0", + "arch": "arm64", + "bitness": 64, + "macos_url": "node-v15.14.0-darwin-x64.tar.gz", + "linux_url": "node-v15.14.0-linux-arm64.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + + { + "id": "node", + "version": "16.20.0", + "bitness": 32, + "arch": "x86", + "windows_url": "node-v16.20.0-win-x86.zip", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "16.20.0", + "arch": "arm", + "bitness": 32, + "linux_url": "node-v16.20.0-linux-armv7l.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "16.20.0", + "bitness": 64, + "arch": "x86_64", + "macos_url": "node-v16.20.0-darwin-x64.tar.gz", + "windows_url": "node-v16.20.0-win-x64.zip", + "linux_url": "node-v16.20.0-linux-x64.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" + }, + { + "id": "node", + "version": "16.20.0", + "arch": "arm64", + "bitness": 64, + "macos_url": "node-v16.20.0-darwin-arm64.tar.gz", + "linux_url": "node-v16.20.0-linux-arm64.tar.xz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, @@ -452,7 +448,7 @@ "id": "python", "version": "3.9.2", "bitness": 64, - "arch": "aarch64", + "arch": "arm64", "macos_url": "python-3.9.2-1-macos-arm64.tar.gz", "activated_cfg": "PYTHON='%installation_dir%/bin/python3'", "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem" @@ -482,7 +478,7 @@ "append_bitness": false, "windows_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.zip", "unix_url": "https://github.com/emscripten-core/emscripten/archive/%tag%.tar.gz", - "zipfile_prefix": "emscripten-e", + "download_prefix": "emscripten-e", "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%%generator_prefix%_32bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%'", "activated_path": "%installation_dir%", "activated_env": "EMSCRIPTEN=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%%generator_prefix%_32bit_optimizer/%cmake_build_type_on_win%optimizer%.exe%", @@ -522,7 +518,7 @@ "append_bitness": false, "windows_url": "https://github.com/WebAssembly/binaryen/archive/%binaryen_tag%.zip", "unix_url": "https://github.com/WebAssembly/binaryen/archive/%binaryen_tag%.tar.gz", - "zipfile_prefix": "binaryen-e", + "download_prefix": "binaryen-e", "activated_cfg": "BINARYEN_ROOT='%installation_dir%%generator_prefix%_64bit_binaryen'", "activated_path": "%installation_dir%%generator_prefix%_64bit_binaryen/bin", "activated_env": "BINARYEN_ROOT=%installation_dir%%generator_prefix%_64bit_binaryen", @@ -538,7 +534,7 @@ "append_bitness": false, "windows_url": "https://github.com/WebAssembly/binaryen/archive/%binaryen_tag%.zip", "unix_url": "https://github.com/WebAssembly/binaryen/archive/%binaryen_tag%.tar.gz", - "zipfile_prefix": "binaryen-e", + "download_prefix": "binaryen-e", "activated_cfg": "BINARYEN_ROOT='%installation_dir%%generator_prefix%_64bit_binaryen'", "activated_path": "%installation_dir%%generator_prefix%_64bit_binaryen/bin", "activated_env": "BINARYEN_ROOT=%installation_dir%%generator_prefix%_64bit_binaryen", @@ -652,180 +648,58 @@ "sdks": [ { - "version": "upstream-main", + "version": "main", "bitness": 64, - "uses": ["python-3.9.2-nuget-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], + "uses": ["python-3.9.2-nuget-64bit", "llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], "os": "win" }, { - "version": "upstream-main", + "version": "main", "bitness": 64, - "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], + "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], "os": "macos" }, { - "version": "upstream-main", + "version": "main", "bitness": 64, - "uses": ["llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], + "uses": ["llvm-git-main-64bit", "node-16.20.0-64bit", "emscripten-main-64bit", "binaryen-main-64bit"], "os": "linux" }, { - "version": "upstream-main", + "version": "main", "bitness": 32, "uses": ["llvm-git-main-32bit", "emscripten-main-32bit", "binaryen-main-32bit"], "os": "linux" }, { - "version": "fastcomp-tag-%tag%", - "bitness": 32, - "uses": ["fastcomp-clang-tag-e%tag%-32bit", "node-8.9.1-32bit", "python-2.7.13.1-32bit", "java-8.152-32bit", "emscripten-tag-%tag%-32bit", "binaryen-tag-%tag%-32bit"], - "os": "win", - "version_filter": [ - ["%tag%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-tag-%tag%", - "bitness": 64, - "uses": ["fastcomp-clang-tag-e%tag%-64bit", "node-8.9.1-64bit", "python-2.7.13.1-64bit", "java-8.152-64bit", "emscripten-tag-%tag%-64bit", "binaryen-tag-%tag%-64bit"], - "os": "win", - "version_filter": [ - ["%tag%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-tag-%tag%", - "bitness": 32, - "uses": ["fastcomp-clang-tag-e%tag%-32bit", "node-8.9.1-32bit", "emscripten-tag-%tag%-32bit", "binaryen-tag-%tag%-32bit"], - "os": "linux", - "version_filter": [ - ["%tag%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-tag-%tag%", + "version": "releases-%releases-tag%", "bitness": 64, - "uses": ["fastcomp-clang-tag-e%tag%-64bit", "node-8.9.1-64bit", "emscripten-tag-%tag%-64bit", "binaryen-tag-%tag%-64bit"], - "os": "unix", - "version_filter": [ - ["%tag%", ">", "1.37.22"] - ] - }, - { - "version": "releases-upstream-%releases-tag%", - "bitness": 64, - "uses": ["node-14.18.2-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-16.20.0-64bit", "releases-%releases-tag%-64bit"], "os": "linux", "custom_install_script": "emscripten_npm_install" }, { - "version": "releases-upstream-%releases-tag%", + "version": "releases-%releases-tag%", "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-16.20.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"], "os": "macos", "arch": "x86_64", "custom_install_script": "emscripten_npm_install" }, { - "version": "releases-upstream-%releases-tag%", + "version": "releases-%releases-tag%", "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-16.20.0-64bit", "python-3.9.2-64bit", "releases-%releases-tag%-64bit"], "os": "macos", - "arch": "aarch64", + "arch": "arm64", "custom_install_script": "emscripten_npm_install" }, { - "version": "releases-upstream-%releases-tag%", + "version": "releases-%releases-tag%", "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.9.2-nuget-64bit", "java-8.152-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-16.20.0-64bit", "python-3.9.2-nuget-64bit", "java-8.152-64bit", "releases-%releases-tag%-64bit"], "os": "win", "custom_install_script": "emscripten_npm_install" - }, - { - "version": "releases-fastcomp-%releases-tag%", - "bitness": 64, - "uses": ["node-14.18.2-64bit", "releases-fastcomp-%releases-tag%-64bit"], - "os": "linux", - "custom_install_script": "emscripten_npm_install" - }, - { - "version": "releases-fastcomp-%releases-tag%", - "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.7.4-2-64bit", "releases-fastcomp-%releases-tag%-64bit"], - "os": "macos", - "arch": "x86_64", - "custom_install_script": "emscripten_npm_install" - }, - { - "version": "releases-fastcomp-%releases-tag%", - "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-fastcomp-%releases-tag%-64bit"], - "os": "macos", - "arch": "aarch64", - "custom_install_script": "emscripten_npm_install" - }, - { - "version": "releases-fastcomp-%releases-tag%", - "bitness": 64, - "uses": ["node-14.18.2-64bit", "python-3.7.4-pywin32-64bit", "java-8.152-64bit", "releases-fastcomp-%releases-tag%-64bit"], - "os": "win", - "custom_install_script": "emscripten_npm_install" - }, - { - "version": "fastcomp-%precompiled_tag32%", - "bitness": 32, - "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-2.7.13.1-32bit", "java-8.152-32bit", "emscripten-%precompiled_tag32%"], - "os": "win", - "version_filter": [ - ["%precompiled_tag32%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-%precompiled_tag64%", - "bitness": 64, - "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-2.7.13.1-64bit", "java-8.152-64bit", "emscripten-%precompiled_tag64%"], - "os": "win", - "version_filter": [ - ["%precompiled_tag64%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-%precompiled_tag32%", - "bitness": 32, - "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "emscripten-%precompiled_tag32%"], - "os": "linux", - "version_filter": [ - ["%precompiled_tag32%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-%precompiled_tag64%", - "bitness": 64, - "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "emscripten-%precompiled_tag64%"], - "os": "linux", - "version_filter": [ - ["%precompiled_tag64%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-%precompiled_tag32%", - "bitness": 32, - "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag32%"], - "os": "macos", - "arch": "x86_64", - "version_filter": [ - ["%precompiled_tag32%", ">", "1.37.22"] - ] - }, - { - "version": "fastcomp-%precompiled_tag64%", - "bitness": 64, - "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag64%"], - "os": "macos", - "arch": "x86_64", - "version_filter": [ - ["%precompiled_tag64%", ">", "1.37.22"] - ] } ] } diff --git a/scripts/create_release.py b/scripts/create_release.py index a7438f2cc1..2d6d312771 100755 --- a/scripts/create_release.py +++ b/scripts/create_release.py @@ -53,19 +53,22 @@ def main(args): f.write(json.dumps(release_info, indent=2)) f.write('\n') - subprocess.check_call([os.path.join(script_dir, 'update_bazel_workspace.sh')], cwd=root_dir) + subprocess.check_call( + [sys.executable, os.path.join(script_dir, 'update_bazel_workspace.py')], + cwd=root_dir) branch_name = 'version_' + new_version # Create a new git branch - subprocess.check_call(['git', 'checkout', '-b', branch_name], cwd=root_dir) + subprocess.check_call(['git', 'checkout', '-b', branch_name, 'origin/main'], cwd=root_dir) # Create auto-generated changes to the new git branch subprocess.check_call(['git', 'add', '-u', '.'], cwd=root_dir) subprocess.check_call(['git', 'commit', '-m', new_version], cwd=root_dir) + print('New release created in branch: `%s`' % branch_name) - print('New relase created in branch: `%s`' % branch_name) - + # Push new branch to origin + subprocess.check_call(['git', 'push', 'origin', branch_name], cwd=root_dir) return 0 diff --git a/scripts/update_bazel_workspace.py b/scripts/update_bazel_workspace.py new file mode 100755 index 0000000000..932cc72e35 --- /dev/null +++ b/scripts/update_bazel_workspace.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# This script will update emsdk/bazel/revisons.bzl to the latest version of +# emscripten. It reads emsdk/emscripten-releases-tags.json to get the latest +# version number. Then, it downloads the prebuilts for that version and computes +# the sha256sum for the archive. It then puts all this information into the +# emsdk/bazel/revisions.bzl file. + +import hashlib +import json +import os +import requests +import sys + +STORAGE_URL = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds' + +EMSDK_ROOT = os.path.dirname(os.path.dirname(__file__)) +RELEASES_TAGS_FILE = EMSDK_ROOT + '/emscripten-releases-tags.json' +BAZEL_REVISIONS_FILE = EMSDK_ROOT + '/bazel/revisions.bzl' + + +def get_latest_info(): + with open(RELEASES_TAGS_FILE) as f: + info = json.load(f) + latest = info['aliases']['latest'] + return latest, info['releases'][latest] + + +def get_sha(platform, archive_fmt, latest_hash, arch_suffix=''): + r = requests.get(f'{STORAGE_URL}/{platform}/{latest_hash}/wasm-binaries{arch_suffix}.{archive_fmt}') + r.raise_for_status() + print(f'Fetching {r.url}') + h = hashlib.new('sha256') + for chunk in r.iter_content(chunk_size=1024): + h.update(chunk) + return h.hexdigest() + + +def revisions_item(version, latest_hash): + return f'''\ + "{version}": struct( + hash = "{latest_hash}", + sha_linux = "{get_sha('linux', 'tbz2', latest_hash)}", + sha_mac = "{get_sha('mac', 'tbz2', latest_hash)}", + sha_mac_arm64 = "{get_sha('mac', 'tbz2', latest_hash, '-arm64')}", + sha_win = "{get_sha('win', 'zip', latest_hash)}", + ), +''' + + +def insert_revision(item): + with open(BAZEL_REVISIONS_FILE, 'r') as f: + lines = f.readlines() + + lines.insert(lines.index('EMSCRIPTEN_TAGS = {\n') + 1, item) + + with open(BAZEL_REVISIONS_FILE, 'w') as f: + f.write(''.join(lines)) + + +def main(argv): + version, latest_hash = get_latest_info() + item = revisions_item(version, latest_hash) + print('inserting item:') + print(item) + insert_revision(item) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/scripts/update_bazel_workspace.sh b/scripts/update_bazel_workspace.sh deleted file mode 100755 index b2f707892d..0000000000 --- a/scripts/update_bazel_workspace.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# This script will update emsdk/bazel/WORKSPACE to the latest version of -# emscripten. It reads emsdk/emscripten-releases-tags.json to get the latest -# version number. Then, it downloads the prebuilts for that version and computes -# the sha256sum for the archive. It then puts all this information into the -# emsdk/bazel/WORKSPACE file. - -ERR=0 -# Attempt to change to the emsdk root directory -cd $(dirname $0)/.. - -# If the previous command succeeded. We are in the emsdk root. Check to make -# sure the files and directories we need are present. -if [[ $? = 0 ]]; then - if [[ ! -f emscripten-releases-tags.json ]]; then - echo "Cannot find emscripten-releases-tags.json." - ERR=1 - fi - - if [[ ! -d bazel ]]; then - echo "Cannot find the bazel directory." - ERR=1 - elif [[ ! -f bazel/WORKSPACE ]]; then - echo "Cannot find bazel/WORKSPACE." - ERR=1 - fi -else - ERR=1 -fi - -if [[ $ERR = 1 ]]; then - echo "Unable to cd into the emsdk root directory." - exit 1 -fi - -URL1=https://storage.googleapis.com/webassembly/emscripten-releases-builds/ -URL2=/wasm-binaries - -# Get commit hash for $1 version -get_hash () { - echo $(grep "$1" emscripten-releases-tags.json | grep -v latest | grep -v asserts | cut -f4 -d\") -} - -# Get sha256 for $1 os $2 extname $3 hash $4 architecture -get_sha () { - echo $(curl "${URL1}$1/$3${URL2}$4.$2" 2>/dev/null | sha256sum | awk '{print $1}') -} - -# Assemble dictionary line -revisions_item () { - hash=$(get_hash $1) - echo \ - "\ \"$1\": struct(\n" \ - "\ hash = \"$(get_hash ${hash})\",\n" \ - "\ sha_linux = \"$(get_sha linux tbz2 ${hash})\",\n" \ - "\ sha_mac = \"$(get_sha mac tbz2 ${hash})\",\n" \ - "\ sha_mac_arm64 = \"$(get_sha mac tbz2 ${hash} -arm64)\",\n" \ - "\ sha_win = \"$(get_sha win zip ${hash})\",\n" \ - "\ )," -} - -append_revision () { - sed -i "5 i $(revisions_item $1)" bazel/revisions.bzl -} - -# Get the latest version number from emscripten-releases-tag.json. -VER=$(grep -oP '(?<=latest\": \")([\d\.]+)(?=\")' \ - emscripten-releases-tags.json \ - | sed --expression "s/\./\\\./g") - -append_revision ${VER} - -echo "Done!" diff --git a/scripts/update_linux_arm64.sh b/scripts/update_linux_arm64.sh new file mode 100755 index 0000000000..bddef745db --- /dev/null +++ b/scripts/update_linux_arm64.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Script for updating the linux-arm64 release +# +# Takes as stdin the JSON string generated by the S3 bucket update when +# amazon produces a new build. e.g. +# {"Records":[{"eventVersion":"2.1","eventSource":"aws:s3","awsRegion":"us-west-2","eventTime":"2023-08-23T19:12:29.218Z","eventName":"ObjectCreated:CompleteMultipartUpload","userIdentity":{"principalId":"AWS:AROAZNAVTCTIWSHR6WFTE:i-0124c978819146a2f"},"requestParameters":{"sourceIPAddress":"35.90.239.236"},"responseElements":{"x-amz-request-id":"3SDMWYMG4BN47DCD","x-amz-id-2":"0mRaqwIIZI8ob1B7TmTSd/s0lXxST73ktgGm94MjKj2bnflqD1zCjbh2LeMIYYPvvfgQu0Ocrlev4nYmiOmS9wR0M/lvpCQz"},"s3":{"s3SchemaVersion":"1.0","configurationId":"arn:aws:cloudformation:us-west-2:646437868753:stack/EmscriptenArm64BuilderStack/eb5d5760-ee9a-11ec-a790-06738944b93b--3752220361625282518","bucket":{"name":"emscripten-arm64-4484191c","ownerIdentity":{"principalId":"A2IEUDOSUKZVHO"},"arn":"arn:aws:s3:::emscripten-arm64-4484191c"},"object":{"key":"emscripten-install-3.1.45-linux-arm64.tbz2","size":247058408,"eTag":"bb7f6835c7900deee6e7881a352ab48c-30","sequencer":"0064E65A1BC2C23A2E"}}}]} + +set -e + +if [ $# -ne 1 ]; then + echo "Please specify a single argument which is an emsdk version (e.g. 3.1.45)" + exit +fi + +VERSION=$1 +SHA=$(jq -r ".releases.\"${VERSION}\"" emscripten-releases-tags.json) +URL=$(jq -r '.Records[0] | "https://\(.s3.bucket.name).s3.\(.awsRegion).amazonaws.com/\(.s3.object.key)"') + +wget $URL -O arm64.tbz2 +gsutil cp -n arm64.tbz2 gs://webassembly/emscripten-releases-builds/linux/${SHA}/wasm-binaries-arm64.tbz2 +sed -i "s/\"latest-arm64-linux\": \".*\"/\"latest-arm64-linux\": \"$VERSION\"/" emscripten-releases-tags.json + +echo "done" diff --git a/scripts/update_node.py b/scripts/update_node.py index 95a245ef82..2c3a786d33 100755 --- a/scripts/update_node.py +++ b/scripts/update_node.py @@ -16,13 +16,15 @@ import os import shutil -version = '14.18.2' -base = 'https://nodejs.org/dist/latest-v14.x/' +version = '16.20.0' +base = 'https://nodejs.org/dist/latest-v16.x/' upload_base = 'gs://webassembly/emscripten-releases-builds/deps/' suffixes = [ + '-win-x86.zip', '-win-x64.zip', '-darwin-x64.tar.gz', + '-darwin-arm64.tar.gz', '-linux-x64.tar.xz', '-linux-arm64.tar.xz', '-linux-armv7l.tar.xz', diff --git a/test/test.py b/test/test.py index f9766d0c22..1b1233c613 100755 --- a/test/test.py +++ b/test/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import os +import platform import shutil import subprocess import sys @@ -9,16 +10,15 @@ WINDOWS = sys.platform.startswith('win') MACOS = sys.platform == 'darwin' +MACOS_ARM64 = MACOS and platform.machine() == 'arm64' emconfig = os.path.abspath('.emscripten') assert os.path.exists(emconfig) upstream_emcc = os.path.join('upstream', 'emscripten', 'emcc') -fastcomp_emcc = os.path.join('fastcomp', 'emscripten', 'emcc') emsdk = './emsdk' if WINDOWS: upstream_emcc += '.bat' - fastcomp_emcc += '.bat' emsdk = 'emsdk.bat' else: emsdk = './emsdk' @@ -164,44 +164,21 @@ def test_config_contents(self): print('test .emscripten contents') with open(emconfig) as f: config = f.read() - assert 'fastcomp' not in config assert 'upstream' in config def test_lib_building(self): print('building proper system libraries') do_lib_building(upstream_emcc) - def test_fastcomp(self): - print('test the last fastcomp release') - run_emsdk('install 1.40.1-fastcomp') - run_emsdk('activate 1.40.1-fastcomp') - - do_lib_building(fastcomp_emcc) - with open(emconfig) as f: - config = f.read() - assert config.count('LLVM_ROOT') == 1 - assert 'upstream' not in config - assert 'fastcomp' in config - - print('verify latest fastcomp version is fixed at 1.40.1') - checked_call_with_output(fastcomp_emcc + ' -v', '1.40.1', stderr=subprocess.STDOUT) - - def test_fastcomp_missing(self): - print('verify that attempting to use newer fastcomp gives an error') - fastcomp_error = 'the fastcomp backend is not getting new builds or releases. Please use the upstream llvm backend or use an older version than 2.0.0 (such as 1.40.1).' - failing_call_with_output(emsdk + ' install latest-fastcomp', fastcomp_error) - failing_call_with_output(emsdk + ' install tot-fastcomp', fastcomp_error) - failing_call_with_output(emsdk + ' install 2.0.0-fastcomp', fastcomp_error) - def test_redownload(self): print('go back to using upstream') run_emsdk('activate latest') # Test the normal tools like node don't re-download on re-install print('another install must re-download') - checked_call_with_output(emsdk + ' uninstall node-14.18.2-64bit') - checked_call_with_output(emsdk + ' install node-14.18.2-64bit', expected='Downloading:', unexpected='already installed') - checked_call_with_output(emsdk + ' install node-14.18.2-64bit', unexpected='Downloading:', expected='already installed') + checked_call_with_output(emsdk + ' uninstall node-15.14.0-64bit') + checked_call_with_output(emsdk + ' install node-15.14.0-64bit', expected='Downloading:', unexpected='already installed') + checked_call_with_output(emsdk + ' install node-15.14.0-64bit', unexpected='Downloading:', expected='already installed') def test_tot_upstream(self): print('test update-tags') @@ -221,26 +198,21 @@ def test_closure(self): # Specificlly test with `--closure` so we know that node_modules is working check_call(upstream_emcc + ' hello_world.c --closure=1') - def test_specific_old(self): - print('test specific release (old, using sdk-* notation)') - run_emsdk('install sdk-fastcomp-1.38.31-64bit') - run_emsdk('activate sdk-fastcomp-1.38.31-64bit') - def test_specific_version(self): + if MACOS_ARM64: + self.skipTest('Old sdk versions do not have ARM64 binaries') print('test specific release (new, short name)') run_emsdk('install 1.38.33') print('another install, but no need for re-download') checked_call_with_output(emsdk + ' install 1.38.33', expected='Skipped', unexpected='Downloading:') run_emsdk('activate 1.38.33') - with open(emconfig) as f: - config = f.read() - assert 'upstream' not in config - assert 'fastcomp' in config def test_specific_version_full(self): + if MACOS_ARM64: + self.skipTest('Old sdk versions do not have ARM64 binaries') print('test specific release (new, full name)') - run_emsdk('install sdk-1.38.33-upstream-64bit') - run_emsdk('activate sdk-1.38.33-upstream-64bit') + run_emsdk('install sdk-1.38.33-64bit') + run_emsdk('activate sdk-1.38.33-64bit') print('test specific release (new, tag name)') run_emsdk('install sdk-tag-1.38.33-64bit') run_emsdk('activate sdk-tag-1.38.33-64bit') @@ -254,7 +226,8 @@ def test_binaryen_from_source(self): def test_no_32bit(self): print('test 32-bit error') emsdk_hacked = hack_emsdk('not is_os_64bit()', 'True') - failing_call_with_output('python %s install latest' % emsdk_hacked, 'this tool is only provided for 64-bit OSes') + failing_call_with_output('%s %s install latest' % (sys.executable, emsdk_hacked), + 'this tool is only provided for 64-bit OSes') os.remove(emsdk_hacked) def test_update_no_git(self): @@ -277,18 +250,18 @@ def test_update_no_git(self): def test_install_arbitrary(self): # Test that its possible to install arbrary emscripten-releases SDKs - run_emsdk('install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2') + run_emsdk('install 1b7f7bc6002a3ca73647f41fc10e1fac7f06f804') # Check that its not re-downloaded - checked_call_with_output(emsdk + ' install 5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2', expected='Skipped', unexpected='Downloading:') + checked_call_with_output(emsdk + ' install 1b7f7bc6002a3ca73647f41fc10e1fac7f06f804', expected='Skipped', unexpected='Downloading:') def test_install_tool(self): # Test that its possible to install emscripten as tool instead of SDK - checked_call_with_output(emsdk + ' install releases-upstream-77b065ace39e6ab21446e13f92897f956c80476a', unexpected='Installing SDK') + checked_call_with_output(emsdk + ' install releases-77b065ace39e6ab21446e13f92897f956c80476a', unexpected='Installing SDK') def test_activate_missing(self): run_emsdk('install latest') - failing_call_with_output(emsdk + ' activate 2.0.1', expected="error: tool is not installed and therefore cannot be activated: 'releases-upstream-13e29bd55185e3c12802bc090b4507901856b2ba-64bit'") + failing_call_with_output(emsdk + ' activate 2.0.1', expected="error: tool is not installed and therefore cannot be activated: 'releases-13e29bd55185e3c12802bc090b4507901856b2ba-64bit'") def test_keep_downloads(self): env = os.environ.copy() @@ -296,7 +269,7 @@ def test_keep_downloads(self): # With EMSDK_KEEP_DOWNLOADS the downloading should happen on the first # install of 2.0.28, and again when we install 2.0.29, but not on the # second install of 2.0.28 because the zip should already be local. - shutil.rmtree('zips') + shutil.rmtree('downloads') checked_call_with_output(emsdk + ' install 2.0.28', expected='Downloading:', env=env) checked_call_with_output(emsdk + ' install 2.0.29', expected='Downloading:', env=env) checked_call_with_output(emsdk + ' install 2.0.28', expected='already downloaded, skipping', unexpected='Downloading:', env=env) diff --git a/test/test.sh b/test/test.sh index 9d8171e79b..5033fd9613 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,26 +1,36 @@ #!/usr/bin/env bash echo "test the standard workflow (as close as possible to how a user would do it, in the shell)" +echo "machine: $(uname -m)" +echo "kernel: $(uname -s)" set -x set -e # Test that arbitrary (non-released) versions can be installed and # activated. -./emsdk install sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2 -./emsdk activate sdk-upstream-5c776e6a91c0cb8edafca16a652ee1ee48f4f6d2 -source ./emsdk_env.sh -which emcc -emcc -v +# This test cannot run on linux-arm64 because only certain binaries +# get uploaded for this architecture. +if [[ !($(uname -s) == "Linux" && $(uname -m) == "aarch64") ]]; then + ./emsdk install sdk-upstream-1b7f7bc6002a3ca73647f41fc10e1fac7f06f804 + ./emsdk activate sdk-upstream-1b7f7bc6002a3ca73647f41fc10e1fac7f06f804 + source ./emsdk_env.sh + which emcc + emcc -v +fi # Install an older version of the SDK that requires EM_CACHE to be # set in the environment, so that we can test it is later removed -./emsdk install sdk-fastcomp-3b8cff670e9233a6623563add831647e8689a86b -./emsdk activate sdk-fastcomp-3b8cff670e9233a6623563add831647e8689a86b -source ./emsdk_env.sh -which emcc -emcc -v -test -n "$EM_CACHE" +# This test only runs on x64 because we didn't build arm binaries +# when this older version of the SDK was built. +if [[ $(uname -m) == "x86_64" ]]; then + ./emsdk install sdk-1.39.15 + ./emsdk activate sdk-1.39.15 + source ./emsdk_env.sh + which emcc + emcc -v + test -n "$EM_CACHE" +fi # Install the latest version of the SDK which is the expected precondition # of test.py. diff --git a/test/test_activation.ps1 b/test/test_activation.ps1 index 6d38ffcbd8..6aeb5e53a9 100644 --- a/test/test_activation.ps1 +++ b/test/test_activation.ps1 @@ -53,10 +53,6 @@ try { if (!$EMSDK_Path) { throw "No path is added!" } - $EMSDK_NODE_Path = $path_split | Where-Object { $_ -like "$repo_root\node*" } - if (!$EMSDK_NODE_Path) { - throw "$repo_root\\node is not added to path." - } $EMSDK_UPSTREAM_Path = $path_split | Where-Object { $_ -like "$repo_root\upstream\emscripten*" } if (!$EMSDK_UPSTREAM_Path) { diff --git a/test/test_bazel.sh b/test/test_bazel.sh index f52daa4473..bc53fcd829 100755 --- a/test/test_bazel.sh +++ b/test/test_bazel.sh @@ -14,7 +14,7 @@ HASH=$(grep "\"${VER}\"" emscripten-releases-tags.json \ | grep -v latest \ | cut -f4 -d\") -FAILMSG="!!! scripts/update_bazel_toolchain.sh needs to be run !!!" +FAILMSG="!!! scripts/update_bazel_workspace.py needs to be run !!!" # Ensure the WORKSPACE file is up to date with the latest version. grep ${VER} bazel/revisions.bzl || (echo ${FAILMSG} && false) diff --git a/test/test_bazel_mac.sh b/test/test_bazel_mac.sh index 0a26a0c245..58aa9f0ad9 100755 --- a/test/test_bazel_mac.sh +++ b/test/test_bazel_mac.sh @@ -14,7 +14,7 @@ HASH=$(grep "\"${VER}\"" emscripten-releases-tags.json \ | grep -v latest \ | cut -f4 -d\") -FAILMSG="!!! scripts/update_bazel_toolchain.sh needs to be run !!!" +FAILMSG="!!! scripts/update_bazel_workspace.py needs to be run !!!" # Ensure the WORKSPACE file is up to date with the latest version. grep ${VER} bazel/revisions.bzl || (echo ${FAILMSG} && false) diff --git a/test/test_node_path.sh b/test/test_node_path.sh new file mode 100755 index 0000000000..582152b4b1 --- /dev/null +++ b/test/test_node_path.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +echo "Test that node is added to that PATH if, and only if, it is not one already present". + +if [ -n "$EMSDK" ]; then + echo "EMSDK is already defined in this shell. Run tests in a shell without sourcing emsdk_env.sh first" + exit 1 +fi + +DIR=$(dirname "$BASH_SOURCE") +cd $DIR/.. + +./emsdk install latest +./emsdk activate latest + +if which node; then + echo "Test should be run without node in the path" + exit 1 +fi + +# Run emsdk_env.sh and confirm that node was added to the PATH +. emsdk_env.sh + +if ! which node; then + echo "node not found in path after emsdk_env.sh" + exit 1 +fi + +# Run emsdk_env.sh again and confirm that node is still in the PATH +. emsdk_env.sh + +if ! which node; then + echo "node not found in path after emsdk_env.sh" + exit 1 +fi