chore: switch to using JavaScript Modules (ESM) (#5233)

This had many different effects that derived from it, making this PR so massive.

**Modules**:
The first stage was to rewrite all the imports to use .js endings everywhere.
Then we needed to replace and bump a bunch of dependencies to their ESM version.

**Bundling**:
Following that we tried to use rollup to bundle everything back into CommonJS so that we can use it in pkg since pkg does not support ESM.
This was unsuccessful, and after trying several different single application solutions, we decided we will have to write our own.

**Single Executable Binary**:
We now use a custom Rust binary which bundles a zipped version of NodeJS, the native extensions and the bundled code.
We bundle the code for tree-shaking and size reduction, but also because otherwise we had issues with how we ensure that native modules are loaded correctly and with resolving the imports correctly cross-platform.
Then on first run, the binary extracts those files to the filesystem and runs node from there.
There are checksum files that we bundle together with the archives and that we store on disk to see if we need to re-extract or not.
This also gives us a huge performance boost in some cases - especially when lots of filesystem reads are required - since pkg was patching and proxying those methods and making things slow.

**Kubernetes client**:
We also had to update the Kubernetes library to the 1.0.0-rc3, with a patch from a fork that contains the changes for kubernetes-client/javascript#1341
That is because the library was using request-promise behind the scenes, which started showing up with unhandled rejection warnings and errors about req not being defined in some cases.
That library update required us to change almost every k8s call since the interface now uses objects instead of positional arguments, which makes things much nicer to read and use. We also had to introduce a workaround for using custom certificates together with proxies, something that was previously globally monkey-patched by global-agent.

**request-promise**:
We no longer depend on request-promise and fully removed the dependency.

Fixes #3841
Fixes #4898

Co-authored-by: Steffen Neubauer <[email protected]>

.circleci/config.yml

@@ -24,4 +24,4 @@ workflows:
             tags:
               only: /.*/
             branches:
-              only: /.*/
+              only: /.*/

.circleci/continue-config.yml

@@ -4,6 +4,8 @@ version: 2.1
 orbs:
   win: circleci/[email protected]
   rok8s-scripts: fairwinds/[email protected]
+  node: circleci/[email protected]
+  macos: circleci/[email protected]
 
 parameters:
   run-test-dockerhub:
@@ -233,7 +235,24 @@ commands:
           command: kubectl get ns | grep -E '(d|h)$' | grep -- '-testing' | awk '{print $1}' | xargs -n 1 kubectl delete namespace --wait=false || true
           when: always
 
-  build_dist:
+  install_rust:
+    description: Installs Rust
+    steps:
+      - run:
+          name: "Install Rust and Cross"
+          command: |
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+            # To be able to run cargo
+            source "$HOME/.cargo/env"
+
+            # Cargo builds the other targets in Docker
+            cargo install cross --git https://github.com/cross-rs/cross
+
+            # So that cross can be called
+            source "$HOME/.cargo/env"
+
+  run_npm_dist:
     description: Package built code into executables and persist to dist directory
     parameters:
       version:
@@ -242,47 +261,39 @@ commands:
           e.g. when creating unstable releases. The script defaults to using the version from core/package.json.
         type: string
         default: ""
+      targets:
+        description: |
+          The version tag used when building. Use to set the version string in the generated zip file names,
+          e.g. when creating unstable releases. The script defaults to using the version from core/package.json.
+        type: string
+        default: ""
     steps:
-      # We need Docker for building the Alpine package
-      - *remote-docker
-      - checkout
-      - npm_install
       - *attach-workspace
-      - restore_cache:
-          keys:
-            - pkg-v3-{{ checksum "package-lock.json" }}
       - run:
-          name: Install Codesigning utility (needed for ARM64 on MacOS)
-          command: |
-            curl -Lo ldid https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_x86_64
-            chmod +x ldid
-            sudo mv ldid /usr/local/bin
-      - run:
-          name: Setup buildx and qemu for cross-platform builds
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y qemu-user-static
-            sudo apt-get install -y binfmt-support
+          name: NPM install
+          command: npm ci
       - run:
           name: Run dist script
-          command: npm run dist -- --version "<<parameters.version>>"
+          command: |
+            # This is already done in the step that installs rust
+            # It also work on the native macos runner,
+            # but somehow it doesn't work for the docker based runner.
+            # We just source it again so we're sure it's available for the next steps
+            source "$HOME/.cargo/env"
+
+            npm run dist -- --version "<<parameters.version>>" <<parameters.targets>>
       - persist_to_workspace:
           root: ./
           paths:
             - dist/
-            # Needed for the alpine docker build
-            - tmp/pkg/
-      - save_cache:
-          key: pkg-v3-{{ checksum "package-lock.json" }}
-          paths:
-            - tmp/pkg-fetch
       - run:
           # No need to store the raw files as artifacts, we just want the archives
           name: Cleanup
           command: rm -rf dist/*-amd64 && rm -rf dist/*-arm64
       - store_artifacts:
           path: dist
           destination: dist
+
   update_buildx:
     description: Update buildx to a version that supports syntax in bakefile
     steps:
@@ -337,17 +348,54 @@ jobs:
             - e2e/
       - fail_fast
 
-  build-dist:
-    <<: *node-config
-    resource_class: xlarge
+  build-dist-macos:
+    macos:
+      xcode: 15.0.0
+    resource_class: macos.m1.medium.gen1
     parameters:
       version:
         description: The version tag/name to set
         type: string
         default: ""
     steps:
-      - build_dist:
+      - checkout
+      - node/install:
+          node-version: '18.18.2'
+      - install_rust
+      # We need rosetta for running the tests for x86_64
+      - macos/install-rosetta
+      - run:
+          name: "Install darwin rustup targets"
+          command: |
+            # We build these on real Apple hardware as we would otherwise violate the Apple Licensing Agreement for Xcode
+            rustup target add x86_64-apple-darwin
+            rustup target add aarch64-apple-darwin
+      - run_npm_dist:
           version: <<parameters.version>>
+          targets: "macos-amd64 macos-arm64"
+      - fail_fast
+
+  build-dist-linux-windows:
+    machine:
+      image: ubuntu-2204:current
+      docker_layer_caching: true
+    resource_class: large
+    parameters:
+      version:
+        description: The version tag/name to set
+        type: string
+        default: ""
+    steps:
+      - checkout
+      - node/install:
+          node-version: '18.18.2'
+      - install_rust
+      - run:
+          name: "Install rollup linux"
+          command: npm install --save-dev @rollup/rollup-linux-x64-gnu
+      - run_npm_dist:
+          version: <<parameters.version>>
+          targets: "linux-amd64 linux-arm64 alpine-amd64 windows-amd64"
       - fail_fast
 
   lint:
@@ -873,7 +921,7 @@ jobs:
 
   test-macos:
     macos:
-      xcode: "13.4.1"
+      xcode: "15.0.0"
     steps:
       - checkout
       - *attach-workspace
@@ -989,9 +1037,11 @@ workflows:
       - build:
           # We only want to run tests on branches (e.g. for the merge queue) and pull requests.
           # On main we do not need to re-run all the tests, as the merge queue guarantees that tests already passed.
-          # For the edge release we declare the `build-edge` job below that only runs on main. 
+          # For the edge release we declare the `build-edge` job below that only runs on main.
           <<: *ignore-main
-      - build-dist:
+      - build-dist-macos:
+          requires: [build]
+      - build-dist-linux-windows:
           requires: [build]
       - lint:
           requires: [build]
@@ -1004,19 +1054,19 @@ workflows:
       - test-dist:
           # Don't attempt to run dist tests for external PRs (they won't have access to the required keys)
           <<: *only-internal-prs
-          requires: [build-dist]
+          requires: [build-dist-macos, build-dist-linux-windows]
       - test-macos:
           <<: *only-internal-prs
-          requires: [build-dist]
+          requires: [build-dist-macos]
       - test-macos-arm:
           <<: *only-internal-prs
-          requires: [build-dist]
+          requires: [build-dist-macos]
       - test-dockerhub:
           <<: *only-prs
-          requires: [build-dist]
+          requires: [build-dist-linux-windows]
       - test-windows:
           <<: *only-internal-prs
-          requires: [build-dist]
+          requires: [build-dist-linux-windows]
 
       # - e2e-project:
       #     <<: *only-internal-prs
@@ -1216,36 +1266,45 @@ workflows:
           <<: *only-main
           name: build-edge
 
-      - build-dist:
+      - build-dist-macos:
           <<: *only-main
-          name: build-dist-edge
+          name: build-dist-macos-edge
+          version: edge-bonsai
+          requires: [build-edge]
+
+      - build-dist-linux-windows:
+          <<: *only-main
+          name: build-dist-linux-windows-edge
           version: edge-bonsai
           requires: [build-edge]
 
       - github-edge-prerelease:
           <<: *only-main
           version: edge-bonsai
-          requires: [build-dist-edge]
+          requires: [build-dist-macos-edge, build-dist-linux-windows]
 
       - dockerhub-release:
           <<: *only-main
           context: docker
-          requires:
-            - build-dist-edge
+          requires: [build-dist-macos-edge, build-dist-linux-windows]
+
       - test-docker-gcloud:
           context: docker
           requires: [dockerhub-release]
   tags:
     jobs:
       - build:
           <<: *only-tags
-      - build-dist:
+      - build-dist-macos:
+          <<: *only-tags
+          requires: [build]
+      - build-dist-linux-windows:
           <<: *only-tags
           requires: [build]
       - dockerhub-release:
           <<: *only-tags
           context: docker
-          requires: [build-dist]
+          requires: [build-dist-macos, build-dist-linux-windows]
       - github-release-tag:
           <<: *only-tags
-          requires: [build-dist]
+          requires: [build-dist-macos, build-dist-linux-windows]

.eslintrc

@@ -1,8 +1,8 @@
 {
-  "extends": [".eslintrc.autogenerated.js", "plugin:@typescript-eslint/recommended", "plugin:prettier/recommended"],
+  "extends": [".eslintrc.autogenerated.cjs", "plugin:@typescript-eslint/recommended", "plugin:prettier/recommended"],
   "root": true,
   "plugins": ["unused-imports", "mocha", "header"],
-  "ignorePatterns": ["*.d.ts*"],
+  "ignorePatterns": ["*.d.ts*", "garden-sea/tmp/**/*"],
   "rules": {
     "@typescript-eslint/no-floating-promises": "warn",
     "@typescript-eslint/no-shadow": [
@@ -49,6 +49,8 @@
       " * License, v. 2.0. If a copy of the MPL was not distributed with this",
       " * file, You can obtain one at http://mozilla.org/MPL/2.0/.",
       " "
-    ]]
+    ]],
+    "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }],
+    "@typescript-eslint/consistent-type-imports": "error"
   }
 }

.gitignore

@@ -14,6 +14,8 @@ node_modules
 .idea
 *.iml
 .vscode
+# We store specific project relevant settings in this file
+!.vscode/settings.json
 .vs
 .history
 *~
@@ -52,4 +54,4 @@ static/**/.garden-version
 #debug files
 debug-info-*
 
-/yarn.lock
+/yarn.lock

bin/garden

@@ -2,4 +2,4 @@
 
 repo_root=$(cd `dirname $0` && cd .. && pwd)
 
-node --max-old-space-size=4096 --max-semi-space-size=64 ${repo_root}/cli/bin/garden "$@"
+node --max-old-space-size=4096 --max-semi-space-size=64 ${repo_root}/cli/bin/garden.js "$@"

bin/garden-debug

@@ -1 +1 @@
-../cli/bin/garden-debug
+../cli/bin/garden-debug.js

cli/.mocharc.yml

@@ -1,6 +1,7 @@
 reporter: spec
-spec:
+require:
   - build/test/setup.js
+spec:
   - build/test/**/*.js
 timeout: 10000
 watch-files:

cli/bin/garden-debug.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node --inspect --stack-trace-limit=1000 --max-semi-space-size=64
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import "./garden.js"