improvement: add GARDEN_IGNORE_UNCAUGHT_EXCEPTION (#6337)

This adds the GARDEN_IGNORE_UNCAUGHT_EXCEPTION env var to work around
#6322 until we find a more
permanent solution.

cli/src/cli.ts

@@ -15,6 +15,9 @@ import { getOtelSDK } from "@garden-io/core/build/src/util/open-telemetry/tracin
 import { withContextFromEnv } from "@garden-io/core/build/src/util/open-telemetry/propagation.js"
 import { wrapActiveSpan } from "@garden-io/core/build/src/util/open-telemetry/spans.js"
 import { InternalError } from "@garden-io/core/build/src/exceptions.js"
+import { styles } from "@garden-io/core/build/src/logger/styles.js"
+import { gardenEnv, IGNORE_UNCAUGHT_EXCEPTION_VARNAME } from "@garden-io/core/build/src/constants.js"
+import { inspect } from "node:util"
 
 // These plugins are always registered
 export const getBundledPlugins = (): GardenPluginReference[] => [
@@ -62,6 +65,23 @@ export const getBundledPlugins = (): GardenPluginReference[] => [
   },
 ]
 
+let ignoredUncaughtExceptions = false
+
+if (gardenEnv.GARDEN_IGNORE_UNCAUGHT_EXCEPTION) {
+  console.warn(
+    styles.warning(
+      `\nWARNING: The environment variable ${IGNORE_UNCAUGHT_EXCEPTION_VARNAME} is set to true. This is not a recommended mode of operation.\n`
+    )
+  )
+
+  process.on("uncaughtException", (e: unknown) => {
+    ignoredUncaughtExceptions = true
+    console.warn(
+      `\n${styles.warning(`WARNING: Ignoring fatal exception because ${IGNORE_UNCAUGHT_EXCEPTION_VARNAME} is set to true`)}: ${inspect(e, { showHidden: true, getters: true })}\n`
+    )
+  })
+}
+
 export async function runCli({
   args,
   cli,
@@ -105,6 +125,15 @@ export async function runCli({
       logUnexpectedError(err, "OTEL shutdown failed")
     }
 
+    if (ignoredUncaughtExceptions) {
+      console.warn(
+        styles.warning(
+          `\nWARNING: Ignored a fatal exception because ${IGNORE_UNCAUGHT_EXCEPTION_VARNAME} is set to true. Exiting with code 1.\n`
+        )
+      )
+      code = 1
+    }
+
     await shutdown(code)
   }
 

core/src/constants.ts

@@ -56,6 +56,8 @@ export const DEFAULT_GARDEN_CLOUD_DOMAIN = "https://app.garden.io"
 
 export const DEFAULT_BROWSER_DIVIDER_WIDTH = 80
 
+export const IGNORE_UNCAUGHT_EXCEPTION_VARNAME = "GARDEN_IGNORE_UNCAUGHT_EXCEPTION"
+
 /**
  * Environment variables, with defaults where appropriate.
  *
@@ -107,4 +109,9 @@ export const gardenEnv = {
   // FIXME: If the environment variable is not set, asBool returns undefined, unlike the type suggests. That's why we cast to `boolean | undefined`.
   GARDEN_CLOUD_BUILDER: env.get("GARDEN_CLOUD_BUILDER").required(false).asBool() as boolean | undefined,
   GARDEN_ENABLE_PARTIAL_RESOLUTION: env.get("GARDEN_ENABLE_PARTIAL_RESOLUTION").required(false).asBool(),
+  GARDEN_IGNORE_UNCAUGHT_EXCEPTION: env
+    .get(IGNORE_UNCAUGHT_EXCEPTION_VARNAME)
+    .required(false)
+    .default("false")
+    .asBool(),
 }