fix(k8s): uninstall-garden-services command now works more reliably

The issue turned out to be that we had to delete the NFS provisioner
last, otherwise PVs/PVCs would get stuck.

garden-service/src/actions.ts

@@ -18,7 +18,7 @@ import { PublishModuleParams, PublishResult } from "./types/plugin/module/publis
 import { SetSecretParams, SetSecretResult } from "./types/plugin/provider/setSecret"
 import { validate } from "./config/common"
 import { defaultProvider } from "./config/provider"
-import { ParameterError, PluginError, ConfigurationError, InternalError } from "./exceptions"
+import { ParameterError, PluginError, ConfigurationError, InternalError, RuntimeError } from "./exceptions"
 import { Garden } from "./garden"
 import { LogEntry } from "./logger/log-entry"
 import { ProcessResults, processServices } from "./process"
@@ -579,14 +579,14 @@ export class ActionRouter implements TypeGuard {
   }
 
   /**
-   * Deletes all services and cleans up the specified environment.
+   * Deletes all or specified services in the environment.
    */
-  async deleteEnvironment(log: LogEntry) {
+  async deleteServices(log: LogEntry, names?: string[]) {
     const graph = await this.garden.getConfigGraph(log)
 
     const servicesLog = log.info({ msg: chalk.white("Deleting services..."), status: "active" })
 
-    const services = await graph.getServices()
+    const services = await graph.getServices(names)
 
     const deleteResults = await this.garden.processTasks(
       services.map((service) => {
@@ -600,12 +600,25 @@ export class ActionRouter implements TypeGuard {
       })
     )
 
+    const failed = Object.values(deleteResults).filter((r) => r && r.error).length
+
+    if (failed) {
+      throw new RuntimeError(`${failed} delete task(s) failed!`, {
+        results: deleteResults,
+      })
+    }
+
     const serviceStatuses = deletedServiceStatuses(deleteResults)
 
     servicesLog.setSuccess()
 
-    log.info("")
+    return serviceStatuses
+  }
 
+  /**
+   * Runs cleanupEnvironment for all configured providers
+   */
+  async cleanupAll(log: LogEntry) {
     const envLog = log.info({ msg: chalk.white("Cleaning up environments..."), status: "active" })
     const environmentStatuses: EnvironmentStatusMap = {}
 
@@ -617,7 +630,7 @@ export class ActionRouter implements TypeGuard {
 
     envLog.setSuccess()
 
-    return { serviceStatuses, environmentStatuses }
+    return environmentStatuses
   }
 
   async getDebugInfo({ log, includeProject }: { log: LogEntry; includeProject: boolean }): Promise<DebugInfoMap> {

garden-service/src/commands/delete.ts

@@ -94,9 +94,14 @@ export class DeleteEnvironmentCommand extends Command {
     printHeader(headerLog, `Deleting ${garden.environmentName} environment`, "skull_and_crossbones")
 
     const actions = await garden.getActionRouter()
-    const result = await actions.deleteEnvironment(log)
 
-    return { result }
+    const serviceStatuses = await actions.deleteServices(log)
+
+    log.info("")
+
+    const environmentStatuses = await actions.cleanupAll(log)
+
+    return { result: { serviceStatuses, environmentStatuses } }
   }
 }
 

garden-service/src/garden.ts

@@ -415,10 +415,15 @@ export class Garden {
     const provider = findByName(providers, name)
 
     if (!provider) {
-      throw new PluginError(`Could not find provider '${name}'`, {
-        name,
-        providers,
-      })
+      const providerNames = providers.map((p) => p.name)
+      throw new PluginError(
+        `Could not find provider '${name}' in environment '${this.environmentName}' ` +
+          `(configured providers: ${providerNames.join(", ")})`,
+        {
+          name,
+          providers,
+        }
+      )
     }
 
     return provider

garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts

@@ -27,10 +27,26 @@ export const uninstallGardenServices: PluginCommand = {
     const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log)
     const actions = await sysGarden.getActionRouter()
 
-    const result = await actions.deleteEnvironment(log)
+    const graph = await sysGarden.getConfigGraph(log)
+    const services = await graph.getServices()
+
+    log.info("")
+
+    // We have to delete all services except nfs-provisioner first to avoid volumes getting stuck
+    const serviceNames = services.map((s) => s.name).filter((name) => name !== "nfs-provisioner")
+    const serviceStatuses = await actions.deleteServices(log, serviceNames)
+
+    if (k8sCtx.provider.config._systemServices.includes("nfs-provisioner")) {
+      const service = await graph.getService("nfs-provisioner")
+      await actions.deleteService({ service, log })
+    }
+
+    log.info("")
+
+    const environmentStatuses = await actions.cleanupAll(log)
 
     log.info(chalk.green("\nDone!"))
 
-    return { result }
+    return { result: { serviceStatuses, environmentStatuses } }
   },
 }

garden-service/src/plugins/kubernetes/init.ts

@@ -236,6 +236,11 @@ export async function prepareSystem({
 
   const remoteCluster = provider.name !== "local-kubernetes"
 
+  // Don't attempt to prepare environment when running the uninstall command
+  if (ctx.command && ctx.command.name === "plugins" && ctx.command.args.command === "uninstall-garden-services") {
+    return {}
+  }
+
   // If we require manual init and system services are ready OR outdated but none are *missing*, we warn
   // in the prepareEnvironment handler, instead of flagging as not ready here. This avoids blocking users where
   // there's variance in configuration between users of the same cluster, that often doesn't affect usage.
@@ -253,8 +258,9 @@ export async function prepareSystem({
           to update them, or contact a cluster admin to do so.
         `),
       })
+
+      return {}
     }
-    return {}
   }
 
   // We require manual init if we're installing any system services to remote clusters, to avoid conflicts
@@ -335,8 +341,12 @@ export async function cleanupEnvironment({ ctx, log }: CleanupEnvironmentParams)
   return {}
 }
 
+export function getNfsStorageClass(config: KubernetesConfig) {
+  return `${config.gardenSystemNamespace}-nfs-v${nfsStorageClassVersion}`
+}
+
 export function getKubernetesSystemVariables(config: KubernetesConfig) {
-  const nfsStorageClass = `${config.gardenSystemNamespace}-nfs-v${nfsStorageClassVersion}`
+  const nfsStorageClass = getNfsStorageClass(config)
   const syncStorageClass = config.storage.sync.storageClass || nfsStorageClass
   const systemNamespace = config.gardenSystemNamespace