Skip to content

Commit

Permalink
improvement(k8s): schedule runners w/o deploying
Browse files Browse the repository at this point in the history 
This overcomes a limitation that had been temporarily introduced in
0.13, where the `helm` or `kubernetes` Deploy that a `kubernetes-pod`
Run or Test targeted would have to be deployed before the Test/Run could
be executed. This was essentially a semantic regression from 0.12.

This was fixed/improved by simply providing manifests to
`kubernetes-pod` Runs and Tests in an equivalent way to how `kubernetes`
Deploys are configured.

Helm Runs and Tests are now implemented with a new `helm-pod` action
definition, which is conceptually similar to the new `kubernetes-pod`
implementation, but which render the manifests from a Helm chart
instead.

Once the `kubernetes-exec` Run and Test action definitions have been
implemented, those will be the generally recommended option unless the
semantics of the `kubernetes-pod` or `helm-pod` action types are
specifically required.
  • Loading branch information
@thsig
thsig committed Mar 16, 2023
1 parent df9d908 commit 3a942ef
Show file tree
Hide file tree
Showing 29 changed files with 2,602 additions and 235 deletions.
2 changes: 1 addition & 1 deletion core/src/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export const DEFAULT_PORT_PROTOCOL = "TCP"
export const DEFAULT_API_VERSION = "garden.io/v0"

export const DEFAULT_TEST_TIMEOUT = 60 * 1000
export const DEFAULT_TASK_TIMEOUT = 60 * 1000
export const DEFAULT_RUN_TIMEOUT = 60 * 1000

export type SupportedPlatform = "linux" | "darwin" | "win32"
export const SUPPORTED_PLATFORMS: SupportedPlatform[] = ["linux", "darwin", "win32"]
Expand Down
33 changes: 33 additions & 0 deletions core/src/plugins/kubernetes/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ import { V1Toleration } from "@kubernetes/client-node"
import { runPodSpecIncludeFields } from "./run"
import { SyncDefaults, syncDefaultsSchema } from "./sync"
import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl"
import { readFileSync } from "fs-extra"
import { join } from "path"
import { STATIC_DIR } from "../../constants"

export const DEFAULT_KANIKO_IMAGE = "gcr.io/kaniko-project/executor:v1.8.1-debug"

Expand Down Expand Up @@ -976,6 +979,36 @@ export const kubernetesCommonRunSchemaKeys = () => ({
namespace: namespaceNameSchema(),
})

export const runPodResourceSchema = (kind: string) =>
targetResourceSpecSchema().description(
dedent`
Specify a Kubernetes resource to derive the Pod spec from for the ${kind}.
This resource will be selected from the manifests provided in this ${kind}'s \`files\` or \`manifests\` config field.
The following fields from the Pod will be used (if present) when executing the ${kind}:
${runPodSpecWhitelistDescription()}
`
)

// Need to use a sync read to avoid having to refactor createGardenPlugin()
// The `podspec-v1.json` file is copied from the handy
// kubernetes-json-schema repo (https://github.com/instrumenta/kubernetes-json-schema/tree/master/v1.18.1-standalone).
const jsonSchema = () => JSON.parse(readFileSync(join(STATIC_DIR, "kubernetes", "podspec-v1.json")).toString())

// TODO: allow reading the pod spec from a file
export const runPodSpecSchema = (kind: string) => joi
.object()
.jsonSchema({ ...jsonSchema(), type: "object" })
.description(
dedent`
Supply a custom Pod specification. This should be a normal Kubernetes Pod manifest. Note that the spec will be modified for the ${kind}, including overriding with other fields you may set here (such as \`args\` and \`env\`), and removing certain fields that are not supported.
The following Pod spec fields from the selected \`resource\` will be used (if present) when executing the ${kind}:
${runPodSpecWhitelistDescription()}
`
)

export const kubernetesTaskSchema = () =>
baseTaskSpecSchema()
.keys({
Expand Down
21 changes: 12 additions & 9 deletions core/src/plugins/kubernetes/helm/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import cryptoRandomString = require("crypto-random-string")
import { PluginContext } from "../../../plugin-context"
import { Log } from "../../../logger/log-entry"
import { getActionNamespace } from "../namespace"
import { KubernetesResource } from "../types"
import { HelmRuntimeAction, KubernetesResource } from "../types"
import { loadAll } from "js-yaml"
import { helm } from "./helm-cli"
import { HelmModule } from "./module-config"
Expand Down Expand Up @@ -48,7 +48,7 @@ async function dependencyUpdate(ctx: KubernetesPluginContext, log: Log, namespac

interface PrepareTemplatesParams {
ctx: KubernetesPluginContext
action: Resolved<HelmDeployAction>
action: Resolved<HelmRuntimeAction>
log: Log
}

Expand Down Expand Up @@ -156,7 +156,7 @@ type PrepareManifestsParams = GetChartResourcesParams & PrepareTemplatesOutput

export async function prepareManifests(params: PrepareManifestsParams): Promise<string> {
const { ctx, action, log, namespace, releaseName, valuesPath, reference } = params
const timeout = action.getSpec("timeout")
const timeout = action.getSpec().timeout

const res = await helm({
ctx,
Expand Down Expand Up @@ -238,8 +238,8 @@ export function getBaseModule(module: HelmModule): HelmModule | undefined {
/**
* Get the full absolute path to the chart, within the action build path, if applicable.
*/
export async function getChartPath(action: Resolved<HelmDeployAction>) {
const chartSpec = action.getSpec("chart") || {}
export async function getChartPath(action: Resolved<HelmRuntimeAction>) {
const chartSpec = action.getSpec().chart || {}
const chartPath = chartSpec.path || "."
const chartDir = resolve(action.getBuildPath(), chartPath)
const yamlPath = resolve(chartDir, helmChartYamlFilename)
Expand Down Expand Up @@ -271,11 +271,14 @@ export async function getChartPath(action: Resolved<HelmDeployAction>) {
/**
* Get the value files arguments that should be applied to any helm install/render command.
*/
export async function getValueArgs({ action, valuesPath }: { action: Resolved<HelmDeployAction>; valuesPath: string }) {
export async function getValueArgs({
action,
valuesPath
}: { action: Resolved<HelmRuntimeAction>; valuesPath: string }) {
// The garden-values.yml file (which is created from the `values` field in the module config) takes precedence,
// so it's added to the end of the list.
const valueFiles = action
.getSpec("valueFiles")
.getSpec().valueFiles
.map((f) => resolve(action.getBuildPath(), f))
.concat([valuesPath])

Expand All @@ -289,8 +292,8 @@ export async function getValueArgs({ action, valuesPath }: { action: Resolved<He
/**
* Get the release name to use for the module/chart (the module name, unless overridden in config).
*/
export function getReleaseName(action: Resolved<HelmDeployAction>) {
return action.getSpec("releaseName") || action.name
export function getReleaseName(action: Resolved<HelmRuntimeAction>) {
return action.getSpec().releaseName || action.name
}

/**
Expand Down
190 changes: 127 additions & 63 deletions core/src/plugins/kubernetes/helm/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,33 +6,42 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

import { DeepPrimitiveMap, joi, joiIdentifier, joiPrimitive, joiSparseArray } from "../../../config/common"
import { createSchema, DeepPrimitiveMap, joi, joiIdentifier, joiPrimitive, joiSparseArray } from "../../../config/common"
import {
kubernetesCommonRunSchemaKeys,
KubernetesCommonRunSpec,
KubernetesTargetResourceSpec,
namespaceNameSchema,
PortForwardSpec,
portForwardsSchema,
runPodResourceSchema,
targetResourceSpecSchema,
} from "../config"
import { kubernetesDeploySyncSchema, KubernetesDeploySyncSpec } from "../sync"
import { DeployAction, DeployActionConfig } from "../../../actions/deploy"
import { dedent, deline } from "../../../util/string"
import { kubernetesLocalModeSchema, KubernetesLocalModeSpec } from "../local-mode"
import { RunActionConfig, RunAction } from "../../../actions/run"
import { KubernetesRunOutputs } from "../kubernetes-type/run"
import { TestAction, TestActionConfig } from "../../../actions/test"
import { ObjectSchema } from "@hapi/joi"

// DEPLOY //

export const defaultHelmTimeout = 300
export const defaultHelmRepo = "https://charts.helm.sh/stable"

interface HelmChartSpec {
name?: string // Formerly `chart` on Helm modules
path?: string // Formerly `chartPath`
repo?: string
url?: string
version?: string
}

interface HelmDeployActionSpec {
atomicInstall: boolean
chart?: {
name?: string // Formerly `chart` on Helm modules
path?: string // Formerly `chartPath`
repo?: string // Formerly `repo`
url?: string
version?: string // Formerly `version`
}
chart?: HelmChartSpec
defaultTarget?: KubernetesTargetResourceSpec
sync?: KubernetesDeploySyncSpec
localMode?: KubernetesLocalModeSpec
Expand All @@ -53,6 +62,31 @@ const parameterValueSchema = () =>
)
.id("parameterValue")

const helmReleaseNameSchema = () => joiIdentifier().description(
"Optionally override the release name used when installing (defaults to the Deploy name)."
)

const helmValuesSchema = () => joi
.object()
.pattern(/.+/, parameterValueSchema())
.default(() => ({})).description(deline`
Map of values to pass to Helm when rendering the templates. May include arrays and nested objects.
When specified, these take precedence over the values in the \`values.yaml\` file (or the files specified
in \`valueFiles\`).
`)

const helmValueFilesSchema = () => joiSparseArray(joi.posixPath()).description(dedent`
Specify value files to use when rendering the Helm chart. These will take precedence over the \`values.yaml\` file
bundled in the Helm chart, and should be specified in ascending order of precedence. Meaning, the last file in
this list will have the highest precedence.
If you _also_ specify keys under the \`values\` field, those will effectively be added as another file at the end
of this list, so they will take precedence over other files listed here.
Note that the paths here should be relative to the _config_ root, and the files should be contained in
this action config's directory.
`)

export const helmCommonSchemaKeys = () => ({
atomicInstall: joi
.boolean()
Expand All @@ -62,35 +96,16 @@ export const helmCommonSchemaKeys = () => ({
),
namespace: namespaceNameSchema(),
portForwards: portForwardsSchema(),
releaseName: joiIdentifier().description(
"Optionally override the release name used when installing (defaults to the module name)."
),
releaseName: helmReleaseNameSchema(),
timeout: joi
.number()
.integer()
.default(defaultHelmTimeout)
.description(
"Time in seconds to wait for Helm to complete any individual Kubernetes operation (like Jobs for hooks)."
),
values: joi
.object()
.pattern(/.+/, parameterValueSchema())
.default(() => ({})).description(deline`
Map of values to pass to Helm when rendering the templates. May include arrays and nested objects.
When specified, these take precedence over the values in the \`values.yaml\` file (or the files specified
in \`valueFiles\`).
`),
valueFiles: joiSparseArray(joi.posixPath()).description(dedent`
Specify value files to use when rendering the Helm chart. These will take precedence over the \`values.yaml\` file
bundled in the Helm chart, and should be specified in ascending order of precedence. Meaning, the last file in
this list will have the highest precedence.
If you _also_ specify keys under the \`values\` field, those will effectively be added as another file at the end
of this list, so they will take precedence over other files listed here.
Note that the paths here should be relative to the _module_ root, and the files should be contained in
your module directory.
`),
values: helmValuesSchema(),
valueFiles: helmValueFilesSchema(),
})

export const helmChartNameSchema = () =>
Expand Down Expand Up @@ -120,42 +135,44 @@ export const defaultTargetSchema = () =>
`
)

const helmChartSpecSchema = () => joi
.object()
.keys({
name: helmChartNameSchema(),
path: joi
.posixPath()
.subPathOnly()
.description(
"The path, relative to the action path, to the chart sources (i.e. where the Chart.yaml file is, if any)."
),
repo: helmChartRepoSchema(),
url: joi.string().uri().description("An absolute URL to a packaged URL."),
version: helmChartVersionSchema(),
})
.with("name", ["version"])
.without("path", ["name", "repo", "version", "url"])
.without("url", ["name", "repo", "version", "path"])
.xor("name", "path", "url")
.description(
dedent`
Specify the Helm chart to use.
If the chart is defined in the same directory as the action, you can skip this, and the chart sources will be detected. If the chart is in the source tree but in a sub-directory, you should set \`chart.path\` to the directory path, relative to the action directory.
If the chart is remote, you must specify \`chart.name\` and \`chart.version\, and optionally \`chart.repo\` (if the chart is not in the default "stable" repo).
You may also specify an absolute URL to a packaged chart via \`chart.url\`.
One of \`chart.name\`, \`chart.path\` or \`chart.url\` must be specified.
`
)

export const helmDeploySchema = () =>
joi
.object()
.keys({
...helmCommonSchemaKeys(),
chart: joi
.object()
.keys({
name: helmChartNameSchema(),
path: joi
.posixPath()
.subPathOnly()
.description(
"The path, relative to the action path, to the chart sources (i.e. where the Chart.yaml file is, if any)."
),
repo: helmChartRepoSchema(),
url: joi.string().uri().description("An absolute URL to a packaged URL."),
version: helmChartVersionSchema(),
})
.with("name", ["version"])
.without("path", ["name", "repo", "version", "url"])
.without("url", ["name", "repo", "version", "path"])
.xor("name", "path", "url")
.description(
dedent`
Specify the Helm chart to deploy.
If the chart is defined in the same directory as the action, you can skip this, and the chart sources will be detected. If the chart is in the source tree but in a sub-directory, you should set \`chart.path\` to the directory path, relative to the action directory.
If the chart is remote, you must specify \`chart.name\` and \`chart.version\, and optionally \`chart.repo\` (if the chart is not in the default "stable" repo).
You may also specify an absolute URL to a packaged chart via \`chart.url\`.
One of \`chart.name\`, \`chart.path\` or \`chart.url\` must be specified.
`
),
chart: helmChartSpecSchema(),
defaultTarget: defaultTargetSchema(),
sync: kubernetesDeploySyncSchema(),
localMode: kubernetesLocalModeSchema(),
Expand All @@ -165,6 +182,53 @@ export const helmDeploySchema = () =>
export type HelmDeployConfig = DeployActionConfig<"helm", HelmDeployActionSpec>
export type HelmDeployAction = DeployAction<HelmDeployConfig, {}>

// NOTE: Runs and Tests are handled as `kubernetes` Run and Test actions
// RUN & TEST //

export interface HelmPodRunActionSpec extends KubernetesCommonRunSpec {
chart?: HelmChartSpec
namespace?: string
releaseName?: string
timeout: number
values: DeepPrimitiveMap
valueFiles: string[]
resource?: KubernetesTargetResourceSpec
}

// Maintaining this cache to avoid errors when `kubernetesRunPodSchema` is called more than once with the same `kind`.
const runSchemas: { [name: string]: ObjectSchema } = {}

export const helmPodRunSchema = (kind: string) => {
const name = `${kind}:helm-pod`
if (runSchemas[name]) {
return runSchemas[name]
}
const schema = createSchema({
name: `${kind}:helm-pod`,
keys: () => ({
...kubernetesCommonRunSchemaKeys(),
releaseName: helmReleaseNameSchema()
.description(`Optionally override the release name used when rendering the templates (defaults to the ${kind} name).`),
chart: helmChartSpecSchema(),
values: helmValuesSchema(),
valueFiles: helmValueFilesSchema(),
resource: runPodResourceSchema("Run"),
timeout: joi
.number()
.integer()
.default(defaultHelmTimeout)
.description("Time in seconds to wait for Helm to render templates."),
}),
xor: ["resource", "podSpec"],
})()
runSchemas[name] = schema
return schema
}

export type HelmPodRunConfig = RunActionConfig<"helm-pod", HelmPodRunActionSpec>
export type HelmPodRunAction = RunAction<HelmPodRunConfig, KubernetesRunOutputs>

export interface HelmPodTestActionSpec extends HelmPodRunActionSpec {}
export type HelmPodTestConfig = TestActionConfig<"helm-pod", HelmPodTestActionSpec>
export type HelmPodTestAction = TestAction<HelmPodTestConfig, KubernetesRunOutputs>

export type HelmActionConfig = HelmDeployConfig
export type HelmActionConfig = HelmDeployConfig | HelmPodRunConfig | HelmPodTestConfig
Loading

0 comments on commit 3a942ef

Please sign in to comment.