From de072ac91bc4b65fefe756820c28d3059dbac40a Mon Sep 17 00:00:00 2001 From: Thorarinn Sigurdsson Date: Thu, 9 Jul 2020 13:15:45 +0200 Subject: [PATCH] feat(enterprise): register workflow runs When the user is logged in to Garden Enterprise, the `run workflow` command will now register a workflow run with the backend, and include the workflow run's UID in streamed events and log entries. This enables `run workflow` command invocations from e.g. a local machine or an external CI system to be represented in Garden Enterprise. --- garden-service/src/cli/cli.ts | 9 +-- garden-service/src/commands/get/get-config.ts | 5 +- garden-service/src/commands/login.ts | 5 +- garden-service/src/commands/run/workflow.ts | 18 ++++++ garden-service/src/constants.ts | 4 ++ .../src/enterprise/buffered-event-stream.ts | 34 ++++++++--- .../src/enterprise/workflow-lifecycle.ts | 59 +++++++++++++++++++ garden-service/src/events.ts | 8 +++ garden-service/src/exceptions.ts | 4 ++ garden-service/src/garden.ts | 36 ++++++----- .../test/unit/src/commands/run/workflow.ts | 38 +++++++----- .../src/platform/buffered-event-stream.ts | 8 ++- 12 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 garden-service/src/enterprise/workflow-lifecycle.ts diff --git a/garden-service/src/cli/cli.ts b/garden-service/src/cli/cli.ts index 29ec410feb..0407f989cf 100644 --- a/garden-service/src/cli/cli.ts +++ b/garden-service/src/cli/cli.ts @@ -341,21 +341,16 @@ export class GardenCli { garden = await Garden.factory(root, contextOpts) } - if (garden.clientAuthToken && garden.enterpriseDomain && garden.projectId) { + if (garden.enterpriseContext) { log.silly(`Connecting Garden instance to BufferedEventStream`) bufferedEventStream.connect({ eventBus: garden.events, - clientAuthToken: garden.clientAuthToken, - enterpriseDomain: garden.enterpriseDomain, - projectId: garden.projectId, + enterpriseContext: garden.enterpriseContext, environmentName: garden.environmentName, namespace: garden.namespace, }) } else { log.silly(`Skip connecting Garden instance to BufferedEventStream`) - log.silly(`clientAuthToken present: ${!!garden.clientAuthToken}`) - log.silly(`enterpriseDomain: ${garden.enterpriseDomain}`) - log.silly(`projectId: ${garden.projectId}`) } // Register log file writers. We need to do this after the Garden class is initialised because diff --git a/garden-service/src/commands/get/get-config.ts b/garden-service/src/commands/get/get-config.ts index 61ee2362ee..a341d522a8 100644 --- a/garden-service/src/commands/get/get-config.ts +++ b/garden-service/src/commands/get/get-config.ts @@ -51,7 +51,10 @@ export class GetConfigCommand extends Command<{}, Opts> { .description("All workflow configs in the project."), projectName: joi.string().description("The name of the project."), projectRoot: joi.string().description("The local path to the project root."), - projectId: joi.string().description("The project ID (Garden Enterprise only)."), + projectId: joi + .string() + .optional() + .description("The project ID (Garden Enterprise only)."), }) options = getConfigOptions diff --git a/garden-service/src/commands/login.ts b/garden-service/src/commands/login.ts index 41fbad74b9..3ba394bc14 100644 --- a/garden-service/src/commands/login.ts +++ b/garden-service/src/commands/login.ts @@ -23,10 +23,11 @@ export class LoginCommand extends Command { async action({ garden, log, headerLog }: CommandParams): Promise { printHeader(headerLog, "Login", "cloud") - if (!garden.enterpriseDomain) { + const enterpriseDomain = garden.enterpriseContext?.enterpriseDomain + if (!enterpriseDomain) { throw new ConfigurationError(`Error: Your project configuration does not specify a domain.`, {}) } - await login(garden.enterpriseDomain, log) + await login(enterpriseDomain, log) return {} } } diff --git a/garden-service/src/commands/run/workflow.ts b/garden-service/src/commands/run/workflow.ts index e5f96dc3e3..c08219702e 100644 --- a/garden-service/src/commands/run/workflow.ts +++ b/garden-service/src/commands/run/workflow.ts @@ -25,6 +25,8 @@ import { getDurationMsec } from "../../util/util" import { runScript } from "../../util/util" import { ExecaError } from "execa" import { LogLevel } from "../../logger/log-node" +import { gardenEnv } from "../../constants" +import { registerWorkflowRun } from "../../enterprise/workflow-lifecycle" const runWorkflowArgs = { workflow: new StringParameter({ @@ -63,6 +65,8 @@ export class RunWorkflowCommand extends Command { const outerLog = log.placeholder() // Prepare any configured files before continuing const workflow = garden.getWorkflowConfig(args.workflow) + await registerAndSetUid(garden, log, workflow) + garden.events.emit("workflowRunning", {}) const templateContext = new WorkflowConfigContext(garden) const files = resolveTemplateStrings(workflow.files || [], templateContext) @@ -175,6 +179,7 @@ export class RunWorkflowCommand extends Command { } printResult({ startedAt, log: outerLog, workflow, success: true }) + garden.events.emit("workflowComplete", {}) return { result } } @@ -318,6 +323,19 @@ export function logErrors( } } +async function registerAndSetUid(garden: Garden, log: LogEntry, config: WorkflowConfig) { + if (garden.enterpriseContext && !gardenEnv.GARDEN_PLATFORM_SCHEDULED) { + const workflowRunUid = await registerWorkflowRun({ + workflowConfig: config, + enterpriseContext: garden.enterpriseContext, + environment: garden.environmentName, + namespace: garden.namespace, + log, + }) + garden.events.emit("_workflowRunRegistered", { workflowRunUid }) + } +} + async function writeWorkflowFile(garden: Garden, file: WorkflowFileSpec) { let data: string diff --git a/garden-service/src/constants.ts b/garden-service/src/constants.ts index e6ac512870..ecc54ecad5 100644 --- a/garden-service/src/constants.ts +++ b/garden-service/src/constants.ts @@ -87,6 +87,10 @@ export const gardenEnv = { .get("GARDEN_LOGGER_TYPE") .required(false) .asString(), + GARDEN_PLATFORM_SCHEDULED: env + .get("GARDEN_PLATFORM_SCHEDULED") + .required(false) + .asBool(), GARDEN_SERVER_PORT: env .get("GARDEN_SERVER_PORT") .required(false) diff --git a/garden-service/src/enterprise/buffered-event-stream.ts b/garden-service/src/enterprise/buffered-event-stream.ts index c2ac392123..fc9838b906 100644 --- a/garden-service/src/enterprise/buffered-event-stream.ts +++ b/garden-service/src/enterprise/buffered-event-stream.ts @@ -14,6 +14,7 @@ import { got } from "../util/http" import { makeAuthHeader } from "./auth" import { LogLevel } from "../logger/log-node" import { gardenEnv } from "../constants" +import { GardenEnterpriseContext } from "../garden" export type StreamEvent = { name: EventName @@ -48,13 +49,13 @@ export const MAX_BATCH_SIZE = 100 export interface ConnectBufferedEventStreamParams { eventBus: EventBus - clientAuthToken: string - enterpriseDomain: string - projectId: string + enterpriseContext: GardenEnterpriseContext environmentName: string namespace: string } +export const controlEventNames: Set = new Set(["_workflowRunRegistered"]) + /** * Buffers events and log entries and periodically POSTs them to Garden Enterprise if the user is logged in. * @@ -73,6 +74,7 @@ export class BufferedEventStream { private projectId: string private environmentName: string private namespace: string + private workflowRunUid: string | undefined /** * We maintain this map to facilitate unsubscribing from a previously connected event bus @@ -96,9 +98,10 @@ export class BufferedEventStream { connect(params: ConnectBufferedEventStreamParams) { this.log.silly("BufferedEventStream: Connected") - this.clientAuthToken = params.clientAuthToken - this.enterpriseDomain = params.enterpriseDomain - this.projectId = params.projectId + const ctx = params.enterpriseContext + this.clientAuthToken = ctx.clientAuthToken + this.enterpriseDomain = ctx.enterpriseDomain + this.projectId = ctx.projectId this.environmentName = params.environmentName this.namespace = params.namespace @@ -155,6 +158,11 @@ export class BufferedEventStream { } streamEvent(name: T, payload: Events[T]) { + if (controlEventNames.has(name)) { + this.handleControlEvent(name, payload) + return + } + this.bufferedEvents.push({ name, payload, @@ -173,7 +181,7 @@ export class BufferedEventStream { } const data = { events, - workflowRunUid: gardenEnv.GARDEN_WORKFLOW_RUN_UID, + workflowRunUid: this.getWorkflowRunUid(), sessionId: this.sessionId, projectUid: this.projectId, environment: this.environmentName, @@ -196,7 +204,7 @@ export class BufferedEventStream { } const data = { logEntries, - workflowRunUid: gardenEnv.GARDEN_WORKFLOW_RUN_UID, + workflowRunUid: this.getWorkflowRunUid(), sessionId: this.sessionId, projectUid: this.projectId, } @@ -220,4 +228,14 @@ export class BufferedEventStream { return Bluebird.all([this.flushEvents(eventsToFlush), this.flushLogEntries(logEntriesToFlush)]) } + + getWorkflowRunUid(): string | undefined { + return gardenEnv.GARDEN_WORKFLOW_RUN_UID || this.workflowRunUid + } + + handleControlEvent(name: T, payload: Events[T]) { + if (name === "_workflowRunRegistered") { + this.workflowRunUid = payload.workflowRunUid + } + } } diff --git a/garden-service/src/enterprise/workflow-lifecycle.ts b/garden-service/src/enterprise/workflow-lifecycle.ts new file mode 100644 index 0000000000..9f1c2bc81c --- /dev/null +++ b/garden-service/src/enterprise/workflow-lifecycle.ts @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2018-2020 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { got, GotResponse } from "../util/http" +import { makeAuthHeader } from "./auth" +import { WorkflowConfig } from "../config/workflow" +import { LogEntry } from "../logger/log-entry" +import { PlatformError } from "../exceptions" +import { GardenEnterpriseContext } from "../garden" + +export interface RegisterWorkflowRunParams { + workflowConfig: WorkflowConfig + enterpriseContext: GardenEnterpriseContext + environment: string + namespace: string + log: LogEntry +} + +/** + * Registers the workflow run with the platform, and returns the UID generated for the run. + */ +export async function registerWorkflowRun({ + workflowConfig, + enterpriseContext, + environment, + namespace, + log, +}: RegisterWorkflowRunParams): Promise { + const { clientAuthToken, projectId, enterpriseDomain } = enterpriseContext + log.debug(`Registering workflow run for ${workflowConfig.name}...`) + const headers = makeAuthHeader(clientAuthToken) + const requestData = { + projectUid: projectId, + environment, + namespace, + workflowName: workflowConfig.name, + } + let res + try { + res = await got.post(`${enterpriseDomain}/workflow-runs`, { json: requestData, headers }).json>() + } catch (err) { + log.error(`An error occurred while registering workflow run: ${err.message}`) + throw err + } + + if (res && res["workflowRunUid"] && res["status"] === "success") { + return res["workflowRunUid"] + } else { + throw new PlatformError(`Error while registering workflow run: Request failed with status ${res["status"]}`, { + status: res["status"], + workflowRunUid: res["workflowRunUid"], + }) + } +} diff --git a/garden-service/src/events.ts b/garden-service/src/events.ts index c2d4e441fe..87caae4977 100644 --- a/garden-service/src/events.ts +++ b/garden-service/src/events.ts @@ -65,6 +65,9 @@ export interface Events extends LoggerEvents { _exit: {} _restart: {} _test: any + _workflowRunRegistered: { + workflowRunUid: string + } // Watcher events configAdded: { @@ -137,6 +140,8 @@ export interface Events extends LoggerEvents { } // Workflow events + workflowRunning: {} + workflowComplete: {} workflowStepProcessing: { index: number } @@ -157,6 +162,7 @@ export const eventNames: EventName[] = [ "_exit", "_restart", "_test", + "_workflowRunRegistered", "configAdded", "configRemoved", "internalError", @@ -175,6 +181,8 @@ export const eventNames: EventName[] = [ "taskStatus", "testStatus", "serviceStatus", + "workflowRunning", + "workflowComplete", "workflowStepProcessing", "workflowStepError", "workflowStepComplete", diff --git a/garden-service/src/exceptions.ts b/garden-service/src/exceptions.ts index 12853582dd..f5e44f2acf 100644 --- a/garden-service/src/exceptions.ts +++ b/garden-service/src/exceptions.ts @@ -94,3 +94,7 @@ export class TimeoutError extends GardenBaseError { export class NotFoundError extends GardenBaseError { type = "not-found" } + +export class PlatformError extends GardenBaseError { + type = "platform" +} diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index d22bf7e26f..08da193870 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -115,10 +115,16 @@ export interface GardenOpts { noEnterprise?: boolean } +export interface GardenEnterpriseContext { + clientAuthToken: string + projectId: string + enterpriseDomain: string +} + export interface GardenParams { artifactsPath: string buildDir: BuildDir - clientAuthToken: string | null + enterpriseContext: GardenEnterpriseContext | null dotIgnoreFiles: string[] environmentName: string environmentConfigs: EnvironmentConfig[] @@ -129,8 +135,6 @@ export interface GardenParams { moduleExcludePatterns?: string[] opts: GardenOpts outputs: OutputSpec[] - projectId: string | null - enterpriseDomain: string | null plugins: RegisterPluginParam[] production: boolean projectName: string @@ -159,13 +163,8 @@ export class Garden { private readonly taskGraph: TaskGraph private watcher: Watcher private asyncLock: any - - // Platform-related instance variables - public clientAuthToken: string | null - public projectId: string | null - public enterpriseDomain: string | null + public enterpriseContext: GardenEnterpriseContext | null public sessionId: string | null - public readonly configStore: ConfigStore public readonly globalConfigStore: GlobalConfigStore public readonly vcs: VcsHandler @@ -200,8 +199,7 @@ export class Garden { constructor(params: GardenParams) { this.buildDir = params.buildDir - this.clientAuthToken = params.clientAuthToken - this.enterpriseDomain = params.enterpriseDomain + this.enterpriseContext = params.enterpriseContext this.sessionId = params.sessionId this.environmentName = params.environmentName this.environmentConfigs = params.environmentConfigs @@ -213,7 +211,6 @@ export class Garden { this.rawOutputs = params.outputs this.production = params.production this.projectName = params.projectName - this.projectId = params.projectId this.projectRoot = params.projectRoot this.projectSources = params.projectSources || [] this.providerConfigs = params.providerConfigs @@ -329,19 +326,20 @@ export class Garden { const { id: projectId, domain: enterpriseDomain } = config let secrets: StringMap = {} - let clientAuthToken: string | null = null + let enterpriseContext: GardenEnterpriseContext | null = null if (!opts.noEnterprise) { const enterpriseInitResult = await enterpriseInit({ log, projectConfig: config, environmentName }) secrets = enterpriseInitResult.secrets - clientAuthToken = enterpriseInitResult.clientAuthToken + const clientAuthToken = enterpriseInitResult.clientAuthToken + if (clientAuthToken && projectId && enterpriseDomain) { + enterpriseContext = { clientAuthToken, projectId, enterpriseDomain } + } } const garden = new this({ artifactsPath, - clientAuthToken, sessionId, - enterpriseDomain: enterpriseDomain || null, - projectId: projectId || null, + enterpriseContext, projectRoot, projectName, environmentName, @@ -1172,7 +1170,7 @@ export class Garden { workflowConfigs: sortBy(workflowConfigs, "name"), projectName: this.projectName, projectRoot: this.projectRoot, - projectId: this.projectId, + projectId: this.enterpriseContext ? this.enterpriseContext.projectId : undefined, } } @@ -1201,5 +1199,5 @@ export interface ConfigDump { workflowConfigs: WorkflowConfig[] projectName: string projectRoot: string - projectId: string | null + projectId?: string } diff --git a/garden-service/test/unit/src/commands/run/workflow.ts b/garden-service/test/unit/src/commands/run/workflow.ts index 82ba07c37b..97918f7c6e 100644 --- a/garden-service/test/unit/src/commands/run/workflow.ts +++ b/garden-service/test/unit/src/commands/run/workflow.ts @@ -106,7 +106,7 @@ describe("RunWorkflowCommand", () => { expect(workflowCompletedEntry!.getMetadata()).to.eql({}, "workflowCompletedEntry") }) - it("should emit workflow step events", async () => { + it("should emit workflow events", async () => { const _garden = await makeTestGardenA() const _log = _garden.log const _defaultParams = { @@ -130,17 +130,20 @@ describe("RunWorkflowCommand", () => { const we = getWorkflowEvents(_garden) - expect(we[0]).to.eql({ name: "workflowStepProcessing", payload: { index: 0 } }) + expect(we[0]).to.eql({ name: "workflowRunning", payload: {} }) + expect(we[1]).to.eql({ name: "workflowStepProcessing", payload: { index: 0 } }) - expect(we[1].name).to.eql("workflowStepComplete") - expect(we[1].payload.index).to.eql(0) - expect(we[1].payload.durationMsec).to.gte(0) + expect(we[2].name).to.eql("workflowStepComplete") + expect(we[2].payload.index).to.eql(0) + expect(we[2].payload.durationMsec).to.gte(0) - expect(we[2]).to.eql({ name: "workflowStepProcessing", payload: { index: 1 } }) + expect(we[3]).to.eql({ name: "workflowStepProcessing", payload: { index: 1 } }) - expect(we[3].name).to.eql("workflowStepComplete") - expect(we[3].payload.index).to.eql(1) - expect(we[3].payload.durationMsec).to.gte(0) + expect(we[4].name).to.eql("workflowStepComplete") + expect(we[4].payload.index).to.eql(1) + expect(we[4].payload.durationMsec).to.gte(0) + + expect(we[5]).to.eql({ name: "workflowComplete", payload: {} }) }) function filterLogEntries(entries: LogEntry[], msgRegex: RegExp): LogEntry[] { @@ -317,10 +320,11 @@ describe("RunWorkflowCommand", () => { const we = getWorkflowEvents(_garden) - expect(we[0]).to.eql({ name: "workflowStepProcessing", payload: { index: 0 } }) - expect(we[1].name).to.eql("workflowStepError") - expect(we[1].payload.index).to.eql(0) - expect(we[1].payload.durationMsec).to.gte(0) + expect(we[0]).to.eql({ name: "workflowRunning", payload: {} }) + expect(we[1]).to.eql({ name: "workflowStepProcessing", payload: { index: 0 } }) + expect(we[2].name).to.eql("workflowStepError") + expect(we[2].payload.index).to.eql(0) + expect(we[2].payload.durationMsec).to.gte(0) }) it("should write a file with string data ahead of the run, before resolving providers", async () => { @@ -634,6 +638,12 @@ describe("RunWorkflowCommand", () => { }) function getWorkflowEvents(garden: TestGarden) { - const eventNames = ["workflowStepProcessing", "workflowStepError", "workflowStepComplete"] + const eventNames = [ + "workflowRunning", + "workflowComplete", + "workflowStepProcessing", + "workflowStepError", + "workflowStepComplete", + ] return garden.events.eventLog.filter((e) => eventNames.includes(e.name)) } diff --git a/garden-service/test/unit/src/platform/buffered-event-stream.ts b/garden-service/test/unit/src/platform/buffered-event-stream.ts index 383ac2408d..bb3a3a2f9f 100644 --- a/garden-service/test/unit/src/platform/buffered-event-stream.ts +++ b/garden-service/test/unit/src/platform/buffered-event-stream.ts @@ -14,9 +14,11 @@ import { EventBus } from "../../../../src/events" describe("BufferedEventStream", () => { const getConnectionParams = (eventBus: EventBus) => ({ eventBus, - clientAuthToken: "dummy-client-token", - enterpriseDomain: "dummy-platform_url", - projectId: "myproject", + enterpriseContext: { + clientAuthToken: "dummy-client-token", + enterpriseDomain: "dummy-platform_url", + projectId: "myproject", + }, environmentName: "my-env", namespace: "my-ns", })