Deploy option to enable continuous deployment #1745
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,10 @@ | ||
| import {exec} from "node:child_process"; | ||
| import {createHash} from "node:crypto"; | ||
| import type {Stats} from "node:fs"; | ||
| import {existsSync} from "node:fs"; | ||
| import {readFile, stat} from "node:fs/promises"; | ||
| import {join} from "node:path/posix"; | ||
| import {promisify} from "node:util"; | ||
| import slugify from "@sindresorhus/slugify"; | ||
| import wrapAnsi from "wrap-ansi"; | ||
| import type {BuildEffects, BuildManifest, BuildOptions} from "./build.js"; | ||
|
|
@@ -15,7 +18,7 @@ import {visitFiles} from "./files.js"; | |
| import type {Logger} from "./logger.js"; | ||
| import type {AuthEffects} from "./observableApiAuth.js"; | ||
| import {defaultEffects as defaultAuthEffects, formatUser, loginInner, validWorkspaces} from "./observableApiAuth.js"; | ||
| import {ObservableApiClient, getObservableUiOrigin} from "./observableApiClient.js"; | ||
| import type { | ||
| DeployManifestFile, | ||
| GetCurrentUserResponse, | ||
|
|
@@ -33,6 +36,26 @@ const DEPLOY_POLL_MAX_MS = 1000 * 60 * 5; | |
| const DEPLOY_POLL_INTERVAL_MS = 1000 * 5; | ||
| const BUILD_AGE_WARNING_MS = 1000 * 60 * 5; | ||
|
|
||
| const OBSERVABLE_UI_ORIGIN = getObservableUiOrigin(); | ||
|
|
||
| function settingsUrl(deployTarget: DeployTargetInfo) { | ||
| if (deployTarget.create) throw new Error("Incorrect deploy target state"); | ||
| return `${OBSERVABLE_UI_ORIGIN}projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}`; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the ownerName and repoName of the first GitHub remote (HTTPS or SSH) | ||
| * on the current repository. Supports both https and ssh URLs: | ||
| * - https://github.com/observablehq/framework.git | ||
| * - [email protected]:observablehq/framework.git | ||
| */ | ||
| async function getGitHubRemote(): Promise<{ownerName: string; repoName: string} | undefined> { | ||
| const firstRemote = (await promisify(exec)("git remote -v")).stdout.match( | ||
| /^\S+\s(https:\/\/github.com\/|[email protected]:)(?<ownerName>[^/]+)\/(?<repoName>[^/]*?)(\.git)?\s/m | ||
| ); | ||
| return firstRemote?.groups as {ownerName: string; repoName: string} | undefined; | ||
| } | ||
|
|
||
| export interface DeployOptions { | ||
| config: Config; | ||
| deployConfigPath: string | undefined; | ||
|
|
@@ -84,7 +107,7 @@ type DeployTargetInfo = | |
| | {create: true; workspace: {id: string; login: string}; projectSlug: string; title: string; accessLevel: string} | ||
| | {create: false; workspace: {id: string; login: string}; project: GetProjectResponse}; | ||
|
|
||
| /** Deploy a project to Observable */ | ||
| export async function deploy(deployOptions: DeployOptions, effects = defaultEffects): Promise<void> { | ||
| Telemetry.record({event: "deploy", step: "start", force: deployOptions.force}); | ||
| effects.clack.intro(`${inverse(" observable deploy ")} ${faint(`v${process.env.npm_package_version}`)}`); | ||
|
|
@@ -180,24 +203,145 @@ class Deployer { | |
| const {deployId} = this.deployOptions; | ||
| if (!deployId) throw new Error("invalid deploy options"); | ||
| await this.checkDeployCreated(deployId); | ||
| await this.uploadFiles(deployId, await this.getBuildFilePaths()); | ||
| await this.markDeployUploaded(deployId); | ||
| return await this.pollForProcessingCompletion(deployId); | ||
| } | ||
|
|
||
| private async cloudBuild(deployTarget: DeployTargetInfo) { | ||
| if (deployTarget.create) throw new Error("Incorrect deploy target state"); | ||
| const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; | ||
| await this.apiClient.postProjectBuild(deployTarget.project.id); | ||
| const spinner = this.effects.clack.spinner(); | ||
| spinner.start("Requesting deploy"); | ||
| const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; | ||
| while (true) { | ||
| if (Date.now() > pollExpiration) { | ||
| spinner.stop("Requesting deploy timed out."); | ||
| throw new CliError("Requesting deploy failed"); | ||
| } | ||
| const {latestCreatedDeployId} = await this.apiClient.getProject({ | ||
| workspaceLogin: deployTarget.workspace.login, | ||
| projectSlug: deployTarget.project.slug | ||
| }); | ||
| if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { | ||
| spinner.stop( | ||
| `Deploy started. Watch logs: ${link(`${settingsUrl(deployTarget)}/deploys/${latestCreatedDeployId}`)}` | ||
| ); | ||
| // latestCreatedDeployId is initially null for a new project, but once | ||
| // it changes to a string it can never change back; since we know it has | ||
| // changed, we assert here that it’s not null | ||
| return latestCreatedDeployId!; | ||
| } | ||
| await new Promise((resolve) => setTimeout(resolve, pollInterval)); | ||
| } | ||
| } | ||
|
|
||
| // Throws error if local and remote GitHub repos don’t match or are invalid. | ||
| // Ignores this.deployOptions.config.root as we only support cloud builds from | ||
| // the root directory. | ||
| private async validateGitHubLink(deployTarget: DeployTargetInfo): Promise<void> { | ||
| if (deployTarget.create) throw new Error("Incorrect deploy target state"); | ||
| if (!deployTarget.project.build_environment_id) throw new CliError("No build environment configured."); | ||
|
There was a problem hiding this comment. Here we could add a link to the build settings page maybe? something like |
||
| if (!existsSync(".git")) throw new CliError("Not at root of a git repository."); | ||
| const remote = await getGitHubRemote(); | ||
| if (!remote) throw new CliError("No GitHub remote found."); | ||
| const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout.trim(); | ||
| if (!branch) throw new Error("Branch not found."); | ||
|
|
||
| // If a source repository has already been configured, check that it’s | ||
| // accessible and matches the linked repository and branch. TODO: validate | ||
| // local/remote refs match, "Your branch is up to date", and "nothing to | ||
| // commit, working tree clean". | ||
| const {source} = deployTarget.project; | ||
| if (source) { | ||
| const linkedRepo = await this.apiClient.getGitHubRepository(remote); | ||
| if (linkedRepo) { | ||
| if (source.provider_id !== linkedRepo.provider_id) { | ||
| throw new CliError( | ||
| `Configured repository does not match local repository; check build settings on ${link( | ||
| `${settingsUrl(deployTarget)}/settings` | ||
| )}` | ||
| ); | ||
| } | ||
| if (source.branch !== branch) { | ||
| // TODO: If source.branch is empty, it'll use the default repository | ||
| // branch (usually main or master), which we don't know from our current | ||
| // getGitHubRepository response, and thus can't check here. | ||
| throw new CliError( | ||
| `Configured branch ${source.branch} does not match local branch ${branch}; check build settings on ${link( | ||
| `${settingsUrl(deployTarget)}/settings` | ||
| )}` | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| if (!(await this.apiClient.getGitHubRepository({providerId: source.provider_id}))) { | ||
| // TODO: This could poll for auth too, but is a distinct case because it | ||
| // means the repo was linked at one point and then something went wrong | ||
| throw new CliError( | ||
| `Cannot access configured repository; check build settings on ${link( | ||
| `${settingsUrl(deployTarget)}/settings` | ||
| )}` | ||
| ); | ||
| } | ||
|
|
||
| // Configured repo is OK; proceed | ||
| return; | ||
| } | ||
|
|
||
| // If the source has not been configured, first check that the remote repo | ||
| // is linked in CD settings. If not, prompt the user to auth & link. | ||
| let linkedRepo = await this.apiClient.getGitHubRepository(remote); | ||
| if (!linkedRepo) { | ||
| if (!this.effects.isTty) | ||
| throw new CliError( | ||
| "Cannot access repository for continuous deployment and cannot request access in non-interactive mode" | ||
| ); | ||
|
|
||
| // Repo is not authorized; link to auth page and poll for auth | ||
| const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); | ||
| authUrl.searchParams.set("owner", remote.ownerName); | ||
| authUrl.searchParams.set("repo", remote.repoName); | ||
| this.effects.clack.log.info( | ||
| `Authorize Observable to access the ${bold(remote.repoName)} repository: ${link(authUrl)}` | ||
| ); | ||
|
|
||
| const spinner = this.effects.clack.spinner(); | ||
| spinner.start("Waiting for authorization"); | ||
| const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; | ||
| const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; | ||
| do { | ||
| await new Promise((resolve) => setTimeout(resolve, pollInterval)); | ||
| if (Date.now() > pollExpiration) { | ||
| spinner.stop("Authorization timed out."); | ||
| throw new CliError("Repository authorization failed"); | ||
| } | ||
| } while (!(linkedRepo = await this.apiClient.getGitHubRepository(remote))); | ||
| spinner.stop("Repository authorized."); | ||
| } | ||
|
|
||
| // Save the linked repo as the configured source. | ||
| const {provider, provider_id, url} = linkedRepo; | ||
| await this.apiClient | ||
| .postProjectEnvironment(deployTarget.project.id, {source: {provider, provider_id, url, branch}}) | ||
| .catch((error) => { | ||
| throw new CliError("Setting source repository for continuous deployment failed", {cause: error}); | ||
| }); | ||
| } | ||
|
|
||
| private async startNewDeploy(): Promise<GetDeployResponse> { | ||
|
Comment on lines
-194
to
-196
There was a problem hiding this comment. fil notes that this could be parallelized: getBuildFilePaths doesn't depend on the previous two; it's filesystem i/o, as opposed to talking to the api. might speed up local deploys a bit There was a problem hiding this comment. not gonna worry about this for now; also, buildFilePaths is only relevant for the local deploy path, so it's not as clean to parallelize anymore |
||
| const {deployConfig, deployTarget} = await this.getDeployTarget(await this.getUpdatedDeployConfig()); | ||
| let deployId: string; | ||
| if (deployConfig.continuousDeployment) { | ||
| await this.validateGitHubLink(deployTarget); | ||
| deployId = await this.cloudBuild(deployTarget); | ||
| } else { | ||
| const buildFilePaths = await this.getBuildFilePaths(); | ||
| deployId = await this.createNewDeploy(deployTarget); | ||
| await this.uploadFiles(deployId, buildFilePaths); | ||
| await this.markDeployUploaded(deployId); | ||
| } | ||
| return await this.pollForProcessingCompletion(deployId); | ||
tophtucker marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
|
|
@@ -210,11 +354,7 @@ class Deployer { | |
| } | ||
| return deployInfo; | ||
| } catch (error) { | ||
| if (isHttpError(error)) throw new CliError(`Deploy ${deployId} not found.`, {cause: error}); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
@@ -274,7 +414,9 @@ class Deployer { | |
| } | ||
|
|
||
| // Get the deploy target, prompting the user as needed. | ||
| private async getDeployTarget( | ||
| deployConfig: DeployConfig | ||
| ): Promise<{deployTarget: DeployTargetInfo; deployConfig: DeployConfig}> { | ||
| let deployTarget: DeployTargetInfo; | ||
| if (deployConfig.workspaceLogin && deployConfig.projectSlug) { | ||
| try { | ||
|
|
@@ -384,18 +526,37 @@ class Deployer { | |
| } | ||
| } | ||
|
|
||
| let {continuousDeployment} = deployConfig; | ||
| if (continuousDeployment === null) { | ||
| const enable = await this.effects.clack.confirm({ | ||
| message: wrapAnsi( | ||
| `Do you want to enable continuous deployment? ${faint( | ||
| "Given a GitHub repository, this builds in the cloud and redeploys whenever you push to the current branch." | ||
| )}`, | ||
| this.effects.outputColumns | ||
| ), | ||
| active: "Yes, enable and build in cloud", | ||
| inactive: "No, build locally" | ||
| }); | ||
| if (this.effects.clack.isCancel(enable)) throw new CliError("User canceled deploy", {print: false, exitCode: 0}); | ||
| continuousDeployment = enable; | ||
| } | ||
|
|
||
| deployConfig = { | ||
| projectId: deployTarget.project.id, | ||
| projectSlug: deployTarget.project.slug, | ||
| workspaceLogin: deployTarget.workspace.login, | ||
| continuousDeployment | ||
| }; | ||
|
|
||
| await this.effects.setDeployConfig( | ||
| this.deployOptions.config.root, | ||
| this.deployOptions.deployConfigPath, | ||
| deployConfig, | ||
| this.effects | ||
| ); | ||
|
|
||
| return {deployConfig, deployTarget}; | ||
| } | ||
|
|
||
| // Create the new deploy on the server. | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
chatting with fil: there's no guarantee that the new deploy ID is the one you just kicked off; maybe your colleague click the deploy button around the same time. currently, the postProjectBuild method can't return the new deploy ID because it just dispatches a message to the job-manager, which at some point will get around to making a new deploy. two options:
but fil thinks this is probably not a blocking issue with this PR
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not planning to do anything about this for now