diff --git a/orchestrator/packages/common/src/types/image-build-service.ts b/orchestrator/packages/common/src/types/image-build-service.ts index ec0b8085..fcc15137 100644 --- a/orchestrator/packages/common/src/types/image-build-service.ts +++ b/orchestrator/packages/common/src/types/image-build-service.ts @@ -1,6 +1,6 @@ export interface GraderImageBuildResult { was_successful: boolean, - logs: Array + logs: Array } export type ImageBuildStep = "Write request contents to Dockerfile." | diff --git a/orchestrator/packages/db/src/image-builder-operations/clean-up-stale-build-info.ts b/orchestrator/packages/db/src/image-builder-operations/clean-stale-build-info.ts similarity index 55% rename from orchestrator/packages/db/src/image-builder-operations/clean-up-stale-build-info.ts rename to orchestrator/packages/db/src/image-builder-operations/clean-stale-build-info.ts index 10812c22..768de0e6 100644 --- a/orchestrator/packages/db/src/image-builder-operations/clean-up-stale-build-info.ts +++ b/orchestrator/packages/db/src/image-builder-operations/clean-stale-build-info.ts @@ -1,8 +1,9 @@ import { graderImageExists } from "@codegrade-orca/common"; +import { GraderImageBuildRequest } from "@codegrade-orca/common"; import prismaInstance from "../prisma-instance" import handleCompletedImageBuild, { EnqueuedJobInfo } from "./handle-completed-image-build" -const cleanStaleBuildInfo = async (): Promise => +const cleanStaleBuildInfo = async (): Promise> => prismaInstance.$transaction(async (tx) => { const possibleStaleBuildInfo = await tx.imageBuildInfo.findMany({ where: { inProgress: true } }); if (!possibleStaleBuildInfo.length) { @@ -12,12 +13,17 @@ const cleanStaleBuildInfo = async (): Promise => return await Promise.all(possibleStaleBuildInfo.map(async (buildInfo) => { const { dockerfileSHA } = buildInfo; if (graderImageExists(dockerfileSHA)) { - return await handleCompletedImageBuild(dockerfileSHA, true) as EnqueuedJobInfo[]; + const originalReq: GraderImageBuildRequest = { + dockerfile_sha_sum: dockerfileSHA, + dockerfile_contents: buildInfo.dockerfileContent, + response_url: buildInfo.responseURL + }; + return [originalReq, await handleCompletedImageBuild(dockerfileSHA, true) as EnqueuedJobInfo[]]; } else { await tx.imageBuildInfo.update({ where: { dockerfileSHA }, data: { inProgress: false } }); return []; } - })).then((lists) => lists.flat()); + })).then((lists) => lists.filter((possiblePair) => possiblePair.length)) as [GraderImageBuildRequest, EnqueuedJobInfo[]][]; }); export default cleanStaleBuildInfo; diff --git a/orchestrator/packages/db/src/image-builder-operations/index.ts b/orchestrator/packages/db/src/image-builder-operations/index.ts index 293a1d97..752a26db 100644 --- a/orchestrator/packages/db/src/image-builder-operations/index.ts +++ b/orchestrator/packages/db/src/image-builder-operations/index.ts @@ -1,6 +1,6 @@ import getNextImageBuild from "./get-next-image-build"; import handleCompletedImageBuild, { EnqueuedJobInfo, CancelJobInfo } from "./handle-completed-image-build"; -import cleanStaleBuildInfo from "./clean-up-stale-build-info"; +import cleanStaleBuildInfo from "./clean-stale-build-info"; export { getNextImageBuild, EnqueuedJobInfo, CancelJobInfo }; export { handleCompletedImageBuild }; diff --git a/orchestrator/packages/image-build-service/src/index.ts b/orchestrator/packages/image-build-service/src/index.ts index 7eebb76d..8a3afd87 100644 --- a/orchestrator/packages/image-build-service/src/index.ts +++ b/orchestrator/packages/image-build-service/src/index.ts @@ -2,21 +2,29 @@ import { GraderImageBuildRequest, toMilliseconds, isImageBuildResult, - pushStatusUpdate + pushStatusUpdate, + getConfig, + GraderImageBuildResult } from "@codegrade-orca/common"; import { getNextImageBuild, handleCompletedImageBuild } from "@codegrade-orca/db"; import { createAndStoreGraderImage, removeStaleImageFiles } from "./process-request"; import { cleanUpDockerFiles, sendJobResultForBuildFail, removeImageFromDockerIfExists, notifyClientOfBuildResult } from "./utils"; import { EnqueuedJobInfo, cleanStaleBuildInfo } from "@codegrade-orca/db"; +import path from "path"; +import { existsSync, rmSync } from "fs"; const LOOP_SLEEP_TIME = 5; // Seconds const main = async () => { console.info("Cleaning up stale build info..."); - const enqueuedJobs = await cleanStaleBuildInfo(); - await Promise.all(enqueuedJobs.map( - ({ response_url, key, ...status }) => pushStatusUpdate(status, response_url, key) - )); + const shaSumJobInfoPairs = await cleanStaleBuildInfo(); + shaSumJobInfoPairs.forEach(([originalReq, enqueuedJobs]) => { + removeDockerfileIfExists(originalReq.dockerfile_sha_sum); + notifyClientOfBuildResult(cleanedImageResult(), originalReq); + enqueuedJobs.forEach( + ({ response_url, key, ...status }) => pushStatusUpdate(status, response_url, key) + ); + }); console.info("Build service initialized."); while (true) { let infoAsBuildReq: GraderImageBuildRequest | undefined = undefined; @@ -29,28 +37,26 @@ const main = async () => { } console.info(`Attempting to build image with SHA ${nextBuildReq.dockerfileSHA}.`); + infoAsBuildReq = { dockerfile_sha_sum: nextBuildReq.dockerfileSHA, dockerfile_contents: nextBuildReq.dockerfileContent, response_url: nextBuildReq.responseURL, }; + const result = await createAndStoreGraderImage(infoAsBuildReq); - // When success is passed as true, we get EnqueuedJobInfo[]. const jobInfo = await handleCompletedImageBuild(nextBuildReq.dockerfileSHA, true) as EnqueuedJobInfo[]; - await notifyClientOfBuildResult(result, infoAsBuildReq); - await Promise.all(jobInfo.map(({ key, response_url, ...status }) => pushStatusUpdate(status, response_url, key))); + + notifyClientOfBuildResult(result, infoAsBuildReq); + jobInfo.forEach(({ key, response_url, ...status }) => pushStatusUpdate(status, response_url, key)); console.info(`Successfully built image with SHA ${nextBuildReq.dockerfileSHA}.`); } catch (err) { if (isImageBuildResult(err) && infoAsBuildReq) { const cancelledJobInfoList = await handleCompletedImageBuild(infoAsBuildReq.dockerfile_sha_sum, false); if (cancelledJobInfoList !== null) { - await Promise.all(cancelledJobInfoList.map((cancelInfo) => { - sendJobResultForBuildFail( - cancelInfo, - ).catch((notifyError) => console.error(notifyError)); // At this point we can't really do anything, but we should at least log out what happened. - })); + cancelledJobInfoList.forEach((cancelInfo) => sendJobResultForBuildFail(cancelInfo)); } - await notifyClientOfBuildResult(err, infoAsBuildReq).catch((notifyError) => console.error(notifyError)); + notifyClientOfBuildResult(err, infoAsBuildReq); await cleanUpDockerFiles(infoAsBuildReq.dockerfile_sha_sum); } console.error(err); @@ -63,6 +69,22 @@ const main = async () => { } }; +const cleanedImageResult = (): GraderImageBuildResult => ({ + was_successful: true, + logs: [ + "This image successfully built but then the system crashed; we have cleaned up extra files and the image can now be used without issue." + ] +}); + +const removeDockerfileIfExists = (dockerfileSHASum: string) => { + const { dockerImageFolder } = getConfig(); + const imagePath = path.join(dockerImageFolder, `${dockerfileSHASum}.Dockerfile}`) + if (!existsSync(imagePath)) { + return; + } + rmSync(imagePath); +} + const sleep = (seconds: number): Promise => { return new Promise((resolve) => { setTimeout(() => { diff --git a/orchestrator/packages/image-build-service/src/utils.ts b/orchestrator/packages/image-build-service/src/utils.ts index 4c7abebb..47fad6c8 100644 --- a/orchestrator/packages/image-build-service/src/utils.ts +++ b/orchestrator/packages/image-build-service/src/utils.ts @@ -47,29 +47,29 @@ const imageExistsInDocker = (dockerfileSHASum: string): Promise => { }); }; -export const sendJobResultForBuildFail = async (cancelInfo: CancelJobInfo) => { +export const sendJobResultForBuildFail = (cancelInfo: CancelJobInfo) => { const result: GradingJobResult = { shell_responses: [], errors: ["The grader image for this job failed to build. Please contact a Professor or Admin."] }; - await fetch(cancelInfo.response_url, { + fetch(cancelInfo.response_url, { method: "POST", headers: { "Accept": "application/json", "Content-Type": "application/json" }, body: JSON.stringify({ ...result, key: cancelInfo.key }) - }); + }).catch((err) => console.error(err)); } -export const notifyClientOfBuildResult = async (result: GraderImageBuildResult, originalReq: GraderImageBuildRequest) => { +export const notifyClientOfBuildResult = (result: GraderImageBuildResult, originalReq: GraderImageBuildRequest) => { const { response_url } = originalReq; - await fetch(response_url, { + fetch(response_url, { method: "POST", headers: { "Accept": "application/json", "Content-Type": "application/json" }, body: JSON.stringify(result) - }); + }).catch((err) => console.error(err)); }