feat: remove stale build dockerfile and notify client of build

orchestrator/packages/common/src/types/image-build-service.ts

@@ -1,6 +1,6 @@
 export interface GraderImageBuildResult {
   was_successful: boolean,
-  logs: Array<ImageBuildLog>
+  logs: Array<ImageBuildLog | string>
 }
 
 export type ImageBuildStep = "Write request contents to Dockerfile." |

orchestrator/packages/db/src/image-builder-operations/clean-up-stale-build-info.ts → orchestrator/packages/db/src/image-builder-operations/clean-stale-build-info.ts

@@ -1,8 +1,9 @@
 import { graderImageExists } from "@codegrade-orca/common";
+import { GraderImageBuildRequest } from "@codegrade-orca/common";
 import prismaInstance from "../prisma-instance"
 import handleCompletedImageBuild, { EnqueuedJobInfo } from "./handle-completed-image-build"
 
-const cleanStaleBuildInfo = async (): Promise<EnqueuedJobInfo[]> =>
+const cleanStaleBuildInfo = async (): Promise<Array<[GraderImageBuildRequest, EnqueuedJobInfo[]]>> =>
   prismaInstance.$transaction(async (tx) => {
     const possibleStaleBuildInfo = await tx.imageBuildInfo.findMany({ where: { inProgress: true } });
     if (!possibleStaleBuildInfo.length) {
@@ -12,12 +13,17 @@ const cleanStaleBuildInfo = async (): Promise<EnqueuedJobInfo[]> =>
     return await Promise.all(possibleStaleBuildInfo.map(async (buildInfo) => {
       const { dockerfileSHA } = buildInfo;
       if (graderImageExists(dockerfileSHA)) {
-        return await handleCompletedImageBuild(dockerfileSHA, true) as EnqueuedJobInfo[];
+        const originalReq: GraderImageBuildRequest = {
+          dockerfile_sha_sum: dockerfileSHA,
+          dockerfile_contents: buildInfo.dockerfileContent,
+          response_url: buildInfo.responseURL
+        };
+        return [originalReq, await handleCompletedImageBuild(dockerfileSHA, true) as EnqueuedJobInfo[]];
       } else {
         await tx.imageBuildInfo.update({ where: { dockerfileSHA }, data: { inProgress: false } });
         return [];
       }
-    })).then((lists) => lists.flat());
+    })).then((lists) => lists.filter((possiblePair) => possiblePair.length)) as [GraderImageBuildRequest, EnqueuedJobInfo[]][];
   });
 
 export default cleanStaleBuildInfo;

orchestrator/packages/db/src/image-builder-operations/index.ts

@@ -1,6 +1,6 @@
 import getNextImageBuild from "./get-next-image-build";
 import handleCompletedImageBuild, { EnqueuedJobInfo, CancelJobInfo } from "./handle-completed-image-build";
-import cleanStaleBuildInfo from "./clean-up-stale-build-info";
+import cleanStaleBuildInfo from "./clean-stale-build-info";
 
 export { getNextImageBuild, EnqueuedJobInfo, CancelJobInfo };
 export { handleCompletedImageBuild };

orchestrator/packages/image-build-service/src/index.ts

@@ -2,21 +2,29 @@ import {
   GraderImageBuildRequest,
   toMilliseconds,
   isImageBuildResult,
-  pushStatusUpdate
+  pushStatusUpdate,
+  getConfig,
+  GraderImageBuildResult
 } from "@codegrade-orca/common";
 import { getNextImageBuild, handleCompletedImageBuild } from "@codegrade-orca/db";
 import { createAndStoreGraderImage, removeStaleImageFiles } from "./process-request";
 import { cleanUpDockerFiles, sendJobResultForBuildFail, removeImageFromDockerIfExists, notifyClientOfBuildResult } from "./utils";
 import { EnqueuedJobInfo, cleanStaleBuildInfo } from "@codegrade-orca/db";
+import path from "path";
+import { existsSync, rmSync } from "fs";
 
 const LOOP_SLEEP_TIME = 5; // Seconds
 
 const main = async () => {
   console.info("Cleaning up stale build info...");
-  const enqueuedJobs = await cleanStaleBuildInfo();
-  await Promise.all(enqueuedJobs.map(
-    ({ response_url, key, ...status }) => pushStatusUpdate(status, response_url, key)
-  ));
+  const shaSumJobInfoPairs = await cleanStaleBuildInfo();
+  shaSumJobInfoPairs.forEach(([originalReq, enqueuedJobs]) => {
+    removeDockerfileIfExists(originalReq.dockerfile_sha_sum);
+    notifyClientOfBuildResult(cleanedImageResult(), originalReq);
+    enqueuedJobs.forEach(
+      ({ response_url, key, ...status }) => pushStatusUpdate(status, response_url, key)
+    );
+  });
   console.info("Build service initialized.");
   while (true) {
     let infoAsBuildReq: GraderImageBuildRequest | undefined = undefined;
@@ -29,28 +37,26 @@ const main = async () => {
       }
 
       console.info(`Attempting to build image with SHA ${nextBuildReq.dockerfileSHA}.`);
+
       infoAsBuildReq = {
         dockerfile_sha_sum: nextBuildReq.dockerfileSHA,
         dockerfile_contents: nextBuildReq.dockerfileContent,
         response_url: nextBuildReq.responseURL,
       };
+
       const result = await createAndStoreGraderImage(infoAsBuildReq);
-      // When success is passed as true, we get EnqueuedJobInfo[].
       const jobInfo = await handleCompletedImageBuild(nextBuildReq.dockerfileSHA, true) as EnqueuedJobInfo[];
-      await notifyClientOfBuildResult(result, infoAsBuildReq);
-      await Promise.all(jobInfo.map(({ key, response_url, ...status }) => pushStatusUpdate(status, response_url, key)));
+
+      notifyClientOfBuildResult(result, infoAsBuildReq);
+      jobInfo.forEach(({ key, response_url, ...status }) => pushStatusUpdate(status, response_url, key));
       console.info(`Successfully built image with SHA ${nextBuildReq.dockerfileSHA}.`);
     } catch (err) {
       if (isImageBuildResult(err) && infoAsBuildReq) {
         const cancelledJobInfoList = await handleCompletedImageBuild(infoAsBuildReq.dockerfile_sha_sum, false);
         if (cancelledJobInfoList !== null) {
-          await Promise.all(cancelledJobInfoList.map((cancelInfo) => {
-            sendJobResultForBuildFail(
-              cancelInfo,
-            ).catch((notifyError) => console.error(notifyError)); // At this point we can't really do anything, but we should at least log out what happened.
-          }));
+          cancelledJobInfoList.forEach((cancelInfo) => sendJobResultForBuildFail(cancelInfo));
         }
-        await notifyClientOfBuildResult(err, infoAsBuildReq).catch((notifyError) => console.error(notifyError));
+        notifyClientOfBuildResult(err, infoAsBuildReq);
         await cleanUpDockerFiles(infoAsBuildReq.dockerfile_sha_sum);
       }
       console.error(err);
@@ -63,6 +69,22 @@ const main = async () => {
   }
 };
 
+const cleanedImageResult = (): GraderImageBuildResult => ({
+  was_successful: true,
+  logs: [
+    "This image successfully built but then the system crashed; we have cleaned up extra files and the image can now be used without issue."
+  ]
+});
+
+const removeDockerfileIfExists = (dockerfileSHASum: string) => {
+  const { dockerImageFolder } = getConfig();
+  const imagePath = path.join(dockerImageFolder, `${dockerfileSHASum}.Dockerfile}`)
+  if (!existsSync(imagePath)) {
+    return;
+  }
+  rmSync(imagePath);
+}
+
 const sleep = (seconds: number): Promise<void> => {
   return new Promise((resolve) => {
     setTimeout(() => {

orchestrator/packages/image-build-service/src/utils.ts

@@ -47,29 +47,29 @@ const imageExistsInDocker = (dockerfileSHASum: string): Promise<boolean> => {
   });
 };
 
-export const sendJobResultForBuildFail = async (cancelInfo: CancelJobInfo) => {
+export const sendJobResultForBuildFail = (cancelInfo: CancelJobInfo) => {
   const result: GradingJobResult = {
     shell_responses: [],
     errors: ["The grader image for this job failed to build. Please contact a Professor or Admin."]
   };
-  await fetch(cancelInfo.response_url, {
+  fetch(cancelInfo.response_url, {
     method: "POST",
     headers: {
       "Accept": "application/json",
       "Content-Type": "application/json"
     },
     body: JSON.stringify({ ...result, key: cancelInfo.key })
-  });
+  }).catch((err) => console.error(err));
 }
 
-export const notifyClientOfBuildResult = async (result: GraderImageBuildResult, originalReq: GraderImageBuildRequest) => {
+export const notifyClientOfBuildResult = (result: GraderImageBuildResult, originalReq: GraderImageBuildRequest) => {
   const { response_url } = originalReq;
-  await fetch(response_url, {
+  fetch(response_url, {
     method: "POST",
     headers: {
       "Accept": "application/json",
       "Content-Type": "application/json"
     },
     body: JSON.stringify(result)
-  });
+  }).catch((err) => console.error(err));
 }