feat: separate endpoint for holding pen status

feat: update client on when job has been pulled from queue

orchestrator/packages/api/src/controllers/grading-queue-controller.ts

@@ -191,8 +191,6 @@ export const jobStatus = async (req: Request, res: Response) => {
   const jobQueueStatus = await getJobStatus(jobID);
   if (!jobQueueStatus) {
     return res.json("We could not find the job you're looking for. Please contact a professor or admin.");
-  } else if (typeof jobQueueStatus === "string") {
-    return res.json(jobQueueStatus);
   }
   const { reservation, queuePosition } = jobQueueStatus;
   if (reservationWaitingOnRelease(reservation.releaseAt)) {

orchestrator/packages/api/src/controllers/holding-pen-controller.ts

@@ -0,0 +1,15 @@
+import { Request, Response } from "express";
+import { errorResponse } from "./utils";
+import { jobConfigInHoldingPen } from "@codegrade-orca/db";
+
+export const holdingPenStatus = async (req: Request, res: Response) => {
+  const jobConfigID = parseInt(req.params.jobConfigID);
+  if (isNaN(jobConfigID)) {
+    return errorResponse(res, 400, [`Given job config ID ${jobConfigID} is not a number.`]);
+  }
+  if (await jobConfigInHoldingPen(jobConfigID)) {
+    res.json("This job is waiting of a grader image to build.");
+  } else {
+    res.json("This job could not be found in a holding pen. Please contact an admin or professor.");
+  }
+}

orchestrator/packages/api/src/index.ts

@@ -5,6 +5,7 @@ import { getConfig } from "@codegrade-orca/common";
 import express = require("express");
 import cors = require("cors");
 import dockerImagesRouter from "./routes/docker-images";
+import holdingPenRouter from "./routes/holding-pen";
 
 const CONFIG = getConfig();
 
@@ -14,7 +15,7 @@ const PORT = process.env.PORT || 4000;
 app.use(cors());
 app.use(express.json());
 
-app.use("/api/v1", gradingQueueRouter, dockerImagesRouter);
+app.use("/api/v1", gradingQueueRouter, dockerImagesRouter, holdingPenRouter);
 app.use("/status", (_req, res) => res.json({"message": "ok"}));
 app.use("/images", express.static(CONFIG.dockerImageFolder));
 

orchestrator/packages/api/src/routes/holding-pen.ts

@@ -0,0 +1,8 @@
+import { Router } from "express";
+import { holdingPenStatus } from "../controllers/holding-pen-controller";
+
+const holdingPenRouter = Router();
+
+holdingPenRouter.get("/holding_pen/:jobConfigID/status", holdingPenStatus);
+
+export default holdingPenRouter;

orchestrator/packages/db/src/holding-pen-operations/index.ts

@@ -0,0 +1,4 @@
+import prismaInstance from '../prisma-instance';
+
+export const jobConfigInHoldingPen = async (jobConfigID: number): Promise<boolean> =>
+  Boolean(await prismaInstance.jobConfigAwaitingImage.count({ where: { id: jobConfigID } }));

orchestrator/packages/db/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./exceptions";
 export * from "./server-operations";
 export * from "./image-builder-operations";
+export * from "./holding-pen-operations";
 export * from "./shared";

orchestrator/packages/db/src/server-operations/job-queue-status.ts

@@ -8,17 +8,14 @@ export interface JobQueueStatus {
   queuePosition: number
 }
 
-const getJobStatus = async (jobID: number): Promise<JobQueueStatus | string | null> =>
+const getJobStatus = async (jobID: number): Promise<JobQueueStatus | null> =>
   prismaInstance.$transaction(async (tx) => {
     const job = await tx.job.findFirst({
       where: { id: jobID },
       include: { reservation: true }
     });
     if (!job) {
-      const jobInHoldingPen = Boolean(await tx.jobConfigAwaitingImage.count(
-        { where: { id: jobID } }
-      ));
-      return jobInHoldingPen ? "This job is waiting on its grader image to be built." : null;
+      return null;
     }
     const reservation: Reservation = job.reservation || await getAssociatedReservation(job, tx);
     return {

worker/orca_grader/common/services/push_results.py

@@ -2,7 +2,7 @@
 import time
 import requests
 from requests import HTTPError
-from typing import Dict, Optional
+from typing import Dict
 from orca_grader.common.grading_job.grading_job_result import GradingJobResult
 from orca_grader.common.services.exceptions import PushResultsFailureException
 from orca_grader.common.types.grading_job_json_types import GradingJobJSON
@@ -37,9 +37,13 @@ def _send_results_with_exponential_backoff(payload: dict, response_url: str, n:
         res.raise_for_status()
         return res
     except HTTPError as e:
-        print(e)
-        if n == _MAX_RETRIES:
-            # TODO: Should we re-enqueue job on raising this exception?
+        if not _should_retry(e.response.status_code) or n == _MAX_RETRIES:
+            print(e)
+            print(f"{e.request.body}, {e.request.headers}")
             raise PushResultsFailureException
         time.sleep(2**n + (random.randint(0, 1000) / 1000.0))
         return _send_results_with_exponential_backoff(payload, response_url, n + 1)
+
+
+def _should_retry(status_code: int) -> bool:
+    return status_code in [503, 504]

worker/orca_grader/common/services/push_status.py

@@ -8,6 +8,12 @@ def post_job_status_to_client(location: str, response_url: str,
         status = {"location": location}
         if id is not None:
             status["id"] = id
-        requests.post(response_url, json=status)
+        requests.post(
+            response_url,
+            json={"key": key, "status": status}
+        ).raise_for_status()
+    except requests.HTTPError as http_e:
+        print(http_e)
+        print(f"{http_e.request.body}, {http_e.request.headers}")
     except Exception as e:
         print(e)