fix: optionally use log dir path; create if not exist

fix: longer sleep period when no job is enqueued

orchestrator/packages/common/src/config.ts

@@ -5,6 +5,7 @@ export interface OrchestratorConfig {
   dockerImageFolder: string;
   postgresURL: string;
   environment: string;
+  orchestratorLogsDir?: string;
 }
 
 interface OrchestratorAPIOptions {
@@ -17,5 +18,6 @@ export const getConfig = (): OrchestratorConfig => ({
     port: process.env.API_PORT ? parseInt(process.env.API_PORT) : 8090,
   },
   dockerImageFolder: join(__dirname, "../../../", "images"),
-  environment: process.env.ENVIRONMENT?.toLowerCase() || 'dev'
+  environment: process.env.ENVIRONMENT?.toLowerCase() || 'dev',
+  orchestratorLogsDir: process.env.ORCHESTRATOR_LOG_DIR
 });

orchestrator/packages/common/src/logger.ts

@@ -1,26 +1,35 @@
 import pino from "pino";
 import { getConfig } from "./config";
 import path from "path";
+import { existsSync, mkdirSync } from "fs";
+
+const _CONFIG = getConfig();
+
+const pinoTarget = {
+  target: 'pino/file',
+  options: {
+    destination: _CONFIG.orchestratorLogsDir ?
+      path.join(_CONFIG.orchestratorLogsDir, `${_CONFIG.environment}.log`) :
+      1 // used by pino for STDOUT
+  }
+}
 
 const transport = pino.transport({
   targets: [
-    {
-      target: 'pino/file',
-      options: {
-        destination: path.join(__dirname, '../../../', 'logs', `${getConfig().environment}.log`),
-        mkdir: true
-      }
-    },
-    {
-      target: 'pino/file',
-      options: { destination: 2 }
-    }
+    pinoTarget
   ]
 });
 
-const logger = pino({
-  level: getConfig().environment === 'production' ? 'info' : 'debug',
-  timestamp: true,
-}, transport);
+const getLogger = () => {
+  if (_CONFIG.orchestratorLogsDir && !existsSync(_CONFIG.orchestratorLogsDir)) {
+    mkdirSync(_CONFIG.orchestratorLogsDir);
+  }
+  return pino({
+    level: getConfig().environment === 'production' ? 'info' : 'debug',
+    timestamp: true,
+  }, transport)
+}
+
+const logger = getLogger();
 
 export default logger;

worker/orca_grader/__main__.py

@@ -28,6 +28,7 @@
 
 CONTAINER_WORKING_DIR = '/home/orca-grader'
 _LOGGER = logging.getLogger(__name__)
+_SLEEP_LENGTH = 5  # seconds
 
 
 def run_local_job(job_path: str, no_container: bool,
@@ -65,11 +66,12 @@ def process_jobs_from_db(no_container: bool,
                 if job_retrieval_future in done:
                     if job_retrieval_future.exception():
                         _LOGGER.exception("Failed to retrieve grading job from postgres.")
-                        time.sleep(1)
+                        time.sleep(_SLEEP_LENGTH)
                         continue
                     grading_job = job_retrieval_future.result()
                     if grading_job is None:
-                        time.sleep(1)
+                        _LOGGER.debug(f"No jobs on the queue; sleeping for {_SLEEP_LENGTH} seconds.")
+                        time.sleep(_SLEEP_LENGTH)
                         continue
 
                 if stop_future in done: