Harden logic around job output saving

Do not attempt to get job output if the helper sidecar is not known to
be running. This will reduce many spurious errors when trying to get
output on of a job when it's not ready or running.

Also don't ignore errors for job shutdown.

Signed-off-by: Darren Shepherd <[email protected]>

pkg/controller/jobs/output.go

@@ -135,13 +135,18 @@ func (h *Handler) SaveJobOutput(req router.Request, resp router.Response) error
 		names.Insert(name)
 	}
 
+	helperRunning := false
 	for _, status := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) {
+		if status.Name == jobs.Helper && status.State.Running != nil {
+			helperRunning = true
+		}
+		// TODO: check that helper is really running, otherwise early exit
 		if status.State.Terminated != nil {
 			names.Delete(status.Name)
 		}
 	}
 
-	if names.Len() > 0 {
+	if names.Len() > 0 || !helperRunning {
 		return nil
 	}
 
@@ -170,7 +175,6 @@ func (h *Handler) SaveJobOutput(req router.Request, resp router.Response) error
 		return err
 	}
 
-	// ignore error, it always exits with exit code 137
-	_, _ = h.runCommand(req.Ctx, pod, "/usr/local/bin/acorn-job-helper-shutdown")
-	return nil
+	_, err = h.runCommand(req.Ctx, pod, "/usr/local/bin/acorn-job-helper-shutdown")
+	return err
 }

scripts/acorn-job-helper-init

@@ -1,3 +1,4 @@
 #!/bin/sh
 mkfifo /tmp/.fifo
 echo > /tmp/.fifo
+sleep .2