From 2a11420bc6548ba62bd1c7f1fc8cde9a8616be34 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 14 Nov 2023 10:58:29 +0100
Subject: [PATCH] Prevent cancelled invocation resetting back to new or ready

---
 lib/galaxy/model/__init__.py              | 17 +++++++++++++++++
 lib/galaxy/workflow/run.py                |  4 ++--
 lib/galaxy/workflow/scheduling_manager.py |  2 +-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 3c19fe0384a6..c437cb1ee65a 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -8216,6 +8216,23 @@ def active(self):
         states = WorkflowInvocation.states
         return self.state in [states.NEW, states.READY]
 
+    def set_state(self, state: "WorkflowInvocation.states"):
+        session = object_session(self)
+        priority_states = (WorkflowInvocation.states.CANCELLING, WorkflowInvocation.states.CANCELLED)
+        if session and self.id and state not in priority_states:
+            # generate statement that will not revert CANCELLING or CANCELLED back to anything non-terminal
+            session.execute(
+                update(WorkflowInvocation.table)
+                .where(
+                    WorkflowInvocation.id == self.id,
+                    or_(~WorkflowInvocation.state.in_(priority_states), WorkflowInvocation.state.is_(None)),
+                )
+                .values(state=state)
+            )
+        else:
+            # Not bound to a session, or setting cancelling/cancelled
+            self.state = state
+
     def cancel(self):
         if self.state not in [WorkflowInvocation.states.CANCELLING, WorkflowInvocation.states.CANCELLED]:
             # No use cancelling workflow again, for all others we may still want to be able to cancel
diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index 59d19a5c079a..776b4e525803 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -193,7 +193,7 @@ def invoke(self) -> Dict[int, Any]:
             log.debug(
                 f"Workflow invocation [{workflow_invocation.id}] exceeded maximum number of seconds allowed for scheduling [{maximum_duration}], failing."
             )
-            workflow_invocation.state = model.WorkflowInvocation.states.FAILED
+            workflow_invocation.set_state(model.WorkflowInvocation.states.FAILED)
             # All jobs ran successfully, so we can save now
             self.trans.sa_session.add(workflow_invocation)
 
@@ -264,7 +264,7 @@ def invoke(self) -> Dict[int, Any]:
             state = model.WorkflowInvocation.states.READY
         else:
             state = model.WorkflowInvocation.states.SCHEDULED
-        workflow_invocation.state = state
+        workflow_invocation.set_state(state)
 
         # All jobs ran successfully, so we can save now
         self.trans.sa_session.add(workflow_invocation)
diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py
index 31f4bca49ae1..3868e24c13a9 100644
--- a/lib/galaxy/workflow/scheduling_manager.py
+++ b/lib/galaxy/workflow/scheduling_manager.py
@@ -155,7 +155,7 @@ def shutdown(self):
             raise exception
 
     def queue(self, workflow_invocation, request_params, flush=True):
-        workflow_invocation.state = model.WorkflowInvocation.states.NEW
+        workflow_invocation.set_state(model.WorkflowInvocation.states.NEW)
         workflow_invocation.scheduler = request_params.get("scheduler", None) or self.default_scheduler_id
         sa_session = self.app.model.context
         sa_session.add(workflow_invocation)