From 0f09dfd751ecc2e9c0e29bc170846c57912db5d9 Mon Sep 17 00:00:00 2001
From: duhow <contacto@duhowpi.net>
Date: Tue, 23 Apr 2024 12:10:40 +0200
Subject: [PATCH] feat: monitor queued jobs

---
 setup.cfg  |  3 ++-
 src/app.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 4ec4b28..b2dfb7a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = github-workflows-monitoring
-version = 0.1.3
+version = 0.2.0
 license-file = LICENSE
 
 [options]
@@ -8,6 +8,7 @@ python_requires = >=3.8
 packages = find:
 install_requires =
         Flask>=2.2,<3
+        Flask-APScheduler==1.13.1
 
 [flake8]
 max-line-length = 120
diff --git a/src/app.py b/src/app.py
index 153603f..86bc2df 100644
--- a/src/app.py
+++ b/src/app.py
@@ -4,6 +4,7 @@
 import os
 
 from flask import Flask, abort, request
+from flask_apscheduler import APScheduler
 
 
 from const import GithubHeaders, LOGGING_CONFIG
@@ -12,6 +13,8 @@
 dictConfig(LOGGING_CONFIG)
 
 app = Flask(__name__)
+scheduler = APScheduler()
+scheduler.init_app(app)
 
 # set to WARNING to disable access log
 log = logging.getLogger("werkzeug")
@@ -131,7 +134,32 @@ def process_workflow_job():
     return True
 
 
+@scheduler.task('interval', id='monitor_queued', seconds=30)
+def monitor_queued_jobs():
+    """ Return the job that has been queued and not starting for long time. """
+    app.logger.debug("Starting monitor_queued_jobs")
+    if not jobs:
+        return
+
+    old_queued_job = min(jobs.items(), key=lambda x: x[1])
+    job_id, time_start = old_queued_job
+    delay = datetime.now().timestamp() - time_start
+
+    if delay <= int(os.getenv("QUEUED_JOBS_DELAY_THRESHOLD", 150)):
+        return
+
+    context_details = {
+        "action": "monitor_queued",
+        "job_id": job_id,
+        "started_at": time_start,
+        "delay": delay,
+    }
+
+    app.logger.info(dict_to_logfmt(context_details))
+
+
 allowed_events = {"workflow_job": process_workflow_job}
+scheduler.start()
 
 
 @app.route("/github-webhook", methods=["POST"])