Merge pull request #51 from JulianFP/dev

Introduce separate route to retrieve audio as binary to get rid of base64, expose IDs to runner

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: "3.8"
+        python-version: "3.12"
 
     - name: Install Python package
       run: |

doc/installation.rst

@@ -585,7 +585,7 @@ You can also run Project-W barebones. This can be a bit more difficult and the f
 Backend
 ```````
 
-1. Install Python (3.8 - 3.11 are tested to work) and pip
+1. Install Python (3.8 or newer, we have tested 3.8 to 3.12) and pip
 2. Clone this repository and enter it:
 
    .. code-block:: console
@@ -645,7 +645,7 @@ The frontend is written in Svelte and needs to be compiled into native Javascrip
 Runner
 ``````
 
-1. Install Python (3.9 or newer), pip, and ffmpeg.
+1. Install Python (3.11 or newer, we have tested 3.11 to 3.12), pip, and ffmpeg.
 2. Clone this repository and enter it:
 
    .. code-block:: bash

nix/overlay.nix

@@ -1,7 +1,6 @@
 (final: prev: {
   python3 = prev.python3.override {
     packageOverrides = pyfinal: pyprev: {
-      sphinx-mdinclude = prev.callPackage ./pkgs/sphinx-mdinclude.nix { };
       sphinx-jsonschema = prev.callPackage ./pkgs/sphinx-jsonschema.nix { };
       project-W = prev.callPackage ./pkgs/project-W.nix { };
     };

project_W/app.py

@@ -1,10 +1,9 @@
-import base64
 import datetime
 import secrets
 from pathlib import Path
 from typing import Optional
 
-from flask import Flask, jsonify, request
+from flask import Flask, jsonify, make_response, request
 from flask_cors import CORS
 from flask_jwt_extended import (
     JWTManager,
@@ -962,6 +961,7 @@ def registerRunner():
 
         :reqheader Authorization: Has to be string "Bearer {Token}", where {Token} is the Runner Token that the /runners/create route returned.
         :resjson string msg: Human-readable success message.
+        :resjson integer runnerID: ID under which the runner is registered at the backend
         :resjson string error: Human-readable error message that tells you why the request failed.
 
         :status 200: Runner successfully registered.
@@ -975,7 +975,10 @@ def registerRunner():
             return jsonify(error="No runner with that token exists!"), 400
         if runner_manager.register_runner(runner):
             logger.info(f"Runner {runner.id} successfully registered!")
-            return jsonify(msg="Runner successfully registered!")
+            return jsonify(
+                msg="Runner successfully registered!",
+                runnerID=runner.id,
+            )
         return jsonify(error="Runner is already registered!"), 400
 
     @app.post("/api/runners/unregister")
@@ -1003,10 +1006,38 @@ def unregisterRunner():
             return jsonify(msg="Runner successfully unregistered!")
         return jsonify(error="Runner is not registered!"), 400
 
-    @app.post("/api/runners/retrieveJob")
-    def retrieveJob():
+    @app.get("/api/runners/retrieveJobAudio")
+    def retrieveJobAudio():
+        """
+        The runner retrieves the binary audio data of its job after it learned from the heartbeat response that the Backend assigned a job to it. Use /api/runners/retrieveJobInfo to get all the other data of the job. The following conditions have to be met for this to succeed:
+
+        - Correct and valid Runner Token (that is known to the Backend) has to be supplied in Authorization header.
+        - The runner has to currently be registered. All runners will be unregistered automatically after 60-70 seconds since the last heartbeat.
+        - There has to be a job assigned to the runner associated with the given Runner Token.
+
+        .. :quickref: Runners; Runner retrieves the job that got assigned to it.
+
+        :reqheader Authorization: Has to be string "Bearer {Token}", where {Token} is the Runner Token that the /runners/create route returned.
+        :resjson string error: Human-readable error message that tells you why the request failed.
+        if successful then returns just the binary audio data. In this case the Content-Type is audio/basic instaed of application/json.
+
+        :status 200: Returning assigned job.
+        :status 400: Failed. Refer to ``error`` field for the reason.
+        """
+        online_runner, error = runner_manager.get_online_runner_for_req(request)
+        if error:
+            return jsonify(error=error), 400
+        job = runner_manager.retrieve_job(online_runner)
+        if not job:
+            return jsonify(error="No job available!"), 400
+        response = make_response(job.file.audio_data)
+        response.headers.set("Content-Type", "audio/basic")
+        return response
+
+    @app.post("/api/runners/retrieveJobInfo")
+    def retrieveJobInfo():
         """
-        The runner retrieves its job after it learned from the heartbeat response that the Backend assigned a job to it. The following conditions have to be met for this to succeed:
+        The runner retrieves the infos about its job after it learned from the heartbeat response that the Backend assigned a job to it. Use /api/runners/retrieveJobAudio to get the audio data. The following conditions have to be met for this to succeed:
 
         - Correct and valid Runner Token (that is known to the Backend) has to be supplied in Authorization header.
         - The runner has to currently be registered. All runners will be unregistered automatically after 60-70 seconds since the last heartbeat.
@@ -1016,7 +1047,7 @@ def retrieveJob():
 
         :reqheader Authorization: Has to be string "Bearer {Token}", where {Token} is the Runner Token that the /runners/create route returned.
         :resjson string error: Human-readable error message that tells you why the request failed.
-        :resjson string audio: Content of audio-file encoded into base64.
+        :resjson integer jobID: ID of the assigned job
         :resjson string model: Whisper model that should be used for the job. May be null.
         :resjson string language: Language of the audio. May be null.
 
@@ -1030,7 +1061,7 @@ def retrieveJob():
         if not job:
             return jsonify(error="No job available!"), 400
         return jsonify(
-            audio=base64.b64encode(job.file.audio_data).decode("ascii"),
+            jobID=job.id,
             model=job.model,
             language=job.language,
         )

project_W/runner_manager.py

@@ -16,7 +16,6 @@
 # every 15 seconds, and if they don't send a heartbeat
 # for 60 seconds they may be automatically unregistered.
 # TODO: Should we make these configurable?
-DEFAULT_HEARTBEAT_INTERVAL = 15
 DEFAULT_HEARTBEAT_TIMEOUT = 60