build: allow uploading to s3

This allows remote workers and unified storage.

Signed-off-by: Paul Spooren <[email protected]>

asu/api.py

@@ -258,6 +258,10 @@ def api_v1_build_post():
         req["repository_allow_list"] = current_app.config["REPOSITORY_ALLOW_LIST"]
         req["request_hash"] = request_hash
         req["base_container"] = current_app.config["BASE_CONTAINER"]
+        req["s3_bucket"] = current_app.config["S3_BUCKET"]
+        req["s3_access_key"] = current_app.config["S3_ACCESS_KEY"]
+        req["s3_secret_key"] = current_app.config["S3_SECRET_KEY"]
+        req["s3_server"] = current_app.config["S3_SERVER"]
 
         job = get_queue().enqueue(
             build,

asu/asu.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 import connexion
-from flask import Flask, render_template, send_from_directory
+from flask import Flask, redirect, render_template, send_from_directory
 from pkg_resources import resource_filename
 from prometheus_client import CollectorRegistry, make_wsgi_app
 from rq import Queue
@@ -38,6 +38,10 @@ def create_app(test_config: dict = None) -> Flask:
         MAX_CUSTOM_ROOTFS_SIZE_MB=100,
         REPOSITORY_ALLOW_LIST=[],
         BASE_CONTAINER="ghcr.io/openwrt/imagebuilder",
+        S3_BUCKET=None,
+        S3_ACCESS_KEY=None,
+        S3_SECRET_KEY=None,
+        S3_SERVER=None,
     )
 
     if not test_config:
@@ -80,7 +84,12 @@ def json_path(path="index.html"):
     @app.route("/store/")
     @app.route("/store/<path:path>")
     def store_path(path="index.html"):
-        return send_from_directory(app.config["PUBLIC_PATH"] / "public", path)
+        if app.config.get("S3_SERVER"):
+            return redirect(
+                f"{app.config['S3_SERVER']}/{app.config['S3_BUCKET']}/{path}"
+            )
+        else:
+            return send_from_directory(app.config["PUBLIC_PATH"] / "public", path)
 
     from . import janitor
 

asu/build.py

@@ -1,10 +1,13 @@
 import json
 import logging
 import re
+import tempfile
 from datetime import datetime
 from os import getenv
 from pathlib import Path
+from shutil import rmtree
 
+import boto3
 from podman import PodmanClient
 from rq import get_current_job
 
@@ -31,7 +34,13 @@ def build(req: dict, job=None):
     Args:
         request (dict): Contains all properties of requested image
     """
-    store_path = Path(req["public_path"]) / "store"
+    if req["s3_server"]:
+        temp_path = tempfile.TemporaryDirectory()
+        store_path = Path(temp_path.name)
+    else:
+        temp_path = None
+        store_path = Path(req["public_path"]) / "store"
+
     store_path.mkdir(parents=True, exist_ok=True)
     log.debug(f"Store path: {store_path}")
 
@@ -320,6 +329,25 @@ def build(req: dict, job=None):
 
     log.debug("JSON content %s", json_content)
 
+    # Upload to S3
+    s3 = boto3.client(
+        "s3",
+        endpoint_url=req["s3_server"],
+        aws_access_key_id=req["s3_access_key"],
+        aws_secret_access_key=req["s3_secret_key"],
+    )
+    for image in json_content["images"]:
+        print(f"Uploading {image['name']} to S3")
+        s3.upload_file(
+            str(store_path / bin_dir / image["name"]),
+            req["s3_bucket"],
+            f"{req['request_hash']}/{image['name']}",
+        )
+
+    if temp_path:
+        temp_path.cleanup()
+        rmtree(store_path, ignore_errors=True)
+
     # Increment stats
     job.connection.hincrby(
         "stats:builds",

pyproject.toml

@@ -18,6 +18,7 @@ prometheus-client = ">=0.13.1,<0.21.0"
 gunicorn = ">=20.1,<22.0"
 podman = ">=4.4.1,<6.0.0"
 setuptools = "^69.0.3"
+boto3 = "^1.34.75"
 
 [tool.poetry.group.dev.dependencies]
 ruff = ">=0.2.2,<0.4.0"