Merge branch 'main' into feat/rabbit-mq-missing-exception

.env-dist

@@ -65,9 +65,6 @@ BYTES_DB_URI=postgresql://${BYTES_DB_USER}:${BYTES_DB_PASSWORD}@postgres:5432/${
 # --- Octopoes --- #
 # See `octopoes/octopoes/config/settings.py`
 
-# Number of Celery workers (for the Octopoes API worker) that need to be started
-CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-4}
-
 # --- Mula --- #
 # See `mula/scheduler/config/settings.py`
 

.env-prod

@@ -21,7 +21,7 @@ SCHEDULER_API=http://scheduler:8000
 KEIKO_API=http://keiko:8000
 KATALOGUS_API=http://katalogus:8000
 XTDB_URI=http://crux:3000
-BOEFJE_API=http://boefje:8000
+BOEFJES_API=http://boefje:8000
 
 # Bytes uses JWT for authentication
 BYTES_API=http://bytes:8000

.github/workflows/pre_commit_checks.yml

@@ -24,7 +24,7 @@ jobs:
           cache: pip
 
       - name: Install pre-commit
-        run: pip install pre-commit==3.8.0
+        run: pip install pre-commit==4.0.1
 
       - uses: actions/cache@v4
         with:

.gitignore

@@ -449,6 +449,8 @@ docs/source/_static/mermaid.min.js
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json
 /boefjes/boefjes/plugins/kat_rpki/rpki-meta.json
+/boefjes/boefjes/plugins/kat_rpki/bgp.jsonl
+/boefjes/boefjes/plugins/kat_rpki/bgp-meta.json
 
 *.pstat
 **/.cache*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: mixed-line-ending
@@ -29,13 +29,13 @@ repos:
         args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"]
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.16
+    rev: v0.23
     hooks:
       - id: validate-pyproject
         files: pyproject.toml$
 
   - repo: https://github.com/rstcheck/rstcheck
-    rev: v6.2.1
+    rev: v6.2.4
     hooks:
       - id: rstcheck
         # https://github.com/rstcheck/rstcheck-core/issues/4
@@ -49,37 +49,37 @@ repos:
         additional_dependencies: ["rstcheck[sphinx]", "autodoc-pydantic==2.1.0"]
 
   - repo: https://github.com/MarketSquare/robotframework-tidy
-    rev: "4.11.0"
+    rev: "4.14.0"
     hooks:
       - id: robotidy
 
   - repo: https://github.com/jendrikseipp/vulture
-    rev: v2.11
+    rev: v2.13
     hooks:
       - id: vulture
         exclude: |
           /tests/
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.3.5"
+    rev: "v0.8.1"
     hooks:
       - id: ruff
       - id: ruff-format
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
 
   - repo: https://github.com/adamchainz/django-upgrade
-    rev: 1.16.0
+    rev: 1.22.1
     hooks:
       - id: django-upgrade
         args: [--target-version, "5.0"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.13.0
     hooks:
       - id: mypy
         additional_dependencies:
@@ -92,6 +92,7 @@ repos:
           - types-python-dateutil
           - types-requests
           - types-croniter
+          - boto3-stubs[s3]
         exclude: |
           (?x)(
           ^boefjes/tools |
@@ -106,7 +107,7 @@ repos:
           )
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -136,7 +137,7 @@ repos:
           )
 
   - repo: https://github.com/Riverside-Healthcare/djLint
-    rev: v1.34.1
+    rev: v1.36.3
     hooks:
       - id: djlint-reformat-django
         files: |
@@ -155,7 +156,7 @@ repos:
         exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
   - repo: https://github.com/thibaudcolas/pre-commit-stylelint
-    rev: v16.3.1
+    rev: v16.10.0
     hooks:
       - id: stylelint
         args: [--fix]
@@ -171,13 +172,13 @@ repos:
         args: ["-e", "SC1091"]
 
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.8.0-1
+    rev: v3.10.0-1
     hooks:
       - id: shfmt
         args: ["-w", "-s", "-i", "4", "-sr"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
+    rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
         additional_dependencies:

Makefile

@@ -109,6 +109,8 @@ debian12-build-image:
 ubuntu22.04-build-image:
 	docker build -t kat-ubuntu22.04-build-image packaging/ubuntu22.04
 
+CHECKSUM_CMD = $(if $(filter $(UNAME), Darwin), shasum -a 256, sha256sum --quiet)
+
 docs:
 	$(call build-settings-doc,keiko,keiko.settings,keiko,Keiko)
 	$(call build-settings-doc,octopoes,octopoes.config.settings,octopoes,Octopoes)
@@ -118,8 +120,9 @@ docs:
 
 	curl -sL -o - https://registry.npmjs.org/d3/-/d3-7.9.0.tgz | tar -Oxzf - package/dist/d3.min.js > docs/source/_static/d3.min.js
 	curl -sL -o - https://registry.npmjs.org/mermaid/-/mermaid-11.3.0.tgz | tar -Oxzf - package/dist/mermaid.min.js > docs/source/_static/mermaid.min.js
-	echo "f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539  docs/source/_static/d3.min.js" | sha256sum --quiet --check || exit 1
-	echo "0d2b6f2361e7e0ce466a6ed458e03daa5584b42ef6926c3beb62eb64670ca261  docs/source/_static/mermaid.min.js" | sha256sum --quiet --check || exit 1
+
+	echo "f2094bbf6141b359722c4fe454eb6c4b0f0e42cc10cc7af921fc158fceb86539  docs/source/_static/d3.min.js" | $(CHECKSUM_CMD) --check || exit 1
+	echo "0d2b6f2361e7e0ce466a6ed458e03daa5584b42ef6926c3beb62eb64670ca261  docs/source/_static/mermaid.min.js" | $(CHECKSUM_CMD) --check || exit 1
 
 	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html --fail-on-warning docs/source docs/_build
 

README.rst

@@ -79,4 +79,4 @@ An OpenKAT installation requires user accounts for users to be able to log in. T
 Security
 ========
 
-OpenKAT is designed to be secure by default in its production setup. In the development setup some debugging flags are enabled by default and it will not include TLS out of the box. To set up a secure production OpenKAT install, please follow the `Production setup guidelines <https://docs.openkat.nl/installation_and_deployment/install.html#production-environments>`_ and `Hardening guidelines <https://docs.openkat.nl/installation_and_deployment/hardening.html>`_.
+OpenKAT is designed to be secure by default in its production setup. In the development setup some debugging flags are enabled by default and it will not include TLS out of the box. To set up a secure production OpenKAT install, please follow the `Production setup guidelines <https://docs.openkat.nl/installation-and-deployment/install.html#production-environments>`_ and `Hardening guidelines <https://docs.openkat.nl/installation-and-deployment/hardening.html>`_.

boefjes/.ci/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       dockerfile: boefjes/Dockerfile
       args:
         - ENVIRONMENT=dev
-    command: bash -c "python -m cProfile -o .ci/bench_$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -v -m slow tests/integration"
+    command: bash -c "python -m cProfile -o .ci/bench_$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -v -m slow tests/integration/test_bench.py::test_migration"
     depends_on:
       - ci_bytes
       - ci_octopoes

boefjes/Makefile

@@ -40,6 +40,7 @@ images:  # Build the images for the containerized boefjes
 	# docker build -f images/base.Dockerfile -t ghcr.io/minvws/openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
 	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t ghcr.io/minvws/openkat/dns-sec:latest .
 	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t ghcr.io/minvws/openkat/nmap:latest  .
+	docker build -f ./boefjes/plugins/kat_export_http/boefje.Dockerfile -t ghcr.io/minvws/openkat/export-http:latest .
 
 
 ##
@@ -75,14 +76,21 @@ itest: ## Run the integration tests.
 	$(ci-docker-compose) build
 	$(ci-docker-compose) down --remove-orphans
 	$(ci-docker-compose) run --rm katalogus_integration
-	$(ci-docker-compose) down
+	$(ci-docker-compose) stop
 
-bench:  ## Run the report benchmark.
+migration_bench:  ## Run the migration benchmark.
 	$(ci-docker-compose) build
 	$(ci-docker-compose) down --remove-orphans
 	$(ci-docker-compose) run --rm migration_bench
 	$(ci-docker-compose) stop
 
+bench:  ## Run the other benchmarks
+	$(ci-docker-compose) build
+	$(ci-docker-compose) down --remove-orphans
+	$(ci-docker-compose) run --rm katalogus_integration \
+	python -m cProfile -o .ci/bench_$$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -m slow --no-cov tests/integration
+	$(ci-docker-compose) stop
+
 debian12:
 	docker run --rm \
 	--env PKG_NAME=kat-boefjes \

boefjes/boefjes/__main__.py

@@ -37,7 +37,7 @@
 @click.command()
 @click.argument("worker_type", type=click.Choice([q.value for q in WorkerManager.Queue]))
 @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]), help="Log level", default="INFO")
-def cli(worker_type: str, log_level: str):
+def cli(worker_type: str, log_level: str) -> None:
     logger.setLevel(log_level)
     logger.info("Starting runtime for %s", worker_type)
 

boefjes/boefjes/api.py

@@ -33,7 +33,7 @@ def __init__(self, config: Config):
         self.server = Server(config=config)
         self.config = config
 
-    def stop(self):
+    def stop(self) -> None:
         self.terminate()
 
     def run(self, *args, **kwargs):
@@ -88,7 +88,7 @@ def boefje_input(
     task_id: UUID,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     plugin_service: PluginService = Depends(get_plugin_service),
-):
+) -> BoefjeInput:
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
@@ -108,7 +108,7 @@ def boefje_output(
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     bytes_client: BytesAPIClient = Depends(get_bytes_client),
     plugin_service: PluginService = Depends(get_plugin_service),
-):
+) -> Response:
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
@@ -127,7 +127,7 @@ def boefje_output(
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes
-            bytes_client.save_raw(task_id, raw, mime_types.union(file.tags))
+            bytes_client.save_raw(task_id, raw, mime_types.union(file.tags) if file.tags else mime_types)
 
     if boefje_output.status == StatusEnum.COMPLETED:
         scheduler_client.patch_task(task_id, TaskStatus.COMPLETED)

boefjes/boefjes/app.py

@@ -80,7 +80,7 @@ def run(self, queue_type: WorkerManager.Queue) -> None:
 
                 raise
 
-    def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
+    def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue) -> None:
         if task_queue.qsize() > self.settings.pool_size:
             time.sleep(self.settings.worker_heartbeat)
             return
@@ -95,7 +95,7 @@ def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
 
         # We do not target a specific queue since we start one runtime for all organisations
         # and queue ids contain the organisation_id
-        queues = [q for q in queues if q.id.startswith(queue_type.value)]
+        queues = [q for q in queues if q.id.startswith(queue_type.value) and q.size > 0]
 
         logger.debug("Found queues: %s", [queue.id for queue in queues])
 
@@ -189,7 +189,7 @@ def _cleanup_pending_worker_task(self, worker: BaseProcess) -> None:
     def _worker_args(self) -> tuple:
         return self.task_queue, self.item_handler, self.scheduler_client, self.handling_tasks
 
-    def exit(self, signum: int | None = None):
+    def exit(self, signum: int | None = None) -> None:
         try:
             if signum:
                 logger.info("Received %s, exiting", signal.Signals(signum).name)
@@ -238,7 +238,7 @@ def _start_working(
     handler: Handler,
     scheduler_client: SchedulerClientInterface,
     handling_tasks: dict[int, str],
-):
+) -> None:
     logger.info("Started listening for tasks from worker[pid=%s]", os.getpid())
 
     while True:

boefjes/boefjes/clients/bytes_client.py

@@ -9,6 +9,7 @@
 import structlog
 from httpx import Client, HTTPStatusError, HTTPTransport, Response
 
+from boefjes.config import settings
 from boefjes.job_models import BoefjeMeta, NormalizerMeta, RawDataMeta
 
 BYTES_API_CLIENT_VERSION = "0.3"
@@ -38,6 +39,7 @@ def __init__(self, base_url: str, username: str, password: str):
             base_url=base_url,
             headers={"User-Agent": f"bytes-api-client/{BYTES_API_CLIENT_VERSION}"},
             transport=(HTTPTransport(retries=6)),
+            timeout=settings.outgoing_request_timeout,
         )
 
         self.credentials = {"username": username, "password": password}

boefjes/boefjes/clients/scheduler_client.py

@@ -5,6 +5,7 @@
 from httpx import Client, HTTPTransport, Response
 from pydantic import BaseModel, TypeAdapter
 
+from boefjes.config import settings
 from boefjes.job_models import BoefjeMeta, NormalizerMeta
 
 
@@ -57,7 +58,9 @@ def push_item(self, p_item: Task) -> None:
 
 class SchedulerAPIClient(SchedulerClientInterface):
     def __init__(self, base_url: str):
-        self._session = Client(base_url=base_url, transport=HTTPTransport(retries=6))
+        self._session = Client(
+            base_url=base_url, transport=HTTPTransport(retries=6), timeout=settings.outgoing_request_timeout
+        )
 
     @staticmethod
     def _verify_response(response: Response) -> None:

boefjes/boefjes/config.py

@@ -122,6 +122,8 @@ class Settings(BaseSettings):
 
     logging_format: Literal["text", "json"] = Field("text", description="Logging format")
 
+    outgoing_request_timeout: int = Field(30, description="Timeout for outgoing HTTP requests")
+
     model_config = SettingsConfigDict(env_prefix="BOEFJES_")
 
     @classmethod