From 5c5645261e7c8a564b2d58ef2b584826f5aa8db6 Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Tue, 31 Dec 2024 10:17:58 +0100 Subject: [PATCH 1/4] catch the schema mismatch error and produce an error raw file --- boefjes/boefjes/job_handler.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index 3de61485a30..df54f98a0e5 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -1,3 +1,4 @@ +import json import os import traceback from collections.abc import Callable @@ -131,15 +132,25 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() + boefje_results: list[tuple[set, bytes | str]] = [] + boefje_meta.runnable_hash = plugin.runnable_hash - boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) + try: + boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) + except SettingsNotConformingToSchema: + logger.exception( + "Error running boefje due to settings/schema mismatch %s[%s]", + boefje_meta.boefje.id, + str(boefje_meta.id), + ) + boefje_results = [({"error/boefje"}, json.dumps(plugin.boefje_schema))] + + raise logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) boefje_meta.started_at = datetime.now(timezone.utc) - boefje_results: list[tuple[set, bytes | str]] = [] - try: boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment) except: From 3c763390cad86c2665fc914fadde45d9a34d72f4 Mon Sep 17 00:00:00 2001 From: Ammar Date: Wed, 8 Jan 2025 13:02:27 +0100 Subject: [PATCH 2/4] Fixed --- boefjes/boefjes/job_handler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index df54f98a0e5..d7d00c65d1a 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -135,8 +135,16 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_results: list[tuple[set, bytes | str]] = [] boefje_meta.runnable_hash = plugin.runnable_hash + + logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) + + boefje_meta.started_at = datetime.now(timezone.utc) + try: boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) + + boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment) + except SettingsNotConformingToSchema: logger.exception( "Error running boefje due to settings/schema mismatch %s[%s]", @@ -147,17 +155,12 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: raise - logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) - - boefje_meta.started_at = datetime.now(timezone.utc) - - try: - boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment) except: logger.exception("Error running boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) boefje_results = [({"error/boefje"}, traceback.format_exc())] raise + finally: boefje_meta.ended_at = datetime.now(timezone.utc) logger.info("Saving to Bytes for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) From b7dc47abf07647cd3adc02ea99ccafd03918e7ac Mon Sep 17 00:00:00 2001 From: Ammar Date: Wed, 8 Jan 2025 14:49:25 +0100 Subject: [PATCH 3/4] Suggestion --- boefjes/boefjes/job_handler.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index d7d00c65d1a..b1d7b6631b8 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -133,16 +133,14 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() boefje_results: list[tuple[set, bytes | str]] = [] - boefje_meta.runnable_hash = plugin.runnable_hash - logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) - - boefje_meta.started_at = datetime.now(timezone.utc) - try: + logger.info("Getting environment settings for boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) + logger.info("Starting boefje %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id)) + boefje_meta.started_at = datetime.now(timezone.utc) boefje_results = self.job_runner.run(boefje_meta, boefje_meta.environment) except SettingsNotConformingToSchema: From 44455e5f60a4448476049d8e98fea1abd7de5c8e Mon Sep 17 00:00:00 2001 From: Jan Klopper Date: Mon, 13 Jan 2025 12:09:54 +0100 Subject: [PATCH 4/4] Update job_handler.py, add error message to error output file --- boefjes/boefjes/job_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index b1d7b6631b8..8d65429cf89 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -149,7 +149,9 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.boefje.id, str(boefje_meta.id), ) - boefje_results = [({"error/boefje"}, json.dumps(plugin.boefje_schema))] + boefje_results = [({"error/boefje"}, "Error running boefje (%s) due to settings/schema mismatch: %s" % ( + boefje_meta.id, json.dumps(plugin.boefje_schema) + ))] raise