✨ add deepfence threatmapper (DefectDojo#9688)

* ✨ add deepfence threatmapper

* 🎇 finished

* update

* update deepfence threatmapper

* fix according to review

* fix ruff

* fix ruff

* 🐛 fix

* remove unecessary file

* update sha sum

docs/content/en/integrations/parsers/file/deepfence_threatmapper.md

@@ -0,0 +1,8 @@
+---
+title: "Deepfence Threatmapper"
+toc_hide: true
+---
+Import compliance, malware, secret, vulnerability reports from [Deepfence Threatmapper](https://github.com/deepfence/ThreatMapper) in XLSX file format. 
+
+### Sample Scan Data
+Sample Threatmapper scans can be found [here](https://github.com/DefectDojo/django-DefectDojo/tree/master/unittests/scans/deepfence_threatmapper). In this link are both .xlsx and .csv listed. They contain the same content, but csv can be read in the Browser, but only xlsx is supported by the parser. 

dojo/settings/.settings.dist.py.sha256sum

@@ -1 +1 @@
-ed4d321ce9ae47f9500965e8494a069fb464a9bd4ea3edf994020523f0dea085
+7b3bb14160f3ffce537d75895ee18cb0a561232fa964bae88b4861f7d289b176

dojo/settings/settings.dist.py

@@ -1265,6 +1265,7 @@ def saml2_attrib_map_format(dict):
     'MobSF Scan': ['title', 'description', 'severity'],
     'OSV Scan': ['title', 'description', 'severity'],
     'Snyk Code Scan': ['vuln_id_from_tool', 'file_path'],
+    'Deepfence Threatmapper Report': ['title', 'description', 'severity'],
     'Bearer CLI': ['title', 'severity'],
     'Nancy Scan': ['title', 'vuln_id_from_tool'],
     'Wiz Scan': ['title', 'description', 'severity'],
@@ -1486,6 +1487,7 @@ def saml2_attrib_map_format(dict):
     'Nosey Parker Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE,
     'Bearer CLI': DEDUPE_ALGO_HASH_CODE,
     'Wiz Scan': DEDUPE_ALGO_HASH_CODE,
+    'Deepfence Threatmapper Report': DEDUPE_ALGO_HASH_CODE,
     'Kubescape JSON Importer': DEDUPE_ALGO_HASH_CODE
 }
 

dojo/tools/deepfence_threatmapper/__init__.py

@@ -0,0 +1 @@
+__author__ = "manuel-sommer"

dojo/tools/deepfence_threatmapper/compliance.py

@@ -0,0 +1,54 @@
+from dojo.models import Finding
+
+
+class DeepfenceThreatmapperCompliance:
+    def get_findings(self, row, headers, test):
+        description = ""
+        compliance_check_type = row[headers["compliance_check_type"]]
+        count = row[headers["count"]]
+        doc_id = row[headers["doc_id"]]
+        host_name = row[headers["host_name"]]
+        cloud_account_id = row[headers["cloud_account_id"]]
+        masked = row[headers["masked"]]
+        node_id = row[headers["node_id"]]
+        node_name = row[headers["node_name"]]
+        node_type = row[headers["node_type"]]
+        status = row[headers["status"]]
+        test_category = row[headers["test_category"]]
+        test_desc = row[headers["test_desc"]]
+        test_info = row[headers["test_info"]]
+        test_number = row[headers["test_number"]]
+        description += "**compliance_check_type:** " + str(compliance_check_type) + "\n"
+        description += "**host_name:** " + str(host_name) + "\n"
+        description += "**cloud_account_id:** " + str(cloud_account_id) + "\n"
+        description += "**masked:** " + str(masked) + "\n"
+        description += "**node_id:** " + str(node_id) + "\n"
+        description += "**node_name:** " + str(node_name) + "\n"
+        description += "**node_type:** " + str(node_type) + "\n"
+        description += "**status:** " + str(status) + "\n"
+        description += "**test_category:** " + str(test_category) + "\n"
+        description += "**test_desc:** " + str(test_desc) + "\n"
+        description += "**test_info:** " + str(test_info) + "\n"
+        description += "**test_number:** " + str(test_number) + "\n"
+        description += "**count:** " + str(count) + "\n"
+        description += "**doc_id:** " + str(doc_id) + "\n"
+        finding = Finding(
+            title="Threatmapper_Compliance_Report-" + test_number,
+            description=description,
+            severity=self.compliance_severity(status),
+            static_finding=False,
+            dynamic_finding=True,
+            test=test,
+        )
+        return finding
+
+    def compliance_severity(self, input):
+        if input == "pass":
+            output = "Info"
+        elif input == "info":
+            output = "Info"
+        elif input == "warn":
+            output = "Medium"
+        else:
+            output = "Info"
+        return output

dojo/tools/deepfence_threatmapper/malware.py

@@ -0,0 +1,39 @@
+from dojo.models import Finding
+
+
+class DeepfenceThreatmapperMalware:
+    def get_findings(self, row, headers, test):
+        description = ""
+        Rule_Name = row[headers["Rule Name"]]
+        Class = row[headers["Class"]]
+        File_Name = row[headers["File Name"]]
+        Summary = row[headers["Summary"]]
+        Severity = row[headers["Severity"]]
+        Node_Name = row[headers["Node Name"]]
+        NodeType = row[headers["NodeType"]]
+        Container_Name = row[headers["Container Name"]]
+        Kubernetes_Cluster_Name = row[headers["Kubernetes Cluster Name"]]
+        description += "**Summary:** " + str(Summary) + "\n"
+        description += "**Rule Name:** " + str(Rule_Name) + "\n"
+        description += "**Class:** " + str(Class) + "\n"
+        description += "**File Name:** " + str(File_Name) + "\n"
+        description += "**Node Name:** " + str(Node_Name) + "\n"
+        description += "**NodeType:** " + str(NodeType) + "\n"
+        description += "**Container Name:** " + str(Container_Name) + "\n"
+        description += "**Kubernetes Cluster Name:** " + str(Kubernetes_Cluster_Name) + "\n"
+        finding = Finding(
+            title=Rule_Name,
+            description=description,
+            file_path=File_Name,
+            severity=self.severity(Severity),
+            static_finding=False,
+            dynamic_finding=True,
+            test=test,
+        )
+        return finding
+
+    def severity(self, input):
+        if input is None:
+            return "Info"
+        else:
+            return input.capitalize()

dojo/tools/deepfence_threatmapper/parser.py

@@ -0,0 +1,40 @@
+from openpyxl import load_workbook
+
+from dojo.tools.deepfence_threatmapper.compliance import DeepfenceThreatmapperCompliance
+from dojo.tools.deepfence_threatmapper.malware import DeepfenceThreatmapperMalware
+from dojo.tools.deepfence_threatmapper.secret import DeepfenceThreatmapperSecret
+from dojo.tools.deepfence_threatmapper.vulnerability import DeepfenceThreatmapperVulnerability
+
+
+class DeepfenceThreatmapperParser:
+    def get_scan_types(self):
+        return ["Deepfence Threatmapper Report"]
+
+    def get_label_for_scan_types(self, scan_type):
+        return scan_type
+
+    def get_description_for_scan_types(self, scan_type):
+        return "Deepfence Threatmapper report in XLSX format."
+
+    def get_findings(self, filename, test):
+        workbook = load_workbook(filename)
+        worksheet = workbook.active
+        findings = []
+        headers = {}
+        first = True
+        for row in worksheet.iter_rows(min_row=1, values_only=True):
+            if first:
+                first = False
+                for i in range(len(row)):
+                    headers[row[i]] = i
+            elif headers.get("Rule Name") is not None and headers.get("Class") is not None:
+                findings.append(DeepfenceThreatmapperMalware().get_findings(row, headers, test))
+            elif headers.get("Filename") is not None and headers.get("Content") is not None:
+                value = DeepfenceThreatmapperSecret().get_findings(row, headers, test)
+                if value is not None:
+                    findings.append(value)
+            elif headers.get("@timestamp") is not None and headers.get("cve_attack_vector") is not None:
+                findings.append(DeepfenceThreatmapperVulnerability().get_findings(row, headers, test))
+            elif headers.get("@timestamp") is not None and headers.get("compliance_check_type") is not None:
+                findings.append(DeepfenceThreatmapperCompliance().get_findings(row, headers, test))
+        return findings

dojo/tools/deepfence_threatmapper/secret.py

@@ -0,0 +1,42 @@
+from dojo.models import Finding
+
+
+class DeepfenceThreatmapperSecret:
+    def get_findings(self, row, headers, test):
+        description = ""
+        Filename = row[headers["Filename"]]
+        Content = row[headers["Content"]]
+        Name = row[headers["Name"]]
+        Rule = row[headers["Rule"]]
+        Severity = row[headers["Severity"]]
+        Node_Name = row[headers["Node Name"]]
+        Container_Name = row[headers["Container Name"]]
+        Kubernetes_Cluster_Name = row[headers["Kubernetes Cluster Name"]]
+        Signature = row[headers["Signature"]]
+        description += "**Filename:** " + str(Filename) + "\n"
+        description += "**Name:** " + str(Name) + "\n"
+        description += "**Rule:** " + str(Rule) + "\n"
+        description += "**Node Name:** " + str(Node_Name) + "\n"
+        description += "**Container Name:** " + str(Container_Name) + "\n"
+        description += "**Kubernetes Cluster Name:** " + str(Kubernetes_Cluster_Name) + "\n"
+        description += "**Content:** " + str(Content) + "\n"
+        description += "**Signature:** " + str(Signature) + "\n"
+        if Name is not None and Severity is not None:
+            finding = Finding(
+                    title=str(Name),
+                    description=description,
+                    file_path=Filename,
+                    severity=self.severity(Severity),
+                    static_finding=False,
+                    dynamic_finding=True,
+                    test=test,
+            )
+        else:
+            finding = None
+        return finding
+
+    def severity(self, input):
+        if input is None:
+            return "Info"
+        else:
+            return input.capitalize()

dojo/tools/deepfence_threatmapper/vulnerability.py

@@ -0,0 +1,50 @@
+from dojo.models import Finding
+
+
+class DeepfenceThreatmapperVulnerability:
+    def get_findings(self, row, headers, test):
+        description = ""
+        cve_attack_vector = row[headers["cve_attack_vector"]]
+        cve_caused_by_package = row[headers["cve_caused_by_package"]]
+        cve_container_image = row[headers["cve_container_image"]]
+        cve_container_image_id = row[headers["cve_container_image_id"]]
+        cve_description = row[headers["cve_description"]]
+        cve_fixed_in = row[headers["cve_fixed_in"]]
+        cve_id = row[headers["cve_id"]]
+        cve_link = row[headers["cve_link"]]
+        cve_severity = row[headers["cve_severity"]]
+        cve_overall_score = row[headers["cve_overall_score"]]
+        cve_type = row[headers["cve_type"]]
+        host_name = row[headers["host_name"]]
+        cloud_account_id = row[headers["cloud_account_id"]]
+        masked = row[headers["masked"]]
+        description += "**cve_attack_vector:** " + str(cve_attack_vector) + "\n"
+        description += "**cve_caused_by_package:** " + str(cve_caused_by_package) + "\n"
+        description += "**cve_container_image:** " + str(cve_container_image) + "\n"
+        description += "**cve_container_image_id:** " + str(cve_container_image_id) + "\n"
+        description += "**cve_description:** " + str(cve_description) + "\n"
+        description += "**cve_severity:** " + str(cve_severity) + "\n"
+        description += "**cve_overall_score:** " + str(cve_overall_score) + "\n"
+        description += "**cve_type:** " + str(cve_type) + "\n"
+        description += "**host_name:** " + str(host_name) + "\n"
+        description += "**cloud_account_id:** " + str(cloud_account_id) + "\n"
+        description += "**masked:** " + str(masked) + "\n"
+        finding = Finding(
+            title="Threatmapper_Vuln_Report-" + cve_id,
+            description=description,
+            component_name=cve_caused_by_package,
+            severity=self.severity(cve_severity),
+            static_finding=False,
+            dynamic_finding=True,
+            mitigation=cve_fixed_in,
+            references=cve_link,
+            cve=cve_id,
+            test=test,
+        )
+        return finding
+
+    def severity(self, input):
+        if input is None:
+            return "Info"
+        else:
+            return input.capitalize()

unittests/scans/deepfence_threatmapper/compliance_report.csv

@@ -0,0 +1,8 @@
+@timestamp,compliance_check_type,count,doc_id,host_name,cloud_account_id,masked,node_id,node_name,node_type,status,test_category,test_desc,test_info,test_number
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,149c4791fc6502e5a30f738d4eaba982,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,pass,Docker Files,3.6 - PASS,Ensure that /fenced/mnt/host/etc/docker directory permissions are set to 755 or more restrictively (Automated),gdpr_3.6
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,47edf84375c0bb90f48fa61684883b04,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,info,Docker Files,3.12 - INFO,Ensure that the Docker server certificate file permissions are set to 444 or more restrictively (Automated),gdpr_3.12
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,ad1965efb22e226df8a95a361a30cbc3,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,info,Docker Files,3.2 - INFO,Ensure that docker.service file permissions are appropriately set (Automated),gdpr_3.2
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,1db7418dc73082cdfc1c9e0d5ba5f6e0,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,warn,Audit,1.1.12 - WARN,1.1.12 Ensure auditing is configured for Dockerfiles and directories - /fenced/mnt/host/etc/containerd/config.toml (Automated),gdpr_1.1.12
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,2c3f915f3e72d6e16d192ae9aa71c704,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,pass,Docker Files,3.16 - PASS,Ensure that the Docker socket file permissions are set to 660 or more restrictively (Automated),gdpr_3.16
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,d158a60b1c623d11ce88cf68555e08af,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,info,Docker Files,3.4 - INFO,Ensure that docker.socket file permissions are set to 644 or more restrictive (Automated),gdpr_3.4
+2024-01-25 11:17:30.272 +0000 UTC,gdpr,,,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,182758849647,False,4d1d5b7a279ce57b0f76be61b461d22c,cf-ngm-dev-cicd-ip-xxx-xxx-xxx-xxx.eu-central-1.compute.internal,host,info,Docker Files,3.14 - INFO,Ensure that the Docker server certificate key file permissions are set to 400 (Automated),gdpr_3.14

unittests/scans/deepfence_threatmapper/malware_report.csv

@@ -0,0 +1,10 @@
+Rule Name,Class,File Name,Summary,Severity,Node Name,NodeType,Container Name,Kubernetes Cluster Name
+MD5_Constants,Crypto Mining,/tmp/Deepfence/YaraHunter/df_db09257b02e615049e0aecc05be2dc2401735e67db4ab74225df777c62c39753/usr/sbin/mkfs.cramfs,The matched rule file's  author  is phoul (@phoul) .The file has a rule match that  It is a crypto signature.Look for MD5 constants .The matched rule file's  Date  is 2014-01 .The matched rule file's  version  is 0.2 .,low,portal / secpipe-core-prd-ip-zzz-zzz-zzz-zzz.eu-west-1.compute.internal,container,portal,secpipe-core-prd
+MD5_Constants,Crypto Mining,/tmp/Deepfence/YaraHunter/df_80ffd64c318595cf17a9ea482315b0c2a03572fb6e41f7ee53ec27786158c27c/usr/sbin/mkfs.cramfs,The matched rule file's  author  is phoul (@phoul) .The file has a rule match that  It is a crypto signature.Look for MD5 constants .The matched rule file's  Date  is 2014-01 .The matched rule file's  version  is 0.2 .,low,portal / secpipe-core-prd-ip-uuu-uuu-uuu-uuu.eu-west-1.compute.internal,container,portal,secpipe-core-prd
+CRC32_table,Crypto Mining,/tmp/Deepfence/YaraHunter/df_0dfa48a10ee6ca92c7d910ecd72a6207978f7f1bdc36870bf1587625f0270d37/lib/libz.so.1.2.13,The matched rule file's  author  is _pusher_ .The file has a rule match that  It is a crypto signature.Look for CRC32 table .The matched rule file's  Date  is 2015-05 .The matched rule file's  version  is 0.1 .,low,nginx / secpipe-core-prd-ip-kkk-kkk-kkk-kkk.eu-west-1.compute.internal,container,nginx,secpipe-core-prd
+CRC32_poly_Constant,Crypto Mining,/tmp/Deepfence/YaraHunter/df_0dfa48a10ee6ca92c7d910ecd72a6207978f7f1bdc36870bf1587625f0270d37/lib/libz.so.1.2.13,The matched rule file's  author  is _pusher_ .The file has a rule match that  It is a crypto signature.Look for CRC32 [poly] .The matched rule file's  Date  is 2015-05 .The matched rule file's  version  is 0.1 .,low,nginx / secpipe-core-prd-ip-kkk-kkk-kkk-kkk.eu-west-1.compute.internal,container,nginx,secpipe-core-prd
+MD5_Constants,Crypto Mining,/tmp/Deepfence/YaraHunter/df_cc54a20c0e1cee5e4951d047e13f69551cfddedbd67a05cc4e3de61939b10e7a/usr/sbin/mkfs.cramfs,The matched rule file's  author  is phoul (@phoul) .The file has a rule match that  It is a crypto signature.Look for MD5 constants .The matched rule file's  Date  is 2014-01 .The matched rule file's  version  is 0.2 .,low,portal / secpipe-core-prd-ip-yyy-yyy-yyy-yyy.eu-west-1.compute.internal,container,portal,secpipe-core-prd
+MD5_Constants,Crypto Mining,/tmp/Deepfence/YaraHunter/df_5e10a8e665e9def9227c98ec630c80d8c8b441c389c3d2b25d7c8d3b07c94eb4/sbin/mkfs.cramfs,The matched rule file's  author  is phoul (@phoul) .The file has a rule match that  It is a crypto signature.Look for MD5 constants .The matched rule file's  Date  is 2014-01 .The matched rule file's  version  is 0.2 .,low,rabbitmq / secpipe-core-prd-ip-xxx-xxx-xxx-xxx.eu-west-1.compute.internal,container,rabbitmq,secpipe-core-prd
+BASE64_table,Crypto Mining,/tmp/Deepfence/YaraHunter/df_5e10a8e665e9def9227c98ec630c80d8c8b441c389c3d2b25d7c8d3b07c94eb4/lib/x86_64-linux-gnu/libresolv-2.31.so,The matched rule file's  author  is _pusher_ .The file has a rule match that  It is a crypto signature.Look for Base64 table .The matched rule file's  Date  is 2015-07 .The matched rule file's  version  is 0.1 .,low,rabbitmq / secpipe-core-prd-ip-xxx-xxx-xxx-xxx.eu-west-1.compute.internal,container,rabbitmq,secpipe-core-prd
+BASE64_table,Crypto Mining,/tmp/Deepfence/YaraHunter/df_5e10a8e665e9def9227c98ec630c80d8c8b441c389c3d2b25d7c8d3b07c94eb4/opt/bitnami/erlang/lib/erlang/erts-13.1.3/bin/beam.smp,The matched rule file's  author  is _pusher_ .The file has a rule match that  It is a crypto signature.Look for Base64 table .The matched rule file's  Date  is 2015-07 .The matched rule file's  version  is 0.1 .,low,rabbitmq / secpipe-core-prd-ip-xxx-xxx-xxx-xxx.eu-west-1.compute.internal,container,rabbitmq,secpipe-core-prd
+CRC32_table,Crypto Mining,/tmp/Deepfence/YaraHunter/df_5e10a8e665e9def9227c98ec630c80d8c8b441c389c3d2b25d7c8d3b07c94eb4/lib/x86_64-linux-gnu/libz.so.1.2.11,The matched rule file's  author  is _pusher_ .The file has a rule match that  It is a crypto signature.Look for CRC32 table .The matched rule file's  Date  is 2015-05 .The matched rule file's  version  is 0.1 .,low,rabbitmq / secpipe-core-prd-ip-xxx-xxx-xxx-xxx.eu-west-1.compute.internal,container,rabbitmq,secpipe-core-prd