diff --git a/octopoes/bits/ask_disallowed_domains/bit.py b/octopoes/bits/ask_disallowed_domains/bit.py deleted file mode 100644 index a9fbdc02690..00000000000 --- a/octopoes/bits/ask_disallowed_domains/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.network import Network - -BIT = BitDefinition( - id="ask-disallowed-domains", - consumes=Network, - parameters=[], - min_scan_level=0, - module="bits.ask_disallowed_domains.ask_disallowed_domains", -) diff --git a/octopoes/bits/ask_port_specification/bit.py b/octopoes/bits/ask_port_specification/bit.py deleted file mode 100644 index 13a12eb5f88..00000000000 --- a/octopoes/bits/ask_port_specification/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.network import Network - -BIT = BitDefinition( - id="ask-port-specification", - consumes=Network, - parameters=[], - min_scan_level=0, - module="bits.ask_port_specification.ask_port_specification", -) diff --git a/octopoes/bits/ask_url_params_to_ignore/bit.py b/octopoes/bits/ask_url_params_to_ignore/bit.py deleted file mode 100644 index f78db5af1ea..00000000000 --- a/octopoes/bits/ask_url_params_to_ignore/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.network import Network - -BIT = BitDefinition( - id="ask_url_params_to_ignore", - consumes=Network, - parameters=[], - min_scan_level=0, - module="bits.ask_url_params_to_ignore.ask_url_params_to_ignore", -) diff --git a/octopoes/bits/check_cve_2021_41773/bit.py b/octopoes/bits/check_cve_2021_41773/bit.py deleted file mode 100644 index 3e32a458c9c..00000000000 --- a/octopoes/bits/check_cve_2021_41773/bit.py +++ /dev/null @@ -1,9 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import HTTPHeader - -BIT = BitDefinition( - id="check_cve_2021_41773", - consumes=HTTPHeader, - parameters=[], - module="bits.check_cve_2021_41773.check_cve_2021_41773", -) diff --git a/octopoes/bits/check_hsts_header/bit.py b/octopoes/bits/check_hsts_header/bit.py deleted file mode 100644 index 6b98f2d86a9..00000000000 --- a/octopoes/bits/check_hsts_header/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import HTTPHeader - -BIT = BitDefinition( - id="check-hsts-header", - consumes=HTTPHeader, - parameters=[], - module="bits.check_hsts_header.check_hsts_header", - config_ooi_relation_path="HTTPHeader.resource.website.hostname.network", -) diff --git a/octopoes/bits/cipher_classification/bit.py b/octopoes/bits/cipher_classification/bit.py deleted file mode 100644 index 6c991b1d05b..00000000000 --- a/octopoes/bits/cipher_classification/bit.py +++ /dev/null @@ -1,9 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.service import TLSCipher - -BIT = BitDefinition( - id="cipher-classification", - consumes=TLSCipher, - parameters=[], - module="bits.cipher_classification.cipher_classification", -) diff --git a/octopoes/bits/default_findingtype_risk/bit.py b/octopoes/bits/default_findingtype_risk/bit.py deleted file mode 100644 index 2a4301f7333..00000000000 --- a/octopoes/bits/default_findingtype_risk/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.findings import FindingType - -BIT = BitDefinition( - id="default-findingtype-risk", - consumes=FindingType, - parameters=[], - module="bits.default_findingtype_risk.default_findingtype_risk", - min_scan_level=0, -) diff --git a/octopoes/bits/disallowed_csp_hostnames/bit.py b/octopoes/bits/disallowed_csp_hostnames/bit.py deleted file mode 100644 index 73d1a2fbc99..00000000000 --- a/octopoes/bits/disallowed_csp_hostnames/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import HTTPHeaderHostname - -BIT = BitDefinition( - id="disallowed-csp-hostnames", - consumes=HTTPHeaderHostname, - parameters=[], - module="bits.disallowed_csp_hostnames.disallowed_csp_hostnames", - config_ooi_relation_path="HTTPHeaderHostname.hostname.network", -) diff --git a/octopoes/bits/domain_owner_verification/bit.py b/octopoes/bits/domain_owner_verification/bit.py deleted file mode 100644 index 98a6d63f87a..00000000000 --- a/octopoes/bits/domain_owner_verification/bit.py +++ /dev/null @@ -1,9 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.dns.records import DNSNSRecord - -BIT = BitDefinition( - id="domain-owner-verification", - consumes=DNSNSRecord, - parameters=[], - module="bits.domain_owner_verification.domain_owner_verification", -) diff --git a/octopoes/bits/expiring_certificate/bit.py b/octopoes/bits/expiring_certificate/bit.py deleted file mode 100644 index 85b55879408..00000000000 --- a/octopoes/bits/expiring_certificate/bit.py +++ /dev/null @@ -1,9 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.certificate import X509Certificate - -BIT = BitDefinition( - id="expiring-certificate", - consumes=X509Certificate, - parameters=[], - module="bits.expiring_certificate.expiring_certificate", -) diff --git a/octopoes/bits/missing_certificate/bit.py b/octopoes/bits/missing_certificate/bit.py deleted file mode 100644 index 330ac1b65d9..00000000000 --- a/octopoes/bits/missing_certificate/bit.py +++ /dev/null @@ -1,6 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="missing-certificate", consumes=Website, parameters=[], module="bits.missing_certificate.missing_certificate" -) diff --git a/octopoes/bits/missing_spf/bit.py b/octopoes/bits/missing_spf/bit.py deleted file mode 100644 index ed46e16f2a0..00000000000 --- a/octopoes/bits/missing_spf/bit.py +++ /dev/null @@ -1,14 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.records import NXDOMAIN -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.email_security import DNSSPFRecord - -BIT = BitDefinition( - id="missing-spf", - consumes=Hostname, - parameters=[ - BitParameterDefinition(ooi_type=DNSSPFRecord, relation_path="dns_txt_record.hostname"), - BitParameterDefinition(ooi_type=NXDOMAIN, relation_path="hostname"), - ], - module="bits.missing_spf.missing_spf", -) diff --git a/octopoes/bits/missing_spf/missing_spf.py b/octopoes/bits/missing_spf/missing_spf.py deleted file mode 100644 index fe7d78d0a03..00000000000 --- a/octopoes/bits/missing_spf/missing_spf.py +++ /dev/null @@ -1,29 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -import tldextract - -from octopoes.models import OOI -from octopoes.models.ooi.dns.records import NXDOMAIN -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.email_security import DNSSPFRecord -from octopoes.models.ooi.findings import Finding, KATFindingType - - -def run(input_ooi: Hostname, additional_oois: list[DNSSPFRecord | NXDOMAIN], config: dict[str, Any]) -> Iterator[OOI]: - spf_records = [ooi for ooi in additional_oois if isinstance(ooi, DNSSPFRecord)] - nxdomains = (ooi for ooi in additional_oois if isinstance(ooi, NXDOMAIN)) - - if any(nxdomains): - return - # only report finding when there is no SPF record - if ( - not tldextract.extract(input_ooi.name).subdomain - and tldextract.extract(input_ooi.name).domain - and not spf_records - ): - ft = KATFindingType(id="KAT-NO-SPF") - yield ft - yield Finding( - ooi=input_ooi.reference, finding_type=ft.reference, description="This hostname does not have an SPF record" - ) diff --git a/octopoes/bits/oois_in_headers/bit.py b/octopoes/bits/oois_in_headers/bit.py deleted file mode 100644 index ef2dd5c40d8..00000000000 --- a/octopoes/bits/oois_in_headers/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import HTTPHeader - -BIT = BitDefinition( - id="oois-in-headers", - consumes=HTTPHeader, - parameters=[], - module="bits.oois_in_headers.oois_in_headers", - config_ooi_relation_path="HTTPHeader.resource.website.hostname.network", -) diff --git a/octopoes/bits/port_common/bit.py b/octopoes/bits/port_common/bit.py deleted file mode 100644 index 256122c9d34..00000000000 --- a/octopoes/bits/port_common/bit.py +++ /dev/null @@ -1,6 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.network import IPPort - -BIT = BitDefinition( - id="port-common", consumes=IPPort, parameters=[], module="bits.port_common.port_common", default_enabled=False -) diff --git a/octopoes/bits/port_common/port_common.py b/octopoes/bits/port_common/port_common.py deleted file mode 100644 index eac9c7d0f7a..00000000000 --- a/octopoes/bits/port_common/port_common.py +++ /dev/null @@ -1,37 +0,0 @@ -from collections.abc import Iterator - -from octopoes.models import OOI -from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPPort, Protocol - -COMMON_TCP_PORTS = [ - 25, # SMTP - 53, # DNS - 80, # HTTP - 110, # POP3 - 143, # IMAP - 443, # HTTPS - 465, # SMTPS - 587, # SMTP (message submmission) - 993, # IMAPS - 995, # POP3S -] - -COMMON_UDP_PORTS = [ - 53 # DNS -] - - -def run(input_ooi: IPPort, additional_oois: list, config: dict) -> Iterator[OOI]: - port = input_ooi.port - protocol = input_ooi.protocol - if (protocol == Protocol.TCP and port in COMMON_TCP_PORTS) or ( - protocol == Protocol.UDP and port in COMMON_UDP_PORTS - ): - kat = KATFindingType(id="KAT-OPEN-COMMON-PORT") - yield kat - yield Finding( - finding_type=kat.reference, - ooi=input_ooi.reference, - description=f"Port {port}/{protocol.value} is a common port and found to be open.", - ) diff --git a/octopoes/bits/spf_discovery/bit.py b/octopoes/bits/spf_discovery/bit.py deleted file mode 100644 index 04737497de1..00000000000 --- a/octopoes/bits/spf_discovery/bit.py +++ /dev/null @@ -1,4 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.dns.records import DNSTXTRecord - -BIT = BitDefinition(id="spf-discovery", consumes=DNSTXTRecord, parameters=[], module="bits.spf_discovery.spf_discovery") diff --git a/octopoes/bits/url_classification/bit.py b/octopoes/bits/url_classification/bit.py deleted file mode 100644 index b8ba4b8d247..00000000000 --- a/octopoes/bits/url_classification/bit.py +++ /dev/null @@ -1,10 +0,0 @@ -from bits.definitions import BitDefinition -from octopoes.models.ooi.web import URL - -BIT = BitDefinition( - id="url-classification", - consumes=URL, - parameters=[], - module="bits.url_classification.url_classification", - min_scan_level=0, -) diff --git a/octopoes/bits/url_discovery/bit.py b/octopoes/bits/url_discovery/bit.py deleted file mode 100644 index a952728f64f..00000000000 --- a/octopoes/bits/url_discovery/bit.py +++ /dev/null @@ -1,14 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.zone import ResolvedHostname -from octopoes.models.ooi.network import IPAddress, IPPort - -BIT = BitDefinition( - id="url-discovery", - consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPPort, relation_path="address"), - BitParameterDefinition(ooi_type=ResolvedHostname, relation_path="address"), - ], - module="bits.url_discovery.url_discovery", - min_scan_level=0, -) diff --git a/octopoes/bits/url_discovery/url_discovery.py b/octopoes/bits/url_discovery/url_discovery.py deleted file mode 100644 index 4cf45d7042d..00000000000 --- a/octopoes/bits/url_discovery/url_discovery.py +++ /dev/null @@ -1,28 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.dns.zone import ResolvedHostname -from octopoes.models.ooi.network import IPAddress, IPPort, Network -from octopoes.models.ooi.web import URL - - -def run( - ip_address: IPAddress, additional_oois: list[IPPort | ResolvedHostname], config: dict[str, Any] -) -> Iterator[OOI]: - hostnames = [resolved.hostname for resolved in additional_oois if isinstance(resolved, ResolvedHostname)] - ip_ports = [ip_port for ip_port in additional_oois if isinstance(ip_port, IPPort)] - - for ip_port in ip_ports: - if ip_port.port == 443: - for hostname in hostnames: - yield URL( - network=Network(name=hostname.tokenized.network.name).reference, - raw=f"https://{hostname.tokenized.name}/", - ) - if ip_port.port == 80: - for hostname in hostnames: - yield URL( - network=Network(name=hostname.tokenized.network.name).reference, - raw=f"http://{hostname.tokenized.name}/", - ) diff --git a/octopoes/bits/website_discovery/bit.py b/octopoes/bits/website_discovery/bit.py deleted file mode 100644 index a4709e6ddaf..00000000000 --- a/octopoes/bits/website_discovery/bit.py +++ /dev/null @@ -1,15 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.zone import ResolvedHostname -from octopoes.models.ooi.network import IPAddress -from octopoes.models.ooi.service import IPService - -BIT = BitDefinition( - id="website-discovery", - consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPService, relation_path="ip_port.address"), - BitParameterDefinition(ooi_type=ResolvedHostname, relation_path="address"), - ], - module="bits.website_discovery.website_discovery", - min_scan_level=0, -) diff --git a/octopoes/bits/website_discovery/website_discovery.py b/octopoes/bits/website_discovery/website_discovery.py deleted file mode 100644 index b6e3c2feb68..00000000000 --- a/octopoes/bits/website_discovery/website_discovery.py +++ /dev/null @@ -1,25 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.dns.zone import ResolvedHostname -from octopoes.models.ooi.network import IPAddressV4 -from octopoes.models.ooi.service import IPService -from octopoes.models.ooi.web import Website - - -def run( - ip_address: IPAddressV4, additional_oois: list[IPService | ResolvedHostname], config: dict[str, Any] -) -> Iterator[OOI]: - def is_service_http(ip_service: IPService) -> bool: - service_name = ip_service.service.tokenized.name.lower().strip() - return service_name in ("http", "https") - - hostnames = [resolved.hostname for resolved in additional_oois if isinstance(resolved, ResolvedHostname)] - services = [ip_service for ip_service in additional_oois if isinstance(ip_service, IPService)] - http_services = filter(is_service_http, services) - - # website is cartesian product of hostname and http services - for http_service in http_services: - for hostname in hostnames: - yield Website(hostname=hostname, ip_service=http_service.reference) diff --git a/octopoes/bits/ask_disallowed_domains/__init__.py b/octopoes/nibbles/__init__.py similarity index 100% rename from octopoes/bits/ask_disallowed_domains/__init__.py rename to octopoes/nibbles/__init__.py diff --git a/octopoes/bits/ask_port_specification/__init__.py b/octopoes/nibbles/ask_disallowed_domains/__init__.py similarity index 100% rename from octopoes/bits/ask_port_specification/__init__.py rename to octopoes/nibbles/ask_disallowed_domains/__init__.py diff --git a/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py b/octopoes/nibbles/ask_disallowed_domains/ask_disallowed_domains.py similarity index 78% rename from octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py rename to octopoes/nibbles/ask_disallowed_domains/ask_disallowed_domains.py index 971f1d68993..cb7f7e9869a 100644 --- a/octopoes/bits/ask_disallowed_domains/ask_disallowed_domains.py +++ b/octopoes/nibbles/ask_disallowed_domains/ask_disallowed_domains.py @@ -1,14 +1,13 @@ import json from collections.abc import Iterator from pathlib import Path -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.network import Network from octopoes.models.ooi.question import Question -def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: Network) -> Iterator[OOI]: network = input_ooi with (Path(__file__).parent / "question_schema.json").open() as f: diff --git a/octopoes/nibbles/ask_disallowed_domains/nibble.py b/octopoes/nibbles/ask_disallowed_domains/nibble.py new file mode 100644 index 00000000000..8ba2ba043d9 --- /dev/null +++ b/octopoes/nibbles/ask_disallowed_domains/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.network import Network + +NIBBLE = NibbleDefinition(id="ask-disallowed-domains", signature=[NibbleParameter(object_type=Network)]) diff --git a/octopoes/bits/ask_disallowed_domains/question_schema.json b/octopoes/nibbles/ask_disallowed_domains/question_schema.json similarity index 100% rename from octopoes/bits/ask_disallowed_domains/question_schema.json rename to octopoes/nibbles/ask_disallowed_domains/question_schema.json diff --git a/octopoes/bits/ask_url_params_to_ignore/__init__.py b/octopoes/nibbles/ask_port_specification/__init__.py similarity index 100% rename from octopoes/bits/ask_url_params_to_ignore/__init__.py rename to octopoes/nibbles/ask_port_specification/__init__.py diff --git a/octopoes/bits/ask_port_specification/ask_port_specification.py b/octopoes/nibbles/ask_port_specification/ask_port_specification.py similarity index 78% rename from octopoes/bits/ask_port_specification/ask_port_specification.py rename to octopoes/nibbles/ask_port_specification/ask_port_specification.py index 971f1d68993..cb7f7e9869a 100644 --- a/octopoes/bits/ask_port_specification/ask_port_specification.py +++ b/octopoes/nibbles/ask_port_specification/ask_port_specification.py @@ -1,14 +1,13 @@ import json from collections.abc import Iterator from pathlib import Path -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.network import Network from octopoes.models.ooi.question import Question -def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: Network) -> Iterator[OOI]: network = input_ooi with (Path(__file__).parent / "question_schema.json").open() as f: diff --git a/octopoes/nibbles/ask_port_specification/nibble.py b/octopoes/nibbles/ask_port_specification/nibble.py new file mode 100644 index 00000000000..c99c38598fa --- /dev/null +++ b/octopoes/nibbles/ask_port_specification/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.network import Network + +NIBBLE = NibbleDefinition(id="ask-port-specification", signature=[NibbleParameter(object_type=Network)]) diff --git a/octopoes/bits/ask_port_specification/question_schema.json b/octopoes/nibbles/ask_port_specification/question_schema.json similarity index 100% rename from octopoes/bits/ask_port_specification/question_schema.json rename to octopoes/nibbles/ask_port_specification/question_schema.json diff --git a/octopoes/bits/check_cve_2021_41773/__init__.py b/octopoes/nibbles/ask_url_params_to_ignore/__init__.py similarity index 100% rename from octopoes/bits/check_cve_2021_41773/__init__.py rename to octopoes/nibbles/ask_url_params_to_ignore/__init__.py diff --git a/octopoes/bits/ask_url_params_to_ignore/ask_url_params_to_ignore.py b/octopoes/nibbles/ask_url_params_to_ignore/ask_url_params_to_ignore.py similarity index 78% rename from octopoes/bits/ask_url_params_to_ignore/ask_url_params_to_ignore.py rename to octopoes/nibbles/ask_url_params_to_ignore/ask_url_params_to_ignore.py index 971f1d68993..cb7f7e9869a 100644 --- a/octopoes/bits/ask_url_params_to_ignore/ask_url_params_to_ignore.py +++ b/octopoes/nibbles/ask_url_params_to_ignore/ask_url_params_to_ignore.py @@ -1,14 +1,13 @@ import json from collections.abc import Iterator from pathlib import Path -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.network import Network from octopoes.models.ooi.question import Question -def run(input_ooi: Network, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: Network) -> Iterator[OOI]: network = input_ooi with (Path(__file__).parent / "question_schema.json").open() as f: diff --git a/octopoes/nibbles/ask_url_params_to_ignore/nibble.py b/octopoes/nibbles/ask_url_params_to_ignore/nibble.py new file mode 100644 index 00000000000..4a2b0fda8dc --- /dev/null +++ b/octopoes/nibbles/ask_url_params_to_ignore/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.network import Network + +NIBBLE = NibbleDefinition(id="ask_url_params_to_ignore", signature=[NibbleParameter(object_type=Network)]) diff --git a/octopoes/bits/ask_url_params_to_ignore/question_schema.json b/octopoes/nibbles/ask_url_params_to_ignore/question_schema.json similarity index 100% rename from octopoes/bits/ask_url_params_to_ignore/question_schema.json rename to octopoes/nibbles/ask_url_params_to_ignore/question_schema.json diff --git a/octopoes/bits/check_hsts_header/__init__.py b/octopoes/nibbles/check_cve_2021_41773/__init__.py similarity index 100% rename from octopoes/bits/check_hsts_header/__init__.py rename to octopoes/nibbles/check_cve_2021_41773/__init__.py diff --git a/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py b/octopoes/nibbles/check_cve_2021_41773/check_cve_2021_41773.py similarity index 81% rename from octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py rename to octopoes/nibbles/check_cve_2021_41773/check_cve_2021_41773.py index 83076aed418..e7b5270edbb 100644 --- a/octopoes/bits/check_cve_2021_41773/check_cve_2021_41773.py +++ b/octopoes/nibbles/check_cve_2021_41773/check_cve_2021_41773.py @@ -1,12 +1,11 @@ from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import CVEFindingType, Finding from octopoes.models.ooi.web import HTTPHeader -def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: HTTPHeader) -> Iterator[OOI]: header = input_ooi if header.key.lower() != "server": return diff --git a/octopoes/nibbles/check_cve_2021_41773/nibble.py b/octopoes/nibbles/check_cve_2021_41773/nibble.py new file mode 100644 index 00000000000..793acd9ec9b --- /dev/null +++ b/octopoes/nibbles/check_cve_2021_41773/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.types import HTTPHeader + +NIBBLE = NibbleDefinition(id="check_cve_2021_41773", signature=[NibbleParameter(object_type=HTTPHeader)]) diff --git a/octopoes/bits/cipher_classification/__init__.py b/octopoes/nibbles/check_hsts_header/__init__.py similarity index 100% rename from octopoes/bits/cipher_classification/__init__.py rename to octopoes/nibbles/check_hsts_header/__init__.py diff --git a/octopoes/bits/check_hsts_header/check_hsts_header.py b/octopoes/nibbles/check_hsts_header/check_hsts_header.py similarity index 87% rename from octopoes/bits/check_hsts_header/check_hsts_header.py rename to octopoes/nibbles/check_hsts_header/check_hsts_header.py index de92e5e38dc..8807daf4069 100644 --- a/octopoes/bits/check_hsts_header/check_hsts_header.py +++ b/octopoes/nibbles/check_hsts_header/check_hsts_header.py @@ -1,20 +1,20 @@ import datetime from collections.abc import Iterator -from typing import Any from octopoes.models import OOI, Reference +from octopoes.models.ooi.config import Config from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.web import HTTPHeader -def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: HTTPHeader, config: Config | None) -> Iterator[OOI]: header = input_ooi if header.key.lower() != "strict-transport-security": return one_year = datetime.timedelta(days=365).total_seconds() - max_age = int(config.get("max-age", one_year)) if config else one_year + max_age = int(str(config.config.get("max-age", one_year))) if config and config.config else one_year findings: list[str] = [] headervalue = header.value.lower() diff --git a/octopoes/nibbles/check_hsts_header/nibble.py b/octopoes/nibbles/check_hsts_header/nibble.py new file mode 100644 index 00000000000..7ee0fd6e20c --- /dev/null +++ b/octopoes/nibbles/check_hsts_header/nibble.py @@ -0,0 +1,113 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.network import Network +from octopoes.models.ooi.web import HTTPHeader + + +def query(targets: list[Reference | None]) -> str: + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "10": + network = str(Network(name=targets[0].split("|")[1]).reference) if targets[0] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeader"] + [?header :HTTPHeader/primary_key "{str(targets[0])}"] + + (or + (and + [?config :Config/ooi "{network}"] + [?config :Config/bit_id "check-hsts-header"] + ) + (and + [(identity nil) ?config] + ) + ) + + ] + }} + }} + """ + elif sgn == "01": + network = str(Network(name=targets[1].split("|")[1]).reference) if targets[1] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "check-hsts-header"] + + (or + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/web_url ?url] + [?url :HostnameHTTPURL/network "{network}"] + ) + (and + [(identity nil) ?header] + [(identity nil) ?resource] + [(identity nil) ?url] + ) + ) + + ] + }} + }} + """ + elif sgn == "11": + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + [?header :object_type "HTTPHeader"] + [?header :HTTPHeader/primary_key "{str(targets[0])}"] + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "check-hsts-header"] + ] + }} + }} + """ + else: + return """ + { + :query { + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeader"] + + (or + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/web_url ?url] + [?url :HostnameHTTPURL/network ?network] + [?config :Config/ooi ?network] + [?config :Config/bit_id "check-hsts-header"] + ) + (and + [(identity nil) ?resource] + [(identity nil) ?url] + [(identity nil) ?network] + [(identity nil) ?config] + ) + ) + + ] + } + } + """ + + +NIBBLE = NibbleDefinition( + id="check-hsts-header", + signature=[ + NibbleParameter(object_type=HTTPHeader, parser="[*][?object_type == 'HTTPHeader'][]"), + NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), + ], + query=query, +) diff --git a/octopoes/bits/default_findingtype_risk/__init__.py b/octopoes/nibbles/cipher_classification/__init__.py similarity index 100% rename from octopoes/bits/default_findingtype_risk/__init__.py rename to octopoes/nibbles/cipher_classification/__init__.py diff --git a/octopoes/bits/cipher_classification/cipher_classification.py b/octopoes/nibbles/cipher_classification/cipher_classification.py similarity index 96% rename from octopoes/bits/cipher_classification/cipher_classification.py rename to octopoes/nibbles/cipher_classification/cipher_classification.py index 88cf99765b3..2ad9f6ea236 100644 --- a/octopoes/bits/cipher_classification/cipher_classification.py +++ b/octopoes/nibbles/cipher_classification/cipher_classification.py @@ -1,7 +1,6 @@ import csv from collections.abc import Iterator from pathlib import Path -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType @@ -77,7 +76,7 @@ def get_highest_severity_and_all_reasons(cipher_suites: dict) -> tuple[str, str] return highest_severity, all_reasons_str -def run(input_ooi: TLSCipher, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: TLSCipher) -> Iterator[OOI]: # Get the highest severity and all reasons for the cipher suite highest_severity, all_reasons = get_highest_severity_and_all_reasons(input_ooi.suites) diff --git a/octopoes/bits/cipher_classification/list-ciphers-openssl-with-finding-type.csv b/octopoes/nibbles/cipher_classification/list-ciphers-openssl-with-finding-type.csv similarity index 100% rename from octopoes/bits/cipher_classification/list-ciphers-openssl-with-finding-type.csv rename to octopoes/nibbles/cipher_classification/list-ciphers-openssl-with-finding-type.csv diff --git a/octopoes/nibbles/cipher_classification/nibble.py b/octopoes/nibbles/cipher_classification/nibble.py new file mode 100644 index 00000000000..7f9b196b2d7 --- /dev/null +++ b/octopoes/nibbles/cipher_classification/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.service import TLSCipher + +NIBBLE = NibbleDefinition(id="cipher-classification", signature=[NibbleParameter(object_type=TLSCipher)]) diff --git a/octopoes/bits/disallowed_csp_hostnames/__init__.py b/octopoes/nibbles/default_findingtype_risk/__init__.py similarity index 100% rename from octopoes/bits/disallowed_csp_hostnames/__init__.py rename to octopoes/nibbles/default_findingtype_risk/__init__.py diff --git a/octopoes/bits/default_findingtype_risk/default_findingtype_risk.py b/octopoes/nibbles/default_findingtype_risk/default_findingtype_risk.py similarity index 77% rename from octopoes/bits/default_findingtype_risk/default_findingtype_risk.py rename to octopoes/nibbles/default_findingtype_risk/default_findingtype_risk.py index 7acca28d705..5596a8b6a84 100644 --- a/octopoes/bits/default_findingtype_risk/default_findingtype_risk.py +++ b/octopoes/nibbles/default_findingtype_risk/default_findingtype_risk.py @@ -1,11 +1,10 @@ from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import FindingType, RiskLevelSeverity -def run(input_ooi: FindingType, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: FindingType) -> Iterator[OOI]: value_set = False if not input_ooi.risk_severity: input_ooi.risk_severity = RiskLevelSeverity.PENDING diff --git a/octopoes/nibbles/default_findingtype_risk/nibble.py b/octopoes/nibbles/default_findingtype_risk/nibble.py new file mode 100644 index 00000000000..d80c2234059 --- /dev/null +++ b/octopoes/nibbles/default_findingtype_risk/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.findings import FindingType + +NIBBLE = NibbleDefinition(id="default-findingtype-risk", signature=[NibbleParameter(object_type=FindingType)]) diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py new file mode 100644 index 00000000000..51d048ed325 --- /dev/null +++ b/octopoes/nibbles/definitions.py @@ -0,0 +1,91 @@ +import importlib +import inspect +import pkgutil +from collections.abc import Callable, Iterable +from pathlib import Path +from types import MethodType, ModuleType +from typing import Any + +import structlog +from pydantic import BaseModel +from xxhash import xxh3_128_hexdigest as xxh3 + +from octopoes.models import OOI, Reference + +NIBBLES_DIR = Path(__file__).parent +NIBBLE_ATTR_NAME = "NIBBLE" +NIBBLE_FUNC_NAME = "nibble" +logger = structlog.get_logger(__name__) + + +class NibbleParameter(BaseModel): + object_type: type[Any] + parser: str = "[]" + optional: bool = False + + def __eq__(self, other): + if isinstance(other, NibbleParameter): + return vars(self) == vars(other) + elif isinstance(other, type): + return self.object_type == other + else: + return False + + +class NibbleDefinition(BaseModel): + id: str + signature: list[NibbleParameter] + query: str | Callable[[list[Reference | None]], str] | None = None + enabled: bool = True + _payload: MethodType | None = None + _checksum: str | None = None + + def __call__(self, args: Iterable[OOI]) -> OOI | Iterable[OOI | None] | None: + if self._payload is None: + raise NotImplementedError + else: + return self._payload(*args) + + def __hash__(self): + return hash(self.id) + + @property + def _ini(self) -> dict[str, Any]: + return {"id": self.id, "enabled": self.enabled, "checksum": self._checksum} + + +def get_nibble_definitions() -> dict[str, NibbleDefinition]: + nibble_definitions = {} + + for package in pkgutil.walk_packages([str(NIBBLES_DIR)]): + if package.name in ["definitions", "runner"]: + continue + + try: + module: ModuleType = importlib.import_module(".nibble", f"{NIBBLES_DIR.name}.{package.name}") + + if hasattr(module, NIBBLE_ATTR_NAME): + nibble_definition: NibbleDefinition = getattr(module, NIBBLE_ATTR_NAME) + + try: + payload: ModuleType = importlib.import_module( + f".{package.name}", f"{NIBBLES_DIR.name}.{package.name}" + ) + if hasattr(payload, NIBBLE_FUNC_NAME): + nibble_definition._payload = getattr(payload, NIBBLE_FUNC_NAME) + nibble_definition._checksum = xxh3(inspect.getsource(module) + inspect.getsource(payload)) + else: + logger.warning('module "%s" has no function %s', package.name, NIBBLE_FUNC_NAME) + + except ModuleNotFoundError: + logger.warning('package "%s" has no function nibble', package.name) + + nibble_definitions[nibble_definition.id] = nibble_definition + + else: + logger.warning('module "%s" has no attribute %s', package.name, NIBBLE_ATTR_NAME) + + except ModuleNotFoundError: + logger.warning('package "%s" has no module nibble', package.name) + + return nibble_definitions diff --git a/octopoes/bits/domain_owner_verification/__init__.py b/octopoes/nibbles/disallowed_csp_hostnames/__init__.py similarity index 100% rename from octopoes/bits/domain_owner_verification/__init__.py rename to octopoes/nibbles/disallowed_csp_hostnames/__init__.py diff --git a/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py b/octopoes/nibbles/disallowed_csp_hostnames/disallowed_csp_hostnames.py similarity index 55% rename from octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py rename to octopoes/nibbles/disallowed_csp_hostnames/disallowed_csp_hostnames.py index c67b8e01bb8..db819389523 100644 --- a/octopoes/bits/disallowed_csp_hostnames/disallowed_csp_hostnames.py +++ b/octopoes/nibbles/disallowed_csp_hostnames/disallowed_csp_hostnames.py @@ -1,9 +1,9 @@ from collections.abc import Iterator -from typing import Any from link_shorteners import link_shorteners_list from octopoes.models import OOI +from octopoes.models.ooi.config import Config from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.web import HTTPHeaderHostname @@ -17,23 +17,29 @@ def get_disallowed_hostnames_from_config(config, config_key, default): return list(disallowed_hostnames.strip().split(",")) if disallowed_hostnames else [] -def run(input_ooi: HTTPHeaderHostname, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: HTTPHeaderHostname, config: Config | None) -> Iterator[OOI]: header_hostname = input_ooi header = header_hostname.header if header.tokenized.key.lower() != "content-security-policy": return - disallow_url_shorteners = config.get("disallow_url_shorteners", True) if config else True + disallow_url_shorteners = config.config.get("disallow_url_shorteners", True) if config and config.config else True hostname = header_hostname.hostname.tokenized.name disallowed_domains = link_shorteners_list() if disallow_url_shorteners else [] - disallowed_hostnames_from_config = get_disallowed_hostnames_from_config(config, "disallowed_hostnames", []) + disallowed_hostnames_from_config = get_disallowed_hostnames_from_config( + config.config if config else {}, "disallowed_hostnames", [] + ) disallowed_domains.extend(disallowed_hostnames_from_config) - - if hostname.lower() in disallowed_domains: - ft = KATFindingType(id="KAT-DISALLOWED-DOMAIN-IN-CSP") - f = Finding(ooi=input_ooi.reference, finding_type=ft.reference) - yield ft - yield f + hostnameparts = hostname.lower().split(".") + + # For e.g. ["www", "example", "com"], check "www.example.com", "example.com" and "com" + for i in range(len(hostnameparts)): + if ".".join(hostnameparts[i:]) in disallowed_domains: + ft = KATFindingType(id="KAT-DISALLOWED-DOMAIN-IN-CSP") + f = Finding(ooi=input_ooi.reference, finding_type=ft.reference) + yield ft + yield f + break diff --git a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py new file mode 100644 index 00000000000..fcf230be3e3 --- /dev/null +++ b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py @@ -0,0 +1,109 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.network import Network +from octopoes.models.ooi.web import HTTPHeaderHostname + + +def query(targets: list[Reference | None]) -> str: + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "10": + network = str(Network(name=targets[0].split("|")[1]).reference) if targets[0] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeaderHostname"] + [?header :HTTPHeaderHostname/primary_key "{str(targets[0])}"] + + (or + (and + [?config :Config/ooi "{network}"] + [?config :Config/bit_id "disallowed-csp-hostnames"] + ) + (and + [(identity nil) ?config] + ) + ) + + ] + }} + }} + """ + elif sgn == "01": + network = str(Network(name=targets[1].split("|")[1]).reference) if targets[1] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "disallowed-csp-hostnames"] + + (or + (and + [?header :HTTPHeaderHostname/hostname ?hostname] + [?hostname :Hostname/network "{network}"] + ) + (and + [(identity nil) ?header] + [(identity nil) ?hostname] + ) + ) + + ] + }} + }} + """ + elif sgn == "11": + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + [?header :object_type "HTTPHeaderHostname"] + [?header :HTTPHeaderHostname/primary_key "{str(targets[0])}"] + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "disallowed-csp-hostnames"] + ] + }} + }} + """ + else: + return """ + { + :query { + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeaderHostname"] + + (or + (and + [?header :HTTPHeaderHostname/hostname ?hostname] + [?hostname :Hostname/network ?network] + [?config :Config/ooi ?network] + [?config :Config/bit_id "disallowed-csp-hostnames"] + ) + (and + [(identity nil) ?hostname] + [(identity nil) ?network] + [(identity nil) ?config] + ) + ) + + ] + } + } + """ + + +NIBBLE = NibbleDefinition( + id="disallowed-csp-hostnames", + signature=[ + NibbleParameter(object_type=HTTPHeaderHostname, parser="[*][?object_type == 'HTTPHeaderHostname'][]"), + NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), + ], + query=query, +) diff --git a/octopoes/bits/expiring_certificate/__init__.py b/octopoes/nibbles/domain_owner_verification/__init__.py similarity index 100% rename from octopoes/bits/expiring_certificate/__init__.py rename to octopoes/nibbles/domain_owner_verification/__init__.py diff --git a/octopoes/bits/domain_owner_verification/domain_owner_verification.py b/octopoes/nibbles/domain_owner_verification/domain_owner_verification.py similarity index 90% rename from octopoes/bits/domain_owner_verification/domain_owner_verification.py rename to octopoes/nibbles/domain_owner_verification/domain_owner_verification.py index e0142442caf..81a0f75d2f9 100644 --- a/octopoes/bits/domain_owner_verification/domain_owner_verification.py +++ b/octopoes/nibbles/domain_owner_verification/domain_owner_verification.py @@ -11,7 +11,7 @@ ] -def run(nameserver_record: DNSNSRecord, additional_oois, config: dict[str, str]) -> Iterator[OOI]: +def nibble(nameserver_record: DNSNSRecord) -> Iterator[OOI]: """Checks to see if a domain has a specific set of dns servers which would indicate domain registrant verification. https://support.dnsimple.com/articles/icann-domain-validation/ """ diff --git a/octopoes/nibbles/domain_owner_verification/nibble.py b/octopoes/nibbles/domain_owner_verification/nibble.py new file mode 100644 index 00000000000..2bd9218022d --- /dev/null +++ b/octopoes/nibbles/domain_owner_verification/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.dns.records import DNSNSRecord + +NIBBLE = NibbleDefinition(id="domain-owner-verification", signature=[NibbleParameter(object_type=DNSNSRecord)]) diff --git a/octopoes/bits/missing_certificate/__init__.py b/octopoes/nibbles/expiring_certificate/__init__.py similarity index 100% rename from octopoes/bits/missing_certificate/__init__.py rename to octopoes/nibbles/expiring_certificate/__init__.py diff --git a/octopoes/bits/expiring_certificate/expiring_certificate.py b/octopoes/nibbles/expiring_certificate/expiring_certificate.py similarity index 82% rename from octopoes/bits/expiring_certificate/expiring_certificate.py rename to octopoes/nibbles/expiring_certificate/expiring_certificate.py index 5ec4278776c..8d31f4c3e4b 100644 --- a/octopoes/bits/expiring_certificate/expiring_certificate.py +++ b/octopoes/nibbles/expiring_certificate/expiring_certificate.py @@ -1,16 +1,14 @@ import datetime from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.certificate import X509Certificate from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.web import Website THRESHOLD = datetime.timedelta(weeks=2) -def run(input_ooi: X509Certificate, additional_oois: list[Website], config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: X509Certificate) -> Iterator[OOI]: # only applies to OOIs referencing the certificate if input_ooi.expired: ft = KATFindingType(id="KAT-CERTIFICATE-EXPIRED") diff --git a/octopoes/nibbles/expiring_certificate/nibble.py b/octopoes/nibbles/expiring_certificate/nibble.py new file mode 100644 index 00000000000..674e69c0a25 --- /dev/null +++ b/octopoes/nibbles/expiring_certificate/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.certificate import X509Certificate + +NIBBLE = NibbleDefinition(id="expiring-certificate", signature=[NibbleParameter(object_type=X509Certificate)]) diff --git a/octopoes/bits/missing_spf/__init__.py b/octopoes/nibbles/max_url_length_config/__init__.py similarity index 100% rename from octopoes/bits/missing_spf/__init__.py rename to octopoes/nibbles/max_url_length_config/__init__.py diff --git a/octopoes/nibbles/max_url_length_config/max_url_length_config.py b/octopoes/nibbles/max_url_length_config/max_url_length_config.py new file mode 100644 index 00000000000..457624bcb0c --- /dev/null +++ b/octopoes/nibbles/max_url_length_config/max_url_length_config.py @@ -0,0 +1,20 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.web import URL + + +def nibble(url: URL, config: Config) -> Iterator[OOI]: + if "max_length" in config.config: + max_length = int(str(config.config["max_length"])) + if len(str(url.raw)) >= max_length: + ft = KATFindingType(id="URL exceeds configured maximum length") + yield ft + yield Finding( + finding_type=ft.reference, + ooi=url.reference, + proof=f"The length of {url.raw} ({len(str(url.raw))}) exceeds the configured maximum length \ +({max_length}).", + ) diff --git a/octopoes/nibbles/max_url_length_config/nibble.py b/octopoes/nibbles/max_url_length_config/nibble.py new file mode 100644 index 00000000000..dd3fe4cbf55 --- /dev/null +++ b/octopoes/nibbles/max_url_length_config/nibble.py @@ -0,0 +1,31 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.web import URL + + +def query(targets: list[Reference | None]) -> str: + links = list(f'"{target}"' if isinstance(target, Reference) else "" for target in targets) + return f"""{{ + :query {{ + :find [(pull ?var [*])] + :where [ + (or + (and [?var :object_type "URL" ] [?var :URL/primary_key {links[0]}]) + (and [?var :object_type "Config" ] [?var :Config/bit_id "superkat"]\ + [?var :Config/primary_key {links[1]}]) + ) + ] + }} + }} + """ + + +NIBBLE = NibbleDefinition( + id="max_url_length_config", + signature=[ + NibbleParameter(object_type=URL, parser="[*][?object_type == 'URL'][]"), + NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]"), + ], + query=query, +) diff --git a/octopoes/bits/oois_in_headers/__init__.py b/octopoes/nibbles/missing_certificate/__init__.py similarity index 100% rename from octopoes/bits/oois_in_headers/__init__.py rename to octopoes/nibbles/missing_certificate/__init__.py diff --git a/octopoes/bits/missing_certificate/missing_certificate.py b/octopoes/nibbles/missing_certificate/missing_certificate.py similarity index 80% rename from octopoes/bits/missing_certificate/missing_certificate.py rename to octopoes/nibbles/missing_certificate/missing_certificate.py index ae08a2a214e..cb124123bbc 100644 --- a/octopoes/bits/missing_certificate/missing_certificate.py +++ b/octopoes/nibbles/missing_certificate/missing_certificate.py @@ -1,12 +1,11 @@ from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.web import Website -def run(input_ooi: Website, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: Website) -> Iterator[OOI]: if input_ooi.ip_service.tokenized.service.name.lower() != "https": return diff --git a/octopoes/nibbles/missing_certificate/nibble.py b/octopoes/nibbles/missing_certificate/nibble.py new file mode 100644 index 00000000000..f164bde962f --- /dev/null +++ b/octopoes/nibbles/missing_certificate/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.web import Website + +NIBBLE = NibbleDefinition(id="missing-certificate", signature=[NibbleParameter(object_type=Website)]) diff --git a/octopoes/bits/port_common/__init__.py b/octopoes/nibbles/missing_spf/__init__.py similarity index 100% rename from octopoes/bits/port_common/__init__.py rename to octopoes/nibbles/missing_spf/__init__.py diff --git a/octopoes/nibbles/missing_spf/missing_spf.py b/octopoes/nibbles/missing_spf/missing_spf.py new file mode 100644 index 00000000000..71f7b814307 --- /dev/null +++ b/octopoes/nibbles/missing_spf/missing_spf.py @@ -0,0 +1,21 @@ +from collections.abc import Iterator + +import tldextract + +from octopoes.models import OOI +from octopoes.models.ooi.dns.records import NXDOMAIN +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.email_security import DNSSPFRecord +from octopoes.models.ooi.findings import Finding, KATFindingType + + +def nibble(hostname: Hostname, spf_record: DNSSPFRecord | None, nx_domain: NXDOMAIN | None) -> Iterator[OOI]: + if nx_domain: + return + # only report finding when there is no SPF record + if not tldextract.extract(hostname.name).subdomain and tldextract.extract(hostname.name).domain and not spf_record: + ft = KATFindingType(id="KAT-NO-SPF") + yield ft + yield Finding( + ooi=hostname.reference, finding_type=ft.reference, description="This hostname does not have an SPF record" + ) diff --git a/octopoes/nibbles/missing_spf/nibble.py b/octopoes/nibbles/missing_spf/nibble.py new file mode 100644 index 00000000000..68ce5fab7ef --- /dev/null +++ b/octopoes/nibbles/missing_spf/nibble.py @@ -0,0 +1,110 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.records import NXDOMAIN +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.email_security import DNSSPFRecord + + +def spf_query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?hostname [*]) (pull ?spf [*]) (pull ?nx [*])] :where [ + {" ".join(statements)} + ] + }} + }} + """ + + optional_spf = """ + (or + (and + [?spf :object_type "DNSSPFRecord"] + [?spf :DNSSPFRecord/dns_txt_record ?txt] + [?txt :DNSTXTRecord/hostname ?hostname] + ) + (and + [(identity nil) ?spf] + [(identity nil) ?txt] + ) + ) + """ + optional_nx = """ + (or + (and + [?nx :object_type "NXDOMAIN"] + [?nx :NXDOMAIN/hostname ?hostname] + ) + (and + [(identity nil) ?nx] + ) + ) + """ + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "100": + return pull( + [ + f""" + [?hostname :object_type "Hostname"] + [?hostname :Hostname/primary_key "{str(targets[0])}"] + """ + ] + + [optional_spf, optional_nx] + ) + elif sgn == "010": + return pull( + [ + f""" + [?spf :object_type "DNSSPFRecord"] + [?spf :DNSSPFRecord/primary_key "{str(targets[1])}"] + [?spf :DNSSPFRecord/dns_txt_record ?txt] + [?txt :DNSTXTRecord/hostname ?hostname] + """ + ] + + [optional_nx] + ) + elif sgn == "001": + return pull( + [ + f""" + [?nx :object_type "NXDOMAIN"] + [?nx :NXDOMAIN/primary_key "{str(targets[2])}"] + [?nx :NXDOMAIN/hostname ?hostname] + """ + ] + + [optional_spf] + ) + elif sgn == "111": + return pull( + [ + f""" + [?hostname :object_type "Hostname"] + [?hostname :Hostname/primary_key "{str(targets[0])}"] + [?spf :object_type "DNSSPFRecord"] + [?spf :DNSSPFRecord/primary_key "{str(targets[1])}"] + [?nx :object_type "NXDOMAIN"] + [?nx :NXDOMAIN/primary_key "{str(targets[2])}"] + """ + ] + ) + else: + return pull( + [ + """ + [?hostname :object_type "Hostname"] + """ + ] + + [optional_spf, optional_nx] + ) + + +NIBBLE = NibbleDefinition( + id="missing_spf", + signature=[ + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'Hostname'][]"), + NibbleParameter(object_type=DNSSPFRecord, parser="[*][?object_type == 'DNSSPFRecord'][]", optional=True), + NibbleParameter(object_type=NXDOMAIN, parser="[*][?object_type == 'NXDOMAIN'][]", optional=True), + ], + query=spf_query, +) diff --git a/octopoes/bits/spf_discovery/__init__.py b/octopoes/nibbles/oois_in_headers/__init__.py similarity index 100% rename from octopoes/bits/spf_discovery/__init__.py rename to octopoes/nibbles/oois_in_headers/__init__.py diff --git a/octopoes/nibbles/oois_in_headers/nibble.py b/octopoes/nibbles/oois_in_headers/nibble.py new file mode 100644 index 00000000000..d4136b731c9 --- /dev/null +++ b/octopoes/nibbles/oois_in_headers/nibble.py @@ -0,0 +1,116 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.network import Network +from octopoes.models.ooi.web import HTTPHeader + + +def query(targets: list[Reference | None]) -> str: + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "10": + network = str(Network(name=targets[0].split("|")[1]).reference) if targets[0] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeader"] + [?header :HTTPHeader/primary_key "{str(targets[0])}"] + + (or + (and + [?config :Config/ooi "{network}"] + [?config :Config/bit_id "oois-in-headers"] + ) + (and + [(identity nil) ?resource] + [(identity nil) ?url] + [(identity nil) ?network] + [(identity nil) ?config] + ) + ) + + ] + }} + }} + """ + elif sgn == "01": + network = str(Network(name=targets[1].split("|")[1]).reference) if targets[1] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "oois-in-headers"] + + (or + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/web_url ?url] + [?url :HostnameHTTPURL/network "{network}"] + ) + (and + [(identity nil) ?header] + [(identity nil) ?resource] + [(identity nil) ?url] + ) + ) + + ] + }} + }} + """ + elif sgn == "11": + return f""" + {{ + :query {{ + :find [(pull ?header [*]) (pull ?config [*])] :where [ + [?header :object_type "HTTPHeader"] + [?header :HTTPHeader/primary_key "{str(targets[0])}"] + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "oois-in-headers"] + ] + }} + }} + """ + else: + return """ + { + :query { + :find [(pull ?header [*]) (pull ?config [*])] :where [ + + [?header :object_type "HTTPHeader"] + + (or + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/web_url ?url] + [?url :HostnameHTTPURL/network ?network] + [?config :Config/ooi ?network] + [?config :Config/bit_id "oois-in-headers"] + ) + (and + [(identity nil) ?resource] + [(identity nil) ?url] + [(identity nil) ?network] + [(identity nil) ?config] + ) + ) + + ] + } + } + """ + + +NIBBLE = NibbleDefinition( + id="oois-in-headers", + signature=[ + NibbleParameter(object_type=HTTPHeader, parser="[*][?object_type == 'HTTPHeader'][]"), + NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), + ], + query=query, +) diff --git a/octopoes/bits/oois_in_headers/oois_in_headers.py b/octopoes/nibbles/oois_in_headers/oois_in_headers.py similarity index 92% rename from octopoes/bits/oois_in_headers/oois_in_headers.py rename to octopoes/nibbles/oois_in_headers/oois_in_headers.py index 23b5eff3008..1452af414b5 100644 --- a/octopoes/bits/oois_in_headers/oois_in_headers.py +++ b/octopoes/nibbles/oois_in_headers/oois_in_headers.py @@ -1,11 +1,11 @@ import re from collections.abc import Iterator -from typing import Any from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse from pydantic import ValidationError from octopoes.models import OOI +from octopoes.models.ooi.config import Config from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.network import Network from octopoes.models.ooi.web import URL, HTTPHeader, HTTPHeaderHostname, HTTPHeaderURL @@ -36,11 +36,11 @@ def remove_ignored_params(url: str, ignored_params: list[str]) -> str: return new_url -def run(input_ooi: HTTPHeader, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: HTTPHeader, config: Config | None) -> Iterator[OOI]: network = Network(name="internet") if input_ooi.key.lower() == "location": - ignored_url_params = get_ignored_url_params(config, "ignored_url_parameters", []) + ignored_url_params = get_ignored_url_params(config.config if config else {}, "ignored_url_parameters", []) if is_url(input_ooi.value): u = URL(raw=remove_ignored_params(input_ooi.value, ignored_url_params), network=network.reference) else: diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py new file mode 100644 index 00000000000..1a8828a7c6b --- /dev/null +++ b/octopoes/nibbles/runner.py @@ -0,0 +1,225 @@ +import json +from collections.abc import Iterable +from datetime import datetime +from typing import Any + +from xxhash import xxh3_128_hexdigest as xxh3 + +from nibbles.definitions import NibbleDefinition, get_nibble_definitions +from octopoes.models import OOI, Reference +from octopoes.models.origin import Origin, OriginType +from octopoes.repositories.nibble_repository import NibbleRepository +from octopoes.repositories.ooi_repository import OOIRepository +from octopoes.repositories.origin_repository import OriginRepository + + +def merge_results( + d1: dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]], d2: dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]] +) -> dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]]: + """ + Merge new runner results with old runner results + d1: runner_results + d2: runner_results + --> runner_results + """ + return { + key: { + nibble_id: { + arg: set(d1.get(key, {}).get(nibble_id, {}).get(arg, set())) + | set(d2.get(key, {}).get(nibble_id, {}).get(arg, set())) + for arg in set(d1.get(key, {}).get(nibble_id, {}).keys()) + | set(d2.get(key, {}).get(nibble_id, {}).keys()) + } + for nibble_id in set(d1.get(key, {}).keys()) | set(d2.get(key, {}).keys()) + } + for key in set(d1.keys()) | set(d2.keys()) + } + + +def flatten(items: Iterable[Any | Iterable[Any | None] | None]) -> Iterable[OOI]: + """ + Retrieve OOIs as returned from the nibble + """ + for item in items: + if isinstance(item, OOI): + yield item + elif item is None: + continue + elif isinstance(item, Iterable): + yield from flatten(item) + else: + continue + + +def nibble_hasher(data: Iterable, additional: str | None = None) -> str: + """ + Hash the nibble generated data with its content together with the nibble checksum + """ + return xxh3( + "".join( + [ + json.dumps(json.loads(ooi.model_dump_json()), sort_keys=True) + if isinstance(ooi, OOI) + else json.dumps(ooi, sort_keys=True) + for ooi in data + ] + ) + + (additional or "") + ) + + +class NibblesRunner: + def __init__( + self, ooi_repository: OOIRepository, origin_repository: OriginRepository, nibble_repository: NibbleRepository + ): + self.ooi_repository = ooi_repository + self.origin_repository = origin_repository + self.cache: dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]] = {} + self.nibble_repository = nibble_repository + self.nibbles: dict[str, NibbleDefinition] = get_nibble_definitions() + self.federated: bool = False + + def __del__(self): + self._write(datetime.now()) + + def update_nibbles(self, valid_time: datetime, new_nibbles: dict[str, NibbleDefinition] = get_nibble_definitions()): + old_checksums = {nibble.id: nibble._checksum for nibble in self.nibbles.values()} + self.nibbles = new_nibbles + new_checksums = {nibble.id: nibble._checksum for nibble in self.nibbles.values()} + if self.federated: + self.register(valid_time) + updated_nibble_ids = [ + nibble_id + for nibble_id in new_checksums + if nibble_id not in old_checksums or old_checksums[nibble_id] != new_checksums[nibble_id] + ] + self.infer(list(flatten(self.retrieve(updated_nibble_ids, valid_time).values())), valid_time) + + def list_nibbles(self) -> list[str]: + return list(self.nibbles.keys()) + + def list_available_nibbles(self) -> list[str]: + return list(get_nibble_definitions()) + + def disable(self): + self.nibbles = {} + + def register(self, valid_time: datetime = datetime.now()): + self.federated = True + self.nibble_repository.put_many([nibble._ini for nibble in self.nibbles.values()], valid_time) + + def sync(self, valid_time: datetime): + if self.federated: + xtdb_nibble_inis = {ni["id"]: ni for ni in self.nibble_repository.get_all(valid_time)} + for nibble in self.nibbles.values(): + xtdb_nibble_ini = xtdb_nibble_inis[nibble.id] + if xtdb_nibble_ini["enabled"] != nibble.enabled: + self.nibbles[nibble.id].enabled = xtdb_nibble_ini["enabled"] + + def toggle_nibbles(self, nibble_ids: list[str], is_enabled: bool | list[bool], valid_time: datetime): + is_enabled = is_enabled if isinstance(is_enabled, list) else [is_enabled] * len(nibble_ids) + for nibble_id, state in zip(nibble_ids, is_enabled): + self.nibbles[nibble_id].enabled = state + if self.federated: + self.nibble_repository.put_many([self.nibbles[nibble_id]._ini for nibble_id in nibble_ids], valid_time) + + def _retrieve(self, nibble_id: str, valid_time: datetime) -> list[list[Any]]: + nibble = self.nibbles[nibble_id] + if len(nibble.signature) > 1: + return [list(args) for args in self.ooi_repository.nibble_query(None, nibble, valid_time)] + else: + t = nibble.signature[0].object_type + if issubclass(t, OOI): + return [[ooi] for ooi in self.ooi_repository.list_oois_by_object_types({t}, valid_time)] + else: + return [[]] + + def retrieve(self, nibble_ids: list[str] | None, valid_time: datetime) -> dict[str, list[list[Any]]]: + return { + nibble_id: self._retrieve(nibble_id, valid_time) + for nibble_id in (nibble_ids if nibble_ids is not None else self.nibbles) + } + + def yields( + self, nibble_ids: list[str] | None, valid_time: datetime + ) -> dict[str, dict[tuple[Reference | None, ...], list[Reference]]]: + return { + nibble_id: { + tuple(nibblet.parameters_references): nibblet.result + for nibblet in self.origin_repository.list_origins( + valid_time, origin_type=OriginType.NIBBLET, method=nibble_id + ) + if nibblet.parameters_references is not None + } + for nibble_id in (nibble_ids if nibble_ids is not None else self.nibbles) + } + + def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...], set[OOI]]]: + return_value: dict[str, dict[tuple[Any, ...], set[OOI]]] = {} + nibblets = self.origin_repository.list_origins( + valid_time, origin_type=OriginType.NIBBLET, parameters_references=[ooi.reference] + ) + for nibblet in nibblets: + if nibblet.method in self.nibbles: + nibble = self.nibbles[nibblet.method] + args = self.ooi_repository.nibble_query( + ooi, + nibble, + valid_time, + nibblet.parameters_references + if nibble.query is not None and (callable(nibble.query) or isinstance(nibble.query, str)) + else None, + ) + results = { + tuple(arg): set(flatten([nibble(arg)])) + for arg in args + if nibblet.parameters_hash != nibble_hasher(arg, nibble._checksum) + } + return_value |= {nibble.id: results} + nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} + + for nibble in filter( + lambda x: any(isinstance(ooi, param.object_type) for param in x.signature) and x not in nibblet_nibbles, + self.nibbles.values(), + ): + if nibble.enabled: + if len(nibble.signature) > 1: + self._write(valid_time) + args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) + results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} + return_value |= {nibble.id: results} + self.cache = merge_results(self.cache, {ooi: return_value}) + return return_value + + def _write(self, valid_time: datetime): + for source_ooi, results in self.cache.items(): + self.ooi_repository.save(source_ooi, valid_time) + for nibble_id, run_result in results.items(): + for arg, result in run_result.items(): + nibble_origin = Origin( + method=nibble_id, + origin_type=OriginType.NIBBLET, + source=source_ooi.reference, + result=[ooi.reference for ooi in result], + parameters_hash=nibble_hasher(arg, self.nibbles[nibble_id]._checksum), + parameters_references=[a.reference if isinstance(a, OOI) else None for a in arg], + ) + for ooi in result: + self.ooi_repository.save(ooi, valid_time=valid_time) + self.origin_repository.save(nibble_origin, valid_time=valid_time) + self.cache = {} + + def infer(self, stack: list[OOI], valid_time: datetime) -> dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]]: + self.sync(valid_time) + inferences: dict[OOI, dict[str, dict[tuple[Any, ...], set[OOI]]]] = {} + blockset = set(stack) + while stack: + ooi = stack.pop() + results = self._run(ooi, valid_time) + if results: + blocks = set.union(set(), *[ooiset for result in results.values() for _, ooiset in result.items()]) + stack += [o for o in blocks if o not in blockset] + blockset |= blocks + inferences |= {ooi: results} + self._write(valid_time) + return inferences diff --git a/octopoes/bits/url_classification/__init__.py b/octopoes/nibbles/spf_discovery/__init__.py similarity index 100% rename from octopoes/bits/url_classification/__init__.py rename to octopoes/nibbles/spf_discovery/__init__.py diff --git a/octopoes/bits/spf_discovery/internetnl_spf_parser.py b/octopoes/nibbles/spf_discovery/internetnl_spf_parser.py similarity index 100% rename from octopoes/bits/spf_discovery/internetnl_spf_parser.py rename to octopoes/nibbles/spf_discovery/internetnl_spf_parser.py diff --git a/octopoes/nibbles/spf_discovery/nibble.py b/octopoes/nibbles/spf_discovery/nibble.py new file mode 100644 index 00000000000..441932aec7b --- /dev/null +++ b/octopoes/nibbles/spf_discovery/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.dns.records import DNSTXTRecord + +NIBBLE = NibbleDefinition(id="spf-discovery", signature=[NibbleParameter(object_type=DNSTXTRecord)]) diff --git a/octopoes/bits/spf_discovery/spf_discovery.py b/octopoes/nibbles/spf_discovery/spf_discovery.py similarity index 97% rename from octopoes/bits/spf_discovery/spf_discovery.py rename to octopoes/nibbles/spf_discovery/spf_discovery.py index a094cc28cfb..06d675c380c 100644 --- a/octopoes/bits/spf_discovery/spf_discovery.py +++ b/octopoes/nibbles/spf_discovery/spf_discovery.py @@ -1,7 +1,6 @@ from collections.abc import Iterator -from typing import Any -from bits.spf_discovery.internetnl_spf_parser import parse +from nibbles.spf_discovery.internetnl_spf_parser import parse from octopoes.models import OOI from octopoes.models.ooi.dns.records import DNSTXTRecord from octopoes.models.ooi.dns.zone import Hostname @@ -10,7 +9,7 @@ from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, Network -def run(input_ooi: DNSTXTRecord, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(input_ooi: DNSTXTRecord) -> Iterator[OOI]: if input_ooi.value.startswith("v=spf1"): spf_value = input_ooi.value.replace("%(d)", input_ooi.hostname.tokenized.name) parsed = parse(spf_value) diff --git a/octopoes/bits/url_discovery/__init__.py b/octopoes/nibbles/url_classification/__init__.py similarity index 100% rename from octopoes/bits/url_discovery/__init__.py rename to octopoes/nibbles/url_classification/__init__.py diff --git a/octopoes/nibbles/url_classification/nibble.py b/octopoes/nibbles/url_classification/nibble.py new file mode 100644 index 00000000000..798c154f9f5 --- /dev/null +++ b/octopoes/nibbles/url_classification/nibble.py @@ -0,0 +1,4 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models.ooi.web import URL + +NIBBLE = NibbleDefinition(id="url_classification", signature=[NibbleParameter(object_type=URL)]) diff --git a/octopoes/bits/url_classification/url_classification.py b/octopoes/nibbles/url_classification/url_classification.py similarity index 94% rename from octopoes/bits/url_classification/url_classification.py rename to octopoes/nibbles/url_classification/url_classification.py index 76228e1db9b..26a34585f42 100644 --- a/octopoes/bits/url_classification/url_classification.py +++ b/octopoes/nibbles/url_classification/url_classification.py @@ -1,6 +1,5 @@ from collections.abc import Iterator from ipaddress import IPv4Address, ip_address -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.dns.zone import Hostname @@ -8,7 +7,7 @@ from octopoes.models.ooi.web import URL, HostnameHTTPURL, IPAddressHTTPURL, WebScheme -def run(url: URL, additional_oois: list, config: dict[str, Any]) -> Iterator[OOI]: +def nibble(url: URL) -> Iterator[OOI]: if url.raw.scheme == "http" or url.raw.scheme == "https": port = url.raw.port if port is None: diff --git a/octopoes/bits/website_discovery/__init__.py b/octopoes/nibbles/url_discovery/__init__.py similarity index 100% rename from octopoes/bits/website_discovery/__init__.py rename to octopoes/nibbles/url_discovery/__init__.py diff --git a/octopoes/nibbles/url_discovery/nibble.py b/octopoes/nibbles/url_discovery/nibble.py new file mode 100644 index 00000000000..4e23ae39dc2 --- /dev/null +++ b/octopoes/nibbles/url_discovery/nibble.py @@ -0,0 +1,47 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import ResolvedHostname +from octopoes.models.ooi.network import IPPort + + +def url_query(targets: list[Reference | None]) -> str: + links = list(f'"{target}"' if isinstance(target, Reference) else "" for target in targets) + return f"""{{ + :query {{ + :find [(pull ?var [*])] + :where [ + (or + (and + [?var :object_type "ResolvedHostname"] + [?var :ResolvedHostname/address ?ip_address] + [?ip_port :IPPort/address ?ip_address] + (or [?ip_port :IPPort/port 443][?ip_port :IPPort/port 80]) + [?var :ResolvedHostname/primary_key {links[1]}] + [?resolved_hostname :object_type] + ) + (and + [?var :object_type "IPPort"] + [?ip_port :IPPort/address ?ip_address] + (or [?ip_port :IPPort/port 443][?ip_port :IPPort/port 80]) + [?resolved_hostname :object_type "ResolvedHostname"] + [?resolved_hostname :ResolvedHostname/address ?ip_address] + [?var :IPPort/primary_key {links[0]}] + [?ip_port :object_type] + ) + ) + ] + }} + }} + """ + + +NIBBLE = NibbleDefinition( + id="url-discovery", + signature=[ + NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), + NibbleParameter( + object_type=ResolvedHostname, parser="[*][?object_type == 'ResolvedHostname'][]", optional=True + ), + ], + query=url_query, +) diff --git a/octopoes/nibbles/url_discovery/url_discovery.py b/octopoes/nibbles/url_discovery/url_discovery.py new file mode 100644 index 00000000000..52a3e98d7bf --- /dev/null +++ b/octopoes/nibbles/url_discovery/url_discovery.py @@ -0,0 +1,13 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.dns.zone import ResolvedHostname +from octopoes.models.ooi.network import IPPort, Network +from octopoes.models.ooi.web import URL + + +def nibble(ip_port: IPPort, resolved_hostname: ResolvedHostname) -> Iterator[OOI]: + yield URL( + network=Network(name=resolved_hostname.hostname.tokenized.network.name).reference, + raw=f"https://{resolved_hostname.hostname.tokenized.name}/", + ) diff --git a/octopoes/nibbles/website_discovery/__init__.py b/octopoes/nibbles/website_discovery/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/octopoes/nibbles/website_discovery/nibble.py b/octopoes/nibbles/website_discovery/nibble.py new file mode 100644 index 00000000000..93f09821a9b --- /dev/null +++ b/octopoes/nibbles/website_discovery/nibble.py @@ -0,0 +1,50 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import ResolvedHostname +from octopoes.models.ooi.service import IPService + + +def query(targets: list[Reference | None]) -> str: + links = list(f'"{target}"' if isinstance(target, Reference) else "" for target in targets) + return f"""{{ + :query {{ + :find [(pull ?var [*])] + :where [ + (or + (and + [?var :object_type "ResolvedHostname"] + [?var :ResolvedHostname/address ?ip_address] + [?ip_service :object_type "IPService"] + [?ip_service :IPService/ip_port ?ip_port] + [?ip_service :IPService/service ?service] + (or [?service :Service/name "http"][?service :Service/name "https"]) + [?ip_port :IPPort/address ?ip_address] + [?var :ResolvedHostname/primary_key {links[0]}] + [?resolved_hostname :object_type] + ) + (and + [?var :object_type "IPService"] + [?var :IPService/ip_port ?ip_port] + [?var :IPService/service ?service] + (or [?service :Service/name "http"][?service :Service/name "https"]) + [?ip_port :IPPort/address ?ip_address] + [?resolved_hostname :object_type "ResolvedHostname"] + [?resolved_hostname :ResolvedHostname/address ?ip_address] + [?var :IPService/primary_key {links[1]}] + [?ip_service :object_type] + ) + ) + ] + }} + }} + """ + + +NIBBLE = NibbleDefinition( + id="website_discovery", + signature=[ + NibbleParameter(object_type=ResolvedHostname, parser="[*][?object_type == 'ResolvedHostname'][]"), + NibbleParameter(object_type=IPService, parser="[*][?object_type == 'IPService'][]"), + ], + query=query, +) diff --git a/octopoes/nibbles/website_discovery/website_discovery.py b/octopoes/nibbles/website_discovery/website_discovery.py new file mode 100644 index 00000000000..2f16a63f39f --- /dev/null +++ b/octopoes/nibbles/website_discovery/website_discovery.py @@ -0,0 +1,10 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.dns.zone import ResolvedHostname +from octopoes.models.ooi.service import IPService +from octopoes.models.ooi.web import Website + + +def nibble(resolved_hostname: ResolvedHostname, ip_service: IPService) -> Iterator[OOI]: + yield Website(hostname=resolved_hostname.hostname, ip_service=ip_service.reference) diff --git a/octopoes/octopoes/api/router.py b/octopoes/octopoes/api/router.py index 5816d2a2d88..9e7f9cff229 100644 --- a/octopoes/octopoes/api/router.py +++ b/octopoes/octopoes/api/router.py @@ -582,3 +582,75 @@ def migrate_origins( session.add((OperationType.DELETE, origin.id, valid_time)) session.commit() # The save-delete order is important to avoid garbage collection of the results + + +@router.get("/nibbles", tags=["nibbles"]) +def nibbles_list(octopoes: OctopoesService = Depends(octopoes_service)) -> list[str]: + return octopoes.nibbler.list_nibbles() + + +@router.get("/nibbles/status", tags=["nibbles"]) +def nibbles_list_enabled(enabled: bool = True, octopoes: OctopoesService = Depends(octopoes_service)) -> list[str]: + return [nibble.id for nibble in octopoes.nibbler.nibbles.values() if nibble.enabled == enabled] + + +@router.get("/nibbles/update", tags=["nibbles"]) +def nibbles_update( + valid_time: datetime = Depends(extract_valid_time), octopoes: OctopoesService = Depends(octopoes_service) +) -> list[str]: + octopoes.nibbler.update_nibbles(valid_time) + return nibbles_list(octopoes) + + +@router.get("/nibbles/disable", tags=["nibbles"]) +def nibbles_disable(octopoes: OctopoesService = Depends(octopoes_service)) -> list[str]: + if octopoes.nibbler.federated: + octopoes.nibbler.toggle_nibbles(list(octopoes.nibbler.nibbles.keys()), False, datetime.now()) + octopoes.nibbler.disable() + return nibbles_list(octopoes) + + +@router.get("/nibbles/available", tags=["nibbles"]) +def nibbles_available(octopoes: OctopoesService = Depends(octopoes_service)) -> list[str]: + return octopoes.nibbler.list_available_nibbles() + + +@router.get("/nibbles/select", tags=["nibbles"]) +def nibbles_toggle( + state: bool, nibble_ids: list[str] = Query(), octopoes: OctopoesService = Depends(octopoes_service) +) -> list[str]: + octopoes.nibbler.toggle_nibbles(nibble_ids, state, datetime.now()) + return nibbles_list_enabled(octopoes) + + +@router.get("/nibbles/checksum", tags=["nibbles"]) +def nibbles_checksum(octopoes: OctopoesService = Depends(octopoes_service)) -> dict[str, str | None]: + return {nibble.id: nibble._checksum for nibble in octopoes.nibbler.nibbles.values()} + + +@router.get("/nibbles/federate", tags=["nibbles"]) +def nibbles_federate(octopoes: OctopoesService = Depends(octopoes_service)): + octopoes.nibbler.federated = True + + +@router.get("/nibbles/register", tags=["nibbles"]) +def nibbles_register(octopoes: OctopoesService = Depends(octopoes_service)): + octopoes.nibbler.register(datetime.now()) + + +@router.get("/nibbles/retrieve", tags=["nibbles"]) +def nibble_retrieve( + nibble_ids: list[str] | None = Query(None), + valid_time: datetime = Depends(extract_valid_time), + octopoes: OctopoesService = Depends(octopoes_service), +) -> dict[str, list[list[Any]]]: + return octopoes.nibbler.retrieve(nibble_ids, valid_time) + + +@router.get("/nibbles/yields", tags=["nibbles"]) +def nibble_yields( + nibble_ids: list[str] | None = Query(None), + valid_time: datetime = Depends(extract_valid_time), + octopoes: OctopoesService = Depends(octopoes_service), +) -> dict[str, dict[tuple[Reference | None, ...], list[Reference]]]: + return octopoes.nibbler.yields(nibble_ids, valid_time) diff --git a/octopoes/octopoes/core/app.py b/octopoes/octopoes/core/app.py index af61d604f11..a9384634623 100644 --- a/octopoes/octopoes/core/app.py +++ b/octopoes/octopoes/core/app.py @@ -4,6 +4,7 @@ from octopoes.config.settings import GATHER_BIT_METRICS, QUEUE_NAME_OCTOPOES, Settings from octopoes.core.service import OctopoesService from octopoes.events.manager import EventManager, get_rabbit_channel +from octopoes.repositories.nibble_repository import XTDBNibbleRepository from octopoes.repositories.ooi_repository import XTDBOOIRepository from octopoes.repositories.origin_parameter_repository import XTDBOriginParameterRepository from octopoes.repositories.origin_repository import XTDBOriginRepository @@ -37,10 +38,18 @@ def bootstrap_octopoes(settings: Settings, client: str, xtdb_session: XTDBSessio origin_repository = XTDBOriginRepository(event_manager, xtdb_session) origin_param_repository = XTDBOriginParameterRepository(event_manager, xtdb_session) scan_profile_repository = XTDBScanProfileRepository(event_manager, xtdb_session) + nibble_repository = XTDBNibbleRepository(xtdb_session) if GATHER_BIT_METRICS: return OctopoesService( - ooi_repository, origin_repository, origin_param_repository, scan_profile_repository, xtdb_session + ooi_repository, + origin_repository, + origin_param_repository, + scan_profile_repository, + nibble_repository, + xtdb_session, ) else: - return OctopoesService(ooi_repository, origin_repository, origin_param_repository, scan_profile_repository) + return OctopoesService( + ooi_repository, origin_repository, origin_param_repository, scan_profile_repository, nibble_repository + ) diff --git a/octopoes/octopoes/core/service.py b/octopoes/octopoes/core/service.py index ce019172c25..df1a71a2846 100644 --- a/octopoes/octopoes/core/service.py +++ b/octopoes/octopoes/core/service.py @@ -8,6 +8,7 @@ import structlog from bits.definitions import get_bit_definitions from bits.runner import BitRunner +from nibbles.runner import NibblesRunner from pydantic import TypeAdapter from octopoes.config.settings import ( @@ -41,6 +42,7 @@ ) from octopoes.models.transaction import TransactionRecord from octopoes.models.tree import ReferenceTree +from octopoes.repositories.nibble_repository import NibbleRepository from octopoes.repositories.ooi_repository import OOIRepository from octopoes.repositories.origin_parameter_repository import OriginParameterRepository from octopoes.repositories.origin_repository import OriginRepository @@ -70,12 +72,14 @@ def __init__( origin_repository: OriginRepository, origin_parameter_repository: OriginParameterRepository, scan_profile_repository: ScanProfileRepository, + nibble_repository: NibbleRepository, session: XTDBSession | None = None, ): self.ooi_repository = ooi_repository self.origin_repository = origin_repository self.origin_parameter_repository = origin_parameter_repository self.scan_profile_repository = scan_profile_repository + self.nibbler = NibblesRunner(ooi_repository, origin_repository, nibble_repository) self.session = session @overload @@ -149,17 +153,23 @@ def get_ooi_tree( return tree def _delete_ooi(self, reference: Reference, valid_time: datetime) -> None: - referencing_origins = self.origin_repository.list_origins(valid_time, result=reference) if not any( origin - for origin in referencing_origins - if not ( - origin.origin_type == OriginType.AFFIRMATION - or (origin.origin_type == OriginType.INFERENCE and origin.source == reference) + for origin in self.origin_repository.list_origins( + valid_time, + origin_type={OriginType.DECLARATION, OriginType.OBSERVATION, OriginType.INFERENCE, OriginType.NIBBLET}, + result=reference, ) + if not (origin.origin_type in [OriginType.INFERENCE, OriginType.NIBBLET] and origin.source == reference) ): self.ooi_repository.delete_if_exists(reference, valid_time) + # Clear out affirmation dangling objects a bit later + residue = self.origin_repository.list_origins(valid_time, result=reference) + if all(map(lambda x: x.origin_type == OriginType.AFFIRMATION, residue)): + for res in residue: + self.origin_repository.delete(res, valid_time) + def save_origin( self, origin: Origin, oois: list[OOI], valid_time: datetime, end_valid_time: datetime | None = None ) -> None: @@ -170,10 +180,10 @@ def save_origin( self.ooi_repository.get(origin.source, valid_time) except ObjectNotFoundException: if ( - origin.origin_type not in [OriginType.DECLARATION, OriginType.AFFIRMATION] + origin.origin_type not in [OriginType.DECLARATION, OriginType.AFFIRMATION, OriginType.NIBBLET] and origin.source not in origin.result ): - raise ValueError("Origin source of observation does not exist") + raise ValueError(f"Origin source [{origin.source}] does not exist") elif origin.origin_type == OriginType.AFFIRMATION: logger.debug("Affirmation source %s already deleted", origin.source) return @@ -181,11 +191,14 @@ def save_origin( if origin.origin_type == OriginType.AFFIRMATION and not any( other_origin for other_origin in self.origin_repository.list_origins( - origin_type={OriginType.DECLARATION, OriginType.OBSERVATION, OriginType.INFERENCE}, + origin_type={OriginType.DECLARATION, OriginType.OBSERVATION, OriginType.INFERENCE, OriginType.NIBBLET}, valid_time=valid_time, result=origin.source, ) - if not (other_origin.origin_type == OriginType.INFERENCE and [other_origin.source] == other_origin.result) + if not ( + other_origin.origin_type in [OriginType.INFERENCE, OriginType.NIBBLET] + and [other_origin.source] == other_origin.result + ) ): logger.debug("Affirmation source %s seems dangling, deleting", origin.source) self.ooi_repository.delete_if_exists(origin.source, valid_time) @@ -196,10 +209,11 @@ def save_origin( self.origin_repository.save(origin, valid_time=valid_time) # Origins that are stale need to be deleted. #3561 - if not origin.result and origin.origin_type != OriginType.INFERENCE: + if not origin.result and origin.origin_type not in [OriginType.INFERENCE, OriginType.NIBBLET]: self.origin_repository.delete(origin, valid_time=valid_time) def _run_inference(self, origin: Origin, valid_time: datetime) -> None: + # The bit part of inferring bit_definition = get_bit_definitions().get(origin.method, None) if bit_definition is None: @@ -234,6 +248,7 @@ def _run_inference(self, origin: Origin, valid_time: datetime) -> None: if len(configs) != 0: config = configs[-1].config + resulting_oois: list[OOI] = [] try: if isinstance(self.session, XTDBSession): start = perf_counter() @@ -252,10 +267,11 @@ def _run_inference(self, origin: Origin, valid_time: datetime) -> None: self.session.client.submit_transaction(ops) else: resulting_oois = BitRunner(bit_definition).run(source, parameters, config=config) - self.save_origin(origin, resulting_oois, valid_time) except Exception as e: logger.exception("Error running inference", exc_info=e) + self.save_origin(origin, resulting_oois, valid_time) + @staticmethod def check_path_level(path_level: int | None, current_level: int) -> bool: return path_level is not None and path_level >= current_level @@ -407,6 +423,12 @@ def _on_create_ooi(self, event: OOIDBEvent) -> None: None, EmptyScanProfile(reference=ooi.reference), valid_time=event.valid_time ) + # The nibble part of inferring + try: + self.nibbler.infer([self.ooi_repository.get(ooi.reference, event.valid_time)], event.valid_time) + except ObjectNotFoundException: + logger.info("OOI not found for inference") + # analyze bit definitions bit_definitions = get_bit_definitions() for bit_id, bit_definition in bit_definitions.items(): @@ -439,6 +461,12 @@ def _on_update_ooi(self, event: OOIDBEvent) -> None: if event.new_data is None: raise ValueError("Update event new_data should not be None") + # The nibble part of inferring + try: + self.nibbler.infer([self.ooi_repository.get(event.new_data.reference, event.valid_time)], event.valid_time) + except ObjectNotFoundException: + logger.info("OOI not found for inference") + if isinstance(event.new_data, Config): relevant_bit_ids = [ bit.id for bit in get_bit_definitions().values() if bit.config_ooi_relation_path is not None @@ -464,6 +492,9 @@ def _on_delete_ooi(self, event: OOIDBEvent) -> None: # delete related origins to which it is a source origins = self.origin_repository.list_origins(event.valid_time, source=reference) + origins += self.origin_repository.list_origins( + event.valid_time, origin_type=OriginType.NIBBLET, parameters_references=[reference] + ) for origin in origins: self.origin_repository.delete(origin, event.valid_time) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index 362b4c0d254..31e9ea82925 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Iterable from enum import Enum, IntEnum from typing import Any, ClassVar, Literal, TypeAlias, TypeVar @@ -240,7 +241,17 @@ def _serialize_value(self, value: Any, required: bool) -> SerializedOOIValue: return str(value) def __hash__(self): - return hash(self.primary_key) + def freeze(items: Iterable[Any | Iterable[Any | None] | None]) -> Iterable[int]: + for item in items: + if isinstance(item, Iterable) and not (isinstance(item, str | bytes)): + if isinstance(item, dict): + yield from freeze(item.items()) + else: + yield from freeze(item) + else: + yield hash(item) + + return hash(tuple(freeze(self.model_dump().items()))) OOIClassType = TypeVar("OOIClassType") diff --git a/octopoes/octopoes/models/origin.py b/octopoes/octopoes/models/origin.py index 755fdf98b4c..889248e69d0 100644 --- a/octopoes/octopoes/models/origin.py +++ b/octopoes/octopoes/models/origin.py @@ -13,6 +13,7 @@ class OriginType(Enum): OBSERVATION = "observation" INFERENCE = "inference" AFFIRMATION = "affirmation" + NIBBLET = "nibblet" class Origin(BaseModel): @@ -21,6 +22,8 @@ class Origin(BaseModel): source: Reference source_method: str | None = None # None for bits and normalizers result: list[Reference] = Field(default_factory=list) + parameters_hash: str | None = None # None for anything other than nibblet + parameters_references: list[Reference | None] | None = None # None for anything other than nibblet task_id: UUID | None = None def __sub__(self, other: Origin) -> set[Reference]: @@ -31,7 +34,21 @@ def __sub__(self, other: Origin) -> set[Reference]: @property def id(self) -> str: - if self.source_method is not None: + if self.origin_type == OriginType.NIBBLET: + return "|".join( + map( + str, + [ + self.__class__.__name__, + self.origin_type.value, + self.method, + f"[{','.join(sorted([str(param) for param in self.parameters_references]))}]" + if self.parameters_references is not None + else "Null", + ], + ) + ) + elif self.source_method is not None: return ( f"{self.__class__.__name__}|{self.origin_type.value}|{self.method}|{self.source_method}|{self.source}" ) diff --git a/octopoes/octopoes/repositories/nibble_repository.py b/octopoes/octopoes/repositories/nibble_repository.py new file mode 100644 index 00000000000..14e14c89bc5 --- /dev/null +++ b/octopoes/octopoes/repositories/nibble_repository.py @@ -0,0 +1,73 @@ +from datetime import datetime +from typing import Any + +from octopoes.models.transaction import TransactionRecord +from octopoes.repositories.repository import Repository +from octopoes.xtdb.client import XTDBSession + +NibbleINI = dict[str, Any] + + +class NibbleRepository(Repository): + def get(self, nibble_id: str, valid_time: datetime) -> NibbleINI: + raise NotImplementedError + + def get_all(self, valid_time: datetime) -> list[NibbleINI]: + raise NotImplementedError + + def put(self, ini: NibbleINI, valid_time: datetime): + raise NotImplementedError + + def put_many(self, inis: list[NibbleINI], valid_time: datetime): + raise NotImplementedError + + def history(self, nibble_id: str, with_docs: bool = False) -> list[TransactionRecord]: + raise NotImplementedError + + +class XTDBNibbleRepository(NibbleRepository): + def __init__(self, session: XTDBSession): + self.session = session + + @classmethod + def _xtid(cls, nibble_id: str) -> str: + return f"NibbleINI|{nibble_id}" + + @classmethod + def _serialize(cls, ini: NibbleINI) -> NibbleINI: + ini["type"] = "NibbleINI" + ini["xt/id"] = cls._xtid(ini["id"]) + return ini + + @classmethod + def _deserialize(cls, ini: NibbleINI) -> NibbleINI: + ini.pop("type", None) + ini.pop("xt/id", None) + return ini + + def get(self, nibble_id: str, valid_time: datetime) -> NibbleINI: + return self._deserialize(self.session.client.get_entity(self._xtid(nibble_id), valid_time)) + + def get_all(self, valid_time: datetime) -> list[NibbleINI]: + result = self.session.client.query( + '{:query {:find [(pull ?var [*])] :where [[?var :type "NibbleINI"]]}}', valid_time + ) + return [self._deserialize(item[0]) for item in result] + + def put(self, ini: NibbleINI, valid_time: datetime): + self.session.put(self._serialize(ini), valid_time) + self.commit() + + def put_many(self, inis: list[NibbleINI], valid_time: datetime): + for ini in inis: + self.session.put(self._serialize(ini), valid_time) + self.commit() + + def history(self, nibble_id: str, with_docs: bool = False) -> list[TransactionRecord]: + return self.session.client.get_entity_history(self._xtid(nibble_id), with_docs=with_docs) + + def status(self): + return self.session.client.status() + + def commit(self): + self.session.commit() diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index 7c432b6250a..c171c42c19a 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -3,12 +3,16 @@ import json import re from collections import Counter +from collections.abc import Iterable from datetime import datetime +from itertools import product from typing import Any, Literal, cast import structlog from bits.definitions import BitDefinition from httpx import HTTPStatusError, codes +from jmespath import search +from nibbles.definitions import NibbleDefinition from pydantic import RootModel, TypeAdapter from octopoes.config.settings import ( @@ -159,7 +163,16 @@ def get_bit_configs(self, source: OOI, bit_definition: BitDefinition, valid_time def list_related(self, ooi: OOI, path: Path, valid_time: datetime) -> list[OOI]: raise NotImplementedError - def query(self, query: Query, valid_time: datetime) -> list[OOI | tuple]: + def query(self, query: str | Query, valid_time: datetime) -> list[OOI | tuple]: + raise NotImplementedError + + def nibble_query( + self, + ooi: OOI | None, + nibble: NibbleDefinition, + valid_time: datetime, + arguments: list[Reference | None] | None = None, + ) -> Iterable[Iterable[Any]]: raise NotImplementedError @@ -237,7 +250,7 @@ def serialize(cls, ooi: OOI) -> dict[str, Any]: @classmethod def deserialize(cls, data: dict[str, Any]) -> OOI: if "object_type" not in data: - raise ValueError("Data is missing object_type") + raise ValueError("OOI data is missing object_type") # pop global attributes object_cls = type_by_name(data.pop("object_type")) @@ -249,6 +262,36 @@ def deserialize(cls, data: dict[str, Any]) -> OOI: stripped["user_id"] = user_id return object_cls.model_validate(stripped) + @classmethod + def parse_as(cls, type_: type | list[type], obj: dict | list | set | Any) -> tuple | frozenset | Any: + """ + parse_as takes a type and a serialized object and tries to turn it into the supplied type or the closest + hashable relative + for instance: + parse_as(int, "6") -> 6 + parse_as(str, "6") -> "6" + see test_ooi_repository.py for more examples + """ + if isinstance(obj, dict): + # assume the dictionary is the serialized object + if isinstance(type_, list): + type_ = type_[0] + if issubclass(type_, OOI): + return cls.deserialize(obj) + else: + return type_(**obj) + elif isinstance(obj, list): + # list --> tuple + if isinstance(type_, list): + return tuple(cls.parse_as(type_t, o) for o, type_t in zip(obj, type_)) + else: + return tuple(cls.parse_as(type_, o) for o in obj) + else: + # assume a simple type + if isinstance(type_, list): + type_ = type_[0] + return type_(obj) + def get(self, reference: Reference, valid_time: datetime) -> OOI: try: res = self.session.client.get_entity(str(reference), valid_time) @@ -860,3 +903,37 @@ def query(self, query: str | Query, valid_time: datetime) -> list[OOI | tuple]: parsed_results.append(tuple(parsed_result)) return parsed_results + + def nibble_query( + self, + ooi: OOI | None, + nibble: NibbleDefinition, + valid_time: datetime, + arguments: list[Reference | None] | None = None, + ) -> Iterable[Iterable[Any]]: + if nibble.query is None: + return [{ooi}] + else: + if arguments is None: + if ooi is not None: + first = True + arguments = [ + ooi.reference + if sgn.object_type == type_by_name(ooi.get_ooi_type()) and (first and not (first := False)) + else None + for sgn in nibble.signature + ] + else: + arguments = [None for _ in nibble.signature] + query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) + data = self.session.client.query(query, valid_time) + objects = [ + {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} + for element in nibble.signature + ] + objects = [ + obj if obj else ({None} if element.optional else set()) + for obj, element in zip(objects, nibble.signature) + ] + + return list(product(*objects)) diff --git a/octopoes/octopoes/repositories/origin_repository.py b/octopoes/octopoes/repositories/origin_repository.py index fa7c102b6f7..b1cded50afb 100644 --- a/octopoes/octopoes/repositories/origin_repository.py +++ b/octopoes/octopoes/repositories/origin_repository.py @@ -37,6 +37,8 @@ def list_origins( source: Reference | None = None, result: Reference | None = None, method: str | list[str] | None = None, + parameters_hash: int | None = None, + parameters_references: list[Reference] | None = None, origin_type: OriginType | list[OriginType] | set[OriginType] | None = None, ) -> list[Origin]: raise NotImplementedError @@ -77,6 +79,8 @@ def list_origins( source: Reference | None = None, result: Reference | None = None, method: str | list[str] | None = None, + parameters_hash: int | None = None, + parameters_references: list[Reference] | None = None, origin_type: OriginType | list[OriginType] | set[OriginType] | None = None, ) -> list[Origin]: where_parameters: dict[str, str | list[str]] = {"type": Origin.__name__} @@ -93,6 +97,12 @@ def list_origins( if method: where_parameters["method"] = method + if parameters_hash: + where_parameters["parameters_hash"] = str(parameters_hash) + + if parameters_references: + where_parameters["parameters_references"] = [str(pr) for pr in parameters_references] + if origin_type: if isinstance(origin_type, OriginType): where_parameters["origin_type"] = origin_type.value diff --git a/octopoes/poetry.lock b/octopoes/poetry.lock index a928a694044..41925cfb6d9 100644 --- a/octopoes/poetry.lock +++ b/octopoes/poetry.lock @@ -359,73 +359,73 @@ files = [ [[package]] name = "coverage" -version = "7.6.9" +version = "7.6.10" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" files = [ - {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, - {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1"}, - {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5"}, - {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073"}, - {file = "coverage-7.6.9-cp310-cp310-win32.whl", hash = "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198"}, - {file = "coverage-7.6.9-cp310-cp310-win_amd64.whl", hash = "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9"}, - {file = "coverage-7.6.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9"}, - {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b"}, - {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3"}, - {file = "coverage-7.6.9-cp311-cp311-win32.whl", hash = "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0"}, - {file = "coverage-7.6.9-cp311-cp311-win_amd64.whl", hash = "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8"}, - {file = "coverage-7.6.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"}, - {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6"}, - {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f"}, - {file = "coverage-7.6.9-cp312-cp312-win32.whl", hash = "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692"}, - {file = "coverage-7.6.9-cp312-cp312-win_amd64.whl", hash = "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664"}, - {file = "coverage-7.6.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00"}, - {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077"}, - {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb"}, - {file = "coverage-7.6.9-cp313-cp313-win32.whl", hash = "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba"}, - {file = "coverage-7.6.9-cp313-cp313-win_amd64.whl", hash = "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419"}, - {file = "coverage-7.6.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae"}, - {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e"}, - {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9"}, - {file = "coverage-7.6.9-cp313-cp313t-win32.whl", hash = "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b"}, - {file = "coverage-7.6.9-cp313-cp313t-win_amd64.whl", hash = "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611"}, - {file = "coverage-7.6.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:adb697c0bd35100dc690de83154627fbab1f4f3c0386df266dded865fc50a902"}, - {file = "coverage-7.6.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:be57b6d56e49c2739cdf776839a92330e933dd5e5d929966fbbd380c77f060be"}, - {file = "coverage-7.6.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1592791f8204ae9166de22ba7e6705fa4ebd02936c09436a1bb85aabca3e599"}, - {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e12ae8cc979cf83d258acb5e1f1cf2f3f83524d1564a49d20b8bec14b637f08"}, - {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb5555cff66c4d3d6213a296b360f9e1a8e323e74e0426b6c10ed7f4d021e464"}, - {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b9389a429e0e5142e69d5bf4a435dd688c14478a19bb901735cdf75e57b13845"}, - {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:592ac539812e9b46046620341498caf09ca21023c41c893e1eb9dbda00a70cbf"}, - {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a27801adef24cc30871da98a105f77995e13a25a505a0161911f6aafbd66e678"}, - {file = "coverage-7.6.9-cp39-cp39-win32.whl", hash = "sha256:8e3c3e38930cfb729cb8137d7f055e5a473ddaf1217966aa6238c88bd9fd50e6"}, - {file = "coverage-7.6.9-cp39-cp39-win_amd64.whl", hash = "sha256:e28bf44afa2b187cc9f41749138a64435bf340adfcacb5b2290c070ce99839d4"}, - {file = "coverage-7.6.9-pp39.pp310-none-any.whl", hash = "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b"}, - {file = "coverage-7.6.9.tar.gz", hash = "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78"}, + {file = "coverage-7.6.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165"}, + {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3"}, + {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5"}, + {file = "coverage-7.6.10-cp310-cp310-win32.whl", hash = "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244"}, + {file = "coverage-7.6.10-cp310-cp310-win_amd64.whl", hash = "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3"}, + {file = "coverage-7.6.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f"}, + {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd"}, + {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377"}, + {file = "coverage-7.6.10-cp311-cp311-win32.whl", hash = "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8"}, + {file = "coverage-7.6.10-cp311-cp311-win_amd64.whl", hash = "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853"}, + {file = "coverage-7.6.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50"}, + {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0"}, + {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852"}, + {file = "coverage-7.6.10-cp312-cp312-win32.whl", hash = "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359"}, + {file = "coverage-7.6.10-cp312-cp312-win_amd64.whl", hash = "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9"}, + {file = "coverage-7.6.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18"}, + {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e"}, + {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694"}, + {file = "coverage-7.6.10-cp313-cp313-win32.whl", hash = "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6"}, + {file = "coverage-7.6.10-cp313-cp313-win_amd64.whl", hash = "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe"}, + {file = "coverage-7.6.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098"}, + {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf"}, + {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2"}, + {file = "coverage-7.6.10-cp313-cp313t-win32.whl", hash = "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312"}, + {file = "coverage-7.6.10-cp313-cp313t-win_amd64.whl", hash = "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d"}, + {file = "coverage-7.6.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a"}, + {file = "coverage-7.6.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27"}, + {file = "coverage-7.6.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4"}, + {file = "coverage-7.6.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f"}, + {file = "coverage-7.6.10-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25"}, + {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315"}, + {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90"}, + {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d"}, + {file = "coverage-7.6.10-cp39-cp39-win32.whl", hash = "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18"}, + {file = "coverage-7.6.10-cp39-cp39-win_amd64.whl", hash = "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59"}, + {file = "coverage-7.6.10-pp39.pp310-none-any.whl", hash = "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"}, + {file = "coverage-7.6.10.tar.gz", hash = "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23"}, ] [package.dependencies] @@ -796,6 +796,16 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jmespath-community" +version = "1.1.3" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath_community-1.1.3-py3-none-any.whl", hash = "sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -2183,6 +2193,138 @@ files = [ {file = "wrapt-1.17.0.tar.gz", hash = "sha256:16187aa2317c731170a88ef35e8937ae0f533c402872c1ee5e6d079fcf320801"}, ] +[[package]] +name = "xxhash" +version = "3.5.0" +description = "Python binding for xxHash" +optional = false +python-versions = ">=3.7" +files = [ + {file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"}, + {file = "xxhash-3.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196"}, + {file = "xxhash-3.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198"}, + {file = "xxhash-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442"}, + {file = "xxhash-3.5.0-cp310-cp310-win32.whl", hash = "sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da"}, + {file = "xxhash-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9"}, + {file = "xxhash-3.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6"}, + {file = "xxhash-3.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1"}, + {file = "xxhash-3.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a"}, + {file = "xxhash-3.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d"}, + {file = "xxhash-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839"}, + {file = "xxhash-3.5.0-cp311-cp311-win32.whl", hash = "sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da"}, + {file = "xxhash-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58"}, + {file = "xxhash-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3"}, + {file = "xxhash-3.5.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00"}, + {file = "xxhash-3.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6"}, + {file = "xxhash-3.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab"}, + {file = "xxhash-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e"}, + {file = "xxhash-3.5.0-cp312-cp312-win32.whl", hash = "sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8"}, + {file = "xxhash-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e"}, + {file = "xxhash-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2"}, + {file = "xxhash-3.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6"}, + {file = "xxhash-3.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb"}, + {file = "xxhash-3.5.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7"}, + {file = "xxhash-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c"}, + {file = "xxhash-3.5.0-cp313-cp313-win32.whl", hash = "sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637"}, + {file = "xxhash-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43"}, + {file = "xxhash-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b"}, + {file = "xxhash-3.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7"}, + {file = "xxhash-3.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d"}, + {file = "xxhash-3.5.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737"}, + {file = "xxhash-3.5.0-cp37-cp37m-win32.whl", hash = "sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306"}, + {file = "xxhash-3.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602"}, + {file = "xxhash-3.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f"}, + {file = "xxhash-3.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec"}, + {file = "xxhash-3.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91"}, + {file = "xxhash-3.5.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd"}, + {file = "xxhash-3.5.0-cp38-cp38-win32.whl", hash = "sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4"}, + {file = "xxhash-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3"}, + {file = "xxhash-3.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301"}, + {file = "xxhash-3.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754"}, + {file = "xxhash-3.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af"}, + {file = "xxhash-3.5.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606"}, + {file = "xxhash-3.5.0-cp39-cp39-win32.whl", hash = "sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4"}, + {file = "xxhash-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558"}, + {file = "xxhash-3.5.0-cp39-cp39-win_arm64.whl", hash = "sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b"}, + {file = "xxhash-3.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57"}, + {file = "xxhash-3.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81"}, + {file = "xxhash-3.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0"}, + {file = "xxhash-3.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240"}, + {file = "xxhash-3.5.0.tar.gz", hash = "sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f"}, +] + [[package]] name = "zipp" version = "3.21.0" @@ -2205,4 +2347,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a78ea191478327e380f113c51948610d797c33de48968295efcff71c3a53d6b2" +content-hash = "e41ab6fe70fe531c50ec680a5620b21e7b5fac4eb45b9841eff945234a445ff2" diff --git a/octopoes/pyproject.toml b/octopoes/pyproject.toml index c993c0d2d94..b682f7c4501 100644 --- a/octopoes/pyproject.toml +++ b/octopoes/pyproject.toml @@ -44,6 +44,8 @@ opentelemetry-util-http = "^0.50b0" fastapi-slim = "^0.115.2" structlog = "^24.2.0" asgiref = "^3.8.1" +jmespath-community = "^1.1.3" +xxhash = "^3.5.0" [tool.poetry.group.dev.dependencies] robotframework = "^7.0" diff --git a/octopoes/requirements-dev.txt b/octopoes/requirements-dev.txt index 0e2d0a6bb05..4162d46aa68 100644 --- a/octopoes/requirements-dev.txt +++ b/octopoes/requirements-dev.txt @@ -133,69 +133,69 @@ click==8.1.8 ; python_version >= "3.10" and python_version < "4.0" \ colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and (sys_platform == "win32" or platform_system == "Windows") \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -coverage[toml]==7.6.9 ; python_version >= "3.10" and python_version < "4.0" \ - --hash=sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4 \ - --hash=sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c \ - --hash=sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f \ - --hash=sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b \ - --hash=sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6 \ - --hash=sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae \ - --hash=sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692 \ - --hash=sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4 \ - --hash=sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4 \ - --hash=sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717 \ - --hash=sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d \ - --hash=sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198 \ - --hash=sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1 \ - --hash=sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3 \ - --hash=sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb \ - --hash=sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d \ - --hash=sha256:4e12ae8cc979cf83d258acb5e1f1cf2f3f83524d1564a49d20b8bec14b637f08 \ - --hash=sha256:592ac539812e9b46046620341498caf09ca21023c41c893e1eb9dbda00a70cbf \ - --hash=sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b \ - --hash=sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710 \ - --hash=sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c \ - --hash=sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae \ - --hash=sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077 \ - --hash=sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00 \ - --hash=sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb \ - --hash=sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664 \ - --hash=sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014 \ - --hash=sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9 \ - --hash=sha256:8e3c3e38930cfb729cb8137d7f055e5a473ddaf1217966aa6238c88bd9fd50e6 \ - --hash=sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e \ - --hash=sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9 \ - --hash=sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa \ - --hash=sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611 \ - --hash=sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b \ - --hash=sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a \ - --hash=sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8 \ - --hash=sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030 \ - --hash=sha256:a27801adef24cc30871da98a105f77995e13a25a505a0161911f6aafbd66e678 \ - --hash=sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015 \ - --hash=sha256:adb697c0bd35100dc690de83154627fbab1f4f3c0386df266dded865fc50a902 \ - --hash=sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97 \ - --hash=sha256:b9389a429e0e5142e69d5bf4a435dd688c14478a19bb901735cdf75e57b13845 \ - --hash=sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419 \ - --hash=sha256:bb5555cff66c4d3d6213a296b360f9e1a8e323e74e0426b6c10ed7f4d021e464 \ - --hash=sha256:be57b6d56e49c2739cdf776839a92330e933dd5e5d929966fbbd380c77f060be \ - --hash=sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9 \ - --hash=sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7 \ - --hash=sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be \ - --hash=sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1 \ - --hash=sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba \ - --hash=sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5 \ - --hash=sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073 \ - --hash=sha256:e28bf44afa2b187cc9f41749138a64435bf340adfcacb5b2290c070ce99839d4 \ - --hash=sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a \ - --hash=sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a \ - --hash=sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3 \ - --hash=sha256:f1592791f8204ae9166de22ba7e6705fa4ebd02936c09436a1bb85aabca3e599 \ - --hash=sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0 \ - --hash=sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b \ - --hash=sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec \ - --hash=sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1 \ - --hash=sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3 +coverage[toml]==7.6.10 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9 \ + --hash=sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f \ + --hash=sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273 \ + --hash=sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994 \ + --hash=sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e \ + --hash=sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50 \ + --hash=sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e \ + --hash=sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e \ + --hash=sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c \ + --hash=sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853 \ + --hash=sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8 \ + --hash=sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8 \ + --hash=sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe \ + --hash=sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165 \ + --hash=sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb \ + --hash=sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59 \ + --hash=sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609 \ + --hash=sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18 \ + --hash=sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098 \ + --hash=sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd \ + --hash=sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3 \ + --hash=sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43 \ + --hash=sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d \ + --hash=sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359 \ + --hash=sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90 \ + --hash=sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78 \ + --hash=sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a \ + --hash=sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99 \ + --hash=sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988 \ + --hash=sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2 \ + --hash=sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0 \ + --hash=sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694 \ + --hash=sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377 \ + --hash=sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d \ + --hash=sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23 \ + --hash=sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312 \ + --hash=sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf \ + --hash=sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6 \ + --hash=sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b \ + --hash=sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c \ + --hash=sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690 \ + --hash=sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a \ + --hash=sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f \ + --hash=sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4 \ + --hash=sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25 \ + --hash=sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd \ + --hash=sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852 \ + --hash=sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0 \ + --hash=sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244 \ + --hash=sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315 \ + --hash=sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078 \ + --hash=sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0 \ + --hash=sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27 \ + --hash=sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132 \ + --hash=sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5 \ + --hash=sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247 \ + --hash=sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022 \ + --hash=sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b \ + --hash=sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3 \ + --hash=sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18 \ + --hash=sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5 \ + --hash=sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f deprecated==1.2.15 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:353bc4a8ac4bfc96800ddab349d89c25dec1079f65fd53acdcc1e0b975b21320 \ --hash=sha256:683e561a90de76239796e6b6feac66b99030d2dd3fcf61ef996330f14bbb9b0d @@ -362,6 +362,8 @@ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 +jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonpatch==1.33 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade \ --hash=sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c @@ -940,6 +942,130 @@ wrapt==1.17.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:f335579a1b485c834849e9075191c9898e0731af45705c2ebf70e0cd5d58beed \ --hash=sha256:fce6fee67c318fdfb7f285c29a82d84782ae2579c0e1b385b7f36c6e8074fffb \ --hash=sha256:fd136bb85f4568fffca995bd3c8d52080b1e5b225dbf1c2b17b66b4c5fa02838 +xxhash==3.5.0 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1 \ + --hash=sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837 \ + --hash=sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb \ + --hash=sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84 \ + --hash=sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd \ + --hash=sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131 \ + --hash=sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622 \ + --hash=sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10 \ + --hash=sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da \ + --hash=sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166 \ + --hash=sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415 \ + --hash=sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57 \ + --hash=sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00 \ + --hash=sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d \ + --hash=sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3 \ + --hash=sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c \ + --hash=sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514 \ + --hash=sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558 \ + --hash=sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54 \ + --hash=sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2 \ + --hash=sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692 \ + --hash=sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c \ + --hash=sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b \ + --hash=sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af \ + --hash=sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520 \ + --hash=sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd \ + --hash=sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644 \ + --hash=sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6 \ + --hash=sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81 \ + --hash=sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3 \ + --hash=sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c \ + --hash=sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2 \ + --hash=sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf \ + --hash=sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6 \ + --hash=sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b \ + --hash=sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482 \ + --hash=sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7 \ + --hash=sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6 \ + --hash=sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4 \ + --hash=sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9 \ + --hash=sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637 \ + --hash=sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2 \ + --hash=sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9 \ + --hash=sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da \ + --hash=sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23 \ + --hash=sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee \ + --hash=sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b \ + --hash=sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4 \ + --hash=sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8 \ + --hash=sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa \ + --hash=sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898 \ + --hash=sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793 \ + --hash=sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da \ + --hash=sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43 \ + --hash=sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c \ + --hash=sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88 \ + --hash=sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade \ + --hash=sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa \ + --hash=sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833 \ + --hash=sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e \ + --hash=sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90 \ + --hash=sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f \ + --hash=sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6 \ + --hash=sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680 \ + --hash=sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da \ + --hash=sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306 \ + --hash=sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1 \ + --hash=sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc \ + --hash=sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43 \ + --hash=sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c \ + --hash=sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91 \ + --hash=sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f \ + --hash=sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6 \ + --hash=sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a \ + --hash=sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7 \ + --hash=sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198 \ + --hash=sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623 \ + --hash=sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839 \ + --hash=sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5 \ + --hash=sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9 \ + --hash=sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0 \ + --hash=sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6 \ + --hash=sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec \ + --hash=sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754 \ + --hash=sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c \ + --hash=sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e \ + --hash=sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084 \ + --hash=sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d \ + --hash=sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d \ + --hash=sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240 \ + --hash=sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58 \ + --hash=sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442 \ + --hash=sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326 \ + --hash=sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301 \ + --hash=sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196 \ + --hash=sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f \ + --hash=sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7 \ + --hash=sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602 \ + --hash=sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3 \ + --hash=sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606 \ + --hash=sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18 \ + --hash=sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3 \ + --hash=sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae \ + --hash=sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148 \ + --hash=sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c \ + --hash=sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7 \ + --hash=sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd \ + --hash=sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab \ + --hash=sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27 \ + --hash=sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1 \ + --hash=sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab \ + --hash=sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296 \ + --hash=sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212 \ + --hash=sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc \ + --hash=sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737 \ + --hash=sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738 \ + --hash=sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be \ + --hash=sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8 \ + --hash=sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e \ + --hash=sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e \ + --hash=sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986 \ + --hash=sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f \ + --hash=sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f zipp==3.21.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4 \ --hash=sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931 diff --git a/octopoes/requirements.txt b/octopoes/requirements.txt index f54a9f12dd5..4538f3b45de 100644 --- a/octopoes/requirements.txt +++ b/octopoes/requirements.txt @@ -293,6 +293,8 @@ idna==3.10 ; python_version >= "3.10" and python_version < "4.0" \ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \ --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7 +jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonschema-specifications==2024.10.1 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf @@ -788,6 +790,130 @@ wrapt==1.17.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:f335579a1b485c834849e9075191c9898e0731af45705c2ebf70e0cd5d58beed \ --hash=sha256:fce6fee67c318fdfb7f285c29a82d84782ae2579c0e1b385b7f36c6e8074fffb \ --hash=sha256:fd136bb85f4568fffca995bd3c8d52080b1e5b225dbf1c2b17b66b4c5fa02838 +xxhash==3.5.0 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:02c2e816896dc6f85922ced60097bcf6f008dedfc5073dcba32f9c8dd786f3c1 \ + --hash=sha256:0691bfcc4f9c656bcb96cc5db94b4d75980b9d5589f2e59de790091028580837 \ + --hash=sha256:07fda5de378626e502b42b311b049848c2ef38784d0d67b6f30bb5008642f8eb \ + --hash=sha256:08424f6648526076e28fae6ea2806c0a7d504b9ef05ae61d196d571e5c879c84 \ + --hash=sha256:0a80ad0ffd78bef9509eee27b4a29e56f5414b87fb01a888353e3d5bda7038bd \ + --hash=sha256:0adfbd36003d9f86c8c97110039f7539b379f28656a04097e7434d3eaf9aa131 \ + --hash=sha256:0ec70a89be933ea49222fafc3999987d7899fc676f688dd12252509434636622 \ + --hash=sha256:1030a39ba01b0c519b1a82f80e8802630d16ab95dc3f2b2386a0b5c8ed5cbb10 \ + --hash=sha256:109b436096d0a2dd039c355fa3414160ec4d843dfecc64a14077332a00aeb7da \ + --hash=sha256:1308fa542bbdbf2fa85e9e66b1077eea3a88bef38ee8a06270b4298a7a62a166 \ + --hash=sha256:1328f6d8cca2b86acb14104e381225a3d7b42c92c4b86ceae814e5c400dbb415 \ + --hash=sha256:13de2b76c1835399b2e419a296d5b38dc4855385d9e96916299170085ef72f57 \ + --hash=sha256:14470ace8bd3b5d51318782cd94e6f94431974f16cb3b8dc15d52f3b69df8e00 \ + --hash=sha256:149b7914451eb154b3dfaa721315117ea1dac2cc55a01bfbd4df7c68c5dd683d \ + --hash=sha256:160e0c19ee500482ddfb5d5570a0415f565d8ae2b3fd69c5dcfce8a58107b1c3 \ + --hash=sha256:2014c5b3ff15e64feecb6b713af12093f75b7926049e26a580e94dcad3c73d8c \ + --hash=sha256:2061188a1ba352fc699c82bff722f4baacb4b4b8b2f0c745d2001e56d0dfb514 \ + --hash=sha256:220f3f896c6b8d0316f63f16c077d52c412619e475f9372333474ee15133a558 \ + --hash=sha256:23241ff6423378a731d84864bf923a41649dc67b144debd1077f02e6249a0d54 \ + --hash=sha256:25b5a51dc3dfb20a10833c8eee25903fd2e14059e9afcd329c9da20609a307b2 \ + --hash=sha256:297595fe6138d4da2c8ce9e72a04d73e58725bb60f3a19048bc96ab2ff31c692 \ + --hash=sha256:2b4154c00eb22e4d543f472cfca430e7962a0f1d0f3778334f2e08a7ba59363c \ + --hash=sha256:2e76e83efc7b443052dd1e585a76201e40b3411fe3da7af4fe434ec51b2f163b \ + --hash=sha256:30eb2efe6503c379b7ab99c81ba4a779748e3830241f032ab46bd182bf5873af \ + --hash=sha256:3171f693dbc2cef6477054a665dc255d996646b4023fe56cb4db80e26f4cc520 \ + --hash=sha256:33513d6cc3ed3b559134fb307aae9bdd94d7e7c02907b37896a6c45ff9ce51bd \ + --hash=sha256:33eac61d0796ca0591f94548dcfe37bb193671e0c9bcf065789b5792f2eda644 \ + --hash=sha256:37889a0d13b0b7d739cfc128b1c902f04e32de17b33d74b637ad42f1c55101f6 \ + --hash=sha256:38c384c434021e4f62b8d9ba0bc9467e14d394893077e2c66d826243025e1f81 \ + --hash=sha256:392f52ebbb932db566973693de48f15ce787cabd15cf6334e855ed22ea0be5b3 \ + --hash=sha256:3dbbd9892c5ebffeca1ed620cf0ade13eb55a0d8c84e0751a6653adc6ac40d0c \ + --hash=sha256:3e5b5e16c5a480fe5f59f56c30abdeba09ffd75da8d13f6b9b6fd224d0b4d0a2 \ + --hash=sha256:3ff2c0a34eae7df88c868be53a8dd56fbdf592109e21d4bfa092a27b0bf4a7bf \ + --hash=sha256:42eca420c8fa072cc1dd62597635d140e78e384a79bb4944f825fbef8bfeeef6 \ + --hash=sha256:4811336f1ce11cac89dcbd18f3a25c527c16311709a89313c3acaf771def2d4b \ + --hash=sha256:4cc2d67fdb4d057730c75a64c5923abfa17775ae234a71b0200346bfb0a7f482 \ + --hash=sha256:4e28503dccc7d32e0b9817aa0cbfc1f45f563b2c995b7a66c4c8a0d232e840c7 \ + --hash=sha256:4e2febf914ace002132aa09169cc572e0d8959d0f305f93d5828c4836f9bc5a6 \ + --hash=sha256:50ac2184ffb1b999e11e27c7e3e70cc1139047e7ebc1aa95ed12f4269abe98d4 \ + --hash=sha256:531af8845aaadcadf951b7e0c1345c6b9c68a990eeb74ff9acd8501a0ad6a1c9 \ + --hash=sha256:53a068fe70301ec30d868ece566ac90d873e3bb059cf83c32e76012c889b8637 \ + --hash=sha256:586886c7e89cb9828bcd8a5686b12e161368e0064d040e225e72607b43858ba2 \ + --hash=sha256:59aa1203de1cb96dbeab595ded0ad0c0056bb2245ae11fac11c0ceea861382b9 \ + --hash=sha256:5a74f23335b9689b66eb6dbe2a931a88fcd7a4c2cc4b1cb0edba8ce381c7a1da \ + --hash=sha256:5d0d307d27099bb0cbeea7260eb39ed4fdb99c5542e21e94bb6fd29e49c57a23 \ + --hash=sha256:5d2a01dcce81789cf4b12d478b5464632204f4c834dc2d064902ee27d2d1f0ee \ + --hash=sha256:5d3a10609c51da2a1c0ea0293fc3968ca0a18bd73838455b5bca3069d7f8e32b \ + --hash=sha256:5ed9ebc46f24cf91034544b26b131241b699edbfc99ec5e7f8f3d02d6eb7fba4 \ + --hash=sha256:6027dcd885e21581e46d3c7f682cfb2b870942feeed58a21c29583512c3f09f8 \ + --hash=sha256:602d339548d35a8579c6b013339fb34aee2df9b4e105f985443d2860e4d7ffaa \ + --hash=sha256:604253b2143e13218ff1ef0b59ce67f18b8bd1c4205d2ffda22b09b426386898 \ + --hash=sha256:61a1ff00674879725b194695e17f23d3248998b843eb5e933007ca743310f793 \ + --hash=sha256:61c722ed8d49ac9bc26c7071eeaa1f6ff24053d553146d5df031802deffd03da \ + --hash=sha256:63107013578c8a730419adc05608756c3fa640bdc6abe806c3123a49fb829f43 \ + --hash=sha256:683b94dbd1ca67557850b86423318a2e323511648f9f3f7b1840408a02b9a48c \ + --hash=sha256:685c4f4e8c59837de103344eb1c8a3851f670309eb5c361f746805c5471b8c88 \ + --hash=sha256:695735deeddfb35da1677dbc16a083445360e37ff46d8ac5c6fcd64917ff9ade \ + --hash=sha256:6e5f70f6dca1d3b09bccb7daf4e087075ff776e3da9ac870f86ca316736bb4aa \ + --hash=sha256:6e93a5ad22f434d7876665444a97e713a8f60b5b1a3521e8df11b98309bff833 \ + --hash=sha256:6fa0b72f2423e2aa53077e54a61c28e181d23effeaafd73fcb9c494e60930c8e \ + --hash=sha256:70dabf941dede727cca579e8c205e61121afc9b28516752fd65724be1355cc90 \ + --hash=sha256:74752ecaa544657d88b1d1c94ae68031e364a4d47005a90288f3bab3da3c970f \ + --hash=sha256:7a46e1d6d2817ba8024de44c4fd79913a90e5f7265434cef97026215b7d30df6 \ + --hash=sha256:7c5d3e570ef46adaf93fc81b44aca6002b5a4d8ca11bd0580c07eac537f36680 \ + --hash=sha256:7cb29a034301e2982df8b1fe6328a84f4b676106a13e9135a0d7e0c3e9f806da \ + --hash=sha256:7ccb800c9418e438b44b060a32adeb8393764da7441eb52aa2aa195448935306 \ + --hash=sha256:7ce379bcaa9fcc00f19affa7773084dd09f5b59947b3fb47a1ceb0179f91aaa1 \ + --hash=sha256:7f85e0108d51092bdda90672476c7d909c04ada6923c14ff9d913c4f7dc8a3bc \ + --hash=sha256:80babcc30e7a1a484eab952d76a4f4673ff601f54d5142c26826502740e70b43 \ + --hash=sha256:82085c2abec437abebf457c1d12fccb30cc8b3774a0814872511f0f0562c768c \ + --hash=sha256:82b833d5563fefd6fceafb1aed2f3f3ebe19f84760fdd289f8b926731c2e6e91 \ + --hash=sha256:84f2caddf951c9cbf8dc2e22a89d4ccf5d86391ac6418fe81e3c67d0cf60b45f \ + --hash=sha256:893074d651cf25c1cc14e3bea4fceefd67f2921b1bb8e40fcfeba56820de80c6 \ + --hash=sha256:89997aa1c4b6a5b1e5b588979d1da048a3c6f15e55c11d117a56b75c84531f5a \ + --hash=sha256:89e66ceed67b213dec5a773e2f7a9e8c58f64daeb38c7859d8815d2c89f39ad7 \ + --hash=sha256:8d47ebd9f5d9607fd039c1fbf4994e3b071ea23eff42f4ecef246ab2b7334198 \ + --hash=sha256:924361811732ddad75ff23e90efd9ccfda4f664132feecb90895bade6a1b4623 \ + --hash=sha256:963be41bcd49f53af6d795f65c0da9b4cc518c0dd9c47145c98f61cb464f4839 \ + --hash=sha256:97a662338797c660178e682f3bc180277b9569a59abfb5925e8620fba00b9fc5 \ + --hash=sha256:9bed5144c6923cc902cd14bb8963f2d5e034def4486ab0bbe1f58f03f042f9a9 \ + --hash=sha256:9c770750cc80e8694492244bca7251385188bc5597b6a39d98a9f30e8da984e0 \ + --hash=sha256:9d32a592cac88d18cc09a89172e1c32d7f2a6e516c3dfde1b9adb90ab5df54a6 \ + --hash=sha256:a5bc08f33c4966f4eb6590d6ff3ceae76151ad744576b5fc6c4ba8edd459fdec \ + --hash=sha256:a606c8070ada8aa2a88e181773fa1ef17ba65ce5dd168b9d08038e2a61b33754 \ + --hash=sha256:a6c50017518329ed65a9e4829154626f008916d36295b6a3ba336e2458824c8c \ + --hash=sha256:a7b1d8315d9b5e9f89eb2933b73afae6ec9597a258d52190944437158b49d38e \ + --hash=sha256:a8fb786fb754ef6ff8c120cb96629fb518f8eb5a61a16aac3a979a9dbd40a084 \ + --hash=sha256:a905ad00ad1e1c34fe4e9d7c1d949ab09c6fa90c919860c1534ff479f40fd12d \ + --hash=sha256:a9d360a792cbcce2fe7b66b8d51274ec297c53cbc423401480e53b26161a290d \ + --hash=sha256:b150b8467852e1bd844387459aa6fbe11d7f38b56e901f9f3b3e6aba0d660240 \ + --hash=sha256:b702f806693201ad6c0a05ddbbe4c8f359626d0b3305f766077d51388a6bac58 \ + --hash=sha256:b96d559e0fcddd3343c510a0fe2b127fbff16bf346dd76280b82292567523442 \ + --hash=sha256:bcd51708a633410737111e998ceb3b45d3dbc98c0931f743d9bb0a209033a326 \ + --hash=sha256:bfc8cdd7f33d57f0468b0614ae634cc38ab9202c6957a60e31d285a71ebe0301 \ + --hash=sha256:c0342aafd421795d740e514bc9858ebddfc705a75a8c5046ac56d85fe97bf196 \ + --hash=sha256:c279f0d2b34ef15f922b77966640ade58b4ccdfef1c4d94b20f2a364617a493f \ + --hash=sha256:c28b2fdcee797e1c1961cd3bcd3d545cab22ad202c846235197935e1df2f8ef7 \ + --hash=sha256:c3bc7bf8cb8806f8d1c9bf149c18708cb1c406520097d6b0a73977460ea03602 \ + --hash=sha256:c4dcb4120d0cc3cc448624147dba64e9021b278c63e34a38789b688fd0da9bf3 \ + --hash=sha256:c8aa771ff2c13dd9cda8166d685d7333d389fae30a4d2bb39d63ab5775de8606 \ + --hash=sha256:cc1276d369452040cbb943300dc8abeedab14245ea44056a2943183822513a18 \ + --hash=sha256:cd2fd827b0ba763ac919440042302315c564fdb797294d86e8cdd4578e3bc7f3 \ + --hash=sha256:d30bbc1644f726b825b3278764240f449d75f1a8bdda892e641d4a688b1494ae \ + --hash=sha256:d5e9db7ef3ecbfc0b4733579cea45713a76852b002cf605420b12ef3ef1ec148 \ + --hash=sha256:dbd2ecfbfee70bc1a4acb7461fa6af7748ec2ab08ac0fa298f281c51518f982c \ + --hash=sha256:dd86b8e7f703ec6ff4f351cfdb9f428955859537125904aa8c963604f2e9d3e7 \ + --hash=sha256:dee1316133c9b463aa81aca676bc506d3f80d8f65aeb0bba2b78d0b30c51d7bd \ + --hash=sha256:e0c48b6300cd0b0106bf49169c3e0536408dfbeb1ccb53180068a18b03c662ab \ + --hash=sha256:e5d0ddaca65ecca9c10dcf01730165fd858533d0be84c75c327487c37a906a27 \ + --hash=sha256:e6a4dd644d72ab316b580a1c120b375890e4c52ec392d4aef3c63361ec4d77d1 \ + --hash=sha256:eade977f5c96c677035ff39c56ac74d851b1cca7d607ab3d8f23c6b859379cab \ + --hash=sha256:ec28adb204b759306a3d64358a5e5c07d7b1dd0ccbce04aa76cb9377b7b70296 \ + --hash=sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212 \ + --hash=sha256:eefc37f6138f522e771ac6db71a6d4838ec7933939676f3753eafd7d3f4c40bc \ + --hash=sha256:f0b48edbebea1b7421a9c687c304f7b44d0677c46498a046079d445454504737 \ + --hash=sha256:f1abffa122452481a61c3551ab3c89d72238e279e517705b8b03847b1d93d738 \ + --hash=sha256:f2f2c61bee5844d41c3eb015ac652a0229e901074951ae48581d58bfb2ba01be \ + --hash=sha256:f7b58d1fd3551b8c80a971199543379be1cee3d0d409e1f6d8b01c1a2eebf1f8 \ + --hash=sha256:fa0cafd3a2af231b4e113fba24a65d7922af91aeb23774a8b78228e6cd785e3e \ + --hash=sha256:fa9f547bd98f5553d03160967866a71056a60960be00356a15ecc44efb40ba8e \ + --hash=sha256:fab81ef75003eda96239a23eda4e4543cedc22e34c373edcaf744e721a163986 \ + --hash=sha256:fd1b2281d01723f076df3c8188f43f2472248a6b63118b036e641243656b1b0f \ + --hash=sha256:fe1a92cfbaa0a1253e339ccec42dbe6db262615e52df591b68726ab10338003f zipp==3.21.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4 \ --hash=sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931 diff --git a/octopoes/tests/conftest.py b/octopoes/tests/conftest.py index ed978733267..042a5187841 100644 --- a/octopoes/tests/conftest.py +++ b/octopoes/tests/conftest.py @@ -34,6 +34,7 @@ Service, Website, ) +from octopoes.repositories.nibble_repository import XTDBNibbleRepository from octopoes.repositories.ooi_repository import OOIRepository, XTDBOOIRepository from octopoes.repositories.origin_parameter_repository import XTDBOriginParameterRepository from octopoes.repositories.origin_repository import XTDBOriginRepository @@ -202,7 +203,7 @@ def app_settings(): @pytest.fixture def octopoes_service() -> OctopoesService: - return OctopoesService(Mock(), Mock(), Mock(), Mock()) + return OctopoesService(Mock(), Mock(), Mock(), Mock(), Mock()) @pytest.fixture @@ -267,7 +268,7 @@ def complete_process_events(self, xtdb_octopoes_service: OctopoesService, repeat @pytest.fixture -def event_manager(xtdb_session: XTDBSession) -> Mock: +def event_manager(xtdb_session: XTDBSession) -> MockEventManager: return MockEventManager() @@ -277,29 +278,41 @@ def xtdb_ooi_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XT @pytest.fixture -def xtdb_origin_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XTDBOOIRepository]: +def xtdb_origin_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XTDBOriginRepository]: yield XTDBOriginRepository(event_manager, xtdb_session) @pytest.fixture -def xtdb_origin_parameter_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XTDBOOIRepository]: +def xtdb_origin_parameter_repository( + xtdb_session: XTDBSession, event_manager +) -> Iterator[XTDBOriginParameterRepository]: yield XTDBOriginParameterRepository(event_manager, xtdb_session) @pytest.fixture -def xtdb_scan_profile_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XTDBOOIRepository]: +def xtdb_scan_profile_repository(xtdb_session: XTDBSession, event_manager) -> Iterator[XTDBScanProfileRepository]: yield XTDBScanProfileRepository(event_manager, xtdb_session) +@pytest.fixture +def xtdb_nibble_repository(xtdb_session: XTDBSession) -> Iterator[XTDBNibbleRepository]: + yield XTDBNibbleRepository(xtdb_session) + + @pytest.fixture def xtdb_octopoes_service( xtdb_ooi_repository: XTDBOOIRepository, xtdb_origin_repository: XTDBOriginRepository, xtdb_origin_parameter_repository: XTDBOriginParameterRepository, xtdb_scan_profile_repository: XTDBScanProfileRepository, + xtdb_nibble_repository: XTDBNibbleRepository, ) -> OctopoesService: return OctopoesService( - xtdb_ooi_repository, xtdb_origin_repository, xtdb_origin_parameter_repository, xtdb_scan_profile_repository + xtdb_ooi_repository, + xtdb_origin_repository, + xtdb_origin_parameter_repository, + xtdb_scan_profile_repository, + xtdb_nibble_repository, ) diff --git a/octopoes/tests/integration/test_api_connector.py b/octopoes/tests/integration/test_api_connector.py index 17dac4724c6..2bc885e1499 100644 --- a/octopoes/tests/integration/test_api_connector.py +++ b/octopoes/tests/integration/test_api_connector.py @@ -15,7 +15,7 @@ from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, PortState, Protocol from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.web import Website -from octopoes.models.origin import OriginType +from octopoes.models.origin import Origin, OriginType if os.environ.get("CI") != "1": pytest.skip("Needs XTDB multinode container.", allow_module_level=True) @@ -49,14 +49,16 @@ def test_bulk_operations(octopoes_api_connector: OctopoesAPIConnector, valid_tim assert len(octopoes_api_connector.list_origins(task_id=uuid.uuid4(), valid_time=valid_time)) == 0 origins = octopoes_api_connector.list_origins(task_id=task_id, valid_time=valid_time) assert len(origins) == 1 - assert origins[0].model_dump() == { - "method": "normalizer_id", - "origin_type": OriginType.OBSERVATION, - "source": network.reference, - "source_method": "manual", - "result": [hostname.reference for hostname in hostnames], - "task_id": task_id, - } + assert origins[0] == Origin.model_validate( + { + "method": "normalizer_id", + "origin_type": OriginType.OBSERVATION, + "source": network.reference, + "source_method": "manual", + "result": [hostname.reference for hostname in hostnames], + "task_id": task_id, + } + ) assert len(octopoes_api_connector.list_origins(result=hostnames[0].reference, valid_time=valid_time)) == 1 diff --git a/octopoes/tests/integration/test_config_nibbles.py b/octopoes/tests/integration/test_config_nibbles.py new file mode 100644 index 00000000000..25f58e15fdf --- /dev/null +++ b/octopoes/tests/integration/test_config_nibbles.py @@ -0,0 +1,268 @@ +import os +import sys +from collections.abc import Iterator +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.definitions import NibbleDefinition, NibbleParameter + +from octopoes.core.service import OctopoesService +from octopoes.models import OOI, Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import Network +from octopoes.models.ooi.web import URL +from octopoes.models.origin import OriginType + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + + +counter = 0 + + +def config_nibble_payload(url: URL, config: Config | None) -> Iterator[OOI]: + global counter + counter += 1 + if config is not None and str(url.raw) in config.config: + kft = KATFindingType(id="URL in config") + yield kft + yield Finding(finding_type=kft.reference, ooi=url.reference, proof=f"{url.reference} in {config.config}") + + +def config_nibble_query(targets: list[Reference | None]) -> str: + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "10": + network = str(Network(name=targets[0].split("|")[1]).reference) if targets[0] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?url [*]) (pull ?config [*])] :where [ + + [?url :object_type "URL"] + [?url :URL/primary_key "{str(targets[0])}"] + + (or + (and + [?config :Config/ooi "{network}"] + [?config :Config/bit_id "config_nibble_test"] + ) + (and + [(identity nil) ?config] + ) + ) + + ] + }} + }} + """ + elif sgn == "01": + network = str(Network(name=targets[1].split("|")[1]).reference) if targets[1] is not None else "" + return f""" + {{ + :query {{ + :find [(pull ?url [*]) (pull ?config [*])] :where [ + + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "config_nibble_test"] + + (or + (and + [?url :URL/network "{network}"] + ) + (and + [(identity nil) ?url] + ) + ) + + ] + }} + }} + """ + elif sgn == "11": + return f""" + {{ + :query {{ + :find [(pull ?url [*]) (pull ?config [*])] :where [ + [?url :object_type "URL"] + [?url :URL/primary_key "{str(targets[0])}"] + [?config :object_type "Config"] + [?config :Config/primary_key "{str(targets[1])}"] + [?config :Config/bit_id "config_nibble_test"] + ] + }} + }} + """ + else: + return """ + { + :query { + :find [(pull ?url [*]) (pull ?config [*])] :where [ + + [?url :object_type "URL"] + + (or + (and + [?url :URL/network ?network] + [?config :Config/ooi ?network] + [?config :object_type "Config"] + [?config :Config/bit_id "config_nibble_test"] + ) + (and + [(identity nil) ?network] + [(identity nil) ?config] + ) + ) + ] + } + } + """ + + +config_nibble = NibbleDefinition( + id="config_nibble_test", + signature=[ + NibbleParameter(object_type=URL, parser="[*][?object_type == 'URL'][]"), + NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), + ], + query=config_nibble_query, +) +config_nibble._payload = getattr(sys.modules[__name__], "config_nibble_payload") + + +def test_inference_without_config(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + global counter + counter = 0 + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + url = URL(network=network.reference, raw="https://mispo.es/") + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 + assert counter == 1 + + +def test_inference_with_config(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + global counter + counter = 0 + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + url = URL(network=network.reference, raw="https://mispo.es/") + config = Config(ooi=network.reference, bit_id="config_nibble_test", config={str(url.raw): None}) + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + assert counter == 2 + + +def test_inference_with_other_config(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + global counter + counter = 0 + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + network_potato = Network(name="potato") + url = URL(network=network.reference, raw="https://mispo.es/") + config = Config(ooi=network_potato.reference, bit_id="config_nibble_test", config={str(url.raw): None}) + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(network_potato, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 + assert counter == 1 + + +def test_inference_with_fake_id_config( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + global counter + counter = 0 + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + url = URL(network=network.reference, raw="https://mispo.es/") + config = Config(ooi=network.reference, bit_id="fake_id", config={str(url.raw): None}) + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 + assert counter == 1 + + +def test_inference_with_changed_config( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + global counter + counter = 0 + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + url = URL(network=network.reference, raw="https://mispo.es/") + config = Config(ooi=network.reference, bit_id="config_nibble_test", config={str(url.raw): None}) + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + assert counter == 2 + + config = Config(ooi=network.reference, bit_id="config_nibble_test", config={}) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 + assert counter == 3 + + assert len(xtdb_octopoes_service.origin_repository.list_origins(valid_time, origin_type=OriginType.NIBBLET)) == 1 + + +def test_retrieve(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {"config_nibble_test": config_nibble} + + network = Network(name="internet") + url = URL(network=network.reference, raw="https://mispo.es/") + config = Config(ooi=network.reference, bit_id="config_nibble_test", config={str(url.raw): None}) + + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + xtdb_url = xtdb_octopoes_service.ooi_repository.get(url.reference, valid_time) + + retrieved = xtdb_octopoes_service.nibbler.retrieve(["config_nibble_test"], valid_time) + assert len(retrieved) == 1 + assert retrieved["config_nibble_test"][0] == [xtdb_url, None] + + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + xtdb_config = xtdb_octopoes_service.ooi_repository.get(config.reference, valid_time) + + retrieved = xtdb_octopoes_service.nibbler.retrieve(["config_nibble_test"], valid_time) + assert len(retrieved) == 1 + assert retrieved["config_nibble_test"][0] == [xtdb_url, xtdb_config] diff --git a/octopoes/tests/integration/test_hsts_nibble.py b/octopoes/tests/integration/test_hsts_nibble.py new file mode 100644 index 00000000000..93eeb8e4d4d --- /dev/null +++ b/octopoes/tests/integration/test_hsts_nibble.py @@ -0,0 +1,145 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.check_hsts_header.nibble import NIBBLE as check_hsts_header_nibble +from nibbles.runner import NibblesRunner + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_hsts_nibble_with_and_without_config( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {check_hsts_header_nibble.id: check_hsts_header_nibble} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_header = xtdb_octopoes_service.ooi_repository.get(header.reference, valid_time) + + result = nibbler.infer([xtdb_header], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_header, None) in result[header][check_hsts_header_nibble.id] + + assert len(result[header][check_hsts_header_nibble.id][(xtdb_header, None)]) == 2 + + config = Config(ooi=network.reference, config={"max-age": 11536000}, bit_id="check-hsts-header") + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_config = xtdb_octopoes_service.ooi_repository.get(config.reference, valid_time) + + result = nibbler.infer([xtdb_config], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_header, xtdb_config) in result[config][check_hsts_header_nibble.id] + + assert len(result[config][check_hsts_header_nibble.id][(xtdb_header, xtdb_config)]) == 0 + + +def test_hsts_nibble_with_config(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {check_hsts_header_nibble.id: check_hsts_header_nibble} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + config = Config(ooi=network.reference, config={"max-age": 11536000}, bit_id="check-hsts-header") + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_header = xtdb_octopoes_service.ooi_repository.get(header.reference, valid_time) + xtdb_config = xtdb_octopoes_service.ooi_repository.get(config.reference, valid_time) + + result = nibbler.infer([xtdb_header], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_header, xtdb_config) in result[header][check_hsts_header_nibble.id] + + assert len(result[header][check_hsts_header_nibble.id][(xtdb_header, xtdb_config)]) == 0 diff --git a/octopoes/tests/integration/test_missing_spf_nibble.py b/octopoes/tests/integration/test_missing_spf_nibble.py new file mode 100644 index 00000000000..ca6650686aa --- /dev/null +++ b/octopoes/tests/integration/test_missing_spf_nibble.py @@ -0,0 +1,111 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.missing_spf.nibble import NIBBLE as missing_spf_nibble +from nibbles.runner import NibblesRunner + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.dns.records import NXDOMAIN, DNSTXTRecord +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.email_security import DNSSPFRecord +from octopoes.models.ooi.network import Network + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + + +def test_missing_spf_nibble_with_and_without_nx( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {missing_spf_nibble.id: missing_spf_nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + hostname = Hostname(name="test", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_hostname = xtdb_octopoes_service.ooi_repository.get(hostname.reference, valid_time) + + result = nibbler.infer([xtdb_hostname], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_hostname, None, None) in result[hostname][missing_spf_nibble.id] + assert len(result[hostname][missing_spf_nibble.id][(xtdb_hostname, None, None)]) == 2 + + nx_domain = NXDOMAIN(hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(nx_domain, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_nx = xtdb_octopoes_service.ooi_repository.get(nx_domain.reference, valid_time) + result = nibbler.infer([xtdb_nx], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_hostname, None, xtdb_nx) in result[nx_domain][missing_spf_nibble.id] + assert len(result[nx_domain][missing_spf_nibble.id][(xtdb_hostname, None, xtdb_nx)]) == 0 + + +def test_missing_spf_nibble_with_and_without_spf( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {missing_spf_nibble.id: missing_spf_nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + hostname = Hostname(name="test", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_hostname = xtdb_octopoes_service.ooi_repository.get(hostname.reference, valid_time) + + result = nibbler.infer([xtdb_hostname], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_hostname, None, None) in result[hostname][missing_spf_nibble.id] + assert len(result[hostname][missing_spf_nibble.id][(xtdb_hostname, None, None)]) == 2 + + txt_record = DNSTXTRecord(hostname=hostname.reference, value="test") + xtdb_octopoes_service.ooi_repository.save(txt_record, valid_time) + spf_record = DNSSPFRecord(dns_txt_record=txt_record.reference, value="test") + xtdb_octopoes_service.ooi_repository.save(spf_record, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_spf_record = xtdb_octopoes_service.ooi_repository.get(spf_record.reference, valid_time) + result = nibbler.infer([xtdb_spf_record], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_hostname, xtdb_spf_record, None) in result[spf_record][missing_spf_nibble.id] + assert len(result[spf_record][missing_spf_nibble.id][(xtdb_hostname, xtdb_spf_record, None)]) == 0 + + +def test_missing_spf_nibble_non_tld(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {missing_spf_nibble.id: missing_spf_nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + hostname = Hostname(name="example.example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_hostname = xtdb_octopoes_service.ooi_repository.get(hostname.reference, valid_time) + + result = nibbler.infer([xtdb_hostname], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_hostname, None, None) in result[hostname][missing_spf_nibble.id] + assert len(result[hostname][missing_spf_nibble.id][(xtdb_hostname, None, None)]) == 0 diff --git a/octopoes/tests/integration/test_nibbles.py b/octopoes/tests/integration/test_nibbles.py new file mode 100644 index 00000000000..6b8a0abcdef --- /dev/null +++ b/octopoes/tests/integration/test_nibbles.py @@ -0,0 +1,403 @@ +import os +import sys +from collections.abc import Iterator +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.definitions import NibbleDefinition, NibbleParameter +from nibbles.runner import NibblesRunner, nibble_hasher + +from octopoes.core.service import OctopoesService +from octopoes.events.events import OOIDBEvent, OperationType, OriginDBEvent +from octopoes.models import OOI, Reference +from octopoes.models.ooi.config import Config +from octopoes.models.ooi.findings import Finding, FindingType, KATFindingType, RiskLevelSeverity +from octopoes.models.ooi.network import Network +from octopoes.models.ooi.web import URL +from octopoes.models.origin import OriginType + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + + +MAX_NETWORK_NAME_LENGTH = 13 + + +def dummy(network: Network) -> Network | None: + if len(network.name) < MAX_NETWORK_NAME_LENGTH: + new_name = network.name + "I" + return Network(name=new_name) + + +dummy_params = [NibbleParameter(object_type=Network)] +dummy_nibble = NibbleDefinition(id="dummy", signature=dummy_params) +dummy_nibble._payload = getattr(sys.modules[__name__], "dummy") + + +def test_dummy_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {dummy_nibble.id: dummy_nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + ctx = 1 + MAX_NETWORK_NAME_LENGTH - len(network.name) + assert xtdb_octopoes_service.ooi_repository.list_oois({Network}, valid_time).count == ctx + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == ctx + + +def test_url_classification_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + nibble = xtdb_octopoes_service.nibbler.nibbles["url_classification"] + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {nibble.id: nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + url = URL(network=network.reference, raw="https://mispo.es/") + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + result = nibbler.infer([url], valid_time) + + assert url in result + assert "url_classification" in result[url] + assert len(result[url]["url_classification"]) == 1 + assert len(result[url]["url_classification"][tuple([url])]) == 3 + + +def find_network_url(network: Network, url: URL) -> Iterator[OOI]: + if len(network.name) == len(str(url.raw)): + yield Finding( + finding_type=KATFindingType(id="Network and URL have same name length").reference, + ooi=network.reference, + proof=url.reference, + ) + + +find_network_url_params = [ + NibbleParameter(object_type=Network, parser="[*][?object_type == 'Network'][]"), + NibbleParameter(object_type=URL, parser="[*][?object_type == 'URL'][]"), +] +find_network_url_nibble = NibbleDefinition( + id="find_network_url", + signature=find_network_url_params, + query=""" + { + :query { + :find [(pull ?var [*])] + :where [ + (or + (and [?var :object_type "URL" ] [?var :URL/raw]) + (and [?var :object_type "Network" ] [?var :Network/name]) + ) + ] + } + } + """, +) +find_network_url_nibble._payload = getattr(sys.modules[__name__], "find_network_url") + + +def test_find_network_url_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {find_network_url_nibble.id: find_network_url_nibble} + network1 = Network(name="internetverbinding") + xtdb_octopoes_service.ooi_repository.save(network1, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + network2 = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network2, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + url1 = URL(network=network1.reference, raw="https://potato.ls/") + xtdb_octopoes_service.ooi_repository.save(url1, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + url2 = URL(network=network2.reference, raw="https://mispo.es/") + xtdb_octopoes_service.ooi_repository.save(url2, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + xtdb_url1 = xtdb_octopoes_service.ooi_repository.get(url1.reference, valid_time) + xtdb_url2 = xtdb_octopoes_service.ooi_repository.get(url2.reference, valid_time) + + result = nibbler.infer([network1], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + target = set(find_network_url(network1, url1)) + + assert network1 in result + assert len(result[network1]["find_network_url"]) == 4 + assert result[network1]["find_network_url"][tuple([network1, xtdb_url1])] == target + assert result[network1]["find_network_url"][tuple([network2, xtdb_url1])] == set() + assert result[network1]["find_network_url"][tuple([network1, xtdb_url2])] == set() + assert result[network1]["find_network_url"][tuple([network2, xtdb_url2])] == set() + + nibblets = xtdb_octopoes_service.origin_repository.list_origins( + origin_type=OriginType.NIBBLET, valid_time=valid_time + ) + + assert len(nibblets) == 4 + for nibblet in nibblets: + assert nibblet.parameters_references is not None + arg = [ + xtdb_octopoes_service.ooi_repository.get(obj, valid_time) + for obj in nibblet.parameters_references + if obj is not None + ] + assert nibblet.parameters_hash == nibble_hasher(tuple(arg)) + if nibblet.result: + assert len(nibblet.result) == 1 + assert nibblet.result == [t.reference for t in target] + + +def test_max_length_config_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + nibble = xtdb_octopoes_service.nibbler.nibbles["max_url_length_config"] + nibbler.nibbles = {"max_url_length_config": nibble} + xtdb_octopoes_service.nibbler.disable() + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + url = URL(network=network.reference, raw="https://mispo.es/") + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + config = Config(ooi=network.reference, bit_id="superkat", config={"max_length": "57"}) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + xtdb_url = xtdb_octopoes_service.ooi_repository.get(url.reference, valid_time) + + result = nibbler.infer([xtdb_url], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_url in result + assert len(result[xtdb_url]["max_url_length_config"]) == 1 + assert result[xtdb_url]["max_url_length_config"][tuple([xtdb_url, config])] == set() + + config = Config(ooi=network.reference, bit_id="superkat", config={"max_length": "13"}) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + result = nibbler.infer([xtdb_url], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_url in result + assert len(result[xtdb_url]["max_url_length_config"]) == 1 + assert result[xtdb_url]["max_url_length_config"][tuple([xtdb_url, config])] == set(nibble([xtdb_url, config])) + + +def callable_query(url1: URL, url2: URL) -> Iterator[OOI]: + if url1.raw == url2.raw and url1.network != url2.network: + yield Finding( + finding_type=KATFindingType(id="Duplicate URL's under different network").reference, + ooi=url1.reference, + proof=f"{url1.reference} matches {url2.reference}.", + ) + + +callable_query_param = [ + NibbleParameter(object_type=URL, parser='[*].{"URL1": @[1]}[?"URL1"] | [?URL1.object_type == \'URL\'].URL1'), + NibbleParameter(object_type=URL, parser='[*].{"URL2": @[3]}[?"URL2"] | [?URL2.object_type == \'URL\'].URL2'), +] + + +def callable_query_query(targets: list[Reference | None]) -> str: + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "10": + return f"""{{ + :query {{ + :find ["URL1" (pull ?url1 [*]) "URL2" (pull ?url2 [*])] + :where [ + [?url1 :URL/primary_key "{str(targets[0])}"] + [?url2 :object_type "URL"] + ] + }} + }} + """ + else: + return f"""{{ + :query {{ + :find ["URL1" (pull ?url1 [*]) "URL2" (pull ?url2 [*])] + :where [ + [?url1 :URL/primary_key "{str(targets[0])}"] + [?url2 :URL/primary_key "{str(targets[1])}"] + ] + }} + }} + """ + + +callable_query_nibble = NibbleDefinition( + id="callable_nibble_query", signature=callable_query_param, query=callable_query_query +) +callable_query_nibble._payload = getattr(sys.modules[__name__], "callable_query") + + +def test_callable_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {"callable_nibble_query": callable_query_nibble} + network1 = Network(name="internet1") + network2 = Network(name="internet2") + + xtdb_octopoes_service.ooi_repository.save(network1, valid_time) + xtdb_octopoes_service.ooi_repository.save(network2, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + for url in ["https://mispo.es", "https://appelmo.es", "https://boesbo.es"]: + xtdb_octopoes_service.ooi_repository.save(URL(network=network1.reference, raw=url), valid_time) + + for url in ["https://tompo.es", "https://smo.es", "https://mispo.es"]: + xtdb_octopoes_service.ooi_repository.save(URL(network=network2.reference, raw=url), valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + url1 = URL(network=network1.reference, raw="https://mispo.es").reference + url2 = URL(network=network2.reference, raw="https://mispo.es").reference + xtdb_url1 = xtdb_octopoes_service.ooi_repository.get(url1, valid_time) + xtdb_url2 = xtdb_octopoes_service.ooi_repository.get(url2, valid_time) + finding = list(callable_query(xtdb_url1, xtdb_url2)) + + result = xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time) + assert result.count == 2 + assert finding[0] in result.items + + +mock_finding_type_nibble = NibbleDefinition( + id="default-findingtype-risk", signature=[NibbleParameter(object_type=FindingType)] +) + + +def set_default_severity(input_ooi: FindingType) -> Iterator[OOI]: + input_ooi.risk_severity = RiskLevelSeverity.PENDING + yield input_ooi + + +mock_finding_type_nibble._payload = getattr(sys.modules[__name__], "set_default_severity") + + +def test_parent_type_in_nibble_signature( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {mock_finding_type_nibble.id: mock_finding_type_nibble} + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + finding_type = KATFindingType(id="test") + xtdb_octopoes_service.ooi_repository.save(finding_type, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + xtdb_finding_type = xtdb_octopoes_service.ooi_repository.get(finding_type.reference, valid_time) + + result = nibbler.infer([xtdb_finding_type], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_finding_type in result + + +def find_network_url_v2(network: Network, url: URL) -> Iterator[OOI]: + if len(network.name) == len(str(url.raw)): + kt = KATFindingType(id="Network and URL have same name length") + yield kt + yield Finding(finding_type=kt.reference, ooi=network.reference, proof=url.reference) + + +def test_nibbles_update(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {find_network_url_nibble.id: find_network_url_nibble} + + network1 = Network(name="internetverbinding") + xtdb_octopoes_service.ooi_repository.save(network1, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + network2 = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network2, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + url1 = URL(network=network1.reference, raw="https://potato.ls/") + xtdb_octopoes_service.ooi_repository.save(url1, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + url2 = URL(network=network2.reference, raw="https://mispo.es/") + xtdb_octopoes_service.ooi_repository.save(url2, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 + + find_network_url_nibble_v2 = find_network_url_nibble.model_copy(deep=True) + find_network_url_nibble_v2._payload = getattr(sys.modules[__name__], "find_network_url_v2") + find_network_url_nibble_v2._checksum = "deadbeef" + + xtdb_octopoes_service.nibbler.update_nibbles( + valid_time, {find_network_url_nibble_v2.id: find_network_url_nibble_v2} + ) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + + +def test_nibble_states(xtdb_octopoes_service: OctopoesService, valid_time: datetime): + nibble_inis = [nibble._ini for nibble in xtdb_octopoes_service.nibbler.nibbles.values()] + xtdb_octopoes_service.nibbler.register() + xtdb_nibble_inis = {ni["id"]: ni for ni in xtdb_octopoes_service.nibbler.nibble_repository.get_all(valid_time)} + for nibble_ini in nibble_inis: + assert xtdb_nibble_inis[nibble_ini["id"]] == nibble_ini + + xtdb_octopoes_service.nibbler.toggle_nibbles(["max_url_length_config"], False, valid_time) + + nibble_inis = [nibble._ini for nibble in xtdb_octopoes_service.nibbler.nibbles.values()] + xtdb_nibble_inis = {ni["id"]: ni for ni in xtdb_octopoes_service.nibbler.nibble_repository.get_all(valid_time)} + for nibble_ini in nibble_inis: + assert xtdb_nibble_inis[nibble_ini["id"]] == nibble_ini + + xtdb_octopoes_service.nibbler.nibbles["max_url_length_config"].enabled = True + xtdb_octopoes_service.nibbler.sync(valid_time) + + nibble_inis = [nibble._ini for nibble in xtdb_octopoes_service.nibbler.nibbles.values()] + xtdb_nibble_inis = {ni["id"]: ni for ni in xtdb_octopoes_service.nibbler.nibble_repository.get_all(valid_time)} + for nibble_ini in nibble_inis: + assert xtdb_nibble_inis[nibble_ini["id"]] == nibble_ini + + +def test_nibble_origin_deletion_propagation( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + url = URL(network=network.reference, raw="https://mispo.es/") + xtdb_octopoes_service.ooi_repository.save(url, valid_time) + config = Config(ooi=network.reference, bit_id="superkat", config={"max_length": "3"}) + xtdb_octopoes_service.ooi_repository.save(config, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count > 3 + + xtdb_octopoes_service.ooi_repository.delete(network.reference, valid_time) + xtdb_octopoes_service.ooi_repository.delete(url.reference, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + for q in event_manager.queue: + if q.operation_type == OperationType.CREATE or q.operation_type == OperationType.UPDATE: + if isinstance(q, OOIDBEvent): + print(f"CREATE: {q.new_data.reference}") + elif isinstance(q, OriginDBEvent): + print(f"CREATE: {q.new_data.id}") + elif q.operation_type == OperationType.DELETE: + if isinstance(q, OOIDBEvent): + print(f"DELETE: {q.old_data.reference}") + elif isinstance(q, OriginDBEvent): + print(f"DELETE: {q.old_data.id}") + + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).items == [config] diff --git a/octopoes/tests/integration/test_ooi_deletion.py b/octopoes/tests/integration/test_ooi_deletion.py index ba288ed942a..d6d45bbe36d 100644 --- a/octopoes/tests/integration/test_ooi_deletion.py +++ b/octopoes/tests/integration/test_ooi_deletion.py @@ -131,9 +131,9 @@ def test_events_created_in_worker_during_handling( xtdb_octopoes_service.process_event(event) xtdb_octopoes_service.commit() - assert len(event_manager.queue) == 8 # Handling OOI delete event triggers Origin delete event + assert len(event_manager.queue) == 5 # Handling OOI delete event triggers Origin delete event - event = event_manager.queue[7] # OOIDelete event + event = event_manager.queue[4] # OOIDelete event assert isinstance(event, OriginDBEvent) assert event.operation_type.value == "delete" @@ -229,7 +229,7 @@ def test_deletion_events_after_nxdomain( event_manager.complete_process_events(xtdb_octopoes_service) assert len(list(filter(lambda x: x.operation_type.value == "delete", event_manager.queue))) == 0 - assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 8 + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 9 nxd = NXDOMAIN(hostname=hostname.reference) xtdb_octopoes_service.ooi_repository.save(nxd, valid_time) @@ -250,7 +250,7 @@ def test_deletion_events_after_nxdomain( event_manager.complete_process_events(xtdb_octopoes_service) assert len(list(filter(lambda x: x.operation_type.value == "delete", event_manager.queue))) >= 3 - assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 6 + assert xtdb_octopoes_service.ooi_repository.list_oois({OOI}, valid_time).count == 8 @pytest.mark.xfail(reason="Wappalyzer works on wrong input objects (to be addressed)") diff --git a/octopoes/tests/integration/test_unicode.py b/octopoes/tests/integration/test_unicode.py index f0fe09137d1..bacff7938e0 100644 --- a/octopoes/tests/integration/test_unicode.py +++ b/octopoes/tests/integration/test_unicode.py @@ -10,7 +10,7 @@ from octopoes.models import DeclaredScanProfile, ScanLevel from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.network import Network -from octopoes.models.origin import OriginType +from octopoes.models.origin import Origin, OriginType if os.environ.get("CI") != "1": pytest.skip("Needs XTDB multinode container.", allow_module_level=True) @@ -67,13 +67,15 @@ def test_unicode_hostname(octopoes_api_connector: OctopoesAPIConnector, valid_ti assert hostname_object.reference == hostname.reference origins = octopoes_api_connector.list_origins(task_id=task_id, valid_time=valid_time) - assert origins[0].dict() == { - "method": NAMES[2], - "origin_type": OriginType.OBSERVATION, - "source": network.reference, - "source_method": "test", - "result": [hostname.reference], - "task_id": task_id, - } + assert origins[0] == Origin.model_validate( + { + "method": NAMES[2], + "origin_type": OriginType.OBSERVATION, + "source": network.reference, + "source_method": "test", + "result": [hostname.reference], + "task_id": task_id, + } + ) assert len(octopoes_api_connector.list_origins(result=hostname.reference, valid_time=valid_time)) == 1 diff --git a/octopoes/tests/integration/test_url_discovery_nibble.py b/octopoes/tests/integration/test_url_discovery_nibble.py new file mode 100644 index 00000000000..97270933a5f --- /dev/null +++ b/octopoes/tests/integration/test_url_discovery_nibble.py @@ -0,0 +1,125 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.runner import NibblesRunner +from nibbles.url_discovery.nibble import NIBBLE as url_discovery_nibble + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.dns.zone import Hostname, ResolvedHostname +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_url_discovery_nibble_simple_port( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {url_discovery_nibble.id: url_discovery_nibble} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + resolved_hostname = ResolvedHostname(address=ip_address.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(resolved_hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_port = xtdb_octopoes_service.ooi_repository.get(port.reference, valid_time) + xtdb_resolved_hostname = xtdb_octopoes_service.ooi_repository.get(resolved_hostname.reference, valid_time) + + result = nibbler.infer([xtdb_port], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_port, xtdb_resolved_hostname) in result[xtdb_port][url_discovery_nibble.id] + assert len(result[xtdb_port][url_discovery_nibble.id][(xtdb_port, xtdb_resolved_hostname)]) == 1 + + +def test_url_discovery_nibble_simple_resolved_hostname( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {url_discovery_nibble.id: url_discovery_nibble} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + resolved_hostname = ResolvedHostname(address=ip_address.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(resolved_hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_port = xtdb_octopoes_service.ooi_repository.get(port.reference, valid_time) + xtdb_resolved_hostname = xtdb_octopoes_service.ooi_repository.get(resolved_hostname.reference, valid_time) + + result = nibbler.infer([xtdb_resolved_hostname], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert (xtdb_port, xtdb_resolved_hostname) in result[xtdb_resolved_hostname][url_discovery_nibble.id] + assert len(result[xtdb_resolved_hostname][url_discovery_nibble.id][(xtdb_port, xtdb_resolved_hostname)]) == 1 + + +def test_url_discovery_nibble_no_http_port( + xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime +): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {url_discovery_nibble.id: url_discovery_nibble} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=1, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + resolved_hostname = ResolvedHostname(address=ip_address.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(resolved_hostname, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + xtdb_port = xtdb_octopoes_service.ooi_repository.get(port.reference, valid_time) + + result = nibbler.infer([xtdb_port], valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert len(result[xtdb_port][url_discovery_nibble.id]) == 0 diff --git a/octopoes/tests/robot/02_list_objects.robot b/octopoes/tests/robot/02_list_objects.robot index 2c1eed6b0f9..d9c9e844040 100644 --- a/octopoes/tests/robot/02_list_objects.robot +++ b/octopoes/tests/robot/02_list_objects.robot @@ -10,7 +10,7 @@ List Objects Insert Normalizer Output Await Sync Object List Should Contain ${REF_HOSTNAME} - Total Object Count Should Be ${6} + Total Object Count Should Be ${8} List Objects With Filter Insert Normalizer Output @@ -44,7 +44,7 @@ Load Bulk *** Keywords *** Verify Object List With Filter ${response_data} Get Objects With ScanLevel 0 - Should Be Equal ${response_data["count"]} ${6} + Should Be Equal ${response_data["count"]} ${8} Get Objects With ScanLevel 0 ${params} Create Dictionary scan_level=0 valid_time=${VALID_TIME} @@ -55,7 +55,7 @@ Get Objects With ScanLevel 0 Verify Object List With SearchString ${response_data} Get Objects With SearchString example.com - Should Be Equal ${response_data["count"]} ${4} + Should Be Equal ${response_data["count"]} ${5} Get Objects With SearchString example.com ${params} Create Dictionary search_string=example.com valid_time=${VALID_TIME} diff --git a/octopoes/tests/robot/08_findings.robot b/octopoes/tests/robot/08_findings.robot index cbd3d903ed0..68764fcc805 100644 --- a/octopoes/tests/robot/08_findings.robot +++ b/octopoes/tests/robot/08_findings.robot @@ -15,8 +15,8 @@ List Findings Declare Scan Profile Hostname|internet|example.com 1 Await Sync - Finding List Should Have Length 1 - Finding Count Per Severity Should Be 'pending' 1 + Finding List Should Have Length 2 + Finding Count Per Severity Should Be 'pending' 2 Finding Count Per Severity Should Be 'low' 0 Finding Count Per Severity Should Be 'critical' 0 diff --git a/octopoes/tests/test_bit_ask_ports.py b/octopoes/tests/test_bit_ask_ports.py index e5e9bff8bb2..900402e9ead 100644 --- a/octopoes/tests/test_bit_ask_ports.py +++ b/octopoes/tests/test_bit_ask_ports.py @@ -1,11 +1,11 @@ -from bits.ask_port_specification.ask_port_specification import run +from nibbles.ask_port_specification.ask_port_specification import nibble from octopoes.models.ooi.network import Network from octopoes.models.ooi.question import Question def test_port_classification_tcp_80(): - results = list(run(Network(name="test1"), [], {})) + results = list(nibble(Network(name="test1"))) assert len(results) == 1 assert isinstance(results[0], Question) diff --git a/octopoes/tests/test_bit_cipher.py b/octopoes/tests/test_bit_cipher.py index 5edfd32db91..1334963086f 100644 --- a/octopoes/tests/test_bit_cipher.py +++ b/octopoes/tests/test_bit_cipher.py @@ -1,4 +1,4 @@ -from bits.cipher_classification.cipher_classification import run as cipher_classification +from nibbles.cipher_classification.cipher_classification import nibble as cipher_classification from octopoes.models.ooi.findings import Finding from octopoes.models.ooi.network import IPAddressV4, IPPort @@ -91,7 +91,7 @@ def test_medium_bad_ciphers(): }, ) - results = list(cipher_classification(cipher, {}, {})) + results = list(cipher_classification(cipher)) assert len(results) == 2 assert results[0].reference == "KATFindingType|KAT-MEDIUM-BAD-CIPHER" @@ -124,6 +124,6 @@ def test_good_ciphers(): }, ) - results = list(cipher_classification(cipher, {}, {})) + results = list(cipher_classification(cipher)) assert len(results) == 0 diff --git a/octopoes/tests/test_bit_default_findingtype_risk.py b/octopoes/tests/test_bit_default_findingtype_risk.py index 785d80fbdf1..3e9b36b6f4b 100644 --- a/octopoes/tests/test_bit_default_findingtype_risk.py +++ b/octopoes/tests/test_bit_default_findingtype_risk.py @@ -1,4 +1,4 @@ -from bits.default_findingtype_risk.default_findingtype_risk import run as run_default_findingtype_risk +from nibbles.default_findingtype_risk.default_findingtype_risk import nibble as run_default_findingtype_risk from octopoes.models.ooi.findings import KATFindingType, RiskLevelSeverity @@ -9,7 +9,7 @@ def test_default_findingtype_risk_pending(): assert test_finding_type.risk_severity is None assert test_finding_type.risk_score is None - results = list(run_default_findingtype_risk(test_finding_type, [], {})) + results = list(run_default_findingtype_risk(test_finding_type)) expected_result = results[0] assert isinstance(expected_result, KATFindingType) @@ -20,6 +20,6 @@ def test_default_findingtype_risk_pending(): def test_default_findingtype_risk_unkown(): test_finding_type = KATFindingType(id="KAT-TEST", risk_severity=RiskLevelSeverity.UNKNOWN, risk_score=5) - results = list(run_default_findingtype_risk(test_finding_type, [], {})) + results = list(run_default_findingtype_risk(test_finding_type)) assert results == [], "Bit should not output anything when risk_severity or risk_score are set" diff --git a/octopoes/tests/test_bit_domain_verification.py b/octopoes/tests/test_bit_domain_verification.py index 1ad9bccc629..770bc014e22 100644 --- a/octopoes/tests/test_bit_domain_verification.py +++ b/octopoes/tests/test_bit_domain_verification.py @@ -1,4 +1,4 @@ -from bits.domain_owner_verification.domain_owner_verification import run +from nibbles.domain_owner_verification.domain_owner_verification import nibble from octopoes.models.ooi.dns.records import DNSNSRecord from octopoes.models.ooi.dns.zone import Hostname @@ -10,7 +10,7 @@ def test_verification_pending(): hostname = Hostname(name="example.com", network=network.reference) ns_hostname = Hostname(name="ns1.registrant-verification.ispapi.net", network=network.reference) ns_record = DNSNSRecord(hostname=hostname.reference, name_server_hostname=ns_hostname.reference, value="x") - results = list(run(ns_record, [], {})) + results = list(nibble(ns_record)) assert len(results) == 2 @@ -20,6 +20,6 @@ def test_no_verification_pending(): hostname = Hostname(name="example.com", network=network.reference) ns_hostname = Hostname(name="ns1.example.com", network=network.reference) ns_record = DNSNSRecord(hostname=hostname.reference, name_server_hostname=ns_hostname.reference, value="x") - results = list(run(ns_record, [], {})) + results = list(nibble(ns_record)) assert len(results) == 0 diff --git a/octopoes/tests/test_bit_expiring_certificate.py b/octopoes/tests/test_bit_expiring_certificate.py index 60c938cb21e..cfc8f30e75a 100644 --- a/octopoes/tests/test_bit_expiring_certificate.py +++ b/octopoes/tests/test_bit_expiring_certificate.py @@ -1,6 +1,6 @@ from _datetime import datetime, timedelta -from bits.expiring_certificate.expiring_certificate import run +from nibbles.expiring_certificate.expiring_certificate import nibble from octopoes.models.ooi.certificate import X509Certificate @@ -13,7 +13,7 @@ def test_expiring_cert_simple_success(): serial_number="abc123", ) - results = list(run(certificate, [], {})) + results = list(nibble(certificate)) assert len(results) == 0 @@ -26,7 +26,7 @@ def test_expiring_cert_simple_expired(): serial_number="abc123", ) - results = list(run(certificate, [], {})) + results = list(nibble(certificate)) assert len(results) == 2 @@ -40,6 +40,6 @@ def test_expiring_cert_simple_expires_soon(): expires_in=timedelta(days=2), ) - results = list(run(certificate, [], {})) + results = list(nibble(certificate)) assert len(results) == 2 diff --git a/octopoes/tests/test_bit_ports.py b/octopoes/tests/test_bit_ports.py index 4efbc3b065d..db1c4d180e0 100644 --- a/octopoes/tests/test_bit_ports.py +++ b/octopoes/tests/test_bit_ports.py @@ -1,5 +1,4 @@ from bits.port_classification_ip.port_classification_ip import run as run_port_classification -from bits.port_common.port_common import run as run_port_common from octopoes.models.ooi.findings import Finding from octopoes.models.ooi.network import IPAddressV4, IPPort @@ -74,37 +73,3 @@ def test_port_classification_udp_80(): finding = results[-1] assert isinstance(finding, Finding) assert finding.description == "Port 80/udp is not a common port and should possibly not be open." - - -def test_port_common_tcp_80(): - port = IPPort(address="fake", protocol="tcp", port=80) - results = list(run_port_common(port, [], {})) - - assert len(results) == 2 - finding = results[-1] - assert isinstance(finding, Finding) - assert finding.description == "Port 80/tcp is a common port and found to be open." - - -def test_port_common_tcp_22(): - port = IPPort(address="fake", protocol="tcp", port=22) - results = list(run_port_common(port, [], {})) - - assert not results - - -def test_port_common_udp_80(): - port = IPPort(address="fake", protocol="udp", port=80) - results = list(run_port_common(port, [], {})) - - assert not results - - -def test_port_common_udp_53(): - port = IPPort(address="fake", protocol="udp", port=53) - results = list(run_port_common(port, [], {})) - - assert len(results) == 2 - finding = results[-1] - assert isinstance(finding, Finding) - assert finding.description == "Port 53/udp is a common port and found to be open." diff --git a/octopoes/tests/test_bit_spf_discovery.py b/octopoes/tests/test_bit_spf_discovery.py index 0219c8cc339..b79c182c7fd 100644 --- a/octopoes/tests/test_bit_spf_discovery.py +++ b/octopoes/tests/test_bit_spf_discovery.py @@ -1,4 +1,4 @@ -from bits.spf_discovery.spf_discovery import run +from nibbles.spf_discovery.spf_discovery import nibble from octopoes.models import Reference from octopoes.models.ooi.dns.records import DNSTXTRecord @@ -6,6 +6,8 @@ from octopoes.models.ooi.findings import KATFindingType from octopoes.models.ooi.network import IPAddressV4 +STATIC_IP = ".".join((4 * "1 ").split()) + def test_spf_discovery_simple_success(): dnstxt_record = DNSTXTRecord( @@ -13,7 +15,7 @@ def test_spf_discovery_simple_success(): value="v=spf1 ip4:1.1.1.1 ~all exp=explain._spf.example.com", ) - results = list(run(dnstxt_record, [], {})) + results = list(nibble(dnstxt_record)) spf_record = DNSSPFRecord( dns_txt_record=dnstxt_record.reference, @@ -23,15 +25,18 @@ def test_spf_discovery_simple_success(): exp="explain._spf.example.com", ) - assert results[-1].dict() == spf_record.dict() + assert results[-1].model_dump() == spf_record.model_dump() - assert results[0].dict() == IPAddressV4(address="1.1.1.1", network=Reference.from_str("Network|internet")).dict() + assert ( + results[0].model_dump() + == IPAddressV4(address=STATIC_IP, network=Reference.from_str("Network|internet")).model_dump() + ) assert ( - results[1].dict() + results[1].model_dump() == DNSSPFMechanismIP( ip=Reference.from_str("IPAddressV4|internet|1.1.1.1"), spf_record=spf_record.reference, mechanism="ip4" - ).dict() + ).model_dump() ) @@ -40,7 +45,7 @@ def test_spf_discovery_invalid_(): hostname=Reference.from_str("Hostname|internet|example.com"), value="v=spf1 assdfsdf w rgw" ) - results = list(run(dnstxt_record, [], {})) + results = list(nibble(dnstxt_record)) assert results[0] == KATFindingType(id="KAT-INVALID-SPF") @@ -51,6 +56,6 @@ def test_spf_discovery_intermediate_success(): value="v=spf1 a:example.com mx mx:deferrals.domain.com ptr:otherdomain.com " "exists:example4.com ?include:example2.com ~all", ) - results = list(run(dnstxt_record, [], {})) + results = list(nibble(dnstxt_record)) assert len(results) == 12 diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index 96ba6b394ab..e2e4f154755 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -1,6 +1,7 @@ from bits.https_availability.https_availability import run as run_https_availability -from bits.oois_in_headers.oois_in_headers import run as run_oois_in_headers +from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers +from octopoes.models.ooi.config import Config from octopoes.models.ooi.findings import Finding from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import URL, HTTPHeader, HTTPHeaderURL, Website @@ -9,7 +10,7 @@ def test_url_extracted_by_oois_in_headers_url(): header = HTTPHeader(resource="", key="Location", value="https://www.example.com/") - results = list(run_oois_in_headers(header, [], {})) + results = list(run_oois_in_headers(header, Config(ooi=header.reference, bit_id="oois-in-headers", config={}))) url = results[0] assert isinstance(url, URL) @@ -25,7 +26,7 @@ def test_url_extracted_by_oois_in_headers_url(): def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): header = HTTPHeader(resource=http_resource_https.reference, key="Location", value="script.php") - results = list(run_oois_in_headers(header, [], {})) + results = list(run_oois_in_headers(header, Config(ooi=header.reference, bit_id="oois-in-headers", config={}))) url = results[0] assert isinstance(url, URL) diff --git a/octopoes/tests/test_disallowed_csp_hostnames.py b/octopoes/tests/test_disallowed_csp_hostnames.py index 60abf4b38c5..c4ae436fbfa 100644 --- a/octopoes/tests/test_disallowed_csp_hostnames.py +++ b/octopoes/tests/test_disallowed_csp_hostnames.py @@ -1,6 +1,7 @@ -from bits.disallowed_csp_hostnames.disallowed_csp_hostnames import run +from nibbles.disallowed_csp_hostnames.disallowed_csp_hostnames import nibble from octopoes.models import Reference +from octopoes.models.ooi.config import Config from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.web import HTTPHeaderHostname @@ -13,7 +14,12 @@ def test_disallowed_csp_headers_no_findings(): ), ) - results = list(run(http_header_hostname, [], {})) + results = list( + nibble( + http_header_hostname, + Config(ooi=http_header_hostname.reference, bit_id="disallowed-csp-hostnames", config={}), + ) + ) assert results == [] @@ -26,7 +32,12 @@ def test_disallowed_csp_headers_simple_finding(): ), ) - results = list(run(http_header_hostname, [], {})) + results = list( + nibble( + http_header_hostname, + Config(ooi=http_header_hostname.reference, bit_id="disallowed-csp-hostnames", config={}), + ) + ) assert results == [ KATFindingType(id="KAT-DISALLOWED-DOMAIN-IN-CSP"), @@ -44,7 +55,16 @@ def test_disallowed_csp_headers_allow_url_shortener(): ), ) - results = list(run(http_header_hostname, [], {"disallow_url_shorteners": False})) + results = list( + nibble( + http_header_hostname, + Config( + ooi=http_header_hostname.reference, + bit_id="disallowed-csp-hostnames", + config={"disallow_url_shorteners": False}, + ), + ) + ) assert results == [] @@ -57,7 +77,16 @@ def test_disallowed_csp_headers_disallow_custom_hostname(): ), ) - results = list(run(http_header_hostname, [], {"disallowed_hostnames": "example.com"})) + results = list( + nibble( + http_header_hostname, + Config( + ooi=http_header_hostname.reference, + bit_id="disallowed-csp-hostnames", + config={"disallowed_hostnames": "example.com"}, + ), + ) + ) assert results == [ KATFindingType(id="KAT-DISALLOWED-DOMAIN-IN-CSP"), diff --git a/octopoes/tests/test_octopoes_service.py b/octopoes/tests/test_octopoes_service.py index fad44d69a27..3d6f719dbc1 100644 --- a/octopoes/tests/test_octopoes_service.py +++ b/octopoes/tests/test_octopoes_service.py @@ -25,6 +25,8 @@ def mocked_bit_definitions(): def test_process_ooi_create_event(octopoes_service, valid_time): # upon creation of a new ooi ooi = Hostname(network=Network(name="internet").reference, name="example.com") + octopoes_service.nibbler = Mock() + octopoes_service.nibbler.infer.return_value = {} octopoes_service.process_event( OOIDBEvent( operation_type=OperationType.CREATE, valid_time=valid_time, client="_dev", old_data=None, new_data=ooi @@ -41,6 +43,8 @@ def test_process_ooi_create_event(octopoes_service, valid_time): def test_process_event_abstract_bit_consumes(octopoes_service, valid_time): # upon creation of a new ooi ooi = IPAddressV4(network=Network(name="internet").reference, address=ip_address("1.1.1.1")) + octopoes_service.nibbler = Mock() + octopoes_service.nibbler.infer.return_value = {} octopoes_service.process_event( OOIDBEvent( operation_type=OperationType.CREATE, valid_time=valid_time, client="_dev", old_data=None, new_data=ooi @@ -94,6 +98,8 @@ def test_on_create_scan_profile(octopoes_service, new_data, old_data, bit_runner octopoes_service.ooi_repository.get.return_value = Mock() octopoes_service.origin_parameter_repository.list_by_origin.return_value = {} octopoes_service.ooi_repository.load_bulk.return_value = {} + octopoes_service.nibbler = Mock() + octopoes_service.nibbler.infer.return_value = {} mock_oois = [Mock(reference="test1"), Mock(reference="test2")] bit_runner().run.return_value = mock_oois diff --git a/octopoes/tests/test_ooi_repository.py b/octopoes/tests/test_ooi_repository.py index b9bec6d4d5e..1c4c6e131de 100644 --- a/octopoes/tests/test_ooi_repository.py +++ b/octopoes/tests/test_ooi_repository.py @@ -9,6 +9,7 @@ from octopoes.models import OOI, Reference from octopoes.models.ooi.dns.zone import DNSZone from octopoes.models.ooi.network import IPAddressV4, Network +from octopoes.models.ooi.web import URL from octopoes.models.path import Direction, Path, Segment from octopoes.models.persistence import ReferenceField from octopoes.repositories.ooi_repository import XTDBOOIRepository @@ -132,3 +133,34 @@ def test_get_neighbours(self): resolved_hostname = neighbours[Path.parse("MockHostname.