From 8dec343e9b53b09df1e1ffa96d078c49d424f27d Mon Sep 17 00:00:00 2001 From: mtraceable Date: Mon, 24 Jul 2023 17:21:25 -0700 Subject: [PATCH 01/23] Added new fields to the Incidents - ipAddressType and apiType --- .../Integrations/Traceable/Traceable.py | 117 ++++++++- .../Integrations/Traceable/Traceable.yml | 4 +- .../Integrations/Traceable/Traceable_test.py | 243 ++++++++++++++++++ 3 files changed, 361 insertions(+), 3 deletions(-) diff --git a/Packs/Traceable/Integrations/Traceable/Traceable.py b/Packs/Traceable/Integrations/Traceable/Traceable.py index e291fc43468e..e6ea955d6381 100644 --- a/Packs/Traceable/Integrations/Traceable/Traceable.py +++ b/Packs/Traceable/Integrations/Traceable/Traceable.py @@ -10,6 +10,7 @@ from concurrent.futures import ThreadPoolExecutor import requests from requests.adapters import HTTPAdapter, Retry +import ipaddress # Disable insecure warnings @@ -196,6 +197,30 @@ } """ +api_entities_query = """query entities +{ + entities( + scope: "API" + limit: $limit + between: { + startTime: "$starttime" + endTime: "$endtime" + } + offset: 0 + $filter_by_clause + ) { + results { + id + name: attribute(expression: { key: "name" }) + isExternal: attribute(expression: { key: "isExternal" }) + isAuthenticated: attribute(expression: { key: "isAuthenticated" }) + riskScore: attribute(expression: { key: "riskScore" }) + riskScoreCategory: attribute(expression: { key: "riskScoreCategory" }) + } + total + } +}""" + class Helper: @staticmethod @@ -207,6 +232,10 @@ def construct_filterby_expression(*clauses): def datetime_to_string(d): return d.strftime(DATE_FORMAT) + @staticmethod + def string_to_datetime(s): + return datetime.strptime(s, DATE_FORMAT) + @staticmethod def construct_key_expression(key, value, _type="ATTRIBUTE", operator="IN"): if key is None: @@ -329,7 +358,7 @@ def set_ip_categories_list(self, ipCategoriesList): _ipCategoriesList.append("IP_LOCATION_TYPE_BOT") else: error = f"Unknown ipCategory {ipCategory} specified." - raise DemistoException(error) + raise Exception(error) self.ipCategoriesList = _ipCategoriesList def set_ip_abuse_velocity_list(self, ipAbuseVelocityList): @@ -479,6 +508,30 @@ def get_threat_events_query( ) return query + def get_api_endpoint_details_query(self, api_id_list, starttime, endtime): + filter_by_clause = Helper.construct_filterby_expression( + Helper.construct_key_expression("id", api_id_list) + ) + return Template(api_entities_query).substitute( + filter_by_clause=filter_by_clause, + limit=self.limit, + starttime=Helper.datetime_to_string(starttime), + endtime=Helper.datetime_to_string(endtime), + ) + + def get_api_endpoint_details(self, api_id_list, starttime, endtime): + if len(api_id_list) == 0: + return [] + query = self.get_api_endpoint_details_query(api_id_list, starttime, endtime) + result = self.graphql_query(query) + + if Helper.is_error(result, "data", "entities", "results"): + msg = "Error Object: " + json.dumps(result) + demisto.error(msg) + raise Exception(msg) + + return result["data"]["entities"]["results"] + def get_threat_events( self, starttime, @@ -500,6 +553,7 @@ def get_threat_events( events = [] first = True future_list = [] + api_id_list = [] with ThreadPoolExecutor(max_workers=self.span_fetch_threadpool) as executor: for domain_event in results: if Helper.is_error(domain_event, "traceId", "value"): @@ -517,6 +571,12 @@ def get_threat_events( trace_id = domain_event["traceId"]["value"] span_id = domain_event["spanId"]["value"] + if ( + domain_event["apiId"] is not None + and domain_event["apiId"]["value"] is not None + and domain_event["apiId"]["value"] != "null" + ): + api_id_list.append(domain_event["apiId"]["value"]) demisto.info("Forking thread for span retrieval") @@ -531,6 +591,14 @@ def get_threat_events( future_list.append((domain_event, future)) demisto.info("Completed thread for span retrieval") + api_endpoint_details = self.get_api_endpoint_details( + api_id_list, starttime, endtime + ) + api_endpoint_details_map = {} + for api_endpoint_detail in api_endpoint_details: + api_endpoint_details_map[api_endpoint_detail["id"]] = api_endpoint_detail + api_endpoint_details = None + for domain_event, future in future_list: trace_results = future.result() domain_event["type"] = "Exploit" @@ -560,6 +628,53 @@ def get_threat_events( domain_event["severity"] = domain_event["securityScoreCategory"][ "value" ] + if ( + domain_event["ipCategories"] is not None + and domain_event["ipCategories"]["value"] is not None + and len(domain_event["ipCategories"]["value"]) > 1 + and domain_event["actorIpAddress"] is not None + and domain_event["actorIpAddress"]["value"] is not None + ): + is_private = False + for ipCategory in domain_event["ipCategories"]["value"]: + if ( + ipCategory == "IP_LOCATION_TYPE_UNSPECIFIED" + and ipaddress.ip_address( + domain_event["actorIpAddress"]["value"] + ).is_private + ): + domain_event["ipAddressType"] = "Internal" + is_private = True + if not is_private: + domain_event["ipAddressType"] = "External" + + if ( + domain_event["apiId"] is not None + and domain_event["apiId"]["value"] is not None + and domain_event["apiId"]["value"] != "null" + and domain_event["apiId"]["value"] in api_endpoint_details_map + and api_endpoint_details_map[domain_event["apiId"]["value"]] is not None + and api_endpoint_details_map[domain_event["apiId"]["value"]][ + "isExternal" + ] + is not None + and api_endpoint_details_map[domain_event["apiId"]["value"]][ + "isExternal" + ] + != "null" + ): + if api_endpoint_details_map[domain_event["apiId"]["value"]][ + "isExternal" + ]: + domain_event["apiType"] = "External" + else: + domain_event["apiType"] = "Internal" + + if "ipAddressType" not in domain_event: + domain_event["ipAddressType"] = "Internal" + + if "apiType" not in domain_event: + domain_event["apiType"] = "Unknown" if Helper.is_error(trace_results, "data", "spans", "results"): msg = "Error Object: " + json.dumps(result) + ". Couldn't get the Span." diff --git a/Packs/Traceable/Integrations/Traceable/Traceable.yml b/Packs/Traceable/Integrations/Traceable/Traceable.yml index 2308d8ebce9c..d73de3bfb980 100644 --- a/Packs/Traceable/Integrations/Traceable/Traceable.yml +++ b/Packs/Traceable/Integrations/Traceable/Traceable.yml @@ -90,8 +90,8 @@ configuration: - UNKNOWN - display: "IP Location Type" name: ipCategories - type: 16 required: false + type: 16 options: - Anonymous VPN - Hosting Provider @@ -114,7 +114,7 @@ script: script: "-" type: python subtype: python3 - dockerimage: demisto/python3:3.10.12.63474 + dockerimage: demisto/python3:3.10.12.65389 fromversion: 5.5.0 tests: - No tests (auto formatted) diff --git a/Packs/Traceable/Integrations/Traceable/Traceable_test.py b/Packs/Traceable/Integrations/Traceable/Traceable_test.py index 846e03ca8669..4594349eabbb 100644 --- a/Packs/Traceable/Integrations/Traceable/Traceable_test.py +++ b/Packs/Traceable/Integrations/Traceable/Traceable_test.py @@ -58,6 +58,40 @@ } }""" +sample_api_result = """{ + "data": { + "entities": { + "results": [ + { + "id": "ea0f77c0-adc2-3a69-89ea-93b1c8341d8f", + "name": "POST /cart", + "isExternal": true, + "isAuthenticated": true, + "riskScore": 2, + "riskScoreCategory": "LOW" + }, + { + "id": "067bb0d7-3740-3ba6-89eb-c457491fbc53", + "name": "POST /get_user", + "isExternal": true, + "isAuthenticated": true, + "riskScore": 3, + "riskScoreCategory": "MEDIUM" + }, + { + "id": "be344182-c100-3287-874a-cb47eac709f2", + "name": "POST /cart", + "isExternal": false, + "isAuthenticated": true, + "riskScore": 2, + "riskScoreCategory": "LOW" + } + ], + "total": 3 + } + } +}""" + empty_domain_event = """{ "data": { "explore": { @@ -66,6 +100,113 @@ } }""" +sample_domain_event_empty_api = """{ + "data": { + "explore": { + "results": [ + { + "threatCategory": { + "value": "null" + }, + "id": { + "value": "9dd9261a-23db-472e-9d2a-a4c3227d6502" + }, + "name": { + "value": "XSS Filter - Category 1: Script Tag Vector" + }, + "type": { + "value": "Cross Site Scripting (XSS)" + }, + "environment": { + "value": "Fintech_app" + }, + "serviceName": { + "value": "frontend" + }, + "apiName": { + "value": "POST /get_user" + }, + "apiId": { + "value": "null" + }, + "serviceId": { + "value": "3d67aadf-4605-385d-bd3a-b297789046fd" + }, + "threatActorScore": { + "value": -2147483648 + }, + "anomalousAttribute": { + "value": "default.password" + }, + "eventDescription": { + "value": "Matched Data: