Skip to content

Commit

Permalink
[greynoise-266] - Add greynoise-similar and greynoise-timeline comman…
Browse files Browse the repository at this point in the history 
…ds (#27067) (#27291)

* add sim and timeline updates

* fix test file prints

* update readme fix tests

* updates to readme for pre-commit

* updates from pre-commit run

* more linting updates

* update secrets

* fix url in timeline

Co-authored-by: Brad Chiappetta <[email protected]>
  • Loading branch information
@content-bot @bradchiappetta @ostolero
2 people authored and ostolero committed Jun 14, 2023
1 parent 643a0bc commit d33f75c
Show file tree
Hide file tree
Showing 36 changed files with 948 additions and 216 deletions.
7 changes: 7 additions & 0 deletions Packs/GreyNoise/.secrets-ignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,10 @@ https://greynoise.io
https://www.greynoise.io
66.249.68.82
103.21.244.0
121.239.23.85
1.145.159.157
45.95.147.229
61.30.129.190
59.88.225.2
1.1.2.2
45.164.214.212
209 changes: 189 additions & 20 deletions Packs/GreyNoise/Integrations/GreyNoise/GreyNoise.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,14 @@
import copy
from typing import Tuple, Dict, Any
from greynoise import GreyNoise, exceptions, util # type: ignore
from greynoise.exceptions import RequestFailure, RateLimitError # type: ignore

# Disable insecure warnings
urllib3.disable_warnings()
util.LOGGER.warning = util.LOGGER.debug

""" CONSTANTS """

TIMEOUT = 10
TIMEOUT = 30
PRETTY_KEY = {
"ip": "IP",
"first_seen": "First Seen",
Expand All @@ -33,24 +32,30 @@
"city": "City",
"country": "Country",
"country_code": "Country Code",
"destination_countries": "Destination Countries",
"destination_country_codes": "Destination Country Codes",
"organization": "Organization",
"category": "Category",
"sensor_count": "Sensor Count",
"sensor_hits": "Sensor Hits",
"source_country": "Source Country",
"source_country_code": "Source Country Code",
"tor": "Tor",
"rdns": "RDNS",
"rdns": "rDNS",
"os": "OS",
"region": "Region",
"vpn": "VPN",
"vpn_service": "VPN Service",
"raw_data": "raw_data",
"scan": "scan",
"port": "port",
"protocol": "protocol",
"web": "web",
"paths": "paths",
"useragents": "useragents",
"raw_data": "Raw Data",
"scan": "Scan",
"port": "Port",
"protocol": "Protocol",
"web": "Web",
"paths": "Paths",
"useragents": "User-Agents",
"ja3": "ja3",
"fingerprint": "fingerprint",
"hassh": "hassh",
"hassh": "HASSH",
"bot": "BOT",
}
IP_CONTEXT_HEADERS = [
Expand All @@ -66,6 +71,27 @@
"First Seen",
"Last Seen",
]
SIMILAR_HEADERS = [
"IP",
"Score",
"Classification",
"Actor",
"Organization",
"Source Country",
"Last Seen",
"Similarity Features"
]
TIMELINE_HEADERS = [
"Date",
"Classification",
"Tags",
"rDNS",
"Organization",
"ASN",
"Ports",
"Web Paths",
"User Agents",
]
RIOT_HEADERS = ["IP", "Category", "Name", "Trust Level", "Description", "Last Updated"]
API_SERVER = util.DEFAULT_CONFIG.get("api_server")
IP_QUICK_CHECK_HEADERS = ["IP", "Noise", "RIOT", "Code", "Code Description"]
Expand All @@ -74,7 +100,8 @@
"spoofable": "Spoofable",
"organizations": "Organizations",
"actors": "Actors",
"countries": "Countries",
"source_countries": "Source Countries",
"destination_countries": "Destination Countries",
"tags": "Tags",
"operating_systems": "Operating Systems",
"categories": "Categories",
Expand Down Expand Up @@ -102,14 +129,12 @@
"COMMAND_FAIL": "Failed to execute {} command.\n Error: {}",
"SERVER_ERROR": "The server encountered an internal error for GreyNoise and was unable to complete your request.",
"CONNECTION_TIMEOUT": "Connection timed out. Check your network connectivity.",
"PROXY": "Proxy Error - cannot connect to proxy. Either try clearing the "
"'Use system proxy' check-box or check the host, "
"authentication details and connection details for the proxy.",
"PROXY": "Proxy Error - cannot connect to proxy. Either try clearing the 'Use system proxy' check-box or check "
"the host, authentication details and connection details for the proxy.",
"INVALID_RESPONSE": "Invalid response from GreyNoise. Response: {}",
"QUERY_STATS_RESPONSE": "GreyNoise request failed. Reason: {}",
}


""" CLIENT CLASS """


Expand All @@ -131,10 +156,10 @@ def authenticate(self):
f"Invalid API Offering ({response['offering']})or Expiration Date ({expiration_date})"
)

except RateLimitError:
except exceptions.RateLimitError:
raise DemistoException(EXCEPTION_MESSAGES["API_RATE_LIMIT"])

except RequestFailure as err:
except exceptions.RequestFailure as err:
status_code = err.args[0]
body = str(err.args[1])

Expand Down Expand Up @@ -245,7 +270,7 @@ def get_ip_context_data(responses: list) -> list:


def get_ip_reputation_score(classification: str) -> Tuple[int, str]:
"""Get DBot score and human readable of score.
"""Get DBot score and human-readable of score.
:type classification: ``str``
:param classification: classification of ip provided from GreyNoise.
Expand Down Expand Up @@ -746,6 +771,8 @@ def stats_command(client: Client, args: dict) -> Any:
hr_list: list = []
if value is None:
continue
if key == "countries":
continue
for rec in value:
hr_rec: dict = {}
header = []
Expand All @@ -767,6 +794,140 @@ def stats_command(client: Client, args: dict) -> Any:
)


@exception_handler
@logger
def similarity_command(client: Client, args: dict) -> Any:
"""Get similarity information for a specified IP.
:type client: ``Client``
:param client: Client object for interaction with GreyNoise.
:type args: ``dict``
:param args: All command arguments, usually passed from ``demisto.args()``.
:return: A ``CommandResults`` object that is then passed to ``return_results``,
that contains the IP information.
:rtype: ``CommandResults``
"""
ip = args.get("ip", "")
min_score = args.get("minimum_score", 90)
limit = args.get("maximum_results", 50)
if isinstance(min_score, str):
min_score = int(min_score)
if isinstance(limit, str):
limit = int(limit)
response = client.similar(ip, min_score=min_score, limit=limit)
original_response = copy.deepcopy(response)
response = remove_empty_elements(response)
if not isinstance(response, dict):
raise DemistoException(EXCEPTION_MESSAGES["INVALID_RESPONSE"].format(response))

if response.get("similar_ips"):
tmp_response = []
for sim_ip in response.get("similar_ips", []):
modified_sim_ip = copy.deepcopy(sim_ip)
modified_sim_ip["IP"] = sim_ip.get("ip")
modified_sim_ip["Score"] = sim_ip.get("score", "0") * 100
modified_sim_ip["Classification"] = sim_ip.get("classification")
modified_sim_ip["Actor"] = sim_ip.get("actor")
modified_sim_ip["Organization"] = sim_ip.get("organization")
modified_sim_ip["Source Country"] = sim_ip.get("source_country")
modified_sim_ip["Last Seen"] = sim_ip.get("last_seen")
modified_sim_ip["Similarity Features"] = sim_ip.get("features")
tmp_response.append(modified_sim_ip)

human_readable = f"### IP: {ip} - Similar Internet Scanners found in GreyNoise\n"
human_readable += f'#### Total Similar IPs with Score above {min_score}%: {response.get("total")}\n'
if response.get('total', 0) > limit:
human_readable += f'##### Displaying {limit} results below. To see all results, visit the GreyNoise ' \
f'Visualizer.\n '

human_readable += tableToMarkdown(
name="GreyNoise Similar IPs", t=tmp_response, headers=SIMILAR_HEADERS, removeNull=True
)

similarity_link = f"https://viz.greynoise.io/ip-similarity/{ip}"
human_readable += f"\n*To view the detailed similarity result please click [here]({similarity_link}).*"

elif response["message"] == "ip not found":
human_readable = "### GreyNoise Similarity Lookup returned No Results."
viz_link = f"https://viz.greynoise.io/ip/{ip}"
human_readable += f"\n*To view this IP on the GreyNoise Visualizer please click [here]({viz_link}).*"

return CommandResults(
outputs_prefix="GreyNoise.Similar",
outputs_key_field="ip",
readable_output=human_readable, outputs=remove_empty_elements(response), raw_response=original_response
)


@exception_handler
@logger
def timeline_command(client: Client, args: dict) -> Any:
"""Get timeline information for a specified IP.
:type client: ``Client``
:param client: Client object for interaction with GreyNoise.
:type args: ``dict``
:param args: All command arguments, usually passed from ``demisto.args()``.
:return: A ``CommandResults`` object that is then passed to ``return_results``,
that contains the IP information.
:rtype: ``CommandResults``
"""
ip = args.get("ip", "")
days = args.get("days", 30)
limit = args.get("maximum_results", 50)
if isinstance(days, str):
days = int(days)
if isinstance(limit, str):
limit = int(limit)
response = client.timelinedaily(ip, days=days, limit=limit)
original_response = copy.deepcopy(response)
response = remove_empty_elements(response)
if not isinstance(response, dict):
raise DemistoException(EXCEPTION_MESSAGES["INVALID_RESPONSE"].format(response))

if response.get("activity"):
tmp_response = []
for activity in response.get("activity", []):
modified_activity = copy.deepcopy(activity)
modified_activity["Date"] = activity.get("timestamp").split("T")[0]
modified_activity["Classification"] = activity.get("classification")
tag_names = [tag["name"] for tag in activity.get("tags", [])]
modified_activity["Tags"] = tag_names
modified_activity["rDNS"] = activity.get("rdns")
modified_activity["Organization"] = activity.get("organization")
modified_activity["ASN"] = activity.get("asn")
ports = [str(item["port"]) + "/" + str(item["transport_protocol"]) for item in activity.get("protocols", [])]
modified_activity["Ports"] = ports
modified_activity["Web Paths"] = activity.get("http_paths")
modified_activity["User Agents"] = activity.get("http_user_agents")
tmp_response.append(modified_activity)

human_readable = f"### IP: {ip} - GreyNoise IP Timeline\n"

human_readable += tableToMarkdown(
name="Internet Scanner Timeline Details - Daily Activity Summary", t=tmp_response, headers=TIMELINE_HEADERS,
removeNull=True
)

timeline_link = f"https://viz.greynoise.io/ip/{ip}?view=timeline"
human_readable += f"\n*To view the detailed timeline result please click [here]({timeline_link}).*"

else:
human_readable = "### GreyNoise IP Timeline Returned No Results."
viz_link = f"https://viz.greynoise.io/ip/{ip}"
human_readable += f"\n*To view this IP on the GreyNoise Visualizer please click [here]({viz_link}).*"

return CommandResults(
outputs_prefix="GreyNoise.Timeline",
outputs_key_field="ip",
readable_output=human_readable, outputs=remove_empty_elements(response), raw_response=original_response
)


@exception_handler
@logger
def riot_command(client: Client, args: Dict, reliability: str) -> CommandResults:
Expand Down Expand Up @@ -963,7 +1124,7 @@ def main() -> None:
else:
packs = []

pack_version = "1.2.0"
pack_version = "1.3.0"
if isinstance(packs, list):
for pack in packs:
if pack["name"] == "GreyNoise":
Expand Down Expand Up @@ -1010,6 +1171,14 @@ def main() -> None:
result = stats_command(client, demisto.args())
return_results(result)

elif demisto.command() == "greynoise-similarity":
result = similarity_command(client, demisto.args())
return_results(result)

elif demisto.command() == "greynoise-timeline":
result = timeline_command(client, demisto.args())
return_results(result)

elif demisto.command() == "greynoise-query":
result = query_command(client, demisto.args())
return_results(result)
Expand Down
Loading

0 comments on commit d33f75c

Please sign in to comment.