-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Marketplace Contribution] Github Maltrail Feed (#30052)
* "pack contribution initial commit" * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml Co-authored-by: Moshe Eichler <[email protected]> * Update GithubMaltrailFeed.yml * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml Co-authored-by: Moshe Eichler <[email protected]> * Update GithubMaltrailFeed.py * Update GithubMaltrailFeed.py * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Update Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py Co-authored-by: Moshe Eichler <[email protected]> * Delete Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/integration-Github_Maltrail_Feed.yml * Update pack_metadata.json * Update GithubMaltrailFeed.yml * Update GithubMaltrailFeed.py * Update GithubMaltrailFeed.py * Update GithubMaltrailFeed.yml * Update GithubMaltrailFeed.py * Update GithubMaltrailFeed.yml * Update GithubMaltrailFeed.py --------- Co-authored-by: Abel S. Santamarina <[email protected]> Co-authored-by: Moshe Eichler <[email protected]>
- Loading branch information
3 people
authored
Nov 5, 2023
1 parent
8ba5554
commit 693e390
Showing
9 changed files
with
354 additions
and
0 deletions.
There are no files selected for viewing
Empty file.
Empty file.
209 changes: 209 additions & 0 deletions
209
Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,209 @@ | ||
| import demistomock as demisto # noqa: F401 | ||
| from CommonServerPython import * # noqa: F401 | ||
| import requests | ||
| import base64 | ||
| from datetime import datetime | ||
| import regex | ||
|
|
||
| # CONSTANTS | ||
| SOURCE_NAME = "Github Maltrail Feed" | ||
| DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' | ||
| COMMIT_LIMIT = 100 | ||
|
|
||
| # ############################## OVERWRITE REGEX FORMATTING ############################### | ||
| regexFlags = re.M # Multi line matching | ||
| REGEX_IP = r"\b(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(?:\[\.\]|\.)){3}(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\b" | ||
|
|
||
| class Client(BaseClient): | ||
|
|
||
| def __init__(self, params: dict): | ||
| self._verify: bool = not params.get('insecure', False) | ||
| self.user = params.get('user') | ||
| self.token = (params.get('api_token') or {}).get('password', '') | ||
| self.repo = params.get('repository') | ||
| self.url = params.get('base_url') | ||
| # self.base_url = f'{self.url}/{self.user}/{self.repo}' | ||
| self.base_url = urljoin(self.url, self.user) | ||
| self.base_url = urljoin(self.base_url, self.repo) | ||
| handle_proxy() | ||
|
|
||
| def http_request_indicators(self): | ||
| res = requests.get( | ||
| url=self.base_url, | ||
| verify=self._verify | ||
| ) | ||
| try: | ||
| res.raise_for_status() | ||
| except Exception: | ||
| demisto.info(f'Github Maltrail Feed - exception in request: {res.status_code!r} {res.content!r}') | ||
| raise | ||
| return res.text | ||
|
|
||
| def getclienturl(self): | ||
| return self.base_url | ||
|
|
||
| def http_request(self, url_endpoint, params: dict = None): | ||
| """The HTTP request for daily feeds. | ||
| Returns: | ||
| list. A list of indicators fetched from the feed. | ||
| """ | ||
| self.headers = { | ||
| 'Authorization': "Bearer " + self.token | ||
| } | ||
| res = requests.request( | ||
| method="GET", | ||
| url=urljoin(self.base_url, url_endpoint), | ||
| verify=self._verify, | ||
| headers=self.headers, | ||
| params=params | ||
| ) | ||
| return res | ||
|
|
||
|
|
||
| def fetch_indicators(client: Client, url: str, limit: int=None, params: dict=None): | ||
| if params: | ||
| feed_tags = argToList(params.get('feedTags', [])) | ||
| tlp_color = params.get('tlp_color') | ||
| response = client.http_request(url) | ||
| indicators_list = [] | ||
| demisto.debug('Fetch of indicators started ###') | ||
|
|
||
| if response.ok: | ||
| content = response.json()["content"] | ||
| file_content = base64.b64decode(content).decode("utf-8") | ||
| lines = file_content.split("\n") | ||
| for line in lines: | ||
| if '#' not in line and line != '': | ||
| type_ = auto_detect_indicator_type(line) | ||
| if regex.search(REGEX_IP, line): | ||
| if line.startswith('http://'): | ||
| line = line.removeprefix('http://') | ||
| elif line.startswith('https://'): | ||
| line = line.removeprefix('https://') | ||
| else: | ||
| line = line.split(':')[0] | ||
| type_ = "IP" | ||
| elif type_ == "URL": | ||
| if not line.startswith('http://') and not line.startswith('https://'): | ||
| line = 'http://' + line | ||
| raw_data = { | ||
| 'value': line, | ||
| 'type': type_, | ||
| } | ||
| indicator_obj = { | ||
| 'value': line, | ||
| 'type': type_, | ||
| 'service': "GitHub Maltrail Feed", | ||
| 'fields': {}, | ||
| 'rawJSON': raw_data | ||
| } | ||
| if feed_tags: | ||
| indicator_obj['fields']['tags'] = feed_tags | ||
| if tlp_color: | ||
| indicator_obj['fields']['trafficlightprotocol'] = tlp_color | ||
| indicators_list.append(indicator_obj) | ||
| # If limit is reached, break loop | ||
| if limit and isinstance(limit, int): | ||
| if len(indicators_list)>=limit: | ||
| break | ||
| else: | ||
| demisto.error(f"Error: {response.status_code} - {response.json()['message']}") | ||
| return indicators_list | ||
|
|
||
|
|
||
| def get_last_commit_date(client): | ||
| api_url = "/commits" | ||
| response = client.http_request(api_url) | ||
| last_commit_date = None | ||
| if response.ok: | ||
| commits = [] | ||
| page = 1 | ||
| while response.ok and page < COMMIT_LIMIT: | ||
| commits.extend(response.json()) | ||
| link_header = response.headers.get('Link') | ||
| if not link_header or 'rel="next"' not in link_header: | ||
| break | ||
| page += 1 | ||
| response = client.http_request(api_url, params={'page': page}) | ||
| for commit in commits: | ||
| if 'qakbot' in commit['commit']['message']: | ||
| commit_date = date_to_timestamp(parse_date_string(commit['commit']['author']['date'], DATE_FORMAT)) | ||
| if not last_commit_date: | ||
| last_commit_date = commit_date | ||
| elif commit_date > last_commit_date: | ||
| last_commit_date = commit_date | ||
|
|
||
| return last_commit_date | ||
|
|
||
|
|
||
| def fetch_indicators_command(client: Client, params: dict=None): | ||
| integration_context = get_integration_context() | ||
| api_url = "/contents/trails/static/malware/qakbot.txt" | ||
| indicators_list = [] | ||
| #First Fetch | ||
| if not integration_context: | ||
| time_of_first_fetch = date_to_timestamp(datetime.now(), DATE_FORMAT) | ||
| set_integration_context({'time_of_last_fetch': time_of_first_fetch}) | ||
| indicators_list = fetch_indicators(client, api_url, None, params) | ||
| else: | ||
| time_from_last_update = integration_context.get('time_of_last_fetch') | ||
| now = date_to_timestamp(datetime.now(), DATE_FORMAT) | ||
| last_commit_date = get_last_commit_date(client) | ||
| if last_commit_date > time_from_last_update: | ||
| indicators_list = fetch_indicators(client, api_url, None, params) | ||
| set_integration_context({'time_of_last_fetch': now}) | ||
| else: | ||
| demisto.debug(f'### Nothing to fetch') | ||
|
|
||
| return indicators_list | ||
|
|
||
|
|
||
| def get_indicators_command(client: Client, params: dict, args: dict): | ||
| try: | ||
| limit = int(args.get('limit', 50)) | ||
| except ValueError: | ||
| raise ValueError('The limit argument must be a number.') | ||
| api_url = "/contents/trails/static/malware/qakbot.txt" | ||
| indicators_list = fetch_indicators(client, api_url, limit, params) | ||
| entry_result = indicators_list[:limit] | ||
| human_readable = tableToMarkdown("Indicators from Github Maltrail:", entry_result, | ||
| headers=['value', 'type', 'firstseenbysource', 'lastseenbysource', 'name'], | ||
| removeNull=True) | ||
| return human_readable, {}, entry_result | ||
|
|
||
|
|
||
| def test_module_command(client: Client, params: dict, args: dict): | ||
| client.http_request_indicators() | ||
| return 'ok', {}, {} | ||
|
|
||
|
|
||
| def main(): | ||
| params = demisto.params() | ||
| args = demisto.args() | ||
|
|
||
| command = demisto.command() | ||
| demisto.info(f'Command being called is {command}') | ||
|
|
||
| # Switch case | ||
| commands = { | ||
| 'test-module': test_module_command, | ||
| 'gh-maltrail-get-indicators': get_indicators_command | ||
| } | ||
|
|
||
| try: | ||
| client = Client(params) | ||
| if command == 'fetch-indicators': | ||
| indicators = fetch_indicators_command(client, params) | ||
| for b in batch(indicators, batch_size=2000): | ||
| demisto.createIndicators(b) | ||
| else: | ||
| readable_output, outputs, raw_response = commands[command](client, params, args) | ||
| return_outputs(readable_output, outputs, raw_response) | ||
|
|
||
| except Exception as e: | ||
| raise Exception(f'Error in {SOURCE_NAME} Integration [{e}]') | ||
|
|
||
|
|
||
|
|
||
| if __name__ in ('__main__', '__builtin__', 'builtins'): | ||
| main() |
123 changes: 123 additions & 0 deletions
123
Packs/GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| category: Utilities | ||
| commonfields: | ||
| id: Github Maltrail Feed | ||
| version: -1 | ||
| configuration: | ||
| - additionalinfo: API Token | ||
| display: "" | ||
| displaypassword: API Token | ||
| hiddenusername: true | ||
| name: api_token | ||
| required: true | ||
| type: 9 | ||
| - display: 'Username of the repository owner, for example: github.com/repos/{user}/{repo}/issues' | ||
| name: user | ||
| required: true | ||
| type: 0 | ||
| - defaultvalue: https://api.github.com/repos | ||
| display: Base URL | ||
| name: base_url | ||
| required: true | ||
| type: 0 | ||
| - display: 'The name of the requested repository, for example: github.com/repos/{user}/{repo}/issues' | ||
| name: repository | ||
| required: true | ||
| type: 0 | ||
| - advanced: true | ||
| display: Trust any certificate (not secure) | ||
| name: insecure | ||
| required: false | ||
| section: Connect | ||
| type: 8 | ||
| - advanced: true | ||
| display: Use system proxy settings | ||
| name: proxy | ||
| required: false | ||
| section: Connect | ||
| type: 8 | ||
| - defaultvalue: "15" | ||
| display: Feed Fetch Interval | ||
| name: feedFetchInterval | ||
| required: false | ||
| type: 19 | ||
| - defaultvalue: "true" | ||
| display: Fetch indicators | ||
| name: feed | ||
| required: false | ||
| type: 8 | ||
| - additionalinfo: Indicators from this integration instance will be marked with this reputation | ||
| defaultvalue: Bad | ||
| display: Indicator Reputation | ||
| name: feedReputation | ||
| options: | ||
| - None | ||
| - Good | ||
| - Suspicious | ||
| - Bad | ||
| required: false | ||
| type: 18 | ||
| - additionalinfo: Reliability of the source providing the intelligence data | ||
| defaultvalue: F - Reliability cannot be judged | ||
| display: Source Reliability | ||
| name: feedReliability | ||
| options: | ||
| - A - Completely reliable | ||
| - B - Usually reliable | ||
| - C - Fairly reliable | ||
| - D - Not usually reliable | ||
| - E - Unreliable | ||
| - F - Reliability cannot be judged | ||
| required: true | ||
| type: 15 | ||
| - defaultvalue: indicatorType | ||
| display: "" | ||
| name: feedExpirationPolicy | ||
| options: | ||
| - never | ||
| - interval | ||
| - indicatorType | ||
| - suddenDeath | ||
| required: false | ||
| type: 17 | ||
| - defaultvalue: "20160" | ||
| display: "" | ||
| name: feedExpirationInterval | ||
| required: false | ||
| type: 1 | ||
| - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | ||
| display: Bypass exclusion list | ||
| name: feedBypassExclusionList | ||
| required: false | ||
| type: 8 | ||
| - additionalinfo: Supports CSV values. | ||
| display: Tags | ||
| name: feedTags | ||
| required: false | ||
| type: 0 | ||
| - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed | ||
| display: Traffic Light Protocol Color | ||
| name: tlp_color | ||
| options: | ||
| - RED | ||
| - AMBER | ||
| - GREEN | ||
| - WHITE | ||
| required: false | ||
| type: 15 | ||
| description: Fetches Indicators from Github Repo https://github.com/stamparm/maltrail | ||
| display: Github Maltrail Feed | ||
| name: Github Maltrail Feed | ||
| script: | ||
| commands: | ||
| - arguments: | ||
| - name: limit | ||
| description: The maximum number of results to return to the output. | ||
| defaultValue: "50" | ||
| name: gh-maltrail-get-indicators | ||
| description: Get indicators from the feed. | ||
| dockerimage: demisto/python3:3.10.13.78623 | ||
| feed: true | ||
| runonce: false | ||
| script: '' | ||
| subtype: python3 | ||
| type: python |
1 change: 1 addition & 0 deletions
1
...bMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed_description.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Fetches Indicators from Github Repo https://github.com/stamparm/maltrail |
Binary file added
BIN
+6.98 KB
...GithubMaltrailFeed/Integrations/GithubMaltrailFeed/GithubMaltrailFeed_image.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Empty file.
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "name": "Github Maltrail Feed", | ||
| "description": "Maltrail is a malicious traffic detection system, utilizing publicly available (black)lists containing malicious and/or generally suspicious trails, along with static trails compiled from various AV reports and custom user defined lists, where trail can be anything from domain name (e.g. zvpprsensinaix.com for Banjori malware), URL (e.g. hXXp://109.162.38.120/harsh02.exe for known malicious executable), IP address (e.g. 185.130.5.231 for known attacker) or HTTP User-Agent header value (e.g. sqlmap for automatic SQL injection and database takeover tool). Also, it uses (optional) advanced heuristic mechanisms that can help in discovery of unknown threats (e.g. new malware).\n\nhttps://github.com/stamparm/maltrail", | ||
| "support": "community", | ||
| "currentVersion": "1.0.0", | ||
| "author": "Abel S. Santamarina", | ||
| "url": "", | ||
| "email": "", | ||
| "created": "2023-10-04T21:52:43Z", | ||
| "categories": ['Utilities', 'Forensics & Malware Analysis', 'Data Enrichment & Threat Intelligence'], | ||
| "tags": [], | ||
| "useCases": [], | ||
| "keywords": [], | ||
| "marketplaces": [ | ||
| "xsoar", | ||
| "marketplacev2" | ||
| ], | ||
| "githubUser": [ | ||
| "asantamarina" | ||
| ] | ||
| } |