From 99a2c962ab856ecfbd7702e2aceaa463fe682782 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 14:29:29 +0200 Subject: [PATCH] Delete feed integration --- .../CategorizedFeeds/CategorizedFeeds.py | 250 ------------------ .../CategorizedFeeds/CategorizedFeeds.yml | 125 --------- .../CategorizedFeeds_description.md | 6 - .../CategorizedFeeds_image.png | Bin 2746 -> 0 bytes .../Integrations/CategorizedFeeds/README.md | 71 ----- .../GoogleThreatIntelligence.yml | 2 +- 6 files changed, 1 insertion(+), 453 deletions(-) delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py deleted file mode 100644 index dfcbad8db2eb..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ /dev/null @@ -1,250 +0,0 @@ -import demistomock as demisto # noqa: F401 -from CommonServerPython import * # noqa: F401 - -import bz2 -import io -import json -import tarfile -import urllib3 - -# Disable insecure warnings. -urllib3.disable_warnings() - - -FEED_STR = { - 'apt': 'APT', - 'cve': 'CVE', - 'iot': 'IoT', - 'mobile': 'Mobile', - 'ransomware': 'Ransomware', -} - - -def _get_current_hour(): - """Gets current hour for Threat feeds.""" - time_obj = datetime.utcnow() - timedelta(hours=2) - hour = time_obj.strftime('%Y%m%d%H') - return hour - - -def _get_indicators(response): - """Gets indicators from response.""" - indicators = [] - decompressed_data = bz2.decompress(response) - tar_bytes = io.BytesIO(decompressed_data) - with tarfile.open(fileobj=tar_bytes, mode='r:') as tar: - for member in tar.getmembers(): - file_data = tar.extractfile(member) - if file_data: - while line := file_data.readline(): - decoded_data = line.decode('utf-8') - indicator = json.loads(decoded_data) - indicators.append(indicator) - return indicators - - -class Client(BaseClient): - """Client for Google Threat Intelligence API.""" - - def fetch_indicators(self, feed_type: str = 'apt', hour: str = None): - """Fetches indicators given a feed type and an hour.""" - if not hour: - hour = _get_current_hour() - return self._http_request( - 'GET', - f'threat_feeds/{feed_type}/hourly/{hour}', - resp_type='content', - ) - - def get_threat_feed(self, feed_type: str) -> list: - """Retrieves matches for a given feed type.""" - last_threat_feed = demisto.getIntegrationContext().get('last_threat_feed') - - hour = _get_current_hour() - - if last_threat_feed == hour: - return [] - - response = self.fetch_indicators(feed_type, hour) - matches = _get_indicators(response) - demisto.setIntegrationContext({'last_threat_feed': hour}) - return matches - - -def test_module(client: Client) -> str: - client.fetch_indicators() - return 'ok' - - -def fetch_indicators_command(client: Client, - feed_type: str, - tlp_color: str = None, - feed_tags: list = None, - limit: int = 40) -> list[dict]: - """Retrieves indicators from the feed - Args: - client (Client): Client object with request - tlp_color (str): Traffic Light Protocol color - feed_tags (list): Tags to assign fetched indicators - limit (int): limit the results - Returns: - Indicators. - """ - iterator = client.get_threat_feed(feed_type) - indicators = [] - if limit > 0: - iterator = iterator[:limit] - - # extract values from iterator - for item in iterator: - attributes = item.get('attributes', {}) - type_ = FeedIndicatorType.File - raw_data = { - 'value': attributes, - 'type': type_, - } - - # Create indicator object for each value. - # The object consists of a dictionary with required and optional keys and values, as described blow. - indicator_obj = { - # The indicator value. - 'value': attributes['sha256'], - # The indicator type as defined in Cortex XSOAR. - # One can use the FeedIndicatorType class under CommonServerPython to populate this field. - 'type': type_, - # The name of the service supplying this feed. - 'service': 'Google Threat Intelligence', - # A dictionary that maps values to existing indicator fields defined in Cortex XSOAR. - # One can use this section in order to map custom indicator fields previously defined - # in Cortex XSOAR to their values. - 'fields': { - 'md5': attributes.get('md5'), - 'sha1': attributes.get('sha1'), - 'sha256': attributes.get('sha256'), - }, - # A dictionary of the raw data returned from the feed source about the indicator. - 'rawJSON': raw_data, - 'sha256': attributes['sha256'], - 'fileType': attributes.get('type_description'), - } - - if feed_tags: - indicator_obj['fields']['tags'] = feed_tags - - if tlp_color: - indicator_obj['fields']['trafficlightprotocol'] = tlp_color - - indicators.append(indicator_obj) - - return indicators - - -def get_indicators_command(client: Client, - params: Dict[str, str], - args: Dict[str, str]) -> CommandResults: - """Wrapper for retrieving indicators from the feed to the war-room. - Args: - client: Client object with request - params: demisto.params() - args: demisto.args() - Returns: - Outputs. - """ - feed_type = params.get('feed_type', 'apt') - limit = int(args.get('limit', params.get('limit', 40))) - tlp_color = params.get('tlp_color') - feed_tags = argToList(params.get('feedTags', '')) - indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) - - human_readable = tableToMarkdown( - f'Indicators from Google Threat Intelligence {FEED_STR.get(feed_type, feed_type)} Feeds:', - indicators, - headers=[ - 'sha256', - 'fileType', - ], - headerTransform=string_to_table_header, - removeNull=True, - ) - - return CommandResults( - readable_output=human_readable, - outputs_prefix='', - outputs_key_field='', - raw_response=indicators, - outputs={}, - ) - - -def reset_last_threat_feed(): - """Reset last threat feed from the integration context""" - demisto.setIntegrationContext({}) - return CommandResults(readable_output='Fetch history deleted successfully') - - -def main(): - """main function, parses params and runs command functions""" - params = demisto.params() - - # If your Client class inherits from BaseClient, SSL verification is - # handled out of the box by it, just pass ``verify_certificate`` to - # the Client constructor - insecure = not params.get('insecure', False) - - # If your Client class inherits from BaseClient, system proxy is handled - # out of the box by it, just pass ``proxy`` to the Client constructor - proxy = params.get('proxy', False) - - command = demisto.command() - args = demisto.args() - - demisto.debug(f'Command being called is {command}') - - try: - client = Client( - base_url='https://www.virustotal.com/api/v3/', - verify=insecure, - proxy=proxy, - headers={ - 'x-apikey': params['credentials']['password'], - 'x-tool': 'CortexGTIFeeds', - } - ) - - if command == 'test-module': - # This is the call made when pressing the integration Test button. - return_results(test_module(client)) - - elif command == 'gti-feed-get-indicators': - # This is the command that fetches a limited number of indicators - # from the feed source and displays them in the war room. - return_results(get_indicators_command(client, params, args)) - - elif command == 'gti-feed-reset-fetch-indicators': - return_results(reset_last_threat_feed()) - - elif command == 'fetch-indicators': - # This is the command that initiates a request to the feed endpoint - # and create new indicators objects from the data fetched. If the - # integration instance is configured to fetch indicators, then this - # is the commandthat will be executed at the specified feed fetch - # interval. - feed_type = params.get('feed_type', 'apt') - tlp_color = params.get('tlp_color') - feed_tags = argToList(params.get('feedTags')) - limit = int(params.get('limit', 40)) - indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) - for iter_ in batch(indicators, batch_size=2000): - demisto.createIndicators(iter_) - - else: - raise NotImplementedError(f'Command {command} is not implemented.') - - # Log exceptions and return errors - except Exception as e: - demisto.error(traceback.format_exc()) # Print the traceback - return_error(f'Failed to execute {command} command.\nError:\n{str(e)}') - - -if __name__ in ['__main__', 'builtin', 'builtins']: - main() diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml deleted file mode 100644 index 4b67a20684d1..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ /dev/null @@ -1,125 +0,0 @@ -category: Data Enrichment & Threat Intelligence -commonfields: - id: Google Threat Intelligence Feeds - version: -1 -configuration: -- display: API Key (leave empty. Fill in the API key in the password field.) - displaypassword: API Key - name: credentials - type: 9 - required: true - hiddenusername: true -- display: Feed type - name: feed_type - defaultvalue: apt - type: 15 - options: - - apt - - cve - - iot - - mobile - - ransomware -- display: Limit - name: limit - defaultvalue: 40 - type: 0 - additionalinfo: Limit of indicators to fetch from retrohunt job results. - required: false -- display: Fetch indicators - name: feed - defaultvalue: "true" - type: 8 - required: false -- display: Indicator Reputation - name: feedReputation - defaultvalue: feedInstanceReputationNotSet - type: 18 - options: - - None - - Good - - Suspicious - - Bad - additionalinfo: Indicators from this integration instance will be marked with this reputation. - required: false -- display: Source Reliability - name: feedReliability - defaultvalue: F - Reliability cannot be judged - type: 15 - required: true - options: - - A - Completely reliable - - B - Usually reliable - - C - Fairly reliable - - D - Not usually reliable - - E - Unreliable - - F - Reliability cannot be judged - additionalinfo: Reliability of the source providing the intelligence data. -- display: "" - name: feedExpirationPolicy - defaultvalue: indicatorType - type: 17 - options: - - never - - interval - - indicatorType - - suddenDeath - required: false -- display: "" - name: feedExpirationInterval - defaultvalue: "20160" - type: 1 - required: false -- display: Feed Fetch Interval - name: feedFetchInterval - defaultvalue: "30" - type: 19 - required: false -- display: Bypass exclusion list - name: feedBypassExclusionList - type: 8 - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. - required: false -- name: feedTags - display: Tags - type: 0 - additionalinfo: Supports CSV values. - required: false -- name: tlp_color - display: Traffic Light Protocol Color - options: - - RED - - AMBER - - GREEN - - WHITE - type: 15 - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. - required: false -- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified. - defaultvalue: 'true' - display: Incremental feed - hidden: true - name: feedIncremental - required: false - type: 8 -description: Use this feed integration to fetch Google Threat Intelligence Feeds matches. -display: Google Threat Intelligence Feeds -name: Google Threat Intelligence Feeds -script: - commands: - - arguments: - - name: limit - defaultValue: "40" - description: The maximum number of results to return. - description: Gets the matches from the latest Feed. - name: gti-feed-get-indicators - - description: "This command will reset your fetch history." - name: gti-feed-reset-fetch-indicators - dockerimage: demisto/python3:3.10.13.84405 - feed: true - runonce: false - script: "-" - subtype: python3 - type: python -fromversion: 5.5.0 -tests: -- No tests (auto formatted) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md deleted file mode 100644 index 37197218b3bf..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md +++ /dev/null @@ -1,6 +0,0 @@ -### Authorization: -Your API key can be found in your Google Threat Intelligence account user menu. -Your API key carries all your privileges, so keep it secure and don't share it with anyone. - -### Cortex XSOAR version 6.0.0 and below: -Fill anything in the "Username" fill box. Use you API key in the password fill box. \ No newline at end of file diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png deleted file mode 100644 index 950d9124d7b215cf15279f45a3fa2564cd170985..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2746 zcmV;r3PtsaP)Y!Wei$ZI^)U3uQ>uAZLp*z;lVN}(R9J=Im!RqcN2 z{j4G~FfcGMFfcGMFfcGMFnDty=oO(hpSnPj#!)Qesz@PjI+k4$!>biC_}UN?{N%)u z%vvBOg_N^{H*OcHN7Uw0Pw?ZmY4Sk!r5(p=MKaiSSh@gm#|6n8D4%&e@`Fk05qNeK zKP6mVOH&q$W~MghEXsv}G#P9wVhR4wvG3;svaZDqQWW;y_7~zJk=+zF%k}T2R3`R_|)u!NG%)`hXDREiK&} zPS)+)cW^;U%2FR#UcR@^U)jI^d+8wXA5ecNV$9Lj%E7_!6o}+z8nlY5f~$=G<>t_j@R;#=85R#FfbM1 zl=W-|3>4VU$>7Rk5znlwI1jKWCe!J3JO77LtrfPQ-G1UeBAGf#G9)xqyHaLxGhUx} zDgn`ycCPR{z4kZ96P8?1FA0)YuRBkBRM%zvQ`K(!Zq_J-sNiKQ^lH(=`&oB8^}tva zNU4{oafOgB>X=|nD1K!DMCYMVh95opXQgS3}EkGZP zfjGDMB}K)y>|q@)f#gg~GUiCxsl%IvQb`qUSrY%ieUJJ?vx*i|%gbLBXbWgn(UxT& zX#K#wBB`R=nVH7i-_A-YCRO6Tap91hDCA>BtA%b$gH|4On@*laI83;sTKew zq!?DYtruv}9R!=y23)q$Tu=np*8Hq)fm>Z{Jf$T|R3O!-y0*z22F;}w<1y_zKc~D) zTQp)kc-|LvxmI`bWY9Mv=QH-;;eaV~s|bdtdr~*pZ?ADjWs@nEUg@z^vrQHxO7S2G zl3`a5znDZltKd37 zuK|{2Au$r@+cf{tnJnetSR|h4R*>dK0vxYiU)UK)rYhI{)P!yMvpRRpBrl)%(JU_p zKV_FveqF`XGH4KoK2;Fv$^@BMBu^tv(b37m>daDi=!W3`I|M;c6)hQaMev!Ch3FQo z^E*}P=f8bd*P!mUD*xg)@3P2+4@!q_oxM_or%jqGIT1rSS#^IG{IBidLokiArvhfZb zAWlIImFX3L$=zzIvK2zu(@RTROI2=devu}MGrQ`0DuM+cv4N8V+ZcPY@P_dr)fS&S zbos;VowHZHU?3=2FKuO8p;pI^UOrQT6jx$mfuommXYcBb_Te2_;rG?Z#yJPx({{;f}Z% zHWP@fEgZT0L7vG2+3BGxACA|By(qP$N-@O?K$U|anGy5UpqNX0&jHDYWUwWG)So2o z)b|YIU}~tnXO5#hW0u%hs)QWNT+6vpeF1)!$}uyMkn|Zy=3-scu&W}KbgMS>;l5mE zsYc&iI2ToTY&Cf_S6JVOuMK)f6rRPQCNf*1`IPDt!M7bI-EA4OX|BMsJu>JU@St15$MAb9g1y3?s!mq({^IfT&$C>< zefBR!O5}@&KKmfHtt_1>=_7kZrxe+^u=6y!>l^+7Ex8)EfBh$D2r~4s4D6`;rK*{@a2@WM3&3aR!)+qI!J#M@$Z7NtJw4(`s{Qj z6*v=YTW_5$@gRJyQ?FU5^cSnCj8?~Y&Rm^Nwy(*?-m@K0FQ+;T65yjcD=TUbIIZij z>EGB5)P=cE-u9AGJo1MZ<6LGEZRrok_E=Jm?d_2(r)Su^Zk>?pSkhm-v>(8BFW|n= z+GphVFQDQZd(S*K8T5?3dkjG4DnLj?0 z3`c1=mSj>6RvrZAhpwFFvKml?=Y!lZxe-AG>*HHNQItUtrG#bdK=kK%T5Y_|Q| z2|2+V|E=}wPF3Fccvo@c{PQ&S?|2dysq;ms>xmm5?NSd%gcT$8oJGGwATml;NF+?e zU5=q|N48fZS~zNa{+<8i9TOI99qb%m;tbkG+CXgdBuFkqB!?YgpBR^1qti+jZ0KjvdYg z>J#k`zJCk8iFI2!V6L_4&c&E0rR4TMjoaYMC zedavR31PL^31xE+n~-SxAUbu(B3>1Kt-xfpdAN)}RY+(>-3#8ZkQtKe?K3Y7cH#`V z+l!6_YjE--u&uE~c+=ekAeLin6AFDj=$c>+mS`*RvtaHdAIk|h??t4-v7t|b9W6+< zyc?|TEP93Q-TyQ&FfcGMFfcGMFfcGMFnDwEKbKb6ZAYp4H~;_u07*qoM6N<$f}eOT AEdT%j diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md deleted file mode 100644 index 3366a1558f86..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md +++ /dev/null @@ -1,71 +0,0 @@ -Use this feed integration to fetch Google Threat Intelligence Feeds matches. It processes the latest finished job retrieving its matches based on the limit parameter (40 by default) in every fetch until there are no more matches for that job. - -## Configure Google Threat Intelligence Feeds on Cortex XSOAR - -1. Navigate to **Settings** > **Integrations** > **Servers & Services**. -2. Search for Google Threat Intelligence Feeds. -3. Click **Add instance** to create and configure a new integration instance. - - | **Parameter** | **Description** | **Required** | - | --- | --- | --- | - | API Key (leave empty. Fill in the API key in the password field.) | | True | - | API Key | | True | - | Feed type | | True | - | Limit | Limit of indicators to fetch from retrohunt job results. | False | - | Fetch indicators | | False | - | Indicator Reputation | Indicators from this integration instance will be marked with this reputation. | False | - | Source Reliability | Reliability of the source providing the intelligence data. | True | - | | | False | - | | | False | - | Feed Fetch Interval | | False | - | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | - | Tags | Supports CSV values. | False | - | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed. | False | - -4. Click **Test** to validate the URLs, token, and connection. -## Commands -You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. -After you successfully execute a command, a DBot message appears in the War Room with the command details. -### gti-feed-get-indicators -*** -Gets the matches from the latest feed. - -### gti-feed-reset-fetch-indicators -*** -Reset the last threat feed. - - - -#### Base Command - -`gti-feed-get-indicators` -#### Input - -| **Argument Name** | **Description** | **Required** | -| --- | --- | --- | -| limit | The maximum number of results to return. Default is 40. | Optional | - - -#### Context Output - -There is no context output for this command. - -#### Command Example -```!gti-feed-get-indicators``` -```!gti-feed-get-indicators limit=10``` - -#### Human Readable Output - -### Indicators from Google Threat Intelligence Feeds: -|Sha256|Filetype| -|---|---|---| -| 80db033dfe2b4e966d46a4ceed36e20b98a13891ce364a1308b90da7ad694cf3 | ELF | -| 6717c568e623551e600d315c7d1d634824a6f4b16e8aedfa298aefe7155313ff | ELF | -| 2c02a593ac714f9bac876d0a3c056384e0038505515d0c8472aa00ea36a6abb2 | ELF | -| e658b64650153c2207a76b2ee390b0fef04712d0da1d75a9eae25e4be596071a | ELF | -| 5ec2e17f25e800825ec5ed592c73303f840fa33cce2c8c4a4e7b6556798ffda0 | ELF | -| 771ba05ca9321dc723fc66b995c1d79a969330fc4242da6737cff1b364f978c8 | ELF | -| 4e3fac63a8b027788a10fd0191adf3ad59b2111324e1aa4eb4441723793c1b11 | ELF | -| ff1bdaf789643c6b934c9a9593fea82912d5974ba6ca0fd8dbf42db09ba82925 | ELF | -| 4371874f35538dc7d3b1d50df8cd0e8ad0744441ed487deb0d7a18a4a4373fea | ELF | - diff --git a/Packs/GoogleThreatIntelligence/Integrations/GoogleThreatIntelligence/GoogleThreatIntelligence.yml b/Packs/GoogleThreatIntelligence/Integrations/GoogleThreatIntelligence/GoogleThreatIntelligence.yml index 30041f3079f6..9bfc78ce1919 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/GoogleThreatIntelligence/GoogleThreatIntelligence.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/GoogleThreatIntelligence/GoogleThreatIntelligence.yml @@ -2303,6 +2303,6 @@ script: - contextPath: GoogleThreatIntelligence.Collection.collections.attributes.targeted_industries description: Targeted industries of the curated threat actors. type: list - dockerimage: demisto/python3:3.10.14.95956 + dockerimage: demisto/python3:3.11.9.101916 tests: - GoogleThreatIntelligence-test