From a50361cc23fb81bdb7a7c886aecadebe664d70c1 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 12:35:33 +0200 Subject: [PATCH 01/19] Add feed integration --- .../CategorizedFeeds/CategorizedFeeds.py | 250 ++++++++++++++++++ .../CategorizedFeeds/CategorizedFeeds.yml | 125 +++++++++ .../CategorizedFeeds_description.md | 6 + .../CategorizedFeeds_image.png | Bin 0 -> 2746 bytes .../Integrations/CategorizedFeeds/README.md | 71 +++++ 5 files changed, 452 insertions(+) create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py new file mode 100644 index 000000000000..dfcbad8db2eb --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py @@ -0,0 +1,250 @@ +import demistomock as demisto # noqa: F401 +from CommonServerPython import * # noqa: F401 + +import bz2 +import io +import json +import tarfile +import urllib3 + +# Disable insecure warnings. +urllib3.disable_warnings() + + +FEED_STR = { + 'apt': 'APT', + 'cve': 'CVE', + 'iot': 'IoT', + 'mobile': 'Mobile', + 'ransomware': 'Ransomware', +} + + +def _get_current_hour(): + """Gets current hour for Threat feeds.""" + time_obj = datetime.utcnow() - timedelta(hours=2) + hour = time_obj.strftime('%Y%m%d%H') + return hour + + +def _get_indicators(response): + """Gets indicators from response.""" + indicators = [] + decompressed_data = bz2.decompress(response) + tar_bytes = io.BytesIO(decompressed_data) + with tarfile.open(fileobj=tar_bytes, mode='r:') as tar: + for member in tar.getmembers(): + file_data = tar.extractfile(member) + if file_data: + while line := file_data.readline(): + decoded_data = line.decode('utf-8') + indicator = json.loads(decoded_data) + indicators.append(indicator) + return indicators + + +class Client(BaseClient): + """Client for Google Threat Intelligence API.""" + + def fetch_indicators(self, feed_type: str = 'apt', hour: str = None): + """Fetches indicators given a feed type and an hour.""" + if not hour: + hour = _get_current_hour() + return self._http_request( + 'GET', + f'threat_feeds/{feed_type}/hourly/{hour}', + resp_type='content', + ) + + def get_threat_feed(self, feed_type: str) -> list: + """Retrieves matches for a given feed type.""" + last_threat_feed = demisto.getIntegrationContext().get('last_threat_feed') + + hour = _get_current_hour() + + if last_threat_feed == hour: + return [] + + response = self.fetch_indicators(feed_type, hour) + matches = _get_indicators(response) + demisto.setIntegrationContext({'last_threat_feed': hour}) + return matches + + +def test_module(client: Client) -> str: + client.fetch_indicators() + return 'ok' + + +def fetch_indicators_command(client: Client, + feed_type: str, + tlp_color: str = None, + feed_tags: list = None, + limit: int = 40) -> list[dict]: + """Retrieves indicators from the feed + Args: + client (Client): Client object with request + tlp_color (str): Traffic Light Protocol color + feed_tags (list): Tags to assign fetched indicators + limit (int): limit the results + Returns: + Indicators. + """ + iterator = client.get_threat_feed(feed_type) + indicators = [] + if limit > 0: + iterator = iterator[:limit] + + # extract values from iterator + for item in iterator: + attributes = item.get('attributes', {}) + type_ = FeedIndicatorType.File + raw_data = { + 'value': attributes, + 'type': type_, + } + + # Create indicator object for each value. + # The object consists of a dictionary with required and optional keys and values, as described blow. + indicator_obj = { + # The indicator value. + 'value': attributes['sha256'], + # The indicator type as defined in Cortex XSOAR. + # One can use the FeedIndicatorType class under CommonServerPython to populate this field. + 'type': type_, + # The name of the service supplying this feed. + 'service': 'Google Threat Intelligence', + # A dictionary that maps values to existing indicator fields defined in Cortex XSOAR. + # One can use this section in order to map custom indicator fields previously defined + # in Cortex XSOAR to their values. + 'fields': { + 'md5': attributes.get('md5'), + 'sha1': attributes.get('sha1'), + 'sha256': attributes.get('sha256'), + }, + # A dictionary of the raw data returned from the feed source about the indicator. + 'rawJSON': raw_data, + 'sha256': attributes['sha256'], + 'fileType': attributes.get('type_description'), + } + + if feed_tags: + indicator_obj['fields']['tags'] = feed_tags + + if tlp_color: + indicator_obj['fields']['trafficlightprotocol'] = tlp_color + + indicators.append(indicator_obj) + + return indicators + + +def get_indicators_command(client: Client, + params: Dict[str, str], + args: Dict[str, str]) -> CommandResults: + """Wrapper for retrieving indicators from the feed to the war-room. + Args: + client: Client object with request + params: demisto.params() + args: demisto.args() + Returns: + Outputs. + """ + feed_type = params.get('feed_type', 'apt') + limit = int(args.get('limit', params.get('limit', 40))) + tlp_color = params.get('tlp_color') + feed_tags = argToList(params.get('feedTags', '')) + indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) + + human_readable = tableToMarkdown( + f'Indicators from Google Threat Intelligence {FEED_STR.get(feed_type, feed_type)} Feeds:', + indicators, + headers=[ + 'sha256', + 'fileType', + ], + headerTransform=string_to_table_header, + removeNull=True, + ) + + return CommandResults( + readable_output=human_readable, + outputs_prefix='', + outputs_key_field='', + raw_response=indicators, + outputs={}, + ) + + +def reset_last_threat_feed(): + """Reset last threat feed from the integration context""" + demisto.setIntegrationContext({}) + return CommandResults(readable_output='Fetch history deleted successfully') + + +def main(): + """main function, parses params and runs command functions""" + params = demisto.params() + + # If your Client class inherits from BaseClient, SSL verification is + # handled out of the box by it, just pass ``verify_certificate`` to + # the Client constructor + insecure = not params.get('insecure', False) + + # If your Client class inherits from BaseClient, system proxy is handled + # out of the box by it, just pass ``proxy`` to the Client constructor + proxy = params.get('proxy', False) + + command = demisto.command() + args = demisto.args() + + demisto.debug(f'Command being called is {command}') + + try: + client = Client( + base_url='https://www.virustotal.com/api/v3/', + verify=insecure, + proxy=proxy, + headers={ + 'x-apikey': params['credentials']['password'], + 'x-tool': 'CortexGTIFeeds', + } + ) + + if command == 'test-module': + # This is the call made when pressing the integration Test button. + return_results(test_module(client)) + + elif command == 'gti-feed-get-indicators': + # This is the command that fetches a limited number of indicators + # from the feed source and displays them in the war room. + return_results(get_indicators_command(client, params, args)) + + elif command == 'gti-feed-reset-fetch-indicators': + return_results(reset_last_threat_feed()) + + elif command == 'fetch-indicators': + # This is the command that initiates a request to the feed endpoint + # and create new indicators objects from the data fetched. If the + # integration instance is configured to fetch indicators, then this + # is the commandthat will be executed at the specified feed fetch + # interval. + feed_type = params.get('feed_type', 'apt') + tlp_color = params.get('tlp_color') + feed_tags = argToList(params.get('feedTags')) + limit = int(params.get('limit', 40)) + indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) + for iter_ in batch(indicators, batch_size=2000): + demisto.createIndicators(iter_) + + else: + raise NotImplementedError(f'Command {command} is not implemented.') + + # Log exceptions and return errors + except Exception as e: + demisto.error(traceback.format_exc()) # Print the traceback + return_error(f'Failed to execute {command} command.\nError:\n{str(e)}') + + +if __name__ in ['__main__', 'builtin', 'builtins']: + main() diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml new file mode 100644 index 000000000000..4b67a20684d1 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -0,0 +1,125 @@ +category: Data Enrichment & Threat Intelligence +commonfields: + id: Google Threat Intelligence Feeds + version: -1 +configuration: +- display: API Key (leave empty. Fill in the API key in the password field.) + displaypassword: API Key + name: credentials + type: 9 + required: true + hiddenusername: true +- display: Feed type + name: feed_type + defaultvalue: apt + type: 15 + options: + - apt + - cve + - iot + - mobile + - ransomware +- display: Limit + name: limit + defaultvalue: 40 + type: 0 + additionalinfo: Limit of indicators to fetch from retrohunt job results. + required: false +- display: Fetch indicators + name: feed + defaultvalue: "true" + type: 8 + required: false +- display: Indicator Reputation + name: feedReputation + defaultvalue: feedInstanceReputationNotSet + type: 18 + options: + - None + - Good + - Suspicious + - Bad + additionalinfo: Indicators from this integration instance will be marked with this reputation. + required: false +- display: Source Reliability + name: feedReliability + defaultvalue: F - Reliability cannot be judged + type: 15 + required: true + options: + - A - Completely reliable + - B - Usually reliable + - C - Fairly reliable + - D - Not usually reliable + - E - Unreliable + - F - Reliability cannot be judged + additionalinfo: Reliability of the source providing the intelligence data. +- display: "" + name: feedExpirationPolicy + defaultvalue: indicatorType + type: 17 + options: + - never + - interval + - indicatorType + - suddenDeath + required: false +- display: "" + name: feedExpirationInterval + defaultvalue: "20160" + type: 1 + required: false +- display: Feed Fetch Interval + name: feedFetchInterval + defaultvalue: "30" + type: 19 + required: false +- display: Bypass exclusion list + name: feedBypassExclusionList + type: 8 + additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. + required: false +- name: feedTags + display: Tags + type: 0 + additionalinfo: Supports CSV values. + required: false +- name: tlp_color + display: Traffic Light Protocol Color + options: + - RED + - AMBER + - GREEN + - WHITE + type: 15 + additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. + required: false +- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified. + defaultvalue: 'true' + display: Incremental feed + hidden: true + name: feedIncremental + required: false + type: 8 +description: Use this feed integration to fetch Google Threat Intelligence Feeds matches. +display: Google Threat Intelligence Feeds +name: Google Threat Intelligence Feeds +script: + commands: + - arguments: + - name: limit + defaultValue: "40" + description: The maximum number of results to return. + description: Gets the matches from the latest Feed. + name: gti-feed-get-indicators + - description: "This command will reset your fetch history." + name: gti-feed-reset-fetch-indicators + dockerimage: demisto/python3:3.10.13.84405 + feed: true + runonce: false + script: "-" + subtype: python3 + type: python +fromversion: 5.5.0 +tests: +- No tests (auto formatted) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md new file mode 100644 index 000000000000..37197218b3bf --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md @@ -0,0 +1,6 @@ +### Authorization: +Your API key can be found in your Google Threat Intelligence account user menu. +Your API key carries all your privileges, so keep it secure and don't share it with anyone. + +### Cortex XSOAR version 6.0.0 and below: +Fill anything in the "Username" fill box. Use you API key in the password fill box. \ No newline at end of file diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png new file mode 100644 index 0000000000000000000000000000000000000000..950d9124d7b215cf15279f45a3fa2564cd170985 GIT binary patch literal 2746 zcmV;r3PtsaP)Y!Wei$ZI^)U3uQ>uAZLp*z;lVN}(R9J=Im!RqcN2 z{j4G~FfcGMFfcGMFfcGMFnDty=oO(hpSnPj#!)Qesz@PjI+k4$!>biC_}UN?{N%)u z%vvBOg_N^{H*OcHN7Uw0Pw?ZmY4Sk!r5(p=MKaiSSh@gm#|6n8D4%&e@`Fk05qNeK zKP6mVOH&q$W~MghEXsv}G#P9wVhR4wvG3;svaZDqQWW;y_7~zJk=+zF%k}T2R3`R_|)u!NG%)`hXDREiK&} zPS)+)cW^;U%2FR#UcR@^U)jI^d+8wXA5ecNV$9Lj%E7_!6o}+z8nlY5f~$=G<>t_j@R;#=85R#FfbM1 zl=W-|3>4VU$>7Rk5znlwI1jKWCe!J3JO77LtrfPQ-G1UeBAGf#G9)xqyHaLxGhUx} zDgn`ycCPR{z4kZ96P8?1FA0)YuRBkBRM%zvQ`K(!Zq_J-sNiKQ^lH(=`&oB8^}tva zNU4{oafOgB>X=|nD1K!DMCYMVh95opXQgS3}EkGZP zfjGDMB}K)y>|q@)f#gg~GUiCxsl%IvQb`qUSrY%ieUJJ?vx*i|%gbLBXbWgn(UxT& zX#K#wBB`R=nVH7i-_A-YCRO6Tap91hDCA>BtA%b$gH|4On@*laI83;sTKew zq!?DYtruv}9R!=y23)q$Tu=np*8Hq)fm>Z{Jf$T|R3O!-y0*z22F;}w<1y_zKc~D) zTQp)kc-|LvxmI`bWY9Mv=QH-;;eaV~s|bdtdr~*pZ?ADjWs@nEUg@z^vrQHxO7S2G zl3`a5znDZltKd37 zuK|{2Au$r@+cf{tnJnetSR|h4R*>dK0vxYiU)UK)rYhI{)P!yMvpRRpBrl)%(JU_p zKV_FveqF`XGH4KoK2;Fv$^@BMBu^tv(b37m>daDi=!W3`I|M;c6)hQaMev!Ch3FQo z^E*}P=f8bd*P!mUD*xg)@3P2+4@!q_oxM_or%jqGIT1rSS#^IG{IBidLokiArvhfZb zAWlIImFX3L$=zzIvK2zu(@RTROI2=devu}MGrQ`0DuM+cv4N8V+ZcPY@P_dr)fS&S zbos;VowHZHU?3=2FKuO8p;pI^UOrQT6jx$mfuommXYcBb_Te2_;rG?Z#yJPx({{;f}Z% zHWP@fEgZT0L7vG2+3BGxACA|By(qP$N-@O?K$U|anGy5UpqNX0&jHDYWUwWG)So2o z)b|YIU}~tnXO5#hW0u%hs)QWNT+6vpeF1)!$}uyMkn|Zy=3-scu&W}KbgMS>;l5mE zsYc&iI2ToTY&Cf_S6JVOuMK)f6rRPQCNf*1`IPDt!M7bI-EA4OX|BMsJu>JU@St15$MAb9g1y3?s!mq({^IfT&$C>< zefBR!O5}@&KKmfHtt_1>=_7kZrxe+^u=6y!>l^+7Ex8)EfBh$D2r~4s4D6`;rK*{@a2@WM3&3aR!)+qI!J#M@$Z7NtJw4(`s{Qj z6*v=YTW_5$@gRJyQ?FU5^cSnCj8?~Y&Rm^Nwy(*?-m@K0FQ+;T65yjcD=TUbIIZij z>EGB5)P=cE-u9AGJo1MZ<6LGEZRrok_E=Jm?d_2(r)Su^Zk>?pSkhm-v>(8BFW|n= z+GphVFQDQZd(S*K8T5?3dkjG4DnLj?0 z3`c1=mSj>6RvrZAhpwFFvKml?=Y!lZxe-AG>*HHNQItUtrG#bdK=kK%T5Y_|Q| z2|2+V|E=}wPF3Fccvo@c{PQ&S?|2dysq;ms>xmm5?NSd%gcT$8oJGGwATml;NF+?e zU5=q|N48fZS~zNa{+<8i9TOI99qb%m;tbkG+CXgdBuFkqB!?YgpBR^1qti+jZ0KjvdYg z>J#k`zJCk8iFI2!V6L_4&c&E0rR4TMjoaYMC zedavR31PL^31xE+n~-SxAUbu(B3>1Kt-xfpdAN)}RY+(>-3#8ZkQtKe?K3Y7cH#`V z+l!6_YjE--u&uE~c+=ekAeLin6AFDj=$c>+mS`*RvtaHdAIk|h??t4-v7t|b9W6+< zyc?|TEP93Q-TyQ&FfcGMFfcGMFfcGMFnDwEKbKb6ZAYp4H~;_u07*qoM6N<$f}eOT AEdT%j literal 0 HcmV?d00001 diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md new file mode 100644 index 000000000000..3366a1558f86 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md @@ -0,0 +1,71 @@ +Use this feed integration to fetch Google Threat Intelligence Feeds matches. It processes the latest finished job retrieving its matches based on the limit parameter (40 by default) in every fetch until there are no more matches for that job. + +## Configure Google Threat Intelligence Feeds on Cortex XSOAR + +1. Navigate to **Settings** > **Integrations** > **Servers & Services**. +2. Search for Google Threat Intelligence Feeds. +3. Click **Add instance** to create and configure a new integration instance. + + | **Parameter** | **Description** | **Required** | + | --- | --- | --- | + | API Key (leave empty. Fill in the API key in the password field.) | | True | + | API Key | | True | + | Feed type | | True | + | Limit | Limit of indicators to fetch from retrohunt job results. | False | + | Fetch indicators | | False | + | Indicator Reputation | Indicators from this integration instance will be marked with this reputation. | False | + | Source Reliability | Reliability of the source providing the intelligence data. | True | + | | | False | + | | | False | + | Feed Fetch Interval | | False | + | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | + | Tags | Supports CSV values. | False | + | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed. | False | + +4. Click **Test** to validate the URLs, token, and connection. +## Commands +You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. +After you successfully execute a command, a DBot message appears in the War Room with the command details. +### gti-feed-get-indicators +*** +Gets the matches from the latest feed. + +### gti-feed-reset-fetch-indicators +*** +Reset the last threat feed. + + + +#### Base Command + +`gti-feed-get-indicators` +#### Input + +| **Argument Name** | **Description** | **Required** | +| --- | --- | --- | +| limit | The maximum number of results to return. Default is 40. | Optional | + + +#### Context Output + +There is no context output for this command. + +#### Command Example +```!gti-feed-get-indicators``` +```!gti-feed-get-indicators limit=10``` + +#### Human Readable Output + +### Indicators from Google Threat Intelligence Feeds: +|Sha256|Filetype| +|---|---|---| +| 80db033dfe2b4e966d46a4ceed36e20b98a13891ce364a1308b90da7ad694cf3 | ELF | +| 6717c568e623551e600d315c7d1d634824a6f4b16e8aedfa298aefe7155313ff | ELF | +| 2c02a593ac714f9bac876d0a3c056384e0038505515d0c8472aa00ea36a6abb2 | ELF | +| e658b64650153c2207a76b2ee390b0fef04712d0da1d75a9eae25e4be596071a | ELF | +| 5ec2e17f25e800825ec5ed592c73303f840fa33cce2c8c4a4e7b6556798ffda0 | ELF | +| 771ba05ca9321dc723fc66b995c1d79a969330fc4242da6737cff1b364f978c8 | ELF | +| 4e3fac63a8b027788a10fd0191adf3ad59b2111324e1aa4eb4441723793c1b11 | ELF | +| ff1bdaf789643c6b934c9a9593fea82912d5974ba6ca0fd8dbf42db09ba82925 | ELF | +| 4371874f35538dc7d3b1d50df8cd0e8ad0744441ed487deb0d7a18a4a4373fea | ELF | + From 6627237b911ea199a6ae68765f6394471abd36fa Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 13:03:33 +0200 Subject: [PATCH 02/19] Add test --- .../CategorizedFeeds/CategorizedFeeds_test.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py new file mode 100644 index 000000000000..e65de1fcf9e4 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py @@ -0,0 +1,107 @@ +import CategorizedFeeds +import demistomock as demisto +from unittest.mock import call + + +MOCK_INDICATORS = [ + { + 'attributes': { + 'md5': 'md5_random', + 'sha1': 'sha1_random', + 'sha256': 'sha256_random', + 'type_description': 'random_type', + }, + 'type': 'file', + 'id': 'sha256_random', + }, + { + 'attributes': { + 'md5': 'md5_random2', + 'sha1': 'sha1_random2', + 'sha256': 'sha256_random2', + 'type_description': 'random_type2', + }, + 'type': 'file', + 'id': 'sha256_random2', + } +] + + +def test_fetch_indicators_command(mocker): + client = CategorizedFeeds.Client('https://fake') + mocker.patch.object(client, 'fetch_indicators', return_value=None) + mocker.patch.object(CategorizedFeeds, '_get_indicators', return_value=MOCK_INDICATORS) + + demisto.setIntegrationContext({}) + indicators = CategorizedFeeds.fetch_indicators_command(client, 'apt', [], limit=10) + + assert len(indicators) == 2 + assert indicators[0]['fields']['sha256'] == 'sha256_random' + assert indicators[0]['sha256'] == 'sha256_random' + assert indicators[0]['fileType'] == 'random_type' + assert indicators[1]['fields']['sha256'] == 'sha256_random2' + assert indicators[1]['sha256'] == 'sha256_random2' + assert indicators[1]['fileType'] == 'random_type2' + + +def test_fetch_indicators_limit_command(mocker): + client = CategorizedFeeds.Client('https://fake') + mocker.patch.object(client, 'fetch_indicators', return_value=None) + mocker.patch.object(CategorizedFeeds, '_get_indicators', return_value=MOCK_INDICATORS) + + demisto.setIntegrationContext({}) + indicators = CategorizedFeeds.fetch_indicators_command(client, 'apt', [], limit=1) + + assert len(indicators) == 1 + assert indicators[0]['fields']['sha256'] == 'sha256_random' + assert indicators[0]['sha256'] == 'sha256_random' + assert indicators[0]['fileType'] == 'random_type' + + +def test_main_manual_command(mocker): + params = { + 'feed_type': 'apt', + 'tlp_color': None, + 'feedTags': [], + 'credentials': {'password': 'xxx'}, + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='gti-feed-get-indicators') + get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'get_threat_feed') + + CategorizedFeeds.main() + + assert get_feed_mock.call_args == call('apt') + + +def test_main_default_command(mocker): + params = { + 'feed_type': 'iot', + 'tlp_color': None, + 'feedTags': [], + 'credentials': {'password': 'xxx'}, + 'limit': 10, + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='fetch-indicators') + get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'get_threat_feed') + + CategorizedFeeds.main() + + assert get_feed_mock.call_args == call('iot') + + +def test_main_test_command(mocker): + params = { + 'credentials': {'password': 'xxx'}, + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='test-module') + get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'fetch_indicators') + + CategorizedFeeds.main() + + assert get_feed_mock.call_count == 1 From f69a444327cbc68633d850d22d31b341d077e28e Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 14:24:32 +0200 Subject: [PATCH 03/19] Update docker image --- .../Integrations/CategorizedFeeds/CategorizedFeeds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml index 4b67a20684d1..941c94bc5b72 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -114,7 +114,7 @@ script: name: gti-feed-get-indicators - description: "This command will reset your fetch history." name: gti-feed-reset-fetch-indicators - dockerimage: demisto/python3:3.10.13.84405 + dockerimage: demisto/python3:3.11.9.101916 feed: true runonce: false script: "-" From 072340093d4eb63cff106481fc770034bf44ecff Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 14:27:00 +0200 Subject: [PATCH 04/19] Update md --- .../CategorizedFeeds/CategorizedFeeds_description.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md index 37197218b3bf..d007773d2cbf 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md @@ -2,5 +2,8 @@ Your API key can be found in your Google Threat Intelligence account user menu. Your API key carries all your privileges, so keep it secure and don't share it with anyone. +### Feed types: +Currently there are four feed types: apt, cve, iot, mobile and ransomware. + ### Cortex XSOAR version 6.0.0 and below: Fill anything in the "Username" fill box. Use you API key in the password fill box. \ No newline at end of file From 381c3975645f60f0fc4823fdb68a04c9175467ce Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 8 Jul 2024 14:27:23 +0200 Subject: [PATCH 05/19] typo --- .../CategorizedFeeds/CategorizedFeeds_description.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md index d007773d2cbf..f4c5c1f9ca70 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md @@ -3,7 +3,7 @@ Your API key can be found in your Google Threat Intelligence account user menu. Your API key carries all your privileges, so keep it secure and don't share it with anyone. ### Feed types: -Currently there are four feed types: apt, cve, iot, mobile and ransomware. +Currently there are five feed types: apt, cve, iot, mobile and ransomware. ### Cortex XSOAR version 6.0.0 and below: Fill anything in the "Username" fill box. Use you API key in the password fill box. \ No newline at end of file From 161773e08b1398f6ee17439da31adfd7f9b5babc Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 9 Jul 2024 15:17:30 +0200 Subject: [PATCH 06/19] Delete limit and add fields --- .../CategorizedFeeds/CategorizedFeeds.py | 39 ++++++++++++++----- .../CategorizedFeeds/CategorizedFeeds.yml | 10 +---- .../CategorizedFeeds/CategorizedFeeds_test.py | 1 - 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py index dfcbad8db2eb..202512cfbaf5 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py @@ -80,30 +80,40 @@ def fetch_indicators_command(client: Client, feed_type: str, tlp_color: str = None, feed_tags: list = None, - limit: int = 40) -> list[dict]: + limit: int = None) -> list[dict]: """Retrieves indicators from the feed Args: client (Client): Client object with request tlp_color (str): Traffic Light Protocol color feed_tags (list): Tags to assign fetched indicators - limit (int): limit the results + limit (int): Limit the results Returns: Indicators. """ iterator = client.get_threat_feed(feed_type) indicators = [] - if limit > 0: + + if limit: iterator = iterator[:limit] # extract values from iterator for item in iterator: attributes = item.get('attributes', {}) + type_ = FeedIndicatorType.File raw_data = { - 'value': attributes, + 'value': attributes['sha256'], 'type': type_, + 'attributes': attributes, } + gti_assessment = attributes.get('gti_assessment', {}) + attribution = attributes.get('attribution', {}) + malware_families = [x['family'] for x in attribution.get('malware_families', [])] + malware_families = list(set(malware_families)) + threat_actors = attribution.get('threat_actors', []) + threat_actors = list(set(threat_actors)) + # Create indicator object for each value. # The object consists of a dictionary with required and optional keys and values, as described blow. indicator_obj = { @@ -121,10 +131,17 @@ def fetch_indicators_command(client: Client, 'md5': attributes.get('md5'), 'sha1': attributes.get('sha1'), 'sha256': attributes.get('sha256'), + 'size': attributes.get('size'), + 'tags': attributes.get('tags'), }, # A dictionary of the raw data returned from the feed source about the indicator. 'rawJSON': raw_data, 'sha256': attributes['sha256'], + 'gti_threat_score': gti_assessment.get('threat_score', {}).get('value'), + 'gti_severity': gti_assessment.get('severity', {}).get('value'), + 'gti_verdict': gti_assessment.get('verdict', {}).get('value'), + 'malware_families': malware_families or None, + 'threat_actors': threat_actors or None, 'fileType': attributes.get('type_description'), } @@ -140,8 +157,8 @@ def fetch_indicators_command(client: Client, def get_indicators_command(client: Client, - params: Dict[str, str], - args: Dict[str, str]) -> CommandResults: + params: dict[str, str], + args: dict[str, str]) -> CommandResults: """Wrapper for retrieving indicators from the feed to the war-room. Args: client: Client object with request @@ -151,9 +168,9 @@ def get_indicators_command(client: Client, Outputs. """ feed_type = params.get('feed_type', 'apt') - limit = int(args.get('limit', params.get('limit', 40))) tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) + limit = int(args.get('limit', 0)) indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) human_readable = tableToMarkdown( @@ -162,6 +179,11 @@ def get_indicators_command(client: Client, headers=[ 'sha256', 'fileType', + 'gti_threat_score', + 'gti_severity', + 'gti_verdict', + 'malware_families', + 'threat_actors', ], headerTransform=string_to_table_header, removeNull=True, @@ -232,8 +254,7 @@ def main(): feed_type = params.get('feed_type', 'apt') tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags')) - limit = int(params.get('limit', 40)) - indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) + indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags) for iter_ in batch(indicators, batch_size=2000): demisto.createIndicators(iter_) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml index 941c94bc5b72..9bb831ddcb1a 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -19,12 +19,6 @@ configuration: - iot - mobile - ransomware -- display: Limit - name: limit - defaultvalue: 40 - type: 0 - additionalinfo: Limit of indicators to fetch from retrohunt job results. - required: false - display: Fetch indicators name: feed defaultvalue: "true" @@ -108,8 +102,8 @@ script: commands: - arguments: - name: limit - defaultValue: "40" - description: The maximum number of results to return. + defaultValue: "0" + description: The maximum number of results to return. If 0 all results will be returned. description: Gets the matches from the latest Feed. name: gti-feed-get-indicators - description: "This command will reset your fetch history." diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py index e65de1fcf9e4..c6dfbcd3472d 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py @@ -81,7 +81,6 @@ def test_main_default_command(mocker): 'tlp_color': None, 'feedTags': [], 'credentials': {'password': 'xxx'}, - 'limit': 10, } mocker.patch.object(demisto, 'params', return_value=params) From 9311327b1010e21050a1c5177f5fc4f411bcb3fa Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 5 Aug 2024 15:17:15 +0200 Subject: [PATCH 07/19] Add feed minimum GTI score --- .../CategorizedFeeds/CategorizedFeeds.py | 16 +++++++++++++--- .../CategorizedFeeds/CategorizedFeeds.yml | 6 ++++++ .../Integrations/CategorizedFeeds/README.md | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py index 202512cfbaf5..a289ef3e6c83 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py @@ -80,7 +80,8 @@ def fetch_indicators_command(client: Client, feed_type: str, tlp_color: str = None, feed_tags: list = None, - limit: int = None) -> list[dict]: + limit: int = None, + minimum_score: int = 0) -> list[dict]: """Retrieves indicators from the feed Args: client (Client): Client object with request @@ -151,7 +152,8 @@ def fetch_indicators_command(client: Client, if tlp_color: indicator_obj['fields']['trafficlightprotocol'] = tlp_color - indicators.append(indicator_obj) + if (indicator_obj.get('gti_threat_score') or 0) >= minimum_score: + indicators.append(indicator_obj) return indicators @@ -171,7 +173,15 @@ def get_indicators_command(client: Client, tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) limit = int(args.get('limit', 0)) - indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags, limit) + minimum_score = int(params.get('feedMinimumGTIScore', 80)) + indicators = fetch_indicators_command( + client, + feed_type, + tlp_color, + feed_tags, + limit, + minimum_score + ) human_readable = tableToMarkdown( f'Indicators from Google Threat Intelligence {FEED_STR.get(feed_type, feed_type)} Feeds:', diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml index 9bb831ddcb1a..f0853e15472e 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -68,6 +68,12 @@ configuration: defaultvalue: "30" type: 19 required: false +- name: feedMinimumGTIScore + type: 0 + display: Feed Minimum GTI Score + required: true + defaultvalue: 80 + additionalinfo: The minimum GTI score to import as part of the feed - display: Bypass exclusion list name: feedBypassExclusionList type: 8 diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md index 3366a1558f86..5186cff32ad0 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md @@ -18,6 +18,7 @@ Use this feed integration to fetch Google Threat Intelligence Feeds matches. It | | | False | | | | False | | Feed Fetch Interval | | False | + | Feed Minimum GTI Score | The minimum GTI score to import as part of the feed | True | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | | Tags | Supports CSV values. | False | | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed. | False | From 561d9ee6f460f990423def42baf3ccda7d3a1fbd Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 5 Aug 2024 16:22:24 +0200 Subject: [PATCH 08/19] Updating old iocs --- .../Integrations/CategorizedFeeds/CategorizedFeeds.py | 6 +++++- .../Integrations/CategorizedFeeds/CategorizedFeeds.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py index a289ef3e6c83..09db3973bbfd 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py @@ -131,7 +131,7 @@ def fetch_indicators_command(client: Client, 'fields': { 'md5': attributes.get('md5'), 'sha1': attributes.get('sha1'), - 'sha256': attributes.get('sha256'), + 'sha256': attributes['sha256'], 'size': attributes.get('size'), 'tags': attributes.get('tags'), }, @@ -154,6 +154,10 @@ def fetch_indicators_command(client: Client, if (indicator_obj.get('gti_threat_score') or 0) >= minimum_score: indicators.append(indicator_obj) + else: + existing_indicators = list(IndicatorsSearcher(value=indicator_obj['value'])) + if len(existing_indicators) > 0 and int(existing_indicators[0].get('total', 0)) > 0: + indicators.append(indicator_obj) return indicators diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml index f0853e15472e..c770b8c22ecc 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -65,7 +65,7 @@ configuration: required: false - display: Feed Fetch Interval name: feedFetchInterval - defaultvalue: "30" + defaultvalue: "60" type: 19 required: false - name: feedMinimumGTIScore From 03fe9cede466206c4b0f9ee05f52afe1d073a0bf Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 5 Aug 2024 16:26:50 +0200 Subject: [PATCH 09/19] nit --- .../Integrations/CategorizedFeeds/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md index 5186cff32ad0..2b119599ac90 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md @@ -18,7 +18,7 @@ Use this feed integration to fetch Google Threat Intelligence Feeds matches. It | | | False | | | | False | | Feed Fetch Interval | | False | - | Feed Minimum GTI Score | The minimum GTI score to import as part of the feed | True | + | Feed Minimum GTI Score | The minimum GTI score to import as part of the feed. | True | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | | Tags | Supports CSV values. | False | | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed. | False | From 863adea828be13d30397988e83a7c5838a150a5d Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Mon, 10 Feb 2025 14:05:04 +0100 Subject: [PATCH 10/19] Update --- .../CategorizedFeeds/CategorizedFeeds.py | 304 +++++++++++++----- .../CategorizedFeeds/CategorizedFeeds.yml | 15 +- 2 files changed, 236 insertions(+), 83 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py index 09db3973bbfd..5a5c30af40e6 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py @@ -1,10 +1,6 @@ import demistomock as demisto # noqa: F401 from CommonServerPython import * # noqa: F401 -import bz2 -import io -import json -import tarfile import urllib3 # Disable insecure warnings. @@ -12,11 +8,20 @@ FEED_STR = { - 'apt': 'APT', - 'cve': 'CVE', + 'cryptominer': 'Cryptominer', + 'first-stage-delivery-vectors': 'First stage delivery vectors', + 'infostealer': 'Infostealer', 'iot': 'IoT', + 'linux': 'Linux', + 'malicious-network-infrastructure': 'Malicious network infrastructure', + 'malware': 'Malware', 'mobile': 'Mobile', + 'osx': 'OSX', + 'phishing': 'Phishing', 'ransomware': 'Ransomware', + 'threat-actor': 'Threat actor', + 'trending': 'Trending', + 'vulnerability-weaponization': 'Vulnerability weaponization', } @@ -27,48 +32,42 @@ def _get_current_hour(): return hour -def _get_indicators(response): - """Gets indicators from response.""" - indicators = [] - decompressed_data = bz2.decompress(response) - tar_bytes = io.BytesIO(decompressed_data) - with tarfile.open(fileobj=tar_bytes, mode='r:') as tar: - for member in tar.getmembers(): - file_data = tar.extractfile(member) - if file_data: - while line := file_data.readline(): - decoded_data = line.decode('utf-8') - indicator = json.loads(decoded_data) - indicators.append(indicator) - return indicators +class DetectionRatio: + """Class for detections.""" + malicious = 0 + total = 0 + + def __init__(self, last_analysis_stats: dict): + self.malicious = last_analysis_stats.get('malicious', 0) + self.total = sum(last_analysis_stats.values()) + + def __repr__(self): + return f'{self.malicious}/{self.total}' class Client(BaseClient): """Client for Google Threat Intelligence API.""" - def fetch_indicators(self, feed_type: str = 'apt', hour: str = None): + def fetch_indicators(self, feed_type: str = 'malware', hour: str = None) -> dict: """Fetches indicators given a feed type and an hour.""" if not hour: hour = _get_current_hour() return self._http_request( 'GET', - f'threat_feeds/{feed_type}/hourly/{hour}', - resp_type='content', + f'threat_lists/{feed_type}/{hour}', ) def get_threat_feed(self, feed_type: str) -> list: """Retrieves matches for a given feed type.""" last_threat_feed = demisto.getIntegrationContext().get('last_threat_feed') - hour = _get_current_hour() if last_threat_feed == hour: return [] response = self.fetch_indicators(feed_type, hour) - matches = _get_indicators(response) demisto.setIntegrationContext({'last_threat_feed': hour}) - return matches + return response.get('iocs', []) def test_module(client: Client) -> str: @@ -76,13 +75,195 @@ def test_module(client: Client) -> str: return 'ok' +def _add_gti_attributes(indicator_obj: dict, item: dict): + """Addes GTI attributes.""" + + # GTI assessment + attributes = item.get('attributes', {}) + gti_assessment = attributes.get('gti_assessment', {}) + gti_threat_score = gti_assessment.get('threat_score', {}).get('value') + gti_severity = gti_assessment.get('severity', {}).get('value') + gti_verdict = gti_assessment.get('verdict', {}).get('value') + + # Relationships + relationships = item.get('relationships', {}) + malware_families: list[str] = [ + x['attributes']['name'] + for x in relationships.get('malware_families', {}).get('data', []) + ] + malware_families = list(set(malware_families)) + threat_actors: list[str] = [ + x['attributes']['name'] + for x in relationships.get('threat_actors', {}).get('data', []) + ] + threat_actors = list(set(threat_actors)) + + indicator_obj['fields'].update({ + 'gtithreatscore': gti_threat_score, + 'gtiseverity': gti_severity, + 'gtiverdict': gti_verdict, + 'malwarefamily': malware_families, + 'actor': threat_actors, + }) + indicator_obj.update({ + 'gti_threat_score': gti_threat_score, + 'gti_severity': gti_severity, + 'gti_verdict': gti_verdict, + 'malware_families': malware_families, + 'threat_actors': threat_actors, + 'relationships': [ + EntityRelationship( + name=EntityRelationship.Relationships.PART_OF, + entity_a=indicator_obj['value'], + entity_a_type=indicator_obj['type'], + entity_b=malware_family.title(), + entity_b_type=ThreatIntel.ObjectsNames.MALWARE, + reverse_name=EntityRelationship.Relationships.CONTAINS, + ).to_indicator() for malware_family in malware_families + ] + [ + EntityRelationship( + name=EntityRelationship.Relationships.ATTRIBUTED_BY, + entity_a=indicator_obj['value'], + entity_a_type=indicator_obj['type'], + entity_b=threat_actor.title(), + entity_b_type=ThreatIntel.ObjectsNames.THREAT_ACTOR, + reverse_name=EntityRelationship.Relationships.ATTRIBUTED_TO, + ).to_indicator() for threat_actor in threat_actors + ], + }) + + return indicator_obj + + +def _get_indicator_type(item: dict): + """Gets indicator type.""" + if item['type'] == 'file': + return FeedIndicatorType.File + if item['type'] == 'domain': + return FeedIndicatorType.Domain + if item['type'] == 'url': + return FeedIndicatorType.URL + if item['type'] == 'ip_address': + return FeedIndicatorType.IP + raise ValueError(f'Unknown type: {item["type"]}. ID: {item["id"]}') + + +def _get_indicator_id(item: dict) -> str: + """Gets indicator ID.""" + if item['type'] == 'url': + return item.get('attributes', {}).get('url') or item['id'] + return item['id'] + + +def _add_file_attributes(indicator_obj: dict, attributes: dict) -> dict: + """Adds file attributes.""" + indicator_obj['fields'].update({ + 'md5': attributes.get('md5'), + 'sha1': attributes.get('sha1'), + 'sha256': attributes.get('sha256'), + 'ssdeep': attributes.get('ssdeep'), + 'fileextension': attributes.get('type_extension'), + 'filetype': attributes.get('type_tag'), + 'imphash': attributes.get('pe_info', {}).get('imphash'), + 'displayname': attributes.get('meaningful_name'), + 'name': attributes.get('meaningful_name'), + 'size': attributes.get('size'), + 'creationdate': attributes.get('creation_date'), + 'firstseenbysource': attributes.get('first_submission_date'), + 'lastseenbysource': attributes.get('last_submission_date'), + }) + + return indicator_obj + + +def _add_domain_attributes(indicator_obj: dict, attributes: dict) -> dict: + """Adds domain attributes.""" + indicator_obj['fields'].update({ + 'creationdate': attributes.get('creation_date'), + 'registrarname': attributes.get('registrar'), + 'firstseenbysource': attributes.get('first_seen_itw_date'), + 'lastseenbysource': attributes.get('last_seen_itw_date'), + }) + + return indicator_obj + + +def _add_url_attributes(indicator_obj: dict, attributes: dict) -> dict: + """Adds URL attributes.""" + indicator_obj['fields'].update({ + 'firstseenbysource': attributes.get('first_submission_date'), + 'lastseenbysource': attributes.get('last_submission_date'), + }) + + return indicator_obj + + +def _add_ip_attributes(indicator_obj: dict, attributes: dict) -> dict: + """Adds IP attributes.""" + indicator_obj['fields'].update({ + 'countrycode': attributes.get('country'), + 'firstseenbysource': attributes.get('first_seen_itw_date'), + 'lastseenbysource': attributes.get('last_seen_itw_date'), + }) + + return indicator_obj + + +def _add_dedicated_attributes(indicator_obj: dict, attributes: dict) -> dict: + """Adds dedicated attributes to indicator object.""" + if indicator_obj['type'] == FeedIndicatorType.File: + return _add_file_attributes(indicator_obj, attributes) + if indicator_obj['type'] == FeedIndicatorType.Domain: + return _add_domain_attributes(indicator_obj, attributes) + if indicator_obj['type'] == FeedIndicatorType.URL: + return _add_url_attributes(indicator_obj, attributes) + if indicator_obj['type'] == FeedIndicatorType.IP: + return _add_ip_attributes(indicator_obj, attributes) + raise ValueError(f'Unknown type: {indicator_obj["type"]}. ID: {indicator_obj["id"]}') + + +def _create_indicator(item: dict) -> dict: + """Creates indicator object.""" + indicator_type = _get_indicator_type(item) + indicator_id = _get_indicator_id(item) + + attributes: dict = item.get('attributes', {}) + + detection_ratio = DetectionRatio(attributes.get('last_analysis_stats', {})) + + indicator_obj = { + 'type': indicator_type, + 'value': indicator_id, + 'service': 'Google Threat Intelligence', + 'fields': { + 'tags': attributes.get('tags') or None, + 'updateddate': attributes.get('last_modification_date'), + 'detectionengines': detection_ratio.total, + 'positivedetections': detection_ratio.malicious, + }, + 'rawJSON': { + 'type': indicator_type, + 'value': indicator_id, + 'attributes': attributes, + 'relationships': item.get('relationships', {}), + }, + 'id': indicator_id, + 'detections': str(detection_ratio), + } + + indicator_obj = _add_gti_attributes(indicator_obj, item) + indicator_obj = _add_dedicated_attributes(indicator_obj, attributes) + + return indicator_obj + + def fetch_indicators_command(client: Client, feed_type: str, tlp_color: str = None, feed_tags: list = None, limit: int = None, minimum_score: int = 0) -> list[dict]: - """Retrieves indicators from the feed + """Retrieves indicators from the feed. Args: client (Client): Client object with request tlp_color (str): Traffic Light Protocol color @@ -99,52 +280,11 @@ def fetch_indicators_command(client: Client, # extract values from iterator for item in iterator: - attributes = item.get('attributes', {}) - - type_ = FeedIndicatorType.File - raw_data = { - 'value': attributes['sha256'], - 'type': type_, - 'attributes': attributes, - } - - gti_assessment = attributes.get('gti_assessment', {}) - attribution = attributes.get('attribution', {}) - malware_families = [x['family'] for x in attribution.get('malware_families', [])] - malware_families = list(set(malware_families)) - threat_actors = attribution.get('threat_actors', []) - threat_actors = list(set(threat_actors)) - - # Create indicator object for each value. - # The object consists of a dictionary with required and optional keys and values, as described blow. - indicator_obj = { - # The indicator value. - 'value': attributes['sha256'], - # The indicator type as defined in Cortex XSOAR. - # One can use the FeedIndicatorType class under CommonServerPython to populate this field. - 'type': type_, - # The name of the service supplying this feed. - 'service': 'Google Threat Intelligence', - # A dictionary that maps values to existing indicator fields defined in Cortex XSOAR. - # One can use this section in order to map custom indicator fields previously defined - # in Cortex XSOAR to their values. - 'fields': { - 'md5': attributes.get('md5'), - 'sha1': attributes.get('sha1'), - 'sha256': attributes['sha256'], - 'size': attributes.get('size'), - 'tags': attributes.get('tags'), - }, - # A dictionary of the raw data returned from the feed source about the indicator. - 'rawJSON': raw_data, - 'sha256': attributes['sha256'], - 'gti_threat_score': gti_assessment.get('threat_score', {}).get('value'), - 'gti_severity': gti_assessment.get('severity', {}).get('value'), - 'gti_verdict': gti_assessment.get('verdict', {}).get('value'), - 'malware_families': malware_families or None, - 'threat_actors': threat_actors or None, - 'fileType': attributes.get('type_description'), - } + try: + indicator_obj = _create_indicator(item['data']) + except ValueError as exc: + demisto.info(str(exc)) + continue if feed_tags: indicator_obj['fields']['tags'] = feed_tags @@ -152,10 +292,14 @@ def fetch_indicators_command(client: Client, if tlp_color: indicator_obj['fields']['trafficlightprotocol'] = tlp_color - if (indicator_obj.get('gti_threat_score') or 0) >= minimum_score: + if int(indicator_obj['fields'].get('gtithreatscore') or 0) >= minimum_score: indicators.append(indicator_obj) else: - existing_indicators = list(IndicatorsSearcher(value=indicator_obj['value'])) + try: + existing_indicators = list(IndicatorsSearcher(value=indicator_obj['value'])) + except SystemExit as exc: + demisto.debug(exc) + existing_indicators = [] if len(existing_indicators) > 0 and int(existing_indicators[0].get('total', 0)) > 0: indicators.append(indicator_obj) @@ -184,7 +328,7 @@ def get_indicators_command(client: Client, tlp_color, feed_tags, limit, - minimum_score + minimum_score, ) human_readable = tableToMarkdown( @@ -213,19 +357,19 @@ def get_indicators_command(client: Client, def reset_last_threat_feed(): - """Reset last threat feed from the integration context""" + """Reset last threat feed from the integration context.""" demisto.setIntegrationContext({}) return CommandResults(readable_output='Fetch history deleted successfully') def main(): - """main function, parses params and runs command functions""" + """main function, parses params and runs command functions.""" params = demisto.params() # If your Client class inherits from BaseClient, SSL verification is # handled out of the box by it, just pass ``verify_certificate`` to # the Client constructor - insecure = not params.get('insecure', False) + secure = not params.get('insecure', False) # If your Client class inherits from BaseClient, system proxy is handled # out of the box by it, just pass ``proxy`` to the Client constructor @@ -239,7 +383,7 @@ def main(): try: client = Client( base_url='https://www.virustotal.com/api/v3/', - verify=insecure, + verify=secure, proxy=proxy, headers={ 'x-apikey': params['credentials']['password'], diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml index c770b8c22ecc..fa9ed00606e9 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml @@ -11,14 +11,23 @@ configuration: hiddenusername: true - display: Feed type name: feed_type - defaultvalue: apt + defaultvalue: malware type: 15 options: - - apt - - cve + - cryptominer + - first-stage-delivery-vectors + - infostealer - iot + - linux + - malicious-network-infrastructure + - malware - mobile + - osx + - phishing - ransomware + - threat-actor + - trending + - vulnerability-weaponization - display: Fetch indicators name: feed defaultvalue: "true" From 6b4c3a3e106b5154bde5a33be372aadd4b7ed2e4 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 4 Mar 2025 10:14:02 +0100 Subject: [PATCH 11/19] Rename --- .../CategorizedFeeds/CategorizedFeeds_test.py | 106 -------- .../Integrations/CategorizedFeeds/README.md | 72 ----- .../FeedThreatLists.py} | 167 ++++++++---- .../FeedThreatLists.yml} | 25 +- .../FeedThreatLists_description.md} | 2 +- .../FeedThreatLists_image.png} | Bin .../FeedThreatLists/FeedThreatLists_test.py | 245 ++++++++++++++++++ .../Integrations/FeedThreatLists/README.md | 58 +++++ .../FeedThreatLists/test_data/domain.json | 102 ++++++++ .../FeedThreatLists/test_data/file.json | 141 ++++++++++ .../FeedThreatLists/test_data/ip.json | 113 ++++++++ .../FeedThreatLists/test_data/url.json | 81 ++++++ 12 files changed, 873 insertions(+), 239 deletions(-) delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py delete mode 100644 Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md rename Packs/GoogleThreatIntelligence/Integrations/{CategorizedFeeds/CategorizedFeeds.py => FeedThreatLists/FeedThreatLists.py} (70%) rename Packs/GoogleThreatIntelligence/Integrations/{CategorizedFeeds/CategorizedFeeds.yml => FeedThreatLists/FeedThreatLists.yml} (85%) rename Packs/GoogleThreatIntelligence/Integrations/{CategorizedFeeds/CategorizedFeeds_description.md => FeedThreatLists/FeedThreatLists_description.md} (58%) rename Packs/GoogleThreatIntelligence/Integrations/{CategorizedFeeds/CategorizedFeeds_image.png => FeedThreatLists/FeedThreatLists_image.png} (100%) create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/domain.json create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/file.json create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/ip.json create mode 100644 Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/url.json diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py deleted file mode 100644 index c6dfbcd3472d..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_test.py +++ /dev/null @@ -1,106 +0,0 @@ -import CategorizedFeeds -import demistomock as demisto -from unittest.mock import call - - -MOCK_INDICATORS = [ - { - 'attributes': { - 'md5': 'md5_random', - 'sha1': 'sha1_random', - 'sha256': 'sha256_random', - 'type_description': 'random_type', - }, - 'type': 'file', - 'id': 'sha256_random', - }, - { - 'attributes': { - 'md5': 'md5_random2', - 'sha1': 'sha1_random2', - 'sha256': 'sha256_random2', - 'type_description': 'random_type2', - }, - 'type': 'file', - 'id': 'sha256_random2', - } -] - - -def test_fetch_indicators_command(mocker): - client = CategorizedFeeds.Client('https://fake') - mocker.patch.object(client, 'fetch_indicators', return_value=None) - mocker.patch.object(CategorizedFeeds, '_get_indicators', return_value=MOCK_INDICATORS) - - demisto.setIntegrationContext({}) - indicators = CategorizedFeeds.fetch_indicators_command(client, 'apt', [], limit=10) - - assert len(indicators) == 2 - assert indicators[0]['fields']['sha256'] == 'sha256_random' - assert indicators[0]['sha256'] == 'sha256_random' - assert indicators[0]['fileType'] == 'random_type' - assert indicators[1]['fields']['sha256'] == 'sha256_random2' - assert indicators[1]['sha256'] == 'sha256_random2' - assert indicators[1]['fileType'] == 'random_type2' - - -def test_fetch_indicators_limit_command(mocker): - client = CategorizedFeeds.Client('https://fake') - mocker.patch.object(client, 'fetch_indicators', return_value=None) - mocker.patch.object(CategorizedFeeds, '_get_indicators', return_value=MOCK_INDICATORS) - - demisto.setIntegrationContext({}) - indicators = CategorizedFeeds.fetch_indicators_command(client, 'apt', [], limit=1) - - assert len(indicators) == 1 - assert indicators[0]['fields']['sha256'] == 'sha256_random' - assert indicators[0]['sha256'] == 'sha256_random' - assert indicators[0]['fileType'] == 'random_type' - - -def test_main_manual_command(mocker): - params = { - 'feed_type': 'apt', - 'tlp_color': None, - 'feedTags': [], - 'credentials': {'password': 'xxx'}, - } - - mocker.patch.object(demisto, 'params', return_value=params) - mocker.patch.object(demisto, 'command', return_value='gti-feed-get-indicators') - get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'get_threat_feed') - - CategorizedFeeds.main() - - assert get_feed_mock.call_args == call('apt') - - -def test_main_default_command(mocker): - params = { - 'feed_type': 'iot', - 'tlp_color': None, - 'feedTags': [], - 'credentials': {'password': 'xxx'}, - } - - mocker.patch.object(demisto, 'params', return_value=params) - mocker.patch.object(demisto, 'command', return_value='fetch-indicators') - get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'get_threat_feed') - - CategorizedFeeds.main() - - assert get_feed_mock.call_args == call('iot') - - -def test_main_test_command(mocker): - params = { - 'credentials': {'password': 'xxx'}, - } - - mocker.patch.object(demisto, 'params', return_value=params) - mocker.patch.object(demisto, 'command', return_value='test-module') - get_feed_mock = mocker.patch.object(CategorizedFeeds.Client, 'fetch_indicators') - - CategorizedFeeds.main() - - assert get_feed_mock.call_count == 1 diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md b/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md deleted file mode 100644 index 2b119599ac90..000000000000 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/README.md +++ /dev/null @@ -1,72 +0,0 @@ -Use this feed integration to fetch Google Threat Intelligence Feeds matches. It processes the latest finished job retrieving its matches based on the limit parameter (40 by default) in every fetch until there are no more matches for that job. - -## Configure Google Threat Intelligence Feeds on Cortex XSOAR - -1. Navigate to **Settings** > **Integrations** > **Servers & Services**. -2. Search for Google Threat Intelligence Feeds. -3. Click **Add instance** to create and configure a new integration instance. - - | **Parameter** | **Description** | **Required** | - | --- | --- | --- | - | API Key (leave empty. Fill in the API key in the password field.) | | True | - | API Key | | True | - | Feed type | | True | - | Limit | Limit of indicators to fetch from retrohunt job results. | False | - | Fetch indicators | | False | - | Indicator Reputation | Indicators from this integration instance will be marked with this reputation. | False | - | Source Reliability | Reliability of the source providing the intelligence data. | True | - | | | False | - | | | False | - | Feed Fetch Interval | | False | - | Feed Minimum GTI Score | The minimum GTI score to import as part of the feed. | True | - | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | - | Tags | Supports CSV values. | False | - | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed. | False | - -4. Click **Test** to validate the URLs, token, and connection. -## Commands -You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. -After you successfully execute a command, a DBot message appears in the War Room with the command details. -### gti-feed-get-indicators -*** -Gets the matches from the latest feed. - -### gti-feed-reset-fetch-indicators -*** -Reset the last threat feed. - - - -#### Base Command - -`gti-feed-get-indicators` -#### Input - -| **Argument Name** | **Description** | **Required** | -| --- | --- | --- | -| limit | The maximum number of results to return. Default is 40. | Optional | - - -#### Context Output - -There is no context output for this command. - -#### Command Example -```!gti-feed-get-indicators``` -```!gti-feed-get-indicators limit=10``` - -#### Human Readable Output - -### Indicators from Google Threat Intelligence Feeds: -|Sha256|Filetype| -|---|---|---| -| 80db033dfe2b4e966d46a4ceed36e20b98a13891ce364a1308b90da7ad694cf3 | ELF | -| 6717c568e623551e600d315c7d1d634824a6f4b16e8aedfa298aefe7155313ff | ELF | -| 2c02a593ac714f9bac876d0a3c056384e0038505515d0c8472aa00ea36a6abb2 | ELF | -| e658b64650153c2207a76b2ee390b0fef04712d0da1d75a9eae25e4be596071a | ELF | -| 5ec2e17f25e800825ec5ed592c73303f840fa33cce2c8c4a4e7b6556798ffda0 | ELF | -| 771ba05ca9321dc723fc66b995c1d79a969330fc4242da6737cff1b364f978c8 | ELF | -| 4e3fac63a8b027788a10fd0191adf3ad59b2111324e1aa4eb4441723793c1b11 | ELF | -| ff1bdaf789643c6b934c9a9593fea82912d5974ba6ca0fd8dbf42db09ba82925 | ELF | -| 4371874f35538dc7d3b1d50df8cd0e8ad0744441ed487deb0d7a18a4a4373fea | ELF | - diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py similarity index 70% rename from Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py index 5a5c30af40e6..deb317ee3006 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py @@ -25,11 +25,11 @@ } -def _get_current_hour(): - """Gets current hour for Threat feeds.""" +def _get_current_package(): + """Gets current package for Threat feeds.""" time_obj = datetime.utcnow() - timedelta(hours=2) - hour = time_obj.strftime('%Y%m%d%H') - return hour + package = time_obj.strftime('%Y%m%d%H') + return package class DetectionRatio: @@ -48,33 +48,69 @@ def __repr__(self): class Client(BaseClient): """Client for Google Threat Intelligence API.""" - def fetch_indicators(self, feed_type: str = 'malware', hour: str = None) -> dict: - """Fetches indicators given a feed type and an hour.""" - if not hour: - hour = _get_current_hour() + def get_threat_list(self, + feed_type: str, + package: str, + limit: int = 10) -> dict: + """Get indicators from GTI API.""" return self._http_request( 'GET', - f'threat_lists/{feed_type}/{hour}', + f'threat_lists/{feed_type}/{package}', + params=assign_params( + limit=limit, + ) ) - def get_threat_feed(self, feed_type: str) -> list: + def fetch_indicators(self, + feed_type: str, + package: str = None, + limit: int = 10, + fetch_command: bool = False) -> list: """Retrieves matches for a given feed type.""" - last_threat_feed = demisto.getIntegrationContext().get('last_threat_feed') - hour = _get_current_hour() + package = package or _get_current_package() - if last_threat_feed == hour: - return [] + if fetch_command: + if self.get_last_run() == package: + return [] + + response = self.get_threat_list(feed_type, package, limit) + + if fetch_command: + self.set_last_run(package) - response = self.fetch_indicators(feed_type, hour) - demisto.setIntegrationContext({'last_threat_feed': hour}) return response.get('iocs', []) + @staticmethod + def set_last_run(package: str = None): + """Sets last threat feed.""" + current_package = package or _get_current_package() + demisto.setLastRun({'last_threat_feed': current_package}) + + @staticmethod + def get_last_run() -> str: + """Gets last threat feed, or '' if no last run.""" + return (demisto.getLastRun() or {}).get('last_threat_feed', '') -def test_module(client: Client) -> str: - client.fetch_indicators() + +def test_module(client: Client, args: dict) -> str: + """Tests module.""" + try: + client.fetch_indicators('malware') + except Exception: + raise Exception("Could not fetch Google Threat Intelligence IoC Threat Lists\n" + "\nCheck your API key and your connection to Google Threat Intelligence.") return 'ok' +def _gti_verdict_to_dbot_score(gti_verdict: str): + """Parses GTI verdict to DBotScore.""" + return { + 'VERDICT_BENIGN': 1, + 'VERDICT_SUSPICIOUS': 2, + 'VERDICT_MALICIOUS': 3, + }.get(gti_verdict, 0) + + def _add_gti_attributes(indicator_obj: dict, item: dict): """Addes GTI attributes.""" @@ -106,6 +142,7 @@ def _add_gti_attributes(indicator_obj: dict, item: dict): 'actor': threat_actors, }) indicator_obj.update({ + 'score': _gti_verdict_to_dbot_score(gti_verdict), 'gti_threat_score': gti_threat_score, 'gti_severity': gti_severity, 'gti_verdict': gti_verdict, @@ -178,8 +215,33 @@ def _add_file_attributes(indicator_obj: dict, attributes: dict) -> dict: def _add_domain_attributes(indicator_obj: dict, attributes: dict) -> dict: """Adds domain attributes.""" + whois: str = attributes.get('whois', '') + + admin_country = re.search(r'Admin Country:\s*([^\n]+)', whois) + admin_email = re.search(r'Admin Email:\s*([^\n]+)', whois) + admin_name = re.search(r'Admin Name:\s*([^\n]+)', whois) + admin_phone = re.search(r'Admin Phone:\s*([^\n]+)', whois) + + registrant_country = re.search(r'Registrant Country:\s*([^\n]+)', whois) + registrant_email = re.search(r'Registrant Email:\s*([^\n]+)', whois) + registrant_name = re.search(r'Registrant Name:\s*([^\n]+)', whois) + registrant_phone = re.search(r'Registrant Phone:\s*([^\n]+)', whois) + + registrar_abuse_email = re.search(r'Registrar Abuse Contact Email:\s*([^\n]+)', whois) + registrar_abuse_phone = re.search(r'Registrar Abuse Contact Phone:\s*([^\n]+)', whois) + indicator_obj['fields'].update({ 'creationdate': attributes.get('creation_date'), + 'admincountry': admin_country.group(1) if admin_country else None, + 'adminemail': admin_email.group(1) if admin_email else None, + 'adminname': admin_name.group(1) if admin_name else None, + 'adminphone': admin_phone.group(1) if admin_phone else None, + 'registrantcountry': registrant_country.group(1) if registrant_country else None, + 'registrantemail': registrant_email.group(1) if registrant_email else None, + 'registrantname': registrant_name.group(1) if registrant_name else None, + 'registrantphone': registrant_phone.group(1) if registrant_phone else None, + 'registrarabuseemail': registrar_abuse_email.group(1) if registrar_abuse_email else None, + 'registrarabusephone': registrar_abuse_phone.group(1) if registrar_abuse_phone else None, 'registrarname': attributes.get('registrar'), 'firstseenbysource': attributes.get('first_seen_itw_date'), 'lastseenbysource': attributes.get('last_seen_itw_date'), @@ -259,10 +321,12 @@ def _create_indicator(item: dict) -> dict: def fetch_indicators_command(client: Client, feed_type: str, + package: str = None, + limit: int = 10, tlp_color: str = None, feed_tags: list = None, - limit: int = None, - minimum_score: int = 0) -> list[dict]: + minimum_score: int = 0, + fetch_command: bool = False) -> list[dict]: """Retrieves indicators from the feed. Args: client (Client): Client object with request @@ -272,14 +336,13 @@ def fetch_indicators_command(client: Client, Returns: Indicators. """ - iterator = client.get_threat_feed(feed_type) indicators = [] - if limit: - iterator = iterator[:limit] + raw_indicators = client.fetch_indicators( + feed_type, package=package, limit=limit, fetch_command=fetch_command) # extract values from iterator - for item in iterator: + for item in raw_indicators: try: indicator_obj = _create_indicator(item['data']) except ValueError as exc: @@ -317,26 +380,29 @@ def get_indicators_command(client: Client, Returns: Outputs. """ - feed_type = params.get('feed_type', 'apt') tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) - limit = int(args.get('limit', 0)) + feed_type = args.get('feed_type') + package = args.get('package') + limit = int(args.get('limit', 10)) minimum_score = int(params.get('feedMinimumGTIScore', 80)) + indicators = fetch_indicators_command( client, feed_type, - tlp_color, - feed_tags, - limit, - minimum_score, + package=package, + limit=limit, + tlp_color=tlp_color, + feed_tags=feed_tags, + minimum_score=minimum_score, ) human_readable = tableToMarkdown( - f'Indicators from Google Threat Intelligence {FEED_STR.get(feed_type, feed_type)} Feeds:', + f'Indicators from Google Threat Intelligence {FEED_STR.get(feed_type, feed_type)} Threat List:', indicators, headers=[ - 'sha256', - 'fileType', + 'id', + 'detections', 'gti_threat_score', 'gti_severity', 'gti_verdict', @@ -356,16 +422,16 @@ def get_indicators_command(client: Client, ) -def reset_last_threat_feed(): - """Reset last threat feed from the integration context.""" - demisto.setIntegrationContext({}) - return CommandResults(readable_output='Fetch history deleted successfully') - - def main(): """main function, parses params and runs command functions.""" params = demisto.params() + feed_type = params.get('feed_type') + limit = int(params.get('limit', 10)) + tlp_color = params.get('tlp_color') + feed_tags = argToList(params.get('feedTags', '')) + minimum_score = int(params.get('feedMinimumGTIScore', 80)) + # If your Client class inherits from BaseClient, SSL verification is # handled out of the box by it, just pass ``verify_certificate`` to # the Client constructor @@ -376,7 +442,6 @@ def main(): proxy = params.get('proxy', False) command = demisto.command() - args = demisto.args() demisto.debug(f'Command being called is {command}') @@ -393,26 +458,26 @@ def main(): if command == 'test-module': # This is the call made when pressing the integration Test button. - return_results(test_module(client)) + return_results(test_module(client, {})) - elif command == 'gti-feed-get-indicators': + elif command == 'gti-threatlists-get-indicators': # This is the command that fetches a limited number of indicators # from the feed source and displays them in the war room. - return_results(get_indicators_command(client, params, args)) - - elif command == 'gti-feed-reset-fetch-indicators': - return_results(reset_last_threat_feed()) + return_results(get_indicators_command(client, params, demisto.args())) elif command == 'fetch-indicators': # This is the command that initiates a request to the feed endpoint # and create new indicators objects from the data fetched. If the # integration instance is configured to fetch indicators, then this - # is the commandthat will be executed at the specified feed fetch + # is the command that will be executed at the specified feed fetch # interval. - feed_type = params.get('feed_type', 'apt') - tlp_color = params.get('tlp_color') - feed_tags = argToList(params.get('feedTags')) - indicators = fetch_indicators_command(client, feed_type, tlp_color, feed_tags) + indicators = fetch_indicators_command(client, + feed_type, + limit=limit, + tlp_color=tlp_color, + feed_tags=feed_tags, + minimum_score=minimum_score, + fetch_command=True) for iter_ in batch(indicators, batch_size=2000): demisto.createIndicators(iter_) diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml similarity index 85% rename from Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml index fa9ed00606e9..cfbf0647c154 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml @@ -1,6 +1,6 @@ category: Data Enrichment & Threat Intelligence commonfields: - id: Google Threat Intelligence Feeds + id: Google Threat Intelligence Threat Lists version: -1 configuration: - display: API Key (leave empty. Fill in the API key in the password field.) @@ -9,7 +9,7 @@ configuration: type: 9 required: true hiddenusername: true -- display: Feed type +- display: Feed type. name: feed_type defaultvalue: malware type: 15 @@ -28,6 +28,11 @@ configuration: - threat-actor - trending - vulnerability-weaponization +- display: The maximum number of results to return. If 0 all results will be returned. + name: limit + defaultvalue: 10 + type: 0 + required: false - display: Fetch indicators name: feed defaultvalue: "true" @@ -110,19 +115,21 @@ configuration: name: feedIncremental required: false type: 8 -description: Use this feed integration to fetch Google Threat Intelligence Feeds matches. -display: Google Threat Intelligence Feeds -name: Google Threat Intelligence Feeds +description: Use this feed integration to fetch Google Threat Intelligence Threat Lists matches. +display: Google Threat Intelligence Threat Lists +name: Google Threat Intelligence Threat Lists script: commands: - arguments: + - name: feed_type + description: Feed type. + - name: package + description: Package in '%Y%m%d%H' format. If not given, the latest package is taken. - name: limit - defaultValue: "0" + defaultValue: '10' description: The maximum number of results to return. If 0 all results will be returned. description: Gets the matches from the latest Feed. - name: gti-feed-get-indicators - - description: "This command will reset your fetch history." - name: gti-feed-reset-fetch-indicators + name: gti-threatlists-get-indicators dockerimage: demisto/python3:3.11.9.101916 feed: true runonce: false diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_description.md similarity index 58% rename from Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_description.md index f4c5c1f9ca70..fbb52f333dac 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_description.md +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_description.md @@ -3,7 +3,7 @@ Your API key can be found in your Google Threat Intelligence account user menu. Your API key carries all your privileges, so keep it secure and don't share it with anyone. ### Feed types: -Currently there are five feed types: apt, cve, iot, mobile and ransomware. +Currently there are 14 feed types: cryptominer, first-stage-delivery-vectors, infostealer, iot, linux, malicious-network-infrastructure, malware, mobile, osx, phishing, ransomware, threat-actor, trending and vulnerability-weaponization. ### Cortex XSOAR version 6.0.0 and below: Fill anything in the "Username" fill box. Use you API key in the password fill box. \ No newline at end of file diff --git a/Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png similarity index 100% rename from Packs/GoogleThreatIntelligence/Integrations/CategorizedFeeds/CategorizedFeeds_image.png rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py new file mode 100644 index 000000000000..65316e59264b --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py @@ -0,0 +1,245 @@ +"""Tests for Google Threat Intelligence IoC Stream Feed integration.""" +import demistomock as demisto # noqa: F401 +from CommonServerPython import FeedIndicatorType # noqa: F401 + +import json +from unittest import mock + +from FeedThreatLists import Client, fetch_indicators_command, get_indicators_command, main + + +def _mock_indicator(indicator_type, gti_score=None): + """Mocks indicator.""" + with open(f'./test_data/{indicator_type}.json', encoding='utf-8') as f: + indicator_mock = json.load(f) + + if gti_score is not None: + indicator_mock['attributes']['gti_assessment']['threat_score']['value'] = gti_score + + return indicator_mock + + +def _mock_file(gti_score=None): + """Mocks file.""" + return _mock_indicator('file', gti_score) + + +def _mock_domain(gti_score=None): + """Mocks domain.""" + return _mock_indicator('domain', gti_score) + + +def _mock_url(gti_score=None): + """Mocks URL.""" + return _mock_indicator('url', gti_score) + + +def _mock_ip(gti_score=None): + """Mocks IP address.""" + return _mock_indicator('ip', gti_score) + + +def test_fetch_indicators_command(mocker): + """Tests fetch indicators command.""" + client = Client('https://fake') + for gti_score, len_response in [ + (0, 4), + (1, 3), + (50, 2), + (95, 1), + (100, 0), + ]: + mocker.patch.object( + client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + + indicators = fetch_indicators_command(client, 'malware', limit=10, minimum_score=gti_score) + + assert len(indicators) == len_response + + for indicator in indicators: + if indicator['type'] == FeedIndicatorType.File: + assert set(indicator['fields'].keys()) == { + 'md5', 'sha1', 'sha256', 'ssdeep', 'fileextension', 'filetype', 'imphash', + 'tags', 'firstseenbysource', 'lastseenbysource', 'creationdate', 'updateddate', + 'detectionengines', 'positivedetections', 'displayname', 'name', 'size', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['value'] == indicator['fields']['sha256'] + assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' + assert indicator['score'] == 3 + elif indicator['type'] == FeedIndicatorType.Domain: + assert set(indicator['fields'].keys()) == { + 'admincountry', 'adminname', 'adminemail', 'adminphone', 'registrantcountry', + 'registrantemail', 'registrantname', 'registrantphone', 'registrarabusephone', + 'registrarabuseemail', 'registrarname', 'firstseenbysource', 'lastseenbysource', + 'tags', 'creationdate', 'updateddate', 'detectionengines', 'positivedetections', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['fields']['adminemail'] == '@google.com' + assert indicator['fields']['registrantcountry'] == 'US' + assert indicator['fields']['registrarabusephone'] == '+34 600 000 000' + assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' + assert indicator['score'] == 3 + elif indicator['type'] == FeedIndicatorType.URL: + assert set(indicator['fields'].keys()) == { + 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', + 'detectionengines', 'positivedetections', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['fields']['firstseenbysource'] == 1722360511 + assert indicator['fields']['gtiverdict'] == 'VERDICT_UNDETECTED' + assert indicator['score'] == 0 + elif indicator['type'] == FeedIndicatorType.IP: + assert set(indicator['fields'].keys()) == { + 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', + 'detectionengines', 'positivedetections', 'countrycode', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == 'X.X.X.X' + assert indicator['fields']['countrycode'] == 'US' + assert indicator['fields']['gtiverdict'] == 'VERDICT_BENIGN' + assert indicator['score'] == 1 + else: + raise ValueError(f'Unknown type: {indicator["type"]}') + + +def test_get_indicators_command(mocker): + """Tests get indicators command.""" + client = Client('https://fake') + + for gti_score, len_response in [ + (None, 2), + (0, 4), + (1, 3), + (50, 2), + (95, 1), + (100, 0), + ]: + mocker.patch.object( + client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + params = { + 'tlp_color': None, + 'feedTags': [], + } + if gti_score is not None: + params['feedMinimumGTIScore'] = gti_score + + result = get_indicators_command(client, params, {}) + + assert len(result.raw_response) == len_response + + +def test_main_manual_command(mocker): + """Tests main manual.""" + params = { + 'tlp_color': None, + 'feedTags': [], + 'credentials': {'password': 'xxx'}, + 'feedMinimumGTIScore': 95, + } + + args = { + 'limit': 7, + 'feed_type': 'malware', + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='gti-threatlists-get-indicators') + mocker.patch.object(demisto, 'args', return_value=args) + get_threat_list_mock = mocker.patch.object( + Client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + return_results_mock = mocker.patch.object(demisto, 'results') + + main() + + assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 7) + assert len(return_results_mock.call_args[0][0]['Contents']) == 1 + + +def test_main_default_command(mocker): + """Tests main default.""" + params = { + 'tlp_color': None, + 'feedTags': [], + 'credentials': {'password': 'xxx'}, + 'limit': 7, + 'feed_type': 'malware', + 'feedMinimumGTIScore': 1, + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='fetch-indicators') + get_threat_list_mock = mocker.patch.object( + Client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + create_indicators_mock = mocker.patch.object(demisto, 'createIndicators') + + main() + + assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 7) + assert len(create_indicators_mock.call_args[0][0]) == 3 + + +def test_main_test_command(mocker): + """Tests main test.""" + params = { + 'credentials': {'password': 'xxx'} + } + + mocker.patch.object(demisto, 'params', return_value=params) + mocker.patch.object(demisto, 'command', return_value='test-module') + get_threat_list_mock = mocker.patch.object( + Client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + ], + }, + ) + + main() + + assert get_threat_list_mock.call_count == 1 diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md new file mode 100644 index 000000000000..e140c8ffe40f --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md @@ -0,0 +1,58 @@ +Use this feed integration to fetch Google Threat Intelligence Threat Lists matches. It processes the latest finished job retrieving its matches based on the limit parameter (40 by default) in every fetch until there are no more matches for that job. + +## Configure Google Threat Intelligence Threat Lists on Cortex XSOAR + +1. Navigate to **Settings** > **Integrations** > **Servers & Services**. +2. Search for Google Threat Intelligence Threat Lists. +3. Click **Add instance** to create and configure a new integration instance. + +| **Parameter** | **Description** | **Required** | +| --- | --- | --- | +| feed | The fetch indicators. | False | +| credentials | API Key. | True | +| feed_type | Feed type. | True | +| limit | The maximum number of results to return. Default is 10. | False | +| feedReputation | The indicator reputation. | False | +| feedReliability | The source's reliability. | True | +| tlp_color | The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. More information about the protocol can be found at https://us-cert.cisa.gov/tlp | False | +| feedExpirationPolicy | The feed's expiration policy. | False | +| feedFetchInterval | The feed fetch interval. | False | +| feedMinimumGTIScore | The minimum GTI score to import as part of the feed. | True | +| feedBypassExclusionList | Whether to bypass exclusion list. | False | + +4. Click **Test** to validate the URLs, token, and connection. + +## Commands +You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. +After you successfully execute a command, a DBot message appears in the War Room with the command details. + +### gti-threatlists-get-indicators +*** +Gets the matches from Google Threat Intelligence Threat Lists. + +#### Base Command + +`gti-threatlists-get-indicators` +#### Input + +| **Argument Name** | **Description** | **Required** | +| --- | --- | --- | +| feed_type | Feed type. | Required | +| package | Package in '%Y%m%d%H' format. If not given, the latest package is taken. | Optional | +| limit | The maximum number of results to return. Default is 10. | Optional | + + +#### Context Output + +There is no context output for this command. + +#### Command Example +```!gti-threatlists-get-indicators``` +```!gti-threatlists-get-indicators feed=malware package=2025021910 limit=10``` + +#### Human Readable Output + +### Indicators from Google Threat Intelligence Threat Lists: +| Id | Detections | Gti Threat Score | Gti Severity | Gti Verdict | Malware Families | Threat Actors | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| f221425286c9073cbb2168f73120b6... | 59/69 | 80 | SEVERITY_LOW | VERDICT_MALICIOUS | beacon | SWEED | diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/domain.json b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/domain.json new file mode 100644 index 000000000000..a3974621bb68 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/domain.json @@ -0,0 +1,102 @@ +{ + "data": { + "id": "", + "type": "domain", + "attributes": { + "last_modification_date": 1726690020, + "reputation": 0, + "tld": "com", + "first_seen_itw_date": 1315830193, + "mandiant_ic_score": 84, + "registrar": "Amazon Registrar, Inc.", + "last_dns_records_date": 1726675835, + "whois": "Admin City: Clearwater\nAdmin Country: US\nAdmin Email: @google.com\nAdmin Organization: \nAdmin Postal Code: 33755\nAdmin State/Province: FL\nCreation Date: 2020-12-07T20:18:23Z\nDNSSEC: unsigned\nDomain Name: \nDomain Status: OK\nName Server: \nRegistrant City: 9f17c16e0cbd11e5\nRegistrant Country: US\nRegistrant Email: @google.com\nRegistrant Fax Ext: 3432650ec337c945\nRegistrant Fax: 3432650ec337c945\nRegistrant Name: 24277ff58446df8f\nRegistrant Organization: c25274193b9137ef\nRegistrant Phone Ext: 3432650ec337c945\nRegistrant Phone: 6099a769d1923d4a\nRegistrant Postal Code: f28adb1ee249d449\nRegistrant State/Province: 6eb233f5a5adbed8\nRegistrant Street: 7d57c43d3cfd7338\nRegistrar Abuse Contact Email: @google.com\nRegistrar Abuse Contact Phone: +34 600 000 000\nRegistrar IANA ID: \nRegistrar Registration Expiration Date: 2024-12-07T20:18:23Z\nRegistrar URL: \nRegistrar URL: \nRegistrar WHOIS Server: \nRegistrar: \nRegistry Admin ID: Not Available From Registry\nRegistry Domain ID: \nRegistry Expiry Date: 2024-12-07T20:18:23Z\nRegistry Registrant ID: Not Available From Registry\nRegistry Tech ID: Not Available From Registry\nTech City: Clearwater\nTech Country: US\nTech Email: @google.com\nTech Organization: \nTech Postal Code: 33755\nTech State/Province: FL\nUpdated Date: 2023-11-03T22:57:14Z", + "last_https_certificate_date": 1593568953, + "threat_verdict": "VERDICT_UNDETECTED", + "whois_date": 1724217310, + "categories": {}, + "last_analysis_stats": { + "malicious": 11, + "suspicious": 1, + "undetected": 31, + "harmless": 51, + "timeout": 0 + }, + "last_analysis_date": 1726675631, + "last_seen_itw_date": 1658186104, + "popularity_ranks": {}, + "creation_date": 1607372303, + "attribution": { + "detailed_threat_actors": [ + { + "name": "", + "source": "Mandiant", + "id": "" + } + ], + "threat_actors": [ + "" + ] + }, + "total_votes": { + "harmless": 0, + "malicious": 0 + }, + "last_update_date": 1699052234, + "threat_severity": { + "version": "D3", + "threat_severity_level": "SEVERITY_NONE", + "threat_severity_data": { + "num_detections": 11, + "belongs_to_bad_collection": true, + "belongs_to_threat_actor": true + }, + "last_analysis_date": "1726675845", + "level_description": "Severity NONE because it has less than 2 detections." + }, + "tags": [], + "gti_assessment": { + "verdict": { + "value": "VERDICT_MALICIOUS" + }, + "contributing_factors": { + "associated_actor": true, + "normalised_categories": [ + "malware" + ], + "mandiant_confidence_score": 84, + "safebrowsing_verdict": "harmless", + "mandiant_association_actor": true, + "mandiant_association_report": true + }, + "threat_score": { + "value": 90 + }, + "severity": { + "value": "SEVERITY_HIGH" + }, + "description": "This indicator is malicious (high severity) with high impact. It is associated with a Mandiant Intelligence Report, Mandiant's scoring pipeline identified this indicator as malicious and it is associated with a tracked Mandiant threat actor. Analysts should prioritize investigation." + } + }, + "context_attributes": { + "notification_id": "19104706970", + "origin": "hunting", + "notification_date": 1726761663, + "sources": [ + { + "id": "19072418300", + "type": "hunting_ruleset", + "label": "Malware Families YARA ruleset" + } + ], + "tags": [ + "malware_families_yara_ruleset", + "malwarefamilyagenttesla" + ], + "hunting_info": { + "rule_name": "MalwareFamilyAgentTesla", + "match_source": "ORIGINAL_FILE" + } + } + } +} diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/file.json b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/file.json new file mode 100644 index 000000000000..fe38dfbb55e8 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/file.json @@ -0,0 +1,141 @@ +{ + "data": { + "id": "", + "type": "file", + "attributes": { + "type_description": "Win32 EXE", + "tlsh": "", + "vhash": "", + "exiftool": { + "MIMEType": "application/octet-stream", + "Subsystem": "Windows GUI", + "MachineType": "Intel 386 or later, and compatibles", + "TimeStamp": "2010:11:20 09:03:08+00:00", + "FileType": "Win32 EXE", + "PEType": "PE32", + "CodeSize": "36864", + "LinkerVersion": "6.0", + "ImageFileCharacteristics": "No relocs, Executable, No line numbers, No symbols, 32-bit", + "FileTypeExtension": "exe", + "InitializedDataSize": "3682304", + "SubsystemVersion": "4.0", + "ImageVersion": "0.0", + "OSVersion": "4.0", + "EntryPoint": "0x9a16", + "UninitializedDataSize": "0" + }, + "trid": [ + { + "file_type": "Win32 Executable MS Visual C++ (generic)", + "probability": 38.8 + }, + { + "file_type": "Microsoft Visual C++ compiled executable (generic)", + "probability": 20.5 + }, + { + "file_type": "Win64 Executable (generic)", + "probability": 13.0 + }, + { + "file_type": "Win32 Dynamic Link Library (generic)", + "probability": 8.1 + }, + { + "file_type": "Win16 NE executable (generic)", + "probability": 6.2 + } + ], + "creation_date": 1290243788, + "names": [ + ".virus" + ], + "last_modification_date": 1635959808, + "type_tag": "peexe", + "times_submitted": 1, + "total_votes": { + "harmless": 0, + "malicious": 0 + }, + "size": 3723264, + "authentihash": "", + "last_submission_date": 1635952526, + "meaningful_name": ".virus", + "downloadable": true, + "sha256": "", + "type_extension": "exe", + "tags": [ + "peexe", + "cve-2017-0147", + "exploit" + ], + "last_analysis_date": 1635952526, + "unique_sources": 1, + "first_submission_date": 1635952526, + "sha1": "", + "ssdeep": "", + "bloom": "", + "packers": { + "PEiD": "Microsoft Visual C++" + }, + "md5": "", + "pe_info": { + "imphash": "" + }, + "magic": "PE32 executable for MS Windows (GUI) Intel 80386 32-bit", + "last_analysis_stats": { + "harmless": 0, + "type-unsupported": 5, + "suspicious": 0, + "confirmed-timeout": 0, + "timeout": 0, + "failure": 0, + "malicious": 60, + "undetected": 9 + }, + "reputation": 0, + "gti_assessment": { + "verdict": { + "value": "VERDICT_MALICIOUS" + }, + "contributing_factors": { + "associated_actor": true, + "normalised_categories": [ + "malware" + ], + "mandiant_confidence_score": 84, + "safebrowsing_verdict": "harmless", + "mandiant_association_actor": true, + "mandiant_association_report": true + }, + "threat_score": { + "value": 95 + }, + "severity": { + "value": "SEVERITY_HIGH" + }, + "description": "This indicator is malicious (high severity) with high impact. It is associated with a Mandiant Intelligence Report, Mandiant's scoring pipeline identified this indicator as malicious and it is associated with a tracked Mandiant threat actor. Analysts should prioritize investigation." + } + }, + "context_attributes": { + "notification_id": "19104706970", + "origin": "hunting", + "notification_date": 1726761663, + "sources": [ + { + "id": "19072418300", + "type": "hunting_ruleset", + "label": "Malware Families YARA ruleset" + } + ], + "tags": [ + "malware_families_yara_ruleset", + "malwarefamilyagenttesla" + ], + "hunting_info": { + "rule_name": "MalwareFamilyAgentTesla", + "match_source": "ORIGINAL_FILE" + } + } + } +} diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/ip.json b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/ip.json new file mode 100644 index 000000000000..db1f87dd351a --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/ip.json @@ -0,0 +1,113 @@ +{ + "data": { + "id": "X.X.X.X", + "type": "ip_address", + "attributes": { + "last_modification_date": 1726781178, + "total_votes": { + "harmless": 203, + "malicious": 32 + }, + "last_analysis_stats": { + "malicious": 1, + "suspicious": 0, + "undetected": 31, + "harmless": 62, + "timeout": 0 + }, + "last_analysis_date": 1726758865, + "first_seen_itw_date": 1409607591, + "reputation": 535, + "tags": [], + "whois": "NetRange: X.X.X.0 - X.X.X.255\nCIDR: X.X.X.0/24\n", + "continent": "NA", + "network": "X.X.X.0/24", + "threat_verdict": "VERDICT_UNDETECTED", + "country": "US", + "regional_internet_registry": "ARIN", + "asn": 15169, + "whois_date": 1726150884, + "as_owner": "GOOGLE", + "last_https_certificate_date": 1726758877, + "mandiant_ic_score": 0, + "attribution": { + "detailed_threat_actors": [ + { + "source": "", + "name": "", + "id": "" + }, + { + "source": "", + "name": "", + "id": "" + } + ], + "malware_families": [ + { + "source": "", + "family": "" + }, + { + "source": "", + "family": "" + } + ], + "threat_actors": [ + "", + "" + ], + "family": "", + "family_pivot": "collection:None" + }, + "last_seen_itw_date": 1726766855, + "jarm": "", + "gti_assessment": { + "contributing_factors": { + "normalised_categories": [ + "infostealer", + "malware", + "control-server", + "phishing", + "malware", + "phishing" + ], + "pervasive_indicator": true, + "mandiant_confidence_score": 0, + "mandiant_analyst_benign": true, + "google_malware_analysis": true + }, + "severity": { + "value": "SEVERITY_NONE" + }, + "threat_score": { + "value": 0 + }, + "verdict": { + "value": "VERDICT_BENIGN" + }, + "description": "This indicator was determined as benign by a Mandiant analyst and likely poses no threat." + } + }, + "context_attributes": { + "notification_id": "19104706970", + "origin": "hunting", + "notification_date": 1726761663, + "sources": [ + { + "id": "19072418300", + "type": "hunting_ruleset", + "label": "Malware Families YARA ruleset" + } + ], + "tags": [ + "malware_families_yara_ruleset", + "malwarefamilyagenttesla" + ], + "hunting_info": { + "rule_name": "MalwareFamilyAgentTesla", + "match_source": "ORIGINAL_FILE" + } + } + } +} diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/url.json b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/url.json new file mode 100644 index 000000000000..439cf34810f7 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/test_data/url.json @@ -0,0 +1,81 @@ +{ + "data": { + "id": "", + "type": "url", + "attributes": { + "last_final_url": "", + "tags": [], + "categories": { + "Forcepoint ThreatSeeker": "elevated exposure" + }, + "private": false, + "last_analysis_stats": { + "malicious": 1, + "suspicious": 1, + "undetected": 25, + "harmless": 67, + "timeout": 0 + }, + "url": "", + "threat_verdict": "VERDICT_UNDETECTED", + "reputation": 0, + "has_content": false, + "last_analysis_date": 1722360511, + "threat_names": [], + "title": "", + "total_votes": { + "harmless": 0, + "malicious": 0 + }, + "tld": "wf", + "threat_severity": { + "version": "U3", + "threat_severity_level": "SEVERITY_NONE", + "threat_severity_data": { + "num_detections": 1 + }, + "last_analysis_date": "1722360525", + "level_description": "Severity NONE because it has less than 2 detections." + }, + "last_submission_date": 1722360511, + "times_submitted": 1, + "last_modification_date": 1722360527, + "first_submission_date": 1722360511, + "gti_assessment": { + "threat_score": { + "value": 1 + }, + "severity": { + "value": "SEVERITY_NONE" + }, + "contributing_factors": { + "safebrowsing_verdict": "harmless" + }, + "verdict": { + "value": "VERDICT_UNDETECTED" + }, + "description": "This indicator did not match our detection criteria and there is currently no evidence of malicious activity." + } + }, + "context_attributes": { + "notification_id": "19104706970", + "origin": "hunting", + "notification_date": 1726761663, + "sources": [ + { + "id": "19072418300", + "type": "hunting_ruleset", + "label": "Malware Families YARA ruleset" + } + ], + "tags": [ + "malware_families_yara_ruleset", + "malwarefamilyagenttesla" + ], + "hunting_info": { + "rule_name": "MalwareFamilyAgentTesla", + "match_source": "ORIGINAL_FILE" + } + } + } +} From fd2a4a7a1c05b491a893287016554cedb7d20fa2 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 4 Mar 2025 10:20:29 +0100 Subject: [PATCH 12/19] Update yml --- .../FeedThreatLists/FeedThreatLists.yml | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml index cfbf0647c154..91c08d1e974a 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml @@ -2,6 +2,9 @@ category: Data Enrichment & Threat Intelligence commonfields: id: Google Threat Intelligence Threat Lists version: -1 +sectionOrder: +- Connect +- Collect configuration: - display: API Key (leave empty. Fill in the API key in the password field.) displaypassword: API Key @@ -9,6 +12,7 @@ configuration: type: 9 required: true hiddenusername: true + section: Connect - display: Feed type. name: feed_type defaultvalue: malware @@ -28,16 +32,19 @@ configuration: - threat-actor - trending - vulnerability-weaponization + section: Connect - display: The maximum number of results to return. If 0 all results will be returned. name: limit defaultvalue: 10 type: 0 required: false + section: Collect - display: Fetch indicators name: feed defaultvalue: "true" type: 8 required: false + section: Collect - display: Indicator Reputation name: feedReputation defaultvalue: feedInstanceReputationNotSet @@ -49,6 +56,7 @@ configuration: - Bad additionalinfo: Indicators from this integration instance will be marked with this reputation. required: false + section: Collect - display: Source Reliability name: feedReliability defaultvalue: F - Reliability cannot be judged @@ -62,6 +70,7 @@ configuration: - E - Unreliable - F - Reliability cannot be judged additionalinfo: Reliability of the source providing the intelligence data. + section: Collect - display: "" name: feedExpirationPolicy defaultvalue: indicatorType @@ -72,32 +81,43 @@ configuration: - indicatorType - suddenDeath required: false + section: Collect + advanced: true - display: "" name: feedExpirationInterval defaultvalue: "20160" type: 1 required: false + section: Collect + advanced: true - display: Feed Fetch Interval name: feedFetchInterval defaultvalue: "60" type: 19 required: false + section: Collect + advanced: true - name: feedMinimumGTIScore type: 0 display: Feed Minimum GTI Score required: true defaultvalue: 80 additionalinfo: The minimum GTI score to import as part of the feed + section: Collect - display: Bypass exclusion list name: feedBypassExclusionList type: 8 additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. required: false + section: Collect + advanced: true - name: feedTags display: Tags type: 0 additionalinfo: Supports CSV values. required: false + section: Collect + advanced: true - name: tlp_color display: Traffic Light Protocol Color options: @@ -108,13 +128,7 @@ configuration: type: 15 additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. required: false -- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified. - defaultvalue: 'true' - display: Incremental feed - hidden: true - name: feedIncremental - required: false - type: 8 + section: Collect description: Use this feed integration to fetch Google Threat Intelligence Threat Lists matches. display: Google Threat Intelligence Threat Lists name: Google Threat Intelligence Threat Lists @@ -130,7 +144,7 @@ script: description: The maximum number of results to return. If 0 all results will be returned. description: Gets the matches from the latest Feed. name: gti-threatlists-get-indicators - dockerimage: demisto/python3:3.11.9.101916 + dockerimage: demisto/python3:3.11.10.111526 feed: true runonce: false script: "-" From 60e6681062c08194eb33b90842dc0043481f45e5 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 4 Mar 2025 10:29:30 +0100 Subject: [PATCH 13/19] Fix precommit --- .../Integrations/FeedThreatLists/FeedThreatLists.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py index deb317ee3006..92ab7bd69189 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py @@ -382,7 +382,7 @@ def get_indicators_command(client: Client, """ tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) - feed_type = args.get('feed_type') + feed_type = args.get('feed_type', 'malware') package = args.get('package') limit = int(args.get('limit', 10)) minimum_score = int(params.get('feedMinimumGTIScore', 80)) @@ -426,7 +426,7 @@ def main(): """main function, parses params and runs command functions.""" params = demisto.params() - feed_type = params.get('feed_type') + feed_type = params.get('feed_type', 'malware') limit = int(params.get('limit', 10)) tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) From 309b289419256763a25d455d70abdf50d12dba71 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 4 Mar 2025 10:35:05 +0100 Subject: [PATCH 14/19] Fix precommit --- .../Integrations/FeedThreatLists/FeedThreatLists.yml | 2 +- Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md | 6 ++++++ Packs/GoogleThreatIntelligence/pack_metadata.json | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml index 91c08d1e974a..8136a9a46979 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml @@ -144,7 +144,7 @@ script: description: The maximum number of results to return. If 0 all results will be returned. description: Gets the matches from the latest Feed. name: gti-threatlists-get-indicators - dockerimage: demisto/python3:3.11.10.111526 + dockerimage: demisto/python3:3.12.8.1983910 feed: true runonce: false script: "-" diff --git a/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md b/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md new file mode 100644 index 000000000000..33db7c20fdc9 --- /dev/null +++ b/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### New: Google Threat Intelligence Threat Lists + +- New: Use this feed integration to fetch Google Threat Intelligence Threat Lists as indicators. diff --git a/Packs/GoogleThreatIntelligence/pack_metadata.json b/Packs/GoogleThreatIntelligence/pack_metadata.json index 7faea84e7d53..eddc6f6fd917 100644 --- a/Packs/GoogleThreatIntelligence/pack_metadata.json +++ b/Packs/GoogleThreatIntelligence/pack_metadata.json @@ -2,7 +2,7 @@ "name": "GoogleThreatIntelligence", "description": "Analyze suspicious hashes, URLs, domains and IP addresses", "support": "partner", - "currentVersion": "1.2.0", + "currentVersion": "1.3.0", "author": "Google Threat Intelligence", "url": "https://www.virustotal.com", "email": "contact@virustotal.com", From 7b51e56fcfcacf40de9bad7d652a848aae7661ed Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Tue, 4 Mar 2025 11:36:52 +0100 Subject: [PATCH 15/19] Add filter --- .../FeedThreatLists/FeedThreatLists.py | 42 ++-- .../FeedThreatLists/FeedThreatLists.yml | 18 +- .../FeedThreatLists/FeedThreatLists_test.py | 200 ++++++++---------- .../Integrations/FeedThreatLists/README.md | 7 +- .../ReleaseNotes/1_3_0.md | 2 +- 5 files changed, 129 insertions(+), 140 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py index 92ab7bd69189..81434d6fe101 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py @@ -26,7 +26,7 @@ def _get_current_package(): - """Gets current package for Threat feeds.""" + """Gets current package for Threat Lists.""" time_obj = datetime.utcnow() - timedelta(hours=2) package = time_obj.strftime('%Y%m%d%H') return package @@ -51,12 +51,14 @@ class Client(BaseClient): def get_threat_list(self, feed_type: str, package: str, + filter_query: str = None, limit: int = 10) -> dict: """Get indicators from GTI API.""" return self._http_request( 'GET', f'threat_lists/{feed_type}/{package}', params=assign_params( + query=filter_query, limit=limit, ) ) @@ -64,16 +66,18 @@ def get_threat_list(self, def fetch_indicators(self, feed_type: str, package: str = None, + filter_query: str = None, limit: int = 10, fetch_command: bool = False) -> list: """Retrieves matches for a given feed type.""" package = package or _get_current_package() + filter_query = filter_query or '' if fetch_command: if self.get_last_run() == package: return [] - response = self.get_threat_list(feed_type, package, limit) + response = self.get_threat_list(feed_type, package, filter_query.strip(), limit) if fetch_command: self.set_last_run(package) @@ -322,24 +326,31 @@ def _create_indicator(item: dict) -> dict: def fetch_indicators_command(client: Client, feed_type: str, package: str = None, + filter_query: str = None, limit: int = 10, tlp_color: str = None, feed_tags: list = None, - minimum_score: int = 0, fetch_command: bool = False) -> list[dict]: """Retrieves indicators from the feed. Args: client (Client): Client object with request + feed_type (str): Feed type + package (string): Package in '%Y%m%d%H' format + filter_query (string): filter query + limit (int): Limit the results tlp_color (str): Traffic Light Protocol color feed_tags (list): Tags to assign fetched indicators - limit (int): Limit the results + fetch_command (bool): Whether command is used as fetch command. Returns: Indicators. """ indicators = [] - raw_indicators = client.fetch_indicators( - feed_type, package=package, limit=limit, fetch_command=fetch_command) + raw_indicators = client.fetch_indicators(feed_type, + package=package, + filter_query=filter_query, + limit=limit, + fetch_command=fetch_command) # extract values from iterator for item in raw_indicators: @@ -355,16 +366,7 @@ def fetch_indicators_command(client: Client, if tlp_color: indicator_obj['fields']['trafficlightprotocol'] = tlp_color - if int(indicator_obj['fields'].get('gtithreatscore') or 0) >= minimum_score: - indicators.append(indicator_obj) - else: - try: - existing_indicators = list(IndicatorsSearcher(value=indicator_obj['value'])) - except SystemExit as exc: - demisto.debug(exc) - existing_indicators = [] - if len(existing_indicators) > 0 and int(existing_indicators[0].get('total', 0)) > 0: - indicators.append(indicator_obj) + indicators.append(indicator_obj) return indicators @@ -384,17 +386,17 @@ def get_indicators_command(client: Client, feed_tags = argToList(params.get('feedTags', '')) feed_type = args.get('feed_type', 'malware') package = args.get('package') + filter_query = args.get('filter') limit = int(args.get('limit', 10)) - minimum_score = int(params.get('feedMinimumGTIScore', 80)) indicators = fetch_indicators_command( client, feed_type, package=package, + filter_query=filter_query, limit=limit, tlp_color=tlp_color, feed_tags=feed_tags, - minimum_score=minimum_score, ) human_readable = tableToMarkdown( @@ -427,10 +429,10 @@ def main(): params = demisto.params() feed_type = params.get('feed_type', 'malware') + filter_query = params.get('filter') limit = int(params.get('limit', 10)) tlp_color = params.get('tlp_color') feed_tags = argToList(params.get('feedTags', '')) - minimum_score = int(params.get('feedMinimumGTIScore', 80)) # If your Client class inherits from BaseClient, SSL verification is # handled out of the box by it, just pass ``verify_certificate`` to @@ -473,10 +475,10 @@ def main(): # interval. indicators = fetch_indicators_command(client, feed_type, + filter_query=filter_query, limit=limit, tlp_color=tlp_color, feed_tags=feed_tags, - minimum_score=minimum_score, fetch_command=True) for iter_ in batch(indicators, batch_size=2000): demisto.createIndicators(iter_) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml index 8136a9a46979..618057ad5b9a 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.yml @@ -33,6 +33,13 @@ configuration: - trending - vulnerability-weaponization section: Connect +- name: filter + display: Filter + type: 0 + additionalinfo: Filter your Threat Lists (e.g., "gti_score:70+ positives:10- has:campaigns"). Leave empty to receive all. + required: false + advanced: true + section: Connect - display: The maximum number of results to return. If 0 all results will be returned. name: limit defaultvalue: 10 @@ -97,13 +104,6 @@ configuration: required: false section: Collect advanced: true -- name: feedMinimumGTIScore - type: 0 - display: Feed Minimum GTI Score - required: true - defaultvalue: 80 - additionalinfo: The minimum GTI score to import as part of the feed - section: Collect - display: Bypass exclusion list name: feedBypassExclusionList type: 8 @@ -129,7 +129,7 @@ configuration: additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. required: false section: Collect -description: Use this feed integration to fetch Google Threat Intelligence Threat Lists matches. +description: Use this feed integration to fetch Google Threat Intelligence Threat Lists matches as indicators. display: Google Threat Intelligence Threat Lists name: Google Threat Intelligence Threat Lists script: @@ -139,6 +139,8 @@ script: description: Feed type. - name: package description: Package in '%Y%m%d%H' format. If not given, the latest package is taken. + - description: Filter your Threat Lists (e.g., "gti_score:70+ positives:10- has:campaigns"). Leave empty to receive all. + name: filter - name: limit defaultValue: '10' description: The maximum number of results to return. If 0 all results will be returned. diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py index 65316e59264b..cbebcda3becd 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_test.py @@ -42,114 +42,98 @@ def _mock_ip(gti_score=None): def test_fetch_indicators_command(mocker): """Tests fetch indicators command.""" client = Client('https://fake') - for gti_score, len_response in [ - (0, 4), - (1, 3), - (50, 2), - (95, 1), - (100, 0), - ]: - mocker.patch.object( - client, - 'get_threat_list', - return_value={ - 'iocs': [ - _mock_file(), - _mock_domain(), - _mock_url(), - _mock_ip(), - ], - }, - ) - - indicators = fetch_indicators_command(client, 'malware', limit=10, minimum_score=gti_score) - - assert len(indicators) == len_response - - for indicator in indicators: - if indicator['type'] == FeedIndicatorType.File: - assert set(indicator['fields'].keys()) == { - 'md5', 'sha1', 'sha256', 'ssdeep', 'fileextension', 'filetype', 'imphash', - 'tags', 'firstseenbysource', 'lastseenbysource', 'creationdate', 'updateddate', - 'detectionengines', 'positivedetections', 'displayname', 'name', 'size', - 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', - } - assert indicator['value'] == '' - assert indicator['value'] == indicator['fields']['sha256'] - assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' - assert indicator['score'] == 3 - elif indicator['type'] == FeedIndicatorType.Domain: - assert set(indicator['fields'].keys()) == { - 'admincountry', 'adminname', 'adminemail', 'adminphone', 'registrantcountry', - 'registrantemail', 'registrantname', 'registrantphone', 'registrarabusephone', - 'registrarabuseemail', 'registrarname', 'firstseenbysource', 'lastseenbysource', - 'tags', 'creationdate', 'updateddate', 'detectionengines', 'positivedetections', - 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', - } - assert indicator['value'] == '' - assert indicator['fields']['adminemail'] == '@google.com' - assert indicator['fields']['registrantcountry'] == 'US' - assert indicator['fields']['registrarabusephone'] == '+34 600 000 000' - assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' - assert indicator['score'] == 3 - elif indicator['type'] == FeedIndicatorType.URL: - assert set(indicator['fields'].keys()) == { - 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', - 'detectionengines', 'positivedetections', - 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', - } - assert indicator['value'] == '' - assert indicator['fields']['firstseenbysource'] == 1722360511 - assert indicator['fields']['gtiverdict'] == 'VERDICT_UNDETECTED' - assert indicator['score'] == 0 - elif indicator['type'] == FeedIndicatorType.IP: - assert set(indicator['fields'].keys()) == { - 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', - 'detectionengines', 'positivedetections', 'countrycode', - 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', - } - assert indicator['value'] == 'X.X.X.X' - assert indicator['fields']['countrycode'] == 'US' - assert indicator['fields']['gtiverdict'] == 'VERDICT_BENIGN' - assert indicator['score'] == 1 - else: - raise ValueError(f'Unknown type: {indicator["type"]}') + + mocker.patch.object( + client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + + indicators = fetch_indicators_command(client, 'malware', limit=10) + + assert len(indicators) == 4 + + for indicator in indicators: + if indicator['type'] == FeedIndicatorType.File: + assert set(indicator['fields'].keys()) == { + 'md5', 'sha1', 'sha256', 'ssdeep', 'fileextension', 'filetype', 'imphash', + 'tags', 'firstseenbysource', 'lastseenbysource', 'creationdate', 'updateddate', + 'detectionengines', 'positivedetections', 'displayname', 'name', 'size', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['value'] == indicator['fields']['sha256'] + assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' + assert indicator['score'] == 3 + elif indicator['type'] == FeedIndicatorType.Domain: + assert set(indicator['fields'].keys()) == { + 'admincountry', 'adminname', 'adminemail', 'adminphone', 'registrantcountry', + 'registrantemail', 'registrantname', 'registrantphone', 'registrarabusephone', + 'registrarabuseemail', 'registrarname', 'firstseenbysource', 'lastseenbysource', + 'tags', 'creationdate', 'updateddate', 'detectionengines', 'positivedetections', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['fields']['adminemail'] == '@google.com' + assert indicator['fields']['registrantcountry'] == 'US' + assert indicator['fields']['registrarabusephone'] == '+34 600 000 000' + assert indicator['fields']['gtiverdict'] == 'VERDICT_MALICIOUS' + assert indicator['score'] == 3 + elif indicator['type'] == FeedIndicatorType.URL: + assert set(indicator['fields'].keys()) == { + 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', + 'detectionengines', 'positivedetections', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == '' + assert indicator['fields']['firstseenbysource'] == 1722360511 + assert indicator['fields']['gtiverdict'] == 'VERDICT_UNDETECTED' + assert indicator['score'] == 0 + elif indicator['type'] == FeedIndicatorType.IP: + assert set(indicator['fields'].keys()) == { + 'tags', 'firstseenbysource', 'lastseenbysource', 'updateddate', + 'detectionengines', 'positivedetections', 'countrycode', + 'gtithreatscore', 'gtiseverity', 'gtiverdict', 'actor', 'malwarefamily', + } + assert indicator['value'] == 'X.X.X.X' + assert indicator['fields']['countrycode'] == 'US' + assert indicator['fields']['gtiverdict'] == 'VERDICT_BENIGN' + assert indicator['score'] == 1 + else: + raise ValueError(f'Unknown type: {indicator["type"]}') def test_get_indicators_command(mocker): """Tests get indicators command.""" client = Client('https://fake') - for gti_score, len_response in [ - (None, 2), - (0, 4), - (1, 3), - (50, 2), - (95, 1), - (100, 0), - ]: - mocker.patch.object( - client, - 'get_threat_list', - return_value={ - 'iocs': [ - _mock_file(), - _mock_domain(), - _mock_url(), - _mock_ip(), - ], - }, - ) - params = { - 'tlp_color': None, - 'feedTags': [], - } - if gti_score is not None: - params['feedMinimumGTIScore'] = gti_score - - result = get_indicators_command(client, params, {}) - - assert len(result.raw_response) == len_response + mocker.patch.object( + client, + 'get_threat_list', + return_value={ + 'iocs': [ + _mock_file(), + _mock_domain(), + _mock_url(), + _mock_ip(), + ], + }, + ) + params = { + 'tlp_color': None, + 'feedTags': [], + } + + result = get_indicators_command(client, params, {}) + + assert len(result.raw_response) == 4 def test_main_manual_command(mocker): @@ -158,12 +142,12 @@ def test_main_manual_command(mocker): 'tlp_color': None, 'feedTags': [], 'credentials': {'password': 'xxx'}, - 'feedMinimumGTIScore': 95, } args = { 'limit': 7, 'feed_type': 'malware', + 'filter': 'gti_score:95+', } mocker.patch.object(demisto, 'params', return_value=params) @@ -185,8 +169,8 @@ def test_main_manual_command(mocker): main() - assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 7) - assert len(return_results_mock.call_args[0][0]['Contents']) == 1 + assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 'gti_score:95+', 7) + assert len(return_results_mock.call_args[0][0]['Contents']) == 4 def test_main_default_command(mocker): @@ -197,7 +181,7 @@ def test_main_default_command(mocker): 'credentials': {'password': 'xxx'}, 'limit': 7, 'feed_type': 'malware', - 'feedMinimumGTIScore': 1, + 'filter': 'gti_score:1+', } mocker.patch.object(demisto, 'params', return_value=params) @@ -218,8 +202,8 @@ def test_main_default_command(mocker): main() - assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 7) - assert len(create_indicators_mock.call_args[0][0]) == 3 + assert get_threat_list_mock.call_args == mock.call('malware', mock.ANY, 'gti_score:1+', 7) + assert len(create_indicators_mock.call_args[0][0]) == 4 def test_main_test_command(mocker): diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md index e140c8ffe40f..57f53e50038e 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/README.md @@ -1,4 +1,4 @@ -Use this feed integration to fetch Google Threat Intelligence Threat Lists matches. It processes the latest finished job retrieving its matches based on the limit parameter (40 by default) in every fetch until there are no more matches for that job. +Use this feed integration to fetch Google Threat Intelligence Threat Lists matches as indicators. It processes the latest finished job retrieving its matches based on the limit parameter (10 by default) in every fetch until there are no more matches for that job. ## Configure Google Threat Intelligence Threat Lists on Cortex XSOAR @@ -11,13 +11,13 @@ Use this feed integration to fetch Google Threat Intelligence Threat Lists match | feed | The fetch indicators. | False | | credentials | API Key. | True | | feed_type | Feed type. | True | +| filter | Filter your Threat Lists (e.g., "gti_score:70+ positives:10- has:campaigns"). Leave empty to receive all. | False | | limit | The maximum number of results to return. Default is 10. | False | | feedReputation | The indicator reputation. | False | | feedReliability | The source's reliability. | True | | tlp_color | The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. More information about the protocol can be found at https://us-cert.cisa.gov/tlp | False | | feedExpirationPolicy | The feed's expiration policy. | False | | feedFetchInterval | The feed fetch interval. | False | -| feedMinimumGTIScore | The minimum GTI score to import as part of the feed. | True | | feedBypassExclusionList | Whether to bypass exclusion list. | False | 4. Click **Test** to validate the URLs, token, and connection. @@ -39,6 +39,7 @@ Gets the matches from Google Threat Intelligence Threat Lists. | --- | --- | --- | | feed_type | Feed type. | Required | | package | Package in '%Y%m%d%H' format. If not given, the latest package is taken. | Optional | +| filter | Filter your Threat Lists (e.g., "gti_score:70+ positives:10- has:campaigns"). Leave empty to receive all. | Optional | | limit | The maximum number of results to return. Default is 10. | Optional | @@ -48,7 +49,7 @@ There is no context output for this command. #### Command Example ```!gti-threatlists-get-indicators``` -```!gti-threatlists-get-indicators feed=malware package=2025021910 limit=10``` +```!gti-threatlists-get-indicators feed=malware package=2025021910 filter="gti_score:70+" limit=10``` #### Human Readable Output diff --git a/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md b/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md index 33db7c20fdc9..7f04fbb1cd7c 100644 --- a/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md +++ b/Packs/GoogleThreatIntelligence/ReleaseNotes/1_3_0.md @@ -3,4 +3,4 @@ ##### New: Google Threat Intelligence Threat Lists -- New: Use this feed integration to fetch Google Threat Intelligence Threat Lists as indicators. +- New: Use this feed integration to fetch Google Threat Intelligence Threat Lists matches as indicators. From 1527e8f218885f520d9613e03de6e5bc185adb7e Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Wed, 5 Mar 2025 10:14:33 +0100 Subject: [PATCH 16/19] Comments --- .../FeedThreatLists/FeedThreatLists.py | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py index 81434d6fe101..9215ed713866 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py @@ -1,11 +1,6 @@ import demistomock as demisto # noqa: F401 from CommonServerPython import * # noqa: F401 -import urllib3 - -# Disable insecure warnings. -urllib3.disable_warnings() - FEED_STR = { 'cryptominer': 'Cryptominer', @@ -128,12 +123,12 @@ def _add_gti_attributes(indicator_obj: dict, item: dict): # Relationships relationships = item.get('relationships', {}) malware_families: list[str] = [ - x['attributes']['name'] + x.get('attributes', {}).get('name') for x in relationships.get('malware_families', {}).get('data', []) ] malware_families = list(set(malware_families)) threat_actors: list[str] = [ - x['attributes']['name'] + x.get('attributes', {}).get('name') for x in relationships.get('threat_actors', {}).get('data', []) ] threat_actors = list(set(threat_actors)) @@ -178,22 +173,22 @@ def _add_gti_attributes(indicator_obj: dict, item: dict): def _get_indicator_type(item: dict): """Gets indicator type.""" - if item['type'] == 'file': + if item.get('type') == 'file': return FeedIndicatorType.File - if item['type'] == 'domain': + if item.get('type') == 'domain': return FeedIndicatorType.Domain - if item['type'] == 'url': + if item.get('type') == 'url': return FeedIndicatorType.URL - if item['type'] == 'ip_address': + if item.get('type') == 'ip_address': return FeedIndicatorType.IP - raise ValueError(f'Unknown type: {item["type"]}. ID: {item["id"]}') + raise ValueError(f'Unknown type: {item.get("type")}. ID: {item.get("id")}') def _get_indicator_id(item: dict) -> str: """Gets indicator ID.""" - if item['type'] == 'url': - return item.get('attributes', {}).get('url') or item['id'] - return item['id'] + if item.get('type') == 'url': + return item.get('attributes', {}).get('url') or item.get('id') + return item.get('id') def _add_file_attributes(indicator_obj: dict, attributes: dict) -> dict: @@ -355,7 +350,7 @@ def fetch_indicators_command(client: Client, # extract values from iterator for item in raw_indicators: try: - indicator_obj = _create_indicator(item['data']) + indicator_obj = _create_indicator(item.get('data', {})) except ValueError as exc: demisto.info(str(exc)) continue @@ -453,7 +448,7 @@ def main(): verify=secure, proxy=proxy, headers={ - 'x-apikey': params['credentials']['password'], + 'x-apikey': params.get('credentials', {}).get('password'), 'x-tool': 'CortexGTIFeeds', } ) From 0197ac460e76eaa887701ca322cd76cf88fb9082 Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Wed, 5 Mar 2025 10:25:23 +0100 Subject: [PATCH 17/19] Fix precommit --- .../Integrations/FeedThreatLists/FeedThreatLists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py index 9215ed713866..f535dc561bf3 100644 --- a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py +++ b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.py @@ -184,7 +184,7 @@ def _get_indicator_type(item: dict): raise ValueError(f'Unknown type: {item.get("type")}. ID: {item.get("id")}') -def _get_indicator_id(item: dict) -> str: +def _get_indicator_id(item: dict): """Gets indicator ID.""" if item.get('type') == 'url': return item.get('attributes', {}).get('url') or item.get('id') From 1184206df84be282e55c46f21cafd0c8dab0950e Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Wed, 5 Mar 2025 12:15:42 +0100 Subject: [PATCH 18/19] Rename png --- ...eedThreatLists_image.png => FeedThreatLists.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/{FeedThreatLists_image.png => FeedThreatLists.png} (100%) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.png similarity index 100% rename from Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.png From 213f9371caa8a7d7331dd646cdccdd4caa5c76cf Mon Sep 17 00:00:00 2001 From: pabloperezj Date: Wed, 5 Mar 2025 14:16:54 +0100 Subject: [PATCH 19/19] Revert --- ...eedThreatLists.png => FeedThreatLists_image.png} | Bin 1 file changed, 0 insertions(+), 0 deletions(-) rename Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/{FeedThreatLists.png => FeedThreatLists_image.png} (100%) diff --git a/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.png b/Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png similarity index 100% rename from Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists.png rename to Packs/GoogleThreatIntelligence/Integrations/FeedThreatLists/FeedThreatLists_image.png