Merge remote-tracking branch 'upstream/master'

.circleci/config.yml

@@ -5,9 +5,9 @@ jobs:
       - image: devdemisto/content-build:3.0.0.7332  # disable-secrets-detection
     parallelism: 2
     environment:
-      CONTENT_VERSION: "20.6.0"
+      CONTENT_VERSION: "20.6.1"
       SERVER_VERSION: "5.5.0"
-      GIT_SHA1: "63646dff0fba977f91d6d9fc2d7fd233bfb5561b" # guardrails-disable-line disable-secrets-detection
+      GIT_SHA1: "85b138633bcc10cedb4d6f4ed6e7074c28dd13d7" # guardrails-disable-line disable-secrets-detection
     steps:
       - checkout
       - setup_remote_docker

Packs/AttivoBotsink/Integrations/AttivoBotsink/AttivoBotsink.py

@@ -72,7 +72,11 @@ def do_request(self, url, data=None, headers=None, files=None, method=None, cont
             demisto.error("Type is: {type}".format(type=e.__class__.__name__))
 
         if r is not None and r.content:
-            return r.json()
+            try:
+                json_res = r.json()
+            except ValueError:
+                return_error('Failed deserializing response JSON - {}'.format(r.content))
+            return json_res
         else:
             return None
 

Packs/AttivoBotsink/ReleaseNotes/1_0_1.md

@@ -0,0 +1,4 @@
+
+#### Integrations
+##### Attivo Botsink
+- Fixed an issue where errors were not handled as expected.

Packs/AttivoBotsink/pack_metadata.json

@@ -1,16 +1,16 @@
 {
-  "name": "Attivo Botsink",
-  "description": "Network-based Threat Deception for Post-Compromise Threat Detection.",
-  "support": "xsoar",
-  "currentVersion": "1.0.0",
-  "author": "Cortex XSOAR",
-  "url": "https://www.paloaltonetworks.com/cortex",
-  "email": "",
-  "created": "2020-04-14T00:00:00Z",
-  "categories": [
-    "Deception"
-  ],
-  "tags": [],
-  "useCases": [],
-  "keywords": []
-}
+    "name": "Attivo Botsink",
+    "description": "Network-based Threat Deception for Post-Compromise Threat Detection.",
+    "support": "xsoar",
+    "currentVersion": "1.0.1",
+    "author": "Cortex XSOAR",
+    "url": "https://www.paloaltonetworks.com/cortex",
+    "email": "",
+    "created": "2020-04-14T00:00:00Z",
+    "categories": [
+        "Deception"
+    ],
+    "tags": [],
+    "useCases": [],
+    "keywords": []
+}

Packs/Base/ReleaseNotes/1_0_14.md

@@ -0,0 +1,5 @@
+<!--
+#### Scripts
+##### DBotSuggestClassifierMapping
+Internal code improvements
+-->

Packs/Base/ReleaseNotes/1_0_15.md

@@ -0,0 +1,5 @@
+
+#### Scripts
+##### CommonServerPython
+ - Fixed an issue where the ***to_context*** function did not return the proper outputs when the **CommandResult** object was supplied with only ***readable_outputs***.
+ - Fixed and issue where ***to_context*** function returned null instead of an empty list when supplied with empty outputs.

Packs/Base/Scripts/CommonServerPython/CommonServerPython.py

@@ -2581,10 +2581,12 @@ class CommandResults:
     :return: None
     :rtype: ``None``
     """
-    def __init__(self, outputs_prefix, outputs_key_field, outputs, indicators=None, readable_output=None,
+    def __init__(self, outputs_prefix=None, outputs_key_field=None, outputs=None, indicators=None, readable_output=None,
                  raw_response=None):
-
         # type: (str, str, object, list, str, object) -> None
+        if raw_response is None:
+            raw_response = outputs
+
         self.indicators = indicators
 
         self.outputs_prefix = outputs_prefix
@@ -2596,7 +2598,10 @@ def __init__(self, outputs_prefix, outputs_key_field, outputs, indicators=None,
 
     def to_context(self):
         outputs = {}  # type: dict
-        human_readable = None
+        if self.readable_output:
+            human_readable = self.readable_output
+        else:
+            human_readable = None
         raw_response = None
 
         if self.indicators:
@@ -2612,16 +2617,10 @@ def to_context(self):
         if self.raw_response:
             raw_response = self.raw_response
 
-        if self.outputs:
+        if self.outputs is not None:
             if not self.readable_output:
                 # if markdown is not provided then create table by default
                 human_readable = tableToMarkdown('Results', self.outputs)
-            else:
-                human_readable = self.readable_output
-
-            if not self.raw_response:
-                raw_response = self.outputs
-
             if self.outputs_prefix and self.outputs_key_field:
                 # if both prefix and key field provided then create DT key
                 outputs_key = '{0}(val.{1} == obj.{1})'.format(self.outputs_prefix, self.outputs_key_field)
@@ -2631,7 +2630,6 @@ def to_context(self):
                 outputs[outputs_key] = self.outputs
             else:
                 outputs = self.outputs
-                human_readable = self.readable_output  # prefix and key field not provided, human readable should
 
         return_entry = {
             'Type': EntryType.NOTE,

Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py

@@ -460,19 +460,12 @@ def test_argToList():
     test2 = 'a,b,c'
     test3 = '["a","b","c"]'
     test4 = 'a;b;c'
-    test5 = 1
-    test6 = '1'
-    test7 = True
 
     results = [argToList(test1), argToList(test2), argToList(test2, ','), argToList(test3), argToList(test4, ';')]
 
     for result in results:
         assert expected == result, 'argToList test failed, {} is not equal to {}'.format(str(result), str(expected))
 
-    assert argToList(test5) == [1]
-    assert argToList(test6) == ['1']
-    assert argToList(test7) == [True]
-
 
 def test_remove_nulls():
     temp_dictionary = {"a": "b", "c": 4, "e": [], "f": {}, "g": None, "h": "", "i": [1], "k": ()}
@@ -922,6 +915,43 @@ def test_file_indicators(self):
 
 
 class TestCommandResults:
+    def test_readable_only_context(self):
+        """
+        Given:
+        - Markdown entry to CommandResults
+
+        When:
+        - Returning results
+
+        Then:
+        - Validate HumanReadable exists
+        """
+        from CommonServerPython import CommandResults
+        markdown = '## Something'
+        context = CommandResults(readable_output=markdown).to_context()
+        assert context.get('HumanReadable') == markdown
+
+    def test_empty_outputs(self):
+        """
+        Given:
+        - Empty outputs
+
+        When:
+        - Returning results
+
+        Then:
+        - Validate EntryContext key value
+
+        """
+        from CommonServerPython import CommandResults
+        res = CommandResults(
+            outputs_prefix='FoundIndicators',
+            outputs_key_field='value',
+            outputs=[]
+        )
+        context = res.to_context()
+        assert {'FoundIndicators(val.value == obj.value)': []} == context.get('EntryContext')
+
     def test_return_command_results(self):
         from CommonServerPython import Common, CommandResults, EntryFormat, EntryType, DBotScoreType
 
@@ -1109,15 +1139,15 @@ def test_return_list_of_items_the_old_way(self):
             raw_response=tickets
         )
 
-        assert results.to_context() == {
+        assert sorted(results.to_context()) == sorted({
             'Type': EntryType.NOTE,
             'ContentsFormat': EntryFormat.JSON,
             'Contents': tickets,
             'HumanReadable': None,
             'EntryContext': {
                 'Jira.Ticket(val.ticket_id == obj.ticket_id)': tickets
             }
-        }
+        })
 
     def test_create_dbot_score_with_invalid_score(self):
         from CommonServerPython import Common, DBotScoreType

Packs/Base/Scripts/DBotSuggestClassifierMapping/DBotSuggestClassifierMapping.py

@@ -370,19 +370,19 @@ def validate(self, validator_name, field_name, value, json_field_name=None):
         return validate_func(field_name, value, json_field_name)
 
 
-def is_sublist_of_list(s, l):
+def is_sublist_of_list(s, lst):
     sub_set = False
     if s == []:
         sub_set = True
-    elif s == l:
+    elif s == lst:
         sub_set = True
-    elif len(s) > len(l):
+    elif len(s) > len(lst):
         sub_set = False
     else:
-        for i in range(len(l)):
-            if l[i] == s[0]:
+        for i in range(len(lst)):
+            if lst[i] == s[0]:
                 n = 1
-                while (n < len(s)) and (i + n) < len(l) and (l[i + n] == s[n]):
+                while (n < len(s)) and (i + n) < len(lst) and (lst[i + n] == s[n]):
                     n += 1
 
                 if n == len(s):

Packs/Base/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Base",
     "description": "The base pack for Cortex XSOAR.",
     "support": "xsoar",
-    "currentVersion": "1.0.13",
+    "currentVersion": "1.0.15",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",
@@ -17,4 +17,4 @@
         "common"
     ],
     "dependencies": {}
-}
+}

Packs/CommonScripts/ReleaseNotes/1_1_10.md

@@ -0,0 +1,4 @@
+
+#### Scripts
+##### ParseEmailFiles
+- Fixed an issue where errors were not handled as expected.

Packs/CommonScripts/Scripts/ExtractDomainFromUrlFormat/ExtractDomainFromUrlFormat_test.py

@@ -5,16 +5,17 @@
 
 
 @pytest.mark.parametrize('input,domain', [
-                        ('http://this.is.test.com', 'test.com'),
-                        ('http:example.com', 'example.com'),
-                        ('http:\\\\example.com', 'example.com'),
-                        ('https://caseapi.phishlabs.com', 'phishlabs.com'),
-                         # output needs to be bytes string utf-8 encoded (otherwise python loop demisto.results fails)
-                        (u'www.bücher.de', u'bücher.de'.encode('utf-8')),
-                        ('https://urldefense.proofpoint.com/v2/url?u=http-3A__go.getpostman.com_y4wULsdG0h0DDMY0Dv00100&d=DwMFaQ&c=ywDJJevdGcjv4rm9P3FcNg&r=s5kA2oIAQRXsacJiBKmTORIWyRN39ZKhobje2GyRgNs&m=vN1dVSiZvEoM9oExtQqEptm9Dbvq9tnjACDZzrBLaWI&s=zroN7KQdBCPBOfhOmv5SP1DDzZKZ1y9I3x4STS5PbHA&e=', 'getpostman.com'),  # noqa: E501
-                        ('hxxps://www[.]demisto[.]com', 'demisto.com'),
-                        ('https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Ftwitter.com%2FPhilipsBeLux&data=02|01||cb2462dc8640484baf7608d638d2a698|1a407a2d76754d178692b3ac285306e4|0|0|636758874714819880&sdata=dnJiphWFhnAKsk5Ps0bj0p%2FvXVo8TpidtGZcW6t8lDQ%3D&reserved=0%3E%5bcid:[email protected]%5d%3C', 'twitter.com'),  # noqa: E501 disable-secrets-detection
-                        ])  # noqa: E124
+    ('http://this.is.test.com', 'test.com'),
+    ('http:example.com', 'example.com'),
+    ('http:\\\\example.com', 'example.com'),
+    ('https://caseapi.phishlabs.com', 'phishlabs.com'),
+    # output needs to be bytes string utf-8 encoded (otherwise python loop demisto.results fails)
+    (u'www.bücher.de', u'bücher.de'.encode('utf-8')),
+    ('https://urldefense.proofpoint.com/v2/url?u=http-3A__go.getpostman.com_y4wULsdG0h0DDMY0Dv00100&d=DwMFaQ&c=ywDJJevdGcjv4rm9P3FcNg&r=s5kA2oIAQRXsacJiBKmTORIWyRN39ZKhobje2GyRgNs&m=vN1dVSiZvEoM9oExtQqEptm9Dbvq9tnjACDZzrBLaWI&s=zroN7KQdBCPBOfhOmv5SP1DDzZKZ1y9I3x4STS5PbHA&e=', 'getpostman.com'),  # noqa: E501
+    ('hxxps://www[.]demisto[.]com', 'demisto.com'),
+    ('https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Ftwitter.com%2FPhilipsBeLux&data=02|01||cb2462dc8640484baf7608d638d2a698|1a407a2d76754d178692b3ac285306e4|0|0|636758874714819880&sdata=dnJiphWFhnAKsk5Ps0bj0p%2FvXVo8TpidtGZcW6t8lDQ%3D&reserved=0%3E%5bcid:[email protected]%5d%3C', 'twitter.com'),  # noqa: E501 disable-secrets-detection
+]
+)  # noqa: E124
 def test_extract_domain(input, domain):
     res = extract_domain(input)
     assert res == domain

Packs/CommonScripts/Scripts/ParseEmailFiles/ParseEmailFiles.py

@@ -3181,9 +3181,8 @@ def parse_email_headers(header, raw=False):
 def get_msg_mail_format(msg_dict):
     try:
         return msg_dict.get('Headers', 'Content-type:').split('Content-type:')[1].split(';')[0]
-    except ValueError:
-        return ''
-    except IndexError:
+    except Exception as e:
+        demisto.debug('Got exception while trying to get msg mail format - {}'.format(str(e)))
         return ''
 
 

Packs/CommonScripts/Scripts/ParseEmailFiles/parse_email_files_test.py

@@ -562,15 +562,20 @@ def test_no_content_type_file(mocker):
 
 
 def test_get_msg_mail_format():
-    format = get_msg_mail_format({
+    msg_mail_format = get_msg_mail_format({
         'Headers': 'Content-type:text/plain;'
     })
-    assert format == 'text/plain'
+    assert msg_mail_format == 'text/plain'
 
-    format = get_msg_mail_format({
+    msg_mail_format = get_msg_mail_format({
         'Something': 'else'
     })
-    assert format == ''
+    assert msg_mail_format == ''
+
+    msg_mail_format = get_msg_mail_format({
+        'Headers': None
+    })
+    assert msg_mail_format == ''
 
 
 def test_no_content_file(mocker):

Packs/CommonScripts/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Common Scripts",
     "description": "Frequently used scripts pack.",
     "support": "xsoar",
-    "currentVersion": "1.1.9",
+    "currentVersion": "1.1.10",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",