From a32c19b01f99e73b38751a4180c4d01270313c52 Mon Sep 17 00:00:00 2001 From: Mai Morag <81917647+maimorag@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:10:21 +0300 Subject: [PATCH] BmcITSM- fix duplicate incidents (#35192) * adding logs * adding a fix to the last ticket create time * adding unit test to demonstrate the bug * adding the fix+precommit_rn * adding reason * docker fix * add to known words * Update Packs/BmcITSM/ReleaseNotes/1_0_23.md Co-authored-by: Dan Tavori <38749041+dantavori@users.noreply.github.com> * cr fix * cr note * cr note --------- Co-authored-by: Dan Tavori <38749041+dantavori@users.noreply.github.com> --- Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.py | 24 ++-- .../BmcITSM/Integrations/BmcITSM/BmcITSM.yml | 2 +- .../Integrations/BmcITSM/BmcITSM_test.py | 36 +++++- .../test_data/list_tickets_not_sorted.json | 112 ++++++++++++++++++ Packs/BmcITSM/ReleaseNotes/1_0_23.md | 7 ++ Packs/BmcITSM/pack_metadata.json | 2 +- 6 files changed, 166 insertions(+), 17 deletions(-) create mode 100644 Packs/BmcITSM/Integrations/BmcITSM/test_data/list_tickets_not_sorted.json create mode 100644 Packs/BmcITSM/ReleaseNotes/1_0_23.md diff --git a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.py b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.py index ecb3b8105e6f..29690795fd97 100644 --- a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.py +++ b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.py @@ -3,7 +3,7 @@ # type: ignore # mypy: ignore-errors from copy import deepcopy -from typing import Callable, Tuple +from collections.abc import Callable from datetime import datetime @@ -351,9 +351,8 @@ def retrieve_access_token(self, username: str, password: str) -> str: """ integration_context = get_integration_context() now = int(datetime.now().timestamp()) - if integration_context.get("token") and integration_context.get("expires_in"): - if now < integration_context["expires_in"]: - return integration_context["token"] + if integration_context.get("token") and integration_context.get("expires_in") and now < integration_context["expires_in"]: + return integration_context["token"] try: token = self._http_request( @@ -2914,7 +2913,7 @@ def get_paginated_records_with_hr( limit: Optional[int], page: int = None, page_size: int = None, -) -> Tuple[list, str]: +) -> tuple[list, str]: """ Retrieve the required page either with Automatic or Manual pagination, and the matching readable output header. @@ -2995,7 +2994,7 @@ def validate_related_arguments_provided(**related_args): def extract_args_from_additional_fields_arg(additional_fields: str, - field_name: str) -> Tuple[Any, List[str]]: + field_name: str) -> tuple[Any, List[str]]: """ Extract dictionary structure from additional field argument. @@ -3298,7 +3297,7 @@ def fetch_relevant_tickets( impact_filter: List[str], urgency_filter: List[str], custom_query: str, -) -> Tuple[list, dict]: +) -> tuple[list, dict]: """ Fetch the relevant tickets according to the provided filter arguments. The Tickets are fetched Iteratively, by their ticket type until the capacity @@ -3340,9 +3339,9 @@ def fetch_relevant_tickets( tickets_capacity -= tickets_amount if fetched_tickets: - last_ticket_create_time = total_tickets[-1].get("CreateDate") - ticket_type_to_last_epoch[ticket_type] = date_to_epoch_for_fetch( - arg_to_datetime(last_ticket_create_time)) + ticket_type_to_last_epoch[ticket_type] = max( + [date_to_epoch_for_fetch(arg_to_datetime(ticket.get("CreateDate"))) + for ticket in total_tickets]) if tickets_capacity <= 0: # no more tickets to retrieve in the current fetch break @@ -3425,10 +3424,7 @@ def all_keys_empty(dict_obj: Dict[str, Any]) -> bool: Returns: bool: Wheter or not all keys have None value. """ - for value in dict_obj.values(): - if value: - return False - return True + return all(not value for value in dict_obj.values()) def gen_multi_filters_statement(filter_mapper: Dict[str, Any], oper_in_filter: str, diff --git a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.yml b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.yml index d378dd20956b..224b419e8cf4 100644 --- a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.yml +++ b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM.yml @@ -169,7 +169,7 @@ script: script: "" type: python subtype: python3 - dockerimage: demisto/python3:3.10.13.87159 + dockerimage: demisto/python3:3.10.14.100715 commands: - name: bmc-itsm-user-list description: Retrieves a list of user profiles from BMC Helix ITSM. The records are retrieved by the query argument or by the filtering arguments. When using filtering arguments, each one defines a 'LIKE' operation and an 'AND' operator is used between them. To see the entire JSON then you can use the raw_response=true at the end of the command. diff --git a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM_test.py b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM_test.py index 675c49954956..2173fd904a4f 100644 --- a/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM_test.py +++ b/Packs/BmcITSM/Integrations/BmcITSM/BmcITSM_test.py @@ -2,6 +2,8 @@ import os import pytest from unittest.mock import patch +from CommonServerPython import * + """MOCK PARAMETERS """ CREDENTIALS = "credentials" ACCOUNT_ID = "account_id" @@ -22,7 +24,7 @@ def load_mock_response(file_name: str) -> str: Returns: str: Mock file content. """ - with open(os.path.join("test_data", file_name), mode="r", encoding="utf-8") as mock_file: + with open(os.path.join("test_data", file_name), encoding="utf-8") as mock_file: return json.loads(mock_file.read()) @@ -1480,3 +1482,35 @@ def test_ticket_list_work_order_command( assert result.outputs_prefix == "BmcITSM.Ticket" assert len(outputs) == expected_outputs_len assert outputs[0]["DisplayID"] == expected_name + + +def test_fetch_command( + mocker +): + """ + Given: + - List tickets. + When: + - fetch_incidents command called. + Then: + - Ensure that the *last_create_time* in *last_run_result* is the last between all incidents. + """ + import BmcITSM + mock_response = load_mock_response("list_tickets_not_sorted.json") + expected_result = 1719671916 + mocker.patch.object(demisto, 'getLastRun', return_value={"SRM:Request": {"last_create_time": '2021-06-29T14:38:36.000+0000'}}) + mocker.patch.object(BmcITSM, "fetch_relevant_tickets_by_ticket_type", return_value=mock_response) + incidents_result, last_run_result = BmcITSM.fetch_incidents(mock_client, + max_fetch=2, + first_fetch="2022-06-29T14:38:36.000+0000", + last_run={"SRM:Request": { + "last_create_time": '2021-06-29T14:38:36.000+0000'}}, + ticket_type_filter=["SRM:Request"], + status_filter=[], + impact_filter=[], + urgency_filter=[], + custom_query=("'Submit Date' <= \"1657032797\" AND 'Submit Date'" + ">\"1657032797\" AND 'Urgency' = \"4-Low\""), + mirror_direction="both", + ) + assert last_run_result["SRM:Request"]["last_create_time"] == expected_result diff --git a/Packs/BmcITSM/Integrations/BmcITSM/test_data/list_tickets_not_sorted.json b/Packs/BmcITSM/Integrations/BmcITSM/test_data/list_tickets_not_sorted.json new file mode 100644 index 000000000000..c4125a410a58 --- /dev/null +++ b/Packs/BmcITSM/Integrations/BmcITSM/test_data/list_tickets_not_sorted.json @@ -0,0 +1,112 @@ +[ + { + "values": { + "SysRequestID": "000000000000402", + "Submitter": "appadmin", + "Submit Date": "2024-06-29T14:38:36.000+0000", + "System Assignee": null, + "Status": "Planning", + "Status-History": { + "Draft": { + "user": "appadmin", + "timestamp": "2022-06-29T14:38:37.000+0000" + }, + "Submitted": { + "user": "appadmin", + "timestamp": "2022-06-29T14:38:37.000+0000" + }, + "Waiting Approval": { + "user": "Remedy Application Service", + "timestamp": "2022-06-29T14:39:49.000+0000" + }, + "Planning": { + "user": "Remedy Application Service", + "timestamp": "2022-06-29T14:39:49.000+0000" + } + }, + "Assignee Groups": "1000000003;'appadmin';", + "InstanceId": "AGGAI7ZXDK9WFAR4IUA2R3J3RVIB6V", + "Vendor Assignee Groups": null, + "Vendor Assignee Groups_parent": null, + "Assignee Groups_parent": "", + "z1D_WorkInfoType": null, + "z1D_WorkInfoSummary": null, + "z1D_WorkInfoDetails": null, + "z1D_WorkInfoSecureLog": null, + "z1D_WorkInfoViewAccess": null, + "z1D_WorkInfoDate": null, + "z1D_CommunicationSource": null, + "Assignee Group": "Backoffice Support", + "Assignee": "Mary Mann", + "Recurring Price Basis": null, + "Location Company": "Calbro Services", + "Organization": null, + "Assigned Support Organization": "IT Support", + "Last Name": "Admin", + "First Name": "App", + "Service Location Address": "1114 Eighth Avenue, 31st Floor. \r\nNew York, New York 10036 \r\nUnited States", + "Internet E-mail": null, + "Phone Number": "###", + "Navigation Tier 1": "File & Print", + "Navigation Tier 2": null, + "Navigation Tier 3": null, + "z1D Action": null, + "Request Manager Group ID": null, + "Company": "Calbro Services", + "Status_Reason": null, + "Details": null, + "Urgency": "2-High", + "Impact": "1-Extensive/Widespread", + "Assigned Group": null, + "Request Manager": null, + "Assigned Support Company": "Calbro Services", + "Request Manager Login": null, + "Total Escalation Level": 0, + "Request Number": "REQ000000000401", + "Date Required": null, + "Next Target Date": null, + "SLM Status": null, + "Customer First Name": "App", + "Customer Last Name": "Admin", + "Customer Company": "Calbro Services", + "Customer Organization": null, + "Customer Department": null, + "Customer Internet E-mail": null, + "Customer Phone Number": "###" + }, + "CreateDate": "2024-06-29T14:38:36.000+0000" + }, + { + "values": { + "SysRequestID": "000000000000403", + "Submitter": "appadmin", + "Submit Date": "2022-06-29T14:38:36.000+0000", + "System Assignee": null, + "Status": "Planning", + "Status-History": { + "Draft": { + "user": "appadmin", + "timestamp": "2022-06-29T14:38:37.000+0000" + }, + "Submitted": { + "user": "appadmin", + "timestamp": "2022-06-29T14:38:37.000+0000" + }, + "Waiting Approval": { + "user": "Remedy Application Service", + "timestamp": "2022-06-29T14:39:43.000+0000" + }, + "Planning": { + "user": "Remedy Application Service", + "timestamp": "2022-06-29T14:39:43.000+0000" + } + }, + "Assignee Groups": "1000000003;'appadmin';", + "InstanceId": "AGGAI7ZXDK9WFAR4IUA2R3J3RVIB6S", + "Vendor Assignee Groups": null, + "Vendor Assignee Groups_parent": null, + "Assignee Groups_parent": "" + }, + "CreateDate": "2022-06-29T14:38:36.000+0000" + } + ] diff --git a/Packs/BmcITSM/ReleaseNotes/1_0_23.md b/Packs/BmcITSM/ReleaseNotes/1_0_23.md new file mode 100644 index 000000000000..35e13be316f4 --- /dev/null +++ b/Packs/BmcITSM/ReleaseNotes/1_0_23.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### BMC Helix ITSM + +- Fixed an issue in the ***fetch-incidents*** command where duplicate incidents were fetched due to the incorrect assumption that tickets pulled from BMC Helix ITSM are sorted in ascending order. +- Updated the Docker image to: *demisto/python3:3.10.14.100715*. diff --git a/Packs/BmcITSM/pack_metadata.json b/Packs/BmcITSM/pack_metadata.json index 75cb746144cf..66cfdbd19553 100644 --- a/Packs/BmcITSM/pack_metadata.json +++ b/Packs/BmcITSM/pack_metadata.json @@ -2,7 +2,7 @@ "name": "BMC Helix ITSM", "description": "BMC Helix ITSM allows customers to manage service request, incident, change request, task, problem investigation, known error and work order tickets.", "support": "xsoar", - "currentVersion": "1.0.22", + "currentVersion": "1.0.23", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",