[SOAR-16858]- Duo Admin- update max lookback to 7 days (#2534)

[SOAR-16858]- Fix lint

[SOAR-16858]- Fix checksums

[SOAR-16858]- Refresh

[SOAR-16858]- Fix checksums

[SOAR-16858]- code changes and fix unit tests

plugins/duo_admin/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "1b1da27fce51b3dd62bb06445931a131",
-	"manifest": "c888effebe3990fcea7d50971a399358",
-	"setup": "6b8afaf1cf5e530029f0f11152b745d0",
+	"spec": "e6f5d47d912d00b20fe51a7ffa082a93",
+	"manifest": "e71860d2685a737f80bf50495cbae80c",
+	"setup": "4271ead3909ea6c4525f05977a8e5347",
 	"schemas": [
 		{
 			"identifier": "add_user/schema.py",
@@ -49,7 +49,7 @@
 		},
 		{
 			"identifier": "monitor_logs/schema.py",
-			"hash": "5ce8b19344da9506c08512370681f783"
+			"hash": "4119a8c82613406e16d830d7b48e0c86"
 		}
 	]
 }

plugins/duo_admin/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 rapid7/insightconnect-python-3-plugin:5
+FROM --platform=linux/amd64 rapid7/insightconnect-python-3-plugin:5.4.9
 
 LABEL organization=rapid7
 LABEL sdk=python

plugins/duo_admin/bin/komand_duo_admin

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Duo Admin API"
 Vendor = "rapid7"
-Version = "4.4.1"
+Version = "4.4.2"
 Description = "Duo is a trusted access solution for organizations. The Duo Admin plugin for Rapid7 InsightConnect allows users to manage and administrate their Duo organization"
 
 

plugins/duo_admin/help.md

@@ -1033,6 +1033,7 @@ A User ID can be obtained by passing a username to the Get User Status action.
 
 # Version History
 
+* 4.4.2 - Updated to include latest SDK v5.4.9 | Task `Monitor Logs` updated to increase max lookback cutoff to 7 days
 * 4.4.1 - `Monitor Logs` task updated to stop logging of trust monitor events response
 * 4.4.0 - `Monitor Logs` task updated to handle `custom_config` parameter for each log type separately | Apply lookback limit of 180 days due to Duo Admin API limitation
 * 4.3.2 - Monitor Logs task: Update to latest SDK | `Monitor Logs` task updated to handle `custom_config` parameter

plugins/duo_admin/komand_duo_admin/tasks/monitor_logs/task.py

@@ -13,8 +13,8 @@
 ADMIN_LOGS_LOG_TYPE = "Admin logs"
 AUTH_LOGS_LOG_TYPE = "Auth logs"
 TRUST_MONITOR_EVENTS_LOG_TYPE = "Trust monitor events"
-CUTOFF_HOURS = 24
-MAX_CUTOFF_HOURS = 72
+INITIAL_CUTOFF_HOURS = 24
+MAX_CUTOFF_HOURS = 168
 API_CUTOFF_HOURS = 4320
 
 
@@ -48,7 +48,7 @@ def get_parameters_for_query(
         # If no previous timestamp retrieved (first run) then query 24 hours
         if not last_log_timestamp:
             self.logger.info(f"First run for {log_type}")
-            filter_time = self._get_filter_time(custom_config, now, CUTOFF_HOURS, log_type)
+            filter_time = self._get_filter_time(custom_config, now, INITIAL_CUTOFF_HOURS, log_type)
             if log_type != "Admin logs":
                 mintime = self.convert_to_milliseconds(filter_time)
                 maxtime = self.convert_to_milliseconds(last_two_minutes)
@@ -65,9 +65,9 @@ def get_parameters_for_query(
                 self.logger.info(f"Subsequent run for {log_type}")
 
             # If for some reason no logs or event have been picked up,
-            # Need to ensure that no more than the previous 3 days is queried - use cutoff check to ensure this
-            # Prevent resuming of task from previous timestamp if beyond 3 days resulting in large data collection
-            max_cutoff_time = self._get_filter_time(custom_config, now, MAX_CUTOFF_HOURS, log_type)
+            # Need to ensure that no more than the previous 7 days is queried - use cutoff check to ensure this
+            # Prevent resuming of task from previous timestamp if beyond 7 days resulting in large data collection
+            max_cutoff_time = self._get_filter_time(custom_config, now, MAX_CUTOFF_HOURS, log_type, last_log_timestamp)
             cutoff_time_secs = self.convert_to_seconds(max_cutoff_time)
             cutoff_time_millisecs = self.convert_to_milliseconds(max_cutoff_time)
             if backward_comp_first_run:
@@ -86,13 +86,13 @@ def get_parameters_for_query(
                     mintime = max(last_log_timestamp_secs, cutoff_time_secs)
                     maxtime = self.convert_to_seconds(last_two_minutes)
             else:
-                if log_type == "Admin logs":
+                if log_type == ADMIN_LOGS_LOG_TYPE:
                     # Use seconds for admin log endpoint
                     mintime = max(last_log_timestamp, cutoff_time_secs)
                     maxtime = self.convert_to_seconds(last_two_minutes)
                 else:
                     if log_type == AUTH_LOGS_LOG_TYPE:
-                        # New method holds logmtimestamps in seconds so convert to milliseconds for auth logs
+                        # New method holds log time stamps in seconds so convert to milliseconds for auth logs
                         last_log_timestamp_millisecs = int(last_log_timestamp * 1000)
                     else:
                         # Trust monitor events hold timestamps in milliseconds(surface_timestamp is recorded)
@@ -389,15 +389,21 @@ def get_trust_monitor_event(self, mintime: int, maxtime: int, next_page_params:
         return trust_monitor_events, parameters
 
     def _get_filter_time(
-        self, custom_config: Dict, current_time: datetime, default_hours: int, log_type: str = None
-    ) -> int:
+        self,
+        custom_config: Dict,
+        current_time: datetime,
+        default_hours: int,
+        log_type: str = None,
+        last_log_timestamp: any = None,
+    ) -> datetime:
         """
         Apply custom_config params (if provided) to the task. If a lookback value exists for that task type, it should
         take precedence (this can allow a larger filter time), otherwise use the cutoff_hours value.
         :param custom_config: dictionary passed containing `cutoff` or `lookback` values
         :param current_time: Datetime of now
         :param default_hours: integer value representing default cutoff hours
         :param log_type: Log type value to be used to determine which lookback to retrieve from custom_config
+        :param last_log_timestamp: Last log timestamp to be used for lookback
         :return: filter_value (epoch seconds) to be applied in request to Duo
         """
         log_types = {
@@ -425,5 +431,13 @@ def _get_filter_time(
                 f"Lookback of {utc_filter_value} is older than 180 days. Looking back to {api_cutoff_date}..."
             )
             utc_filter_value = api_cutoff_date
+        # Check if last_log_timestamp is within 7 days and if it is then use that as the lookback value
+        if last_log_timestamp:
+            if log_type == TRUST_MONITOR_EVENTS_LOG_TYPE:
+                last_log_datetime = datetime.utcfromtimestamp(last_log_timestamp / 1000).replace(tzinfo=timezone.utc)
+            else:
+                last_log_datetime = datetime.utcfromtimestamp(last_log_timestamp).replace(tzinfo=timezone.utc)
+            if last_log_datetime > utc_filter_value:
+                utc_filter_value = last_log_datetime
         self.logger.info(f"Task execution for {log_type} will be applying a lookback to {utc_filter_value} UTC...")
         return utc_filter_value

plugins/duo_admin/plugin.spec.yaml

@@ -10,10 +10,10 @@ status: []
 supported_versions: ["Duo Admin API 2023-05-19"]
 sdk:
   type: full
-  version: 5
+  version: 5.4.9
   user: nobody
 description: Duo is a trusted access solution for organizations. The Duo Admin plugin for Rapid7 InsightConnect allows users to manage and administrate their Duo organization
-version: 4.4.1
+version: 4.4.2
 connection_version: 4
 resources:
   source_url: https://github.com/rapid7/insightconnect-plugins/tree/master/plugins/duo_admin

plugins/duo_admin/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="duo_admin-rapid7-plugin",
-      version="4.4.1",
+      version="4.4.2",
       description="Duo is a trusted access solution for organizations. The Duo Admin plugin for Rapid7 InsightConnect allows users to manage and administrate their Duo organization",
       author="rapid7",
       author_email="",

plugins/duo_admin/unit_test/expected/monitor_logs_2.json.exp

@@ -160,7 +160,6 @@
       }
 ],
   "state": {
-    "last_collection_timestamp": null,
     "previous_admin_log_hashes": [
       "cd9aa0fc02dd4a4538a3368025b2fab7480fbc7a",
       "79ac88bf9a2b241fc385a1501b614c39cdf7cd8c"
@@ -172,17 +171,17 @@
     "previous_trust_monitor_event_hashes": [
       "b7db98ed56420f228cf72b3b80a26c44e14e0dec"
     ],
-    "trust_monitor_last_log_timestamp": 1682843686000,
+    "trust_monitor_last_log_timestamp": 1682930026000,
     "trust_monitor_next_page_params": {
-      "mintime": "1682843686000",
+      "mintime": "1682930026000",
       "maxtime": "1682929966000",
       "limit": "200",
       "offset": "1591014"
      },
-     "admin_logs_last_log_timestamp": 1682843686,
+     "admin_logs_last_log_timestamp": 1682930026,
      "auth_logs_last_log_timestamp": 1684749133,
      "auth_logs_next_page_params": {
-       "mintime": "1682843686000",
+       "mintime": "1682930026000",
        "maxtime": "1682929966000",
        "limit": "1000",
        "sort": "ts:asc",

plugins/duo_admin/unit_test/expected/monitor_logs_3.json.exp

@@ -111,7 +111,6 @@
       "next_offset": "1683730665255,9de5069c-5afe-602b-2ea0-a04b66beb2c0",
       "sort": "ts:asc"
     },
-    "last_collection_timestamp": null,
     "previous_admin_log_hashes": [
       "cd9aa0fc02dd4a4538a3368025b2fab7480fbc7a",
       "79ac88bf9a2b241fc385a1501b614c39cdf7cd8c"

plugins/duo_admin/unit_test/inputs/monitor_logs_next_page.json.inp

@@ -8,7 +8,6 @@
       "9de5069c-5afe-602b-2ea0-a04b66beb2c0"
     ]
   },
-  "last_collection_timestamp": 1682930026000,
   "previous_admin_log_hashes": [
     "9044a3c0964c859ae26aee09a0ad031b0546b0b4",
     "29f17fa591dc5b46c1dfac976364059d80208fdb"
@@ -24,5 +23,8 @@
     "maxtime": "1682930026000",
     "mintime": "1682843686000",
     "offset": "1591014"
-  }
+  },
+  "admin_logs_last_log_timestamp": 1682930026,
+  "auth_logs_last_log_timestamp": 1682930026,
+  "trust_monitor_last_log_timestamp": 1682930026000
 }

plugins/duo_admin/unit_test/inputs/monitor_logs_with_state.json.inp

@@ -1,8 +1,10 @@
 {
-  "last_collection_timestamp": 1682843686000,
   "previous_admin_log_hashes": [
     "9044a3c0964c859ae26aee09a0ad031b0546b0b4"
   ],
   "previous_auth_log_hashes": [],
-  "previous_trust_monitor_event_hashes": []
+  "previous_trust_monitor_event_hashes": [],
+  "admin_logs_last_log_timestamp": 1682930026,
+  "auth_logs_last_log_timestamp": 1682930026,
+  "trust_monitor_last_log_timestamp": 1682930026000
 }

plugins/duo_admin/unit_test/util.py

@@ -59,6 +59,8 @@ def json(self):
                 return MockResponse(200, "get_auth_logs.json.resp")
             if params == {"mintime": "1684009458000", "maxtime": "1684209458000", "limit": "1000"}:
                 return MockResponse(200, "get_auth_logs.json.resp")
+            if params == {"mintime": "1682930026000", "maxtime": "1682929966000", "limit": "1000", "sort": "ts:asc"}:
+                return MockResponse(200, "get_auth_logs.json.resp")
             if params == {"mintime": "1683009458000", "maxtime": "1683209458000", "limit": "1000"}:
                 return MockResponse(200, "get_auth_logs_empty.json.resp")
             if params == {
@@ -91,11 +93,15 @@ def json(self):
                 return MockResponse(200, "get_admin_logs.json.resp")
             if params == {"mintime": "1682930026"}:
                 return MockResponse(200, "get_admin_logs.json.resp")
+            if params == {"mintime": "112321582426"}:
+                return MockResponse(200, "get_admin_logs.json.resp")
         if url == "https://example.com/admin/v1/trust_monitor/events":
             if params == {"mintime": "1682843686000", "maxtime": "1682929966000", "limit": "200"}:
                 return MockResponse(200, "get_trust_monitor_events.json.resp")
             if params == {"mintime": "1682670886000", "maxtime": "1682929966000", "limit": "200"}:
                 return MockResponse(200, "get_trust_monitor_events.json.resp")
+            if params == {"mintime": "1682930026000", "maxtime": "1682929966000", "limit": "200"}:
+                return MockResponse(200, "get_trust_monitor_events.json.resp")
             if params == {"mintime": "1682843686000", "maxtime": "1682930026000", "limit": "200", "offset": "1591014"}:
                 return MockResponse(200, "get_trust_monitor_events_2.json.resp")
             if params == {"mintime": "1682670886400", "maxtime": "1682929966000", "limit": "200"}: