Fix pagination bug that skipped events when more than one page is pre…

…sent. (#3140)
elastic · Apr 21, 2022 · f7e7d87 · f7e7d87
1 parent d2b6e24
commit f7e7d87
Show file tree

Hide file tree

Showing 8 changed files with 12 additions and 7 deletions.
diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.4"
+  changes:
+    - description: Fix pagination to prevent skipped events when more than one page is present.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3140
 - version: "1.3.3"
   changes:
     - description: Add documentation for multi-fields

diff --git a/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs
@@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}}
 request.transforms:
   - set:
       target: url.params.startTime
-      value: "[[.cursor.last_execution_datetime]]"
+      value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]'
       default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]'
 response.split:
   target: body.items

diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml
@@ -1,6 +1,6 @@
 name: google_workspace
 title: Google Workspace Audit Reports
-version: 1.3.3
+version: 1.3.4
 release: ga
 description: Collect audit reports from Google Workspaces with Elastic Agent.
 type: integration