From f7e7d87093b73680dc401424abf3e564c0189d1d Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 21 Apr 2022 10:26:09 +0200 Subject: [PATCH] Fix pagination bug that skipped events when more than one page is present. (#3140) --- packages/google_workspace/changelog.yml | 5 +++++ .../data_stream/admin/agent/stream/httpjson.yml.hbs | 2 +- .../data_stream/drive/agent/stream/httpjson.yml.hbs | 2 +- .../data_stream/groups/agent/stream/httpjson.yml.hbs | 2 +- .../data_stream/login/agent/stream/httpjson.yml.hbs | 2 +- .../data_stream/saml/agent/stream/httpjson.yml.hbs | 2 +- .../data_stream/user_accounts/agent/stream/httpjson.yml.hbs | 2 +- packages/google_workspace/manifest.yml | 2 +- 8 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml index 77c6fec1a31..b354fb6856c 100644 --- a/packages/google_workspace/changelog.yml +++ b/packages/google_workspace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.4" + changes: + - description: Fix pagination to prevent skipped events when more than one page is present. + type: bugfix + link: https://github.com/elastic/integrations/pull/3140 - version: "1.3.3" changes: - description: Add documentation for multi-fields diff --git a/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs index 973357f53b1..0f8ae6142e0 100644 --- a/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/admin/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs index f4cd56a3411..39433017889 100644 --- a/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/drive/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs index e7db1c71787..d1486b825af 100644 --- a/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/groups/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs index edcd159d1f7..3d83580dca0 100644 --- a/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/login/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs index dd1365492ee..c5e1386ebe3 100644 --- a/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/saml/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs b/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs index 872b3c6f4d9..9c4e11a5fea 100644 --- a/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs +++ b/packages/google_workspace/data_stream/user_accounts/agent/stream/httpjson.yml.hbs @@ -12,7 +12,7 @@ request.timeout: {{http_client_timeout}} request.transforms: - set: target: url.params.startTime - value: "[[.cursor.last_execution_datetime]]" + value: '[[if eq .last_response.page 0]][[.cursor.last_execution_datetime]][[else]][[.last_response.url.params.Get "startTime"]][[end]]' default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]' response.split: target: body.items diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml index 4b70efeca05..35e8d3b026b 100644 --- a/packages/google_workspace/manifest.yml +++ b/packages/google_workspace/manifest.yml @@ -1,6 +1,6 @@ name: google_workspace title: Google Workspace Audit Reports -version: 1.3.3 +version: 1.3.4 release: ga description: Collect audit reports from Google Workspaces with Elastic Agent. type: integration