From 308d9ca79131e3791cd7ce951ae90dfb0bba2779 Mon Sep 17 00:00:00 2001 From: Quan Zhou Date: Fri, 30 Aug 2024 17:19:02 +0200 Subject: [PATCH] [elk] Handle OpenSearch page info on pagination This code allows to handle the page info returned when iterating over paginated results in OpenSearch. Signed-off-by: Quan Zhou --- grimoire_elk/elk.py | 7 +++++-- .../handle-opensearch-page-info-on-pagination.yml | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 releases/unreleased/handle-opensearch-page-info-on-pagination.yml diff --git a/grimoire_elk/elk.py b/grimoire_elk/elk.py index 47f631c8e..0e100530e 100755 --- a/grimoire_elk/elk.py +++ b/grimoire_elk/elk.py @@ -683,7 +683,9 @@ def get_uuids_in_index(target_uuids): ) hits = [] - if page['hits']['total'] != 0: + total = page['hits']['total'] + total_value = total['value'] if isinstance(total, dict) else total + if total_value != 0: hits = page['hits']['hits'] return hits @@ -810,7 +812,8 @@ def delete_inactive_unique_identities(es, sortinghat_db, before_date): ) sid = page['_scroll_id'] - scroll_size = page['hits']['total'] + total = page['hits']['total'] + scroll_size = total['value'] if isinstance(total, dict) else total if scroll_size == 0: logging.warning("[identities retention] No inactive identities found in {} after {}!".format( diff --git a/releases/unreleased/handle-opensearch-page-info-on-pagination.yml b/releases/unreleased/handle-opensearch-page-info-on-pagination.yml new file mode 100644 index 000000000..a2cfae615 --- /dev/null +++ b/releases/unreleased/handle-opensearch-page-info-on-pagination.yml @@ -0,0 +1,8 @@ +--- +title: Handle OpenSearch page info on pagination +category: fixed +author: Quan Zhou +issue: null +notes: > + In OpenSearch and ElasticSearch < 7.x the page info on + pagination is different. This will handle both of them.