From 743345d7e63bfcabd2255daa22336def9322d38b Mon Sep 17 00:00:00 2001 From: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> Date: Tue, 21 Nov 2023 01:58:53 +0530 Subject: [PATCH] Handle canMatchSearchAfter for frozen context scenario (#11249) * Handle canMatchSearchAfter for frozen context scenario Signed-off-by: Chaitanya Gohel * Addressig review comments Signed-off-by: Chaitanya Gohel * Enable test for 2.12.0 for nano date now Signed-off-by: Chaitanya Gohel --------- Signed-off-by: Chaitanya Gohel Signed-off-by: Chaitanya Gohel <104654647+gashutos@users.noreply.github.com> --- CHANGELOG.md | 1 + .../test/search/240_date_nanos.yml | 96 +++++++++++++++++++ .../test/search/90_search_after.yml | 2 +- .../search/internal/ContextIndexSearcher.java | 5 +- .../search/sort/FieldSortBuilder.java | 22 +++-- 5 files changed, 116 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 412b4a99c8b16..0e2446ab64595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix per request latency last phase not tracked ([#10934](https://github.com/opensearch-project/OpenSearch/pull/10934)) - Adding version condition while adding geoshape doc values to the index, to ensure backward compatibility.([#11095](https://github.com/opensearch-project/OpenSearch/pull/11095)) - Fix SuggestSearch.testSkipDuplicates by forcing refresh when indexing its test documents ([#11068](https://github.com/opensearch-project/OpenSearch/pull/11068)) +- Handle canMatchSearchAfter for frozen context scenario ([#11249](https://github.com/opensearch-project/OpenSearch/pull/11249)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index 1ddba45c97c72..fb5c3268a3c82 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -164,3 +164,99 @@ setup: - match: { aggregations.date.buckets.1.key: 1540857600000 } - match: { aggregations.date.buckets.1.key_as_string: "2018-10-30T00:00:00.000Z" } - match: { aggregations.date.buckets.1.doc_count: 2 } + + +--- +"date with nested sort now": + - skip: + version: " - 2.11.99" + reason: fixed in 2.12.0 + + # This tests cover scenario where nested sort have now() in date field type. + # For this test, we have date field as nested field and we trigger asc/desc sort + # on that nested field. `filter` clause is needed when we sort any nested field, + # like in this case, "gte": "now/h" says sort nested field date_field only where + # document is having value greater than current time now(). + # Nested field sort query doesn't sort documents if it is not qualified through + # `filter` clause. + # Only adding tests for `gte` as `lte` would be same behaviour + + - do: + indices.create: + index: test + body: + mappings: + properties: + nested_field: + type: nested + properties: + date_field: + type: date + format: date_optional_time + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"nested_field": [{"date_field": "3023-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "3024-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "3025-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "3026-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "3027-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "2022-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "2023-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "2021-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "2020-10-26T12:00:00+09:00"}]} + {"index":{}} + {"nested_field": [{"date_field": "2019-10-26T12:00:00+09:00"}]} + + # gte: now/h with the desc sort + - do: + search: + index: test + body: + size: 5 + sort: [{ nested_field.date_field: { mode: max, order: desc, nested: { path: nested_field, filter: { bool: { filter : [{ range : { nested_field.date_field: { gte: now/h, time_zone: +09:00} } }] } } } } } ] + - match: {hits.total.value: 10 } + - length: {hits.hits: 5 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._source.nested_field.0.date_field: "3027-10-26T12:00:00+09:00" } + - match: { hits.hits.0.sort: [33381428400000] } + - match: { hits.hits.1._source.nested_field.0.date_field: "3026-10-26T12:00:00+09:00" } + - match: { hits.hits.1.sort: [ 33349892400000 ] } + - match: { hits.hits.2._source.nested_field.0.date_field: "3025-10-26T12:00:00+09:00" } + - match: { hits.hits.2.sort: [ 33318356400000 ] } + - match: { hits.hits.3._source.nested_field.0.date_field: "3024-10-26T12:00:00+09:00" } + - match: { hits.hits.3.sort: [ 33286820400000 ] } + - match: { hits.hits.4._source.nested_field.0.date_field: "3023-10-26T12:00:00+09:00" } + - match: { hits.hits.4.sort: [ 33255198000000 ] } + + # gte: now/h with the asc sort + - do: + search: + index: test + body: + size: 5 + sort: [ { nested_field.date_field: { mode: max, order: asc, nested: { path: nested_field, filter: { bool: { filter: [ { range: { nested_field.date_field: { gte: now/h, time_zone: +09:00 } } } ] } } } } } ] + - match: { hits.total.value: 10 } + - length: { hits.hits: 5 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0._source.nested_field.0.date_field: "3023-10-26T12:00:00+09:00" } + - match: { hits.hits.0.sort: [ 33255198000000 ] } + - match: { hits.hits.1._source.nested_field.0.date_field: "3024-10-26T12:00:00+09:00" } + - match: { hits.hits.1.sort: [ 33286820400000 ] } + - match: { hits.hits.2._source.nested_field.0.date_field: "3025-10-26T12:00:00+09:00" } + - match: { hits.hits.2.sort: [ 33318356400000 ] } + - match: { hits.hits.3._source.nested_field.0.date_field: "3026-10-26T12:00:00+09:00" } + - match: { hits.hits.3.sort: [ 33349892400000 ] } + - match: { hits.hits.4._source.nested_field.0.date_field: "3027-10-26T12:00:00+09:00" } + - match: { hits.hits.4.sort: [ 33381428400000 ] } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 41c8626afcd75..b384038a04550 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -219,7 +219,7 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828733" } - match: {hits.hits.0.sort: [1571646604828733000] } - # search_after with the sort + # search_after with the asc sort - do: search: index: test diff --git a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java index 5c1c2a0e410c8..b042f3cf41d61 100644 --- a/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java @@ -507,14 +507,15 @@ private boolean canMatch(LeafReaderContext ctx) throws IOException { } private boolean canMatchSearchAfter(LeafReaderContext ctx) throws IOException { - if (searchContext.request() != null && searchContext.request().source() != null) { + if (searchContext.searchAfter() != null && searchContext.request() != null && searchContext.request().source() != null) { // Only applied on primary sort field and primary search_after. FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(searchContext.request().source()); if (primarySortField != null) { MinAndMax minMax = FieldSortBuilder.getMinMaxOrNullForSegment( this.searchContext.getQueryShardContext(), ctx, - primarySortField + primarySortField, + searchContext.sort() ); return SearchService.canMatchSearchAfter( searchContext.searchAfter(), diff --git a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java index 8403a65111307..ff20502115c50 100644 --- a/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/opensearch/search/sort/FieldSortBuilder.java @@ -616,7 +616,8 @@ public static FieldSortBuilder getPrimaryFieldSortOrNull(SearchSourceBuilder sou * and configurations return null. */ public static MinAndMax getMinMaxOrNull(QueryShardContext context, FieldSortBuilder sortBuilder) throws IOException { - return getMinMaxOrNullInternal(context.getIndexReader(), context, sortBuilder); + final SortAndFormats sort = SortBuilder.buildSort(Collections.singletonList(sortBuilder), context).get(); + return getMinMaxOrNullInternal(context.getIndexReader(), context, sortBuilder, sort); } /** @@ -624,14 +625,21 @@ public static MinAndMax getMinMaxOrNull(QueryShardContext context, FieldSortB * The value can be extracted on non-nested indexed mapped fields of type keyword, numeric or date, other fields * and configurations return null. */ - public static MinAndMax getMinMaxOrNullForSegment(QueryShardContext context, LeafReaderContext ctx, FieldSortBuilder sortBuilder) - throws IOException { - return getMinMaxOrNullInternal(ctx.reader(), context, sortBuilder); + public static MinAndMax getMinMaxOrNullForSegment( + QueryShardContext context, + LeafReaderContext ctx, + FieldSortBuilder sortBuilder, + SortAndFormats sort + ) throws IOException { + return getMinMaxOrNullInternal(ctx.reader(), context, sortBuilder, sort); } - private static MinAndMax getMinMaxOrNullInternal(IndexReader reader, QueryShardContext context, FieldSortBuilder sortBuilder) - throws IOException { - SortAndFormats sort = SortBuilder.buildSort(Collections.singletonList(sortBuilder), context).get(); + private static MinAndMax getMinMaxOrNullInternal( + IndexReader reader, + QueryShardContext context, + FieldSortBuilder sortBuilder, + SortAndFormats sort + ) throws IOException { SortField sortField = sort.sort.getSort()[0]; if (sortField.getField() == null) { return null;