From e839a048e2ec5ecfea09a72a93eb7fc0dea52115 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 11 May 2023 10:22:50 +0200 Subject: [PATCH 1/3] (dsl): Support LocalDate elastic primitive and date format for range queries --- .../zio/elasticsearch/HttpExecutorSpec.scala | 27 +++++++++++ .../zio/elasticsearch/ElasticPrimitive.scala | 5 ++ .../zio/elasticsearch/query/Queries.scala | 13 ++++-- .../zio/elasticsearch/ElasticQuerySpec.scala | 46 ++++++++++++++----- 4 files changed, 76 insertions(+), 15 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index ac53ab568..21002b546 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -532,6 +532,33 @@ object HttpExecutorSpec extends IntegrationSpec { Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ), + test("search for first 2 documents using range query with date format") { + checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument, genDocumentId, genTestDocument) { + (firstDocumentId, firstDocument, secondDocumentId, secondDocument, thirdDocumentId, thirdDocument) => + val firstDocumentUpdated = firstDocument.copy(dateField = LocalDate.now.minusDays(2)) + val secondDocumentUpdated = secondDocument.copy(dateField = LocalDate.now) + val thirdDocumentUpdated = thirdDocument.copy(dateField = LocalDate.now.plusDays(2)) + for { + _ <- Executor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) + _ <- Executor.execute( + ElasticRequest.upsert[TestDocument](firstSearchIndex, firstDocumentId, firstDocumentUpdated) + ) + _ <- Executor.execute( + ElasticRequest.upsert[TestDocument](firstSearchIndex, secondDocumentId, secondDocumentUpdated) + ) + _ <- Executor.execute( + ElasticRequest + .upsert[TestDocument](firstSearchIndex, thirdDocumentId, thirdDocumentUpdated) + .refreshTrue + ) + query = range(TestDocument.dateField).gte(LocalDate.now).format("uuuu-MM-dd") + res <- Executor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[TestDocument] + } yield assert(res)(equalTo(List(secondDocumentUpdated, thirdDocumentUpdated))) + } + } @@ around( + Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), + Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie + ), test("search for documents with source filtering") { checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument, genDocumentId, genTestDocument) { (firstDocumentId, firstDocument, secondDocumentId, secondDocument, thirdDocumentId, thirdDocument) => diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticPrimitive.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticPrimitive.scala index f8b43a4db..c61face51 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticPrimitive.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticPrimitive.scala @@ -19,6 +19,7 @@ package zio.elasticsearch import zio.json.ast.Json import zio.json.ast.Json.{Num, Str} +import java.time.LocalDate import java.util.UUID object ElasticPrimitive { @@ -54,6 +55,10 @@ object ElasticPrimitive { def toJson(value: UUID): Json = Str(value.toString) } + implicit object ElasticLocalDate extends ElasticPrimitive[LocalDate] { + def toJson(value: LocalDate): Json = Str(value.toString) + } + final implicit class ElasticPrimitiveOps[A](private val value: A) extends AnyVal { def toJson(implicit EP: ElasticPrimitive[A]): Json = EP.toJson(value) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala index b3d1f1371..225ece252 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala @@ -18,6 +18,7 @@ package zio.elasticsearch.query import zio.elasticsearch.ElasticPrimitive._ import zio.elasticsearch.query.options._ +import zio.elasticsearch.query.sort.options.HasFormat import zio.json.ast.Json import zio.json.ast.Json.{Arr, Num, Obj, Str} import zio.schema.Schema @@ -353,7 +354,8 @@ private[elasticsearch] case object Unbounded extends LowerBound with UpperBound sealed trait RangeQuery[S, A, LB <: LowerBound, UB <: UpperBound] extends ElasticQuery[S] - with HasBoost[RangeQuery[S, A, LB, UB]] { + with HasBoost[RangeQuery[S, A, LB, UB]] + with HasFormat[RangeQuery[S, A, LB, UB]] { def gt[B <: A: ElasticPrimitive](value: B)(implicit @unused ev: LB =:= Unbounded.type ): RangeQuery[S, B, GreaterThan[B], UB] @@ -375,11 +377,15 @@ private[elasticsearch] final case class Range[S, A, LB <: LowerBound, UB <: Uppe field: String, lower: LB, upper: UB, - boost: Option[Double] + boost: Option[Double], + format: Option[String] ) extends RangeQuery[S, A, LB, UB] { self => def boost(value: Double): RangeQuery[S, A, LB, UB] = self.copy(boost = Some(value)) + def format(value: String): RangeQuery[S, A, LB, UB] = + self.copy(format = Some(value)) + def gt[B <: A: ElasticPrimitive](value: B)(implicit @unused ev: LB =:= Unbounded.type ): RangeQuery[S, B, GreaterThan[B], UB] = @@ -414,7 +420,8 @@ private[elasticsearch] object Range { field = field, lower = Unbounded, upper = Unbounded, - boost = None + boost = None, + format = None ) } diff --git a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala index 97bc6638b..7cf1a766c 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala @@ -259,7 +259,13 @@ object ElasticQuerySpec extends ZIOSpecDefault { must = List(Terms(field = "stringField", values = List("a", "b", "c"), boost = None)), mustNot = List(Match(field = "intField", value = 50, boost = None)), should = List( - Range(field = "intField", lower = GreaterThan(1), upper = LessThanOrEqualTo(100), boost = None), + Range( + field = "intField", + lower = GreaterThan(1), + upper = LessThanOrEqualTo(100), + boost = None, + format = None + ), Match(field = "stringField", value = "test", boost = None) ), boost = None, @@ -302,7 +308,13 @@ object ElasticQuerySpec extends ZIOSpecDefault { must = List(Terms(field = "stringField", values = List("a", "b", "c"), boost = None)), mustNot = List(Match(field = "intField", value = 50, boost = None)), should = List( - Range(field = "intField", lower = GreaterThan(1), upper = LessThanOrEqualTo(100), boost = None), + Range( + field = "intField", + lower = GreaterThan(1), + upper = LessThanOrEqualTo(100), + boost = None, + format = None + ), Match(field = "stringField", value = "test", boost = None) ), boost = Some(3.14), @@ -595,7 +607,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "testField", lower = Unbounded, upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -605,7 +618,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "stringField", lower = Unbounded, upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -615,7 +629,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "intField", lower = Unbounded, upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -625,7 +640,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "stringField.test", lower = Unbounded, upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -635,7 +651,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "doubleField", lower = GreaterThan(3.14), upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -645,7 +662,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "doubleField", lower = Unbounded, upper = LessThan(10.21), - boost = None + boost = None, + format = None ) ) ) && @@ -655,7 +673,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "intField", lower = GreaterThanOrEqualTo(10), upper = Unbounded, - boost = None + boost = None, + format = None ) ) ) && @@ -665,7 +684,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "intField", lower = Unbounded, upper = LessThanOrEqualTo(21), - boost = None + boost = None, + format = None ) ) ) && @@ -675,7 +695,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "doubleField", lower = GreaterThan(3.14), upper = LessThanOrEqualTo(21.0), - boost = None + boost = None, + format = None ) ) ) && @@ -685,7 +706,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { field = "doubleField", lower = GreaterThan(3.14), upper = LessThanOrEqualTo(21), - boost = Some(2.8) + boost = Some(2.8), + format = None ) ) ) From 4e40c18e3bf9fdbb0b4d24befe24848e6e58d561 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 11 May 2023 10:44:12 +0200 Subject: [PATCH 2/3] Add Query DSL tests --- .../zio/elasticsearch/query/Queries.scala | 17 ++++++---- .../zio/elasticsearch/ElasticQuerySpec.scala | 32 ++++++++++++++++++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala index 225ece252..fd77d8343 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala @@ -16,6 +16,7 @@ package zio.elasticsearch.query +import zio.Chunk import zio.elasticsearch.ElasticPrimitive._ import zio.elasticsearch.query.options._ import zio.elasticsearch.query.sort.options.HasFormat @@ -406,12 +407,16 @@ private[elasticsearch] final case class Range[S, A, LB <: LowerBound, UB <: Uppe ): RangeQuery[S, B, LB, LessThanOrEqualTo[B]] = self.copy(upper = LessThanOrEqualTo(value)) - def paramsToJson(fieldPath: Option[String]): Json = { - val rangeFields = Some( - fieldPath.foldRight(field)(_ + "." + _) -> Obj(List(lower.toJson, upper.toJson).flatten: _*) - ) ++ boost.map("boost" -> Num(_)) - Obj("range" -> Obj(rangeFields.toList: _*)) - } + def paramsToJson(fieldPath: Option[String]): Json = + Obj( + "range" -> Obj( + Chunk( + Some(fieldPath.foldRight(field)(_ + "." + _) -> Obj(List(lower.toJson, upper.toJson).flatten: _*)), + boost.map("boost" -> Num(_)), + format.map("format" -> Str(_)) + ).flatten: _* + ) + ) } private[elasticsearch] object Range { diff --git a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala index 7cf1a766c..0e61e806b 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala @@ -25,6 +25,8 @@ import zio.prelude.Validation import zio.test.Assertion.equalTo import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assert} +import java.time.LocalDate + object ElasticQuerySpec extends ZIOSpecDefault { def spec: Spec[TestEnvironment, Any] = suite("ElasticQuery")( @@ -600,6 +602,7 @@ object ElasticQuerySpec extends ZIOSpecDefault { val queryInclusiveUpperBound = range(TestDocument.intField).lte(21) val queryMixedBounds = queryLowerBound.lte(21.0) val queryWithBoostParam = queryMixedBounds.boost(2.8) + val queryWithFormatParam = range(TestDocument.dateField).gt(LocalDate.of(2023, 5, 11)).format("uuuu-MM-dd") assert(query)( equalTo( @@ -710,6 +713,17 @@ object ElasticQuerySpec extends ZIOSpecDefault { format = None ) ) + ) && + assert(queryWithFormatParam)( + equalTo( + Range[TestDocument, LocalDate, GreaterThan[LocalDate], Unbounded.type]( + field = "dateField", + lower = GreaterThan(LocalDate.of(2023, 5, 11)), + upper = Unbounded, + boost = None, + format = Some("uuuu-MM-dd") + ) + ) ) }, test("startsWith") { @@ -2000,6 +2014,7 @@ object ElasticQuerySpec extends ZIOSpecDefault { val queryInclusiveUpperBound = range(TestDocument.intField).lte(45) val queryMixedBounds = range(TestDocument.intField).gt(10).lte(99) val queryMixedBoundsWithBoost = range(TestDocument.intField).gt(10).lte(99).boost(3.14) + val queryWithFormat = range(TestDocument.dateField).gt(LocalDate.of(2020, 1, 10)).format("uuuu-MM-dd") val expectedEmpty = """ @@ -2107,6 +2122,20 @@ object ElasticQuerySpec extends ZIOSpecDefault { |} |""".stripMargin + val expectedWithFormat = + """ + |{ + | "query": { + | "range": { + | "dateField": { + | "gt": "2020-01-10" + | }, + | "format": "uuuu-MM-dd" + | } + | } + |} + |""".stripMargin + assert(queryEmpty.toJson)(equalTo(expectedEmpty.toJson)) && assert(queryEmptyWithBoost.toJson)(equalTo(expectedWithBoost.toJson)) && assert(queryLowerBound.toJson)(equalTo(expectedLowerBound.toJson)) && @@ -2114,7 +2143,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { assert(queryInclusiveLowerBound.toJson)(equalTo(expectedInclusiveLowerBound.toJson)) && assert(queryInclusiveUpperBound.toJson)(equalTo(expectedInclusiveUpperBound.toJson)) && assert(queryMixedBounds.toJson)(equalTo(expectedMixedBounds.toJson)) && - assert(queryMixedBoundsWithBoost.toJson)(equalTo(expectedMixedBoundsWithBoost.toJson)) + assert(queryMixedBoundsWithBoost.toJson)(equalTo(expectedMixedBoundsWithBoost.toJson)) && + assert(queryWithFormat.toJson)(equalTo(expectedWithFormat.toJson)) }, test("startsWith") { val query = startsWith(TestDocument.stringField, "test") From 3fdfa397064f43e874759365050fd4c73feddc71 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 11 May 2023 11:16:47 +0200 Subject: [PATCH 3/3] Fix encoding to json and failing tests --- .../zio/elasticsearch/HttpExecutorSpec.scala | 2 +- .../scala/zio/elasticsearch/query/Queries.scala | 13 ++++++++----- .../zio/elasticsearch/ElasticQuerySpec.scala | 16 ++++++++-------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 21002b546..1f6372f4c 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -551,7 +551,7 @@ object HttpExecutorSpec extends IntegrationSpec { .upsert[TestDocument](firstSearchIndex, thirdDocumentId, thirdDocumentUpdated) .refreshTrue ) - query = range(TestDocument.dateField).gte(LocalDate.now).format("uuuu-MM-dd") + query = range(TestDocument.dateField).gte(LocalDate.now).format("uuuu-MM-dd").boost(1.0) res <- Executor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[TestDocument] } yield assert(res)(equalTo(List(secondDocumentUpdated, thirdDocumentUpdated))) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala index fd77d8343..75f103930 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala @@ -410,11 +410,14 @@ private[elasticsearch] final case class Range[S, A, LB <: LowerBound, UB <: Uppe def paramsToJson(fieldPath: Option[String]): Json = Obj( "range" -> Obj( - Chunk( - Some(fieldPath.foldRight(field)(_ + "." + _) -> Obj(List(lower.toJson, upper.toJson).flatten: _*)), - boost.map("boost" -> Num(_)), - format.map("format" -> Str(_)) - ).flatten: _* + fieldPath.foldRight(field)(_ + "." + _) -> Obj( + Chunk( + lower.toJson, + upper.toJson, + boost.map("boost" -> Num(_)), + format.map("format" -> Str(_)) + ).flatten: _* + ) ) ) } diff --git a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala index 0e61e806b..02c09bba5 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala @@ -2034,8 +2034,8 @@ object ElasticQuerySpec extends ZIOSpecDefault { | "query": { | "range": { | "intField": { - | }, - | "boost": 3.14 + | "boost": 3.14 + | } | } | } |} @@ -2114,9 +2114,9 @@ object ElasticQuerySpec extends ZIOSpecDefault { | "range": { | "intField": { | "gt": 10, - | "lte": 99 - | }, - | "boost": 3.14 + | "lte": 99, + | "boost": 3.14 + | } | } | } |} @@ -2128,9 +2128,9 @@ object ElasticQuerySpec extends ZIOSpecDefault { | "query": { | "range": { | "dateField": { - | "gt": "2020-01-10" - | }, - | "format": "uuuu-MM-dd" + | "gt": "2020-01-10", + | "format": "uuuu-MM-dd" + | } | } | } |}