Store inner hits as Hit (#288)

lambdaworks · Jul 17, 2023 · 7f5fe52 · 7f5fe52
1 parent 470a813
commit 7f5fe52
Show file tree

Hide file tree

Showing 11 changed files with 97 additions and 39 deletions.
diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala
@@ -27,12 +27,14 @@ import zio.elasticsearch.domain.{PartialTestDocument, TestDocument, TestSubDocum
 import zio.elasticsearch.executor.Executor
 import zio.elasticsearch.query.DistanceUnit.Kilometers
 import zio.elasticsearch.query.FunctionScoreFunction.randomScoreFunction
+import zio.elasticsearch.query.{FunctionScoreBoostMode, FunctionScoreFunction, InnerHits}
 import zio.elasticsearch.query.sort.SortMode.Max
 import zio.elasticsearch.query.sort.SortOrder._
 import zio.elasticsearch.query.sort.SourceType.NumberType
 import zio.elasticsearch.query.{Distance, FunctionScoreBoostMode, FunctionScoreFunction}
 import zio.elasticsearch.request.{CreationOutcome, DeletionOutcome}
 import zio.elasticsearch.result._
+import zio.elasticsearch.result.{Item, MaxAggregationResult, UpdateByQueryResult}
 import zio.elasticsearch.script.{Painless, Script}
 import zio.json.ast.Json.{Arr, Str}
 import zio.schema.codec.JsonCodec
@@ -1141,6 +1143,55 @@ object HttpExecutorSpec extends IntegrationSpec {
             Executor.execute(ElasticRequest.createIndex(firstSearchIndex)),
             Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie
           ),
+          test("successfully find inner hit document with highlight") {
+            checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument) {
+              (firstDocumentId, firstDocument, secondDocumentId, secondDocument) =>
+                for {
+                  _ <- Executor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll))
+                  _ <-
+                    Executor.execute(
+                      ElasticRequest.upsert[TestDocument](firstSearchIndex, firstDocumentId, firstDocument)
+                    )
+                  _ <- Executor.execute(
+                         ElasticRequest
+                           .upsert[TestDocument](firstSearchIndex, secondDocumentId, secondDocument)
+                           .refreshTrue
+                       )
+                  query = nested(
+                            path = TestDocument.subDocumentList,
+                            query = must(
+                              matches(
+                                TestSubDocument.stringField,
+                                secondDocument.subDocumentList.headOption.map(_.stringField).getOrElse("foo")
+                              )
+                            )
+                          ).innerHits(
+                            InnerHits().highlights(highlight(TestSubDocument.stringField))
+                          )
+                  result <- Executor.execute(ElasticRequest.search(firstSearchIndex, query))
+                  items  <- result.items
+                  res = items
+                          .flatMap(_.innerHit("subDocumentList"))
+                          .flatten
+                          .flatMap(_.highlight("subDocumentList.stringField"))
+                          .flatten
+                } yield assert(res)(
+                  Assertion.contains(
+                    secondDocument.subDocumentList.headOption
+                      .map(doc => s"<em>${doc.stringField}</em>")
+                      .getOrElse("<em>foo</em>")
+                  )
+                )
+            }
+          } @@ around(
+            Executor.execute(
+              ElasticRequest.createIndex(
+                firstSearchIndex,
+                """{ "mappings": { "properties": { "subDocumentList": { "type": "nested" } } } }"""
+              )
+            ),
+            Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie
+          ),
           test("successfully find document with highlight using field accessor") {
             checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument) {
               (firstDocumentId, firstDocument, secondDocumentId, secondDocument) =>

diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala
@@ -660,7 +660,7 @@ object ElasticRequest {
 
       val sizeJson: Json = size.fold(Obj())(s => Obj("size" -> s.toJson))
 
-      val highlightsJson: Json = highlights.fold(Obj())(h => Obj("highlight" -> h.toJson))
+      val highlightsJson: Json = highlights.fold(Obj())(h => Obj("highlight" -> h.toJson(None)))
 
       val searchAfterJson: Json = searchAfter.fold(Obj())(sa => Obj("search_after" -> sa))
 
@@ -751,7 +751,7 @@ object ElasticRequest {
 
       val sizeJson: Json = size.fold(Obj())(s => Obj("size" -> s.toJson))
 
-      val highlightsJson: Json = highlights.fold(Obj())(h => Obj("highlight" -> h.toJson))
+      val highlightsJson: Json = highlights.fold(Obj())(h => Obj("highlight" -> h.toJson(None)))
 
       val searchAfterJson: Json = searchAfter.fold(Obj())(sa => Obj("search_after" -> sa))
 

diff --git a/modules/library/src/main/scala/zio/elasticsearch/executor/HttpExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/executor/HttpExecutor.scala
@@ -40,6 +40,7 @@ import zio.elasticsearch.executor.response.{
   CreateResponse,
   DocumentWithHighlightsAndSort,
   GetResponse,
+  Hit,
   SearchWithAggregationsResponse,
   UpdateByQueryResponse
 }
@@ -382,7 +383,7 @@ private[elasticsearch] final class HttpExecutor private (esConfig: ElasticConfig
         case HttpOk =>
           response.body.fold(
             e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")),
-            value =>
+            value => {
               ZIO
                 .fromEither(value.innerHitsResults)
                 .map { innerHitsResults =>
@@ -395,6 +396,7 @@ private[elasticsearch] final class HttpExecutor private (esConfig: ElasticConfig
                   )
                 }
                 .mapError(error => DecodingException(s"Could not parse inner_hits: $error"))
+            }
           )
         case _ =>
           ZIO.fail(handleFailuresFromCustomResponse(response))
@@ -610,7 +612,7 @@ private[elasticsearch] final class HttpExecutor private (esConfig: ElasticConfig
 
   private def itemsFromDocumentsWithHighlightsSortAndInnerHits(
     results: Chunk[DocumentWithHighlightsAndSort],
-    innerHits: Chunk[Map[String, Chunk[Json]]]
+    innerHits: Chunk[Map[String, Chunk[Hit]]]
   ): Chunk[Item] =
     results.zip(innerHits).map { case (r, innerHits) =>
       Item(raw = r.source, highlight = r.highlight, innerHits = innerHits, sort = r.sort)

diff --git a/...y/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala b/...y/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala
@@ -35,19 +35,19 @@ private[elasticsearch] final case class SearchWithAggregationsResponse(
   hits: Hits,
   aggregations: Option[Json]
 ) {
-  lazy val innerHitsResults: Either[String, Chunk[Map[String, Chunk[Json]]]] =
+  lazy val innerHitsResults: Either[String, Chunk[Map[String, Chunk[Hit]]]] =
     Validation
       .validateAll(
         hits.hits
-          .map(_.innerHits.fold[Validation[String, Map[String, Chunk[Json]]]](Validation.succeed(Map.empty)) {
+          .map(_.innerHits.fold[Validation[String, Map[String, Chunk[Hit]]]](Validation.succeed(Map.empty)) {
             innerHits =>
               Validation
                 .validateAll(
                   innerHits.fields.map { case (name, response) =>
                     Validation.fromEither(
                       response
                         .as[InnerHitsResponse]
-                        .map(innerHitsResponse => (name, innerHitsResponse.hits.hits.map(_.source)))
+                        .map(innerHitsResponse => (name, innerHitsResponse.hits.hits))
                     )
                   }
                 )

diff --git a/modules/library/src/main/scala/zio/elasticsearch/highlights/Highlights.scala b/modules/library/src/main/scala/zio/elasticsearch/highlights/Highlights.scala
@@ -99,15 +99,15 @@ private[elasticsearch] final case class Highlights(
   def withHighlight(field: String, config: HighlightConfig): Highlights =
     self.copy(fields = HighlightField(field, config) +: self.fields)
 
-  private[elasticsearch] def toJson: Json = Obj(configChunk) merge fieldsJson
+  private[elasticsearch] def toJson(fieldPath: Option[String]): Json = Obj(configChunk) merge fieldsJson(fieldPath)
 
   private lazy val configChunk: Chunk[(String, Json)] = Chunk.fromIterable(config)
 
-  private lazy val fieldsJson: Json =
+  private def fieldsJson(fieldPath: Option[String]): Json =
     if (explicitFieldOrder) {
-      Obj("fields" -> Arr(fields.reverse.map(_.toJsonObj)))
+      Obj("fields" -> Arr(fields.reverse.map(_.toJsonObj(fieldPath))))
     } else {
-      Obj("fields" -> Obj(fields.reverse.map(_.toStringJsonPair)))
+      Obj("fields" -> Obj(fields.reverse.map(_.toStringJsonPair(fieldPath))))
     }
 }
 
@@ -116,7 +116,8 @@ object Highlights {
 }
 
 private[elasticsearch] final case class HighlightField(field: String, config: HighlightConfig = Map.empty) {
-  def toStringJsonPair: (String, Obj) = field -> Obj(Chunk.fromIterable(config))
+  def toStringJsonPair(fieldPath: Option[String]): (String, Obj) =
+    fieldPath.map(_ + "." + field).getOrElse(field) -> Obj(Chunk.fromIterable(config))
 
-  def toJsonObj: Json = Obj(toStringJsonPair)
+  def toJsonObj(fieldPath: Option[String]): Json = Obj(toStringJsonPair(fieldPath))
 }
diff --git a/modules/library/src/main/scala/zio/elasticsearch/query/InnerHits.scala b/modules/library/src/main/scala/zio/elasticsearch/query/InnerHits.scala
@@ -132,7 +132,7 @@ final case class InnerHits private[elasticsearch] (
   def size(value: Int): InnerHits =
     self.copy(size = Some(value))
 
-  private[elasticsearch] def toStringJsonPair: (String, Json) = {
+  private[elasticsearch] def toStringJsonPair(fieldPath: Option[String]): (String, Json) = {
     val sourceJson: Option[Json] =
       (included, excluded) match {
         case (Chunk(), Chunk()) =>
@@ -148,7 +148,7 @@ final case class InnerHits private[elasticsearch] (
         from.map("from" -> Num(_)),
         size.map("size" -> Num(_)),
         name.map("name" -> Str(_)),
-        highlights.map("highlight" -> _.toJson),
+        highlights.map("highlight" -> _.toJson(fieldPath)),
         sourceJson.map("_source" -> _)
       ).flatten
     )

diff --git a/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala b/modules/library/src/main/scala/zio/elasticsearch/query/Queries.scala
@@ -466,7 +466,7 @@ private[elasticsearch] final case class HasChild[S](
           Some("type"  -> childType.toJson),
           Some("query" -> query.toJson(None)),
           ignoreUnmapped.map("ignore_unmapped" -> _.toJson),
-          innerHitsField.map(_.toStringJsonPair),
+          innerHitsField.map(_.toStringJsonPair(None)),
           maxChildren.map("max_children" -> _.toJson),
           minChildren.map("min_children" -> _.toJson),
           scoreMode.map("score_mode" -> _.toString.toLowerCase.toJson)
@@ -544,7 +544,7 @@ private[elasticsearch] final case class HasParent[S](
           boost.map("boost" -> _.toJson),
           ignoreUnmapped.map("ignore_unmapped" -> _.toJson),
           score.map("score" -> _.toJson),
-          innerHitsField.map(_.toStringJsonPair)
+          innerHitsField.map(_.toStringJsonPair(None))
         ).flatten
       )
     )
@@ -614,7 +614,7 @@ private[elasticsearch] final case class Nested[S](
           Some("query" -> query.toJson(fieldPath.map(_ + "." + path).orElse(Some(path)))),
           scoreMode.map("score_mode" -> _.toString.toLowerCase.toJson),
           ignoreUnmapped.map("ignore_unmapped" -> _.toJson),
-          innerHitsField.map(_.toStringJsonPair)
+          innerHitsField.map(_.toStringJsonPair(fieldPath.map(_ + "." + path).orElse(Some(path))))
         ).flatten
       )
     )

diff --git a/modules/library/src/main/scala/zio/elasticsearch/result/Item.scala b/modules/library/src/main/scala/zio/elasticsearch/result/Item.scala
@@ -18,6 +18,7 @@ package zio.elasticsearch.result
 
 import zio.Chunk
 import zio.elasticsearch.Field
+import zio.elasticsearch.executor.response.Hit
 import zio.json.DecoderOps
 import zio.json.ast.{Json, JsonCursor}
 import zio.prelude.Validation
@@ -27,8 +28,8 @@ import zio.schema.codec.JsonCodec.JsonDecoder
 
 final case class Item(
   raw: Json,
-  private val highlight: Option[Json] = None,
-  private val innerHits: Map[String, Chunk[Json]] = Map.empty,
+  highlight: Option[Json] = None,
+  private val innerHits: Map[String, Chunk[Hit]] = Map.empty,
   sort: Option[Json] = None
 ) {
   def documentAs[A](implicit schema: Schema[A]): Either[DecodeError, A] = JsonDecoder.decode(schema, raw.toString)
@@ -43,14 +44,17 @@ final case class Item(
   def highlight(field: Field[_, _]): Option[Chunk[String]] =
     highlight(field.toString)
 
+  def innerHit(name: String): Option[Chunk[Item]] =
+    innerHits.get(name).map(_.map(hit => Item(hit.source, hit.highlight)))
+
   def innerHitAs[A](name: String)(implicit schema: Schema[A]): Either[DecodingException, Chunk[A]] =
     for {
-      innerHitsJson <- innerHits.get(name).toRight(DecodingException(s"Could not find inner hits with name $name"))
+      innerHitItems <- innerHit(name).toRight(DecodingException(s"Could not find inner hits with name $name"))
       innerHits <-
         Validation
           .validateAll(
-            innerHitsJson.map(json =>
-              Validation.fromEither(JsonDecoder.decode(schema, json.toString)).mapError(_.message)
+            innerHitItems.map(item =>
+              Validation.fromEither(JsonDecoder.decode(schema, item.raw.toString)).mapError(_.message)
             )
           )
           .toEitherWith(errors =>

diff --git a/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala b/modules/library/src/test/scala/zio/elasticsearch/ElasticQuerySpec.scala
@@ -2550,7 +2550,7 @@ object ElasticQuerySpec extends ZIOSpecDefault {
               |      "name": "innerHitName",
               |      "highlight" : {
               |        "fields" : {
-              |          "stringField" : {}
+              |          "subDocumentList.stringField" : {}
               |        }
               |      },
               |      "_source" : {

diff --git a/modules/library/src/test/scala/zio/elasticsearch/HighlightsSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/HighlightsSpec.scala
@@ -228,32 +228,32 @@ object HighlightsSpec extends ZIOSpecDefault {
               |}
               |""".stripMargin
 
-          assert(highlightObject.toJson)(
+          assert(highlightObject.toJson(None))(
             equalTo(
               expected.toJson
             )
-          ) && assert(highlightWithHighlight.toJson)(
+          ) && assert(highlightWithHighlight.toJson(None))(
             equalTo(
               expectedWithFirstName.toJson
             )
-          ) && assert(highlightWithHighlightAndGlobalConfig.toJson)(
+          ) && assert(highlightWithHighlightAndGlobalConfig.toJson(None))(
             equalTo(
               expectedPlainWithFirstName.toJson
             )
-          ) && assert(highlightWithConfig.toJson)(
+          ) && assert(highlightWithConfig.toJson(None))(
             equalTo(
               expectedPlainWithRequiredFieldMatch.toJson
             )
-          ) && assert(highlightWithConfigAndHighlight.toJson)(
+          ) && assert(highlightWithConfigAndHighlight.toJson(None))(
             equalTo(
               expectedPlainWithMatchedFields.toJson
             )
-          ) && assert(highlightWithConfigHighlightAndExplicitFieldOrder.toJson)(
+          ) && assert(highlightWithConfigHighlightAndExplicitFieldOrder.toJson(None))(
             equalTo(
               expectedPlainWithArrayOfFields.toJson
             )
           ) &&
-          assert(highlightWithMultipleConfig.toJson)(
+          assert(highlightWithMultipleConfig.toJson(None))(
             equalTo(
               expectedFvhType.toJson
             )

diff --git a/modules/library/src/test/scala/zio/elasticsearch/InnerHitsSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/InnerHitsSpec.scala
@@ -231,14 +231,14 @@ object InnerHitsSpec extends ZIOSpecDefault {
             |}
             |""".stripMargin
 
-        assert(Obj(innerHits.toStringJsonPair))(equalTo(expected.toJson)) &&
-        assert(Obj(innerHitsWithExcluded.toStringJsonPair))(equalTo(expectedWithExcluded.toJson)) &&
-        assert(Obj(innerHitsWithFrom.toStringJsonPair))(equalTo(expectedWithFrom.toJson)) &&
-        assert(Obj(innerHitsWithHighlights.toStringJsonPair))(equalTo(expectedWithHighlights.toJson)) &&
-        assert(Obj(innerHitsWithIncluded.toStringJsonPair))(equalTo(expectedWithIncluded.toJson)) &&
-        assert(Obj(innerHitsWithName.toStringJsonPair))(equalTo(expectedWithName.toJson)) &&
-        assert(Obj(innerHitsWithSize.toStringJsonPair))(equalTo(expectedWithSize.toJson)) &&
-        assert(Obj(innerHitsWithAllParams.toStringJsonPair))(equalTo(expectedWithAllParams.toJson))
+        assert(Obj(innerHits.toStringJsonPair(None)))(equalTo(expected.toJson)) &&
+        assert(Obj(innerHitsWithExcluded.toStringJsonPair(None)))(equalTo(expectedWithExcluded.toJson)) &&
+        assert(Obj(innerHitsWithFrom.toStringJsonPair(None)))(equalTo(expectedWithFrom.toJson)) &&
+        assert(Obj(innerHitsWithHighlights.toStringJsonPair(None)))(equalTo(expectedWithHighlights.toJson)) &&
+        assert(Obj(innerHitsWithIncluded.toStringJsonPair(None)))(equalTo(expectedWithIncluded.toJson)) &&
+        assert(Obj(innerHitsWithName.toStringJsonPair(None)))(equalTo(expectedWithName.toJson)) &&
+        assert(Obj(innerHitsWithSize.toStringJsonPair(None)))(equalTo(expectedWithSize.toJson)) &&
+        assert(Obj(innerHitsWithAllParams.toStringJsonPair(None)))(equalTo(expectedWithAllParams.toJson))
       }
     )
 }