Improve highlighting settings (#3136)

* Handle fragmenter parameter * Handle tagsSchema parameter * Add fragmentOffset and matchedFields parameters * Add encoder, maxAnalyzedOffset and tagsSchema parameters * Handle/fix options parameter * Deprecate requireFieldMatchScan method * Deprecate forceSource method/parameter
Philippus · Aug 21, 2024 · d786b3f · d786b3f
1 parent f1bece0
commit d786b3f
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 10 deletions.
diff --git a/.../scala/com/sksamuel/elastic4s/requests/searches/queries/HighlightFieldBuilderFnTest.scala b/.../scala/com/sksamuel/elastic4s/requests/searches/queries/HighlightFieldBuilderFnTest.scala
@@ -17,9 +17,10 @@ class HighlightFieldBuilderFnTest extends AnyFunSuite with Matchers {
     val highlight = HighlightField("text").boundaryMaxScan(20)
     searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"boundary_max_scan":20}"""
   }
-  test("'forceSource' generates 'force_source' field.") {
+  test("'forceSource' has no effect.") {
     val highlight = HighlightField("text").forceSource(true)
-    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"force_source":true}"""
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe
+      """{}""".stripMargin
   }
   test("'fragmentOffset' generates 'fragment_offset' field.") {
     val highlight = HighlightField("text").fragmentOffset(100)
@@ -126,4 +127,24 @@ class HighlightFieldBuilderFnTest extends AnyFunSuite with Matchers {
     val highlight = HighlightField("text").requireFieldMatch(false)
     searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"require_field_match":false}"""
   }
+  test("'fragmenter' generates 'fragmenter' field.") {
+    val highlight = HighlightField("text").fragmenter("abc")
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"fragmenter":"abc"}"""
+  }
+  test("'encoder' generates 'encoder' field.") {
+    val highlight = HighlightField("text").encoder("abc")
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"encoder":"abc"}"""
+  }
+  test("'maxAnalyzedOffset' generates 'maxAnalyzedOffset' field.") {
+    val highlight = HighlightField("text").maxAnalyzedOffset(100)
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"max_analyzed_offset":100}"""
+  }
+  test("'tagsSchema' generates 'tagsSchema' field.") {
+    val highlight = HighlightField("text").tagsSchema("abc")
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"tags_schema":"abc"}"""
+  }
+  test("'options' generates 'options' fields.") {
+    val highlight = HighlightField("text").options(Map("a" -> "a", "b" -> 3, "c" -> true))
+    searches.HighlightFieldBuilderFn(highlight).string shouldBe """{"a":"a","b":3,"c":true}"""
+  }
 }
diff --git a/...tic4s-domain/src/main/scala/com/sksamuel/elastic4s/requests/searches/HighlightField.scala b/...tic4s-domain/src/main/scala/com/sksamuel/elastic4s/requests/searches/HighlightField.scala
@@ -6,6 +6,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query
 case class HighlightField(field: String,
                           boundaryChars: Option[Array[Char]] = None,
                           boundaryMaxScan: Option[Int] = None,
+                          @deprecated("This parameter has no effect", "8.15.0")
                           forceSource: Option[Boolean] = None,
                           fragmenter: Option[String] = None,
                           fragmentOffset: Option[Int] = None,
@@ -23,7 +24,10 @@ case class HighlightField(field: String,
                           phraseLimit: Option[Int] = None,
                           boundaryScanner: Option[String] = None,
                           boundaryScannerLocale: Option[String] = None,
-                          options: Option[Map[String, AnyRef]] = None) {
+                          options: Option[Map[String, Any]] = None,
+                          encoder: Option[String] = None,
+                          maxAnalyzedOffset: Option[Int] = None,
+                          tagsSchema: Option[String] = None) {
 
   def boundaryChars(chars: Array[Char]): HighlightField = copy(boundaryChars = chars.some)
   def boundaryChars(chars: String): HighlightField = copy(boundaryChars = chars.toCharArray.some)
@@ -32,13 +36,15 @@ case class HighlightField(field: String,
   def boundaryScanner(scanner: String): HighlightField = copy(boundaryScanner = scanner.some)
   def boundaryScannerLocale(locale: String): HighlightField = copy(boundaryScannerLocale = locale.some)
 
+  def encoder(encoder: String): HighlightField = copy(encoder = encoder.some)
   def fragmenter(fragmenter: String): HighlightField = copy(fragmenter = fragmenter.some)
   def fragmentOffset(fragmentOffset: Int): HighlightField = copy(fragmentOffset = fragmentOffset.some)
   def fragmentSize(fragmentSize: Int): HighlightField = copy(fragmentSize = fragmentSize.some)
 
   def requireFieldMatch(requireFieldMatch: Boolean): HighlightField =
     copy(requireFieldMatch = requireFieldMatch.some)
 
+  @deprecated("This method has no effect", "8.15.0")
   def forceSource(forceSource: Boolean): HighlightField = copy(forceSource = forceSource.some)
 
   def highlightFilter(highlightFilter: Boolean): HighlightField = copy(highlightFilter = highlightFilter.some)
@@ -47,6 +53,8 @@ case class HighlightField(field: String,
   def matchedFields(first: String, rest: String*): HighlightField = matchedFields(first +: rest)
   def matchedFields(fields: Iterable[String]): HighlightField = copy(matchedFields = fields.toSeq)
 
+  def maxAnalyzedOffset(maxAnalyzedOffset: Int): HighlightField = copy(maxAnalyzedOffset = maxAnalyzedOffset.some)
+
   def noMatchSize(noMatchSize: Int): HighlightField = copy(noMatchSize = noMatchSize.some)
 
   def numberOfFragments(numOfFragments: Int): HighlightField = copy(numOfFragments = numOfFragments.some)
@@ -63,8 +71,11 @@ case class HighlightField(field: String,
   def postTag(tags: String*): HighlightField = postTag(tags)
   def postTag(tags: Iterable[String]): HighlightField = copy(postTags = tags.toSeq)
 
-  def requireFieldMatchScan(req: Boolean): HighlightField = copy(requireFieldMatch = req.some)
+  @deprecated("Use requireFieldMatch", "8.15.0")
+  def requireFieldMatchScan(req: Boolean): HighlightField = requireFieldMatch(req)
+
+  def tagsSchema(tagsSchema: String): HighlightField = copy(tagsSchema = tagsSchema.some)
 
-  def options(newOptions: Map[String, AnyRef]): HighlightField = copy(options = newOptions.some)
+  def options(newOptions: Map[String, Any]): HighlightField = copy(options = newOptions.some)
 
 }
diff --git a/...c4s-domain/src/main/scala/com/sksamuel/elastic4s/requests/searches/HighlightOptions.scala b/...c4s-domain/src/main/scala/com/sksamuel/elastic4s/requests/searches/HighlightOptions.scala
@@ -12,6 +12,7 @@ case class HighlightOptions(encoder: Option[String] = None,
                             boundaryScannerLocale: Option[String] = None,
                             fragmenter: Option[String] = None,
                             fragmentSize: Option[Int] = None,
+                            @deprecated("This parameter has no effect", "8.15.0")
                             forceSource: Option[Boolean] = None,
                             highlighterType: Option[String] = None,
                             highlightFilter: Option[Boolean] = None,
@@ -24,7 +25,9 @@ case class HighlightOptions(encoder: Option[String] = None,
                             postTags: Seq[String] = Nil,
                             preTags: Seq[String] = Nil,
                             requireFieldMatch: Option[Boolean] = None,
-                            options: Option[Map[String, AnyRef]] = None) {
+                            options: Option[Map[String, Any]] = None,
+                            fragmentOffset: Option[Int] = None,
+                            matchedFields: Seq[String] = Nil) {
 
   def boundaryChars(boundaryChars: String): HighlightOptions = copy(boundaryChars = boundaryChars.some)
   def boundaryMaxScan(boundaryMaxScan: Int): HighlightOptions = copy(boundaryMaxScan = boundaryMaxScan.some)
@@ -39,7 +42,9 @@ case class HighlightOptions(encoder: Option[String] = None,
     copy(useExplicitFieldOrder = useExplicitFieldOrder.some)
 
   def fragmenter(fragmenter: String): HighlightOptions = copy(fragmenter = fragmenter.some)
+  def fragmentOffset(fragmentOffset: Int): HighlightOptions = copy(fragmentOffset = fragmentOffset.some)
   def fragmentSize(fragmentSize: Int): HighlightOptions = copy(fragmentSize = fragmentSize.some)
+  @deprecated("This method has no effect", "8.15.0")
   def forceSource(forceSource: Boolean): HighlightOptions = copy(forceSource = forceSource.some)
 
   def highlighterType(highlighterType: String): HighlightOptions =
@@ -51,6 +56,9 @@ case class HighlightOptions(encoder: Option[String] = None,
   def highlightQuery(highlightQuery: Query): HighlightOptions =
     copy(highlightQuery = highlightQuery.some)
 
+  def matchedFields(first: String, rest: String*): HighlightOptions = matchedFields(first +: rest)
+  def matchedFields(fields: Iterable[String]): HighlightOptions = copy(matchedFields = fields.toSeq)
+
   def noMatchSize(noMatchSize: Int): HighlightOptions = copy(noMatchSize = noMatchSize.some)
   def numOfFragments(numOfFragments: Int): HighlightOptions = copy(numOfFragments = numOfFragments.some)
   def order(order: String): HighlightOptions = copy(order = order.some)
@@ -63,7 +71,7 @@ case class HighlightOptions(encoder: Option[String] = None,
   def postTags(postTags: Iterable[String]): HighlightOptions = copy(postTags = postTags.toSeq)
   def preTags(preTags: Iterable[String]): HighlightOptions = copy(preTags = preTags.toSeq)
 
-  def options(newOptions: Map[String, AnyRef]): HighlightOptions = copy(options = newOptions.some)
+  def options(newOptions: Map[String, Any]): HighlightOptions = copy(options = newOptions.some)
 
   def requireFieldMatch(requireFieldMatch: Boolean): HighlightOptions =
     copy(requireFieldMatch = requireFieldMatch.some)

diff --git a/...handlers/src/main/scala/com/sksamuel/elastic4s/handlers/searches/HighlightBuilderFn.scala b/...handlers/src/main/scala/com/sksamuel/elastic4s/handlers/searches/HighlightBuilderFn.scala
@@ -15,19 +15,22 @@ object HighlightBuilderFn {
     highlight.options.boundaryChars.foreach(chars => builder.field("boundary_chars", String.valueOf(chars)))
     highlight.options.boundaryMaxScan.foreach(builder.field("boundary_max_scan", _))
     highlight.options.fragmenter.foreach(builder.field("fragmenter", _))
+    highlight.options.fragmentOffset.foreach(builder.field("fragment_offset", _))
     highlight.options.fragmentSize.foreach(builder.field("fragment_size", _))
     highlight.options.numOfFragments.foreach(builder.field("number_of_fragments", _))
     highlight.options.encoder.foreach(builder.field("encoder", _))
-    highlight.options.forceSource.foreach(builder.field("force_source", _))
     highlight.options.highlighterType.foreach(builder.field("type", _))
     highlight.options.highlightQuery.map(QueryBuilderFn.apply).foreach { highlight =>
       builder.rawField("highlight_query", highlight)
     }
+    if (highlight.options.matchedFields.nonEmpty)
+      builder.array("matched_fields", highlight.options.matchedFields.toArray)
     highlight.options.noMatchSize.foreach(builder.field("no_match_size", _))
     highlight.options.order.foreach(builder.field("order", _))
     highlight.options.phraseLimit.foreach(builder.field("phrase_limit", _))
     highlight.options.maxAnalyzedOffset.foreach(builder.field("max_analyzed_offset", _))
     highlight.options.requireFieldMatch.foreach(builder.field("require_field_match", _))
+    highlight.options.tagsSchema.foreach(builder.field("tags_schema", _))
 
     if (highlight.options.postTags.nonEmpty || highlight.options.preTags.nonEmpty) {
       if (highlight.options.postTags.isEmpty) builder.array("post_tags", Array("</em>"))
@@ -44,6 +47,8 @@ object HighlightBuilderFn {
       builder.endObject()
     }
 
+    highlight.options.options.foreach(options => options.foreach { case (k, v) => builder.autofield(k, v) } )
+
     builder.endObject()
     builder
   }

diff --git a/...ers/src/main/scala/com/sksamuel/elastic4s/handlers/searches/HighlightFieldBuilderFn.scala b/...ers/src/main/scala/com/sksamuel/elastic4s/handlers/searches/HighlightFieldBuilderFn.scala
@@ -12,7 +12,8 @@ object HighlightFieldBuilderFn {
 
     field.boundaryChars.foreach(chars => builder.field("boundary_chars", String.valueOf(chars)))
     field.boundaryMaxScan.foreach(builder.field("boundary_max_scan", _))
-    field.forceSource.foreach(builder.field("force_source", _))
+    field.encoder.foreach(builder.field("encoder", _))
+    field.fragmenter.foreach(builder.field("fragmenter", _))
     field.fragmentOffset.foreach(builder.field("fragment_offset", _))
     field.fragmentSize.foreach(builder.field("fragment_size", _))
     field.highlightQuery.map(QueryBuilderFn.apply).foreach { highlight =>
@@ -26,6 +27,8 @@ object HighlightFieldBuilderFn {
     field.order.foreach(builder.field("order", _))
     field.phraseLimit.foreach(builder.field("phrase_limit", _))
     field.requireFieldMatch.foreach(builder.field("require_field_match", _))
+    field.maxAnalyzedOffset.foreach(builder.field("max_analyzed_offset", _))
+    field.tagsSchema.foreach(builder.field("tags_schema", _))
     field.boundaryScanner.foreach(builder.field("boundary_scanner", _))
     field.boundaryScannerLocale.foreach(builder.field("boundary_scanner_locale", _))
 
@@ -37,6 +40,8 @@ object HighlightFieldBuilderFn {
       else builder.array("pre_tags", field.preTags.toArray)
     }
 
+    field.options.foreach(options => options.foreach{case (k, v) => builder.autofield(k, v)})
+
     builder
   }
 }
diff --git a/elastic4s-tests/src/test/resources/json/search/search_highlighting.json b/elastic4s-tests/src/test/resources/json/search/search_highlighting.json
@@ -4,13 +4,14 @@
         "boundary_max_scan": 4,
         "encoder": "html",
         "order": "score",
+        "max_analyzed_offset": 900000,
+        "tags_schema": "styled",
         "post_tags": [
             "</b>"
         ],
         "pre_tags": [
             "<b>"
         ],
-        "max_analyzed_offset": 900000,
         "fields": {
             "name": {
                 "fragment_offset": 4,