stac-utils · pomadchin · Mar 30, 2022 · Mar 30, 2022 · Mar 30, 2022
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -64,7 +64,7 @@ workflows:
       - openjdk8-build:
           matrix:
             parameters:
-              scala-version: ["2.12.15", "2.13.7"]
+              scala-version: ["2.12.15", "2.13.8"]
           # required since openjdk8-deploy has tag filters AND requires
           # openjdk8
           # https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag
@@ -89,7 +89,7 @@ workflows:
       - openjdk8-deploy:
           matrix:
             parameters:
-              scala-version: ["2.13.7"]
+              scala-version: ["2.13.8"]
           context: sonatype-azavea-signing-key
           requires:
             - openjdk8-build

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [Unreleased]
 ### Changed
 - Pagination Improvements [#496](https://github.com/azavea/stac4s/pull/496)
+- Make SearchFilters pagination agnostic [#502](https://github.com/azavea/stac4s/pull/502) 
 
 ## [0.7.2] - 2021-10-12
 ### Changed

diff --git a/build.sbt b/build.sbt
@@ -220,8 +220,6 @@ lazy val client = crossProject(JSPlatform, JVMPlatform)
   .settings(publishSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "com.github.julien-truffaut"    %%% "monocle-core"  % Versions.Monocle,
-      "com.github.julien-truffaut"    %%% "monocle-macro" % Versions.Monocle,
       "io.circe"                      %%% "circe-core"    % Versions.Circe,
       "io.circe"                      %%% "circe-generic" % Versions.Circe,
       "io.circe"                      %%% "circe-refined" % Versions.Circe,

diff --git a/modules/client/js/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala b/modules/client/js/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala
@@ -1,16 +1,14 @@
 package com.azavea.stac4s.api.client
 
-import com.azavea.stac4s.api.client.SttpStacClientF.PaginationToken
 import com.azavea.stac4s.api.client.util.ClientCodecs
 import com.azavea.stac4s.geometry.Geometry
-import com.azavea.stac4s.{Bbox, TemporalExtent}
+import com.azavea.stac4s.{Bbox, TemporalExtent, productFieldNames}
 
 import cats.syntax.option._
 import eu.timepit.refined.types.numeric.NonNegInt
 import io.circe._
 import io.circe.refined._
-import monocle.Lens
-import monocle.macros.GenLens
+import io.circe.syntax._
 
 case class SearchFilters(
     bbox: Option[Bbox] = None,
@@ -20,22 +18,24 @@ case class SearchFilters(
     items: List[String] = Nil,
     limit: Option[NonNegInt] = None,
     query: Map[String, List[Query]] = Map.empty,
-    next: Option[PaginationToken] = None
+    // according to the STAC Spec, any fields can be used to represent pagination
+    // for more details see https://github.com/radiantearth/stac-api-spec/tree/v1.0.0-rc.1/item-search#pagination
+    paginationBody: JsonObject = JsonObject.empty
 )
 
 object SearchFilters extends ClientCodecs {
-  implicit val paginationTokenLens: Lens[SearchFilters, Option[PaginationToken]] = GenLens[SearchFilters](_.next)
+  val searchFilterFields = productFieldNames[SearchFilters]
 
   implicit val searchFiltersDecoder: Decoder[SearchFilters] = { c =>
     for {
-      bbox              <- c.downField("bbox").as[Option[Bbox]]
-      datetime          <- c.downField("datetime").as[Option[TemporalExtent]]
-      intersects        <- c.downField("intersects").as[Option[Geometry]]
-      collectionsOption <- c.downField("collections").as[Option[List[String]]]
-      itemsOption       <- c.downField("ids").as[Option[List[String]]]
-      limit             <- c.downField("limit").as[Option[NonNegInt]]
+      bbox              <- c.get[Option[Bbox]]("bbox")
+      datetime          <- c.get[Option[TemporalExtent]]("datetime")
+      intersects        <- c.get[Option[Geometry]]("intersects")
+      collectionsOption <- c.get[Option[List[String]]]("collections")
+      itemsOption       <- c.get[Option[List[String]]]("ids")
+      limit             <- c.get[Option[NonNegInt]]("limit")
       query             <- c.get[Option[Map[String, List[Query]]]]("query")
-      paginationToken   <- c.get[Option[PaginationToken]]("next")
+      document          <- c.value.as[JsonObject]
     } yield {
       SearchFilters(
         bbox,
@@ -45,30 +45,32 @@ object SearchFilters extends ClientCodecs {
         itemsOption getOrElse Nil,
         limit,
         query getOrElse Map.empty,
-        paginationToken
+        document.filter { case (k, _) => !searchFilterFields.contains(k) }
       )
     }
   }
 
-  implicit val searchFiltersEncoder: Encoder[SearchFilters] = Encoder.forProduct8(
-    "bbox",
-    "datetime",
-    "intersects",
-    "collections",
-    "ids",
-    "limit",
-    "query",
-    "next"
-  )(filters =>
-    (
-      filters.bbox,
-      filters.datetime,
-      filters.intersects,
-      filters.collections.some.filter(_.nonEmpty),
-      filters.items.some.filter(_.nonEmpty),
-      filters.limit,
-      filters.query.some.filter(_.nonEmpty),
-      filters.next
-    )
-  )
+  implicit val searchFiltersEncoder: Encoder[SearchFilters] = { filters =>
+    val fieldsEncoder = Encoder.forProduct7(
+      "bbox",
+      "datetime",
+      "intersects",
+      "collections",
+      "ids",
+      "limit",
+      "query"
+    ) { filters: SearchFilters =>
+      (
+        filters.bbox,
+        filters.datetime,
+        filters.intersects,
+        filters.collections.some.filter(_.nonEmpty),
+        filters.items.some.filter(_.nonEmpty),
+        filters.limit,
+        filters.query.some.filter(_.nonEmpty)
+      )
+    }
+
+    fieldsEncoder(filters).deepMerge(filters.paginationBody.asJson)
+  }
 }
diff --git a/modules/client/jvm/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala b/modules/client/jvm/src/main/scala/com/azavea/stac4s/api/client/SearchFilters.scala
@@ -1,16 +1,14 @@
 package com.azavea.stac4s.api.client
 
-import com.azavea.stac4s.api.client.SttpStacClientF.PaginationToken
 import com.azavea.stac4s.api.client.util.ClientCodecs
-import com.azavea.stac4s.{Bbox, TemporalExtent}
+import com.azavea.stac4s.{Bbox, TemporalExtent, productFieldNames}
 
 import cats.syntax.option._
 import eu.timepit.refined.types.numeric.NonNegInt
 import geotrellis.vector.{io => _, _}
 import io.circe._
 import io.circe.refined._
-import monocle.Lens
-import monocle.macros.GenLens
+import io.circe.syntax._
 
 case class SearchFilters(
     bbox: Option[Bbox] = None,
@@ -20,22 +18,24 @@ case class SearchFilters(
     items: List[String] = Nil,
     limit: Option[NonNegInt] = None,
     query: Map[String, List[Query]] = Map.empty,
-    next: Option[PaginationToken] = None
+    // according to the STAC Spec, any fields can be used to represent pagination
+    // for more details see https://github.com/radiantearth/stac-api-spec/tree/v1.0.0-rc.1/item-search#pagination
+    paginationBody: JsonObject = JsonObject.empty
 )
 
 object SearchFilters extends ClientCodecs {
-  implicit val paginationTokenLens: Lens[SearchFilters, Option[PaginationToken]] = GenLens[SearchFilters](_.next)
+  val searchFilterFields = productFieldNames[SearchFilters]
 
   implicit val searchFiltersDecoder: Decoder[SearchFilters] = { c =>
     for {
-      bbox              <- c.downField("bbox").as[Option[Bbox]]
-      datetime          <- c.downField("datetime").as[Option[TemporalExtent]]
-      intersects        <- c.downField("intersects").as[Option[Geometry]]
-      collectionsOption <- c.downField("collections").as[Option[List[String]]]
-      itemsOption       <- c.downField("ids").as[Option[List[String]]]
-      limit             <- c.downField("limit").as[Option[NonNegInt]]
+      bbox              <- c.get[Option[Bbox]]("bbox")
+      datetime          <- c.get[Option[TemporalExtent]]("datetime")
+      intersects        <- c.get[Option[Geometry]]("intersects")
+      collectionsOption <- c.get[Option[List[String]]]("collections")
+      itemsOption       <- c.get[Option[List[String]]]("ids")
+      limit             <- c.get[Option[NonNegInt]]("limit")
       query             <- c.get[Option[Map[String, List[Query]]]]("query")
-      paginationToken   <- c.get[Option[PaginationToken]]("next")
+      document          <- c.value.as[JsonObject]
     } yield {
       SearchFilters(
         bbox,
@@ -45,30 +45,32 @@ object SearchFilters extends ClientCodecs {
         itemsOption getOrElse Nil,
         limit,
         query getOrElse Map.empty,
-        paginationToken
+        document.filter { case (k, _) => !searchFilterFields.contains(k) }
       )
     }
   }
 
-  implicit val searchFiltersEncoder: Encoder[SearchFilters] = Encoder.forProduct8(
-    "bbox",
-    "datetime",
-    "intersects",
-    "collections",
-    "ids",
-    "limit",
-    "query",
-    "next"
-  )(filters =>
-    (
-      filters.bbox,
-      filters.datetime,
-      filters.intersects,
-      filters.collections.some.filter(_.nonEmpty),
-      filters.items.some.filter(_.nonEmpty),
-      filters.limit,
-      filters.query.some.filter(_.nonEmpty),
-      filters.next
-    )
-  )
+  implicit val searchFiltersEncoder: Encoder[SearchFilters] = { filters =>
+    val fieldsEncoder = Encoder.forProduct7(
+      "bbox",
+      "datetime",
+      "intersects",
+      "collections",
+      "ids",
+      "limit",
+      "query"
+    ) { filters: SearchFilters =>
+      (
+        filters.bbox,
+        filters.datetime,
+        filters.intersects,
+        filters.collections.some.filter(_.nonEmpty),
+        filters.items.some.filter(_.nonEmpty),
+        filters.limit,
+        filters.query.some.filter(_.nonEmpty)
+      )
+    }
+
+    fieldsEncoder(filters).deepMerge(filters.paginationBody.asJson)
+  }
 }
diff --git a/modules/client/shared/src/main/scala/com/azavea/stac4s/api/client/SttpStacClientF.scala b/modules/client/shared/src/main/scala/com/azavea/stac4s/api/client/SttpStacClientF.scala
@@ -14,7 +14,6 @@ import eu.timepit.refined.types.string.NonEmptyString
 import fs2.Stream
 import io.circe.syntax._
 import io.circe.{Encoder, Json, JsonObject}
-import monocle.Lens
 import sttp.client3.circe.asJson
 import sttp.client3.{Response, SttpBackend, UriContext, basicRequest}
 import sttp.model.{MediaType, Uri}
@@ -31,16 +30,16 @@ case class SttpStacClientF[F[_]: MonadThrow, S: Encoder](
 
   def search(filter: Option[S]): Stream[F, StacItem] = {
     val emptyJson = JsonObject.empty.asJson
-    // the initial filter may contain the paginationToken that is used for the initial query
-    val initialBody = filter.map(_.asJson).getOrElse(emptyJson)
+    // the initial filter may contain the paginationBody that is used for the initial query
+    val initialBody = filter.map(_.asJson.deepDropNullValues).getOrElse(emptyJson)
     Stream
       .unfoldLoopEval((baseUri.addPath("search"), initialBody)) { case (link, request) =>
         client
           .send(
             basicRequest
               .post(link)
               .contentType(MediaType.ApplicationJson)
-              .body(request.deepDropNullValues.noSpaces)
+              .body(request.noSpaces)
               .response(asJson[Json])
           )
           .flatMap { response =>
@@ -147,8 +146,6 @@ case class SttpStacClientF[F[_]: MonadThrow, S: Encoder](
 }
 
 object SttpStacClientF {
-  // TODO: should be a newtype
-  type PaginationToken = NonEmptyString
 
   implicit class ResponseEitherJsonOps[E <: Exception](val self: Response[Either[E, Json]]) extends AnyVal {
 
@@ -170,7 +167,7 @@ object SttpStacClientF {
                 // to make the case described above more generic, we can take the entire body
                 // and pass it forward by merging with the body (SearchFilters in a form of Json)
                 // with the paginationBody
-                // see https://github.com/azavea/stac4s/pull/496 for details
+                // see https://github.com/azavea/stac4s/pull/496 and https://github.com/azavea/stac4s/pull/502 for details
                 val paginationBody: Option[Json] = l.extensionFields("body").map(_.deepDropNullValues)
 
                 uri"${l.href}" -> paginationBody
@@ -196,14 +193,6 @@ object SttpStacClientF {
       self.body.flatMap(_.hcursor.downField("collections").as[List[StacCollection]]).liftTo[F]
   }
 
-  implicit class StacFilterOps[S](val self: Option[S]) extends AnyVal {
-
-    def setPaginationToken(
-        token: Option[PaginationToken]
-    )(implicit l: Lens[S, Option[PaginationToken]], enc: Encoder[S]): Json =
-      self.map(l.set(token)(_).asJson).getOrElse(JsonObject.empty.asJson)
-  }
-
   implicit class JsonOps[S](val self: Json) extends AnyVal {
 
     def setPaginationBody(body: Option[Json]): Json = {