Pagination Improvements (#496)

* Improve Pagination

* Bump deps versions

.scalafmt.conf

@@ -1,5 +1,5 @@
 runner.dialect = scala212
-version = 3.3.1
+version = 3.3.3
 align.preset = more // For pretty alignment.
 maxColumn = 120
 newlines.topLevelStatementBlankLines = [

CHANGELOG.md

@@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Changed
+- Pagination Improvements [#496](https://github.com/azavea/stac4s/pull/496)
 
 ## [0.7.2] - 2021-10-12
 ### Changed

build.sbt

@@ -13,7 +13,7 @@ lazy val commonSettings = Seq(
       git.gitDescribedVersion.value.get
   },
   scalaVersion                 := "2.12.15",
-  crossScalaVersions           := List("2.12.15", "2.13.7"),
+  crossScalaVersions           := List("2.12.15", "2.13.8"),
   Global / cancelable          := true,
   scalafmtOnCompile            := true,
   ThisBuild / scapegoatVersion := Versions.Scapegoat,
@@ -109,7 +109,7 @@ lazy val credentialSettings = Seq(
 
 val jvmGeometryDependencies = Def.setting {
   Seq(
-    "org.locationtech.jts"         % "jts-core"          % Versions.Jts.value,
+    "org.locationtech.jts"         % "jts-core"          % Versions.Jts,
     "org.locationtech.geotrellis" %% "geotrellis-vector" % Versions.GeoTrellis.value
   )
 }
@@ -124,7 +124,7 @@ val coreDependenciesJVM = Def.setting {
 val testingDependenciesJVM = Def.setting {
   Seq(
     "org.locationtech.geotrellis" %% "geotrellis-vector" % Versions.GeoTrellis.value,
-    "org.locationtech.jts"         % "jts-core"          % Versions.Jts.value,
+    "org.locationtech.jts"         % "jts-core"          % Versions.Jts,
     "org.threeten"                 % "threeten-extra"    % Versions.ThreeTenExtra
   )
 }

modules/client/shared/src/main/scala/com/azavea/stac4s/api/client/SttpStacClientF.scala

@@ -1,6 +1,5 @@
 package com.azavea.stac4s.api.client
 
-import com.azavea.stac4s.api.client.SttpStacClientF.PaginationToken
 import com.azavea.stac4s.api.client.util.syntax._
 import com.azavea.stac4s.{StacCollection, StacItem, StacLink, StacLinkType}
 
@@ -13,15 +12,14 @@ import cats.syntax.nested._
 import cats.syntax.option._
 import eu.timepit.refined.types.string.NonEmptyString
 import fs2.Stream
-import io.circe.refined._
 import io.circe.syntax._
 import io.circe.{Encoder, Json, JsonObject}
 import monocle.Lens
 import sttp.client3.circe.asJson
 import sttp.client3.{Response, SttpBackend, UriContext, basicRequest}
 import sttp.model.{MediaType, Uri}
 
-case class SttpStacClientF[F[_]: MonadThrow, S: Lens[*, Option[PaginationToken]]: Encoder](
+case class SttpStacClientF[F[_]: MonadThrow, S: Encoder](
     client: SttpBackend[F, Any],
     baseUri: Uri
 ) extends StreamingStacClientF[F, Stream[F, *], S] {
@@ -47,7 +45,7 @@ case class SttpStacClientF[F[_]: MonadThrow, S: Lens[*, Option[PaginationToken]]
           )
           .flatMap { response =>
             val items = response.stacItems
-            val next  = response.nextPage(filter)
+            val next  = response.nextPage(request)
             (items, next).tupled
           }
       }
@@ -154,8 +152,8 @@ object SttpStacClientF {
 
   implicit class ResponseEitherJsonOps[E <: Exception](val self: Response[Either[E, Json]]) extends AnyVal {
 
-    /** Get the next page Uri from the retrieved Json body and the next PaginationToken. */
-    def nextPage[F[_]: MonadThrow]: F[Option[(Uri, Option[PaginationToken])]] =
+    /** Get the next page Uri from the retrieved Json body and the next pagination body. */
+    def nextPage[F[_]: MonadThrow]: F[Option[(Uri, Option[Json])]] =
       self.body
         .flatMap {
           _.hcursor
@@ -168,23 +166,21 @@ object SttpStacClientF {
                 // Some STAC API implementations (i.e. Franklin)
                 // encode pagination into the next page Uri (put it into the l.href):
                 // in this case, the pagination token is always set to None and only Uri is used for the pagination purposes.
-                val paginationToken: Option[PaginationToken] =
-                  l
-                    .extensionFields("body")
-                    .flatMap(_.asObject)
-                    .flatMap(_("next"))
-                    .flatMap(_.as[PaginationToken].toOption)
-
-                uri"${l.href}" -> paginationToken
+
+                // to make the case described above more generic, we can take the entire body
+                // and pass it forward by merging with the body (SearchFilters in a form of Json)
+                // with the paginationBody
+                // see https://github.com/azavea/stac4s/pull/496 for details
+                val paginationBody: Option[Json] = l.extensionFields("body").map(_.deepDropNullValues)
+
+                uri"${l.href}" -> paginationBody
             }))
         }
         .liftTo[F]
 
     /** Get the next page Uri and the next page Json request body (that has a correctly set next page token). */
-    def nextPage[F[_]: MonadThrow, S](
-        filter: Option[S]
-    )(implicit l: Lens[S, Option[PaginationToken]], enc: Encoder[S]): F[Option[(Uri, Json)]] =
-      nextPage.nested.map { case (uri, token) => (uri, filter.setPaginationToken(token)) }.value
+    def nextPage[F[_]: MonadThrow](filter: Json): F[Option[(Uri, Json)]] =
+      nextPage.nested.map { case (uri, body) => (uri, filter.setPaginationBody(body)) }.value
 
     /** Get the next page Uri and drop the next page token / body. Useful for get requests with no POST pagination
       * support.
@@ -207,4 +203,26 @@ object SttpStacClientF {
     )(implicit l: Lens[S, Option[PaginationToken]], enc: Encoder[S]): Json =
       self.map(l.set(token)(_).asJson).getOrElse(JsonObject.empty.asJson)
   }
+
+  implicit class JsonOps[S](val self: Json) extends AnyVal {
+
+    def setPaginationBody(body: Option[Json]): Json = {
+      val selfNotNull = self.deepDropNullValues
+      val bodyNotNull = body.map(_.deepDropNullValues).getOrElse(JsonObject.empty.asJson)
+
+      val filter = selfNotNull.deepMerge(bodyNotNull)
+
+      // bbox and intersection can't be present at the same time
+      if (filter.hcursor.downField("bbox").succeeded && filter.hcursor.downField("intersects").succeeded) {
+        // let's see which field is present in the nextPageBody
+        val field =
+          if (bodyNotNull.hcursor.downField("bbox").succeeded && bodyNotNull.hcursor.downField("intersects").failed)
+            filter.hcursor.downField("intersects")
+          else
+            filter.hcursor.downField("bbox")
+
+        field.delete.top.getOrElse(filter)
+      } else filter
+    }
+  }
 }

project/Versions.scala

@@ -16,15 +16,15 @@ object Versions {
   val CirceJsonSchema         = "0.2.0"
   val DisciplineScalatest     = "2.1.5"
   val Enumeratum              = "1.7.0"
-  val GeoTrellis              = Def.setting(ver("3.6.0", "3.6.1-SNAPSHOT").value)
-  val Jts                     = Def.setting(ver("1.16.1", "1.17.0").value)
+  val GeoTrellis              = Def.setting(ver("3.6.1", "3.6.1+2-9cc96096-SNAPSHOT").value)
+  val Jts                     = "1.18.1"
   val Monocle                 = "2.1.0"
   val Refined                 = "0.9.28"
   val ScalacheckCats          = "0.3.1"
   val Scalacheck              = "1.15.4"
   val ScalatestPlusScalacheck = "3.2.2.0"
-  val Scalatest               = "3.2.10"
-  val Scapegoat               = "1.4.11"
+  val Scalatest               = "3.2.11"
+  val Scapegoat               = "1.4.12"
   val Shapeless               = "2.3.7"
   val Sttp                    = "3.3.18"
   val SttpModel               = "1.4.18"

project/build.properties

@@ -1 +1 @@
-sbt.version=1.6.1
+sbt.version=1.6.2

project/plugins.sbt

@@ -1,13 +1,13 @@
-addSbtPlugin("org.xerial.sbt"            % "sbt-sonatype"              % "3.9.10")
+addSbtPlugin("org.xerial.sbt"            % "sbt-sonatype"              % "3.9.12")
 addSbtPlugin("com.github.cb372"          % "sbt-explicit-dependencies" % "0.2.16")
 addSbtPlugin("io.crashbox"               % "sbt-gpg"                   % "0.2.1")
-addSbtPlugin("com.typesafe.sbt"          % "sbt-git"                   % "1.0.2")
+addSbtPlugin("com.github.sbt"            % "sbt-git"                   % "2.0.0")
 addSbtPlugin("io.spray"                  % "sbt-revolver"              % "0.9.1")
 addSbtPlugin("org.scalameta"             % "sbt-scalafmt"              % "2.4.6")
 addSbtPlugin("com.sksamuel.scapegoat"   %% "sbt-scapegoat"             % "1.1.1")
-addSbtPlugin("ch.epfl.scala"             % "sbt-scalafix"              % "0.9.33")
-addSbtPlugin("com.eed3si9n"              % "sbt-assembly"              % "1.1.0")
-addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat"              % "0.1.20")
+addSbtPlugin("ch.epfl.scala"             % "sbt-scalafix"              % "0.9.34")
+addSbtPlugin("com.eed3si9n"              % "sbt-assembly"              % "1.2.0")
+addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat"              % "0.1.22")
 addSbtPlugin("org.jmotor.sbt"            % "sbt-dependency-updates"    % "1.2.2")
-addSbtPlugin("org.scala-js"              % "sbt-scalajs"               % "1.8.0")
-addSbtPlugin("org.portable-scala"        % "sbt-scalajs-crossproject"  % "1.1.0")
+addSbtPlugin("org.scala-js"              % "sbt-scalajs"               % "1.9.0")
+addSbtPlugin("org.portable-scala"        % "sbt-scalajs-crossproject"  % "1.2.0")

sbt

@@ -34,11 +34,11 @@
 
 set -o pipefail
 
-declare -r sbt_release_version="1.5.2"
-declare -r sbt_unreleased_version="1.5.2"
+declare -r sbt_release_version="1.6.2"
+declare -r sbt_unreleased_version="1.6.2"
 
-declare -r latest_213="2.13.5"
-declare -r latest_212="2.12.13"
+declare -r latest_213="2.13.8"
+declare -r latest_212="2.12.15"
 declare -r latest_211="2.11.12"
 declare -r latest_210="2.10.7"
 declare -r latest_29="2.9.3"
@@ -216,7 +216,8 @@ getJavaVersion() {
   # but on 9 and 10 it's 9.x.y and 10.x.y.
   if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then
     echo "${BASH_REMATCH[1]}"
-  elif [[ "$str" =~ ^([0-9]+)(\..*)?$ ]]; then
+  # Fixes https://github.com/dwijnand/sbt-extras/issues/326
+  elif [[ "$str" =~ ^([0-9]+)(\..*)?(-ea)?$ ]]; then
     echo "${BASH_REMATCH[1]}"
   elif [[ -n "$str" ]]; then
     echoerr "Can't parse java version from: $str"
@@ -252,7 +253,9 @@ is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]
 # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+
 default_jvm_opts() {
   local -r v="$(java_version)"
-  if [[ $v -ge 10 ]]; then
+  if [[ $v -ge 17 ]]; then
+    echo "$default_jvm_opts_common"
+  elif [[ $v -ge 10 ]]; then
     if is_apple_silicon; then
       # As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI
       echo "$default_jvm_opts_common"
@@ -385,10 +388,12 @@ usage() {
   set_sbt_version
   cat <<EOM
 Usage: $script_name [options]
+
 Note that options which are passed along to sbt begin with -- whereas
 options to this runner use a single dash. Any sbt command can be scheduled
 to run first by prefixing the command with --, so --warn, --error and so on
 are not special.
+
   -h | -help         print this message
   -v                 verbose operation (this runner is chattier)
   -d, -w, -q         aliases for --debug, --warn, --error (q means quiet)
@@ -406,13 +411,15 @@ are not special.
   -batch             Disable interactive mode
   -prompt <expr>     Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
   -script <file>     Run the specified file as a scala script
+
   # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
   -sbt-version <version>  use the specified version of sbt (default: $sbt_release_version)
   -sbt-force-latest       force the use of the latest release of sbt: $sbt_release_version
   -sbt-dev                use the latest pre-release version of sbt: $sbt_unreleased_version
   -sbt-jar      <path>    use the specified jar as the sbt launcher
   -sbt-launch-dir <path>  directory to hold sbt launchers (default: $sbt_launch_dir)
   -sbt-launch-repo <url>  repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version"))
+
   # scala version (default: as chosen by sbt)
   -28                        use $latest_28
   -29                        use $latest_29
@@ -423,8 +430,10 @@ are not special.
   -scala-home <path>         use the scala build at the specified directory
   -scala-version <version>   use the specified version of scala
   -binary-version <version>  use the specified scala version when searching for dependencies
+
   # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
   -java-home <path>          alternate JAVA_HOME
+
   # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
   # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
   <default>         $(default_jvm_opts)
@@ -434,12 +443,14 @@ are not special.
   -jvm-opts <path>  file containing jvm args (if not given, .jvmopts in project root is used if present)
   -Dkey=val         pass -Dkey=val directly to the jvm
   -J-X              pass option -X directly to the jvm (-J is stripped)
+
   # passing options to sbt, OR to this runner
   SBT_OPTS          environment variable holding either the sbt args directly, or
                     the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
                     Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
   -sbt-opts <path>  file containing sbt args (if not given, .sbtopts in project root is used if present)
   -S-X              add -X to sbt's scalacOptions (-S is stripped)
+
   # passing options exclusively to this runner
   SBTX_OPTS         environment variable holding either the sbt-extras args directly, or
                     the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts')
@@ -592,6 +603,7 @@ fi
 $(pwd) doesn't appear to be an sbt project.
 If you want to start sbt anyway, run:
   $0 -sbt-create
+
 EOM
   exit 1
 }

scripts/test

@@ -18,20 +18,32 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
         usage
     else
         echo "Linting Scala source code and executing tests"
-        ./sbt "++$SCALA_VERSION" ";\
-            scalafix --check; \
-            scalafmtCheck; \
-            scalafmtSbtCheck; \
-            scapegoat; \
-            coreJVM/undeclaredCompileDependenciesTest; \
-            testingJVM/undeclaredCompileDependenciesTest; \
-            coreTestJVM/undeclaredCompileDependenciesTest; \
-            clientJVM/undeclaredCompileDependenciesTest; \
-            coreJVM/unusedCompileDependenciesTest; \
-            testingJVM/unusedCompileDependenciesTest; \
-            coreTestJVM/unusedCompileDependenciesTest; \
-            clientJVM/unusedCompileDependenciesTest; \
-            test
-        "
+
+        # due to a specific deps in the class path, we verify declared deps only for the 2.12 builds
+        if [[ "$SCALA_VERSION" =~ ^2.12.* ]]; then
+          ./sbt "++$SCALA_VERSION" ";\
+              scalafix --check; \
+              scalafmtCheck; \
+              scalafmtSbtCheck; \
+              scapegoat; \
+              coreJVM/undeclaredCompileDependenciesTest; \
+              testingJVM/undeclaredCompileDependenciesTest; \
+              coreTestJVM/undeclaredCompileDependenciesTest; \
+              clientJVM/undeclaredCompileDependenciesTest; \
+              coreJVM/unusedCompileDependenciesTest; \
+              testingJVM/unusedCompileDependenciesTest; \
+              coreTestJVM/unusedCompileDependenciesTest; \
+              clientJVM/unusedCompileDependenciesTest; \
+              test
+          "
+        else
+          ./sbt "++$SCALA_VERSION" ";\
+              scalafix --check; \
+              scalafmtCheck; \
+              scalafmtSbtCheck; \
+              scapegoat; \
+              test
+          "
+        fi
     fi
 fi