From f08258f46677352a6b0fc8e0d3eea35c25b0eb1e Mon Sep 17 00:00:00 2001 From: Arnold Lacko Date: Wed, 1 Mar 2023 09:06:45 +0100 Subject: [PATCH 01/14] POC: refactor removing Map and phantom type parameter ERT --- .../example/RepositoriesElasticsearch.scala | 6 +- .../zio/elasticsearch/ElasticExecutor.scala | 4 +- .../zio/elasticsearch/ElasticRequest.scala | 225 +++++++++--------- .../zio/elasticsearch/ElasticResult.scala | 22 ++ .../zio/elasticsearch/Elasticsearch.scala | 6 +- .../elasticsearch/HttpElasticExecutor.scala | 13 +- .../scala/zio/elasticsearch/Refresh.scala | 77 ------ .../scala/zio/elasticsearch/Routing.scala | 84 ------- .../zio/elasticsearch/TestExecutor.scala | 155 ++++++++++++ .../scala/zio/elasticsearch/package.scala | 11 + .../HttpElasticExecutorSpec.scala | 6 +- 11 files changed, 318 insertions(+), 291 deletions(-) create mode 100644 modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala delete mode 100644 modules/library/src/main/scala/zio/elasticsearch/Refresh.scala create mode 100644 modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala diff --git a/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala b/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala index 8a84df0d1..bc44a98bf 100644 --- a/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala +++ b/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala @@ -32,12 +32,12 @@ import zio.prelude.Newtype.unsafeWrap final case class RepositoriesElasticsearch(elasticsearch: Elasticsearch) { def findAll(): Task[List[GitHubRepo]] = - elasticsearch.execute(ElasticRequest.search[GitHubRepo](Index, matchAll)) + elasticsearch.execute(ElasticRequest.search(Index, matchAll)).result[GitHubRepo] def findById(organization: String, id: String): Task[Option[GitHubRepo]] = for { routing <- routingOf(organization) - res <- elasticsearch.execute(ElasticRequest.getById[GitHubRepo](Index, DocumentId(id)).routing(routing)) + res <- elasticsearch.execute(ElasticRequest.getById(Index, DocumentId(id)).routing(routing)).result[GitHubRepo] } yield res def create(repository: GitHubRepo): Task[CreationOutcome] = @@ -75,7 +75,7 @@ final case class RepositoriesElasticsearch(elasticsearch: Elasticsearch) { } yield res def search(query: ElasticQuery[_]): Task[List[GitHubRepo]] = - elasticsearch.execute(ElasticRequest.search[GitHubRepo](Index, query)) + elasticsearch.execute(ElasticRequest.search(Index, query)).result[GitHubRepo] private def routingOf(value: String): IO[IllegalArgumentException, Routing.Type] = Routing.make(value).toZIO.mapError(e => new IllegalArgumentException(e)) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index 067e4f47f..aad57629d 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -20,7 +20,7 @@ import sttp.client3.SttpBackend import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { - def execute[A](request: ElasticRequest[A, _]): Task[A] + def execute[A](request: ElasticRequest[A]): Task[A] } object ElasticExecutor { @@ -30,6 +30,6 @@ object ElasticExecutor { lazy val local: URLayer[SttpBackend[Task, Any], ElasticExecutor] = ZLayer.succeed(ElasticConfig.Default) >>> live - private[elasticsearch] def execute[A](request: ElasticRequest[A, _]): RIO[ElasticExecutor, A] = + private[elasticsearch] def execute[A](request: ElasticRequest[A]): RIO[ElasticExecutor, A] = ZIO.serviceWithZIO[ElasticExecutor](_.execute(request)) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index 4ece49d9f..88e43c6ff 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -16,43 +16,34 @@ package zio.elasticsearch -import zio.elasticsearch.Refresh.WithRefresh -import zio.elasticsearch.Routing.{Routing, WithRouting} -import zio.prelude._ +import zio.elasticsearch.Routing.Routing import zio.schema.Schema -import zio.schema.codec.JsonCodec.JsonDecoder -import scala.annotation.unused -import scala.language.implicitConversions +trait HasRefresh[A] { // TODO extend this on all appropritate requests + def refresh(value: Boolean): ElasticRequest[A] -sealed trait ElasticRequest[+A, ERT <: ElasticRequestType] { self => + def refreshFalse: ElasticRequest[A] - final def map[B](f: A => Either[DecodingException, B]): ElasticRequest[B, ERT] = ElasticRequest.Map(self, f) + def refreshTrue: ElasticRequest[A] +} - final def refresh(value: Boolean)(implicit wr: WithRefresh[ERT]): ElasticRequest[A, ERT] = - wr.withRefresh(request = self, value = value) +trait HasRouting[A] { // TODO extend this on all appropritate requests + def routing(value: Routing): ElasticRequest[A] +} - final def refreshFalse(implicit wr: WithRefresh[ERT]): ElasticRequest[A, ERT] = - wr.withRefresh(request = self, value = false) +sealed trait ElasticRequest[A] - final def refreshTrue(implicit wr: WithRefresh[ERT]): ElasticRequest[A, ERT] = - wr.withRefresh(request = self, value = true) - - final def routing(value: Routing)(implicit wr: WithRouting[ERT]): ElasticRequest[A, ERT] = - wr.withRouting(request = self, routing = value) -} +sealed trait BulkableRequest[A] extends ElasticRequest[A] object ElasticRequest { - import ElasticRequestType._ - - def bulk(requests: BulkableRequest*): ElasticRequest[Unit, Bulk] = + def bulk(requests: BulkableRequest[_]*): BulkRequest = BulkRequest.of(requests: _*) - def create[A: Schema](index: IndexName, doc: A): ElasticRequest[DocumentId, Create] = + def create[A: Schema](index: IndexName, doc: A): CreateRequest = CreateRequest(index = index, document = Document.from(doc), refresh = false, routing = None) - def create[A: Schema](index: IndexName, id: DocumentId, doc: A): ElasticRequest[CreationOutcome, CreateWithId] = + def create[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateWithIdRequest = CreateWithIdRequest(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) def createIndex(name: IndexName): ElasticRequest[CreationOutcome, CreateIndex] = @@ -61,69 +52,47 @@ object ElasticRequest { def createIndex(name: IndexName, definition: String): ElasticRequest[CreationOutcome, CreateIndex] = CreateIndexRequest(name = name, definition = Some(definition)) - def deleteById(index: IndexName, id: DocumentId): ElasticRequest[DeletionOutcome, DeleteById] = + def deleteById(index: IndexName, id: DocumentId): DeleteByIdRequest = DeleteByIdRequest(index = index, id = id, refresh = false, routing = None) - def deleteByQuery(index: IndexName, query: ElasticQuery[_]): ElasticRequest[DeletionOutcome, DeleteByQuery] = + def deleteByQuery(index: IndexName, query: ElasticQuery[_]): DeleteByQueryRequest = DeleteByQueryRequest(index = index, query = query, refresh = false, routing = None) - def deleteIndex(name: IndexName): ElasticRequest[DeletionOutcome, DeleteIndex] = + def deleteIndex(name: IndexName): DeleteIndexRequest = DeleteIndexRequest(name) - def exists(index: IndexName, id: DocumentId): ElasticRequest[Boolean, Exists] = + def exists(index: IndexName, id: DocumentId): ExistsRequest = ExistsRequest(index = index, id = id, routing = None) - def getById[A: Schema](index: IndexName, id: DocumentId): ElasticRequest[Option[A], GetById] = - GetByIdRequest(index = index, id = id, routing = None).map { - case Some(document) => - document.decode match { - case Left(e) => Left(DecodingException(s"Could not parse the document: ${e.message}")) - case Right(doc) => Right(Some(doc)) - } - case None => - Right(None) - } - - def search[A](index: IndexName, query: ElasticQuery[_])(implicit - schema: Schema[A] - ): ElasticRequest[List[A], GetByQuery] = - GetByQueryRequest(index = index, query = query, routing = None).map { response => - Validation - .validateAll(response.results.map { json => - ZValidation.fromEither(JsonDecoder.decode(schema, json.toString)) - }) - .toEitherWith { errors => - DecodingException(s"Could not parse all documents successfully: ${errors.map(_.message).mkString(",")})") - } - } + def getById(index: IndexName, id: DocumentId): GetByIdRequest = + GetByIdRequest(index = index, id = id, routing = None) + + def search(index: IndexName, query: ElasticQuery[_]): GetByQueryRequest = + GetByQueryRequest(index = index, query = query, routing = None) def upsert[A: Schema](index: IndexName, id: DocumentId, doc: A): ElasticRequest[Unit, Upsert] = CreateOrUpdateRequest(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) - private[elasticsearch] final case class BulkableRequest private (request: ElasticRequest[_, _]) - - object BulkableRequest { - implicit def toBulkable[ERT <: ElasticRequestType](request: ElasticRequest[_, ERT])(implicit - @unused ev: ERT <:< BulkableRequestType - ): BulkableRequest = - BulkableRequest(request) - - implicit def toBulkableList[ERT <: ElasticRequestType](requests: List[ElasticRequest[_, ERT]])(implicit - @unused ev: ERT <:< BulkableRequestType - ): List[BulkableRequest] = - requests.map(BulkableRequest(_)) - } - - private[elasticsearch] final case class BulkRequest( - requests: List[BulkableRequest], + final case class BulkRequest( + requests: List[BulkableRequest[_]], index: Option[IndexName], refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[Unit, Bulk] { + ) extends ElasticRequest[Unit] + with HasRouting[Unit] + with HasRefresh[Unit] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + + override def refresh(value: Boolean) = self.copy(refresh = value) + + override def refreshFalse = refresh(false) + + override def refreshTrue = refresh(true) + lazy val body: String = requests.flatMap { r => // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen // because these are only Bulkable Requests and other matches will not occur. - (r.request: @unchecked) match { + r match { case CreateRequest(index, document, _, maybeRouting) => List(getActionAndMeta("create", List(("_index", Some(index)), ("routing", maybeRouting))), document.json) case CreateWithIdRequest(index, id, document, _, maybeRouting) => @@ -143,77 +112,127 @@ object ElasticRequest { } object BulkRequest { - def of(requests: BulkableRequest*): BulkRequest = + def of(requests: BulkableRequest[_]*): BulkRequest = BulkRequest(requests = requests.toList, index = None, refresh = false, routing = None) } - private[elasticsearch] final case class CreateRequest( + final case class CreateRequest( index: IndexName, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[DocumentId, Create] + ) extends BulkableRequest[DocumentId] + with HasRouting[DocumentId] + with HasRefresh[DocumentId] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + + override def refresh(value: Boolean) = self.copy(refresh = value) - private[elasticsearch] final case class CreateWithIdRequest( + override def refreshFalse = refresh(false) + + override def refreshTrue = refresh(true) + } + + final case class CreateWithIdRequest( index: IndexName, id: DocumentId, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[CreationOutcome, CreateWithId] + ) extends BulkableRequest[CreationOutcome] + with HasRouting[CreationOutcome] + with HasRefresh[CreationOutcome] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + + override def refresh(value: Boolean) = self.copy(refresh = value) + + override def refreshFalse = refresh(false) - private[elasticsearch] final case class CreateIndexRequest( + override def refreshTrue = refresh(true) + } + + final case class CreateIndexRequest( name: IndexName, definition: Option[String] - ) extends ElasticRequest[CreationOutcome, CreateIndex] + ) extends ElasticRequest[CreationOutcome] - private[elasticsearch] final case class CreateOrUpdateRequest( + final case class CreateOrUpdateRequest( index: IndexName, id: DocumentId, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[Unit, Upsert] + ) extends BulkableRequest[Unit] + with HasRouting[Unit] + with HasRefresh[Unit] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + + override def refresh(value: Boolean) = self.copy(refresh = value) + + override def refreshFalse = refresh(false) + + override def refreshTrue = refresh(true) + } - private[elasticsearch] final case class DeleteByIdRequest( + final case class DeleteByIdRequest( index: IndexName, id: DocumentId, refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[DeletionOutcome, DeleteById] + ) extends BulkableRequest[DeletionOutcome] + with HasRouting[DeletionOutcome] + with HasRefresh[DeletionOutcome] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + + override def refresh(value: Boolean) = self.copy(refresh = value) + + override def refreshFalse = refresh(false) + + override def refreshTrue = refresh(true) + } - private[elasticsearch] final case class DeleteByQueryRequest( + final case class DeleteByQueryRequest( index: IndexName, query: ElasticQuery[_], refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[DeletionOutcome, DeleteByQuery] + ) extends ElasticRequest[DeletionOutcome] + with HasRouting[DeletionOutcome] + with HasRefresh[DeletionOutcome] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) - private[elasticsearch] final case class DeleteIndexRequest(name: IndexName) - extends ElasticRequest[DeletionOutcome, DeleteIndex] + override def refresh(value: Boolean) = self.copy(refresh = value) - private[elasticsearch] final case class ExistsRequest( + override def refreshFalse = refresh(false) + + override def refreshTrue = refresh(true) + } + + final case class DeleteIndexRequest(name: IndexName) extends ElasticRequest[DeletionOutcome] + + final case class ExistsRequest( index: IndexName, id: DocumentId, routing: Option[Routing] - ) extends ElasticRequest[Boolean, Exists] + ) extends ElasticRequest[Boolean] + with HasRouting[Boolean] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + } - private[elasticsearch] final case class GetByIdRequest( + final case class GetByIdRequest( index: IndexName, id: DocumentId, routing: Option[Routing] - ) extends ElasticRequest[Option[Document], GetById] + ) extends ElasticRequest[GetResult] + with HasRouting[GetResult] { self => + override def routing(value: Routing) = self.copy(routing = Some(value)) + } - private[elasticsearch] final case class GetByQueryRequest( + final case class GetByQueryRequest( index: IndexName, query: ElasticQuery[_], routing: Option[Routing] - ) extends ElasticRequest[ElasticQueryResponse, GetByQuery] - - private[elasticsearch] final case class Map[A, B, ERT <: ElasticRequestType]( - request: ElasticRequest[A, ERT], - mapper: A => Either[DecodingException, B] - ) extends ElasticRequest[B, ERT] + ) extends ElasticRequest[SearchResult] private def getActionAndMeta(requestType: String, parameters: List[(String, Any)]): String = parameters.collect { case (name, Some(value)) => s""""$name" : "${value.toString}"""" } @@ -221,24 +240,6 @@ object ElasticRequest { } -sealed trait ElasticRequestType - -sealed trait BulkableRequestType extends ElasticRequestType - -object ElasticRequestType { - sealed trait Bulk extends ElasticRequestType - sealed trait CreateIndex extends ElasticRequestType - sealed trait Create extends BulkableRequestType - sealed trait CreateWithId extends BulkableRequestType - sealed trait DeleteById extends BulkableRequestType - sealed trait DeleteByQuery extends ElasticRequestType - sealed trait DeleteIndex extends ElasticRequestType - sealed trait Exists extends ElasticRequestType - sealed trait GetById extends ElasticRequestType - sealed trait GetByQuery extends ElasticRequestType - sealed trait Upsert extends BulkableRequestType -} - sealed abstract class CreationOutcome case object Created extends CreationOutcome diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala new file mode 100644 index 000000000..fdd67007f --- /dev/null +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -0,0 +1,22 @@ +package zio.elasticsearch + +import zio.{Task, ZIO} +import zio.schema.Schema +import zio.prelude.{ForEachOps, ZValidation} + +sealed trait ElasticResult[F[_]] { + def result[A: Schema]: Task[F[A]] +} + +final class GetResult(private val doc: Option[Document]) extends ElasticResult[Option] { + override def result[A: Schema]: Task[Option[A]] = + ZIO.fromEither(doc.forEach(_.decode)).mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) +} +final class SearchResult(private val hits: List[Document]) extends ElasticResult[List] { + override def result[A: Schema]: Task[List[A]] = + ZIO.fromEither { + ZValidation.validateAll(hits.map(d => ZValidation.fromEither(d.decode))).toEitherWith { errors => + DecodingException(s"Could not parse all documents successfully: ${errors.map(_.message).mkString(",")})") + } + } +} diff --git a/modules/library/src/main/scala/zio/elasticsearch/Elasticsearch.scala b/modules/library/src/main/scala/zio/elasticsearch/Elasticsearch.scala index a06ab56d9..8ad6a1338 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/Elasticsearch.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/Elasticsearch.scala @@ -19,17 +19,17 @@ package zio.elasticsearch import zio.{RIO, Task, URLayer, ZIO, ZLayer} trait Elasticsearch { - def execute[A](request: ElasticRequest[A, _]): Task[A] + def execute[A](request: ElasticRequest[A]): Task[A] } object Elasticsearch { - def execute[A](request: ElasticRequest[A, _]): RIO[Elasticsearch, A] = + def execute[A](request: ElasticRequest[A]): RIO[Elasticsearch, A] = ZIO.serviceWithZIO[Elasticsearch](_.execute(request)) lazy val layer: URLayer[ElasticExecutor, Elasticsearch] = ZLayer.fromFunction { executor: ElasticExecutor => new Elasticsearch { - def execute[A](request: ElasticRequest[A, _]): Task[A] = executor.execute(request) + def execute[A](request: ElasticRequest[A]): Task[A] = executor.execute(request) } } } diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index e8504ca87..c437581e1 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -37,7 +37,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC import HttpElasticExecutor._ - def execute[A](request: ElasticRequest[A, _]): Task[A] = + def execute[A](request: ElasticRequest[A]): Task[A] = request match { case r: BulkRequest => executeBulk(r) case r: CreateRequest => executeCreate(r) @@ -50,7 +50,6 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: ExistsRequest => executeExists(r) case r: GetByIdRequest => executeGetById(r) case r: GetByQueryRequest => executeGetByQuery(r) - case map @ Map(_, _) => execute(map.request).flatMap(a => ZIO.fromEither(map.mapper(a))) } private def executeBulk(r: BulkRequest): Task[Unit] = { @@ -193,7 +192,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetById(r: GetByIdRequest): Task[Option[Document]] = { + private def executeGetById(r: GetByIdRequest): Task[GetResult] = { val uri = uri"${config.uri}/${r.index}/$Doc/${r.id}".withParams(getQueryParams(List(("routing", r.routing)))) sendRequestWithCustomResponse[ElasticGetResponse]( @@ -202,14 +201,14 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC .response(asJson[ElasticGetResponse]) ).flatMap { response => response.code match { - case HttpOk => ZIO.attempt(response.body.toOption.map(d => Document.from(d.source))) - case HttpNotFound => ZIO.succeed(None) + case HttpOk => ZIO.attempt(new GetResult(response.body.toOption.map(d => Document.from(d.source)))) + case HttpNotFound => ZIO.succeed(new GetResult(None)) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) } } } - private def executeGetByQuery(r: GetByQueryRequest): Task[ElasticQueryResponse] = + private def executeGetByQuery(r: GetByQueryRequest): Task[SearchResult] = sendRequestWithCustomResponse( request .post(uri"${config.uri}/${r.index}/$Search") @@ -221,7 +220,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case HttpOk => response.body.fold( e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed(value) + value => ZIO.succeed(new SearchResult(value.results.map(_.toString).map(Document(_)))) // TODO have Json in Document instead of String??? ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) diff --git a/modules/library/src/main/scala/zio/elasticsearch/Refresh.scala b/modules/library/src/main/scala/zio/elasticsearch/Refresh.scala deleted file mode 100644 index 095947511..000000000 --- a/modules/library/src/main/scala/zio/elasticsearch/Refresh.scala +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2022 LambdaWorks - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package zio.elasticsearch - -import zio.elasticsearch.ElasticRequest._ -import zio.elasticsearch.ElasticRequestType._ - -object Refresh { - - trait WithRefresh[ERT <: ElasticRequestType] { - def withRefresh[A](request: ElasticRequest[A, ERT], value: Boolean): ElasticRequest[A, ERT] - } - - object WithRefresh { - implicit val bulkWithRefresh: WithRefresh[Bulk] = new WithRefresh[Bulk] { - def withRefresh[A](request: ElasticRequest[A, Bulk], value: Boolean): ElasticRequest[A, Bulk] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: BulkRequest => r.copy(refresh = value) - } - } - - implicit val createWithRefresh: WithRefresh[Create] = new WithRefresh[Create] { - def withRefresh[A](request: ElasticRequest[A, Create], value: Boolean): ElasticRequest[A, Create] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: CreateRequest => r.copy(refresh = value) - } - } - - implicit val createWithIdWithRefresh: WithRefresh[CreateWithId] = new WithRefresh[CreateWithId] { - def withRefresh[A](request: ElasticRequest[A, CreateWithId], value: Boolean): ElasticRequest[A, CreateWithId] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: CreateWithIdRequest => r.copy(refresh = value) - } - } - - implicit val deleteByIdWithRefresh: WithRefresh[DeleteById] = new WithRefresh[DeleteById] { - def withRefresh[A](request: ElasticRequest[A, DeleteById], value: Boolean): ElasticRequest[A, DeleteById] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: DeleteByIdRequest => r.copy(refresh = value) - } - } - - implicit val deleteByQueryWithRefresh: WithRefresh[DeleteByQuery] = new WithRefresh[DeleteByQuery] { - def withRefresh[A](request: ElasticRequest[A, DeleteByQuery], value: Boolean): ElasticRequest[A, DeleteByQuery] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: DeleteByQueryRequest => r.copy(refresh = value) - } - } - - implicit val upsertWithRefresh: WithRefresh[Upsert] = new WithRefresh[Upsert] { - def withRefresh[A](request: ElasticRequest[A, Upsert], value: Boolean): ElasticRequest[A, Upsert] = - request match { - case Map(r, mapper) => Map(withRefresh(r, value), mapper) - case r: CreateOrUpdateRequest => r.copy(refresh = value) - } - } - } -} diff --git a/modules/library/src/main/scala/zio/elasticsearch/Routing.scala b/modules/library/src/main/scala/zio/elasticsearch/Routing.scala index 28cc7cd9b..6f82fc2d8 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/Routing.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/Routing.scala @@ -16,17 +16,6 @@ package zio.elasticsearch -import zio.elasticsearch.ElasticRequest._ -import zio.elasticsearch.ElasticRequestType.{ - Bulk, - Create, - CreateWithId, - DeleteById, - DeleteByQuery, - Exists, - GetById, - Upsert -} import zio.prelude.Assertion.isEmptyString import zio.prelude.Newtype @@ -34,77 +23,4 @@ object Routing extends Newtype[String] { override def assertion = assert(!isEmptyString) // scalafix:ok type Routing = Routing.Type - - trait WithRouting[ERT <: ElasticRequestType] { - def withRouting[A](request: ElasticRequest[A, ERT], routing: Routing): ElasticRequest[A, ERT] - } - - object WithRouting { - implicit val bulkWithRouting: WithRouting[Bulk] = new WithRouting[Bulk] { - def withRouting[A](request: ElasticRequest[A, Bulk], routing: Routing): ElasticRequest[A, Bulk] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: BulkRequest => r.copy(routing = Some(routing)) - } - } - - implicit val createWithRouting: WithRouting[Create] = new WithRouting[Create] { - def withRouting[A](request: ElasticRequest[A, Create], routing: Routing): ElasticRequest[A, Create] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: CreateRequest => r.copy(routing = Some(routing)) - } - } - - implicit val createWithIdWithRouting: WithRouting[CreateWithId] = new WithRouting[CreateWithId] { - def withRouting[A](request: ElasticRequest[A, CreateWithId], routing: Routing): ElasticRequest[A, CreateWithId] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: CreateWithIdRequest => r.copy(routing = Some(routing)) - } - } - - implicit val deleteByIdWithRouting: WithRouting[DeleteById] = new WithRouting[DeleteById] { - def withRouting[A](request: ElasticRequest[A, DeleteById], routing: Routing): ElasticRequest[A, DeleteById] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: DeleteByIdRequest => r.copy(routing = Some(routing)) - } - } - - implicit val deleteByQueryWithRouting: WithRouting[DeleteByQuery] = new WithRouting[DeleteByQuery] { - def withRouting[A]( - request: ElasticRequest[A, DeleteByQuery], - routing: Routing - ): ElasticRequest[A, DeleteByQuery] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: DeleteByQueryRequest => r.copy(routing = Some(routing)) - } - } - - implicit val existsWithRouting: WithRouting[Exists] = new WithRouting[Exists] { - def withRouting[A](request: ElasticRequest[A, Exists], routing: Routing): ElasticRequest[A, Exists] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: ExistsRequest => r.copy(routing = Some(routing)) - } - } - - implicit val getByIdWithRouting: WithRouting[GetById] = new WithRouting[GetById] { - def withRouting[A](request: ElasticRequest[A, GetById], routing: Routing): ElasticRequest[A, GetById] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: GetByIdRequest => r.copy(routing = Some(routing)) - } - } - - implicit val upsertWithRouting: WithRouting[Upsert] = new WithRouting[Upsert] { - def withRouting[A](request: ElasticRequest[A, Upsert], routing: Routing): ElasticRequest[A, Upsert] = - request match { - case Map(r, mapper) => Map(withRouting(r, routing), mapper) - case r: CreateOrUpdateRequest => r.copy(routing = Some(routing)) - } - } - } } diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala new file mode 100644 index 000000000..9c1497c71 --- /dev/null +++ b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala @@ -0,0 +1,155 @@ +/* + * Copyright 2022 LambdaWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package zio.elasticsearch + +import zio.Random.nextUUID +import zio.elasticsearch.ElasticRequest._ +import zio.json.ast.Json +import zio.stm.{STM, TMap, ZSTM} +import zio.{Task, ZIO} + +private[elasticsearch] final case class TestExecutor private (data: TMap[IndexName, TMap[DocumentId, Document]]) + extends ElasticExecutor { + self => + + def execute[A](request: ElasticRequest[A]): Task[A] = + request match { + case BulkRequest(requests, _, _, _) => + fakeBulk(requests) + case CreateRequest(index, document, _, _) => + fakeCreate(index, document) + case CreateWithIdRequest(index, id, document, _, _) => + fakeCreateWithId(index, id, document) + case CreateIndexRequest(name, _) => + fakeCreateIndex(name) + case CreateOrUpdateRequest(index, id, document, _, _) => + fakeCreateOrUpdate(index, id, document) + case DeleteByIdRequest(index, id, _, _) => + fakeDeleteById(index, id) + case DeleteByQueryRequest(index, _, _, _) => + fakeDeleteByQuery(index) + case DeleteIndexRequest(name) => + fakeDeleteIndex(name) + case ExistsRequest(index, id, _) => + fakeExists(index, id) + case GetByIdRequest(index, id, _) => + fakeGetById(index, id) + case GetByQueryRequest(index, _, _) => + fakeGetByQuery(index) + } + + private def fakeBulk(requests: List[BulkableRequest[_]]): Task[Unit] = + ZIO.attempt { + requests.map { r => + execute(r) + } + }.unit + + private def fakeCreate(index: IndexName, document: Document): Task[DocumentId] = + for { + uuid <- nextUUID + documents <- getDocumentsFromIndex(index).commit + documentId = DocumentId(uuid.toString) + _ <- documents.put(documentId, document).commit + } yield documentId + + private def fakeCreateWithId(index: IndexName, documentId: DocumentId, document: Document): Task[CreationOutcome] = + (for { + documents <- getDocumentsFromIndex(index) + alreadyExists <- documents.contains(documentId) + _ <- documents.putIfAbsent(documentId, document) + } yield if (alreadyExists) AlreadyExists else Created).commit + + private def fakeCreateIndex(index: IndexName): Task[CreationOutcome] = + (for { + alreadyExists <- self.data.contains(index) + emptyDocuments <- TMap.empty[DocumentId, Document] + _ <- self.data.putIfAbsent(index, emptyDocuments) + } yield if (alreadyExists) AlreadyExists else Created).commit + + private def fakeCreateOrUpdate(index: IndexName, documentId: DocumentId, document: Document): Task[Unit] = + (for { + documents <- getDocumentsFromIndex(index) + _ <- documents.put(documentId, document) + } yield ()).commit + + private def fakeDeleteById(index: IndexName, documentId: DocumentId): Task[DeletionOutcome] = + (for { + documents <- getDocumentsFromIndex(index) + exists <- documents.contains(documentId) + _ <- documents.delete(documentId) + } yield if (exists) Deleted else NotFound).commit + + private def fakeDeleteByQuery(index: IndexName): Task[DeletionOutcome] = + (for { + exists <- self.data.contains(index) + } yield if (exists) Deleted else NotFound).commit + // until we have a way of using query to delete we can either delete all or delete none documents + + private def fakeDeleteIndex(index: IndexName): Task[DeletionOutcome] = + (for { + exists <- self.data.contains(index) + _ <- self.data.delete(index) + } yield if (exists) Deleted else NotFound).commit + + private def fakeExists(index: IndexName, documentId: DocumentId): Task[Boolean] = + (for { + documents <- getDocumentsFromIndex(index) + exists <- documents.contains(documentId) + } yield exists).commit + + private def fakeGetById(index: IndexName, documentId: DocumentId): Task[GetResult] = + (for { + documents <- getDocumentsFromIndex(index) + maybeDocument <- documents.get(documentId) + } yield new GetResult(maybeDocument)).commit + + private def fakeGetByQuery(index: IndexName): Task[SearchResult] = { + def createSearchResult( + index: IndexName, + documents: TMap[DocumentId, Document] + ): ZSTM[Any, Nothing, SearchResult] = { + for { + items <- + documents.toList.map( + _.map { case (id, document) => + Item( + index = index.toString, + `type` = "type", + id = id.toString, + score = 1, + source = Json.Str(document.json) + ) + } + ) + } yield new SearchResult(items.map(_.source.toString).map(Document(_))) + } + + (for { + documents <- getDocumentsFromIndex(index) + response <- createSearchResult(index, documents) + } yield response).commit + } + + private def getDocumentsFromIndex(index: IndexName): ZSTM[Any, ElasticException, TMap[DocumentId, Document]] = + for { + maybeDocuments <- self.data.get(index) + documents <- maybeDocuments.fold[STM[ElasticException, TMap[DocumentId, Document]]]( + STM.fail[ElasticException](new ElasticException(s"Index $index does not exists!")) + )(STM.succeed(_)) + } yield documents +} diff --git a/modules/library/src/main/scala/zio/elasticsearch/package.scala b/modules/library/src/main/scala/zio/elasticsearch/package.scala index 353380e4e..ecb3f6d66 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/package.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/package.scala @@ -20,6 +20,7 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.StringUtils._ import zio.prelude.AssertionError.failure import zio.prelude.Newtype +import zio.schema.Schema package object elasticsearch { private[elasticsearch] class ElasticException(message: String) extends RuntimeException(message) @@ -61,4 +62,14 @@ package object elasticsearch { def containsAny(name: String, params: List[String]): Boolean = params.exists(StringUtils.contains(name, _)) + // TODO decide if this extension is favorable to avoid having an additional flatMap in user code + final implicit class ResultRIO[R, F[_]](zio: RIO[R, ElasticResult[F]]) { + def result[A: Schema]: RIO[R, F[A]] = zio.flatMap(_.result[A]) + } + + // TODO decide if this extension is favorable to avoid having an additional flatMap in user code + final implicit class ResultTask[F[_]](zio: Task[ElasticResult[F]]) { + def result[A: Schema]: Task[F[A]] = zio.flatMap(_.result[A]) + } + } diff --git a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala index 71699cc86..a8226335f 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala @@ -308,9 +308,9 @@ object HttpElasticExecutorSpec extends WireMockSpec { assertZIO( addStubMapping *> ElasticExecutor.execute( ElasticRequest - .getById[GitHubRepo](index = index, id = DocumentId("V4x8q4UB3agN0z75fv5r")) + .getById(index = index, id = DocumentId("V4x8q4UB3agN0z75fv5r")) .routing(Routing("routing")) - ) + ).result[GitHubRepo] )(isSome(equalTo(repo))) }, test("getting by query request") { @@ -361,7 +361,7 @@ object HttpElasticExecutorSpec extends WireMockSpec { ) assertZIO( - addStubMapping *> ElasticExecutor.execute(ElasticRequest.search[GitHubRepo](index = index, query = matchAll)) + addStubMapping *> ElasticExecutor.execute(ElasticRequest.search(index = index, query = matchAll)).result[GitHubRepo] )( equalTo(List(repo)) ) From 98b04b46d92f467b954e089fd8c77390c6e1e5ea Mon Sep 17 00:00:00 2001 From: markaya Date: Wed, 1 Mar 2023 13:49:40 +0100 Subject: [PATCH 02/14] Make case classes private again --- .../zio/elasticsearch/HttpExecutorSpec.scala | 33 ++-- .../zio/elasticsearch/ElasticRequest.scala | 147 ++++++++++-------- .../elasticsearch/HttpElasticExecutor.scala | 49 +++--- .../zio/elasticsearch/TestExecutor.scala | 25 ++- .../zio/elasticsearch/QueryDSLSpec.scala | 6 +- 5 files changed, 144 insertions(+), 116 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index a6db87fca..2d905bd9b 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -31,7 +31,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genCustomer) { customer => for { docId <- ElasticExecutor.execute(ElasticRequest.create[CustomerDocument](index, customer)) - res <- ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, docId)) + res <- ElasticExecutor.execute(ElasticRequest.getById(index, docId)).result[CustomerDocument] } yield assert(res)(isSome(equalTo(customer))) } }, @@ -67,7 +67,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId, genCustomer) { (documentId, customer) => for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, customer)) - doc <- ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, documentId)) + doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] } yield assert(doc)(isSome(equalTo(customer))) } }, @@ -76,7 +76,7 @@ object HttpExecutorSpec extends IntegrationSpec { for { _ <- ElasticExecutor.execute(ElasticRequest.create[CustomerDocument](index, documentId, firstCustomer)) _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, secondCustomer)) - doc <- ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, documentId)) + doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] } yield assert(doc)(isSome(equalTo(secondCustomer))) } } @@ -131,20 +131,22 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId, genCustomer) { (documentId, customer) => for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, customer)) - res <- ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, documentId)) + res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] } yield assert(res)(isSome(equalTo(customer))) } }, test("return None if the document does not exist") { checkOnce(genDocumentId) { documentId => - assertZIO(ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, documentId)))(isNone) + assertZIO(ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument])( + isNone + ) } }, test("fail with throwable if decoding fails") { checkOnce(genDocumentId, genEmployee) { (documentId, employee) => val result = for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[EmployeeDocument](index, documentId, employee)) - res <- ElasticExecutor.execute(ElasticRequest.getById[CustomerDocument](index, documentId)) + res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] } yield res assertZIO(result.exit)( @@ -170,7 +172,8 @@ object HttpExecutorSpec extends IntegrationSpec { .refreshTrue ) query = range("balance").gte(100) - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](firstSearchIndex, query)) + res <- + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] } yield assert(res)(isNonEmpty) } } @@ around( @@ -193,7 +196,8 @@ object HttpExecutorSpec extends IntegrationSpec { .refreshTrue ) query = range("age").gte(0) - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](secondSearchIndex, query)) + res <- + ElasticExecutor.execute(ElasticRequest.search(secondSearchIndex, query)).result[CustomerDocument] } yield res assertZIO(result.exit)( @@ -224,7 +228,8 @@ object HttpExecutorSpec extends IntegrationSpec { .refreshTrue ) query = ElasticQuery.contains("name.keyword", firstCustomer.name.take(3)) - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](firstSearchIndex, query)) + res <- + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -247,7 +252,8 @@ object HttpExecutorSpec extends IntegrationSpec { .refreshTrue ) query = ElasticQuery.startsWith("name.keyword", firstCustomer.name.take(3)) - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](firstSearchIndex, query)) + res <- + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -271,7 +277,8 @@ object HttpExecutorSpec extends IntegrationSpec { ) query = wildcard("name.keyword", s"${firstCustomer.name.take(2)}*${firstCustomer.name.takeRight(2)}") - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](firstSearchIndex, query)) + res <- + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -314,7 +321,9 @@ object HttpExecutorSpec extends IntegrationSpec { deleteQuery = range("balance").gte(300) _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(deleteByQueryIndex, deleteQuery).refreshTrue) - res <- ElasticExecutor.execute(ElasticRequest.search[CustomerDocument](deleteByQueryIndex, matchAll)) + res <- ElasticExecutor + .execute(ElasticRequest.search(deleteByQueryIndex, matchAll)) + .result[CustomerDocument] } yield assert(res)(hasSameElements(List(firstCustomer.copy(balance = 150)))) } } @@ around( diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index 88e43c6ff..aa0b81439 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -37,50 +37,50 @@ sealed trait BulkableRequest[A] extends ElasticRequest[A] object ElasticRequest { - def bulk(requests: BulkableRequest[_]*): BulkRequest = - BulkRequest.of(requests: _*) + def bulk(requests: BulkableRequest[_]*): Bulk = + Bulk.of(requests: _*) - def create[A: Schema](index: IndexName, doc: A): CreateRequest = - CreateRequest(index = index, document = Document.from(doc), refresh = false, routing = None) + def create[A: Schema](index: IndexName, doc: A): Create = + Create(index = index, document = Document.from(doc), refresh = false, routing = None) - def create[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateWithIdRequest = - CreateWithIdRequest(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) + def create[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateWithId = + CreateWithId(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) - def createIndex(name: IndexName): ElasticRequest[CreationOutcome, CreateIndex] = - CreateIndexRequest(name = name, definition = None) + def createIndex(name: IndexName): CreateIndex = + CreateIndex(name = name, definition = None) - def createIndex(name: IndexName, definition: String): ElasticRequest[CreationOutcome, CreateIndex] = - CreateIndexRequest(name = name, definition = Some(definition)) + def createIndex(name: IndexName, definition: String): CreateIndex = + CreateIndex(name = name, definition = Some(definition)) - def deleteById(index: IndexName, id: DocumentId): DeleteByIdRequest = - DeleteByIdRequest(index = index, id = id, refresh = false, routing = None) + def deleteById(index: IndexName, id: DocumentId): DeleteById = + DeleteById(index = index, id = id, refresh = false, routing = None) - def deleteByQuery(index: IndexName, query: ElasticQuery[_]): DeleteByQueryRequest = - DeleteByQueryRequest(index = index, query = query, refresh = false, routing = None) + def deleteByQuery(index: IndexName, query: ElasticQuery[_]): DeleteByQuery = + DeleteByQuery(index = index, query = query, refresh = false, routing = None) - def deleteIndex(name: IndexName): DeleteIndexRequest = - DeleteIndexRequest(name) + def deleteIndex(name: IndexName): DeleteIndex = + DeleteIndex(name) - def exists(index: IndexName, id: DocumentId): ExistsRequest = - ExistsRequest(index = index, id = id, routing = None) + def exists(index: IndexName, id: DocumentId): Exists = + Exists(index = index, id = id, routing = None) - def getById(index: IndexName, id: DocumentId): GetByIdRequest = - GetByIdRequest(index = index, id = id, routing = None) + def getById(index: IndexName, id: DocumentId): GetById = + GetById(index = index, id = id, routing = None) - def search(index: IndexName, query: ElasticQuery[_]): GetByQueryRequest = - GetByQueryRequest(index = index, query = query, routing = None) + def search(index: IndexName, query: ElasticQuery[_]): GetByQuery = + GetByQuery(index = index, query = query, routing = None) - def upsert[A: Schema](index: IndexName, id: DocumentId, doc: A): ElasticRequest[Unit, Upsert] = - CreateOrUpdateRequest(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) + def upsert[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateOrUpdate = + CreateOrUpdate(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) - final case class BulkRequest( + sealed trait BulkRequest extends ElasticRequest[Unit] with HasRouting[Unit] with HasRefresh[Unit] + + private[elasticsearch] final case class Bulk( requests: List[BulkableRequest[_]], index: Option[IndexName], refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[Unit] - with HasRouting[Unit] - with HasRefresh[Unit] { self => + ) extends BulkRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -93,37 +93,37 @@ object ElasticRequest { // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen // because these are only Bulkable Requests and other matches will not occur. r match { - case CreateRequest(index, document, _, maybeRouting) => + case Create(index, document, _, maybeRouting) => List(getActionAndMeta("create", List(("_index", Some(index)), ("routing", maybeRouting))), document.json) - case CreateWithIdRequest(index, id, document, _, maybeRouting) => + case CreateWithId(index, id, document, _, maybeRouting) => List( getActionAndMeta("create", List(("_index", Some(index)), ("_id", Some(id)), ("routing", maybeRouting))), document.json ) - case CreateOrUpdateRequest(index, id, document, _, maybeRouting) => + case CreateOrUpdate(index, id, document, _, maybeRouting) => List( getActionAndMeta("index", List(("_index", Some(index)), ("_id", Some(id)), ("routing", maybeRouting))), document.json ) - case DeleteByIdRequest(index, id, _, maybeRouting) => + case DeleteById(index, id, _, maybeRouting) => List(getActionAndMeta("delete", List(("_index", Some(index)), ("_id", Some(id)), ("routing", maybeRouting)))) } }.mkString(start = "", sep = "\n", end = "\n") } - object BulkRequest { - def of(requests: BulkableRequest[_]*): BulkRequest = - BulkRequest(requests = requests.toList, index = None, refresh = false, routing = None) + object Bulk { + def of(requests: BulkableRequest[_]*): Bulk = + Bulk(requests = requests.toList, index = None, refresh = false, routing = None) } - final case class CreateRequest( + sealed trait CreateRequest extends BulkableRequest[DocumentId] with HasRouting[DocumentId] with HasRefresh[DocumentId] + + private[elasticsearch] final case class Create( index: IndexName, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends BulkableRequest[DocumentId] - with HasRouting[DocumentId] - with HasRefresh[DocumentId] { self => + ) extends CreateRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -133,15 +133,18 @@ object ElasticRequest { override def refreshTrue = refresh(true) } - final case class CreateWithIdRequest( + sealed trait CreateWithIdRequest + extends BulkableRequest[CreationOutcome] + with HasRouting[CreationOutcome] + with HasRefresh[CreationOutcome] + + private[elasticsearch] final case class CreateWithId( index: IndexName, id: DocumentId, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends BulkableRequest[CreationOutcome] - with HasRouting[CreationOutcome] - with HasRefresh[CreationOutcome] { self => + ) extends CreateWithIdRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -151,20 +154,22 @@ object ElasticRequest { override def refreshTrue = refresh(true) } - final case class CreateIndexRequest( + sealed trait CreateIndexRequest extends ElasticRequest[CreationOutcome] + + private[elasticsearch] final case class CreateIndex( name: IndexName, definition: Option[String] ) extends ElasticRequest[CreationOutcome] - final case class CreateOrUpdateRequest( + sealed trait CreateOrUpdateRequest extends BulkableRequest[Unit] with HasRouting[Unit] with HasRefresh[Unit] + + private[elasticsearch] final case class CreateOrUpdate( index: IndexName, id: DocumentId, document: Document, refresh: Boolean, routing: Option[Routing] - ) extends BulkableRequest[Unit] - with HasRouting[Unit] - with HasRefresh[Unit] { self => + ) extends CreateOrUpdateRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -174,14 +179,17 @@ object ElasticRequest { override def refreshTrue = refresh(true) } - final case class DeleteByIdRequest( + sealed trait DeleteByIdRequest + extends BulkableRequest[DeletionOutcome] + with HasRouting[DeletionOutcome] + with HasRefresh[DeletionOutcome] + + private[elasticsearch] final case class DeleteById( index: IndexName, id: DocumentId, refresh: Boolean, routing: Option[Routing] - ) extends BulkableRequest[DeletionOutcome] - with HasRouting[DeletionOutcome] - with HasRefresh[DeletionOutcome] { self => + ) extends DeleteByIdRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -191,14 +199,17 @@ object ElasticRequest { override def refreshTrue = refresh(true) } - final case class DeleteByQueryRequest( + sealed trait DeleteByQueryRequest + extends ElasticRequest[DeletionOutcome] + with HasRouting[DeletionOutcome] + with HasRefresh[DeletionOutcome] + + private[elasticsearch] final case class DeleteByQuery( index: IndexName, query: ElasticQuery[_], refresh: Boolean, routing: Option[Routing] - ) extends ElasticRequest[DeletionOutcome] - with HasRouting[DeletionOutcome] - with HasRefresh[DeletionOutcome] { self => + ) extends DeleteByQueryRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) override def refresh(value: Boolean) = self.copy(refresh = value) @@ -208,31 +219,37 @@ object ElasticRequest { override def refreshTrue = refresh(true) } - final case class DeleteIndexRequest(name: IndexName) extends ElasticRequest[DeletionOutcome] + sealed trait DeleteIndexRequest extends ElasticRequest[DeletionOutcome] - final case class ExistsRequest( + final case class DeleteIndex(name: IndexName) extends DeleteIndexRequest + + sealed trait ExistRequest extends ElasticRequest[Boolean] with HasRouting[Boolean] + + private[elasticsearch] final case class Exists( index: IndexName, id: DocumentId, routing: Option[Routing] - ) extends ElasticRequest[Boolean] - with HasRouting[Boolean] { self => + ) extends ExistRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) } - final case class GetByIdRequest( + sealed trait GetByIdRequest extends ElasticRequest[GetResult] with HasRouting[GetResult] + + private[elasticsearch] final case class GetById( index: IndexName, id: DocumentId, routing: Option[Routing] - ) extends ElasticRequest[GetResult] - with HasRouting[GetResult] { self => + ) extends GetByIdRequest { self => override def routing(value: Routing) = self.copy(routing = Some(value)) } - final case class GetByQueryRequest( + sealed trait GetByQueryRequest extends ElasticRequest[SearchResult] + + private[elasticsearch] final case class GetByQuery( index: IndexName, query: ElasticQuery[_], routing: Option[Routing] - ) extends ElasticRequest[SearchResult] + ) extends GetByQueryRequest private def getActionAndMeta(requestType: String, parameters: List[(String, Any)]): String = parameters.collect { case (name, Some(value)) => s""""$name" : "${value.toString}"""" } diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index c437581e1..768fb7a9c 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -39,20 +39,20 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC def execute[A](request: ElasticRequest[A]): Task[A] = request match { - case r: BulkRequest => executeBulk(r) - case r: CreateRequest => executeCreate(r) - case r: CreateWithIdRequest => executeCreateWithId(r) - case r: CreateIndexRequest => executeCreateIndex(r) - case r: CreateOrUpdateRequest => executeCreateOrUpdate(r) - case r: DeleteByIdRequest => executeDeleteById(r) - case r: DeleteByQueryRequest => executeDeleteByQuery(r) - case r: DeleteIndexRequest => executeDeleteIndex(r) - case r: ExistsRequest => executeExists(r) - case r: GetByIdRequest => executeGetById(r) - case r: GetByQueryRequest => executeGetByQuery(r) + case r: Bulk => executeBulk(r) + case r: Create => executeCreate(r) + case r: CreateWithId => executeCreateWithId(r) + case r: CreateIndex => executeCreateIndex(r) + case r: CreateOrUpdate => executeCreateOrUpdate(r) + case r: DeleteById => executeDeleteById(r) + case r: DeleteByQuery => executeDeleteByQuery(r) + case r: DeleteIndex => executeDeleteIndex(r) + case r: Exists => executeExists(r) + case r: GetById => executeGetById(r) + case r: GetByQuery => executeGetByQuery(r) } - private def executeBulk(r: BulkRequest): Task[Unit] = { + private def executeBulk(r: Bulk): Task[Unit] = { val uri = (r.index match { case Some(index) => uri"${config.uri}/$index/$Bulk" case None => uri"${config.uri}/$Bulk" @@ -68,7 +68,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeCreate(r: CreateRequest): Task[DocumentId] = { + private def executeCreate(r: Create): Task[DocumentId] = { val uri = uri"${config.uri}/${r.index}/$Doc" .withParams(getQueryParams(List(("refresh", Some(r.refresh)), ("routing", r.routing)))) @@ -94,7 +94,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } - private def executeCreateWithId(r: CreateWithIdRequest): Task[CreationOutcome] = { + private def executeCreateWithId(r: CreateWithId): Task[CreationOutcome] = { val uri = uri"${config.uri}/${r.index}/$Create/${r.id}" .withParams(getQueryParams(List(("refresh", Some(r.refresh)), ("routing", r.routing)))) @@ -112,7 +112,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeCreateIndex(createIndex: CreateIndexRequest): Task[CreationOutcome] = + private def executeCreateIndex(createIndex: CreateIndex): Task[CreationOutcome] = sendRequest( request .put(uri"${config.uri}/${createIndex.name}") @@ -126,7 +126,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeCreateOrUpdate(r: CreateOrUpdateRequest): Task[Unit] = { + private def executeCreateOrUpdate(r: CreateOrUpdate): Task[Unit] = { val uri = uri"${config.uri}/${r.index}/$Doc/${r.id}" .withParams(getQueryParams(List(("refresh", Some(r.refresh)), ("routing", r.routing)))) @@ -138,7 +138,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeDeleteById(r: DeleteByIdRequest): Task[DeletionOutcome] = { + private def executeDeleteById(r: DeleteById): Task[DeletionOutcome] = { val uri = uri"${config.uri}/${r.index}/$Doc/${r.id}" .withParams(getQueryParams(List(("refresh", Some(r.refresh)), ("routing", r.routing)))) @@ -151,7 +151,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeDeleteByQuery(r: DeleteByQueryRequest): Task[DeletionOutcome] = { + private def executeDeleteByQuery(r: DeleteByQuery): Task[DeletionOutcome] = { val uri = uri"${config.uri}/${r.index}/$DeleteByQuery".withParams( getQueryParams(List(("refresh", Some(r.refresh)), ("routing", r.routing))) @@ -171,7 +171,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeDeleteIndex(r: DeleteIndexRequest): Task[DeletionOutcome] = + private def executeDeleteIndex(r: DeleteIndex): Task[DeletionOutcome] = sendRequest(request.delete(uri"${config.uri}/${r.name}")).flatMap { response => response.code match { case HttpOk => ZIO.succeed(Deleted) @@ -180,7 +180,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeExists(r: ExistsRequest): Task[Boolean] = { + private def executeExists(r: Exists): Task[Boolean] = { val uri = uri"${config.uri}/${r.index}/$Doc/${r.id}".withParams(getQueryParams(List(("routing", r.routing)))) sendRequest(request.head(uri)).flatMap { response => @@ -192,7 +192,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetById(r: GetByIdRequest): Task[GetResult] = { + private def executeGetById(r: GetById): Task[GetResult] = { val uri = uri"${config.uri}/${r.index}/$Doc/${r.id}".withParams(getQueryParams(List(("routing", r.routing)))) sendRequestWithCustomResponse[ElasticGetResponse]( @@ -208,7 +208,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetByQuery(r: GetByQueryRequest): Task[SearchResult] = + private def executeGetByQuery(r: GetByQuery): Task[SearchResult] = sendRequestWithCustomResponse( request .post(uri"${config.uri}/${r.index}/$Search") @@ -220,7 +220,10 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case HttpOk => response.body.fold( e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed(new SearchResult(value.results.map(_.toString).map(Document(_)))) // TODO have Json in Document instead of String??? + value => + ZIO.succeed( + new SearchResult(value.results.map(_.toString).map(Document(_))) + ) // TODO have Json in Document instead of String??? ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala index 9c1497c71..11b1ba080 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala @@ -28,27 +28,27 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa def execute[A](request: ElasticRequest[A]): Task[A] = request match { - case BulkRequest(requests, _, _, _) => + case Bulk(requests, _, _, _) => fakeBulk(requests) - case CreateRequest(index, document, _, _) => + case Create(index, document, _, _) => fakeCreate(index, document) - case CreateWithIdRequest(index, id, document, _, _) => + case CreateWithId(index, id, document, _, _) => fakeCreateWithId(index, id, document) - case CreateIndexRequest(name, _) => + case CreateIndex(name, _) => fakeCreateIndex(name) - case CreateOrUpdateRequest(index, id, document, _, _) => + case CreateOrUpdate(index, id, document, _, _) => fakeCreateOrUpdate(index, id, document) - case DeleteByIdRequest(index, id, _, _) => + case DeleteById(index, id, _, _) => fakeDeleteById(index, id) - case DeleteByQueryRequest(index, _, _, _) => + case DeleteByQuery(index, _, _, _) => fakeDeleteByQuery(index) - case DeleteIndexRequest(name) => + case DeleteIndex(name) => fakeDeleteIndex(name) - case ExistsRequest(index, id, _) => + case Exists(index, id, _) => fakeExists(index, id) - case GetByIdRequest(index, id, _) => + case GetById(index, id, _) => fakeGetById(index, id) - case GetByQueryRequest(index, _, _) => + case GetByQuery(index, _, _) => fakeGetByQuery(index) } @@ -122,7 +122,7 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa def createSearchResult( index: IndexName, documents: TMap[DocumentId, Document] - ): ZSTM[Any, Nothing, SearchResult] = { + ): ZSTM[Any, Nothing, SearchResult] = for { items <- documents.toList.map( @@ -137,7 +137,6 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa } ) } yield new SearchResult(items.map(_.source.toString).map(Document(_))) - } (for { documents <- getDocumentsFromIndex(index) diff --git a/modules/library/src/test/scala/zio/elasticsearch/QueryDSLSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/QueryDSLSpec.scala index 7f177dc1d..0ea88bc54 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/QueryDSLSpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/QueryDSLSpec.scala @@ -18,7 +18,7 @@ package zio.elasticsearch import zio.Scope import zio.elasticsearch.ElasticQuery._ -import zio.elasticsearch.ElasticRequest.BulkRequest +import zio.elasticsearch.ElasticRequest.Bulk import zio.elasticsearch.utils._ import zio.prelude.Newtype.unsafeWrap import zio.prelude.Validation @@ -1073,8 +1073,8 @@ object QueryDSLSpec extends ZIOSpecDefault { val req4 = ElasticRequest.deleteById(index, DocumentId("1VNzFt2XUFZfXZheDc")).routing(unsafeWrap(Routing)(user.id)) ElasticRequest.bulk(req1, req2, req3, req4) match { - case r: BulkRequest => Some(r.body) - case _ => None + case r: Bulk => Some(r.body) + case _ => None } } From 63f09127656188e1ca443fecc493add4add3b16f Mon Sep 17 00:00:00 2001 From: markaya Date: Wed, 1 Mar 2023 14:35:13 +0100 Subject: [PATCH 03/14] Fix linter --- .../zio/elasticsearch/ElasticResult.scala | 24 ++++++++++++++++--- .../HttpElasticExecutorSpec.scala | 16 ++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala index fdd67007f..c5775d7d1 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -1,8 +1,24 @@ +/* + * Copyright 2022 LambdaWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package zio.elasticsearch -import zio.{Task, ZIO} -import zio.schema.Schema import zio.prelude.{ForEachOps, ZValidation} +import zio.schema.Schema +import zio.{Task, ZIO} sealed trait ElasticResult[F[_]] { def result[A: Schema]: Task[F[A]] @@ -10,7 +26,9 @@ sealed trait ElasticResult[F[_]] { final class GetResult(private val doc: Option[Document]) extends ElasticResult[Option] { override def result[A: Schema]: Task[Option[A]] = - ZIO.fromEither(doc.forEach(_.decode)).mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) + ZIO + .fromEither(doc.forEach(_.decode)) + .mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) } final class SearchResult(private val hits: List[Document]) extends ElasticResult[List] { override def result[A: Schema]: Task[List[A]] = diff --git a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala index a8226335f..6ebff2481 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala @@ -306,11 +306,13 @@ object HttpElasticExecutorSpec extends WireMockSpec { ) assertZIO( - addStubMapping *> ElasticExecutor.execute( - ElasticRequest - .getById(index = index, id = DocumentId("V4x8q4UB3agN0z75fv5r")) - .routing(Routing("routing")) - ).result[GitHubRepo] + addStubMapping *> ElasticExecutor + .execute( + ElasticRequest + .getById(index = index, id = DocumentId("V4x8q4UB3agN0z75fv5r")) + .routing(Routing("routing")) + ) + .result[GitHubRepo] )(isSome(equalTo(repo))) }, test("getting by query request") { @@ -361,7 +363,9 @@ object HttpElasticExecutorSpec extends WireMockSpec { ) assertZIO( - addStubMapping *> ElasticExecutor.execute(ElasticRequest.search(index = index, query = matchAll)).result[GitHubRepo] + addStubMapping *> ElasticExecutor + .execute(ElasticRequest.search(index = index, query = matchAll)) + .result[GitHubRepo] )( equalTo(List(repo)) ) From 8baf47915f43c47cc3edb102215167fd5b07131b Mon Sep 17 00:00:00 2001 From: markaya Date: Wed, 1 Mar 2023 15:39:21 +0100 Subject: [PATCH 04/14] fix tests for scala 2.12 and fix code remarks --- .../zio/elasticsearch/ElasticRequest.scala | 52 +++++++++---------- .../zio/elasticsearch/ElasticResult.scala | 13 ++++- .../zio/elasticsearch/TestExecutor.scala | 4 +- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index aa0b81439..e58345533 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -81,13 +81,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends BulkRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) lazy val body: String = requests.flatMap { r => // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen @@ -124,13 +124,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) } sealed trait CreateWithIdRequest @@ -145,13 +145,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateWithIdRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) } sealed trait CreateIndexRequest extends ElasticRequest[CreationOutcome] @@ -170,13 +170,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateOrUpdateRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) } sealed trait DeleteByIdRequest @@ -190,13 +190,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByIdRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) } sealed trait DeleteByQueryRequest @@ -210,13 +210,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByQueryRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) - override def refresh(value: Boolean) = self.copy(refresh = value) + def refresh(value: Boolean) = self.copy(refresh = value) - override def refreshFalse = refresh(false) + def refreshFalse = refresh(false) - override def refreshTrue = refresh(true) + def refreshTrue = refresh(true) } sealed trait DeleteIndexRequest extends ElasticRequest[DeletionOutcome] @@ -230,7 +230,7 @@ object ElasticRequest { id: DocumentId, routing: Option[Routing] ) extends ExistRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) } sealed trait GetByIdRequest extends ElasticRequest[GetResult] with HasRouting[GetResult] @@ -240,7 +240,7 @@ object ElasticRequest { id: DocumentId, routing: Option[Routing] ) extends GetByIdRequest { self => - override def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing) = self.copy(routing = Some(value)) } sealed trait GetByQueryRequest extends ElasticRequest[SearchResult] diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala index c5775d7d1..cea2eb357 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -16,7 +16,7 @@ package zio.elasticsearch -import zio.prelude.{ForEachOps, ZValidation} +import zio.prelude.ZValidation import zio.schema.Schema import zio.{Task, ZIO} @@ -27,9 +27,18 @@ sealed trait ElasticResult[F[_]] { final class GetResult(private val doc: Option[Document]) extends ElasticResult[Option] { override def result[A: Schema]: Task[Option[A]] = ZIO - .fromEither(doc.forEach(_.decode)) + .fromEither(doc match { + case Some(document) => + document.decode match { + case Left(e) => Left(DecodingException(s"Could not parse the document: ${e.message}")) + case Right(doc) => Right(Some(doc)) + } + case None => + Right(None) + }) .mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) } + final class SearchResult(private val hits: List[Document]) extends ElasticResult[List] { override def result[A: Schema]: Task[List[A]] = ZIO.fromEither { diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala index 11b1ba080..dd3749354 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala @@ -19,7 +19,7 @@ package zio.elasticsearch import zio.Random.nextUUID import zio.elasticsearch.ElasticRequest._ import zio.json.ast.Json -import zio.stm.{STM, TMap, ZSTM} +import zio.stm.{STM, TMap, USTM, ZSTM} import zio.{Task, ZIO} private[elasticsearch] final case class TestExecutor private (data: TMap[IndexName, TMap[DocumentId, Document]]) @@ -122,7 +122,7 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa def createSearchResult( index: IndexName, documents: TMap[DocumentId, Document] - ): ZSTM[Any, Nothing, SearchResult] = + ): USTM[SearchResult] = for { items <- documents.toList.map( From e686839c19d5eeb377e53c4cc447eddbf6f5c331 Mon Sep 17 00:00:00 2001 From: markaya Date: Wed, 1 Mar 2023 15:58:52 +0100 Subject: [PATCH 05/14] Fix code remarks --- .../zio/elasticsearch/ElasticRequest.scala | 60 ++++++------------- 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index e58345533..12536403f 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -19,15 +19,15 @@ package zio.elasticsearch import zio.elasticsearch.Routing.Routing import zio.schema.Schema -trait HasRefresh[A] { // TODO extend this on all appropritate requests +trait HasRefresh[A] { def refresh(value: Boolean): ElasticRequest[A] - def refreshFalse: ElasticRequest[A] + def refreshFalse: ElasticRequest[A] = refresh(false) - def refreshTrue: ElasticRequest[A] + def refreshTrue: ElasticRequest[A] = refresh(true) } -trait HasRouting[A] { // TODO extend this on all appropritate requests +trait HasRouting[A] { def routing(value: Routing): ElasticRequest[A] } @@ -81,13 +81,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends BulkRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing): Bulk = self.copy(routing = Some(value)) - def refresh(value: Boolean) = self.copy(refresh = value) - - def refreshFalse = refresh(false) - - def refreshTrue = refresh(true) + def refresh(value: Boolean): Bulk = self.copy(refresh = value) lazy val body: String = requests.flatMap { r => // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen @@ -124,13 +120,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) - - def refresh(value: Boolean) = self.copy(refresh = value) + def routing(value: Routing): Create = self.copy(routing = Some(value)) - def refreshFalse = refresh(false) - - def refreshTrue = refresh(true) + def refresh(value: Boolean): Create = self.copy(refresh = value) } sealed trait CreateWithIdRequest @@ -145,13 +137,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateWithIdRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) - - def refresh(value: Boolean) = self.copy(refresh = value) - - def refreshFalse = refresh(false) + def routing(value: Routing): CreateWithId = self.copy(routing = Some(value)) - def refreshTrue = refresh(true) + def refresh(value: Boolean): CreateWithId = self.copy(refresh = value) } sealed trait CreateIndexRequest extends ElasticRequest[CreationOutcome] @@ -170,13 +158,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateOrUpdateRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing): CreateOrUpdate = self.copy(routing = Some(value)) - def refresh(value: Boolean) = self.copy(refresh = value) - - def refreshFalse = refresh(false) - - def refreshTrue = refresh(true) + def refresh(value: Boolean): CreateOrUpdate = self.copy(refresh = value) } sealed trait DeleteByIdRequest @@ -190,13 +174,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByIdRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) - - def refresh(value: Boolean) = self.copy(refresh = value) + def routing(value: Routing): DeleteById = self.copy(routing = Some(value)) - def refreshFalse = refresh(false) - - def refreshTrue = refresh(true) + def refresh(value: Boolean): DeleteById = self.copy(refresh = value) } sealed trait DeleteByQueryRequest @@ -210,13 +190,9 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByQueryRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) - - def refresh(value: Boolean) = self.copy(refresh = value) - - def refreshFalse = refresh(false) + def routing(value: Routing): DeleteByQuery = self.copy(routing = Some(value)) - def refreshTrue = refresh(true) + def refresh(value: Boolean): DeleteByQuery = self.copy(refresh = value) } sealed trait DeleteIndexRequest extends ElasticRequest[DeletionOutcome] @@ -230,7 +206,7 @@ object ElasticRequest { id: DocumentId, routing: Option[Routing] ) extends ExistRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing): Exists = self.copy(routing = Some(value)) } sealed trait GetByIdRequest extends ElasticRequest[GetResult] with HasRouting[GetResult] @@ -240,7 +216,7 @@ object ElasticRequest { id: DocumentId, routing: Option[Routing] ) extends GetByIdRequest { self => - def routing(value: Routing) = self.copy(routing = Some(value)) + def routing(value: Routing): GetById = self.copy(routing = Some(value)) } sealed trait GetByQueryRequest extends ElasticRequest[SearchResult] From 75c0b1713dbf2401396cfe21e4b665362d3cb6d2 Mon Sep 17 00:00:00 2001 From: markaya Date: Wed, 1 Mar 2023 17:00:59 +0100 Subject: [PATCH 06/14] Code remarks --- .../zio/elasticsearch/ElasticRequest.scala | 32 ++++++++++++++++--- .../scala/zio/elasticsearch/package.scala | 7 ++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index 12536403f..240373e05 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -19,15 +19,15 @@ package zio.elasticsearch import zio.elasticsearch.Routing.Routing import zio.schema.Schema -trait HasRefresh[A] { +trait HasRefresh[A] { // TODO extend this on all appropritate requests def refresh(value: Boolean): ElasticRequest[A] - def refreshFalse: ElasticRequest[A] = refresh(false) + def refreshFalse: ElasticRequest[A] - def refreshTrue: ElasticRequest[A] = refresh(true) + def refreshTrue: ElasticRequest[A] } -trait HasRouting[A] { +trait HasRouting[A] { // TODO extend this on all appropritate requests def routing(value: Routing): ElasticRequest[A] } @@ -85,6 +85,10 @@ object ElasticRequest { def refresh(value: Boolean): Bulk = self.copy(refresh = value) + def refreshFalse: Bulk = refresh(false) + + def refreshTrue: Bulk = refresh(true) + lazy val body: String = requests.flatMap { r => // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen // because these are only Bulkable Requests and other matches will not occur. @@ -123,6 +127,10 @@ object ElasticRequest { def routing(value: Routing): Create = self.copy(routing = Some(value)) def refresh(value: Boolean): Create = self.copy(refresh = value) + + def refreshFalse: Create = refresh(false) + + def refreshTrue: Create = refresh(true) } sealed trait CreateWithIdRequest @@ -140,6 +148,10 @@ object ElasticRequest { def routing(value: Routing): CreateWithId = self.copy(routing = Some(value)) def refresh(value: Boolean): CreateWithId = self.copy(refresh = value) + + def refreshFalse: CreateWithId = refresh(false) + + def refreshTrue: CreateWithId = refresh(true) } sealed trait CreateIndexRequest extends ElasticRequest[CreationOutcome] @@ -161,6 +173,10 @@ object ElasticRequest { def routing(value: Routing): CreateOrUpdate = self.copy(routing = Some(value)) def refresh(value: Boolean): CreateOrUpdate = self.copy(refresh = value) + + def refreshFalse: CreateOrUpdate = refresh(false) + + def refreshTrue: CreateOrUpdate = refresh(true) } sealed trait DeleteByIdRequest @@ -177,6 +193,10 @@ object ElasticRequest { def routing(value: Routing): DeleteById = self.copy(routing = Some(value)) def refresh(value: Boolean): DeleteById = self.copy(refresh = value) + + def refreshFalse: DeleteById = refresh(false) + + def refreshTrue: DeleteById = refresh(true) } sealed trait DeleteByQueryRequest @@ -193,6 +213,10 @@ object ElasticRequest { def routing(value: Routing): DeleteByQuery = self.copy(routing = Some(value)) def refresh(value: Boolean): DeleteByQuery = self.copy(refresh = value) + + def refreshFalse: DeleteByQuery = refresh(false) + + def refreshTrue: DeleteByQuery = refresh(true) } sealed trait DeleteIndexRequest extends ElasticRequest[DeletionOutcome] diff --git a/modules/library/src/main/scala/zio/elasticsearch/package.scala b/modules/library/src/main/scala/zio/elasticsearch/package.scala index ecb3f6d66..63967b9f7 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/package.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/package.scala @@ -62,7 +62,7 @@ package object elasticsearch { def containsAny(name: String, params: List[String]): Boolean = params.exists(StringUtils.contains(name, _)) - // TODO decide if this extension is favorable to avoid having an additional flatMap in user code + /*// TODO decide if this extension is favorable to avoid having an additional flatMap in user code final implicit class ResultRIO[R, F[_]](zio: RIO[R, ElasticResult[F]]) { def result[A: Schema]: RIO[R, F[A]] = zio.flatMap(_.result[A]) } @@ -70,6 +70,9 @@ package object elasticsearch { // TODO decide if this extension is favorable to avoid having an additional flatMap in user code final implicit class ResultTask[F[_]](zio: Task[ElasticResult[F]]) { def result[A: Schema]: Task[F[A]] = zio.flatMap(_.result[A]) - } + }*/ + final implicit class Result[R, F[_]](zio: ZIO[R, Throwable, ElasticResult[F]]) { + def result[A: Schema]: ZIO[R, Throwable, F[A]] = zio.flatMap(_.result[A]) + } } From 02375e88b08d3f77990b9e21bf8e78da5a4de8a7 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 2 Mar 2023 12:23:14 +0100 Subject: [PATCH 07/14] Implement streaming raw --- .../zio/elasticsearch/HttpExecutorSpec.scala | 31 ++++++++++++ .../zio/elasticsearch/ElasticExecutor.scala | 8 +++ .../elasticsearch/ElasticQueryResponse.scala | 2 + .../zio/elasticsearch/ElasticResult.scala | 4 +- .../elasticsearch/HttpElasticExecutor.scala | 50 ++++++++++++++++++- .../zio/elasticsearch/TestExecutor.scala | 3 ++ 6 files changed, 95 insertions(+), 3 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 2d905bd9b..6310e80cc 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -16,7 +16,10 @@ package zio.elasticsearch +import zio.Chunk import zio.elasticsearch.ElasticQuery._ +import zio.json.ast.Json +import zio.stream.ZSink import zio.test.Assertion._ import zio.test.TestAspect._ import zio.test._ @@ -286,6 +289,34 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ) ) @@ shrinks(0), + suite("searching documents")( + test("search for document using range query") { + checkOnce(genDocumentId, genCustomer, genDocumentId, genCustomer) { + (firstDocumentId, firstCustomer, secondDocumentId, secondCustomer) => + val sink: ZSink[Any, Throwable, Json, Nothing, Chunk[Json]] = ZSink.collectAll[Json] + + for { + _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) + _ <- + ElasticExecutor.execute( + ElasticRequest.upsert[CustomerDocument](firstSearchIndex, firstDocumentId, firstCustomer) + ) + _ <- + ElasticExecutor.execute( + ElasticRequest + .upsert[CustomerDocument](firstSearchIndex, secondDocumentId, secondCustomer) + .refreshTrue + ) + query = range("balance").gte(100) + res <- + ElasticExecutor.stream(ElasticRequest.search(firstSearchIndex, query)).run(sink) + } yield assert(res)(isNonEmpty) + } + } @@ around( + ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie + ) + ) @@ shrinks(0), suite("deleting by query")( test("successfully delete all matched documents") { checkOnce(genDocumentId, genCustomer, genDocumentId, genCustomer, genDocumentId, genCustomer) { diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index aad57629d..c61d0e09d 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -17,10 +17,15 @@ package zio.elasticsearch import sttp.client3.SttpBackend +import zio.elasticsearch.ElasticRequest.GetByQuery +import zio.json.ast.Json +import zio.stream.ZStream import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { def execute[A](request: ElasticRequest[A]): Task[A] + + def stream(request: GetByQuery): ZStream[Any, Throwable, Json] } object ElasticExecutor { @@ -32,4 +37,7 @@ object ElasticExecutor { private[elasticsearch] def execute[A](request: ElasticRequest[A]): RIO[ElasticExecutor, A] = ZIO.serviceWithZIO[ElasticExecutor](_.execute(request)) + + private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, Json] = + ZStream.serviceWithStream[ElasticExecutor](_.stream(request)) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala index cc3870b17..8c9439679 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala @@ -20,6 +20,8 @@ import zio.json.ast.Json import zio.json.{DeriveJsonDecoder, JsonDecoder, jsonField} private[elasticsearch] final case class ElasticQueryResponse( + @jsonField("_scroll_id") + scrollId: Option[String], took: Int, @jsonField("timed_out") timedOut: Boolean, diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala index cea2eb357..86e7d4708 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -24,7 +24,7 @@ sealed trait ElasticResult[F[_]] { def result[A: Schema]: Task[F[A]] } -final class GetResult(private val doc: Option[Document]) extends ElasticResult[Option] { +final class GetResult private[elasticsearch] (private val doc: Option[Document]) extends ElasticResult[Option] { override def result[A: Schema]: Task[Option[A]] = ZIO .fromEither(doc match { @@ -39,7 +39,7 @@ final class GetResult(private val doc: Option[Document]) extends ElasticResult[O .mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) } -final class SearchResult(private val hits: List[Document]) extends ElasticResult[List] { +final class SearchResult private[elasticsearch] (private val hits: List[Document]) extends ElasticResult[List] { override def result[A: Schema]: Task[List[A]] = ZIO.fromEither { ZValidation.validateAll(hits.map(d => ZValidation.fromEither(d.decode))).toEitherWith { errors => diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index 768fb7a9c..65fb8fa6a 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -28,7 +28,10 @@ import sttp.model.StatusCode.{ } import zio.ZIO.logDebug import zio.elasticsearch.ElasticRequest._ -import zio.{Task, ZIO} +import zio.json.ast.Json +import zio.json.ast.Json.{Obj, Str} +import zio.stream.ZStream +import zio.{Chunk, Task, ZIO} import scala.collection.immutable.{Map => ScalaMap} @@ -52,6 +55,51 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: GetByQuery => executeGetByQuery(r) } + def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + sendRequestWithCustomResponse( + request + .post( + uri"${config.uri}/${r.index}/$Search".withParams(("scroll", "1m")) + ) + .response(asJson[ElasticQueryResponse]) + .contentType(ApplicationJson) + .body(r.query.toJson) + ).flatMap { response => + response.code match { + case HttpOk => + response.body.fold( + e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), + value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) + ) + case _ => + ZIO.fail(createElasticExceptionFromCustomResponse(response)) + } + } + + def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + sendRequestWithCustomResponse( + request + .post(uri"${config.uri}/$Search/scroll") + .response(asJson[ElasticQueryResponse]) + .contentType(ApplicationJson) + .body(Obj("scroll_id" -> Str(scrollId))) + ).flatMap { response => + response.code match { + case HttpOk => + response.body.fold( + e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), + value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) + ) + case _ => + ZIO.fail(createElasticExceptionFromCustomResponse(response)) + } + } + + def stream(r: GetByQuery): ZStream[Any, Throwable, Json] = + ZStream.paginateChunkZIO("") { s => + if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) + } + private def executeBulk(r: Bulk): Task[Unit] = { val uri = (r.index match { case Some(index) => uri"${config.uri}/$index/$Bulk" diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala index dd3749354..de9101acb 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala @@ -20,6 +20,7 @@ import zio.Random.nextUUID import zio.elasticsearch.ElasticRequest._ import zio.json.ast.Json import zio.stm.{STM, TMap, USTM, ZSTM} +import zio.stream.ZStream import zio.{Task, ZIO} private[elasticsearch] final case class TestExecutor private (data: TMap[IndexName, TMap[DocumentId, Document]]) @@ -52,6 +53,8 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa fakeGetByQuery(index) } + override def stream(request: GetByQuery): ZStream[Any, Throwable, Json] = ??? + private def fakeBulk(requests: List[BulkableRequest[_]]): Task[Unit] = ZIO.attempt { requests.map { r => From b094ab3f2827395c24d4e759f3faa15d349143d1 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 2 Mar 2023 16:13:47 +0100 Subject: [PATCH 08/14] Fix issues with pagination on stream --- .../zio/elasticsearch/HttpExecutorSpec.scala | 40 ++++++++- .../elasticsearch/HttpElasticExecutor.scala | 90 +++++++++++-------- 2 files changed, 89 insertions(+), 41 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 6310e80cc..0a7cd8fa9 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -19,11 +19,15 @@ package zio.elasticsearch import zio.Chunk import zio.elasticsearch.ElasticQuery._ import zio.json.ast.Json -import zio.stream.ZSink +import zio.schema.Schema +import zio.schema.codec.DecodeError +import zio.stream.{ZPipeline, ZSink} import zio.test.Assertion._ import zio.test.TestAspect._ import zio.test._ +import scala.util.Random + object HttpExecutorSpec extends IntegrationSpec { def spec: Spec[TestEnvironment, Any] = { @@ -289,7 +293,7 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ) ) @@ shrinks(0), - suite("searching documents")( + suite("searching documents and returning them as ZStream")( test("search for document using range query") { checkOnce(genDocumentId, genCustomer, genDocumentId, genCustomer) { (firstDocumentId, firstCustomer, secondDocumentId, secondCustomer) => @@ -315,6 +319,38 @@ object HttpExecutorSpec extends IntegrationSpec { } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex, None)), ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie + ), + test("search for document using range query with multiple pages") { + checkOnce(genCustomer) { customer => + def sink: ZSink[Any, Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = + ZSink.collectAll[CustomerDocument] + + val pipeline: ZPipeline[Any, Nothing, Json, Document] = ZPipeline.map(Document.from) + def pipeline2[A: Schema]: ZPipeline[Any, Nothing, Document, Either[DecodeError, A]] = + ZPipeline.map(_.decode) + def pipeline3[A]: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight + + for { + _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) + reqs = (0 to 50).map { _ => + ElasticRequest.create[CustomerDocument]( + secondSearchIndex, + customer.copy(id = Random.alphanumeric.take(5).mkString, balance = 150) + ) + } + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*).refreshTrue) + query = range("balance").gte(100) + res <- (ElasticExecutor.stream( + ElasticRequest.search(secondSearchIndex, query) + ) >>> pipeline >>> pipeline2[CustomerDocument] >>> pipeline3[CustomerDocument]).run(sink) + } yield assert(res)(hasSize(Assertion.equalTo(204))) + } + } @@ around( + ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie ) ) @@ shrinks(0), suite("deleting by query")( diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index 65fb8fa6a..b14fddc13 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -55,45 +55,15 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: GetByQuery => executeGetByQuery(r) } - def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = - sendRequestWithCustomResponse( - request - .post( - uri"${config.uri}/${r.index}/$Search".withParams(("scroll", "1m")) - ) - .response(asJson[ElasticQueryResponse]) - .contentType(ApplicationJson) - .body(r.query.toJson) - ).flatMap { response => - response.code match { - case HttpOk => - response.body.fold( - e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) - ) - case _ => - ZIO.fail(createElasticExceptionFromCustomResponse(response)) - } - } - - def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = - sendRequestWithCustomResponse( - request - .post(uri"${config.uri}/$Search/scroll") - .response(asJson[ElasticQueryResponse]) - .contentType(ApplicationJson) - .body(Obj("scroll_id" -> Str(scrollId))) - ).flatMap { response => - response.code match { - case HttpOk => - response.body.fold( - e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) - ) - case _ => - ZIO.fail(createElasticExceptionFromCustomResponse(response)) - } - } +// def stream[A: Schema](r: GetByQuery): ZStream[Any, Throwable, A] = { +// val pipeline: ZPipeline[Any, Nothing, Json, Document] = ZPipeline.map(Document.from) +// val pipeline2: ZPipeline[Any, Nothing, Document, Either[DecodeError, A]] = ZPipeline.map(_.decode) +// val pipeline3: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight +// +// ZStream.paginateChunkZIO("") { s => +// if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) +// } >>> pipeline >>> pipeline2 >>> pipeline3 +// } def stream(r: GetByQuery): ZStream[Any, Throwable, Json] = ZStream.paginateChunkZIO("") { s => @@ -278,6 +248,48 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } + private def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + sendRequestWithCustomResponse( + request + .post( + uri"${config.uri}/${r.index}/$Search".withParams(("scroll", "1m")) + ) + .response(asJson[ElasticQueryResponse]) + .contentType(ApplicationJson) + .body(r.query.toJson) + ).flatMap { response => + response.code match { + case HttpOk => + response.body.fold( + e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), + value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) + ) + case _ => + ZIO.fail(createElasticExceptionFromCustomResponse(response)) + } + } + + private def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + sendRequestWithCustomResponse( + request + .post(uri"${config.uri}/$Search/scroll".withParams(("scroll", "1m"))) + .response(asJson[ElasticQueryResponse]) + .contentType(ApplicationJson) + .body(Obj("scroll_id" -> Str(scrollId))) + ).flatMap { response => + response.code match { + case HttpOk => + response.body.fold( + e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), + value => + if (value.results.isEmpty) ZIO.succeed((Chunk.empty, None)) + else ZIO.succeed((Chunk.fromIterable(value.results), Some(scrollId))) + ) + case _ => + ZIO.fail(createElasticExceptionFromCustomResponse(response)) + } + } + private def sendRequest( req: RequestT[Identity, Either[String, String], Any] ): Task[Response[Either[String, String]]] = From 271edf2c158fe5ffe799953072943192828625c6 Mon Sep 17 00:00:00 2001 From: markaya Date: Thu, 2 Mar 2023 16:28:29 +0100 Subject: [PATCH 09/14] Add raw item --- .../scala/zio/elasticsearch/HttpExecutorSpec.scala | 12 +++++------- .../scala/zio/elasticsearch/ElasticExecutor.scala | 5 ++--- .../zio/elasticsearch/HttpElasticExecutor.scala | 11 +++++------ .../src/main/scala/zio/elasticsearch/RawItem.scala | 10 ++++++++++ .../main/scala/zio/elasticsearch/TestExecutor.scala | 2 +- 5 files changed, 23 insertions(+), 17 deletions(-) create mode 100644 modules/library/src/main/scala/zio/elasticsearch/RawItem.scala diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 0a7cd8fa9..3f2e47707 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -18,7 +18,6 @@ package zio.elasticsearch import zio.Chunk import zio.elasticsearch.ElasticQuery._ -import zio.json.ast.Json import zio.schema.Schema import zio.schema.codec.DecodeError import zio.stream.{ZPipeline, ZSink} @@ -297,7 +296,7 @@ object HttpExecutorSpec extends IntegrationSpec { test("search for document using range query") { checkOnce(genDocumentId, genCustomer, genDocumentId, genCustomer) { (firstDocumentId, firstCustomer, secondDocumentId, secondCustomer) => - val sink: ZSink[Any, Throwable, Json, Nothing, Chunk[Json]] = ZSink.collectAll[Json] + val sink: ZSink[Any, Throwable, RawItem, Nothing, Chunk[RawItem]] = ZSink.collectAll[RawItem] for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) @@ -325,10 +324,9 @@ object HttpExecutorSpec extends IntegrationSpec { def sink: ZSink[Any, Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = ZSink.collectAll[CustomerDocument] - val pipeline: ZPipeline[Any, Nothing, Json, Document] = ZPipeline.map(Document.from) - def pipeline2[A: Schema]: ZPipeline[Any, Nothing, Document, Either[DecodeError, A]] = - ZPipeline.map(_.decode) - def pipeline3[A]: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight + def pipeline[A: Schema]: ZPipeline[Any, Nothing, RawItem, Either[DecodeError, A]] = + ZPipeline.map(_.documentAs[A]) + def pipeline_2[A]: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) @@ -345,7 +343,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = range("balance").gte(100) res <- (ElasticExecutor.stream( ElasticRequest.search(secondSearchIndex, query) - ) >>> pipeline >>> pipeline2[CustomerDocument] >>> pipeline3[CustomerDocument]).run(sink) + ) >>> pipeline[CustomerDocument] >>> pipeline_2[CustomerDocument]).run(sink) } yield assert(res)(hasSize(Assertion.equalTo(204))) } } @@ around( diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index c61d0e09d..12664c79f 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -18,14 +18,13 @@ package zio.elasticsearch import sttp.client3.SttpBackend import zio.elasticsearch.ElasticRequest.GetByQuery -import zio.json.ast.Json import zio.stream.ZStream import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { def execute[A](request: ElasticRequest[A]): Task[A] - def stream(request: GetByQuery): ZStream[Any, Throwable, Json] + def stream(request: GetByQuery): ZStream[Any, Throwable, RawItem] } object ElasticExecutor { @@ -38,6 +37,6 @@ object ElasticExecutor { private[elasticsearch] def execute[A](request: ElasticRequest[A]): RIO[ElasticExecutor, A] = ZIO.serviceWithZIO[ElasticExecutor](_.execute(request)) - private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, Json] = + private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, RawItem] = ZStream.serviceWithStream[ElasticExecutor](_.stream(request)) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index b14fddc13..fe295312b 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -28,7 +28,6 @@ import sttp.model.StatusCode.{ } import zio.ZIO.logDebug import zio.elasticsearch.ElasticRequest._ -import zio.json.ast.Json import zio.json.ast.Json.{Obj, Str} import zio.stream.ZStream import zio.{Chunk, Task, ZIO} @@ -65,7 +64,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC // } >>> pipeline >>> pipeline2 >>> pipeline3 // } - def stream(r: GetByQuery): ZStream[Any, Throwable, Json] = + def stream(r: GetByQuery): ZStream[Any, Throwable, RawItem] = ZStream.paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) } @@ -248,7 +247,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + private def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[RawItem], Option[String])] = sendRequestWithCustomResponse( request .post( @@ -262,14 +261,14 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case HttpOk => response.body.fold( e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed((Chunk.fromIterable(value.results), value.scrollId)) + value => ZIO.succeed((Chunk.fromIterable(value.results).map(RawItem), value.scrollId)) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) } } - private def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[Json], Option[String])] = + private def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[RawItem], Option[String])] = sendRequestWithCustomResponse( request .post(uri"${config.uri}/$Search/scroll".withParams(("scroll", "1m"))) @@ -283,7 +282,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), value => if (value.results.isEmpty) ZIO.succeed((Chunk.empty, None)) - else ZIO.succeed((Chunk.fromIterable(value.results), Some(scrollId))) + else ZIO.succeed((Chunk.fromIterable(value.results).map(RawItem), Some(scrollId))) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) diff --git a/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala b/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala new file mode 100644 index 000000000..c99dc1858 --- /dev/null +++ b/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala @@ -0,0 +1,10 @@ +package zio.elasticsearch + +import zio.json.ast.Json +import zio.schema.Schema +import zio.schema.codec.DecodeError +import zio.schema.codec.JsonCodec.JsonDecoder + +final case class RawItem(raw: Json) { + def documentAs[A](implicit schema: Schema[A]): Either[DecodeError, A] = JsonDecoder.decode(schema, raw.toString) +} diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala index de9101acb..ccd153dc4 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala @@ -53,7 +53,7 @@ private[elasticsearch] final case class TestExecutor private (data: TMap[IndexNa fakeGetByQuery(index) } - override def stream(request: GetByQuery): ZStream[Any, Throwable, Json] = ??? + override def stream(request: GetByQuery): ZStream[Any, Throwable, RawItem] = ??? private def fakeBulk(requests: List[BulkableRequest[_]]): Task[Unit] = ZIO.attempt { From 37555ab740f4c24dbbeefdb170be78d350da4e10 Mon Sep 17 00:00:00 2001 From: markaya Date: Fri, 3 Mar 2023 12:36:06 +0100 Subject: [PATCH 10/14] Fix code remarks and add typed stream --- .../example/RepositoriesElasticsearch.scala | 7 +- .../zio/elasticsearch/HttpExecutorSpec.scala | 71 +++++--- .../scala/zio/elasticsearch/Document.scala | 10 +- .../zio/elasticsearch/ElasticExecutor.scala | 10 +- .../elasticsearch/ElasticQueryResponse.scala | 8 +- .../zio/elasticsearch/ElasticResult.scala | 18 +- .../elasticsearch/HttpElasticExecutor.scala | 54 +++--- .../main/scala/zio/elasticsearch/Item.scala | 26 +++ .../scala/zio/elasticsearch/RawItem.scala | 10 -- .../zio/elasticsearch/TestExecutor.scala | 157 ------------------ .../scala/zio/elasticsearch/package.scala | 14 +- .../HttpElasticExecutorSpec.scala | 4 +- 12 files changed, 137 insertions(+), 252 deletions(-) create mode 100644 modules/library/src/main/scala/zio/elasticsearch/Item.scala delete mode 100644 modules/library/src/main/scala/zio/elasticsearch/RawItem.scala delete mode 100644 modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala diff --git a/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala b/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala index bc44a98bf..e2ee2f4eb 100644 --- a/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala +++ b/modules/example/src/main/scala/example/RepositoriesElasticsearch.scala @@ -32,12 +32,13 @@ import zio.prelude.Newtype.unsafeWrap final case class RepositoriesElasticsearch(elasticsearch: Elasticsearch) { def findAll(): Task[List[GitHubRepo]] = - elasticsearch.execute(ElasticRequest.search(Index, matchAll)).result[GitHubRepo] + elasticsearch.execute(ElasticRequest.search(Index, matchAll)).documentAs[GitHubRepo] def findById(organization: String, id: String): Task[Option[GitHubRepo]] = for { routing <- routingOf(organization) - res <- elasticsearch.execute(ElasticRequest.getById(Index, DocumentId(id)).routing(routing)).result[GitHubRepo] + res <- + elasticsearch.execute(ElasticRequest.getById(Index, DocumentId(id)).routing(routing)).documentAs[GitHubRepo] } yield res def create(repository: GitHubRepo): Task[CreationOutcome] = @@ -75,7 +76,7 @@ final case class RepositoriesElasticsearch(elasticsearch: Elasticsearch) { } yield res def search(query: ElasticQuery[_]): Task[List[GitHubRepo]] = - elasticsearch.execute(ElasticRequest.search(Index, query)).result[GitHubRepo] + elasticsearch.execute(ElasticRequest.search(Index, query)).documentAs[GitHubRepo] private def routingOf(value: String): IO[IllegalArgumentException, Routing.Type] = Routing.make(value).toZIO.mapError(e => new IllegalArgumentException(e)) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 3f2e47707..238e1d939 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -20,7 +20,7 @@ import zio.Chunk import zio.elasticsearch.ElasticQuery._ import zio.schema.Schema import zio.schema.codec.DecodeError -import zio.stream.{ZPipeline, ZSink} +import zio.stream.{Sink, ZPipeline, ZSink} import zio.test.Assertion._ import zio.test.TestAspect._ import zio.test._ @@ -37,7 +37,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genCustomer) { customer => for { docId <- ElasticExecutor.execute(ElasticRequest.create[CustomerDocument](index, customer)) - res <- ElasticExecutor.execute(ElasticRequest.getById(index, docId)).result[CustomerDocument] + res <- ElasticExecutor.execute(ElasticRequest.getById(index, docId)).documentAs[CustomerDocument] } yield assert(res)(isSome(equalTo(customer))) } }, @@ -73,7 +73,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId, genCustomer) { (documentId, customer) => for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, customer)) - doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] + doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] } yield assert(doc)(isSome(equalTo(customer))) } }, @@ -82,7 +82,7 @@ object HttpExecutorSpec extends IntegrationSpec { for { _ <- ElasticExecutor.execute(ElasticRequest.create[CustomerDocument](index, documentId, firstCustomer)) _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, secondCustomer)) - doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] + doc <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] } yield assert(doc)(isSome(equalTo(secondCustomer))) } } @@ -137,13 +137,15 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId, genCustomer) { (documentId, customer) => for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[CustomerDocument](index, documentId, customer)) - res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] + res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] } yield assert(res)(isSome(equalTo(customer))) } }, test("return None if the document does not exist") { checkOnce(genDocumentId) { documentId => - assertZIO(ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument])( + assertZIO( + ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] + )( isNone ) } @@ -152,7 +154,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId, genEmployee) { (documentId, employee) => val result = for { _ <- ElasticExecutor.execute(ElasticRequest.upsert[EmployeeDocument](index, documentId, employee)) - res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).result[CustomerDocument] + res <- ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] } yield res assertZIO(result.exit)( @@ -179,7 +181,7 @@ object HttpExecutorSpec extends IntegrationSpec { ) query = range("balance").gte(100) res <- - ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] } yield assert(res)(isNonEmpty) } } @@ around( @@ -203,7 +205,9 @@ object HttpExecutorSpec extends IntegrationSpec { ) query = range("age").gte(0) res <- - ElasticExecutor.execute(ElasticRequest.search(secondSearchIndex, query)).result[CustomerDocument] + ElasticExecutor + .execute(ElasticRequest.search(secondSearchIndex, query)) + .documentAs[CustomerDocument] } yield res assertZIO(result.exit)( @@ -235,7 +239,7 @@ object HttpExecutorSpec extends IntegrationSpec { ) query = ElasticQuery.contains("name.keyword", firstCustomer.name.take(3)) res <- - ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -259,7 +263,7 @@ object HttpExecutorSpec extends IntegrationSpec { ) query = ElasticQuery.startsWith("name.keyword", firstCustomer.name.take(3)) res <- - ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -284,7 +288,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = wildcard("name.keyword", s"${firstCustomer.name.take(2)}*${firstCustomer.name.takeRight(2)}") res <- - ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).result[CustomerDocument] + ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( @@ -292,11 +296,11 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ) ) @@ shrinks(0), - suite("searching documents and returning them as ZStream")( - test("search for document using range query") { + suite("searching documents and returning them as a stream")( + test("search for documents using range query") { checkOnce(genDocumentId, genCustomer, genDocumentId, genCustomer) { (firstDocumentId, firstCustomer, secondDocumentId, secondCustomer) => - val sink: ZSink[Any, Throwable, RawItem, Nothing, Chunk[RawItem]] = ZSink.collectAll[RawItem] + val sink: Sink[Throwable, Item, Nothing, Chunk[Item]] = ZSink.collectAll[Item] for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) @@ -319,12 +323,12 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex, None)), ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ), - test("search for document using range query with multiple pages") { + test("search for documents using range query with multiple pages") { checkOnce(genCustomer) { customer => - def sink: ZSink[Any, Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = + def sink: Sink[Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = ZSink.collectAll[CustomerDocument] - def pipeline[A: Schema]: ZPipeline[Any, Nothing, RawItem, Either[DecodeError, A]] = + def pipeline[A: Schema]: ZPipeline[Any, Nothing, Item, Either[DecodeError, A]] = ZPipeline.map(_.documentAs[A]) def pipeline_2[A]: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight @@ -349,6 +353,35 @@ object HttpExecutorSpec extends IntegrationSpec { } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie + ), + test("search for documents using range query with multiple pages and return type") { + checkOnce(genCustomer) { customer => + def sink: Sink[Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = + ZSink.collectAll[CustomerDocument] + + for { + _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) + reqs = (1 to 50).map { _ => + ElasticRequest.create[CustomerDocument]( + secondSearchIndex, + customer.copy(id = Random.alphanumeric.take(5).mkString, balance = 150) + ) + } + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) + _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*).refreshTrue) + query = range("balance").gte(100) + res <- ElasticExecutor + .streamAs[CustomerDocument]( + ElasticRequest.search(secondSearchIndex, query) + ) + .run(sink) + } yield assert(res)(hasSize(Assertion.equalTo(200))) + } + } @@ around( + ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie ) ) @@ shrinks(0), suite("deleting by query")( @@ -388,7 +421,7 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticExecutor.execute(ElasticRequest.deleteByQuery(deleteByQueryIndex, deleteQuery).refreshTrue) res <- ElasticExecutor .execute(ElasticRequest.search(deleteByQueryIndex, matchAll)) - .result[CustomerDocument] + .documentAs[CustomerDocument] } yield assert(res)(hasSameElements(List(firstCustomer.copy(balance = 150)))) } } @@ around( diff --git a/modules/library/src/main/scala/zio/elasticsearch/Document.scala b/modules/library/src/main/scala/zio/elasticsearch/Document.scala index bd24d8660..c6c8f5de0 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/Document.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/Document.scala @@ -16,19 +16,13 @@ package zio.elasticsearch -import zio.json.ast.Json import zio.schema.Schema -import zio.schema.codec.JsonCodec.JsonDecoder -import zio.schema.codec.{DecodeError, JsonCodec} +import zio.schema.codec.JsonCodec -private[elasticsearch] final case class Document(json: String) { - def decode[A](implicit schema: Schema[A]): Either[DecodeError, A] = JsonDecoder.decode(schema, json) -} +private[elasticsearch] final case class Document(json: String) private[elasticsearch] object Document { def from[A](doc: A)(implicit schema: Schema[A]): Document = Document( JsonCodec.jsonEncoder(schema).encodeJson(doc, indent = None).toString ) - - def from(json: Json): Document = new Document(json.toString) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index 12664c79f..559c3f075 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -19,12 +19,15 @@ package zio.elasticsearch import sttp.client3.SttpBackend import zio.elasticsearch.ElasticRequest.GetByQuery import zio.stream.ZStream +import zio.schema.Schema import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { def execute[A](request: ElasticRequest[A]): Task[A] - def stream(request: GetByQuery): ZStream[Any, Throwable, RawItem] + def stream(request: GetByQuery): ZStream[Any, Throwable, Item] + + def streamAs[A: Schema](request: GetByQuery): ZStream[Any, Throwable, A] } object ElasticExecutor { @@ -37,6 +40,9 @@ object ElasticExecutor { private[elasticsearch] def execute[A](request: ElasticRequest[A]): RIO[ElasticExecutor, A] = ZIO.serviceWithZIO[ElasticExecutor](_.execute(request)) - private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, RawItem] = + private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, Item] = ZStream.serviceWithStream[ElasticExecutor](_.stream(request)) + + private[elasticsearch] def streamAs[A: Schema](request: GetByQuery): ZStream[ElasticExecutor, Throwable, A] = + ZStream.serviceWithStream[ElasticExecutor](_.streamAs[A](request)) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala index 8c9439679..6fb0f422e 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticQueryResponse.scala @@ -52,7 +52,7 @@ private[elasticsearch] final case class Hits( total: Total, @jsonField("max_score") maxScore: Option[Double] = None, - hits: List[Item] + hits: List[Hit] ) private[elasticsearch] object Hits { @@ -65,7 +65,7 @@ private[elasticsearch] object Total { implicit val decoder: JsonDecoder[Total] = DeriveJsonDecoder.gen[Total] } -private[elasticsearch] final case class Item( +private[elasticsearch] final case class Hit( @jsonField("_index") index: String, @jsonField("_type") @@ -78,6 +78,6 @@ private[elasticsearch] final case class Item( source: Json ) -private[elasticsearch] object Item { - implicit val decoder: JsonDecoder[Item] = DeriveJsonDecoder.gen[Item] +private[elasticsearch] object Hit { + implicit val decoder: JsonDecoder[Hit] = DeriveJsonDecoder.gen[Hit] } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala index 86e7d4708..3fe44e209 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -18,18 +18,18 @@ package zio.elasticsearch import zio.prelude.ZValidation import zio.schema.Schema -import zio.{Task, ZIO} +import zio.{IO, Task, ZIO} sealed trait ElasticResult[F[_]] { - def result[A: Schema]: Task[F[A]] + def documentAs[A: Schema]: Task[F[A]] } -final class GetResult private[elasticsearch] (private val doc: Option[Document]) extends ElasticResult[Option] { - override def result[A: Schema]: Task[Option[A]] = +final class GetResult private[elasticsearch] (private val doc: Option[Item]) extends ElasticResult[Option] { + override def documentAs[A: Schema]: IO[DecodingException, Option[A]] = ZIO .fromEither(doc match { - case Some(document) => - document.decode match { + case Some(item) => + item.documentAs match { case Left(e) => Left(DecodingException(s"Could not parse the document: ${e.message}")) case Right(doc) => Right(Some(doc)) } @@ -39,10 +39,10 @@ final class GetResult private[elasticsearch] (private val doc: Option[Document]) .mapError(e => DecodingException(s"Could not parse the document: ${e.message}")) } -final class SearchResult private[elasticsearch] (private val hits: List[Document]) extends ElasticResult[List] { - override def result[A: Schema]: Task[List[A]] = +final class SearchResult private[elasticsearch] (private val hits: List[Item]) extends ElasticResult[List] { + override def documentAs[A: Schema]: IO[DecodingException, List[A]] = ZIO.fromEither { - ZValidation.validateAll(hits.map(d => ZValidation.fromEither(d.decode))).toEitherWith { errors => + ZValidation.validateAll(hits.map(item => ZValidation.fromEither(item.documentAs))).toEitherWith { errors => DecodingException(s"Could not parse all documents successfully: ${errors.map(_.message).mkString(",")})") } } diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index fe295312b..e389145a9 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -29,6 +29,7 @@ import sttp.model.StatusCode.{ import zio.ZIO.logDebug import zio.elasticsearch.ElasticRequest._ import zio.json.ast.Json.{Obj, Str} +import zio.schema.Schema import zio.stream.ZStream import zio.{Chunk, Task, ZIO} @@ -54,21 +55,19 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: GetByQuery => executeGetByQuery(r) } -// def stream[A: Schema](r: GetByQuery): ZStream[Any, Throwable, A] = { -// val pipeline: ZPipeline[Any, Nothing, Json, Document] = ZPipeline.map(Document.from) -// val pipeline2: ZPipeline[Any, Nothing, Document, Either[DecodeError, A]] = ZPipeline.map(_.decode) -// val pipeline3: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight -// -// ZStream.paginateChunkZIO("") { s => -// if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) -// } >>> pipeline >>> pipeline2 >>> pipeline3 -// } - - def stream(r: GetByQuery): ZStream[Any, Throwable, RawItem] = + def stream(r: GetByQuery): ZStream[Any, Throwable, Item] = ZStream.paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) } + def streamAs[A: Schema](r: GetByQuery): ZStream[Any, Throwable, A] = + ZStream + .paginateChunkZIO("") { s => + if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) + } + .map(_.documentAs[A]) + .collectWhileRight + private def executeBulk(r: Bulk): Task[Unit] = { val uri = (r.index match { case Some(index) => uri"${config.uri}/$index/$Bulk" @@ -218,7 +217,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC .response(asJson[ElasticGetResponse]) ).flatMap { response => response.code match { - case HttpOk => ZIO.attempt(new GetResult(response.body.toOption.map(d => Document.from(d.source)))) + case HttpOk => ZIO.attempt(new GetResult(response.body.toOption.map(r => Item(r.source)))) case HttpNotFound => ZIO.succeed(new GetResult(None)) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) } @@ -239,19 +238,19 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), value => ZIO.succeed( - new SearchResult(value.results.map(_.toString).map(Document(_))) - ) // TODO have Json in Document instead of String??? + new SearchResult(value.results.map(Item)) + ) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) } } - private def executeGetByQueryWithScroll(r: GetByQuery): ZIO[Any, Throwable, (Chunk[RawItem], Option[String])] = + private def executeGetByQueryWithScroll(r: GetByQuery): Task[(Chunk[Item], Option[String])] = sendRequestWithCustomResponse( request .post( - uri"${config.uri}/${r.index}/$Search".withParams(("scroll", "1m")) + uri"${config.uri}/${r.index}/$Search".withParams((Scroll, ScrollDefaultDuration)) ) .response(asJson[ElasticQueryResponse]) .contentType(ApplicationJson) @@ -261,20 +260,20 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case HttpOk => response.body.fold( e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => ZIO.succeed((Chunk.fromIterable(value.results).map(RawItem), value.scrollId)) + value => ZIO.succeed((Chunk.fromIterable(value.results).map(Item), value.scrollId)) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) } } - private def executeGetByScroll(scrollId: String): ZIO[Any, Throwable, (Chunk[RawItem], Option[String])] = + private def executeGetByScroll(scrollId: String): Task[(Chunk[Item], Option[String])] = sendRequestWithCustomResponse( request - .post(uri"${config.uri}/$Search/scroll".withParams(("scroll", "1m"))) + .post(uri"${config.uri}/$Search/$Scroll".withParams((Scroll, ScrollDefaultDuration))) .response(asJson[ElasticQueryResponse]) .contentType(ApplicationJson) - .body(Obj("scroll_id" -> Str(scrollId))) + .body(Obj(ScrollId -> Str(scrollId))) ).flatMap { response => response.code match { case HttpOk => @@ -282,7 +281,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), value => if (value.results.isEmpty) ZIO.succeed((Chunk.empty, None)) - else ZIO.succeed((Chunk.fromIterable(value.results).map(RawItem), Some(scrollId))) + else ZIO.succeed((Chunk.fromIterable(value.results).map(Item), value.scrollId.orElse(Some(scrollId)))) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) @@ -326,11 +325,14 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC private[elasticsearch] object HttpElasticExecutor { - private final val Bulk = "_bulk" - private final val Create = "_create" - private final val DeleteByQuery = "_delete_by_query" - private final val Doc = "_doc" - private final val Search = "_search" + private final val Bulk = "_bulk" + private final val Create = "_create" + private final val DeleteByQuery = "_delete_by_query" + private final val Doc = "_doc" + private final val Search = "_search" + private final val Scroll = "scroll" + private final val ScrollId = "scroll_id" + private final val ScrollDefaultDuration = "1m" def apply(config: ElasticConfig, client: SttpBackend[Task, Any]) = new HttpElasticExecutor(config, client) diff --git a/modules/library/src/main/scala/zio/elasticsearch/Item.scala b/modules/library/src/main/scala/zio/elasticsearch/Item.scala new file mode 100644 index 000000000..760e37c79 --- /dev/null +++ b/modules/library/src/main/scala/zio/elasticsearch/Item.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2022 LambdaWorks + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package zio.elasticsearch + +import zio.json.ast.Json +import zio.schema.Schema +import zio.schema.codec.DecodeError +import zio.schema.codec.JsonCodec.JsonDecoder + +final case class Item(raw: Json) { + def documentAs[A](implicit schema: Schema[A]): Either[DecodeError, A] = JsonDecoder.decode(schema, raw.toString) +} diff --git a/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala b/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala deleted file mode 100644 index c99dc1858..000000000 --- a/modules/library/src/main/scala/zio/elasticsearch/RawItem.scala +++ /dev/null @@ -1,10 +0,0 @@ -package zio.elasticsearch - -import zio.json.ast.Json -import zio.schema.Schema -import zio.schema.codec.DecodeError -import zio.schema.codec.JsonCodec.JsonDecoder - -final case class RawItem(raw: Json) { - def documentAs[A](implicit schema: Schema[A]): Either[DecodeError, A] = JsonDecoder.decode(schema, raw.toString) -} diff --git a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala deleted file mode 100644 index ccd153dc4..000000000 --- a/modules/library/src/main/scala/zio/elasticsearch/TestExecutor.scala +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2022 LambdaWorks - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package zio.elasticsearch - -import zio.Random.nextUUID -import zio.elasticsearch.ElasticRequest._ -import zio.json.ast.Json -import zio.stm.{STM, TMap, USTM, ZSTM} -import zio.stream.ZStream -import zio.{Task, ZIO} - -private[elasticsearch] final case class TestExecutor private (data: TMap[IndexName, TMap[DocumentId, Document]]) - extends ElasticExecutor { - self => - - def execute[A](request: ElasticRequest[A]): Task[A] = - request match { - case Bulk(requests, _, _, _) => - fakeBulk(requests) - case Create(index, document, _, _) => - fakeCreate(index, document) - case CreateWithId(index, id, document, _, _) => - fakeCreateWithId(index, id, document) - case CreateIndex(name, _) => - fakeCreateIndex(name) - case CreateOrUpdate(index, id, document, _, _) => - fakeCreateOrUpdate(index, id, document) - case DeleteById(index, id, _, _) => - fakeDeleteById(index, id) - case DeleteByQuery(index, _, _, _) => - fakeDeleteByQuery(index) - case DeleteIndex(name) => - fakeDeleteIndex(name) - case Exists(index, id, _) => - fakeExists(index, id) - case GetById(index, id, _) => - fakeGetById(index, id) - case GetByQuery(index, _, _) => - fakeGetByQuery(index) - } - - override def stream(request: GetByQuery): ZStream[Any, Throwable, RawItem] = ??? - - private def fakeBulk(requests: List[BulkableRequest[_]]): Task[Unit] = - ZIO.attempt { - requests.map { r => - execute(r) - } - }.unit - - private def fakeCreate(index: IndexName, document: Document): Task[DocumentId] = - for { - uuid <- nextUUID - documents <- getDocumentsFromIndex(index).commit - documentId = DocumentId(uuid.toString) - _ <- documents.put(documentId, document).commit - } yield documentId - - private def fakeCreateWithId(index: IndexName, documentId: DocumentId, document: Document): Task[CreationOutcome] = - (for { - documents <- getDocumentsFromIndex(index) - alreadyExists <- documents.contains(documentId) - _ <- documents.putIfAbsent(documentId, document) - } yield if (alreadyExists) AlreadyExists else Created).commit - - private def fakeCreateIndex(index: IndexName): Task[CreationOutcome] = - (for { - alreadyExists <- self.data.contains(index) - emptyDocuments <- TMap.empty[DocumentId, Document] - _ <- self.data.putIfAbsent(index, emptyDocuments) - } yield if (alreadyExists) AlreadyExists else Created).commit - - private def fakeCreateOrUpdate(index: IndexName, documentId: DocumentId, document: Document): Task[Unit] = - (for { - documents <- getDocumentsFromIndex(index) - _ <- documents.put(documentId, document) - } yield ()).commit - - private def fakeDeleteById(index: IndexName, documentId: DocumentId): Task[DeletionOutcome] = - (for { - documents <- getDocumentsFromIndex(index) - exists <- documents.contains(documentId) - _ <- documents.delete(documentId) - } yield if (exists) Deleted else NotFound).commit - - private def fakeDeleteByQuery(index: IndexName): Task[DeletionOutcome] = - (for { - exists <- self.data.contains(index) - } yield if (exists) Deleted else NotFound).commit - // until we have a way of using query to delete we can either delete all or delete none documents - - private def fakeDeleteIndex(index: IndexName): Task[DeletionOutcome] = - (for { - exists <- self.data.contains(index) - _ <- self.data.delete(index) - } yield if (exists) Deleted else NotFound).commit - - private def fakeExists(index: IndexName, documentId: DocumentId): Task[Boolean] = - (for { - documents <- getDocumentsFromIndex(index) - exists <- documents.contains(documentId) - } yield exists).commit - - private def fakeGetById(index: IndexName, documentId: DocumentId): Task[GetResult] = - (for { - documents <- getDocumentsFromIndex(index) - maybeDocument <- documents.get(documentId) - } yield new GetResult(maybeDocument)).commit - - private def fakeGetByQuery(index: IndexName): Task[SearchResult] = { - def createSearchResult( - index: IndexName, - documents: TMap[DocumentId, Document] - ): USTM[SearchResult] = - for { - items <- - documents.toList.map( - _.map { case (id, document) => - Item( - index = index.toString, - `type` = "type", - id = id.toString, - score = 1, - source = Json.Str(document.json) - ) - } - ) - } yield new SearchResult(items.map(_.source.toString).map(Document(_))) - - (for { - documents <- getDocumentsFromIndex(index) - response <- createSearchResult(index, documents) - } yield response).commit - } - - private def getDocumentsFromIndex(index: IndexName): ZSTM[Any, ElasticException, TMap[DocumentId, Document]] = - for { - maybeDocuments <- self.data.get(index) - documents <- maybeDocuments.fold[STM[ElasticException, TMap[DocumentId, Document]]]( - STM.fail[ElasticException](new ElasticException(s"Index $index does not exists!")) - )(STM.succeed(_)) - } yield documents -} diff --git a/modules/library/src/main/scala/zio/elasticsearch/package.scala b/modules/library/src/main/scala/zio/elasticsearch/package.scala index 63967b9f7..78c8d0466 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/package.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/package.scala @@ -62,17 +62,7 @@ package object elasticsearch { def containsAny(name: String, params: List[String]): Boolean = params.exists(StringUtils.contains(name, _)) - /*// TODO decide if this extension is favorable to avoid having an additional flatMap in user code - final implicit class ResultRIO[R, F[_]](zio: RIO[R, ElasticResult[F]]) { - def result[A: Schema]: RIO[R, F[A]] = zio.flatMap(_.result[A]) - } - - // TODO decide if this extension is favorable to avoid having an additional flatMap in user code - final implicit class ResultTask[F[_]](zio: Task[ElasticResult[F]]) { - def result[A: Schema]: Task[F[A]] = zio.flatMap(_.result[A]) - }*/ - - final implicit class Result[R, F[_]](zio: ZIO[R, Throwable, ElasticResult[F]]) { - def result[A: Schema]: ZIO[R, Throwable, F[A]] = zio.flatMap(_.result[A]) + final implicit class ZIOResultOps[R, F[_]](zio: RIO[R, ElasticResult[F]]) { + def documentAs[A: Schema]: RIO[R, F[A]] = zio.flatMap(_.documentAs[A]) } } diff --git a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala index 6ebff2481..1ebb33b70 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/HttpElasticExecutorSpec.scala @@ -312,7 +312,7 @@ object HttpElasticExecutorSpec extends WireMockSpec { .getById(index = index, id = DocumentId("V4x8q4UB3agN0z75fv5r")) .routing(Routing("routing")) ) - .result[GitHubRepo] + .documentAs[GitHubRepo] )(isSome(equalTo(repo))) }, test("getting by query request") { @@ -365,7 +365,7 @@ object HttpElasticExecutorSpec extends WireMockSpec { assertZIO( addStubMapping *> ElasticExecutor .execute(ElasticRequest.search(index = index, query = matchAll)) - .result[GitHubRepo] + .documentAs[GitHubRepo] )( equalTo(List(repo)) ) From 03b1f7e76ade476551236a972e934ba7e851bd43 Mon Sep 17 00:00:00 2001 From: markaya Date: Fri, 3 Mar 2023 12:49:19 +0100 Subject: [PATCH 11/14] Fix linter --- .../src/main/scala/zio/elasticsearch/ElasticExecutor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index 559c3f075..2213ee491 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -18,8 +18,8 @@ package zio.elasticsearch import sttp.client3.SttpBackend import zio.elasticsearch.ElasticRequest.GetByQuery -import zio.stream.ZStream import zio.schema.Schema +import zio.stream.ZStream import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { From 54ca8d68785fefeccb0189c60351d1797bd2edfd Mon Sep 17 00:00:00 2001 From: markaya Date: Fri, 3 Mar 2023 14:37:49 +0100 Subject: [PATCH 12/14] Fix code remarks --- .../zio/elasticsearch/HttpExecutorSpec.scala | 40 ++++++---------- .../zio/elasticsearch/ElasticExecutor.scala | 6 +-- .../zio/elasticsearch/ElasticRequest.scala | 48 +++++++++---------- .../zio/elasticsearch/ElasticResult.scala | 4 +- .../elasticsearch/HttpElasticExecutor.scala | 13 ++--- 5 files changed, 48 insertions(+), 63 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 238e1d939..151c381ba 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -18,9 +18,7 @@ package zio.elasticsearch import zio.Chunk import zio.elasticsearch.ElasticQuery._ -import zio.schema.Schema -import zio.schema.codec.DecodeError -import zio.stream.{Sink, ZPipeline, ZSink} +import zio.stream.{Sink, ZSink} import zio.test.Assertion._ import zio.test.TestAspect._ import zio.test._ @@ -240,7 +238,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = ElasticQuery.contains("name.keyword", firstCustomer.name.take(3)) res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(Assertion.contains(firstCustomer)) + } yield assert(res)(contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -264,7 +262,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = ElasticQuery.startsWith("name.keyword", firstCustomer.name.take(3)) res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(Assertion.contains(firstCustomer)) + } yield assert(res)(contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -289,7 +287,7 @@ object HttpExecutorSpec extends IntegrationSpec { wildcard("name.keyword", s"${firstCustomer.name.take(2)}*${firstCustomer.name.takeRight(2)}") res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(Assertion.contains(firstCustomer)) + } yield assert(res)(contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -325,30 +323,25 @@ object HttpExecutorSpec extends IntegrationSpec { ), test("search for documents using range query with multiple pages") { checkOnce(genCustomer) { customer => - def sink: Sink[Throwable, CustomerDocument, Nothing, Chunk[CustomerDocument]] = - ZSink.collectAll[CustomerDocument] - - def pipeline[A: Schema]: ZPipeline[Any, Nothing, Item, Either[DecodeError, A]] = - ZPipeline.map(_.documentAs[A]) - def pipeline_2[A]: ZPipeline[Any, Nothing, Either[DecodeError, A], A] = ZPipeline.collectWhileRight + def sink: Sink[Throwable, Item, Nothing, Chunk[Item]] = + ZSink.collectAll[Item] for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) - reqs = (0 to 50).map { _ => + reqs = (0 to 203).map { _ => ElasticRequest.create[CustomerDocument]( secondSearchIndex, customer.copy(id = Random.alphanumeric.take(5).mkString, balance = 150) ) } - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*).refreshTrue) query = range("balance").gte(100) - res <- (ElasticExecutor.stream( - ElasticRequest.search(secondSearchIndex, query) - ) >>> pipeline[CustomerDocument] >>> pipeline_2[CustomerDocument]).run(sink) - } yield assert(res)(hasSize(Assertion.equalTo(204))) + res <- ElasticExecutor + .stream( + ElasticRequest.search(secondSearchIndex, query) + ) + .run(sink) + } yield assert(res)(hasSize(equalTo(204))) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), @@ -361,15 +354,12 @@ object HttpExecutorSpec extends IntegrationSpec { for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) - reqs = (1 to 50).map { _ => + reqs = (0 to 200).map { _ => ElasticRequest.create[CustomerDocument]( secondSearchIndex, customer.copy(id = Random.alphanumeric.take(5).mkString, balance = 150) ) } - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) - _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*)) _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*).refreshTrue) query = range("balance").gte(100) res <- ElasticExecutor @@ -377,7 +367,7 @@ object HttpExecutorSpec extends IntegrationSpec { ElasticRequest.search(secondSearchIndex, query) ) .run(sink) - } yield assert(res)(hasSize(Assertion.equalTo(200))) + } yield assert(res)(hasSize(equalTo(201))) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index 2213ee491..9466944de 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -19,15 +19,15 @@ package zio.elasticsearch import sttp.client3.SttpBackend import zio.elasticsearch.ElasticRequest.GetByQuery import zio.schema.Schema -import zio.stream.ZStream +import zio.stream.{Stream, ZStream} import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { def execute[A](request: ElasticRequest[A]): Task[A] - def stream(request: GetByQuery): ZStream[Any, Throwable, Item] + def stream(request: GetByQuery): Stream[Throwable, Item] - def streamAs[A: Schema](request: GetByQuery): ZStream[Any, Throwable, A] + def streamAs[A: Schema](request: GetByQuery): Stream[Throwable, A] } object ElasticExecutor { diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index 240373e05..e0d82ae5f 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -19,7 +19,7 @@ package zio.elasticsearch import zio.elasticsearch.Routing.Routing import zio.schema.Schema -trait HasRefresh[A] { // TODO extend this on all appropritate requests +trait HasRefresh[A] { def refresh(value: Boolean): ElasticRequest[A] def refreshFalse: ElasticRequest[A] @@ -27,7 +27,7 @@ trait HasRefresh[A] { // TODO extend this on all appropritate requests def refreshTrue: ElasticRequest[A] } -trait HasRouting[A] { // TODO extend this on all appropritate requests +trait HasRouting[A] { def routing(value: Routing): ElasticRequest[A] } @@ -38,7 +38,7 @@ sealed trait BulkableRequest[A] extends ElasticRequest[A] object ElasticRequest { def bulk(requests: BulkableRequest[_]*): Bulk = - Bulk.of(requests: _*) + Bulk.of(requests = requests: _*) def create[A: Schema](index: IndexName, doc: A): Create = Create(index = index, document = Document.from(doc), refresh = false, routing = None) @@ -59,7 +59,7 @@ object ElasticRequest { DeleteByQuery(index = index, query = query, refresh = false, routing = None) def deleteIndex(name: IndexName): DeleteIndex = - DeleteIndex(name) + DeleteIndex(name = name) def exists(index: IndexName, id: DocumentId): Exists = Exists(index = index, id = id, routing = None) @@ -73,7 +73,7 @@ object ElasticRequest { def upsert[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateOrUpdate = CreateOrUpdate(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) - sealed trait BulkRequest extends ElasticRequest[Unit] with HasRouting[Unit] with HasRefresh[Unit] + sealed trait BulkRequest extends ElasticRequest[Unit] with HasRefresh[Unit] with HasRouting[Unit] private[elasticsearch] final case class Bulk( requests: List[BulkableRequest[_]], @@ -81,17 +81,15 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends BulkRequest { self => - def routing(value: Routing): Bulk = self.copy(routing = Some(value)) - def refresh(value: Boolean): Bulk = self.copy(refresh = value) def refreshFalse: Bulk = refresh(false) def refreshTrue: Bulk = refresh(true) + def routing(value: Routing): Bulk = self.copy(routing = Some(value)) + lazy val body: String = requests.flatMap { r => - // We use @unchecked to ignore 'pattern match not exhaustive' error since we guarantee that it will not happen - // because these are only Bulkable Requests and other matches will not occur. r match { case Create(index, document, _, maybeRouting) => List(getActionAndMeta("create", List(("_index", Some(index)), ("routing", maybeRouting))), document.json) @@ -116,7 +114,7 @@ object ElasticRequest { Bulk(requests = requests.toList, index = None, refresh = false, routing = None) } - sealed trait CreateRequest extends BulkableRequest[DocumentId] with HasRouting[DocumentId] with HasRefresh[DocumentId] + sealed trait CreateRequest extends BulkableRequest[DocumentId] with HasRefresh[DocumentId] with HasRouting[DocumentId] private[elasticsearch] final case class Create( index: IndexName, @@ -124,19 +122,19 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateRequest { self => - def routing(value: Routing): Create = self.copy(routing = Some(value)) - def refresh(value: Boolean): Create = self.copy(refresh = value) def refreshFalse: Create = refresh(false) def refreshTrue: Create = refresh(true) + + def routing(value: Routing): Create = self.copy(routing = Some(value)) } sealed trait CreateWithIdRequest extends BulkableRequest[CreationOutcome] - with HasRouting[CreationOutcome] with HasRefresh[CreationOutcome] + with HasRouting[CreationOutcome] private[elasticsearch] final case class CreateWithId( index: IndexName, @@ -145,13 +143,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateWithIdRequest { self => - def routing(value: Routing): CreateWithId = self.copy(routing = Some(value)) - def refresh(value: Boolean): CreateWithId = self.copy(refresh = value) def refreshFalse: CreateWithId = refresh(false) def refreshTrue: CreateWithId = refresh(true) + + def routing(value: Routing): CreateWithId = self.copy(routing = Some(value)) } sealed trait CreateIndexRequest extends ElasticRequest[CreationOutcome] @@ -161,7 +159,7 @@ object ElasticRequest { definition: Option[String] ) extends ElasticRequest[CreationOutcome] - sealed trait CreateOrUpdateRequest extends BulkableRequest[Unit] with HasRouting[Unit] with HasRefresh[Unit] + sealed trait CreateOrUpdateRequest extends BulkableRequest[Unit] with HasRefresh[Unit] with HasRouting[Unit] private[elasticsearch] final case class CreateOrUpdate( index: IndexName, @@ -170,19 +168,19 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends CreateOrUpdateRequest { self => - def routing(value: Routing): CreateOrUpdate = self.copy(routing = Some(value)) - def refresh(value: Boolean): CreateOrUpdate = self.copy(refresh = value) def refreshFalse: CreateOrUpdate = refresh(false) def refreshTrue: CreateOrUpdate = refresh(true) + + def routing(value: Routing): CreateOrUpdate = self.copy(routing = Some(value)) } sealed trait DeleteByIdRequest extends BulkableRequest[DeletionOutcome] - with HasRouting[DeletionOutcome] with HasRefresh[DeletionOutcome] + with HasRouting[DeletionOutcome] private[elasticsearch] final case class DeleteById( index: IndexName, @@ -190,19 +188,19 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByIdRequest { self => - def routing(value: Routing): DeleteById = self.copy(routing = Some(value)) - def refresh(value: Boolean): DeleteById = self.copy(refresh = value) def refreshFalse: DeleteById = refresh(false) def refreshTrue: DeleteById = refresh(true) + + def routing(value: Routing): DeleteById = self.copy(routing = Some(value)) } sealed trait DeleteByQueryRequest extends ElasticRequest[DeletionOutcome] - with HasRouting[DeletionOutcome] with HasRefresh[DeletionOutcome] + with HasRouting[DeletionOutcome] private[elasticsearch] final case class DeleteByQuery( index: IndexName, @@ -210,13 +208,13 @@ object ElasticRequest { refresh: Boolean, routing: Option[Routing] ) extends DeleteByQueryRequest { self => - def routing(value: Routing): DeleteByQuery = self.copy(routing = Some(value)) - def refresh(value: Boolean): DeleteByQuery = self.copy(refresh = value) def refreshFalse: DeleteByQuery = refresh(false) def refreshTrue: DeleteByQuery = refresh(true) + + def routing(value: Routing): DeleteByQuery = self.copy(routing = Some(value)) } sealed trait DeleteIndexRequest extends ElasticRequest[DeletionOutcome] @@ -259,8 +257,8 @@ object ElasticRequest { sealed abstract class CreationOutcome -case object Created extends CreationOutcome case object AlreadyExists extends CreationOutcome +case object Created extends CreationOutcome sealed abstract class DeletionOutcome diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala index 3fe44e209..68fbc12d1 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticResult.scala @@ -25,7 +25,7 @@ sealed trait ElasticResult[F[_]] { } final class GetResult private[elasticsearch] (private val doc: Option[Item]) extends ElasticResult[Option] { - override def documentAs[A: Schema]: IO[DecodingException, Option[A]] = + def documentAs[A: Schema]: IO[DecodingException, Option[A]] = ZIO .fromEither(doc match { case Some(item) => @@ -40,7 +40,7 @@ final class GetResult private[elasticsearch] (private val doc: Option[Item]) ext } final class SearchResult private[elasticsearch] (private val hits: List[Item]) extends ElasticResult[List] { - override def documentAs[A: Schema]: IO[DecodingException, List[A]] = + def documentAs[A: Schema]: IO[DecodingException, List[A]] = ZIO.fromEither { ZValidation.validateAll(hits.map(item => ZValidation.fromEither(item.documentAs))).toEitherWith { errors => DecodingException(s"Could not parse all documents successfully: ${errors.map(_.message).mkString(",")})") diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index e389145a9..4ad668617 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -30,7 +30,7 @@ import zio.ZIO.logDebug import zio.elasticsearch.ElasticRequest._ import zio.json.ast.Json.{Obj, Str} import zio.schema.Schema -import zio.stream.ZStream +import zio.stream.{Stream, ZStream} import zio.{Chunk, Task, ZIO} import scala.collection.immutable.{Map => ScalaMap} @@ -55,12 +55,12 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: GetByQuery => executeGetByQuery(r) } - def stream(r: GetByQuery): ZStream[Any, Throwable, Item] = + def stream(r: GetByQuery): Stream[Throwable, Item] = ZStream.paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) } - def streamAs[A: Schema](r: GetByQuery): ZStream[Any, Throwable, A] = + def streamAs[A: Schema](r: GetByQuery): Stream[Throwable, A] = ZStream .paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) @@ -236,10 +236,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case HttpOk => response.body.fold( e => ZIO.fail(new ElasticException(s"Exception occurred: ${e.getMessage}")), - value => - ZIO.succeed( - new SearchResult(value.results.map(Item)) - ) + value => ZIO.succeed(new SearchResult(value.results.map(Item))) ) case _ => ZIO.fail(createElasticExceptionFromCustomResponse(response)) @@ -331,8 +328,8 @@ private[elasticsearch] object HttpElasticExecutor { private final val Doc = "_doc" private final val Search = "_search" private final val Scroll = "scroll" - private final val ScrollId = "scroll_id" private final val ScrollDefaultDuration = "1m" + private final val ScrollId = "scroll_id" def apply(config: ElasticConfig, client: SttpBackend[Task, Any]) = new HttpElasticExecutor(config, client) From 0c13270e63c48c1643d5cbec211f73ebc6fcab58 Mon Sep 17 00:00:00 2001 From: markaya Date: Fri, 3 Mar 2023 14:46:09 +0100 Subject: [PATCH 13/14] Fix ambigous method call --- .../scala/zio/elasticsearch/HttpExecutorSpec.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index 151c381ba..a9519741a 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -238,7 +238,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = ElasticQuery.contains("name.keyword", firstCustomer.name.take(3)) res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(contains(firstCustomer)) + } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -262,7 +262,7 @@ object HttpExecutorSpec extends IntegrationSpec { query = ElasticQuery.startsWith("name.keyword", firstCustomer.name.take(3)) res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(contains(firstCustomer)) + } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -287,7 +287,7 @@ object HttpExecutorSpec extends IntegrationSpec { wildcard("name.keyword", s"${firstCustomer.name.take(2)}*${firstCustomer.name.takeRight(2)}") res <- ElasticExecutor.execute(ElasticRequest.search(firstSearchIndex, query)).documentAs[CustomerDocument] - } yield assert(res)(contains(firstCustomer)) + } yield assert(res)(Assertion.contains(firstCustomer)) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), @@ -318,7 +318,7 @@ object HttpExecutorSpec extends IntegrationSpec { } yield assert(res)(isNonEmpty) } } @@ around( - ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ), test("search for documents using range query with multiple pages") { @@ -344,7 +344,7 @@ object HttpExecutorSpec extends IntegrationSpec { } yield assert(res)(hasSize(equalTo(204))) } } @@ around( - ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex)), ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie ), test("search for documents using range query with multiple pages and return type") { @@ -370,7 +370,7 @@ object HttpExecutorSpec extends IntegrationSpec { } yield assert(res)(hasSize(equalTo(201))) } } @@ around( - ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex, None)), + ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex)), ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie ) ) @@ shrinks(0), From 1fba35fe9efd4491c329d4531b255c1194cb1614 Mon Sep 17 00:00:00 2001 From: Dimitrije Bulaja Date: Mon, 6 Mar 2023 12:28:17 +0100 Subject: [PATCH 14/14] Refactor code --- .../zio/elasticsearch/HttpExecutorSpec.scala | 51 ++++++++++--------- .../zio/elasticsearch/ElasticExecutor.scala | 10 ++-- .../zio/elasticsearch/ElasticRequest.scala | 12 ++--- .../elasticsearch/HttpElasticExecutor.scala | 10 ++-- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala index a9519741a..8b9750aa9 100644 --- a/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/library/src/it/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -143,9 +143,7 @@ object HttpExecutorSpec extends IntegrationSpec { checkOnce(genDocumentId) { documentId => assertZIO( ElasticExecutor.execute(ElasticRequest.getById(index, documentId)).documentAs[CustomerDocument] - )( - isNone - ) + )(isNone) } }, test("fail with throwable if decoding fails") { @@ -202,10 +200,9 @@ object HttpExecutorSpec extends IntegrationSpec { .refreshTrue ) query = range("age").gte(0) - res <- - ElasticExecutor - .execute(ElasticRequest.search(secondSearchIndex, query)) - .documentAs[CustomerDocument] + res <- ElasticExecutor + .execute(ElasticRequest.search(secondSearchIndex, query)) + .documentAs[CustomerDocument] } yield res assertZIO(result.exit)( @@ -302,19 +299,16 @@ object HttpExecutorSpec extends IntegrationSpec { for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) - _ <- - ElasticExecutor.execute( - ElasticRequest.upsert[CustomerDocument](firstSearchIndex, firstDocumentId, firstCustomer) - ) - _ <- - ElasticExecutor.execute( - ElasticRequest - .upsert[CustomerDocument](firstSearchIndex, secondDocumentId, secondCustomer) - .refreshTrue - ) + _ <- ElasticExecutor.execute( + ElasticRequest.upsert[CustomerDocument](firstSearchIndex, firstDocumentId, firstCustomer) + ) + _ <- ElasticExecutor.execute( + ElasticRequest + .upsert[CustomerDocument](firstSearchIndex, secondDocumentId, secondCustomer) + .refreshTrue + ) query = range("balance").gte(100) - res <- - ElasticExecutor.stream(ElasticRequest.search(firstSearchIndex, query)).run(sink) + res <- ElasticExecutor.stream(ElasticRequest.search(firstSearchIndex, query)).run(sink) } yield assert(res)(isNonEmpty) } } @@ around( @@ -323,8 +317,7 @@ object HttpExecutorSpec extends IntegrationSpec { ), test("search for documents using range query with multiple pages") { checkOnce(genCustomer) { customer => - def sink: Sink[Throwable, Item, Nothing, Chunk[Item]] = - ZSink.collectAll[Item] + def sink: Sink[Throwable, Item, Nothing, Chunk[Item]] = ZSink.collectAll[Item] for { _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(secondSearchIndex, matchAll)) @@ -363,15 +356,25 @@ object HttpExecutorSpec extends IntegrationSpec { _ <- ElasticExecutor.execute(ElasticRequest.bulk(reqs: _*).refreshTrue) query = range("balance").gte(100) res <- ElasticExecutor - .streamAs[CustomerDocument]( - ElasticRequest.search(secondSearchIndex, query) - ) + .streamAs[CustomerDocument](ElasticRequest.search(secondSearchIndex, query)) .run(sink) } yield assert(res)(hasSize(equalTo(201))) } } @@ around( ElasticExecutor.execute(ElasticRequest.createIndex(secondSearchIndex)), ElasticExecutor.execute(ElasticRequest.deleteIndex(secondSearchIndex)).orDie + ), + test("search for documents using range query - empty stream") { + val sink: Sink[Throwable, Item, Nothing, Chunk[Item]] = ZSink.collectAll[Item] + + for { + _ <- ElasticExecutor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) + query = range("balance").gte(100) + res <- ElasticExecutor.stream(ElasticRequest.search(firstSearchIndex, query)).run(sink) + } yield assert(res)(hasSize(equalTo(0))) + } @@ around( + ElasticExecutor.execute(ElasticRequest.createIndex(firstSearchIndex)), + ElasticExecutor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ) ) @@ shrinks(0), suite("deleting by query")( diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala index 9466944de..30e209eab 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticExecutor.scala @@ -17,7 +17,7 @@ package zio.elasticsearch import sttp.client3.SttpBackend -import zio.elasticsearch.ElasticRequest.GetByQuery +import zio.elasticsearch.ElasticRequest.Search import zio.schema.Schema import zio.stream.{Stream, ZStream} import zio.{RIO, Task, URLayer, ZIO, ZLayer} @@ -25,9 +25,9 @@ import zio.{RIO, Task, URLayer, ZIO, ZLayer} private[elasticsearch] trait ElasticExecutor { def execute[A](request: ElasticRequest[A]): Task[A] - def stream(request: GetByQuery): Stream[Throwable, Item] + def stream(request: Search): Stream[Throwable, Item] - def streamAs[A: Schema](request: GetByQuery): Stream[Throwable, A] + def streamAs[A: Schema](request: Search): Stream[Throwable, A] } object ElasticExecutor { @@ -40,9 +40,9 @@ object ElasticExecutor { private[elasticsearch] def execute[A](request: ElasticRequest[A]): RIO[ElasticExecutor, A] = ZIO.serviceWithZIO[ElasticExecutor](_.execute(request)) - private[elasticsearch] def stream(request: GetByQuery): ZStream[ElasticExecutor, Throwable, Item] = + private[elasticsearch] def stream(request: Search): ZStream[ElasticExecutor, Throwable, Item] = ZStream.serviceWithStream[ElasticExecutor](_.stream(request)) - private[elasticsearch] def streamAs[A: Schema](request: GetByQuery): ZStream[ElasticExecutor, Throwable, A] = + private[elasticsearch] def streamAs[A: Schema](request: Search): ZStream[ElasticExecutor, Throwable, A] = ZStream.serviceWithStream[ElasticExecutor](_.streamAs[A](request)) } diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala index e0d82ae5f..cb5a0df1f 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticRequest.scala @@ -31,10 +31,10 @@ trait HasRouting[A] { def routing(value: Routing): ElasticRequest[A] } -sealed trait ElasticRequest[A] - sealed trait BulkableRequest[A] extends ElasticRequest[A] +sealed trait ElasticRequest[A] + object ElasticRequest { def bulk(requests: BulkableRequest[_]*): Bulk = @@ -67,8 +67,8 @@ object ElasticRequest { def getById(index: IndexName, id: DocumentId): GetById = GetById(index = index, id = id, routing = None) - def search(index: IndexName, query: ElasticQuery[_]): GetByQuery = - GetByQuery(index = index, query = query, routing = None) + def search(index: IndexName, query: ElasticQuery[_]): Search = + Search(index = index, query = query, routing = None) def upsert[A: Schema](index: IndexName, id: DocumentId, doc: A): CreateOrUpdate = CreateOrUpdate(index = index, id = id, document = Document.from(doc), refresh = false, routing = None) @@ -157,7 +157,7 @@ object ElasticRequest { private[elasticsearch] final case class CreateIndex( name: IndexName, definition: Option[String] - ) extends ElasticRequest[CreationOutcome] + ) extends CreateIndexRequest sealed trait CreateOrUpdateRequest extends BulkableRequest[Unit] with HasRefresh[Unit] with HasRouting[Unit] @@ -243,7 +243,7 @@ object ElasticRequest { sealed trait GetByQueryRequest extends ElasticRequest[SearchResult] - private[elasticsearch] final case class GetByQuery( + private[elasticsearch] final case class Search( index: IndexName, query: ElasticQuery[_], routing: Option[Routing] diff --git a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala index 4ad668617..4bd79a4e3 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/HttpElasticExecutor.scala @@ -52,15 +52,15 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC case r: DeleteIndex => executeDeleteIndex(r) case r: Exists => executeExists(r) case r: GetById => executeGetById(r) - case r: GetByQuery => executeGetByQuery(r) + case r: Search => executeSearch(r) } - def stream(r: GetByQuery): Stream[Throwable, Item] = + def stream(r: Search): Stream[Throwable, Item] = ZStream.paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) } - def streamAs[A: Schema](r: GetByQuery): Stream[Throwable, A] = + def streamAs[A: Schema](r: Search): Stream[Throwable, A] = ZStream .paginateChunkZIO("") { s => if (s.isEmpty) executeGetByQueryWithScroll(r) else executeGetByScroll(s) @@ -224,7 +224,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetByQuery(r: GetByQuery): Task[SearchResult] = + private def executeSearch(r: Search): Task[SearchResult] = sendRequestWithCustomResponse( request .post(uri"${config.uri}/${r.index}/$Search") @@ -243,7 +243,7 @@ private[elasticsearch] final class HttpElasticExecutor private (config: ElasticC } } - private def executeGetByQueryWithScroll(r: GetByQuery): Task[(Chunk[Item], Option[String])] = + private def executeGetByQueryWithScroll(r: Search): Task[(Chunk[Item], Option[String])] = sendRequestWithCustomResponse( request .post(