feat: upgrade ZIO http client to improve performance (#850)

Signed-off-by: David Poltorak <[email protected]> Signed-off-by: Benjamin Voiturier <[email protected]> Co-authored-by: Benjamin Voiturier <[email protected]> Signed-off-by: Shota Jolbordi <[email protected]>
hyperledger · Mar 18, 2024 · cc61eb9 · cc61eb9
1 parent 348045f
commit cc61eb9
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 1 deletion.
diff --git a/build.sbt b/build.sbt
@@ -59,7 +59,7 @@ lazy val V = new {
   val mockito = "3.2.16.0"
   val monocle = "3.1.0"
 
-  // https://mvnrepository.com/artifact/io.circe/circe-core
+    // https://mvnrepository.com/artifact/io.circe/circe-core
   val circe = "0.14.6"
 
   val tapir = "1.6.4"

diff --git a/infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh b/infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh
@@ -35,7 +35,10 @@ echo "--------------------------------------"
     yarn webpack
     k6 run -e SCENARIO_LABEL=st-create-prism-did-smoke dist/create-prism-did-test.js -o experimental-prometheus-rw
     k6 run -e SCENARIO_LABEL=st-credential-offer-smoke dist/credential-offer-test.js -o experimental-prometheus-rw
+<<<<<<< HEAD
     k6 run -e SCENARIO_LABEL=st-credential-definition-smoke dist/credential-definition-test.js -o experimental-prometheus-rw
+=======
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
     k6 run -e SCENARIO_LABEL=st-credential-schema-smoke dist/credential-schema-test.js -o experimental-prometheus-rw
     k6 run -e SCENARIO_LABEL=st-did-publishing-smoke dist/did-publishing-test.js -o experimental-prometheus-rw
     k6 run -e SCENARIO_LABEL=st-connection-flow-smoke dist/connection-flow-test.js -o experimental-prometheus-rw

diff --git a/...x/lib/core/src/main/scala/io/iohk/atala/pollux/core/service/HttpURIDereferencerImpl.scala b/...x/lib/core/src/main/scala/io/iohk/atala/pollux/core/service/HttpURIDereferencerImpl.scala
@@ -3,6 +3,10 @@ package io.iohk.atala.pollux.core.service
 import io.iohk.atala.pollux.core.service.URIDereferencerError.{ConnectionError, ResourceNotFound, UnexpectedError}
 import zio.*
 import zio.http.*
+<<<<<<< HEAD
+=======
+import zio.stream.ZSink
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
 
 import java.net.URI
 import java.nio.charset.StandardCharsets

diff --git a/prism-agent/service/server/src/main/resources/application.conf b/prism-agent/service/server/src/main/resources/application.conf
@@ -100,9 +100,15 @@ agent {
     httpClient {
         connectionPoolSize = 0
         connectionPoolSize = ${?AGENT_HTTP_CLIENT_CONNECTION_POOL_SIZE}
+<<<<<<< HEAD
         idleTimeout = 5.seconds
         idleTimeout = ${?AGENT_HTTP_CLIENT_IDLE_TIMEOUT}
         connectionTimeout = 5.seconds
+=======
+        idleTimeout = 2.seconds
+        idleTimeout = ${?AGENT_HTTP_CLIENT_IDLE_TIMEOUT}
+        connectionTimeout = 2.seconds
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
         connectionTimeout = ${?AGENT_HTTP_CLIENT_CONNECTION_TIMEOUT}
     }
     authentication {

diff --git a/prism-agent/service/server/src/main/scala/io/iohk/atala/agent/server/Main.scala b/prism-agent/service/server/src/main/scala/io/iohk/atala/agent/server/Main.scala
@@ -98,6 +98,26 @@ object MainApp extends ZIOAppDefault {
     _ <- AgentMigrations.validateRLS.provide(RepoModule.agentContextAwareTransactorLayer)
   } yield ()
 
+  private val zioHttpClientLayer = {
+    import zio.http.netty.NettyConfig
+    import zio.http.{ConnectionPoolConfig, DnsResolver, ZClient}
+    (ZLayer.fromZIO(
+      for {
+        appConfig <- ZIO.service[AppConfig].provide(SystemModule.configLayer)
+      } yield ZClient.Config.default.copy(
+        connectionPool = {
+          val cpSize = appConfig.agent.httpClient.connectionPoolSize
+          if (cpSize > 0) ConnectionPoolConfig.Fixed(cpSize)
+          else ConnectionPoolConfig.Disabled
+        },
+        idleTimeout = Some(appConfig.agent.httpClient.idleTimeout),
+        connectionTimeout = Some(appConfig.agent.httpClient.connectionTimeout),
+      )
+    ) ++
+      ZLayer.succeed(NettyConfig.default) ++
+      DnsResolver.default) >>> ZClient.live
+  }
+
   override def run: ZIO[Any, Throwable, Unit] = {
 
     val app = for {
@@ -193,7 +213,11 @@ object MainApp extends ZIOAppDefault {
           // event notification service
           ZLayer.succeed(500) >>> EventNotificationServiceImpl.layer,
           // HTTP client
+<<<<<<< HEAD
           SystemModule.zioHttpClientLayer,
+=======
+          zioHttpClientLayer,
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
           Scope.default,
         )
     } yield app

diff --git a/prism-agent/service/server/src/main/scala/io/iohk/atala/agent/server/PrismAgentApp.scala b/prism-agent/service/server/src/main/scala/io/iohk/atala/agent/server/PrismAgentApp.scala
@@ -37,6 +37,8 @@ import io.iohk.atala.shared.utils.DurationOps.toMetricsSeconds
 import io.iohk.atala.system.controller.SystemServerEndpoints
 import zio.*
 import zio.metrics.*
+import java.util.concurrent.Executors
+import scala.concurrent.ExecutionContext
 
 object PrismAgentApp {
 

diff --git a/...m-agent/service/server/src/main/scala/io/iohk/atala/agent/server/http/ZioHttpClient.scala b/...m-agent/service/server/src/main/scala/io/iohk/atala/agent/server/http/ZioHttpClient.scala
@@ -3,6 +3,11 @@ package io.iohk.atala.agent.server.http
 import io.iohk.atala.mercury.*
 import zio.*
 import zio.http.{Header as _, *}
+<<<<<<< HEAD
+=======
+
+import java.time.Instant
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
 
 object ZioHttpClient {
   val layer: URLayer[Client, ZioHttpClient] = ZLayer.fromFunction(new ZioHttpClient(_))

diff --git a/tests/integration-tests/src/test/kotlin/features/system/SystemSteps.kt b/tests/integration-tests/src/test/kotlin/features/system/SystemSteps.kt
@@ -0,0 +1,52 @@
+package features.system
+
+import interactions.Get
+import io.cucumber.java.en.Then
+import io.cucumber.java.en.When
+import io.iohk.atala.automation.extensions.get
+import io.iohk.atala.automation.serenity.ensure.Ensure
+import io.iohk.atala.prism.models.HealthInfo
+import net.serenitybdd.rest.SerenityRest
+import net.serenitybdd.screenplay.Actor
+import org.apache.http.HttpStatus
+
+class SystemSteps {
+    @When("{actor} makes a request to the health endpoint")
+    fun actorRequestsHealthEndpoint(actor: Actor) {
+        actor.attemptsTo(
+            Get.resource("/_system/health")
+        )
+        actor.attemptsTo(
+            Ensure.thatTheLastResponse().statusCode().isEqualTo(HttpStatus.SC_OK)
+        )
+    }
+
+    @Then("{actor} knows what version of the service is running")
+    fun actorUnderstandsVersion(actor: Actor) {
+        val healthResponse = SerenityRest.lastResponse().get<HealthInfo>()
+        actor.attemptsTo(
+            Ensure.that(healthResponse.version).isNotBlank()
+        )
+    }
+
+    @When("{actor} makes a request to the metrics endpoint")
+    fun actorRequestsMetricEndpoint(actor: Actor) {
+        actor.attemptsTo(
+            Get.resource("/_system/metrics")
+        )
+        actor.attemptsTo(
+            Ensure.thatTheLastResponse().statusCode().isEqualTo(HttpStatus.SC_OK)
+        )
+    }
+
+    @Then("{actor} sees that the metrics contain background job stats")
+    fun actorSeesMetrics(actor: Actor) {
+        val metricsResponse = SerenityRest.lastResponse()
+        actor.attemptsTo(
+            Ensure.that(metricsResponse.body.asString()).contains("present_proof_flow_did_com_exchange_job_ms_gauge"),
+            Ensure.that(metricsResponse.body.asString()).contains("connection_flow_did_com_exchange_job_ms_gauge"),
+            Ensure.that(metricsResponse.body.asString()).contains("issuance_flow_did_com_exchange_job_ms_gauge")
+        )
+    }
+
+}
diff --git a/tests/performance-tests/atala-performance-tests-k6/README.md b/tests/performance-tests/atala-performance-tests-k6/README.md
@@ -33,7 +33,11 @@ Once that is done, we can run our script the same way we usually do, for instanc
 $ k6 run dist/connection-flow-test.js
 ```
 
+<<<<<<< HEAD
 # Debugging Tests
+=======
+## Debugging Tests
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
 
 k6 can be configured to log the HTTP request and responses that it makes during test execution. This is useful to debug errors that happen in tests when logs or k6 output does not contain the reason for a failure.
 

diff --git a/tests/performance-tests/atala-performance-tests-k6/src/common/Config.ts b/tests/performance-tests/atala-performance-tests-k6/src/common/Config.ts
@@ -4,7 +4,11 @@
  * Maximum number of iterations for the waiting loop.
  * If not provided, the default value is 100.
  */
+<<<<<<< HEAD
 export const WAITING_LOOP_MAX_ITERATIONS = Number(__ENV.MY_USER_AGENT) || 1000;
+=======
+export const WAITING_LOOP_MAX_ITERATIONS = Number(__ENV.MY_USER_AGENT) || 100;
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
 
 /**
  * Pause interval in seconds for each iteration of the waiting loop.

diff --git a/tests/performance-tests/atala-performance-tests-k6/src/common/CredentialsService.ts b/tests/performance-tests/atala-performance-tests-k6/src/common/CredentialsService.ts
@@ -1,6 +1,12 @@
+<<<<<<< HEAD
 import { fail, sleep } from "k6";
 import { HttpService, statusChangeTimeouts } from "./HttpService";
 import { ISSUER_AGENT_URL, WAITING_LOOP_MAX_ITERATIONS, WAITING_LOOP_PAUSE_INTERVAL } from "./Config";
+=======
+import {fail, sleep} from "k6";
+import { HttpService, statusChangeTimeouts } from "./HttpService";
+import {ISSUER_AGENT_URL, WAITING_LOOP_MAX_ITERATIONS, WAITING_LOOP_PAUSE_INTERVAL} from "./Config";
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
 import { IssueCredentialRecord, Connection, CredentialSchemaResponse } from "@input-output-hk/prism-typescript-client";
 import { crypto } from "k6/experimental/webcrypto";
 
@@ -21,7 +27,11 @@ export class CredentialsService extends HttpService {
         "claims": {
           "emailAddress": "${crypto.randomUUID()}-@atala.io",
           "familyName": "Test",
+<<<<<<< HEAD
           "schemaId": "${ISSUER_AGENT_URL.replace("localhost", "host.docker.internal")}/schema-registry/schemas/${schema.guid}/schema",
+=======
+          "schemaId": "${ISSUER_AGENT_URL.replace("localhost", "host.docker.internal")}/schema-registry/schemas/${schema.guid}",
+>>>>>>> 7aa9b4c2 (feat: upgrade ZIO http client to improve performance (#850))
           "dateOfIssuance": "${new Date()}",
           "drivingLicenseID": "Test",
           "drivingClass": 1