diff --git a/build.sbt b/build.sbt index 3c74ab3c30..9865a11c43 100644 --- a/build.sbt +++ b/build.sbt @@ -5,6 +5,7 @@ import com.softwaremill.UpdateVersionInDocs import com.typesafe.tools.mima.core.{Problem, ProblemFilters} import sbt.Reference.display import sbt.internal.ProjectMatrix +import sbtassembly.AssemblyPlugin.autoImport.assembly // explicit import to avoid clash with gatling plugin import java.net.URL import scala.concurrent.duration.DurationInt @@ -357,8 +358,8 @@ val http4sVanillaMulti = taskKey[Unit]("http4s-vanilla-multi") val http4sTapirMulti = taskKey[Unit]("http4s-tapir-multi") def genPerfTestTask(servName: String, simName: String) = Def.taskDyn { Def.task { - (Compile / runMain).toTask(s" perfTests.${servName}Server").value - (Gatling / testOnly).toTask(s" perfTests.${simName}Simulation").value + (Compile / runMain).toTask(s" sttp.tapir.perf.${servName}Server").value + (Gatling / testOnly).toTask(s" sttp.tapir.perf.${simName}Simulation").value } } @@ -385,14 +386,14 @@ lazy val perfTests: ProjectMatrix = (projectMatrix in file("perf-tests")) fork := true, connectInput := true ) - .settings(akkaHttpVanilla := { (genPerfTestTask("AkkaHttp.Vanilla", "OneRoute")).value }) - .settings(akkaHttpTapir := { (genPerfTestTask("AkkaHttp.Tapir", "OneRoute")).value }) - .settings(akkaHttpVanillaMulti := { (genPerfTestTask("AkkaHttp.VanillaMulti", "MultiRoute")).value }) - .settings(akkaHttpTapirMulti := { (genPerfTestTask("AkkaHttp.TapirMulti", "MultiRoute")).value }) - .settings(http4sVanilla := { (genPerfTestTask("Http4s.Vanilla", "OneRoute")).value }) - .settings(http4sTapir := { (genPerfTestTask("Http4s.Tapir", "OneRoute")).value }) - .settings(http4sVanillaMulti := { (genPerfTestTask("Http4s.VanillaMulti", "MultiRoute")).value }) - .settings(http4sTapirMulti := { (genPerfTestTask("Http4s.TapirMulti", "MultiRoute")).value }) + .settings(akkaHttpVanilla := { (genPerfTestTask("akka.Vanilla", "OneRoute")).value }) + .settings(akkaHttpTapir := { (genPerfTestTask("akka.Tapir", "OneRoute")).value }) + .settings(akkaHttpVanillaMulti := { (genPerfTestTask("akka.VanillaMulti", "MultiRoute")).value }) + .settings(akkaHttpTapirMulti := { (genPerfTestTask("akka.TapirMulti", "MultiRoute")).value }) + .settings(http4sVanilla := { (genPerfTestTask("http4s.Vanilla", "OneRoute")).value }) + .settings(http4sTapir := { (genPerfTestTask("http4s.Tapir", "OneRoute")).value }) + .settings(http4sVanillaMulti := { (genPerfTestTask("http4s.VanillaMulti", "MultiRoute")).value }) + .settings(http4sTapirMulti := { (genPerfTestTask("http4s.TapirMulti", "MultiRoute")).value }) .jvmPlatform(scalaVersions = examplesScalaVersions) .dependsOn(core, akkaHttpServer, http4sServer) diff --git a/perf-tests/README.md b/perf-tests/README.md index 5bab44a635..0103ab3fca 100644 --- a/perf-tests/README.md +++ b/perf-tests/README.md @@ -1,15 +1,19 @@ # seperate testing -To start a server, run `perfTests/run` and select the server you want to test. +To start a server, run `perfTests/run` and select the server you want to test, or in a single command: -To test vanilla akka-http load, run this command: ``` -perfTests / Gatling / testOnly perfTests.AkkaHttpVanillaSimulation +sbt "perfTests/runMain sttp.tapir.perf.akka.VanillaMultiServer" +// or +sbt "perfTests/runMain sttp.tapir.perf.akka.TapirMultiServer" +// or others ... ``` -To test tapir with akka-http load, run this command: +Then run the test: ``` -perfTests / Gatling / testOnly perfTests.AkkaHttpTapirSimulation +sbt "perfTests/Gatling/testOnly sttp.tapir.perf.OneRouteSimulation" +// or +sbt "perfTests/Gatling/testOnly sttp.tapir.perf.MultiRouteSimulation" ``` This method yields the most performant results, but requires running the commands in two seperate sbt instacnes. diff --git a/perf-tests/results/100users-1min-18-03-2022-local-adamw.md b/perf-tests/results/100users-1min-18-03-2022-local-adamw.md new file mode 100644 index 0000000000..37df1fe612 --- /dev/null +++ b/perf-tests/results/100users-1min-18-03-2022-local-adamw.md @@ -0,0 +1,8 @@ +# Setup + +2019 mbp, 2,3 GHz 8-Core Intel Core i9 + +# Results + +* vanilla akka-http, multi-route: 23278 req/s +* tapir akka-http, multi-route: 8277 req/s \ No newline at end of file diff --git a/perf-tests/results/100users-5min-09-03-2022-ec2.md b/perf-tests/results/100users-5min-09-03-2022-ec2.md index 4bc3ba344b..59c9b4c041 100644 --- a/perf-tests/results/100users-5min-09-03-2022-ec2.md +++ b/perf-tests/results/100users-5min-09-03-2022-ec2.md @@ -11,17 +11,17 @@ Network communication through local IP on a shared EC2 network; ## Summary ``` Name : requests sent -[1] perfTests.AkkaHttp.TapirMultiServer : 568554 -[2] perfTests.AkkaHttp.TapirServer : 1310373 -[3] perfTests.AkkaHttp.VanillaMultiServer : 1301316 -[4] perfTests.AkkaHttp.VanillaServer : 1311758 -[5] perfTests.Http4s.TapirMultiServer : 368887 -[6] perfTests.Http4s.TapirServer : 788788 -[7] perfTests.Http4s.VanillaMultiServer : 1074490 -[8] perfTests.Http4s.VanillaServer : 1235687 +[1] sttp.tapir.perf.akka.TapirMultiServer : 568554 +[2] sttp.tapir.perf.akka.TapirServer : 1310373 +[3] sttp.tapir.perf.akka.VanillaMultiServer : 1301316 +[4] sttp.tapir.perf.akka.VanillaServer : 1311758 +[5] sttp.tapir.perf.akka.TapirMultiServer : 368887 +[6] sttp.tapir.perf.akka.TapirServer : 788788 +[7] sttp.tapir.perf.akka.VanillaMultiServer : 1074490 +[8] sttp.tapir.perf.akka.VanillaServer : 1235687 ``` ## Full results -### [1] perfTests.AkkaHttp.TapirMultiServer +### [1] sttp.tapir.perf.akka.TapirMultiServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -42,7 +42,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [2] perfTests.AkkaHttp.TapirServer +### [2] sttp.tapir.perf.akka.TapirServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -63,7 +63,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [3] perfTests.AkkaHttp.VanillaMultiServer +### [3] sttp.tapir.perf.akka.VanillaMultiServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -84,7 +84,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [4] perfTests.AkkaHttp.VanillaServer +### [4] sttp.tapir.perf.akka.VanillaServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -105,7 +105,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [5] perfTests.Http4s.TapirMultiServer +### [5] sttp.tapir.perf.akka.TapirMultiServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -126,7 +126,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [6] perfTests.Http4s.TapirServer +### [6] sttp.tapir.perf.akka.TapirServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -147,7 +147,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [7] perfTests.Http4s.VanillaMultiServer +### [7] sttp.tapir.perf.akka.VanillaMultiServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- @@ -168,7 +168,7 @@ Network communication through local IP on a shared EC2 network; > failed 0 ( 0%) ================================================================================ ``` -### [8] perfTests.Http4s.VanillaServer +### [8] sttp.tapir.perf.akka.VanillaServer ``` ================================================================================ ---- Global Information -------------------------------------------------------- diff --git a/perf-tests/src/main/resources/logback.xml b/perf-tests/src/main/resources/logback.xml new file mode 100644 index 0000000000..e6cee15ae7 --- /dev/null +++ b/perf-tests/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + + %date [%thread] %-5level %logger{36} - %msg%n + + + + + + + \ No newline at end of file diff --git a/perf-tests/src/main/scala/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala similarity index 62% rename from perf-tests/src/main/scala/Common.scala rename to perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index b880f5a9ce..d44b39738d 100644 --- a/perf-tests/src/main/scala/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -1,16 +1,17 @@ -package perfTests +package sttp.tapir.perf + +import sttp.tapir.{PublicEndpoint, endpoint, path, stringBody} -import sttp.tapir._ import scala.io.StdIn object Common { - def genTapirEndpoint(n: Int) = endpoint.get + def genTapirEndpoint(n: Int): PublicEndpoint[Int, String, String, Any] = endpoint.get .in("path" + n.toString) .in(path[Int]("id")) .errorOut(stringBody) .out(stringBody) - def blockServer() = { + def blockServer(): Unit = { println(Console.BLUE + "Server now online. Please navigate to http://localhost:8080/path0/1\nPress RETURN to stop..." + Console.RESET) StdIn.readLine() println("Server terminated") diff --git a/perf-tests/src/main/scala/AkkaHttp.scala b/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala similarity index 69% rename from perf-tests/src/main/scala/AkkaHttp.scala rename to perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala index fda90792b0..9760c0a9e6 100644 --- a/perf-tests/src/main/scala/AkkaHttp.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/akka/AkkaHttp.scala @@ -1,14 +1,17 @@ -package perfTests.AkkaHttp +package sttp.tapir.perf.akka -import sttp.tapir.server.akkahttp.AkkaHttpServerInterpreter import akka.actor.ActorSystem +import akka.http.scaladsl.Http import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import akka.http.scaladsl.Http -import scala.concurrent.Future +import sttp.tapir.perf +import sttp.tapir.perf.Common +import sttp.tapir.server.akkahttp.AkkaHttpServerInterpreter + +import scala.concurrent.{ExecutionContextExecutor, Future} object Vanilla { - val router = (nRoutes: Int) => + val router: Int => Route = (nRoutes: Int) => concat( (0 to nRoutes).map((n: Int) => get { @@ -21,11 +24,11 @@ object Vanilla { } object Tapir { - val router = (nRoutes: Int) => + val router: Int => Route = (nRoutes: Int) => AkkaHttpServerInterpreter().toRoute( (0 to nRoutes) .map((n: Int) => - perfTests.Common + perf.Common .genTapirEndpoint(n) .serverLogic((id: Int) => Future.successful(Right((id + n).toString)): Future[Either[String, String]]) ) @@ -34,14 +37,14 @@ object Tapir { } object AkkaHttp { - implicit val actorSystem = ActorSystem("akka-http") - implicit val executionContext = actorSystem.dispatcher + implicit val actorSystem: ActorSystem = ActorSystem("akka-http") + implicit val executionContext: ExecutionContextExecutor = actorSystem.dispatcher - def runServer(router: Route) = { + def runServer(router: Route): Unit = { Http() .newServerAt("127.0.0.1", 8080) .bind(router) - .flatMap((x) => { perfTests.Common.blockServer(); x.unbind() }) + .flatMap((x) => { Common.blockServer(); x.unbind() }) .onComplete(_ => actorSystem.terminate()) } } diff --git a/perf-tests/src/main/scala/Http4s.scala b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala similarity index 74% rename from perf-tests/src/main/scala/Http4s.scala rename to perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala index 0112d79957..588d30c846 100644 --- a/perf-tests/src/main/scala/Http4s.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/http4s/Http4s.scala @@ -1,17 +1,19 @@ -package perfTests.Http4s +package sttp.tapir.perf.http4s import cats.effect._ import cats.effect.unsafe.implicits.global import cats.syntax.all._ import org.http4s._ +import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.dsl._ import org.http4s.implicits._ -import org.http4s.server.blaze.BlazeServerBuilder import org.http4s.server.Router +import sttp.tapir.perf +import sttp.tapir.perf.Common import sttp.tapir.server.http4s.Http4sServerInterpreter object Vanilla { - val router = (nRoutes: Int) => + val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router( (0 to nRoutes).map((n: Int) => ("/path" + n.toString) -> { @@ -26,11 +28,11 @@ object Vanilla { } object Tapir { - val router = (nRoutes: Int) => + val router: Int => HttpRoutes[IO] = (nRoutes: Int) => Router("/" -> { Http4sServerInterpreter[IO]().toRoutes( (0 to nRoutes) - .map((n: Int) => perfTests.Common.genTapirEndpoint(n).serverLogic(id => IO(((id + n).toString).asRight[String]))) + .map((n: Int) => Common.genTapirEndpoint(n).serverLogic(id => IO(((id + n).toString).asRight[String]))) .toList ) }) @@ -42,7 +44,7 @@ object Http4s { .bindHttp(8080, "localhost") .withHttpApp(router.orNotFound) .resource - .use(_ => { perfTests.Common.blockServer(); IO.pure(ExitCode.Success) }) + .use(_ => { perf.Common.blockServer(); IO.pure(ExitCode.Success) }) } } diff --git a/perf-tests/src/test/scala/Simulations.scala b/perf-tests/src/test/scala/Simulations.scala deleted file mode 100644 index f82712db73..0000000000 --- a/perf-tests/src/test/scala/Simulations.scala +++ /dev/null @@ -1,29 +0,0 @@ -package perfTests - -import io.gatling.core.Predef._ -import io.gatling.http.Predef._ - -object CommonSimulations { - val scn = scenario("get plaintext") - .during(5 * 60) { - exec( - http("first plaintext") - .get("/4") - ) - } - val userCount = 100 - val baseUrl = "http://127.0.0.1:8080" - - def genericInjection(n: Int) = { - val httpProtocol = http.baseUrl(baseUrl + "/path" + n.toString) - scn.inject(atOnceUsers(userCount)).protocols(httpProtocol) - } -} - -class OneRouteSimulation extends Simulation { - setUp(CommonSimulations.genericInjection(0)) -} - -class MultiRouteSimulation extends Simulation { - setUp(CommonSimulations.genericInjection(127)) -} diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala new file mode 100644 index 0000000000..fb4837b997 --- /dev/null +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -0,0 +1,30 @@ +package sttp.tapir.perf + +import io.gatling.core.Predef._ +import io.gatling.core.structure.PopulationBuilder +import io.gatling.http.Predef._ + +import scala.concurrent.duration.{DurationInt, FiniteDuration} + +object CommonSimulations { + private val userCount = 100 + private val baseUrl = "http://127.0.0.1:8080" + + def testScenario(duration: FiniteDuration, routeNumber: Int): PopulationBuilder = { + val httpProtocol = http.baseUrl(baseUrl) + val execHttpGet = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) + + scenario(s"Repeatedly invoke GET of route number $routeNumber") + .during(duration.toSeconds.toInt)(execHttpGet) + .inject(atOnceUsers(userCount)) + .protocols(httpProtocol) + } +} + +class OneRouteSimulation extends Simulation { + setUp(CommonSimulations.testScenario(1.minute, 0)) +} + +class MultiRouteSimulation extends Simulation { + setUp(CommonSimulations.testScenario(1.minute, 127)) +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 64a6b62100..9e64076230 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -11,5 +11,5 @@ addSbtPlugin("org.jetbrains.scala" % "sbt-ide-settings" % "1.1.1") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.9.0") addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") -addSbtPlugin("io.gatling" % "gatling-sbt" % "3.2.2") +addSbtPlugin("io.gatling" % "gatling-sbt" % "4.1.3") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0")