diff --git a/perf-tests/README.md b/perf-tests/README.md index 00e5e7b0bf..e0f6a71397 100644 --- a/perf-tests/README.md +++ b/perf-tests/README.md @@ -4,11 +4,13 @@ Performance tests are executed by running `PerfTestSuiteRunner`, which is a stan each test consist of: 1. Starting a HTTP server (Like Tapir-based Pekko, Vartx, http4s, or a "vanilla", tapirless one) -2. Sending a bunch of warmup requests -3. Sending simulation-specific requests +2. Running a simulation in warm-up mode (5 seconds, 3 concurrent users) +3. Running a simulation with user-defined duration and concurrent user count 4. Closing the server +5. Reading Gatling's simulation.log and building simulation results -The sequence is repeated for a set of servers multiplied by simulations, all configurable as arguments. Command parameters can be viewed by running: +The sequence is repeated for a set of servers multiplied by simulations. Afterwards, all individual simulation results will be aggregated into a single report. +Command parameters can be viewed by running: ``` perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner @@ -57,8 +59,8 @@ containing aggregated results from the entire suite: These reports include information about throughput and latency of each server for each simulation. -How the aggregation works: After each test the results are read from `simulation.log` produced by Gatling and aggregated by `GatlingLogProcessor`. -Entires related to warm-up process are not counted. The processor then uses 'com.codehale.metrics.Histogram' to calculate +How the aggregation works: After each non-warmup test the results are read from `simulation.log` produced by Gatling and aggregated by `GatlingLogProcessor`. +The processor then uses 'com.codehale.metrics.Histogram' to calculate p99, p95, p75, and p50 percentiles for latencies of all requests sent during the simulation. ## Adding new servers and simulations diff --git a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala index ef90fa1322..0af7ce7dc0 100644 --- a/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala +++ b/perf-tests/src/main/scala/sttp/tapir/perf/Common.scala @@ -11,6 +11,7 @@ object Common { val rootPackage = "sttp.tapir.perf" val LargeInputSize = 5 * 1024 * 1024 val WarmupDuration = 5.seconds + val WarmupUsers = 3 val Port = 8080 val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile def newTempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}") diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala index bb43a449c7..c356dffa10 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala @@ -30,7 +30,7 @@ object GatlingLogProcessor { .through(text.lines) .fold[State](State.initial) { (state, line) => val parts = line.split("\\s+") - if (parts.length >= 5 && parts(0) == "REQUEST" && parts(3) != "Warm-Up") { + if (parts.length >= 5 && parts(0) == "REQUEST") { val requestStartTime = parts(4).toLong val minRequestTs = state.minRequestTs.min(requestStartTime) val requestEndTime = parts(5).toLong diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala index ca724039b2..0822bc0fef 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala @@ -4,12 +4,13 @@ import cats.effect.{ExitCode, IO, IOApp} import cats.syntax.all._ import fs2.io.file import fs2.text -import sttp.tapir.perf.apis.ServerRunner import sttp.tapir.perf.Common._ +import sttp.tapir.perf.apis.ServerRunner import java.nio.file.Paths import java.time.LocalDateTime import java.time.format.DateTimeFormatter +import scala.concurrent.duration.FiniteDuration import scala.reflect.runtime.universe /** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling @@ -23,8 +24,6 @@ object PerfTestSuiteRunner extends IOApp { def run(args: List[String]): IO[ExitCode] = { val params = PerfTestSuiteParams.parse(args) - System.setProperty("tapir.perf.user-count", params.users.toString) - System.setProperty("tapir.perf.duration-seconds", params.durationSeconds.toString) println("===========================================================================================") println(s"Running a suite of ${params.totalTests} tests, each for ${params.users} users and ${params.duration}.") println(s"Additional warm-up phase of $WarmupDuration will be performed before each simulation.") @@ -42,8 +41,14 @@ object PerfTestSuiteRunner extends IOApp { for { serverKillSwitch <- startServerByTypeName(serverName) _ <- IO.println(s"Running server $shortServerName") - _ <- IO - .blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) + _ <- (for { + _ <- IO.println("======================== WARM-UP ===============================================") + _ = setSimulationParams(users = WarmupUsers, duration = WarmupDuration, warmup = true) + _ <- IO.blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) // warm-up + _ <- IO.println("==================== WARM-UP COMPLETED =========================================") + _ = setSimulationParams(users = params.users, duration = params.duration, warmup = false) + simResultCode <- IO.blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) // actual test + } yield simResultCode) .guarantee(serverKillSwitch) .ensureOr(errCode => new Exception(s"Gatling failed with code $errCode"))(_ == 0) serverSimulationResult <- GatlingLogProcessor.processLast(shortSimulationName, shortServerName) @@ -67,6 +72,15 @@ object PerfTestSuiteRunner extends IOApp { } } + /** Gatling doesn't allow to pass parameters to simulations when they are run using `Gatling.fromMap()`, that's why we're using system + * parameters as global variables to customize some params. + */ + private def setSimulationParams(users: Int, duration: FiniteDuration, warmup: Boolean): Unit = { + System.setProperty("tapir.perf.user-count", users.toString) + System.setProperty("tapir.perf.duration-seconds", duration.toSeconds.toString) + System.setProperty("tapir.perf.is-warm-up", warmup.toString) : Unit + } + private def writeCsvReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = { val csv = CsvReportPrinter.print(results) writeReportFile(csv, "csv", currentTime) diff --git a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala index 4952055caf..4923ca867e 100644 --- a/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala +++ b/perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala @@ -5,7 +5,6 @@ import io.gatling.core.structure.PopulationBuilder import io.gatling.http.Predef._ import sttp.tapir.perf.Common._ -import scala.concurrent.duration._ import scala.util.Random import io.gatling.core.structure.ChainBuilder @@ -34,31 +33,15 @@ object CommonSimulations { ) ) - private lazy val userCount = getParam("user-count").toInt - private lazy val duration = getParam("duration-seconds").toInt - private val httpProtocol = http.baseUrl(baseUrl) - - // Scenarios - val warmUpScenario = scenario("Warm-Up Scenario") - .during(WarmupDuration)( - exec( - http("HTTP GET Warm-Up") - .get("/path0/1") - ) - .exec( - http("HTTP POST Warm-Up") - .post("/path0") - .body(StringBody("warmup")) - .header("Content-Type", "text/plain") - ) - ) - .inject(atOnceUsers(3)) - .protocols(httpProtocol) + def userCount = getParam("user-count").toInt + def duration = getParam("duration-seconds").toInt + def namePrefix = if (getParamOpt("is-warm-up").map(_.toBoolean) == Some(true)) "[WARMUP] " else "" + val httpProtocol = http.baseUrl(baseUrl) def scenario_simple_get(routeNumber: Int): PopulationBuilder = { val execHttpGet: ChainBuilder = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4")) - scenario(s"Repeatedly invoke GET of route number $routeNumber") + scenario(s"${namePrefix}Repeatedly invoke GET of route number $routeNumber") .during(duration)(execHttpGet) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -72,7 +55,7 @@ object CommonSimulations { .header("Content-Type", "text/plain") ) - scenario(s"Repeatedly invoke POST with short string body") + scenario(s"${namePrefix}Repeatedly invoke POST with short string body") .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -86,7 +69,7 @@ object CommonSimulations { .header("Content-Type", "text/plain") // otherwise Play complains ) - scenario(s"Repeatedly invoke POST with short byte array body") + scenario(s"${namePrefix}Repeatedly invoke POST with short byte array body") .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -100,7 +83,7 @@ object CommonSimulations { .header("Content-Type", "application/octet-stream") ) - scenario(s"Repeatedly invoke POST with file body") + scenario(s"${namePrefix}Repeatedly invoke POST with file body") .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -114,7 +97,7 @@ object CommonSimulations { .header("Content-Type", "text/plain") // otherwise Play complains ) - scenario(s"Repeatedly invoke POST with large byte array") + scenario(s"${namePrefix}Repeatedly invoke POST with large byte array") .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -128,7 +111,7 @@ object CommonSimulations { .header("Content-Type", "text/plain") ) - scenario(s"Repeatedly invoke POST with large byte array, interpreted to a String") + scenario(s"${namePrefix}Repeatedly invoke POST with large byte array, interpreted to a String") .during(duration)(execHttpPost) .inject(atOnceUsers(userCount)) .protocols(httpProtocol) @@ -138,29 +121,29 @@ object CommonSimulations { import CommonSimulations._ class SimpleGetSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_simple_get(0))): Unit + setUp(scenario_simple_get(0)): Unit } class SimpleGetMultiRouteSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_simple_get(127))): Unit + setUp(scenario_simple_get(127)): Unit } class PostBytesSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_post_bytes(0))): Unit + setUp(scenario_post_bytes(0)): Unit } class PostLongBytesSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_post_long_bytes(0))): Unit + setUp(scenario_post_long_bytes(0)): Unit } class PostFileSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_post_file(0))): Unit + setUp(scenario_post_file(0)): Unit } class PostStringSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_post_string(0))): Unit + setUp(scenario_post_string(0)): Unit } class PostLongStringSimulation extends Simulation { - setUp(warmUpScenario.andThen(scenario_post_long_string(0))): Unit + setUp(scenario_post_long_string(0)): Unit }