Performance tests: Fix warm-up phase

perf-tests/README.md

@@ -4,11 +4,13 @@ Performance tests are executed by running `PerfTestSuiteRunner`, which is a stan
 each test consist of:
 
 1. Starting a HTTP server (Like Tapir-based Pekko, Vartx, http4s, or a "vanilla", tapirless one)
-2. Sending a bunch of warmup requests
-3. Sending simulation-specific requests
+2. Running a simulation in warm-up mode (5 seconds, 3 concurrent users)
+3. Running a simulation with user-defined duration and concurrent user count
 4. Closing the server
+5. Reading Gatling's simulation.log and building simulation results
 
-The sequence is repeated for a set of servers multiplied by simulations, all configurable as arguments. Command parameters can be viewed by running:
+The sequence is repeated for a set of servers multiplied by simulations. Afterwards, all individual simulation results will be aggregated into a single report. 
+Command parameters can be viewed by running:
 
 ```
 perfTests/Test/runMain sttp.tapir.perf.PerfTestSuiteRunner
@@ -57,8 +59,8 @@ containing aggregated results from the entire suite:
 
 These reports include information about throughput and latency of each server for each simulation.
 
-How the aggregation works: After each test the results are read from `simulation.log` produced by Gatling and aggregated by `GatlingLogProcessor`. 
-Entires related to warm-up process are not counted. The processor then uses 'com.codehale.metrics.Histogram' to calculate 
+How the aggregation works: After each non-warmup test the results are read from `simulation.log` produced by Gatling and aggregated by `GatlingLogProcessor`. 
+The processor then uses 'com.codehale.metrics.Histogram' to calculate 
 p99, p95, p75, and p50 percentiles for latencies of all requests sent during the simulation.
 
 ## Adding new servers and simulations

perf-tests/src/main/scala/sttp/tapir/perf/Common.scala

@@ -11,6 +11,7 @@ object Common {
   val rootPackage = "sttp.tapir.perf"
   val LargeInputSize = 5 * 1024 * 1024
   val WarmupDuration = 5.seconds
+  val WarmupUsers = 3
   val Port = 8080
   val TmpDir: File = new java.io.File(System.getProperty("java.io.tmpdir")).getAbsoluteFile
   def newTempFilePath(): Path = TmpDir.toPath.resolve(s"tapir-${new Date().getTime}-${Random.nextLong()}")

perf-tests/src/test/scala/sttp/tapir/perf/GatlingLogProcessor.scala

@@ -30,7 +30,7 @@ object GatlingLogProcessor {
         .through(text.lines)
         .fold[State](State.initial) { (state, line) =>
           val parts = line.split("\\s+")
-          if (parts.length >= 5 && parts(0) == "REQUEST" && parts(3) != "Warm-Up") {
+          if (parts.length >= 5 && parts(0) == "REQUEST") {
             val requestStartTime = parts(4).toLong
             val minRequestTs = state.minRequestTs.min(requestStartTime)
             val requestEndTime = parts(5).toLong

perf-tests/src/test/scala/sttp/tapir/perf/PerfTestSuiteRunner.scala

@@ -4,12 +4,13 @@ import cats.effect.{ExitCode, IO, IOApp}
 import cats.syntax.all._
 import fs2.io.file
 import fs2.text
-import sttp.tapir.perf.apis.ServerRunner
 import sttp.tapir.perf.Common._
+import sttp.tapir.perf.apis.ServerRunner
 
 import java.nio.file.Paths
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import scala.concurrent.duration.FiniteDuration
 import scala.reflect.runtime.universe
 
 /** Main entry point for running suites of performance tests and generating aggregated reports. A suite represents a set of Gatling
@@ -23,8 +24,6 @@ object PerfTestSuiteRunner extends IOApp {
 
   def run(args: List[String]): IO[ExitCode] = {
     val params = PerfTestSuiteParams.parse(args)
-    System.setProperty("tapir.perf.user-count", params.users.toString)
-    System.setProperty("tapir.perf.duration-seconds", params.durationSeconds.toString)
     println("===========================================================================================")
     println(s"Running a suite of ${params.totalTests} tests, each for ${params.users} users and ${params.duration}.")
     println(s"Additional warm-up phase of $WarmupDuration will be performed before each simulation.")
@@ -42,8 +41,14 @@ object PerfTestSuiteRunner extends IOApp {
         for {
           serverKillSwitch <- startServerByTypeName(serverName)
           _ <- IO.println(s"Running server $shortServerName")
-          _ <- IO
-            .blocking(GatlingRunner.runSimulationBlocking(simulationName, params))
+          _ <- (for {
+            _ <- IO.println("======================== WARM-UP ===============================================")
+            _ = setSimulationParams(users = WarmupUsers, duration = WarmupDuration, warmup = true)
+            _ <- IO.blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) // warm-up
+            _ <- IO.println("==================== WARM-UP COMPLETED =========================================")
+            _ = setSimulationParams(users = params.users, duration = params.duration, warmup = false)
+            simResultCode <- IO.blocking(GatlingRunner.runSimulationBlocking(simulationName, params)) // actual test
+          } yield simResultCode)
             .guarantee(serverKillSwitch)
             .ensureOr(errCode => new Exception(s"Gatling failed with code $errCode"))(_ == 0)
           serverSimulationResult <- GatlingLogProcessor.processLast(shortSimulationName, shortServerName)
@@ -67,6 +72,15 @@ object PerfTestSuiteRunner extends IOApp {
     }
   }
 
+  /** Gatling doesn't allow to pass parameters to simulations when they are run using `Gatling.fromMap()`, that's why we're using system
+    * parameters as global variables to customize some params.
+    */
+  private def setSimulationParams(users: Int, duration: FiniteDuration, warmup: Boolean): Unit = {
+    System.setProperty("tapir.perf.user-count", users.toString)
+    System.setProperty("tapir.perf.duration-seconds", duration.toSeconds.toString)
+    System.setProperty("tapir.perf.is-warm-up", warmup.toString) : Unit
+  }
+
   private def writeCsvReport(currentTime: String)(results: List[GatlingSimulationResult]): IO[Unit] = {
     val csv = CsvReportPrinter.print(results)
     writeReportFile(csv, "csv", currentTime)

perf-tests/src/test/scala/sttp/tapir/perf/Simulations.scala

@@ -5,7 +5,6 @@ import io.gatling.core.structure.PopulationBuilder
 import io.gatling.http.Predef._
 import sttp.tapir.perf.Common._
 
-import scala.concurrent.duration._
 import scala.util.Random
 import io.gatling.core.structure.ChainBuilder
 
@@ -34,31 +33,15 @@ object CommonSimulations {
       )
     )
 
-  private lazy val userCount = getParam("user-count").toInt
-  private lazy val duration = getParam("duration-seconds").toInt
-  private val httpProtocol = http.baseUrl(baseUrl)
-
-  // Scenarios
-  val warmUpScenario = scenario("Warm-Up Scenario")
-    .during(WarmupDuration)(
-      exec(
-        http("HTTP GET Warm-Up")
-          .get("/path0/1")
-      )
-        .exec(
-          http("HTTP POST Warm-Up")
-            .post("/path0")
-            .body(StringBody("warmup"))
-            .header("Content-Type", "text/plain")
-        )
-    )
-    .inject(atOnceUsers(3))
-    .protocols(httpProtocol)
+  def userCount = getParam("user-count").toInt
+  def duration = getParam("duration-seconds").toInt
+  def namePrefix = if (getParamOpt("is-warm-up").map(_.toBoolean) == Some(true)) "[WARMUP] " else ""
+  val httpProtocol = http.baseUrl(baseUrl)
 
   def scenario_simple_get(routeNumber: Int): PopulationBuilder = {
     val execHttpGet: ChainBuilder = exec(http(s"HTTP GET /path$routeNumber/4").get(s"/path$routeNumber/4"))
 
-    scenario(s"Repeatedly invoke GET of route number $routeNumber")
+    scenario(s"${namePrefix}Repeatedly invoke GET of route number $routeNumber")
       .during(duration)(execHttpGet)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -72,7 +55,7 @@ object CommonSimulations {
         .header("Content-Type", "text/plain")
     )
 
-    scenario(s"Repeatedly invoke POST with short string body")
+    scenario(s"${namePrefix}Repeatedly invoke POST with short string body")
       .during(duration)(execHttpPost)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -86,7 +69,7 @@ object CommonSimulations {
         .header("Content-Type", "text/plain") // otherwise Play complains
     )
 
-    scenario(s"Repeatedly invoke POST with short byte array body")
+    scenario(s"${namePrefix}Repeatedly invoke POST with short byte array body")
       .during(duration)(execHttpPost)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -100,7 +83,7 @@ object CommonSimulations {
         .header("Content-Type", "application/octet-stream")
     )
 
-    scenario(s"Repeatedly invoke POST with file body")
+    scenario(s"${namePrefix}Repeatedly invoke POST with file body")
       .during(duration)(execHttpPost)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -114,7 +97,7 @@ object CommonSimulations {
         .header("Content-Type", "text/plain") // otherwise Play complains
     )
 
-    scenario(s"Repeatedly invoke POST with large byte array")
+    scenario(s"${namePrefix}Repeatedly invoke POST with large byte array")
       .during(duration)(execHttpPost)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -128,7 +111,7 @@ object CommonSimulations {
         .header("Content-Type", "text/plain")
     )
 
-    scenario(s"Repeatedly invoke POST with large byte array, interpreted to a String")
+    scenario(s"${namePrefix}Repeatedly invoke POST with large byte array, interpreted to a String")
       .during(duration)(execHttpPost)
       .inject(atOnceUsers(userCount))
       .protocols(httpProtocol)
@@ -138,29 +121,29 @@ object CommonSimulations {
 import CommonSimulations._
 
 class SimpleGetSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_simple_get(0))): Unit
+  setUp(scenario_simple_get(0)): Unit
 }
 
 class SimpleGetMultiRouteSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_simple_get(127))): Unit
+  setUp(scenario_simple_get(127)): Unit
 }
 
 class PostBytesSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_post_bytes(0))): Unit
+  setUp(scenario_post_bytes(0)): Unit
 }
 
 class PostLongBytesSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_post_long_bytes(0))): Unit
+  setUp(scenario_post_long_bytes(0)): Unit
 }
 
 class PostFileSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_post_file(0))): Unit
+  setUp(scenario_post_file(0)): Unit
 }
 
 class PostStringSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_post_string(0))): Unit
+  setUp(scenario_post_string(0)): Unit
 }
 
 class PostLongStringSimulation extends Simulation {
-  setUp(warmUpScenario.andThen(scenario_post_long_string(0))): Unit
+  setUp(scenario_post_long_string(0)): Unit
 }