From 2667175d136ad2f748d807b6cf083eb6581bb5a9 Mon Sep 17 00:00:00 2001 From: Milad Khajavi Date: Sat, 11 May 2024 14:38:36 +0330 Subject: [PATCH] Enable Framework Benchmark Workflow (#2830) * enable benchmark. * remove extra label zio-http. * use ubuntu-latest. * remove extra steps temporary. * Revert "remove extra label zio-http." This reverts commit 2ff090e6b9ceefa84b22345b8a145f85d78a2fef. * test on personal repo. * remove condition. * remove extra labels. * clone the project. * workaround: git init. * git tag. * initial commit. * config git user and email. * do not freeze. * do not freeze plain text * remove extra stuffs. * revert changes. * update ci. * run only on pull request. * reduce the performance floor. * fmt. * remove trigger check on pull request events. * decrease performance floor to 300k/s. --- .github/workflows/ci.yml | 156 ++++++++++++++++++ build.sbt | 2 +- project/BenchmarkWorkFlow.scala | 28 +++- .../example/PlainTextBenchmarkServer.scala | 31 ++-- 4 files changed, 190 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6f91f0e3e..edfbf78596 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -689,3 +689,159 @@ jobs: with: path: Main.txt key: jmh_benchmarks_${{ github.sha }} + + runBenchmarks-simple: + name: Performance Benchmarks (PlainTextBenchmarkServer) + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.13.12] + java: [temurin@8] + runs-on: ${{ matrix.os }} + steps: + - name: Clean up + id: clean_up + env: + GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}} + run: sudo rm -rf * + + - uses: actions/checkout@v2 + with: + path: zio-http + + - uses: actions/checkout@v2 + with: + repository: khajavi/FrameworkBenchmarks + path: FrameworkBenchMarks + + - id: result + env: + GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}} + run: | + mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala + cp ./zio-http/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala + cd ./FrameworkBenchMarks + cd ./frameworks/Scala/zio-http + git init . + git config user.email 'benchamrk@example.com' + git config user.name 'ZIO Benchmark' + git add build.sbt + git commit -m 'initial commit' + git tag v1.0.0 + git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git + cd zio-http + git checkout ${{github.event.pull_request.head.sha}} + cd ../../../.. + ./tfb --test zio-http | tee result + RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1) + RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+") + echo "request_per_second=$RESULT_REQUEST" >> $GITHUB_OUTPUT + echo "concurrency=$RESULT_CONCURRENCY" >> $GITHUB_OUTPUT + + - if: ${{github.event.pull_request.head.repo.full_name == 'zio/zio-http'}} + uses: peter-evans/commit-comment@v2 + with: + sha: ${{github.event.pull_request.head.sha}} + body: | + + **🚀 :** Performance Benchmarks (PlainTextBenchmarkServer) + + concurrency: ${{steps.result.outputs.concurrency}} + requests/sec: ${{steps.result.outputs.request_per_second}} + + + - name: Performance Report + id: perf-report + env: + REQUESTS_PER_SECOND: ${{steps.result.outputs.request_per_second}} + CONCURRENCY: ${{steps.result.outputs.concurrency}} + PERFORMANCE_FLOOR: 300000 + run: | + echo "** 🚀 Performance Benchmarks (PlainTextBenchmarkServer) Report 🚀 **" + + echo "$REQUESTS_PER_SECOND requests/sec for $CONCURRENCY concurrent requests" + + if (( REQUESTS_PER_SECOND > PERFORMANCE_FLOOR )); then + echo "Woohoo! Performance is good! $REQUESTS_PER_SECOND requests/sec exceeds the performance floor of $PERFORMANCE_FLOOR requests/sec." + else + echo "Performance benchmark failed with $REQUESTS_PER_SECOND req/sec! Performance must exceed $PERFORMANCE_FLOOR req/sec." + exit 1 + fi + + runBenchmarks-effectful: + name: Performance Benchmarks (SimpleEffectBenchmarkServer) + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.13.12] + java: [temurin@8] + runs-on: ${{ matrix.os }} + steps: + - name: Clean up + id: clean_up + env: + GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}} + run: sudo rm -rf * + + - uses: actions/checkout@v2 + with: + path: zio-http + + - uses: actions/checkout@v2 + with: + repository: khajavi/FrameworkBenchmarks + path: FrameworkBenchMarks + + - id: result + env: + GITHUB_TOKEN: ${{secrets.ACTIONS_PAT}} + run: | + mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala + cp ./zio-http/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala + cd ./FrameworkBenchMarks + cd ./frameworks/Scala/zio-http + git init . + git config user.email 'benchamrk@example.com' + git config user.name 'ZIO Benchmark' + git add build.sbt + git commit -m 'initial commit' + git tag v1.0.0 + git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git + cd zio-http + git checkout ${{github.event.pull_request.head.sha}} + cd ../../../.. + ./tfb --test zio-http | tee result + RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1) + RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+") + echo "request_per_second=$RESULT_REQUEST" >> $GITHUB_OUTPUT + echo "concurrency=$RESULT_CONCURRENCY" >> $GITHUB_OUTPUT + + - if: ${{github.event.pull_request.head.repo.full_name == 'zio/zio-http'}} + uses: peter-evans/commit-comment@v2 + with: + sha: ${{github.event.pull_request.head.sha}} + body: | + + **🚀 :** Performance Benchmarks (SimpleEffectBenchmarkServer) + + concurrency: ${{steps.result.outputs.concurrency}} + requests/sec: ${{steps.result.outputs.request_per_second}} + + + - name: Performance Report + id: perf-report + env: + REQUESTS_PER_SECOND: ${{steps.result.outputs.request_per_second}} + CONCURRENCY: ${{steps.result.outputs.concurrency}} + PERFORMANCE_FLOOR: 300000 + run: | + echo "** 🚀 Performance Benchmarks (SimpleEffectBenchmarkServer) Report 🚀 **" + + echo "$REQUESTS_PER_SECOND requests/sec for $CONCURRENCY concurrent requests" + + if (( REQUESTS_PER_SECOND > PERFORMANCE_FLOOR )); then + echo "Woohoo! Performance is good! $REQUESTS_PER_SECOND requests/sec exceeds the performance floor of $PERFORMANCE_FLOOR requests/sec." + else + echo "Performance benchmark failed with $REQUESTS_PER_SECOND req/sec! Performance must exceed $PERFORMANCE_FLOOR req/sec." + exit 1 + fi diff --git a/build.sbt b/build.sbt index b076e681ff..eecdaa1e04 100644 --- a/build.sbt +++ b/build.sbt @@ -28,7 +28,7 @@ ThisBuild / githubWorkflowAddedJobs := steps = List(WorkflowStep.Use(UseRef.Public("release-drafter", "release-drafter", s"v${releaseDrafterVersion}"))), cond = Option("${{ github.base_ref == 'main' }}"), ), - ) ++ ScoverageWorkFlow(50, 60) ++ JmhBenchmarkWorkflow(1) // ++ BenchmarkWorkFlow() + ) ++ ScoverageWorkFlow(50, 60) ++ JmhBenchmarkWorkflow(1) ++ BenchmarkWorkFlow() ThisBuild / githubWorkflowTargetTags ++= Seq("v*") ThisBuild / githubWorkflowPublishTargetBranches += RefPredicate.StartsWith(Ref.Tag("v")) diff --git a/project/BenchmarkWorkFlow.scala b/project/BenchmarkWorkFlow.scala index 06b40f8029..0a6c98606e 100644 --- a/project/BenchmarkWorkFlow.scala +++ b/project/BenchmarkWorkFlow.scala @@ -6,26 +6,26 @@ object BenchmarkWorkFlow { makeBenchmarkPass( "runBenchmarks-simple", "Performance Benchmarks (PlainTextBenchmarkServer)", - 800000, + 300000, "PlainTextBenchmarkServer", ), makeBenchmarkPass( "runBenchmarks-effectful", "Performance Benchmarks (SimpleEffectBenchmarkServer)", - 500000, + 300000, "SimpleEffectBenchmarkServer", ), ) private def makeBenchmarkPass(id: String, name: String, performanceFloor: Int, server: String) = WorkflowJob( - runsOnExtraLabels = List("zio-http"), +// runsOnExtraLabels = List("zio-http"), id = id, name = name, - oses = List("centos"), - cond = Some( - "${{ github.event_name == 'pull_request'}}", - ), + oses = List("ubuntu-latest"), +// cond = Some( +// "${{ github.event_name == 'pull_request'}}", +// ), scalas = List(Scala213), steps = List( WorkflowStep.Run( @@ -43,7 +43,7 @@ object BenchmarkWorkFlow { WorkflowStep.Use( UseRef.Public("actions", "checkout", s"v2"), Map( - "repository" -> "zio/FrameworkBenchmarks", + "repository" -> "khajavi/FrameworkBenchmarks", "path" -> "FrameworkBenchMarks", ), ), @@ -54,7 +54,17 @@ object BenchmarkWorkFlow { "mkdir -p ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala", s"cp ./zio-http/zio-http-example/src/main/scala/example/${server}.scala ./FrameworkBenchMarks/frameworks/Scala/zio-http/src/main/scala/Main.scala", "cd ./FrameworkBenchMarks", - """sed -i "s/---COMMIT_SHA---/${{github.event.pull_request.head.repo.owner.login}}\/zio-http.git#${{github.event.pull_request.head.sha}}/g" frameworks/Scala/zio-http/build.sbt""", + "cd ./frameworks/Scala/zio-http", + "git init .", + "git config user.email 'benchamrk@example.com'", + "git config user.name 'ZIO Benchmark'", + "git add build.sbt", + "git commit -m 'initial commit'", + "git tag v1.0.0", + "git clone https://github.com/${{github.event.pull_request.head.repo.owner.login}}/zio-http.git", + "cd zio-http", + "git checkout ${{github.event.pull_request.head.sha}}", + "cd ../../../..", "./tfb --test zio-http | tee result", """RESULT_REQUEST=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "requests/sec: [0-9]+.[0-9]+" | grep -oiE "[0-9]+" | head -1)""", """RESULT_CONCURRENCY=$(echo $(grep -B 1 -A 17 "Concurrency: 256 for plaintext" result) | grep -oiE "concurrency: [0-9]+" | grep -oiE "[0-9]+")""", diff --git a/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala b/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala index c76af9aba7..a21caa0eae 100644 --- a/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala +++ b/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala @@ -19,23 +19,20 @@ object PlainTextBenchmarkServer extends ZIOAppDefault { private val STATIC_SERVER_NAME = "zio-http" - private val frozenJsonResponse = Response - .json(jsonMessage) -// .serverTime - .addHeader(Header.Server(STATIC_SERVER_NAME)) - - private val frozenPlainTextResponse = Response - .text(plainTextMessage) -// .serverTime - .addHeader(Header.Server(STATIC_SERVER_NAME)) - - private def plainTextApp(response: Response): Routes[Any, Response] = - Routes(Method.GET / plaintextPath -> Handler.fromResponse(response)) - - private def jsonApp(json: Response): Routes[Any, Response] = - Routes(Method.GET / jsonPath -> Handler.fromResponse(json)) - - val app: Routes[Any, Response] = plainTextApp(frozenPlainTextResponse) ++ jsonApp(frozenJsonResponse) + val app: Routes[Any, Response] = Routes( + Method.GET / plaintextPath -> + Handler.fromResponse( + Response + .text(plainTextMessage) + .addHeader(Header.Server(STATIC_SERVER_NAME)), + ), + Method.GET / jsonPath -> + Handler.fromResponse( + Response + .json(jsonMessage) + .addHeader(Header.Server(STATIC_SERVER_NAME)), + ), + ) private val config = Server.Config.default .port(8080)