Merge af9f630 into 0d03653

SaltieRL · May 1, 2020 · b7c55ce · b7c55ce
2 parents 0d03653 + af9f630
commit b7c55ce
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 9 deletions.
diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
@@ -2,11 +2,14 @@ name: Benchmarking
 
 on:
   push:
-    branches: [ master, action-benchmark ]
+    branches: [ master, benchmarking-data ]
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        benchtest: [short_sample, short_dropshot, full_rumble, oce_rlcs]
 
     steps:
     - uses: actions/checkout@v2
@@ -40,7 +43,38 @@ jobs:
     - name: Run benchmark.
       run: |
         cd carball/tests/benchmarking
-        pytest benchmarking.py
+        pytest benchmarking.py --benchmark-json carball_benchmarking.json -k 'test_${{matrix.benchtest}}'
 
-#    TO BE DONE.
-#    - name: Store benchmark result(s).
+    # always comment on every push
+    - name: Process and store benchmark ${{matrix.benchtest}}.
+      uses: rhysd/github-action-benchmark@v1
+      with:
+        name: Carball Benchmarks ${{matrix.benchtest}}
+        tool: 'pytest'
+        output-file-path: carball/tests/benchmarking/carball_benchmarking.json
+        benchmark-data-dir-path: dev/bench/${{matrix.benchtest}}/
+        # Use personal access token instead of GITHUB_TOKEN due to
+        # https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        auto-push: false
+        alert-threshold: '125%' # If performance is 1.25x worse, there will be an alert
+        comment-always: true
+        fail-on-alert: true
+
+    # REQUIRES gh-pages BRANCH IN THE MAIN REPO (for the graphs).
+    # Only upload to master branch on push
+    - name: Process and store benchmark ${{matrix.benchtest}}.
+      if: contains(github.ref, 'master')
+      uses: rhysd/github-action-benchmark@v1
+      with:
+        name: Carball Benchmarks ${{matrix.benchtest}}
+        tool: 'pytest'
+        output-file-path: carball/tests/benchmarking/carball_benchmarking.json
+        benchmark-data-dir-path: dev/bench/${{matrix.benchtest}}/
+        # Use personal access token instead of GITHUB_TOKEN due to
+        # https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false
+        github-token: ${{ secrets.GH_WIKI_EDIT_TOKEN }}
+        auto-push: true
+        alert-threshold: '125%' # If performance is 1.25x worse, there will be an alert
+        comment-always: false
+        comment-on-alert: true
diff --git a/carball/tests/benchmarking/benchmarking.py b/carball/tests/benchmarking/benchmarking.py
@@ -14,32 +14,32 @@ def test_short_sample(benchmark):
     replay_path = get_replay_path("SHORT_SAMPLE.replay")
 
     benchmark.pedantic(analyze_replay_file,
-                       kwargs={"replay_path":replay_path}, rounds=50, iterations=3)
+                       kwargs={"replay_path":replay_path}, rounds=10, iterations=3)
 
 
 def test_short_dropshot(benchmark):
     replay_path = get_replay_path("DROPSHOT_PHASE2_BALL.replay")
 
     benchmark.pedantic(analyze_replay_file,
-                       kwargs={"replay_path":replay_path}, rounds=30, iterations=3)
+                       kwargs={"replay_path":replay_path}, rounds=10, iterations=3)
 
 
 def test_full_rumble(benchmark):
     replay_path = get_replay_path("RUMBLE_FULL.replay")
 
     benchmark.pedantic(analyze_replay_file,
-                       kwargs={"replay_path":replay_path}, rounds=20, iterations=3)
+                       kwargs={"replay_path":replay_path}, rounds=10, iterations=3)
 
 
 def test_oce_rlcs(benchmark):
     replay_path = get_replay_path("OCE_RLCS_7_CARS.replay")
 
     benchmark.pedantic(analyze_replay_file,
-                       kwargs={"replay_path":replay_path}, rounds=20, iterations=3)
+                       kwargs={"replay_path":replay_path}, rounds=10, iterations=3)
 
 
 def test_oce_rlcs_intensive(benchmark):
     replay_path = get_replay_path("OCE_RLCS_7_CARS.replay")
 
     benchmark.pedantic(analyze_replay_file,
-                       kwargs={"replay_path":replay_path, "calculate_intensive_events":True}, rounds=5, iterations=3)
+                       kwargs={"replay_path":replay_path, "calculate_intensive_events":True}, rounds=5, iterations=1)