diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 4d2d9f4a..bfa6ba00 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -2,11 +2,14 @@ name: Benchmarking on: push: - branches: [ master, action-benchmark ] + branches: [ master, benchmarking-data ] jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + benchtest: [short_sample, short_dropshot, full_rumble, oce_rlcs] steps: - uses: actions/checkout@v2 @@ -40,7 +43,38 @@ jobs: - name: Run benchmark. run: | cd carball/tests/benchmarking - pytest benchmarking.py + pytest benchmarking.py --benchmark-json carball_benchmarking.json -k 'test_${{matrix.benchtest}}' -# TO BE DONE. -# - name: Store benchmark result(s). \ No newline at end of file + # always comment on every push + - name: Process and store benchmark ${{matrix.benchtest}}. + uses: rhysd/github-action-benchmark@v1 + with: + name: Carball Benchmarks ${{matrix.benchtest}} + tool: 'pytest' + output-file-path: carball/tests/benchmarking/carball_benchmarking.json + benchmark-data-dir-path: dev/bench/${{matrix.benchtest}}/ + # Use personal access token instead of GITHUB_TOKEN due to + # https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: false + alert-threshold: '125%' # If performance is 1.25x worse, there will be an alert + comment-always: true + fail-on-alert: true + + # REQUIRES gh-pages BRANCH IN THE MAIN REPO (for the graphs). + # Only upload to master branch on push + - name: Process and store benchmark ${{matrix.benchtest}}. + if: contains(github.ref, 'master') + uses: rhysd/github-action-benchmark@v1 + with: + name: Carball Benchmarks ${{matrix.benchtest}} + tool: 'pytest' + output-file-path: carball/tests/benchmarking/carball_benchmarking.json + benchmark-data-dir-path: dev/bench/${{matrix.benchtest}}/ + # Use personal access token instead of GITHUB_TOKEN due to + # https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false + github-token: ${{ secrets.GH_WIKI_EDIT_TOKEN }} + auto-push: true + alert-threshold: '125%' # If performance is 1.25x worse, there will be an alert + comment-always: false + comment-on-alert: true diff --git a/carball/tests/benchmarking/benchmarking.py b/carball/tests/benchmarking/benchmarking.py index 24a5d785..df11b978 100644 --- a/carball/tests/benchmarking/benchmarking.py +++ b/carball/tests/benchmarking/benchmarking.py @@ -14,32 +14,32 @@ def test_short_sample(benchmark): replay_path = get_replay_path("SHORT_SAMPLE.replay") benchmark.pedantic(analyze_replay_file, - kwargs={"replay_path":replay_path}, rounds=50, iterations=3) + kwargs={"replay_path":replay_path}, rounds=10, iterations=3) def test_short_dropshot(benchmark): replay_path = get_replay_path("DROPSHOT_PHASE2_BALL.replay") benchmark.pedantic(analyze_replay_file, - kwargs={"replay_path":replay_path}, rounds=30, iterations=3) + kwargs={"replay_path":replay_path}, rounds=10, iterations=3) def test_full_rumble(benchmark): replay_path = get_replay_path("RUMBLE_FULL.replay") benchmark.pedantic(analyze_replay_file, - kwargs={"replay_path":replay_path}, rounds=20, iterations=3) + kwargs={"replay_path":replay_path}, rounds=10, iterations=3) def test_oce_rlcs(benchmark): replay_path = get_replay_path("OCE_RLCS_7_CARS.replay") benchmark.pedantic(analyze_replay_file, - kwargs={"replay_path":replay_path}, rounds=20, iterations=3) + kwargs={"replay_path":replay_path}, rounds=10, iterations=3) def test_oce_rlcs_intensive(benchmark): replay_path = get_replay_path("OCE_RLCS_7_CARS.replay") benchmark.pedantic(analyze_replay_file, - kwargs={"replay_path":replay_path, "calculate_intensive_events":True}, rounds=5, iterations=3) + kwargs={"replay_path":replay_path, "calculate_intensive_events":True}, rounds=5, iterations=1)