Skip to content

Simulator Tests

Simulator Tests #29

name: Simulator Tests
concurrency:
group: ${{ github.workflow }}
on:
schedule:
- cron: '0 9 * * *' # equivalent to 1am PST
workflow_dispatch:
inputs:
sui_ref:
description: "Branch / commit to test"
type: string
required: true
default: main
test_num:
description: "MSIM_TEST_NUM (test iterations)"
type: string
required: false
default: "30"
env:
SUI_REF: "${{ github.event.inputs.sui_ref || 'main' }}"
TEST_NUM: "${{ github.event.inputs.test_num || '30' }}"
jobs:
simtest:
# Allow running the job for up to 6 hours, maximum for GitHub Actions.
timeout-minutes: 360
permissions:
# The "id-token: write" permission is required or Machine ID will not be
# able to authenticate with the cluster.
id-token: write
contents: read
runs-on: ubuntu-latest
steps:
- name: Install Teleport
uses: teleport-actions/setup@176c25dfcd19cd31a252f275d579822b243e7b9c # [email protected]
with:
version: 11.3.1
- name: Authorize against Teleport
id: auth
uses: teleport-actions/auth@9091dad16a564f3c5b9c2ec520b234a4872b6879 # pin@v1
with:
# Specify the publically accessible address of your Teleport proxy.
proxy: proxy.mysten-int.com:443
# Specify the name of the join token for your bot.
token: sui-simtest-token
# Specify the length of time that the generated credentials should be
# valid for. This is optional and defaults to "1h"
certificate-ttl: 2h
# Cargo clean and git restore on any left over files from git checkout, and deletes all remote tracking branches
- name: Environment clean
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && rm -rf ~/sui"
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/ && git clone [email protected]:MystenLabs/sui.git"
# Deleting files in tmpfs that usually fill up pretty quickly after each run. Cargo clean to free up space as well.
- name: Tmpfs and cargo clean
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "sudo rm -rf /tmp/*"
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 5 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && cargo clean"
# Checkout out the latest sui repo
- name: Checkout sui repo
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && git fetch origin && git checkout ${{ env.SUI_REF }}"
# Setting up cargo and simtest
- name: Install simtest
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 10 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && ./scripts/simtest/install.sh"
# Run simulator tests
- name: Run simtest
run: |
tsh -i ${{ steps.auth.outputs.identity-file }} --ttl 120 ssh ubuntu@simtest-01 "source ~/.bashrc && source ~/.cargo/env && cd ~/sui && RUSTUP_MAX_RETRIES=10 CARGO_TERM_COLOR=always CARGO_INCREMENTAL=0 CARGO_NET_RETRY=10 RUST_BACKTRACE=short RUST_LOG=off NUM_CPUS=24 TEST_NUM=${{ env.TEST_NUM }} ./scripts/simtest/simtest-run.sh"
notify:
name: Notify
needs: [simtest]
runs-on: ubuntu-latest
if: github.event_name == 'schedule' && failure()
steps:
- uses: technote-space/workflow-conclusion-action@45ce8e0eb155657ab8ccf346ade734257fd196a5 # Pin v4.1.1
- name: Checkout sui repo main branch
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
- name: Get sui commit
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
export sui_sha=$(git rev-parse HEAD)
echo "sui_sha=${sui_sha}" >> $GITHUB_ENV
- name: Get link to logs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh_job_link=$(gh api -X GET 'repos/MystenLabs/sui/actions/runs/${{ github.run_id }}/jobs' --jq '.jobs.[0].html_url')
echo "gh_job_link=${gh_job_link}" >> $GITHUB_ENV
- name: Get current oncall
run: |
export current_oncall=$(curl -s --request GET \
--url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \
--header 'Accept: application/json' \
--header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \
--header 'Content-Type: application/json' \
| jq '.oncalls[].user.summary' | tr -d '"')
echo "current_oncall=$(echo ${current_oncall})" >> $GITHUB_ENV
export oncall_name=$(curl -s --request GET \
--url 'https://api.pagerduty.com/oncalls?schedule_ids[]=PGCQ3YS' \
--header 'Accept: application/json' \
--header 'Authorization: Token token=${{ secrets.PAGERDUTY_ACCESS_KEY }}' \
--header 'Content-Type: application/json' \
| jq '.oncalls[].escalation_policy.summary' | tr -d '"')
echo "oncall_name=$(echo ${oncall_name})" >> $GITHUB_ENV
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # pin v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Get slack id for the oncall
run: |
export slack_id=$(aws s3 cp s3://mysten-employees-dir/employees.json - | jq --arg ONCALL "${{ env.current_oncall }}" '.[] | if .name == $ONCALL then .slack_id else empty end')
echo "slack_id=$(echo ${slack_id} | tr -d '"')" >> $GITHUB_ENV
- name: Post to slack
uses: slackapi/slack-github-action@34c3fd73326693ef04728f8611669d918a2d781d # [email protected]
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SUI_SHA: ${{ env.sui_sha }}
GH_JOB_LINK: ${{ env.gh_job_link }}
SLACK_ID: ${{ env.slack_id }}
ONCALL_NAME: ${{ env.oncall_name }}
with:
channel-id: 'simtest-nightly'
slack-message: |
:sui::error-cross: *${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`
Sui commit: <https://github.com/MystenLabs/sui/commit/${{ env.SUI_SHA }}|${{ env.SUI_SHA }}>
Run: <${{ env.GH_JOB_LINK }}|${{ github.run_id }}>
<@${{ env.SLACK_ID }}>, current `${{ env.ONCALL_NAME }}` oncall, please debug failures: `tsh ssh ubuntu@simtest-01` and look in the `/home/ubuntu/simtest_logs/{date}` folder for test results