Multireaderfilestream Redesign #8104
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Test | |
on: | |
push: | |
branches: [ master ] | |
pull_request: | |
jobs: | |
format: | |
name: Format | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-node@v3 | |
with: | |
node-version: 14.x | |
- name: Set up Python 3.7 | |
uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- run: npm ci | |
working-directory: ./frontend | |
- run: npm run check-ci | |
working-directory: ./frontend | |
env: | |
CI: true | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.dev.txt') }} | |
restore-keys: | | |
pip- | |
- run: ./pre-commit.sh && git diff --exit-code | |
install: | |
name: Install | |
runs-on: ${{ matrix.os }} | |
strategy: | |
matrix: | |
python-version: [3.7, 3.8, 3.9] | |
os: [ubuntu-20.04, macos-latest] | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} | |
restore-keys: | | |
pip- | |
- run: pip install -e . | |
- run: cl | |
test_frontend: | |
name: Test Frontend | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-node@v3 | |
with: | |
node-version: 14.x | |
- run: npm ci | |
working-directory: ./frontend | |
- run: npm test | |
working-directory: ./frontend | |
env: | |
CI: true | |
build: | |
name: Build | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
service: [rest-server, worker, frontend] | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- run: python3 codalab_service.py build --pull --version ${VERSION} -s ${SERVICE} $([ -z "${CODALAB_DOCKER_USERNAME}" ] || echo "--push") | |
env: | |
CODALAB_DOCKER_USERNAME: ${{ secrets.CODALAB_DOCKER_USERNAME }} | |
CODALAB_DOCKER_PASSWORD: ${{ secrets.CODALAB_DOCKER_PASSWORD }} | |
# Gives us the branch name of the PR if on a pull_request-triggered build, | |
# otherwise, "master" if on a push-triggered build | |
VERSION: ${{ github.head_ref || 'master' }} | |
SERVICE: ${{ matrix.service }} | |
test_backend: | |
name: Test backend | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- unittest gen-rest-docs gen-cli-docs gen-readthedocs basic auth status batch anonymous competition unicode rest1 upload1 upload2 upload3 upload4 download disk | |
- refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups | |
- worker_manager service | |
- run time | |
- run2 | |
- search link read kill write mimic workers edit_user sharing_workers | |
- resources | |
- memoize | |
- copy | |
- netcat netcurl | |
- edit | |
- open wopen | |
- store_add parallel | |
runtime: [docker, kubernetes] | |
exclude: | |
# netcat / netcurl not supported for kubernetes. | |
- test: netcat netcurl | |
runtime: kubernetes | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests using Docker runtime | |
if: matrix.runtime == 'docker' | |
run: | | |
sh ./tests/test-setup.sh | |
python3 codalab_service.py start --services default --version ${VERSION} | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_LINK_MOUNTS: /tmp | |
- uses: actions/setup-go@v3 | |
if: matrix.runtime == 'kubernetes' | |
with: | |
go-version: '1.18.1' | |
- name: Run tests using Kubernetes runtime | |
if: matrix.runtime == 'kubernetes' | |
run: | | |
sh ./tests/test-setup.sh | |
sh ./scripts/local-k8s/setup-ci.sh | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_LINK_MOUNTS: /tmp | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Save kubernetes logs | |
if: (failure() || cancelled()) && matrix.runtime == 'kubernetes' | |
run: | | |
kubectl config use-context kind-codalab | |
kubectl cluster-info dump --output-directory /tmp/logs | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.runtime }}-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_on_worker_restart: | |
name: Test backend - on worker restart | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: [run] | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
# Make sure restarting worker doesn't cause any issues (ie in serialization/deserialization) | |
run: | | |
python3 codalab_service.py start --services default --version ${VERSION} | |
docker restart codalab_worker_1 | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Save logs | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_sharedfs: | |
name: Test backend - shared FS | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: [run,run2,link read write kill resources] | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run shared filesystem tests | |
run: | | |
sh ./tests/test-setup.sh | |
python3 codalab_service.py start --services default --version ${VERSION} --shared-file-system | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_LINK_MOUNTS: /tmp | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-sharedfs-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_protected_mode: | |
name: Test backend - protected mode | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- basic status batch anonymous unicode rest1 upload1 download disk | |
- refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups | |
- run | |
- search read kill write mimic workers | |
- copy netcat | |
- protected_mode | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
python3 codalab_service.py start --services default --version ${VERSION} --protected-mode | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-protectedmode-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_default_bundle_store: | |
name: Test backend - default bundle store | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- default_bundle_store | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
CODALAB_DEFAULT_BUNDLE_STORE_NAME=store$(date +%s) python3 codalab_service.py start --services default --version ${VERSION} --protected-mode | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_default_bundle_store_azure: | |
name: Test backend - use azure as default bundle store | |
runs-on: ubuntu-20.04 | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- upload1 upload2 upload3 upload4 download | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- run: pip install -e . | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
python3 codalab_service.py start --services default azurite --version ${VERSION} | |
sh ./tests/test-setup-default-store.sh | |
CODALAB_DEFAULT_BUNDLE_STORE_NAME=azure-store-default python3 codalab_service.py start --services default azurite --version ${VERSION} | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_USERNAME: codalab | |
CODALAB_PASSWORD: codalab | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_preemptible_worker: | |
name: Test backend - preemptible workers | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- preemptible | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- run: pip install -e . | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
python3 codalab_service.py start --services default no-worker worker-preemptible --version ${VERSION} | |
sleep 20 | |
python3 codalab_service.py start --services worker-preemptible2 --version ${VERSION} | |
./tests/test-setup-preemptible.sh | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_USERNAME: codalab | |
CODALAB_PASSWORD: codalab | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.test }} | |
path: /tmp/logs | |
test_backend_azure_blob: | |
name: Test backend with Azure Blob Storage | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: | |
- unittest gen-rest-docs gen-cli-docs gen-readthedocs basic auth status batch anonymous competition unicode rest1 upload1 upload2 upload3 upload4 download disk | |
- refs binary rm make worksheet_search worksheet_tags bundle_freeze_unfreeze worksheet_freeze_unfreeze detach perm search_time groups | |
- worker_manager service | |
- run time | |
- run2 | |
- search read kill write mimic workers edit_user sharing_workers | |
# - search link read kill write mimic workers edit_user sharing_workers | |
- resources | |
- memoize | |
- copy netcat netcurl | |
- edit blob | |
- open wopen | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
python3 codalab_service.py start --services default azurite --version ${VERSION} | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
CODALAB_LINK_MOUNTS: /tmp | |
CODALAB_ALWAYS_USE_AZURE_BLOB_BETA: 1 | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-azblob-${{ matrix.test }} | |
path: /tmp/logs | |
test_ui: | |
name: End-to-end UI Tests | |
runs-on: ubuntu-latest | |
needs: [build] | |
strategy: | |
matrix: | |
test: [frontend] | |
steps: | |
- name: Clear free space | |
run: | | |
sudo rm -rf /opt/ghc | |
df -h | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: 3.7 | |
- uses: actions/cache@v3 | |
with: | |
path: ~/.cache/pip | |
key: pip-${{ hashFiles('requirements.txt') }} | |
restore-keys: | | |
pip- | |
- run: pip install -r requirements.txt | |
- name: Setup tests | |
run: | | |
sudo service mysql stop | |
python3 codalab_service.py build services --version ${VERSION} --pull | |
env: | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Run tests | |
run: | | |
python3 codalab_service.py start --services default --version ${VERSION} | |
docker exec codalab_rest-server_1 /bin/bash -c "python3 scripts/create_sample_worksheet.py --test-print" | |
python3 test_runner.py --version ${VERSION} ${TEST} | |
env: | |
TEST: ${{ matrix.test }} | |
VERSION: ${{ github.head_ref || 'master' }} | |
- name: Upload screenshots on failure | |
uses: actions/upload-artifact@v1 | |
if: failure() | |
with: | |
name: screenshots-test-${{ matrix.test }} | |
path: tests/ui | |
- name: Save logs | |
if: (failure() || cancelled()) | |
run: | | |
mkdir /tmp/logs | |
for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done | |
- name: Upload logs | |
if: (failure() || cancelled()) | |
uses: actions/upload-artifact@v1 | |
with: | |
name: logs-test-${{ matrix.test }} | |
path: /tmp/logs | |
ci: | |
name: All CI tasks complete | |
runs-on: ubuntu-latest | |
needs: [format, install, test_frontend, build, test_backend, test_backend_on_worker_restart, test_backend_sharedfs, test_backend_protected_mode, test_ui] | |
steps: | |
- uses: actions/checkout@v3 | |
- run: echo Done |