Skip to content
Build and Test

refactor: changing mode in volume is enough #3758

Sign in to view logs

refactor: changing mode in volume is enough

refactor: changing mode in volume is enough #3758

Workflow file for this run

.github/workflows/build.yml at a025d1b
name: Build and Test
on:
push:
branches-ignore:
- release
paths-ignore:
- .github/workflows/manual-integration-test.yml
tags-ignore: [ '**' ]
jobs:
versionning:
name: Versionning
runs-on: ubuntu-latest
outputs:
version: ${{ steps.genver.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
fetch-depth: 0
- name: Generate Version
id: genver
uses: codacy/git-version@80c816f11db8dea5e3a81025f598193015b51832 # 2.8.0
with:
minor-identifier: "feat:"
release-branch: release
dev-branch: main
- name: Print version
run: echo VERSION ${{ steps.genver.outputs.version }} >> $GITHUB_STEP_SUMMARY
tests:
strategy:
matrix:
projects:
- Common/tests
- Adaptors/MongoDB/tests
- Adaptors/Memory/tests
- Adaptors/S3/tests
os:
- ubuntu-latest
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Minio Server UP
if: ${{ matrix.projects }} == "Adaptors/S3/tests"
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just object=minio deployTargetObject
- name: Run tests
run: |
MONITOR_PREFIX="monitor/test/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \
dotnet test --logger "trx;LogFileName=test-results.trx" -p:RunAnalyzers=false -p:WarningLevel=0
- name: Test Report
uses: dorny/test-reporter@v1
if: success() || failure()
with:
name: Test - ${{ matrix.os }} ${{ matrix.projects }}
path: ${{ matrix.projects }}/TestResults/test-results.trx
reporter: dotnet-trx
- name: Upload monitor profile
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
project="$(echo ${{ matrix.projects }} | sed 's:/:_:')"
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/tests-$project-monitor.tar.gz
testsQueueProtos:
strategy:
matrix:
queue:
- activemq
- rabbitmq
projects:
- Adaptors/Amqp/tests
os:
- ubuntu-latest
include:
- queue: rabbitmq091
projects: Adaptors/RabbitMQ/tests
os: ubuntu-latest
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Set up queue
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just queue=${{ matrix.queue }} deployTargetQueue
- name: Run tests
run: |
MONITOR_PREFIX="monitor/test/" MONITOR_CD=${{ matrix.projects }} tools/monitor.sh \
dotnet test --logger "trx;LogFileName=test-results.trx" -p:RunAnalyzers=false -p:WarningLevel=0
- name: Test Report
uses: dorny/test-reporter@v1
if: success() || failure()
with:
name: Test - ${{ matrix.queue }} ${{ matrix.projects }}
path: ${{ matrix.projects }}/TestResults/test-results.trx
reporter: dotnet-trx
- name: Upload monitor profile
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
project="$(echo ${{ matrix.projects }} | sed 's:/:_:')"
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/test-${{ matrix.queue }}-$project-monitor.tar.gz
testsWinOnly:
strategy:
matrix:
projects:
- Adaptors/Redis/tests
- Common/tests
- Adaptors/MongoDB/tests
- Adaptors/Memory/tests
fail-fast: false
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Run tests
run: |
cd ${{ matrix.projects }}
dotnet test --logger "trx;LogFileName=test-results.trx" -p:RunAnalyzers=false -p:WarningLevel=0
- name: Test Report
uses: dorny/test-reporter@v1
if: success() || failure()
with:
name: Test - windows ${{ matrix.projects }}
path: ${{ matrix.projects }}/TestResults/test-results.trx
reporter: dotnet-trx
buildProjects:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Build the solution
run: dotnet build ArmoniK.Core.sln -c Release
publish-nuget:
runs-on: ubuntu-latest
needs:
- versionning
- buildProjects
env:
VERSION: ${{ needs.versionning.outputs.version }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
- name: Build the package
run: |
dotnet build Base/src/ArmoniK.Core.Base.csproj -c Release -p:RunAnalyzers=false -p:WarningLevel=0
- name: Pack the package
run: |
dotnet pack Base/src/ArmoniK.Core.Base.csproj -c Release -o /tmp/packages -p:PackageVersion=$VERSION -p:Version=$VERSION
- name: Push the package
run: dotnet nuget push /tmp/packages/ArmoniK.Core.*.nupkg -k ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json
images:
runs-on: ubuntu-latest
needs:
- versionning
- buildProjects
env:
VERSION: ${{ needs.versionning.outputs.version }}
strategy:
fail-fast: true
matrix:
type:
- build-core
- buildBenchClient
- worker=bench buildWorker
- buildStreamClient
- worker=stream buildWorker
- buildHtcmockClient
- worker=htcmock buildWorker
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: login
run: echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_LOGIN }} --password-stdin
- name: Build
run: |
just tag=$VERSION platform=linux/arm64,linux/amd64 load=false push=true ${{ matrix.type }}
testStreamDC:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
name: Stream ${{ matrix.queue }} ${{ matrix.log-level }}
strategy:
fail-fast: false
matrix:
queue:
- activemq
- rabbitmq
- rabbitmq091
log-level:
- Information
- Verbose
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just log_level=${{ matrix.log-level }} tag=${VERSION} queue=${{ matrix.queue }} worker=stream deploy
sleep 10
- name: Run Stream Test
timeout-minutes: 4
run: |
export GrpcClient__Endpoint=http://localhost:5001
export Partition=TestPartition0
MONITOR_PREFIX="monitor/test/" MONITOR_CD=./Tests/Stream/Client/ tools/monitor.sh \
dotnet test --logger "trx;LogFileName=test-results.trx" -p:RunAnalyzers=false -p:WarningLevel=0
- name: Test Report
uses: dorny/test-reporter@v1
if: success() || failure()
with:
name: Test - ${{ matrix.queue }} x ${{ matrix.log-level }}
path: ./Tests/Stream/Client/TestResults/test-results.trx
reporter: dotnet-trx
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/stream-${{ matrix.queue }}-${{ matrix.log-level }}.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/stream-${{ matrix.queue }}-${{ matrix.log-level }}-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/stream-${{ matrix.queue }}-${{ matrix.log-level }}-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/stream-${{ matrix.queue }}-${{ matrix.log-level }}-database.tar.gz
testHtcMockDC:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
queue:
- activemq
- rabbitmq
- rabbitmq091
- pubsub
object:
- redis
- minio
log-level:
- Information
- Verbose
name: HtcMock ${{ matrix.queue }} ${{ matrix.object }} ${{ matrix.log-level }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just log_level=${{ matrix.log-level }} tag=${VERSION} queue=${{ matrix.queue }} object=${{ matrix.object }} worker=htcmock deploy
sleep 10
- name: Print And Time Metrics
run: |
set -x
time curl localhost:5002/metrics
time curl localhost:5003/metrics
- name: Run HtcMock test 100 tasks 1 level
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-100-1/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=1 \
-e HtcMock__Partition=TestPartition0 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Run HtcMock test 100 tasks 4 levels
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-100-4/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=4 \
-e HtcMock__Partition=TestPartition0 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Run HtcMock test 100 tasks 4 levels with retries
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-100-4-r/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=4 \
-e HtcMock__Partition=TestPartition0 \
-e HtcMock__TaskRpcException=a \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Run HtcMock test 100 tasks 4 levels with errors
timeout-minutes: 3
run: |
! MONITOR_PREFIX="monitor/htcmock-100-4-e/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=4 \
-e HtcMock__Partition=TestPartition0 \
-e HtcMock__TaskError=a \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Run HtcMock test 1000 tasks 1 level
timeout-minutes: 3
if: ${{ matrix.log-level != 'Verbose' }}
run: |
MONITOR_PREFIX="monitor/htcmock-1000-1/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=1000 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=1 \
-e HtcMock__Partition=TestPartition0 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Run HtcMock test 1000 tasks 4 levels
timeout-minutes: 3
if: ${{ matrix.log-level != 'Verbose' }}
run: |
MONITOR_PREFIX="monitor/htcmock-1000-4/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=1000 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=4 \
-e HtcMock__Partition=TestPartition0 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
- name: Print And Time Metrics
run: |
set -x
time curl localhost:5002/metrics
time curl localhost:5003/metrics
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-database.tar.gz
testConnectivity:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
ca: [true, false]
name: "Test connectivity - Ca installed ${{ matrix.ca }}"
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
just tag=${VERSION} deploy
sleep 10
- name: Add CA to store
if: ${{ matrix.ca }}
run: |
sudo apt install ca-certificates
sudo mkdir -p /usr/local/share/ca-certificates/
sudo cp terraform/generated/ingress_tls/server/ca.crt /usr/local/share/ca-certificates/ca_tls.crt
sudo cp terraform/generated/ingress_mtls/server/ca.crt /usr/local/share/ca-certificates/ca_mtls.crt
sudo update-ca-certificates
- name: Run connection test
run: CA_INSTALLED=${{ matrix.ca }} dotnet test Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj --logger "trx;LogFileName=test-results.trx" -p:RunAnalyzers=false -p:WarningLevel=0
- name: Test Report
uses: dorny/test-reporter@v1
if: always()
with:
name: Test - Connectivity CA ${{ matrix.ca }}
path: ./Tests/Connectivity/src/TestResults/test-results.trx
reporter: dotnet-trx
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/connectivity-ca${{ matrix.ca }}.json.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/connectivity-ca${{ matrix.ca }}-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/connectivity-ca${{ matrix.ca }}-database.tar.gz
runBench:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just log_level=Information tag=${VERSION} queue=activemq worker=bench deploy
sleep 10
ls -la terraform/logs/*.json
- name: Run Bench test tasks - many tasks (400)
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/bench-many-tasks-400/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e BenchOptions__NTasks=400 \
-e BenchOptions__TaskDurationMs=10 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_bench_test_client:$VERSION
- name: Run Bench test tasks - many tasks (200) with events
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/bench-many-tasks-200-e/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e BenchOptions__NTasks=200 \
-e BenchOptions__TaskDurationMs=100 \
-e BenchOptions__ShowEvents=true \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_bench_test_client:$VERSION
- name: Run Bench test time - long tasks (10s)
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/bench-long-tasks-10s/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e BenchOptions__NTasks=2 \
-e BenchOptions__TaskDurationMs=10000 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_bench_test_client:$VERSION
- name: Run Bench test time - large payloads (10MB)
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/bench-large-payload-10m/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e BenchOptions__NTasks=10 \
-e BenchOptions__TaskDurationMs=10 \
-e BenchOptions__ResultSize=1 \
-e BenchOptions__PayloadSize=10000 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_bench_test_client:$VERSION
- name: Run Bench test time - large results (10MB)
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/bench-large-result-10m/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e BenchOptions__NTasks=10 \
-e BenchOptions__TaskDurationMs=10 \
-e BenchOptions__ResultSize=10000 \
-e BenchOptions__PayloadSize=1 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_bench_test_client:$VERSION
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runBench.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runBench-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runBench-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runBench-database.tar.gz
runCrash:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just log_level=Information tag=${VERSION} queue=activemq worker=crashingworker replicas=4 partitions=1 build-deploy
sleep 10
ls -la terraform/logs/*.json
- name: Build client
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just tag=${VERSION} buildCrashingWorkerClient
- name: Run Crash test
timeout-minutes: 4
run: |
MONITOR_PREFIX="monitor/crash" tools/monitor.sh \
docker run --net armonik_network --rm \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_crashingworker_test_client:$VERSION
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runCrash.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runCrash-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runCrash-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/runCrash-database.tar.gz
defaultPartitionMock:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just log_level=Information tag=${VERSION} worker=htcmock queue=activemq deploy
sleep 10
ls -la terraform/logs/*.json
- name: Run HtcMock test
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-default-partition/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=1000 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=3 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
ls -la terraform/logs/*.json
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-defaultpartition.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-defaultPartition-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-defaultpartition-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-defaultpartition-database.tar.gz
healthCheckTest:
needs:
- versionning
- images
env:
VERSION: ${{ needs.versionning.outputs.version }}
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
with:
ref: ${{ github.ref }}
submodules: true
- name: Install AWSCLI (the one in the Github runner does not work)
run: |
pip install awscli
- name: Setup just
run: |
sudo snap install --edge --classic just
- name: Setup Terraform
uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3
- name: Deploy Core
run: |
MONITOR_PREFIX="monitor/deploy/" tools/monitor.sh \
just tag=${VERSION} object=redis deploy
sleep 10
ls -la terraform/logs/*.json
- name: Run HtcMock test
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-base/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=3 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
ls -la terraform/logs/*.json
- name: Health Check Test when database is down
timeout-minutes: 10
run: |
just tag=${VERSION} object=redis stop database
just healthChecks
just tag=${VERSION} object=redis restoreDeployment database
sleep 10
just healthChecks
- name: Run HtcMock test
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-restarted-db/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=3 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
ls -la terraform/logs/*.json
- name: Health Check Test when queue is down
timeout-minutes: 10
run: |
just tag=${VERSION} object=redis stop queue
just healthChecks
just tag=${VERSION} object=redis restoreDeployment queue
sleep 10
just healthChecks
- name: Run HtcMock test
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-restarted-redis/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=3 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
ls -la terraform/logs/*.json
- name: Health Check Test when object is down
timeout-minutes: 10
run: |
just tag=${VERSION} object=redis stop object
just healthChecks
just tag=${VERSION} object=redis restoreDeployment object
sleep 10
just healthChecks
- name: Run HtcMock test
timeout-minutes: 3
run: |
MONITOR_PREFIX="monitor/htcmock-restarted-redis2/" tools/monitor.sh \
docker run --net armonik_network --rm \
-e HtcMock__NTasks=100 \
-e HtcMock__TotalCalculationTime=00:00:00.100 \
-e HtcMock__DataSize=1 \
-e HtcMock__MemorySize=1 \
-e HtcMock__SubTasksLevels=3 \
-e GrpcClient__Endpoint=http://armonik.control.submitter:1080 \
dockerhubaneo/armonik_core_htcmock_test_client:$VERSION
ls -la terraform/logs/*.json
- name: Show logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -czf - terraform/logs/armonik-logs.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-healthchecktest.json.tar.gz
tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-healthchecktest-monitor.tar.gz
- name: Collect docker container logs
uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2
if: always()
with:
dest: './container-logs'
- name: Upload docker container logs
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-healthchecktest-container-logs.tar.gz
- name: Export and upload database
if: always()
run: |
export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}
export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}
bash tools/export_mongodb.sh
tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-healthchecktest-database.tar.gz
canMerge:
needs:
- tests
- testsQueueProtos
- testsWinOnly
- testHtcMockDC
- defaultPartitionMock
- testStreamDC
- images
- runBench
- runCrash
- healthCheckTest
- testConnectivity
runs-on: ubuntu-latest
steps:
- name: Echo OK
run: echo OK