Skip to content

Remove secrets and use dummy values #5

Remove secrets and use dummy values

Remove secrets and use dummy values #5

Workflow file for this run

name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
run-tests:
name: Setup and Run Tests
runs-on: ubuntu-latest
env:
RETRY_COUNT: 12 # number of retries for health checks
SLEEP_INTERVAL: 5 # Sleep duration in seconds between retries
MINIO_HEALTH_URL: http://localhost:9000/minio/health/live
DREMIO_HEALTH_URL: http://localhost:9047
MINIO_ROOT_USER: admin
MINIO_ROOT_PASSWORD: password
DREMIO_SOFTWARE_USERNAME: dremio
DREMIO_SOFTWARE_PASSWORD: dremio123
DREMIO_SOFTWARE_HOST: localhost
DREMIO_DATALAKE: dbt_test_source
DREMIO_DATABASE: dbt_test
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Create Docker Network
run: |
docker network create ci-network || echo "Network already exists"
- name: Start MinIO Service
run: bash .github/scripts/start_minio.sh
- name: Start Dremio Service
run: bash .github/scripts/start_dremio.sh
- name: Install MinIO Client (mc)
run: bash .github/scripts/install_minio_client.sh
- name: Create MinIO bucket
run: bash .github/scripts/create_minio_bucket.sh
- name: Create Dremio S3 Source
run: bash .github/scripts/create_dremio_s3_source.sh
- name: Install Dependencies
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install Python Dependencies
run: |
pip install --upgrade pip
pip install -r dev_requirements.txt
pip install .
- name: Create dbt test users
run: bash .github/scripts/create_dbt_test_users.sh
- name: Create dbt projects
run: bash .github/scripts/create_dbt_projects.sh
- name: Clean up __pycache__ directories
run: |
find . -type d -name "__pycache__" -exec rm -r {} +
- name: Create .env file for tests
run: bash .github/scripts/create_env_file.sh
- name: Run tests
run: bash .github/scripts/run_tests.sh
- name: Upload tests report as artifact
uses: actions/upload-artifact@v3
with:
name: all-tests-reports
path: reports/
upload-individual-test-reports:
name: Upload Tests Artifacts
runs-on: ubuntu-latest
needs: run-tests
steps:
- name: Download test reports
uses: actions/download-artifact@v3
with:
name: all-tests-reports
path: reports/
- name: Upload individual test reports
uses: actions/upload-artifact@v3
with:
name: individual-test-reports
path: reports/*.txt
verify-failures:
name: Verify Expected Test Failures
runs-on: ubuntu-latest
needs: [run-tests, upload-individual-test-reports]
steps:
- name: Check out repository
uses: actions/[email protected]
- name: Download All Test Reports
uses: actions/download-artifact@v3
with:
name: all-tests-reports
path: reports/
- name: Set Up Python Environment
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Extract Actual Failed Tests
run: |
shopt -s globstar
grep "FAILED" reports/**/*.txt | awk '{print $2}' | sort > actual_failures_sorted.txt
- name: Sort Expected Failures
run: sort .github/expected_failures.txt > expected_failures_sorted.txt
- name: Compare Actual Failures with Expected Failures
run: |
echo "Expected Failures:"
cat expected_failures_sorted.txt
echo ""
echo "Actual Failures:"
cat actual_failures_sorted.txt
echo ""
# Identify unexpected failures
unexpected_failures=$(comm -13 expected_failures_sorted.txt actual_failures_sorted.txt)
# Identify missing expected failures
missing_failures=$(comm -23 expected_failures_sorted.txt actual_failures_sorted.txt)
# Initialize exit code
exit_code=0
if [ -n "$unexpected_failures" ]; then
echo "Unexpected test failures detected:"
echo "$unexpected_failures"
exit_code=1
fi
if [ -n "$missing_failures" ]; then
echo "Expected test failures that did not occur (they passed):"
echo "$missing_failures"
exit_code=1
fi
if [ $exit_code -eq 0 ]; then
echo "All failed tests are expected, and all expected failures have occurred."
else
echo "Verification failed: There are unexpected or missing test failures."
fi
exit $exit_code