From 244f50b755a33a7c35706c63fdd2ffe356520eda Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Wed, 11 Dec 2024 13:12:40 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20Create=20=20a=20new=20CI=20Pipel?= =?UTF-8?q?ine=20(#2461)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add new gh actions based ci pipeline Signed-off-by: Samet Akcay * Add new gh actions based ci pipeline Signed-off-by: Samet Akcay * Remove nodejs from reusable workflows Signed-off-by: Samet Akcay * Remove sudo in clamav Signed-off-by: Samet Akcay * Further refactor test suite Signed-off-by: Samet Akcay * update release validation workflow Signed-off-by: Samet Akcay * Add documentation about the release strategy Signed-off-by: Samet Akcay --------- Signed-off-by: Samet Akcay --- .ci/ipas_default.config | 408 ------------------ .ci/trivy.yaml | 11 - .github/CODEOWNERS | 1 - .../code-quality/pre-commit/action.yaml | 116 +++++ .github/actions/pytest/action.yaml | 177 ++++++++ .github/actions/security/bandit/action.yaml | 142 ++++++ .github/actions/security/clamav/action.yaml | 167 +++++++ .github/actions/security/semgrep/action.yaml | 136 ++++++ .github/actions/security/trivy/action.yaml | 183 ++++++++ .../workflows/_reusable-artifact-builder.yaml | 115 +++++ .github/workflows/_reusable-code-quality.yaml | 67 +++ .../_reusable-production-release-process.yaml | 109 +++++ .../_reusable-rc-release-process.yaml | 241 +++++++++++ .../_reusable-release-publisher.yaml | 101 +++++ .../workflows/_reusable-release-status.yaml | 76 ++++ .../_reusable-release-validation.yaml | 178 ++++++++ .../workflows/_reusable-security-scan.yaml | 181 ++++++++ .github/workflows/_reusable-test-suite.yaml | 109 +++++ .../workflows/_reusable-version-check.yaml | 104 +++++ .github/workflows/code_scan.yml | 52 --- .github/workflows/pr.yaml | 84 ++++ .github/workflows/publish.yml | 28 -- .github/workflows/release.yaml | 104 +++++ .github/workflows/security-checks.yaml | 96 +++++ .github/workflows/upload_coverage.yml | 30 -- .../source/markdown/guides/developer/index.md | 1 + .../guides/developer/release_guidelines.md | 367 ++++++++++++++++ 27 files changed, 2854 insertions(+), 530 deletions(-) delete mode 100644 .ci/ipas_default.config delete mode 100644 .ci/trivy.yaml create mode 100644 .github/actions/code-quality/pre-commit/action.yaml create mode 100644 .github/actions/pytest/action.yaml create mode 100644 .github/actions/security/bandit/action.yaml create mode 100644 .github/actions/security/clamav/action.yaml create mode 100644 .github/actions/security/semgrep/action.yaml create mode 100644 .github/actions/security/trivy/action.yaml create mode 100644 .github/workflows/_reusable-artifact-builder.yaml create mode 100644 .github/workflows/_reusable-code-quality.yaml create mode 100644 .github/workflows/_reusable-production-release-process.yaml create mode 100644 .github/workflows/_reusable-rc-release-process.yaml create mode 100644 .github/workflows/_reusable-release-publisher.yaml create mode 100644 .github/workflows/_reusable-release-status.yaml create mode 100644 .github/workflows/_reusable-release-validation.yaml create mode 100644 .github/workflows/_reusable-security-scan.yaml create mode 100644 .github/workflows/_reusable-test-suite.yaml create mode 100644 .github/workflows/_reusable-version-check.yaml delete mode 100644 .github/workflows/code_scan.yml create mode 100644 .github/workflows/pr.yaml delete mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/security-checks.yaml delete mode 100644 .github/workflows/upload_coverage.yml create mode 100644 docs/source/markdown/guides/developer/release_guidelines.md diff --git a/.ci/ipas_default.config b/.ci/ipas_default.config deleted file mode 100644 index 5cca904cfa..0000000000 --- a/.ci/ipas_default.config +++ /dev/null @@ -1,408 +0,0 @@ - -### Bandit config file generated from: -# './bandit/bandit/cli/config_generator.py --out ipas_default.config' - -### This config may optionally select a subset of tests to run or skip by -### filling out the 'tests' and 'skips' lists given below. If no tests are -### specified for inclusion then it is assumed all tests are desired. The skips -### set will remove specific tests from the include set. This can be controlled -### using the -t/-s CLI options. Note that the same test ID should not appear -### in both 'tests' and 'skips', this would be nonsensical and is detected by -### Bandit at runtime. - -# Available tests: -# B101 : assert_used -# B102 : exec_used -# B103 : set_bad_file_permissions -# B104 : hardcoded_bind_all_interfaces -# B105 : hardcoded_password_string -# B106 : hardcoded_password_funcarg -# B107 : hardcoded_password_default -# B108 : hardcoded_tmp_directory -# B110 : try_except_pass -# B112 : try_except_continue -# B201 : flask_debug_true -# B301 : pickle -# B302 : marshal -# B303 : md5 -# B304 : ciphers -# B305 : cipher_modes -# B306 : mktemp_q -# B307 : eval -# B308 : mark_safe -# B310 : urllib_urlopen -# B311 : random -# B312 : telnetlib -# B313 : xml_bad_cElementTree -# B314 : xml_bad_ElementTree -# B315 : xml_bad_expatreader -# B316 : xml_bad_expatbuilder -# B317 : xml_bad_sax -# B318 : xml_bad_minidom -# B319 : xml_bad_pulldom -# B320 : xml_bad_etree -# B321 : ftplib -# B323 : unverified_context -# B324 : hashlib_new_insecure_functions -# B401 : import_telnetlib -# B402 : import_ftplib -# B403 : import_pickle -# B404 : import_subprocess -# B405 : import_xml_etree -# B406 : import_xml_sax -# B407 : import_xml_expat -# B408 : import_xml_minidom -# B409 : import_xml_pulldom -# B410 : import_lxml -# B411 : import_xmlrpclib -# B412 : import_httpoxy -# B413 : import_pycrypto -# B501 : request_with_no_cert_validation -# B502 : ssl_with_bad_version -# B503 : ssl_with_bad_defaults -# B504 : ssl_with_no_version -# B505 : weak_cryptographic_key -# B506 : yaml_load -# B507 : ssh_no_host_key_verification -# B601 : paramiko_calls -# B602 : subprocess_popen_with_shell_equals_true -# B603 : subprocess_without_shell_equals_true -# B604 : any_other_function_with_shell_equals_true -# B605 : start_process_with_a_shell -# B606 : start_process_with_no_shell -# B607 : start_process_with_partial_path -# B608 : hardcoded_sql_expressions -# B609 : linux_commands_wildcard_injection -# B610 : django_extra_used -# B611 : django_rawsql_used -# B701 : jinja2_autoescape_false -# B702 : use_of_mako_templates -# B703 : django_mark_safe - -# (optional) list included test IDs here, eg '[B101, B406]': -# IPAS Required Checkers. Do not disable these -# Additional checkers may be added if desired -tests: - [ 'B301', 'B302', 'B303', 'B304', 'B305', 'B306', 'B308', 'B310', 'B311', 'B312', 'B313', 'B314', 'B315', 'B316', 'B317', 'B318', 'B319', 'B320', 'B321', 'B323', 'B324', 'B401', 'B402', 'B403', 'B404', 'B405', 'B406', 'B407', 'B408', 'B409', 'B410', 'B411', 'B412', 'B413'] - -# (optional) list skipped test IDs here, eg '[B101, B406]': -# The following checkers are not required but be added to tests list if desired -skips: - [ 'B101', 'B102', 'B103', 'B104', 'B105', 'B106', 'B107', 'B108', 'B110', 'B112', 'B201', 'B501', 'B502', 'B503', 'B504', 'B505', 'B506', 'B507', 'B601', 'B602', 'B603', 'B604', 'B605', 'B606', 'B607', 'B608', 'B609', 'B610', 'B611', 'B701', 'B702', 'B703'] - - -# Added to exclude some path which are not actual source code for this project -exclude_dirs: [ - '.tox/', - '.vscode/', - '.git/', -] - -### (optional) plugin settings - some test plugins require configuration data -### that may be given here, per-plugin. All bandit test plugins have a built in -### set of sensible defaults and these will be used if no configuration is -### provided. It is not necessary to provide settings for every (or any) plugin -### if the defaults are acceptable. - -any_other_function_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -assert_used: - skips: [] -hardcoded_tmp_directory: - tmp_dirs: - - /tmp - - /var/tmp - - /dev/shm -linux_commands_wildcard_injection: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -ssl_with_bad_defaults: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD -ssl_with_bad_version: - bad_protocol_versions: - - PROTOCOL_SSLv2 - - SSLv2_METHOD - - SSLv23_METHOD - - PROTOCOL_SSLv3 - - PROTOCOL_TLSv1 - - SSLv3_METHOD - - TLSv1_METHOD -start_process_with_a_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_no_shell: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -start_process_with_partial_path: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_popen_with_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -subprocess_without_shell_equals_true: - no_shell: - - os.execl - - os.execle - - os.execlp - - os.execlpe - - os.execv - - os.execve - - os.execvp - - os.execvpe - - os.spawnl - - os.spawnle - - os.spawnlp - - os.spawnlpe - - os.spawnv - - os.spawnve - - os.spawnvp - - os.spawnvpe - - os.startfile - shell: - - os.system - - os.popen - - os.popen2 - - os.popen3 - - os.popen4 - - popen2.popen2 - - popen2.popen3 - - popen2.popen4 - - popen2.Popen3 - - popen2.Popen4 - - commands.getoutput - - commands.getstatusoutput - subprocess: - - subprocess.Popen - - subprocess.call - - subprocess.check_call - - subprocess.check_output - - subprocess.run -try_except_continue: - check_typed_exception: false -try_except_pass: - check_typed_exception: false -weak_cryptographic_key: - weak_key_size_dsa_high: 1024 - weak_key_size_dsa_medium: 2048 - weak_key_size_ec_high: 160 - weak_key_size_ec_medium: 224 - weak_key_size_rsa_high: 1024 - weak_key_size_rsa_medium: 2048 diff --git a/.ci/trivy.yaml b/.ci/trivy.yaml deleted file mode 100644 index 0b20468b5b..0000000000 --- a/.ci/trivy.yaml +++ /dev/null @@ -1,11 +0,0 @@ -ignore-policy: "" -ignorefile: .trivyignore -insecure: false -scan: - scanners: - - vuln - - secret - slow: false -severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL -vulnerability: - ignore-unfixed: false diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2960d480a7..5703b7174c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -20,7 +20,6 @@ /notebooks/200_models @samet-akcay /notebooks/300_benchmarking @ashwinvaidya17 /notebooks/400_openvino @samet-akcay -/notebooks/500_use_cases @paularamo /notebooks/README.md @samet-akcay # Requirements diff --git a/.github/actions/code-quality/pre-commit/action.yaml b/.github/actions/code-quality/pre-commit/action.yaml new file mode 100644 index 0000000000..acbcf87379 --- /dev/null +++ b/.github/actions/code-quality/pre-commit/action.yaml @@ -0,0 +1,116 @@ +# Pre-commit Quality Action +# +# This composite action executes pre-commit hooks for code quality checks +# with configurable Python and Node.js environments. +# +# Key Features: +# - Pre-commit hook execution +# - Environment configuration +# - Cache management +# - Multi-language support +# - Dependency handling +# +# Process Stages: +# 1. Environment Setup: +# - Python installation +# - Node.js installation +# - Cache configuration +# +# 2. Dependency Management: +# - Pre-commit installation +# - Hook installation +# - Cache restoration +# +# 3. Quality Checks: +# - Hook execution +# - Error reporting +# - Result caching +# +# Required Inputs: +# - python-version: Python version to use +# - node-version: Node.js version to use (defaults to "20") +# +# Example Usage: +# steps: +# - uses: ./.github/actions/code-quality/pre-commit +# with: +# python-version: "3.11" +# +# Note: Requires configured pre-commit hooks in repository + +name: "Pre-commit Quality Checks" +description: "Runs pre-commit hooks for code quality checks" + +inputs: + python-version: + description: "Python version to use" + required: false + default: "3.10" + node-version: + description: "Node.js version to use" + required: false + default: "20" + skip: + description: "Comma-separated list of hooks to skip" + required: false + default: "" + cache: + description: "Whether to use caching" + required: false + default: "true" + +outputs: + cache-hit: + description: "Whether the cache was hit" + value: ${{ steps.pre-commit-cache.outputs.cache-hit }} + +runs: + using: composite + steps: + # Set up Python environment with caching + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: pip # Enable pip caching + cache-dependency-path: .pre-commit-config.yaml + + # Set up Node.js for JavaScript-related hooks + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + # Install pre-commit with latest pip + - name: Install pre-commit + shell: bash + run: | + python -m pip install --upgrade pip + pip install pre-commit + + # Cache pre-commit hooks to speed up subsequent runs + - name: Cache pre-commit hooks + if: inputs.cache == 'true' + id: pre-commit-cache + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + # Cache key includes Python and Node versions to ensure correct environment + key: pre-commit-${{ runner.os }}-py${{ inputs.python-version }}-node${{ inputs.node-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit-${{ runner.os }}-py${{ inputs.python-version }}-node${{ inputs.node-version }}- + pre-commit-${{ runner.os }}-py${{ inputs.python-version }}- + + # Execute pre-commit checks with optional hook skipping + - name: Run pre-commit checks + shell: bash + env: + SKIP: ${{ inputs.skip }} + run: | + if [ -n "$SKIP" ]; then + # Run specific hooks if skip parameter is provided + pre-commit run --all-files --hook-stage="$SKIP" + else + # Run all hooks if no skip parameter + pre-commit run --all-files + fi diff --git a/.github/actions/pytest/action.yaml b/.github/actions/pytest/action.yaml new file mode 100644 index 0000000000..8617ea148e --- /dev/null +++ b/.github/actions/pytest/action.yaml @@ -0,0 +1,177 @@ +# Test Runner Action +# +# This composite action executes Python tests with pytest, providing +# comprehensive test execution and reporting capabilities. +# +# Key Features: +# - Multiple test type support +# - Parallel execution +# - Coverage reporting +# - Performance tracking +# - Result analysis +# +# Process Stages: +# 1. Environment Setup: +# - Python configuration +# - Virtual environment creation +# - Dependency installation +# +# 2. Test Execution: +# - Test scope determination +# - Parallel processing +# - Coverage tracking +# - Performance monitoring +# +# 3. Results Processing: +# - Coverage analysis +# - Performance reporting +# - Results aggregation +# +# Required Inputs: +# - python-version: Python version for tests +# - test-type: Type of tests to run +# - codecov-token: Token for coverage upload +# - max-test-time: Maximum test duration +# +# Outputs: +# - coverage-percentage: Total coverage +# - tests-passed: Test success status +# - test-duration: Execution time +# +# Example Usage: +# steps: +# - uses: ./.github/actions/pytest +# with: +# python-version: "3.11" +# test-type: "unit" +# codecov-token: ${{ secrets.CODECOV_TOKEN }} +# +# Note: Requires proper pytest configuration in pyproject.toml + +name: "Python Tests Runner" +description: "Runs Python unit and integration tests with pytest and uploads coverage to Codecov" + +inputs: + python-version: + description: "Python version to use" + required: false + default: "3.10" + test-type: + description: "Type of tests to run (unit/integration/all)" + required: false + default: "all" + codecov-token: + description: "Codecov upload token" + required: true + max-test-time: + description: "Maximum time in seconds for the test suite to run" + required: false + default: "300" + +outputs: + coverage-percentage: + description: "Total coverage percentage" + value: ${{ steps.coverage.outputs.percentage }} + tests-passed: + description: "Whether all tests passed" + value: ${{ steps.test-execution.outputs.success }} + test-duration: + description: "Total test duration in seconds" + value: ${{ steps.test-execution.outputs.duration }} + +runs: + using: composite + steps: + # Set up Python with pip caching + - name: Set up Python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: pip # Enable pip caching + cache-dependency-path: pyproject.toml + + # Create and configure virtual environment + - name: Configure virtual environment + id: setup-venv + shell: bash + run: | + # Create isolated test environment + python -m venv .venv + source .venv/bin/activate + # Install dependencies with dev extras + python -m pip install --upgrade pip + pip install ".[dev]" + pip install codecov + + # Determine which tests to run based on input + - name: Determine test scope + id: test-scope + shell: bash + run: | + case "${{ inputs.test-type }}" in + "unit") + echo "path=tests/unit" >> $GITHUB_OUTPUT + ;; + "integration") + echo "path=tests/integration" >> $GITHUB_OUTPUT + ;; + *) + # Run both test types if not specified + echo "path=tests/unit tests/integration" >> $GITHUB_OUTPUT + ;; + esac + + # Execute test suite with performance tracking + - name: Execute test suite + id: test-execution + shell: bash + run: | + source .venv/bin/activate + start_time=$(date +%s) + + # Run pytest with: + # - Auto parallel execution (-n auto) + # - Duration reporting for slow tests + # - Configurable timeout + # - JSON report generation + PYTHONPATH=src pytest ${{ steps.test-scope.outputs.path }} \ + -n auto \ + --durations=10 \ + --durations-min=1.0 \ + --timeout=${{ inputs.max-test-time }} \ + --json-report --json-report-file=pytest.json \ + && echo "success=true" >> $GITHUB_OUTPUT \ + || echo "success=false" >> $GITHUB_OUTPUT + + # Calculate total test duration + end_time=$(date +%s) + duration=$((end_time - start_time)) + echo "duration=${duration}" >> $GITHUB_OUTPUT + + # Analyze and report test performance + - name: Analyze test performance + if: always() # Run even if tests fail + shell: bash + run: | + echo "Test Duration: ${{ steps.test-execution.outputs.duration }} seconds" + + # Report slowest tests for optimization + echo "Top 10 slowest tests:" + cat pytest.json | jq -r '.tests[] | select(.duration >= 1) | "\(.duration)s \(.name)"' | sort -rn | head -n 10 + + # Warn if tests exceed time limit + if [ "${{ steps.test-execution.outputs.duration }}" -gt "${{ inputs.max-test-time }}" ]; then + echo "::warning::Test suite exceeded recommended duration of ${{ inputs.max-test-time }} seconds" + fi + + # Upload coverage data to Codecov + - name: Upload coverage to Codecov + if: steps.test-execution.outputs.success == 'true' + shell: bash + run: | + source .venv/bin/activate + # Upload with test type and Python version tags + codecov --token "${{ inputs.codecov-token }}" \ + --file coverage.xml \ + --flags "${{ inputs.test-type }}_py${{ inputs.python-version }}" \ + --name "${{ inputs.test-type }} tests (Python ${{ inputs.python-version }})" diff --git a/.github/actions/security/bandit/action.yaml b/.github/actions/security/bandit/action.yaml new file mode 100644 index 0000000000..a339cb3fca --- /dev/null +++ b/.github/actions/security/bandit/action.yaml @@ -0,0 +1,142 @@ +# Bandit Scanner Action +# +# This composite action executes Python security scanning using Bandit, +# providing configurable security analysis capabilities. +# +# Key Features: +# - Python code scanning +# - Severity configuration +# - Flexible scan scope +# - Multiple report formats +# - Custom rule support +# +# Process Stages: +# 1. Environment Setup: +# - Python installation +# - Bandit configuration +# - Cache preparation +# +# 2. Scan Execution: +# - Target determination +# - Rule application +# - Security analysis +# +# 3. Results Processing: +# - Report generation +# - Finding analysis +# - Output formatting +# +# Required Inputs: +# - scan_scope: Files to scan +# - severity_level: Issue severity threshold +# - fail_on_findings: Whether to fail on issues +# +# Outputs: +# - scan_result: Scan exit code +# - report_path: Results location +# +# Example Usage: +# steps: +# - uses: ./.github/actions/security/bandit +# with: +# scan_scope: "changed" +# severity_level: "MEDIUM" +# +# Note: Configure Bandit settings in pyproject.toml for best results + +name: "Bandit Security Scan" +description: "Runs Bandit security scanner with configurable options" + +inputs: + scan_scope: + description: "Scope of files to scan (all/changed)" + required: false + default: "changed" + paths: + description: "Paths to scan when using all scope" + required: false + default: "./src" + config_file: + description: "Path to pyproject.toml or custom bandit config" + required: false + default: "pyproject.toml" + severity_level: + description: "Minimum severity level to report (all/LOW/MEDIUM/HIGH)" + required: false + default: "LOW" + confidence_level: + description: "Minimum confidence level to report (all/LOW/MEDIUM/HIGH)" + required: false + default: "LOW" + output_format: + description: "Format for scan results (json/txt/html/csv)" + required: false + default: "json" + fail_on_findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" + +outputs: + scan_result: + description: "Exit code of the Bandit scan" + value: ${{ steps.run-bandit.outputs.exit_code }} + report_path: + description: "Path to the generated report file" + value: ${{ steps.run-bandit.outputs.report_path }} + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Bandit + shell: bash + run: | + python -m pip install --upgrade pip + pip install bandit[toml] + + - name: Get changed files + if: inputs.scan_scope == 'changed' + id: changed-files + uses: tj-actions/changed-files@v41 + with: + files: | + **/*.py + **/*.pyx + **/*.pyi + + - name: Run Bandit scan + id: run-bandit + shell: bash + run: | + REPORT_FILE="bandit-report.${{ inputs.output_format }}" + + if [[ "${{ inputs.scan_scope }}" == "changed" && -n "${{ steps.changed-files.outputs.all_changed_files }}" ]]; then + echo "Running Bandit on changed files" + FILES="${{ steps.changed-files.outputs.all_changed_files }}" + else + echo "Running Bandit on all files in ${{ inputs.paths }}" + FILES="${{ inputs.paths }}" + fi + + # Convert severity and confidence to lowercase + SEVERITY=$(echo "${{ inputs.severity_level }}" | tr '[:upper:]' '[:lower:]') + CONFIDENCE=$(echo "${{ inputs.confidence_level }}" | tr '[:upper:]' '[:lower:]') + + bandit \ + -c ${{ inputs.config_file }} \ + --severity-level ${SEVERITY} \ + --confidence-level ${CONFIDENCE} \ + -f ${{ inputs.output_format }} \ + -o "${REPORT_FILE}" \ + -r ${FILES} || echo "exit_code=$?" >> $GITHUB_OUTPUT + + echo "report_path=${REPORT_FILE}" >> $GITHUB_OUTPUT + + if [[ "${{ inputs.fail_on_findings }}" == "true" && -n "$exit_code" && "$exit_code" != "0" ]]; then + exit $exit_code + fi diff --git a/.github/actions/security/clamav/action.yaml b/.github/actions/security/clamav/action.yaml new file mode 100644 index 0000000000..8ed31fc7fa --- /dev/null +++ b/.github/actions/security/clamav/action.yaml @@ -0,0 +1,167 @@ +# ClamAV Scanner Action +# +# This composite action executes antivirus scanning using ClamAV, +# providing malware detection and security analysis. +# +# Key Features: +# - Malware detection +# - Real-time database updates +# - Configurable scan scope +# - Exclusion support +# - Performance optimization +# +# Process Stages: +# 1. Environment Setup: +# - ClamAV installation +# - Database updates +# - Cache configuration +# +# 2. Scan Execution: +# - Target selection +# - Exclusion application +# - Threat detection +# +# 3. Results Processing: +# - Report generation +# - Threat analysis +# - Finding summary +# +# Required Inputs: +# - scan_scope: Files to scan +# - exclude_dirs: Directories to skip +# - max_file_size: Size limit for scanning +# +# Outputs: +# - scan_result: Scan exit code +# - report_path: Results location +# - threats_found: Number of threats +# +# Example Usage: +# steps: +# - uses: ./.github/actions/security/clamav +# with: +# scan_scope: "changed" +# exclude_dirs: ".git,node_modules" +# +# Note: Requires sufficient disk space for virus database + +name: "ClamAV Security Scan" +description: "Runs ClamAV antivirus scanner with configurable options" + +inputs: + scan_scope: + description: "Scope of files to scan (all/changed)" + required: false + default: "changed" + paths: + description: "Paths to scan when using all scope" + required: false + default: "." + exclude_dirs: + description: "Directories to exclude from scan" + required: false + default: ".git,node_modules,venv" + max_file_size: + description: "Maximum file size to scan in MB" + required: false + default: "100" + output_format: + description: "Format for scan results (json/txt)" + required: false + default: "json" + fail_on_findings: + description: "Whether to fail the action if threats are found" + required: false + default: "true" + +outputs: + scan_result: + description: "Exit code of the ClamAV scan" + value: ${{ steps.run-clamav.outputs.exit_code }} + report_path: + description: "Path to the generated report file" + value: ${{ steps.run-clamav.outputs.report_path }} + threats_found: + description: "Number of threats found during scan" + value: ${{ steps.run-clamav.outputs.threats_found }} + +runs: + using: composite + steps: + - name: Get changed files + if: inputs.scan_scope == 'changed' + id: changed-files + uses: tj-actions/changed-files@v41 + + - name: Run ClamAV scan + id: run-clamav + uses: docker://clamav/clamav:stable + env: + GITHUB_OUTPUT: /tmp/gh_output + with: + entrypoint: sh + args: | + -c " + # Update virus definitions + freshclam --quiet + + # Prepare exclude dirs + EXCLUDE_DIRS=`echo '${{ inputs.exclude_dirs }}' | tr ',' ' ' | sed 's/[^ ]* */--exclude-dir=&/g'` + + # Convert MB to bytes + MAX_FILE_SIZE=`expr ${{ inputs.max_file_size }} \* 1024 \* 1024` + + # Create output directory + mkdir -p security-results/clamav + + # Run scan based on scope + if [ '${{ inputs.scan_scope }}' = 'changed' ] && [ -n '${{ steps.changed-files.outputs.all_changed_files }}' ]; then + echo 'Running ClamAV on changed files' + FILES='${{ steps.changed-files.outputs.all_changed_files }}' + SCAN_CMD='clamscan' + else + echo 'Running ClamAV on all files in ${{ inputs.paths }}' + FILES='${{ inputs.paths }}' + SCAN_CMD='clamscan -r' + fi + + # Run scan and capture output + $SCAN_CMD \ + --max-filesize=$MAX_FILE_SIZE \ + $EXCLUDE_DIRS \ + $FILES \ + > scan_output.txt 2>&1 + + SCAN_EXIT_CODE=$? + + # Count infected files + INFECTED_FILES=`grep 'Infected files:' scan_output.txt | awk '{print $3}'` + if [ -z \"$INFECTED_FILES\" ]; then + INFECTED_FILES=0 + fi + + # Generate report + if [ '${{ inputs.output_format }}' = 'json' ]; then + echo '{ + \"scan_summary\": { + \"files_scanned\": '`grep 'Scanned files:' scan_output.txt | awk '{print $3}'`', + \"threats_found\": '$INFECTED_FILES', + \"start_time\": \"'`grep 'Start time:' scan_output.txt | cut -d: -f2- | xargs`'\", + \"end_time\": \"'`grep 'End time:' scan_output.txt | cut -d: -f2- | xargs`'\" + } + }' > security-results/clamav/report.json + else + cp scan_output.txt security-results/clamav/report.txt + fi + + # Write to outputs file + { + echo \"exit_code=$SCAN_EXIT_CODE\" + echo \"threats_found=$INFECTED_FILES\" + echo \"report_path=security-results/clamav/report.${{ inputs.output_format }}\" + } > \"$GITHUB_OUTPUT\" + + if [ '${{ inputs.fail_on_findings }}' = 'true' ] && [ $INFECTED_FILES -gt 0 ]; then + exit 1 + fi + " diff --git a/.github/actions/security/semgrep/action.yaml b/.github/actions/security/semgrep/action.yaml new file mode 100644 index 0000000000..e33c6f7851 --- /dev/null +++ b/.github/actions/security/semgrep/action.yaml @@ -0,0 +1,136 @@ +# Semgrep Scanner Action +# +# This composite action executes static analysis security testing using Semgrep, +# providing comprehensive code analysis capabilities. +# +# Key Features: +# - Multi-language support +# - Custom rule sets +# - Incremental scanning +# - SARIF reporting +# - Performance optimization +# +# Process Stages: +# 1. Environment Setup: +# - Python installation +# - Semgrep configuration +# - Rule preparation +# +# 2. Scan Execution: +# - Target selection +# - Rule application +# - Code analysis +# +# 3. Results Processing: +# - Report generation +# - Finding analysis +# - Output formatting +# +# Required Inputs: +# - scan_scope: Files to scan +# - config: Rule configuration +# - severity: Issue threshold +# +# Outputs: +# - scan_result: Scan exit code +# - report_path: Results location +# +# Example Usage: +# steps: +# - uses: ./.github/actions/security/semgrep +# with: +# scan_scope: "changed" +# config: "p/owasp-top-ten" +# +# Note: Consider using custom rule sets for project-specific checks + +name: "Semgrep SAST Scan" +description: "Runs Semgrep security scanner with configurable options" + +inputs: + scan_scope: + description: "Scope of files to scan (all/changed)" + required: false + default: "changed" + paths: + description: "Paths to scan when using all scope" + required: false + default: "." + config: + description: "Semgrep rules or config to use" + required: false + default: "p/default" + severity: + description: "Minimum severity level to report (ERROR/WARNING/INFO)" + required: false + default: "WARNING" + timeout: + description: "Maximum time to run semgrep in seconds" + required: false + default: "300" + output_format: + description: "Format for scan results (text/json/sarif)" + required: false + default: "sarif" + fail_on_findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" + +outputs: + scan_result: + description: "Exit code of the Semgrep scan" + value: ${{ steps.run-semgrep.outputs.exit_code }} + report_path: + description: "Path to the generated report file" + value: ${{ steps.run-semgrep.outputs.report_path }} + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Semgrep + shell: bash + run: | + python -m pip install --upgrade pip + pip install semgrep + + - name: Get changed files + if: inputs.scan_scope == 'changed' + id: changed-files + uses: tj-actions/changed-files@v41 + with: + files: | + **/*.* + + - name: Run Semgrep scan + id: run-semgrep + shell: bash + run: | + REPORT_FILE="semgrep-results.${{ inputs.output_format }}" + + if [[ "${{ inputs.scan_scope }}" == "changed" && -n "${{ steps.changed-files.outputs.all_changed_files }}" ]]; then + echo "Running Semgrep on changed files" + FILES="${{ steps.changed-files.outputs.all_changed_files }}" + else + echo "Running Semgrep on all files in ${{ inputs.paths }}" + FILES="${{ inputs.paths }}" + fi + + semgrep \ + --config ${{ inputs.config }} \ + --severity ${{ inputs.severity }} \ + --timeout ${{ inputs.timeout }} \ + --${{ inputs.output_format }} \ + -o "${REPORT_FILE}" \ + ${FILES} || echo "exit_code=$?" >> $GITHUB_OUTPUT + + echo "report_path=${REPORT_FILE}" >> $GITHUB_OUTPUT + + if [[ "${{ inputs.fail_on_findings }}" == "true" && -n "$exit_code" && "$exit_code" != "0" ]]; then + exit $exit_code + fi diff --git a/.github/actions/security/trivy/action.yaml b/.github/actions/security/trivy/action.yaml new file mode 100644 index 0000000000..44f3ca4e95 --- /dev/null +++ b/.github/actions/security/trivy/action.yaml @@ -0,0 +1,183 @@ +# Trivy Scanner Action +# +# This composite action executes comprehensive security scanning using Trivy, +# supporting multiple targets and vulnerability detection methods. +# +# Key Features: +# - Multi-target scanning +# - Vulnerability detection +# - Secret scanning +# - SBOM generation +# - IaC analysis +# +# Process Stages: +# 1. Environment Setup: +# - Trivy installation +# - Database updates +# - Cache configuration +# +# 2. Scan Execution: +# - Target analysis +# - Vulnerability detection +# - Configuration checks +# +# 3. Results Processing: +# - Report generation +# - SBOM creation +# - Finding analysis +# +# Required Inputs: +# - scan_type: Type of scan +# - scan_target: Target to analyze +# - severity: Issue threshold +# +# Outputs: +# - scan_result: Scan exit code +# - report_path: Results location +# +# Example Usage: +# steps: +# - uses: ./.github/actions/security/trivy +# with: +# scan_type: "fs" +# scan_target: "./src" +# severity: "HIGH,CRITICAL" +# +# Note: Requires appropriate permissions for scanning + +name: "Trivy Security Scanner" +description: "Comprehensive security scanner for vulnerabilities, IaC issues, and secrets" + +inputs: + scan_type: + description: "Type of scan to perform (fs/config/image/repo/rootfs)" + required: false + default: "fs" + scan_scope: + description: "Scope of files to scan (all/changed)" + required: false + default: "changed" + scan_target: + description: "Target to scan (path, image name, or repo URL)" + required: false + default: "." + severity: + description: "Minimum severity level (UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL)" + required: false + default: "MEDIUM,HIGH,CRITICAL" + ignore_unfixed: + description: "Ignore unpatched/unfixed vulnerabilities" + required: false + default: "true" + scanners: + description: "Scanners to enable (vuln,secret,config)" + required: false + default: "vuln" + misconfig_scanners: + description: "Misconfig scanners to enable (azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan)" + required: false + default: "azure-arm,cloudformation,dockerfile,helm,kubernetes,terraform,terraformplan" + format: + description: "Output format (table,json,sarif,template)" + required: false + default: "sarif" + timeout: + description: "Timeout duration (e.g., 5m, 10m)" + required: false + default: "10m" + generate_sbom: + description: "Generate Software Bill of Materials (SBOM)" + required: false + default: "false" + sbom_format: + description: "SBOM output format (cyclonedx, spdx, spdx-json)" + required: false + default: "cyclonedx" + +outputs: + scan_result: + description: "Exit code of the Trivy scan" + value: ${{ steps.run-trivy.outputs.exit_code }} + report_path: + description: "Path to the generated report file" + value: ${{ steps.run-trivy.outputs.report_path }} + +runs: + using: composite + steps: + - name: Get changed files + if: inputs.scan_scope == 'changed' + id: changed-files + uses: tj-actions/changed-files@v41 + + - name: Cache Trivy vulnerability database + uses: actions/cache@v3 + with: + path: ~/.cache/trivy + key: trivy-db-${{ runner.os }}-${{ hashFiles('**/trivy-db/**') }} + restore-keys: | + trivy-db-${{ runner.os }}- + + - name: Install Trivy + shell: bash + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.48.0 + # Download DB in advance with retry mechanism + for i in {1..3}; do + echo "Attempt $i to download vulnerability database..." + trivy --cache-dir ~/.cache/trivy image --download-db-only && break || sleep 10 + done + + - name: Run Trivy scan + id: run-trivy + shell: bash + run: | + # Create output directory + mkdir -p reports + REPORT_FILE="reports/trivy-${{ inputs.scan_type }}-${{ inputs.scanners }}.sarif" + + echo "Running Trivy with scan type: ${{ inputs.scan_type }}" + echo "Output will be saved to: ${REPORT_FILE}" + + # Always scan the entire directory but use different paths based on scope + if [[ "${{ inputs.scan_scope }}" == "changed" && -n "${{ steps.changed-files.outputs.all_changed_files }}" ]]; then + echo "Changed files detected, scanning repository" + SCAN_TARGET="." + else + echo "Scanning target: ${{ inputs.scan_target }}" + SCAN_TARGET="${{ inputs.scan_target }}" + fi + + # Build the base command + CMD="trivy --cache-dir ~/.cache/trivy ${{ inputs.scan_type }} --severity ${{ inputs.severity }} --format ${{ inputs.format }} --output ${REPORT_FILE} --timeout ${{ inputs.timeout }}" + + # Add scanner-specific flags based on scan type + if [[ "${{ inputs.scan_type }}" == "config" ]]; then + # For config scans, use all default misconfig scanners or specified ones + CMD="$CMD --misconfig-scanners ${{ inputs.misconfig_scanners }}" + elif [[ "${{ inputs.scan_type }}" == "fs" ]]; then + # For filesystem scans, use --scanners + CMD="$CMD --scanners ${{ inputs.scanners }} --ignore-unfixed=${{ inputs.ignore_unfixed }}" + fi + + # Add the scan target and execute + CMD="$CMD ${SCAN_TARGET}" + echo "Executing command: $CMD" + eval $CMD || echo "::warning::Trivy scan completed with findings" + + if [ -f "${REPORT_FILE}" ]; then + echo "report_path=${REPORT_FILE}" >> $GITHUB_OUTPUT + echo "Scan report generated at ${REPORT_FILE}" + else + echo "::error::Report file was not generated" + exit 1 + fi + + # Generate SBOM if requested + if [[ "${{ inputs.generate_sbom }}" == "true" ]]; then + echo "Generating SBOM in ${{ inputs.sbom_format }} format" + trivy fs \ + --format ${{ inputs.sbom_format }} \ + --output "sbom.${{ inputs.sbom_format }}" \ + ${SCAN_TARGET} + fi diff --git a/.github/workflows/_reusable-artifact-builder.yaml b/.github/workflows/_reusable-artifact-builder.yaml new file mode 100644 index 0000000000..292a6dbe61 --- /dev/null +++ b/.github/workflows/_reusable-artifact-builder.yaml @@ -0,0 +1,115 @@ +# Artifact Builder Workflow +# +# This reusable workflow handles Python package building and artifact creation, +# providing a standardized build process with verification and caching. +# +# Key Features: +# - Python package building +# - Package verification +# - Build artifact caching +# - Configurable Python versions +# - Artifact retention management +# +# Process Stages: +# 1. Environment Setup: +# - Python environment configuration +# - Dependency installation +# - Cache initialization +# +# 2. Build Process: +# - Package building +# - Distribution creation +# - Build verification +# +# 3. Artifact Management: +# - Artifact naming +# - Upload and caching +# - Retention configuration +# +# Required Inputs: +# - python-version: Python version for building (default: "3.10") +# - verify-package: Enable package verification (default: true) +# +# Outputs: +# - artifact-name: Name of the uploaded artifact +# +# Example Usage: +# 1. Basic Build: +# jobs: +# build: +# uses: ./.github/workflows/_reusable-artifact-builder.yaml +# with: +# python-version: "3.11" +# +# 2. Build with Verification: +# jobs: +# build: +# uses: ./.github/workflows/_reusable-artifact-builder.yaml +# with: +# python-version: "3.10" +# verify-package: true +# +# Note: Requires proper Python project structure and build configuration + +name: Reusable Artifact Builder + +on: + workflow_call: + inputs: + python-version: + description: "Python version for building" + type: string + default: "3.10" + verify-package: + description: "Run package verification" + type: boolean + default: true + outputs: + artifact-name: + description: "Name of the uploaded artifact" + value: ${{ jobs.build.outputs.artifact-name }} + +jobs: + build: + runs-on: ubuntu-latest + outputs: + artifact-name: ${{ steps.set-artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Build package + run: | + python -m pip install --upgrade pip build + python -m build + - name: Verify package + if: inputs.verify-package + run: | + pip install twine + twine check dist/* + - name: Set artifact name + id: set-artifact-name + run: echo "name=dist-$(date +%s)" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.set-artifact-name.outputs.name }} + path: dist/ + retention-days: 5 + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cache/pip + .venv + key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Cache build artifacts + uses: actions/cache@v3 + with: + path: | + dist/ + *.egg-info/ + build/ + key: ${{ runner.os }}-build-${{ hashFiles('**/pyproject.toml') }} diff --git a/.github/workflows/_reusable-code-quality.yaml b/.github/workflows/_reusable-code-quality.yaml new file mode 100644 index 0000000000..077285a972 --- /dev/null +++ b/.github/workflows/_reusable-code-quality.yaml @@ -0,0 +1,67 @@ +# Code Quality Workflow +# +# This reusable workflow executes code quality checks using pre-commit hooks +# and other quality assurance tools across multiple languages. +# +# Key Features: +# - Pre-commit hook execution +# - Multi-language support +# - Dependency caching +# - Configurable environments +# - Parallel check execution +# +# Process Stages: +# 1. Environment Preparation: +# - Python setup +# - Cache configuration +# +# 2. Quality Checks: +# - Code linting +# - Style verification +# - Type checking +# - Best practices validation +# +# 3. Results Processing: +# - Error reporting +# - Check summaries +# - Status updates +# +# Required Inputs: +# - python-version: Python version for checks (default: "3.10") +# +# Example Usage: +# 1. Default Configuration: +# jobs: +# quality: +# uses: ./.github/workflows/_reusable-code-quality.yaml +# +# 2. Custom Versions: +# jobs: +# quality: +# uses: ./.github/workflows/_reusable-code-quality.yaml +# with: +# python-version: "3.11" +# +# Note: Requires configured pre-commit hooks in repository + +name: Reusable Code Quality + +on: + workflow_call: + inputs: + python-version: + description: "Python version for checks" + type: string + default: "3.10" + +jobs: + pre-commit: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/code-quality/pre-commit + with: + python-version: ${{ inputs.python-version }} diff --git a/.github/workflows/_reusable-production-release-process.yaml b/.github/workflows/_reusable-production-release-process.yaml new file mode 100644 index 0000000000..19d36f46cc --- /dev/null +++ b/.github/workflows/_reusable-production-release-process.yaml @@ -0,0 +1,109 @@ +# Production Release Process Workflow +# +# This reusable workflow manages the production release process, including +# validation, preparation, and publication steps. +# +# Key Features: +# - Release readiness validation +# - RC approval verification +# - Production deployment +# - Artifact management +# - Release publication +# +# Process Stages: +# 1. Release Validation: +# - RC approval verification +# - Version compatibility check +# - Release readiness assessment +# +# 2. Release Preparation: +# - Artifact collection +# - Production bundle creation +# - Documentation updates +# +# 3. Publication: +# - Production PyPI deployment +# - GitHub release creation +# - Documentation publishing +# +# Required Inputs: +# - version: Release version +# - artifact-name: Name of validated artifact +# +# Required Secrets: +# - pypi-token: Production PyPI token +# +# Example Usage: +# 1. Production Release: +# jobs: +# release: +# uses: ./.github/workflows/_reusable-production-release-process.yaml +# with: +# version: "v1.2.3" +# artifact-name: "dist-123456789" +# secrets: +# pypi-token: ${{ secrets.PYPI_TOKEN }} +# +# Note: Should only be triggered after successful RC process completion + +name: Production Release Process + +on: + workflow_call: + inputs: + version: + required: true + type: string + artifact-name: + required: true + type: string + secrets: + pypi-token: + required: true + +jobs: + validate-release-readiness: + runs-on: ubuntu-latest + steps: + - name: Check for approved RC + run: | + VERSION="${{ inputs.version }}" + ARTIFACTS_JSON=$(curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/artifacts") + + RC_APPROVAL=$(echo "$ARTIFACTS_JSON" | jq -r --arg ver "${VERSION%-*}" \ + '.artifacts[] | select(.name | startswith("rc-approval-v" + $ver))') + + if [ -z "$RC_APPROVAL" ]; then + echo "::error::No approved RC found for version $VERSION" + exit 1 + fi + + prepare-release: + needs: [validate-release-readiness] + environment: + name: production + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: dist + + - name: Upload for production release + uses: actions/upload-artifact@v4 + with: + name: production-release-artifacts + path: dist/ + retention-days: 1 + + publish: + needs: [prepare-release] + uses: ./.github/workflows/_reusable-release-publisher.yaml + with: + version: ${{ inputs.version }} + artifact-name: production-release-artifacts + is-prerelease: false + secrets: + pypi-token: ${{ secrets.pypi-token }} diff --git a/.github/workflows/_reusable-rc-release-process.yaml b/.github/workflows/_reusable-rc-release-process.yaml new file mode 100644 index 0000000000..93e51a25cb --- /dev/null +++ b/.github/workflows/_reusable-rc-release-process.yaml @@ -0,0 +1,241 @@ +# Release Candidate Process Workflow +# +# This reusable workflow implements a comprehensive release candidate (RC) process +# with multiple validation stages and approvals. +# +# Key Features: +# - Multi-stage validation +# - Technical review process +# - QA validation steps +# - Product review integration +# - Approval tracking +# +# Process Stages: +# 1. Technical Review: +# - Test PyPI deployment +# - Build validation +# - Security verification +# +# 2. QA Validation: +# - Test results review +# - Deployment verification +# - Documentation check +# +# 3. Product Review: +# - Feature verification +# - Release notes review +# - Version compatibility +# +# 4. Final Approval: +# - Sign-off collection +# - Approval recording +# - Release readiness +# +# Required Inputs: +# - version: Version to release +# - artifact-name: Name of build artifact +# +# Required Secrets: +# - test-pypi-token: PyPI token for test deployments +# +# Example Usage: +# 1. Standard RC Process: +# jobs: +# rc-release: +# uses: ./.github/workflows/_reusable-rc-release-process.yaml +# with: +# version: "v1.2.3-rc1" +# artifact-name: "dist-123456789" +# secrets: +# test-pypi-token: ${{ secrets.TEST_PYPI_TOKEN }} +# +# Note: Requires configured environments with appropriate approvals + +name: RC Release Process + +on: + workflow_call: + inputs: + version: + required: true + type: string + artifact-name: + required: true + type: string + secrets: + test-pypi-token: + required: true + +jobs: + technical-review: + environment: + name: technical-review + url: ${{ steps.review-url.outputs.url }} + runs-on: ubuntu-latest + outputs: + deployment-status: ${{ steps.validate-deployment.outputs.status }} + steps: + - name: Generate review URL + id: review-url + run: | + echo "url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: dist + + - name: Download test results + uses: actions/download-artifact@v4 + with: + pattern: "*-test-results" + merge-multiple: true + path: test-results + + - name: Download security results + uses: actions/download-artifact@v4 + with: + pattern: "*-security-results" + merge-multiple: true + path: security-results + + - name: Download quality results + uses: actions/download-artifact@v4 + with: + pattern: "*-quality-results" + merge-multiple: true + path: quality-results + + - name: Deploy to Test PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.test-pypi-token }} + TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ + run: | + pip install --upgrade pip twine + twine upload dist/* + + - name: Validate deployment + id: validate-deployment + run: | + WHEEL_FILE=$(ls dist/*.whl | head -n 1) + PACKAGE_NAME=$(basename $WHEEL_FILE | cut -d'-' -f1) + + sleep 30 + + python -m venv test-env + source test-env/bin/activate + + pip install --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple \ + "${PACKAGE_NAME}==${VERSION#v}" + + if python -c "import ${PACKAGE_NAME}"; then + echo "status=success" >> $GITHUB_OUTPUT + else + echo "status=failure" >> $GITHUB_OUTPUT + exit 1 + fi + env: + VERSION: ${{ inputs.version }} + + - name: Generate technical review report + run: | + cat << EOF > technical-review-report.md + # Technical Review Report + + ## Version Information + - Version: ${{ inputs.version }} + - Package Name: $(ls dist/*.whl | head -n 1 | xargs basename) + + ## Test Results + \`\`\` + $(cat test-results/*/report.xml || echo "No test results found") + \`\`\` + + ## Security Scan Results + \`\`\` + $(cat security-results/*/report.json || echo "No security results found") + \`\`\` + + ## Code Quality Results + \`\`\` + $(cat quality-results/*/report.txt || echo "No quality results found") + \`\`\` + + ## Test PyPI Deployment + - Status: ${{ steps.validate-deployment.outputs.status }} + - URL: https://test.pypi.org/project/${PACKAGE_NAME}/${VERSION#v}/ + + EOF + + - name: Upload technical review report + uses: actions/upload-artifact@v4 + with: + name: technical-review-report + path: technical-review-report.md + + qa-validation: + needs: [technical-review] + environment: + name: qa-validation + url: ${{ steps.qa-url.outputs.url }} + runs-on: ubuntu-latest + steps: + - name: Generate QA URL + id: qa-url + run: | + echo "url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT + + - name: Download technical review report + uses: actions/download-artifact@v4 + with: + name: technical-review-report + path: qa-review + + - name: Generate QA dashboard + run: | + echo "QA Review Dashboard" + echo "===================" + echo + echo "Technical Review Status: ${{ needs.technical-review.outputs.deployment-status }}" + echo + echo "Review the full technical report at: qa-review/technical-review-report.md" + echo + echo "Please validate the following:" + echo "1. All tests have passed" + echo "2. No security issues found" + echo "3. Code quality meets standards" + echo "4. Package is installable from Test PyPI" + echo "5. Documentation is up to date" + + product-review: + needs: [qa-validation] + environment: + name: product-review + runs-on: ubuntu-latest + steps: + - name: Download technical review report + uses: actions/download-artifact@v4 + with: + name: technical-review-report + + - name: Display release information + run: | + echo "Release Information for ${{ inputs.version }}" + echo "----------------------------------------" + cat technical-review-report.md + + release-approval: + needs: [product-review] + environment: + name: release-approval + runs-on: ubuntu-latest + steps: + - name: Record approval + run: | + echo "RC Approval Record:" + echo "- Version: ${{ inputs.version }}" + echo "- Timestamp: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" + echo "- Approver: ${{ github.actor }}" diff --git a/.github/workflows/_reusable-release-publisher.yaml b/.github/workflows/_reusable-release-publisher.yaml new file mode 100644 index 0000000000..b8765e8ca1 --- /dev/null +++ b/.github/workflows/_reusable-release-publisher.yaml @@ -0,0 +1,101 @@ +# Release Publisher Workflow +# +# This reusable workflow handles package publication to PyPI and GitHub, +# supporting both production and pre-release deployments. +# +# Key Features: +# - PyPI package publishing +# - GitHub release creation +# - Pre-release support +# - Release notes generation +# - Artifact management +# +# Process Stages: +# 1. Artifact Processing: +# - Download validation +# - Package verification +# - Distribution preparation +# +# 2. PyPI Publication: +# - Environment selection +# - Package upload +# - Publication verification +# +# 3. GitHub Release: +# - Release creation +# - Asset attachment +# - Notes generation +# +# Required Inputs: +# - version: Version to release +# - artifact-name: Name of artifact to publish +# - is-prerelease: Whether this is a pre-release +# +# Required Secrets: +# - pypi-token: Production PyPI token +# - test-pypi-token: Test PyPI token (for pre-releases) +# +# Example Usage: +# 1. Production Release: +# jobs: +# publish: +# uses: ./.github/workflows/_reusable-release-publisher.yaml +# with: +# version: "v1.0.0" +# artifact-name: "dist-123456789" +# is-prerelease: false +# secrets: +# pypi-token: ${{ secrets.PYPI_TOKEN }} +# +# Note: Requires appropriate tokens and permissions for publishing + +name: Reusable Release Publisher + +on: + workflow_call: + inputs: + version: + description: "Version to release" + required: true + type: string + artifact-name: + description: "Name of the artifact to publish" + required: true + type: string + is-prerelease: + description: "Whether this is a pre-release" + type: boolean + default: false + secrets: + pypi-token: + required: true + description: "PyPI token for package publishing" + test-pypi-token: + required: false + description: "Test PyPI token for pre-releases" + +jobs: + publish: + runs-on: ubuntu-latest + environment: ${{ inputs.is-prerelease && 'staging' || 'production' }} + steps: + - uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: dist + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ inputs.is-prerelease && secrets.test-pypi-token || secrets.pypi-token }} + TWINE_REPOSITORY_URL: ${{ inputs.is-prerelease && 'https://test.pypi.org/legacy/' || '' }} + run: | + pip install --upgrade pip twine + twine upload dist/* + - uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ inputs.version }} + name: Release ${{ inputs.version }} + draft: false + prerelease: ${{ inputs.is-prerelease }} + files: dist/* + generate_release_notes: true diff --git a/.github/workflows/_reusable-release-status.yaml b/.github/workflows/_reusable-release-status.yaml new file mode 100644 index 0000000000..b855f284ea --- /dev/null +++ b/.github/workflows/_reusable-release-status.yaml @@ -0,0 +1,76 @@ +# Release Status Workflow +# +# This reusable workflow validates and reports the status of release processes, +# handling both RC and production releases. +# +# Key Features: +# - Status verification +# - Error detection +# - Release type handling +# - Status reporting +# - Process validation +# +# Process Stages: +# 1. Status Collection: +# - RC status check +# - Production status check +# - Process completion verification +# +# 2. Validation: +# - Error detection +# - Status analysis +# - Release type determination +# +# 3. Reporting: +# - Status summary +# - Error reporting +# - Success confirmation +# +# Required Inputs: +# - version: Release version +# - rc-status: RC process status +# - prod-status: Production process status +# +# Example Usage: +# 1. Status Check: +# jobs: +# status: +# uses: ./.github/workflows/_reusable-release-status.yaml +# with: +# version: "v1.2.3" +# rc-status: "success" +# prod-status: "success" +# +# Note: Should be used as final step in release workflows + +name: Release Status Check + +on: + workflow_call: + inputs: + version: + required: true + type: string + rc-status: + required: true + type: string + prod-status: + required: true + type: string + +jobs: + check-status: + runs-on: ubuntu-latest + steps: + - name: Verify workflow status + run: | + if [[ "${{ inputs.rc-status }}" == "failure" || "${{ inputs.prod-status }}" == "failure" ]]; then + echo "::error::Release workflow failed" + exit 1 + fi + + if [[ "${{ inputs.version }}" =~ -rc ]]; then + echo "Release candidate ${{ inputs.version }} processed successfully" + else + echo "Production release ${{ inputs.version }} completed successfully" + fi diff --git a/.github/workflows/_reusable-release-validation.yaml b/.github/workflows/_reusable-release-validation.yaml new file mode 100644 index 0000000000..5119531c1c --- /dev/null +++ b/.github/workflows/_reusable-release-validation.yaml @@ -0,0 +1,178 @@ +# Release Validation Workflow +# +# This reusable workflow performs comprehensive validation of releases, +# including quality checks, testing, security scanning, and artifact building. +# +# Key Features: +# - Version format validation +# - Code quality verification +# - Test suite execution +# - Security scanning +# - Package building +# - Pre-release validation +# +# Process Stages: +# 1. Version Validation: +# - Checks version string format +# - Validates pre-release compatibility +# - Ensures version consistency +# +# 2. Quality Assurance: +# - Runs code quality checks +# - Performs style verification +# - Validates documentation +# +# 3. Testing: +# - Executes unit tests +# - Runs integration tests +# - Generates coverage reports +# +# 4. Security: +# - Performs security scans +# - Checks dependencies +# - Validates compliance +# +# 5. Build Process: +# - Creates distribution packages +# - Verifies package integrity +# - Prepares artifacts +# +# Required Inputs: +# - version: Version string to validate +# - python-version: Python version for building +# - verify-package: Whether to verify built package +# - dry-run: Run without creating artifacts +# - allow-prerelease: Allow RC versions +# +# Required Secrets: +# - codecov-token: Token for coverage reporting +# +# Outputs: +# - version: Validated version string +# - artifact-name: Name of built artifact +# +# Example Usage: +# 1. Basic Validation: +# jobs: +# validate: +# uses: ./.github/workflows/_reusable-release-validation.yaml +# with: +# version: "v1.2.3" +# python-version: "3.10" +# secrets: +# codecov-token: ${{ secrets.CODECOV_TOKEN }} +# +# 2. Pre-release Validation: +# jobs: +# rc-validate: +# uses: ./.github/workflows/_reusable-release-validation.yaml +# with: +# version: "v1.2.3-rc1" +# python-version: "3.10" +# allow-prerelease: true +# secrets: +# codecov-token: ${{ secrets.CODECOV_TOKEN }} +# +# Note: This workflow is a critical part of the release pipeline and should +# be completed successfully before proceeding with RC or production releases. + +name: Release Validation + +on: + workflow_call: + inputs: + version: + required: true + type: string + python-version: + required: true + type: string + verify-package: + required: false + type: boolean + default: true + dry-run: + required: false + type: boolean + default: false + allow-prerelease: + required: false + type: boolean + default: false + secrets: + codecov-token: + required: true + outputs: + version: + description: "Validated version string" + value: ${{ jobs.version-check.outputs.version }} + artifact-name: + description: "Name of the built artifact" + value: ${{ jobs.build.outputs.artifact-name }} + +jobs: + version-check: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.validate-version.outputs.version }} + steps: + - name: Validate version + id: validate-version + run: | + VERSION="${{ inputs.version }}" + if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Invalid version format: $VERSION" + exit 1 + fi + if [[ $VERSION =~ -rc[0-9]+$ ]] && [[ "${{ inputs.allow-prerelease }}" != "true" ]]; then + echo "::error::Pre-release versions not allowed" + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + + quality: + needs: [version-check] + uses: ./.github/workflows/_reusable-code-quality.yaml + with: + python-version: ${{ inputs.python-version }} + + unit-tests: + needs: [version-check] + uses: ./.github/workflows/_reusable-test-suite.yaml + with: + python-version: ${{ inputs.python-version }} + test-type: "unit" + runner: "ubuntu-latest" + timeout: 10 + secrets: + codecov-token: ${{ secrets.codecov-token }} + + integration-tests: + needs: [version-check] + uses: ./.github/workflows/_reusable-test-suite.yaml + with: + python-version: ${{ inputs.python-version }} + test-type: "integration" + runner: "self-hosted" + timeout: 30 + secrets: + codecov-token: ${{ secrets.codecov-token }} + + security: + needs: [version-check] + uses: ./.github/workflows/_reusable-security-scan.yaml + with: + tools: "bandit,semgrep,trivy,clamav" + scan-scope: "all" + severity-level: "LOW" + fail-on-findings: true + + build: + needs: [version-check, quality, unit-tests, integration-tests, security] + if: | + !inputs.dry_run && + !failure() && !cancelled() + uses: ./.github/workflows/_reusable-artifact-builder.yaml + with: + python-version: ${{ inputs.python-version }} + verify-package: ${{ inputs.verify-package }} diff --git a/.github/workflows/_reusable-security-scan.yaml b/.github/workflows/_reusable-security-scan.yaml new file mode 100644 index 0000000000..332804282c --- /dev/null +++ b/.github/workflows/_reusable-security-scan.yaml @@ -0,0 +1,181 @@ +# Reusable Security Scan Workflow +# +# This reusable workflow orchestrates multiple security scanning tools to provide +# comprehensive security analysis of the codebase. +# +# Key Features: +# - Parallel security tool execution +# - Configurable tool selection +# - Comprehensive result aggregation +# - Artifact preservation +# - Customizable failure thresholds +# +# Process Stages: +# 1. Tool Selection and Configuration +# 2. Parallel Security Scans +# 3. Result Aggregation +# 4. Report Generation +# +# Required Inputs: +# - tools: Comma-separated list of tools to run +# - scan-scope: Scope of scanning +# - severity-level: Minimum severity threshold +# - fail-on-findings: Whether to fail on security findings +# +# Outputs: +# - has-findings: Boolean indicating if security issues were found +# +# Example Usage: +# jobs: +# security: +# uses: ./.github/workflows/_reusable-security-scan.yaml +# with: +# tools: "bandit,semgrep" +# scan-scope: "changed" +# severity-level: "MEDIUM" +# fail-on-findings: true +# +# Note: Different security tools may require specific permissions +# or configurations. + +name: Reusable Security Scan + +on: + workflow_call: + inputs: + tools: + description: "Security tools to run (comma-separated: bandit,clamav,semgrep,trivy)" + type: string + default: "bandit,semgrep" + scan-scope: + description: "Scan scope (all/changed)" + type: string + default: "changed" + severity-level: + description: "Minimum severity level (LOW/MEDIUM/HIGH)" + type: string + default: "LOW" + fail-on-findings: + description: "Fail workflow if issues found" + type: boolean + default: true + outputs: + has-findings: + description: "Whether any security issues were found" + value: ${{ jobs.summarize.outputs.has_findings }} + +jobs: + bandit: + if: contains(inputs.tools, 'bandit') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Bandit scan + uses: ./.github/actions/security/bandit + with: + scan-scope: ${{ inputs.scan-scope }} + severity-level: ${{ inputs.severity-level }} + fail-on-findings: ${{ inputs.fail-on-findings }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bandit-results + path: security-results/bandit + retention-days: 7 + + semgrep: + if: contains(inputs.tools, 'semgrep') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run Semgrep scan + uses: ./.github/actions/security/semgrep + with: + scan-scope: ${{ inputs.scan-scope }} + severity-level: ${{ inputs.severity-level }} + fail-on-findings: ${{ inputs.fail-on-findings }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: semgrep-results + path: security-results/semgrep + retention-days: 7 + + trivy: + if: contains(inputs.tools, 'trivy') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Required for changed files detection + + - name: Run Trivy scan + id: trivy + uses: ./.github/actions/security/trivy + with: + scan_type: "fs" + scan_scope: ${{ inputs.scan-scope }} + severity: ${{ inputs.severity-level }},HIGH,CRITICAL + scanners: "vuln,secret" + format: "sarif" + timeout: "15m" + ignore_unfixed: "true" + + - name: Move Trivy results + if: always() && steps.trivy.outputs.report_path + run: | + mkdir -p security-results/trivy + mv ${{ steps.trivy.outputs.report_path }} security-results/trivy/ + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trivy-results + path: security-results/trivy + retention-days: 7 + + clamav: + if: contains(inputs.tools, 'clamav') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run ClamAV scan + uses: ./.github/actions/security/clamav + with: + scan-scope: ${{ inputs.scan-scope }} + fail-on-findings: ${{ inputs.fail-on-findings }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: clamav-results + path: security-results/clamav + retention-days: 7 + + summarize: + needs: [bandit, semgrep, trivy, clamav] + if: always() + runs-on: ubuntu-latest + outputs: + has_findings: ${{ steps.check-findings.outputs.has_findings }} + steps: + - id: check-findings + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "has_findings=true" >> $GITHUB_OUTPUT + else + echo "has_findings=false" >> $GITHUB_OUTPUT + fi + + - name: Download all results + uses: actions/download-artifact@v4 + with: + pattern: "*-results" + merge-multiple: true + path: all-results + + - name: Upload combined results + uses: actions/upload-artifact@v4 + with: + name: security-scan-results + path: all-results + retention-days: 7 diff --git a/.github/workflows/_reusable-test-suite.yaml b/.github/workflows/_reusable-test-suite.yaml new file mode 100644 index 0000000000..4277fd9b49 --- /dev/null +++ b/.github/workflows/_reusable-test-suite.yaml @@ -0,0 +1,109 @@ +# Test Suite Workflow +# +# This reusable workflow executes comprehensive test suites with configurable +# parameters for different test types and environments. +# +# Key Features: +# - Parallel test execution +# - Multiple test types support +# - Coverage reporting +# - Configurable timeouts +# - Result aggregation +# +# Process Stages: +# 1. Unit Testing: +# - Environment setup +# - Test execution +# - Coverage generation +# +# 2. Integration Testing: +# - Environment preparation +# - Test execution +# - Result collection +# +# 3. Results Processing: +# - Coverage reporting +# - Result aggregation +# - Artifact creation +# +# Required Inputs: +# - python-version: Python version for tests (default: "3.10") +# - test-type: Type of test to run (unit/integration/e2e) +# - runner: Runner to use for the tests +# - timeout: Test timeout in minutes +# +# Required Secrets: +# - codecov-token: Token for coverage reporting (optional) +# +# Outputs: +# - test-results: Path to test results artifact +# +# Example Usage: +# 1. Basic Test Run: +# jobs: +# test: +# uses: ./.github/workflows/_reusable-test-suite.yaml +# with: +# test-type: "unit" +# +# 2. Custom Configuration: +# jobs: +# test: +# uses: ./.github/workflows/_reusable-test-suite.yaml +# with: +# python-version: "3.11" +# test-type: "unit" +# timeout: 15 +# secrets: +# codecov-token: ${{ secrets.CODECOV_TOKEN }} +# +# Note: Requires properly configured pytest environment and test structure + +name: Reusable Test Suite + +on: + workflow_call: + inputs: + python-version: + description: "Python version to use for tests" + type: string + default: "3.10" + test-type: + description: "Type of test to run (unit/integration/e2e)" + type: string + required: true + runner: + description: "Runner to use for the tests" + type: string + default: "ubuntu-latest" + timeout: + description: "Test timeout in minutes" + type: number + default: 10 + secrets: + codecov-token: + required: false + description: "Token for Codecov upload" + outputs: + test-results: + description: "Path to test results artifact" + value: ${{ jobs.test.outputs.artifact-path }} + +jobs: + test: + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout }} + steps: + - name: Verify GPU availability + if: contains(inputs.runner, 'self-hosted') + shell: bash + run: | + nvidia-smi || echo "::error::No GPU found" + + - uses: actions/checkout@v4 + - name: Run tests + uses: ./.github/actions/pytest + with: + python-version: ${{ inputs.python-version }} + test-type: ${{ inputs.test-type }} + codecov-token: ${{ secrets.codecov-token }} diff --git a/.github/workflows/_reusable-version-check.yaml b/.github/workflows/_reusable-version-check.yaml new file mode 100644 index 0000000000..6f565e87a2 --- /dev/null +++ b/.github/workflows/_reusable-version-check.yaml @@ -0,0 +1,104 @@ +# Version Check Workflow +# +# This reusable workflow validates version strings and determines their release type, +# supporting both explicit version inputs and git tags. +# +# Key Features: +# - Semantic version validation +# - Pre-release version support +# - Git tag compatibility +# - Version format enforcement +# - Configurable pre-release rules +# +# Process Stages: +# 1. Version Extraction: +# - Input parsing +# - Git tag reading +# - Format validation +# +# 2. Version Analysis: +# - Semantic version parsing +# - Pre-release detection +# - Format compliance check +# +# 3. Validation Rules: +# - Version prefix check +# - Component validation +# - Pre-release acceptance +# +# Required Inputs: +# - version: Version string to validate (optional) +# - allow-prerelease: Allow pre-release versions (default: true) +# +# Outputs: +# - version: Validated version string +# - is_prerelease: Whether version is a pre-release +# +# Example Usage: +# 1. Basic Version Check: +# jobs: +# validate: +# uses: ./.github/workflows/_reusable-version-check.yaml +# with: +# version: "v1.2.3" +# +# 2. Pre-release Check: +# jobs: +# validate: +# uses: ./.github/workflows/_reusable-version-check.yaml +# with: +# version: "v1.2.3-rc1" +# allow-prerelease: true +# +# Note: Version must follow format vX.Y.Z or vX.Y.Z-rcN + +name: Reusable Version Check + +on: + workflow_call: + inputs: + version: + description: "Version to validate" + required: false + type: string + allow-prerelease: + description: "Allow pre-release versions" + type: boolean + default: true + outputs: + version: + description: "Validated version" + value: ${{ jobs.validate.outputs.version }} + is_prerelease: + description: "Whether version is a pre-release" + value: ${{ jobs.validate.outputs.is_prerelease }} + +jobs: + validate: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get-version.outputs.version }} + is_prerelease: ${{ steps.check-prerelease.outputs.is_prerelease }} + steps: + - uses: actions/checkout@v4 + - name: Validate version + id: get-version + run: | + VERSION="${{ inputs.version || github.ref_name }}" + if ! [[ $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Invalid version format. Must be vX.Y.Z or vX.Y.Z-rcN" + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Check pre-release + id: check-prerelease + run: | + if [[ "${{ steps.get-version.outputs.version }}" =~ -rc[0-9]+$ ]]; then + if [[ "${{ inputs.allow-prerelease }}" != "true" ]]; then + echo "::error::Pre-release versions are not allowed" + exit 1 + fi + echo "is_prerelease=true" >> $GITHUB_OUTPUT + else + echo "is_prerelease=false" >> $GITHUB_OUTPUT + fi diff --git a/.github/workflows/code_scan.yml b/.github/workflows/code_scan.yml deleted file mode 100644 index 2ebcf03770..0000000000 --- a/.github/workflows/code_scan.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Code Scanning -permissions: read-all - -on: - workflow_dispatch: # run on request (no need for PR) - schedule: - # every UTC 6PM from Mon to Fri - - cron: "0 18 * * 1-5" - -jobs: - Bandit: - runs-on: ubuntu-20.04 - steps: - - name: CHECKOUT REPOSITORY - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install dependencies - run: python -m pip install tox - - name: Bandit Scanning - run: tox -e bandit-scan - - name: UPLOAD BANDIT REPORT - uses: actions/upload-artifact@v4 - with: - name: bandit-report - path: .tox/bandit-report.txt - # Use always() to always run this step to publish scan results when there are test failures - if: ${{ always() }} - Trivy-scan: - runs-on: ubuntu-20.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install dependencies - run: python -m pip install tox - - name: Trivy Scanning - env: - TRIVY_DOWNLOAD_URL: ${{ vars.TRIVY_DOWNLOAD_URL }} - run: tox -vv -e trivy-scan - - name: Upload Trivy results artifact - uses: actions/upload-artifact@v4 - with: - name: trivy-results - path: | - .tox/trivy-scan-results.txt - .tox/trivy-spdx-anomalib.json diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 0000000000..fa2cb8e0f8 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,84 @@ +# Pull Request Workflow +# +# This workflow orchestrates quality checks, tests, and security scans for +# pull requests using reusable workflows. +# +# Key Features: +# - Code quality validation +# - Test suite execution +# - Security scanning +# - Concurrent execution handling +# - Automated feedback +# +# Process Stages: +# 1. Quality Checks: +# - Code style verification +# - Type checking +# - Pre-commit hook validation +# +# 2. Testing: +# - Unit test execution +# - Integration testing +# - Coverage reporting +# +# 3. Security: +# - Changed files scanning +# - Vulnerability detection +# - Security report generation +# +# Required Secrets: +# - CODECOV_TOKEN: Coverage reporting token +# +# Example Usage: +# Automatically triggered on: +# 1. Pull requests to main branch +# 2. Pull requests to feature/* branches +# +# Note: Configured to cancel outdated runs when new commits are pushed + +name: PR Checks + +on: + pull_request: + branches: [main, "feature/**"] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Code quality job using reusable workflow + quality: + uses: ./.github/workflows/_reusable-code-quality.yaml + with: + python-version: "3.10" + + # Test suite job using reusable workflow + unit-tests: + uses: ./.github/workflows/_reusable-test-suite.yaml + with: + test-type: "unit" + runner: "ubuntu-latest" + timeout: 10 + secrets: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + integration-tests: + uses: ./.github/workflows/_reusable-test-suite.yaml + with: + test-type: "integration" + runner: "self-hosted" + timeout: 30 + secrets: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + # NOTE: When we have e2e or other tests, we can add them here. + + # Security scanning job using reusable workflow + security: + needs: [] # No dependencies, can run in parallel + uses: ./.github/workflows/_reusable-security-scan.yaml + with: + tools: "semgrep" # Security tools to run + scan-scope: "changed" # Only scan changed files + severity-level: "MEDIUM" # Minimum severity to report diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 777f9b0645..0000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Upload Python Package -permissions: read-all - -on: - release: - types: [published] - workflow_dispatch: # run on request (no need for PR) - -jobs: - deploy: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build twine - - name: Build the python package - run: | - python -m build - - name: Upload to PyPI - run: twine upload dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..bcb203506f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,104 @@ +# Release Workflow +# +# This workflow manages the complete release process for both production +# and release candidate versions. +# +# Key Features: +# - Version validation +# - Comprehensive testing +# - Security scanning +# - RC and production releases +# - Manual and automated triggers +# +# Process Stages: +# 1. Release Validation: +# - Version format check +# - Code quality verification +# - Test suite execution +# - Security assessment +# +# 2. RC Process: +# - Test PyPI deployment +# - Validation steps +# - Approval collection +# +# 3. Production Release: +# - Production PyPI deployment +# - GitHub release creation +# - Documentation updates +# +# Required Secrets: +# - CODECOV_TOKEN: Coverage reporting +# - TEST_PYPI_TOKEN: RC deployments +# - PYPI_TOKEN: Production deployments +# +# Example Usage: +# 1. Automated (Tag Push): +# - Production: Push tag v1.2.3 +# - RC: Push tag v1.2.3-rc1 +# +# 2. Manual Trigger: +# - With version input +# - Optional dry-run mode +# +# Note: Supports both automated tag-based and manual releases + +name: Release + +on: + push: + tags: + - "v*.*.*" + - "v*.*.*-rc*" + workflow_dispatch: + inputs: + version: + description: "Version to release (e.g., v1.2.3 or v1.2.3-rc1)" + required: true + type: string + dry_run: + description: "Perform a dry run without creating a release" + required: false + type: boolean + default: false + +jobs: + validation: + uses: ./.github/workflows/_reusable-release-validation.yaml + with: + version: ${{ github.event_name == 'push' && github.ref_name || inputs.version }} + python-version: "3.10" + verify-package: true + dry-run: ${{ github.event.inputs.dry_run || false }} + allow-prerelease: true + secrets: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + rc-release-process: + needs: [validation] + if: contains(needs.validation.outputs.version, '-rc') + uses: ./.github/workflows/_reusable-rc-release-process.yaml + with: + version: ${{ needs.validation.outputs.version }} + artifact-name: ${{ needs.validation.outputs.artifact-name }} + secrets: + test-pypi-token: ${{ secrets.TEST_PYPI_TOKEN }} + + production-release-process: + needs: [validation] + if: ${{ !contains(needs.validation.outputs.version, '-rc') }} + uses: ./.github/workflows/_reusable-production-release-process.yaml + with: + version: ${{ needs.validation.outputs.version }} + artifact-name: ${{ needs.validation.outputs.artifact-name }} + secrets: + pypi-token: ${{ secrets.PYPI_TOKEN }} + + status: + needs: [validation, rc-release-process, production-release-process] + if: always() && !inputs.dry_run + uses: ./.github/workflows/_reusable-release-status.yaml + with: + version: ${{ needs.validation.outputs.version }} + rc-status: ${{ needs.rc-release-process.result }} + prod-status: ${{ needs.production-release-process.result }} diff --git a/.github/workflows/security-checks.yaml b/.github/workflows/security-checks.yaml new file mode 100644 index 0000000000..0fd15148f7 --- /dev/null +++ b/.github/workflows/security-checks.yaml @@ -0,0 +1,96 @@ +# Security Checks Workflow +# +# This workflow orchestrates comprehensive security scanning using multiple tools and +# configurable parameters. It supports both scheduled and manual execution modes. +# +# Key Features: +# - Multiple security tool integration +# - Scheduled daily scans +# - Manual trigger with customization +# - Configurable severity thresholds +# - Flexible scan scope options +# +# Process Stages: +# 1. Scheduled Execution (Daily at 2 AM UTC): +# - Full security toolset +# - Complete codebase scan +# - LOW severity threshold +# +# 2. Manual Execution: +# - Selectable security tools +# - Adjustable scan scope +# - Customizable severity level +# +# Security Tools: +# - Bandit: Python-specific security scanning +# - ClamAV: Malware detection +# - Semgrep: Static Application Security Testing (SAST) +# - Trivy: Vulnerability scanning +# +# Required Permissions: +# - contents: read +# - security-events: write +# +# Example Usage: +# 1. Scheduled Run: +# Automatically runs with full configuration +# +# 2. Manual Trigger: +# workflow_dispatch: +# inputs: +# tools: "bandit,semgrep,trivy" +# scan-scope: "changed" +# severity-level: "MEDIUM" +# +# Note: Results are available as workflow artifacts and in the +# Security tab when integrated with GitHub Advanced Security. + +name: Security Checks + +on: + schedule: + # Run security checks every day at 2 AM UTC + - cron: "0 2 * * *" + + workflow_dispatch: + inputs: + tools: + description: "Security tools to run" + required: true + type: choice + options: + - "bandit,semgrep,trivy" # Default set + - "bandit,clamav,semgrep,trivy" # Full set + - "bandit,semgrep" # Minimal set + default: "bandit,semgrep,trivy" + scan-scope: + description: "Scan scope" + required: true + type: choice + options: + - all + - changed + default: "all" + severity-level: + description: "Minimum severity level" + required: true + type: choice + options: + - LOW + - MEDIUM + - HIGH + default: "LOW" + +permissions: + contents: read + security-events: write + +jobs: + security: + uses: ./.github/workflows/_reusable-security-scan.yaml + with: + # For scheduled runs, use full scan configuration + tools: ${{ github.event_name == 'schedule' && 'bandit,clamav,semgrep,trivy' || inputs.tools }} + scan-scope: ${{ github.event_name == 'schedule' && 'all' || inputs.scan-scope }} + severity-level: ${{ github.event_name == 'schedule' && 'LOW' || inputs.severity-level }} + fail-on-findings: true diff --git a/.github/workflows/upload_coverage.yml b/.github/workflows/upload_coverage.yml deleted file mode 100644 index 7770cb8b31..0000000000 --- a/.github/workflows/upload_coverage.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Upload coverage -permissions: read-all - -on: - workflow_run: - workflows: ["Pre-Merge Checks"] - types: - - completed - -jobs: - Upload-Coverage: - runs-on: ubuntu-latest - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: ${{ github.event.workflow_run.artifact_url }} - path: coverage - - name: Upload coverage report - run: | - COMMIT_ID=${{ github.event.workflow_run.head_sha }} - # Pass token from secrets if available. Otherwise it takes it from the environment variable of the CI - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - if [ -n "${{ secrets.CODECOV_TOKEN }}" ] - then - ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f coverage/coverage.xml - else - ./codecov -t "${CODECOV_TOKEN}" --sha $COMMIT_ID -U $HTTP_PROXY -f coverage/coverage.xml - fi diff --git a/docs/source/markdown/guides/developer/index.md b/docs/source/markdown/guides/developer/index.md index 33bb6e950a..3c7b83cf39 100644 --- a/docs/source/markdown/guides/developer/index.md +++ b/docs/source/markdown/guides/developer/index.md @@ -34,4 +34,5 @@ Learn the criteria for reviewing code. ./sdd ./contributing ./code_review_checklist +./release_guidelines ``` diff --git a/docs/source/markdown/guides/developer/release_guidelines.md b/docs/source/markdown/guides/developer/release_guidelines.md new file mode 100644 index 0000000000..3fdf7f12b7 --- /dev/null +++ b/docs/source/markdown/guides/developer/release_guidelines.md @@ -0,0 +1,367 @@ +# {octicon}`rocket` Release Process Guide + +This document outlines the release process for our Python package, including environment setup, workflow configuration, and step-by-step release instructions. + +## {octicon}`checklist` Prerequisites + +### {octicon}`key` Required Tokens + +:::{dropdown} Test PyPI Token +:animate: fade-in-slide-down + +1. Create account on [Test PyPI](https://test.pypi.org) +2. Generate API token: Account Settings → API tokens +3. Scope: Upload to project +4. Save as `TEST_PYPI_TOKEN` in GitHub Actions secrets + ::: + +:::{dropdown} Production PyPI Token +:animate: fade-in-slide-down + +1. Create account on [PyPI](https://pypi.org) +2. Generate API token: Account Settings → API tokens +3. Scope: Upload to project +4. Save as `PYPI_TOKEN` in GitHub Actions secrets + ::: + +:::{dropdown} Codecov Token +:animate: fade-in-slide-down + +1. Create account on [Codecov](https://codecov.io) +2. Generate repository upload token +3. Scope: Repository-specific access +4. Save as `CODECOV_TOKEN` in GitHub Actions secrets + ::: + +### {octicon}`repo` GitHub Repository Setup + +:::{dropdown} Branch Protection +:animate: fade-in-slide-down + +```text +main branch: +☑️ Require pull request reviews +☑️ Require status checks to pass +☑️ Require linear history +☑️ Include administrators +``` + +::: + +:::{dropdown} Required Status Checks +:animate: fade-in-slide-down + +- Unit Tests +- Integration Tests +- Security Scans +- Quality Checks + ::: + +## {octicon}`gear` Environment Setup + +### {octicon}`project` 1. Create GitHub Environments + +Create the following environments in your repository: + +1. Go to Repository Settings → Environments → New environment + +:::{dropdown} Staging Environment +:animate: fade-in-slide-down + +```yaml +Name: staging +Protection rules: None (allows automated RC deployments) +``` + +::: + +:::{dropdown} QA Environment +:animate: fade-in-slide-down + +```yaml +Name: qa +Protection rules: + - Required reviewers: [QA team members] + - Wait timer: 0 minutes + - Deployment branches: main +Environment secrets: None +``` + +::: + +:::{dropdown} Production Release Environment +:animate: fade-in-slide-down + +```yaml +Name: production-release +Protection rules: + - Required reviewers: [Release managers] + - Wait timer: 30 minutes + - Deployment branches: main +Environment secrets: None +``` + +::: + +:::{dropdown} Production Deploy Environment +:animate: fade-in-slide-down + +```yaml +Name: production-deploy +Protection rules: + - Required reviewers: [Senior engineers, DevOps] + - Wait timer: 0 minutes + - Deployment branches: main +Environment secrets: + - PYPI_TOKEN: [Production PyPI token] +``` + +::: + +### {octicon}`key-asterisk` 2. Configure Repository Secrets + +Go to Repository Settings → Secrets and variables → Actions → New repository secret + +```yaml +TEST_PYPI_TOKEN: [Test PyPI token] +PYPI_TOKEN: [Production PyPI token] +``` + +## {octicon}`versions` Release Types + +### {octicon}`git-branch` Release Candidate (RC) + +- Format: `vX.Y.Z-rcN` +- Examples: `v1.2.3-rc1`, `v1.2.3-rc2` +- Purpose: Testing and validation +- Deploys to: Test PyPI + +### {octicon}`git-merge` Production Release + +- Format: `vX.Y.Z` +- Examples: `v1.2.3`, `v2.0.0` +- Purpose: Production deployment +- Deploys to: Production PyPI + +## {octicon}`workflow` Release Process + +### {octicon}`pencil` 1. Prepare Release + +1. Update version in package files: + + ```python + # src/__init__.py or similar + __version__ = "1.2.3" + ``` + +2. Update CHANGELOG.md: + + ```markdown + ## [1.2.3] - YYYY-MM-DD + + ### Added + + - New feature X + + ### Changed + + - Modified behavior Y + + ### Fixed + + - Bug fix Z + ``` + +3. Create release branch: + + ```bash + git checkout -b release/v1.2.3 + git add . + git commit -m "chore: prepare release v1.2.3" + git push origin release/v1.2.3 + ``` + +4. Create and merge PR to main + +### {octicon}`git-branch` 2. Create Release Candidate + +```bash +# Ensure you're on main and up-to-date +git checkout main +git pull origin main + +# Create and push RC tag +git tag v1.2.3-rc1 +git push origin v1.2.3-rc1 +``` + +### {octicon}`checklist` 3. RC Validation Process + +:::{dropdown} Automated Checks +:animate: fade-in-slide-down + +- Quality checks +- Security scans +- Test suite +- Build verification + ::: + +:::{dropdown} RC Deployment +:animate: fade-in-slide-down + +- Automatic upload to Test PyPI +- Creates draft GitHub release + ::: + +:::{dropdown} QA Validation +:animate: fade-in-slide-down + +- Install from Test PyPI +- Run smoke tests +- Validate functionality + ::: + +:::{dropdown} Review Approvals +:animate: fade-in-slide-down + +- QA team approves in QA environment +- Release managers approve in production-release environment + ::: + +### {octicon}`git-merge` 4. Production Release + +After successful RC validation: + +```bash +# Create and push production tag +git tag v1.2.3 +git push origin v1.2.3 +``` + +The workflow will: + +1. Run all checks +2. Create GitHub release +3. Await approvals +4. Deploy to PyPI + +## {octicon}`bug` Troubleshooting + +### {octicon}`alert` Common Issues + +:::{dropdown} RC Upload Fails +:animate: fade-in-slide-down + +- Check Test PyPI token permissions +- Verify version doesn't exist on Test PyPI +- Ensure version follows PEP 440 + ::: + +:::{dropdown} Workflow Failures +:animate: fade-in-slide-down + +```bash +# View workflow logs +gh run list +gh run view [run-id] +``` + +::: + +:::{dropdown} Environment Approval Issues +:animate: fade-in-slide-down + +- Verify reviewer permissions +- Check environment protection rules +- Ensure reviewers are in correct teams + ::: + +### {octicon}`sync` Recovery Steps + +:::{dropdown} Failed RC +:animate: fade-in-slide-down + +```bash +# Delete failed RC tag +git tag -d v1.2.3-rc1 +git push origin :refs/tags/v1.2.3-rc1 + +# Create new RC +git tag v1.2.3-rc2 +git push origin v1.2.3-rc2 +``` + +::: + +:::{dropdown} Failed Production Release +:animate: fade-in-slide-down + +- Do not delete production tags +- Create new patch version if needed + ::: + +## {octicon}`light-bulb` Best Practices + +### {octicon}`versions` Version Management + +- Follow semantic versioning +- Use RCs for significant changes +- Include build metadata in package + +### {octicon}`note` Release Notes + +- Use consistent format +- Include upgrade instructions +- Document breaking changes + +### {octicon}`shield-lock` Security + +- Never share or commit tokens +- Review dependency updates +- Monitor security advisories + +### {octicon}`megaphone` Communication + +- Announce release schedule +- Document known issues +- Maintain changelog + +## {octicon}`bookmark` Quick Reference + +### {octicon}`terminal` Commands Cheatsheet + +```bash +# Create RC +git tag v1.2.3-rc1 +git push origin v1.2.3-rc1 + +# Create Release +git tag v1.2.3 +git push origin v1.2.3 + +# Delete Tag (only for RCs) +git tag -d v1.2.3-rc1 +git push origin :refs/tags/v1.2.3-rc1 + +# View Tags +git tag -l "v*" + +# Check Workflow Status +gh workflow list +gh run list +``` + +### {octicon}`file-directory` Required Files + +```text +repository/ +├── .github/ +│ ├── workflows/ +│ │ └── release.yaml +│ └── actions/ +├── src/ +│ └── __init__.py +├── tests/ +├── CHANGELOG.md +└── pyproject.toml +```