Add text2sql tasks #1398
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Performance | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| performance: | |
| runs-on: ubuntu-latest | |
| env: | |
| OS: ubuntu-latest | |
| UNITXT_DEFAULT_VERBOSITY: error | |
| UNITXT_MOCK_INFERENCE_MODE: "True" | |
| DATASETS_VERBOSITY: error | |
| HF_HUB_VERBOSITY: error | |
| HF_DATASETS_DISABLE_PROGRESS_BARS: "True" | |
| TQDM_DISABLE: "True" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Requirements | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| uv pip install --system ".[tests,watsonx,inference-tests]" | |
| uv pip install --system litellm | |
| uv pip install --system diskcache | |
| huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} | |
| - name: Prepare the dirs for performance evaluation in main | |
| run: | | |
| mkdir -p performance_action | |
| cp performance/bluebench_profiler.py performance_action/bluebench_profiler.py | |
| cp performance/compare_benchmark_performance_results.py performance_action/compare_benchmark_performance_results.py | |
| - name: Run performance on PR just to warm the cache, output will be overwritten | |
| run : | | |
| python performance_action/bluebench_profiler.py --output_file performance_action/pr_results.json | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| clean: false | |
| - name: Run performance on main branch | |
| run: | | |
| python performance_action/bluebench_profiler.py --output_file performance_action/main_results.json | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| clean: false | |
| - name: Run performance on PR branch | |
| run: | | |
| python performance_action/bluebench_profiler.py --output_file performance_action/pr_results.json | |
| - name: Compare main and PR performance results | |
| run: | | |
| python performance_action/compare_benchmark_performance_results.py performance_action/main_results.json performance_action/pr_results.json >> $GITHUB_STEP_SUMMARY |