LanceDB - Remove Orphaned Chunks #1661
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | clickhouse | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - devel | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752 | |
| RUNTIME__LOG_LEVEL: ERROR | |
| DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} | |
| ACTIVE_DESTINATIONS: "[\"clickhouse\"]" | |
| ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" | |
| jobs: | |
| get_docs_changes: | |
| name: docs changes | |
| uses: ./.github/workflows/get_docs_changes.yml | |
| if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} | |
| run_loader: | |
| name: test | clickhouse tests | |
| needs: get_docs_changes | |
| if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@master | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10.x" | |
| - name: Install Poetry | |
| uses: snok/[email protected] | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp | |
| - name: Install dependencies | |
| run: poetry install --no-interaction -E clickhouse --with providers -E parquet --with sentry-sdk --with pipeline | |
| - name: create secrets.toml | |
| run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml | |
| # OSS ClickHouse | |
| - run: | | |
| docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" up -d | |
| echo "Waiting for ClickHouse to be healthy..." | |
| timeout 30s bash -c 'until docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" ps | grep -q "healthy"; do sleep 1; done' | |
| echo "ClickHouse is up and running" | |
| name: Start ClickHouse OSS | |
| - run: poetry run pytest tests/load -m "essential" | |
| name: Run essential tests Linux (ClickHouse OSS) | |
| if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} | |
| env: | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 | |
| - run: poetry run pytest tests/load | |
| name: Run all tests Linux (ClickHouse OSS) | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} | |
| env: | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 | |
| - name: Stop ClickHouse OSS | |
| if: always() | |
| run: docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" down -v | |
| # ClickHouse Cloud | |
| - run: | | |
| poetry run pytest tests/load -m "essential" | |
| name: Run essential tests Linux (ClickHouse Cloud) | |
| if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} | |
| - run: | | |
| poetry run pytest tests/load | |
| name: Run all tests Linux (ClickHouse Cloud) | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} | |