ci.yml

name: ci
on:
  push:
    branches:
      - master
  pull_request:

jobs:
  files-changed:
    name: detect what files changed
    runs-on: ubuntu-latest
    timeout-minutes: 3
    # Map a step output to a job output
    outputs:
      api_changes: ${{ steps.changes.outputs.api_changes }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v2

      - name: Check for backend file changes
        uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2
        id: changes
        with:
          token: ${{ github.token }}
          filters: .github/file-filters.yml

  linting:
    name: "pre-commit hooks" # (includes Python formatting + linting)
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - name: Internal github app token
        id: token
        uses: getsentry/action-github-app-token@d4b5da6c5e37703f8c3b3e43abb5705b46e159cc # v3.0.0
        continue-on-error: true
        with:
          app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
          private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Setup pre-commit
        run: make setup-git
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - uses: getsentry/paths-filter@v2
        id: files
        with:
          # Enable listing of files matching each filter.
          # Paths to files will be available in `${FILTER_NAME}_files` output variable.
          # Paths will be escaped and space-delimited.
          # Output is usable as command line argument list in linux shell
          list-files: shell

          # It doesn't make sense to lint deleted files.
          # Therefore we specify we are only interested in added or modified files.
          filters: |
            all:
              - added|modified: '**/*'
      - name: Run pre-commit checks
        # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master.
        # XXX: there is a very small chance that it'll expand to exceed Linux's limits
        #      `getconf ARG_MAX` - max # bytes of args + environ for exec()
        # we skip the `no-commit-to-branch` because in CI we are in fact on master already
        # and we have merged to it
        run: |
          SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }}

      # If working tree is dirty, commit and update if we have a token
      - name: Apply any pre-commit fixed files
        if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
        uses: getsentry/action-github-commit@v2.1.0
        with:
          github-token: ${{ steps.token.outputs.token }}

  rust-linting:
    name: "Linting - Rust"
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - name: Run linter
        run: |
          make lint-rust format-rust-ci

  typing:
    name: "mypy typing"
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - name: Run mypy
        run: |
          make backend-typing

  config-validation:
    name: "Dataset Config Validation"
    runs-on: ubuntu-latest
    timeout-minutes: 2
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          make install-python-dependencies
      - name: Validate configs
        run: |
          make validate-configs

  snuba-image:
    name: Build snuba CI image
    runs-on: ubuntu-latest
    timeout-minutes: 20
    outputs:
      branch: ${{ steps.branch.outputs.branch }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Get branch name
        id: branch
        # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that
        # it can be used as a docker tag
        run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT"

      # We are only using ghcr here for CI as `setup-gcloud` is a bit slow
      # Should revisit this when we move off of google cloud build (we may want to move these to GCR)
      - name: Registry login
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

      - name: Build snuba docker image for CI
        uses: docker/build-push-action@v6
        if: github.repository_owner == 'getsentry'
        with:
          context: .
          # push: true
          load: true
          target: testing
          tags: |
            ghcr.io/getsentry/snuba-ci:${{ github.sha }}
            ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
            ghcr.io/getsentry/snuba-ci:latest
          cache-from: |
            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
            type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
            type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
          # cache-to is re-implemented using the next step, to run conditionally for contributors

      - name: Publish images for cache
        if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository
        # outside contributors won't be able to push to the docker registry
        # ignore the failures in this step
        continue-on-error: ${{ github.event_name == 'pull_request' }}
        run: |
          # Useful to speed up CI
          docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
          docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }}
          if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then
            # The latest tag should only be published on `master`
            docker push ghcr.io/getsentry/snuba-ci:latest
          fi

  tests:
    needs: [linting, snuba-image]
    name: Tests and code coverage
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        snuba_settings:
          [
            "test",
            "test_rust",
            "test_distributed",
            "test_distributed_migrations",
          ]
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Registry login
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

      - name: Docker set up
        run: |
          docker network create --attachable cloudbuild

      - name: Docker Snuba Rust tests
        run: |
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test-rust
        if: ${{ matrix.snuba_settings == 'test_rust' }}

      - name: Docker Snuba tests
        run: |
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker compose -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings  == 'test_distributed' }}

      - name: Docker Snuba Multi-Node Tests
        run: |
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml up -d
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }}

      - name: Docker Snuba Init Tests
        run: |
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker compose -f docker-compose.gcb.yml run --rm snuba-test
        if: ${{ matrix.snuba_settings == 'test' }}

      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Upload to codecov
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

  admin-tests:
    needs: [linting]
    name: Front end tests for snuba admin
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - uses: volta-cli/action@v4
      - name: Set up and run tests through yarn
        run: cd snuba/admin && yarn install && yarn run test --coverage
      - name: Upload to codecov
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

  sentry:
    needs: [snuba-image, files-changed]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        instance: [0, 1, 2, 3]
    env:
      MIGRATIONS_TEST_MIGRATE: 1
      # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
      MATRIX_INSTANCE_TOTAL: 4
      TEST_GROUP_STRATEGY: ROUND_ROBIN

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Registry login
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

      - name: Checkout sentry
        uses: actions/checkout@v4
        with:
          repository: getsentry/sentry
          path: sentry

      - name: Setup steps
        id: setup
        run: |
          pip install --upgrade pip wheel
          # We cannot execute actions that are not placed under .github of the main repo
          mkdir -p .github/actions
          cp -r sentry/.github/actions/* .github/actions

      - name: Setup Sentry
        id: setup-sentry
        uses: ./.github/actions/setup-sentry
        with:
          workdir: sentry
          snuba: false
          kafka: true
          clickhouse: true

      - name: Start snuba
        run: |
          docker run -d --rm \
            -p 127.0.0.1:1218:1218 \
            -e PYTHONUNBUFFERED=1 \
            -e SNUBA_SETTINGS=docker \
            -e DEBUG=1 \
            -e DEFAULT_BROKERS=sentry_kafka:9092 \
            -e CLICKHOUSE_HOST=sentry_clickhouse \
            -e CLICKHOUSE_PORT=9000 \
            -e CLICKHOUSE_HTTP_PORT=8123 \
            -e REDIS_HOST=sentry_redis \
            -e REDIS_PORT=6379 \
            -e REDIS_DB=1 \
            --name sentry_snuba \
            --network sentry \
            ghcr.io/getsentry/snuba-ci:${{ github.sha }}
          docker exec sentry_snuba snuba migrations migrate --force

      - name: Run snuba tests
        if: needs.files-changed.outputs.api_changes == 'false'
        working-directory: sentry
        run: |
          pytest -k 'not __In' tests \
            -m snuba_ci \
            -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"

      - name: Run full tests
        # TODO: Adjust which changed files we care about
        if: needs.files-changed.outputs.api_changes == 'true'
        working-directory: sentry
        run: |
          pytest -k 'not __In' tests/snuba \
              tests/sentry/eventstream/kafka \
              tests/sentry/post_process_forwarder \
              tests/sentry/snuba \
              tests/sentry/eventstore/snuba \
              tests/sentry/search/events \
              tests/sentry/event_manager \
              tests/sentry/api/endpoints/test_organization_profiling_functions.py \
              tests/snuba/api/endpoints/test_organization_events_stats_mep.py \
              tests/sentry/sentry_metrics/querying \
              tests/snuba/test_snql_snuba.py \
              tests/snuba/test_metrics_layer.py \
              tests/sentry/api/endpoints/test_organization_spans_fields.py \
              tests/snuba/api/endpoints/test_organization_events_span_indexed.py \
              -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"

      - name: Run CI module tests
        if: needs.files-changed.outputs.api_changes == 'true'
        working-directory: sentry
        run: pytest -k 'not __In' tests -vv -m snuba_ci

  clickhouse-versions:
    needs: [linting, snuba-image]
    name: Tests on multiple clickhouse versions
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        version:
          [
            "23.8.11.29.altinitystable",
            "24.3.5.47.altinitystable",
          ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Registry login
        run: |
          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

      - name: Docker set up
        run: |
          docker network create --attachable cloudbuild

      - name: Docker Snuba Test other ClickHouse versions
        run: |
          export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}}
          SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test

      - name: Upload to codecov
        run: |
          curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

  self-hosted-end-to-end:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout Snuba
        uses: actions/checkout@v4
      - name: Run Sentry self-hosted e2e CI
        uses: getsentry/action-self-hosted-e2e-tests@main
        with:
          project_name: snuba
          docker_repo: getsentry/snuba
          image_url: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.event.pull_request.head.sha || github.sha }}
          docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }}

  publish-to-dockerhub:
    name: Publish Snuba to DockerHub
    runs-on: ubuntu-20.04
    if: ${{ (github.ref_name == 'master') }}
    steps:
      - uses: actions/checkout@v4 # v3.1.0
      - name: Pull the test image
        id: image_pull
        env:
          IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }}
        shell: bash
        run: |
          echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out."
          echo "Polling for $IMAGE_URL"
          timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done'
      - name: Get short SHA for docker tag
        id: short_sha
        shell: bash
        run: |
          SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          if [[ -z "$SHORT_SHA" ]]; then
            echo "Short SHA empty? Re-running rev-parse."
            git rev-parse --short "$GITHUB_SHA"
          else
            echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT
          fi
      - name: Push built docker image
        shell: bash
        env:
          SHORT_SHA: ${{ steps.short_sha.outputs.sha }}
          IMAGE_URL: us-central1-docker.pkg.dev/sentryio/snuba/image:${{ github.sha }}
        run: |
          # only login if the password is set
          if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi
          # We push 3 tags to Dockerhub:
          # first, the full sha of the commit
          docker tag ${IMAGE_URL} getsentry/snuba:${GITHUB_SHA}
          docker push getsentry/snuba:${GITHUB_SHA}
          # second, the short sha of the commit
          docker tag ${IMAGE_URL} getsentry/snuba:${SHORT_SHA}
          docker push getsentry/snuba:${SHORT_SHA}
          # finally, nightly
          docker tag ${IMAGE_URL} getsentry/snuba:nightly
          docker push getsentry/snuba:nightly