go.yml

name: Go Workflows

# note, because of generate tests using path filters here is too complicated
on:
  pull_request:
    branches-ignore:
      - 'gh-pages'
      - 'fe-release'
  push:
    branches-ignore:
      - 'gh-pages'

jobs:
  cancel-outdated:
    name: Cancel Outdated Jobs
    runs-on: ubuntu-latest
    steps:
      - id: skip_check
        if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && !contains(github.event.head_commit.message, '[no_skip]') }}
        uses: fkirc/skip-duplicate-actions@v5
        with:
          cancel_others: 'true'


  changes:
    name: Change Detection
    runs-on: ubuntu-latest
    outputs:
      # Expose matched filters as job 'packages' output variable with a trailing _deps for dependencies
      packages_deps: ${{ steps.filter_go.outputs.changed_modules_deps }}
      package_count_deps: ${{ steps.length.outputs.FILTER_LENGTH_DEPS }}

      unchanged_deps: ${{ steps.filter_go.outputs.unchanged_modules_deps }}
      unchanged_package_count_deps: ${{ steps.length.outputs.FILTER_LENGTH_UNCHANGED_DEPS }}

      # These have no dependencies included in the change outputs
      packages_nodeps: ${{ steps.filter_go.outputs.changed_modules }}
      package_count_nodeps: ${{ steps.length.outputs.FILTER_LENGTH_NODEPS }}
      solidity_changes: ${{ steps.filter_solidity.outputs.any_changed }}
      golangci_changed: ${{ steps.golangci_changed.outputs.any_changed }}
      all_packages: ${{ steps.filter_go.outputs.all_modules }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # note: after this action is pushed, whatever this ends up being should go back to latest.
      - uses: docker://ghcr.io/synapsecns/sanguine/git-changes-action:latest
        id: filter_go
        with:
          github_token: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }}
          timeout: '10m'

      - name: Check For Solidity Changes
        id: filter_solidity
        uses: tj-actions/changed-files@v26.1
        with:
          files: |
            **/*.sol

      - name: Check For GolangCI Changes
        id: golangci_changed
        uses: tj-actions/changed-files@v26.1
        with:
          # note: without building a yaml tree of our workflow, we won't be able to tell if golangci version changed so any ci change triggers full lint.
          files: |
           .golangci.yml
           .github/workflows/go.yml


      - name: Run step if any of the listed files above change
        if: steps.filter_solidity.outputs.any_changed == 'true'
        run: |
          echo "One or more files listed above has changed."

      - name: Golangci changed
        if: steps.golangci_changed.outputs.any_changed == 'true'
        run: |
          echo "Golangci has changed."


      - id: length
        run: |
          export FILTER_LENGTH_DEPS=$(echo $FILTERED_PATHS_DEPS | jq '. | length')
          echo "##[set-output name=FILTER_LENGTH_DEPS;]$(echo $FILTER_LENGTH_DEPS)"

          export FILTER_LENGTH_UNCHANGED_DEPS=$(echo $UNCHANGED_DEPS | jq '. | length')
          echo "##[set-output name=FILTER_LENGTH_UNCHANGED_DEPS;]$(echo $FILTER_LENGTH_UNCHANGED_DEPS)"

          export FILTER_LENGTH_NODEPS=$(echo $FILTERED_PATHS_NODEPS | jq '. | length')
          echo "##[set-output name=FILTER_LENGTH_NODEPS;]$(echo $FILTER_LENGTH_NODEPS)"
        env:
          FILTERED_PATHS_DEPS: ${{ steps.filter_go.outputs.changed_modules_deps }}
          UNCHANGED_DEPS: ${{ steps.filter_go.outputs.unchanged_modules_deps }}
          FILTERED_PATHS_NODEPS: ${{ steps.filter_go.outputs.changed_modules }}

  test:
    name: Go Coverage
    runs-on: ${{ (matrix.package == 'core' || matrix.package == 'tools' || contains(matrix.package, 'explorer') || contains(matrix.package, 'contrib')) && 'ubuntu-latest' || 'namespace-profile-default-coverage-runner' }}
    if: ${{ needs.changes.outputs.package_count_deps > 0 && (github.event_name != 'push' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) }}
    needs: changes
    strategy:
      fail-fast: false
      matrix:
        go-version:
          - 1.22.x
        package: ${{ fromJSON(needs.changes.outputs.packages_deps) }}
        exclude:
          - package: agents
    services:
      mariadb:
        image: mariadb:10.11.3
        ports:
          - 3306
        env:
          MYSQL_USER: user
          MYSQL_PASSWORD: password
          MYSQL_DATABASE: test
          MYSQL_ROOT_PASSWORD: password
        options: --health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=30
    defaults:
      run:
        working-directory: ${{ matrix.package }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
          submodules: 'recursive'

      - name: Go modules cache
        uses: actions/cache@v4
        if: ${{ !contains(runner.name, 'nsc') }}
        with:
          # see https://github.com/mvdan/github-actions-golang
          # also: https://glebbahmutov.com/blog/do-not-let-npm-cache-snowball/ w/ go build (workaround now is having a cache that just gets expired at night)
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
            ~/Library/Caches/go-build
            %LocalAppData%\go-build
          key: ${{ runner.os }}-test-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-test-${{matrix.package}}

      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go-version }}
          cache: ${{ contains(runner.name, 'nsc') && 'false' || 'true' }}

      # TODO: consider setting up cache key, see: https://discord.com/channels/975088590705012777/1263347501721714793/1263534719954452560
      - name: Setup Go cache
        uses: namespacelabs/nscloud-cache-action@v1
        if: ${{ contains(runner.name, 'nsc') }}
        with:
          cache: go

      - name: Verify MariaDB connection
        env:
          PORT: ${{ job.services.mariadb.ports[3306] }}
        run: |
          while ! mysqladmin ping -h"127.0.0.1" -P"$PORT" --silent; do
          sleep 1
          done

      - name: Install gotestsum
        uses: jaxxstorm/action-install-gh-release@v1.10.0
        with: # Grab the latest version
          repo: gotestyourself/gotestsum
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Get Coverage name
        id: coverage
        run: |
          echo "::set-output name=flag::$(echo "${{ matrix.package }}" | rev | cut -d/ -f1 | rev)"

      - name: Precompile Tests
        working-directory: ${{ matrix.package }}
        run: go test -run=nope ./...
        env:
          # no other containers are running at this point so we can use all available memory for compilation
          GOMEMLIMIT: 5GiB
          GOGC: -1

      - name: Test
        run: |
          set -e
          max_num_retries=3
          echo "mode: atomic" > coverage.txt

          # Loop through each package and run tests, with package level retries
          # as well as individual test retries.
          for pkg in $(go list ./...); do
            retries=0

            while [ $retries -lt $max_num_retries ]; do
              if gotestsum --rerun-fails=3 --packages="$pkg" --format standard-verbose -- -v -timeout 30m -coverpkg="$pkg" -coverprofile=profile.cov $pkg; then
                break
              else
                retries=$((retries+1))
                echo "Retry #$retries: 'gotestsum' command failed for $pkg"
              fi
            done

            # Check if max retries is exceeded for this package.
            if [ $retries -eq $max_num_retries ]; then
                echo "Max retries exceeded for $pkg"
                exit 1
            fi

            if [ -f profile.cov ]; then
              tail -n +2 profile.cov >> coverage.txt
              rm profile.cov
            fi
          done

          cp coverage.txt profile.cov
          docker ps
        env:
          ENABLE_MYSQL_TEST: true
          MYSQL_HOST: 0.0.0.0
          MYSQL_USER: user
          MYSQL_PASSWORD: password
          MYSQL_DATABASE: test
          MYSQL_ROOT_PASSWORD: password
          MYSQL_PORT: ${{ job.services.mariadb.ports[3306] }}
          GOMEMLIMIT: 8GiB
          GOGC: -1
          ETHEREUM_RPC_URI: ${{ secrets.ETHEREUM_RPC_URI }}
          GOPROXY: https://proxy.golang.org

      # TODO: these 3 steps should be moved into a reusable action.
      # also,should be noted these aren't comprehensive, as many of the containers are automatically removed
      # and we'd need something like this for all logs: https://github.com/synapsecns/sanguine/pull/1280#discussion_r1320889916
      # but this should be better than nothing as long as you don't assume the logs are complete
      #
      # there's a good case to have this in addition to the suggestions referenced in the comment above because
      # logs are not collected there in the case of sigterms (because logs stop piping when app breaks), which are caught here
      #
      # in any case this should be considered an early implementation
      - name: Collect docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
        with:
          dest: '/tmp/logs'

      - name: Replace forward slashes with dashes
        id: replace
        run: echo "::set-output name=package_name::$(echo ${{ matrix.package }} | sed 's/\//-/g')"
        shell: bash

      - name: Upload logs to GitHub
        #        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.replace.outputs.package_name }}
          path: /tmp/logs/**

      - name: Send Coverage (Codecov)
        uses: Wandalen/wretry.action@v1.0.36
        with:
          action: codecov/codecov-action@v3
          with: |
            token: ${{ secrets.CODECOV }}
            files: profile.cov
            flags: ${{ steps.coverage.outputs.flag }}
            fail_ci_if_error: true # optional (default = false)
            verbose: true # optional (default = false)
          attempt_limit: 5
          attempt_delay: 30000

  # make sure the build works
  build:
    name: Build
    needs: changes
    runs-on: ubuntu-latest
    if: ${{ needs.changes.outputs.package_count_deps > 0 && github.event_name != 'pull_request' }}
    strategy:
      fail-fast: false
      matrix:
        go-version:
          - 1.22.x
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go-version }}

      - name: Go modules cache
        uses: actions/cache@v4
        with:
          # see https://github.com/mvdan/github-actions-golang
          # also: https://glebbahmutov.com/blog/do-not-let-npm-cache-snowball/ w/ go build (workaround now is having a cache that just gets expired at night)
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
            ~/Library/Caches/go-build
            %LocalAppData%\go-build
          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-build-
      - name: Build
        # go build all workspaces
        run: go build $(go work edit -json | jq -c -r '[.Use[].DiskPath] | map_values(. + "/...")[]')
        # see: https://www.reddit.com/r/golang/comments/476pae/how_to_speed_up_go_compiler_and_many_other_go/
        # this cannot use gomemlimit because we are running multiple builds in paralell
        env:
          GOGC: 2000
          GOMEMLIMIT: 6GiB

  #note: right now this is not run against all work dirs
  lint:
    name: Lint
    runs-on: ubuntu-latest
    needs: changes
    if: ${{ (needs.changes.outputs.package_count_deps > 0 || needs.changes.outputs.golangci_changed == 'true') && (github.event_name != 'push' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) }}
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix:
        package: ${{ needs.changes.outputs.golangci_changed == 'true' && fromJSON(needs.changes.outputs.all_packages) || fromJSON(needs.changes.outputs.packages_nodeps) }}
    steps:
      - uses: actions/setup-go@v5
        with:
          go-version: 1.23.x

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # workaround for: https://github.com/golangci/golangci-lint-action/issues/479
      - name: Setup cache key
        run: cp ${{matrix.package}}/go.mod go.mod -v

      - name: golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
          working-directory: ${{matrix.package}}/
          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
          version: v1.60.3
          # see: https://github.com/golangci/golangci-lint/issues/2654
          args: --timeout=60m
        env:
          # GitHub token for annotations (optional)
          GITHUB_TOKEN: ${{ secrets.WORKFLOW_PAT || secrets.GITHUB_TOKEN }}
          GOMEMLIMIT: 6GiB
          GOGC: -1
  pr_metadata:
    # this is needed to prevent us from hitting the github api rate limit
    name: Get PR Metadata
    runs-on: ubuntu-latest
    # not stricly true, but this job is fast enough to not block and we want to prioritize canceling outdated because downstream jobs can use many workers
    needs: cancel-outdated
    # currently, this matches the logic in the go generate check. If we ever add more checks that run on all packages, we should
    # change this to run on those pushes
    if: ${{ github.event_name != 'push' && format('refs/heads/{0}', github.event.repository.default_branch) != github.ref }}
    outputs:
      issue_number: ${{ steps.find_pr.outputs.pr }}
      metadata: ${{ steps.metadata.outputs.METADATA }}
      labels: ${{ steps.metadata.outputs.LABELS }}
    steps:
      # TODO: https://stackoverflow.com/a/75429845 consider splitting w/ gql to reduce limit hit
      - name: Fetch Metadata for ${{github.event.number}}
        run: |
          # Fetch the metadata
          metadata="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER)"

          # Extract the labels in JSON format from the metadata
          labels_json="$(echo "$metadata" | jq -r -c '[.labels[].name]')"

          # Set the full metadata including the labels as the GitHub Actions step output
          echo "::set-output name=METADATA::$metadata"
          echo "::set-output name=LABELS::$labels_json"

        id: metadata
        shell: bash
        continue-on-error: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          OWNER: ${{ github.repository_owner }}
          REPO_NAME: ${{ github.event.repository.name }}
          PULL_REQUEST_NUMBER: ${{github.event.number}}

  # check if we need to rerun go generate as a result of solidity changes. Note, this will only run on solidity changes.
  # TODO: consolidate w/ go change check. This will run twice on agents
  check-generation-solidity:
    name: Go Generate (Solidity Only)
    runs-on: ubuntu-latest
    needs: [changes, pr_metadata]
    if: ${{ github.event_name != 'push' && needs.changes.outputs.solidity_changes == 'true' }}
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        # only do on agents for now. Anything that relies on solidity in a package should do this
        package: ['agents', 'services/rfq']
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
          submodules: 'recursive'

      - name: Setup NodeJS
        uses: ./.github/actions/setup-nodejs

      # Foundry is required for flattening
      - name: Install Foundry
        uses: foundry-rs/foundry-toolchain@v1
        with:
          version: nightly

      # Generate flattened files
      - name: Run flattener
        run: npx lerna exec npm run build:go --parallel

      # Setup Go
      - uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Go modules cache
        uses: actions/cache@v4
        with:
          # see https://github.com/mvdan/github-actions-golang
          # also: https://glebbahmutov.com/blog/do-not-let-npm-cache-snowball/ w/ go build (workaround now is having a cache that just gets expired at night)
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
            ~/Library/Caches/go-build
            %LocalAppData%\go-build
          # use seperate cache for generate, builds less stuff
          # TODO: consider scoping to package
          key: ${{ runner.os }}-go-generate-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-generate-${{matrix.package}}

      # TODO: remove
      - name: authenticate with github for private go modules
        if: ${{github.event.repository.private}}
        uses: fusion-engineering/setup-git-credentials@v2
        with:
          credentials: https://trajan0x:${{secrets.GIT_TOKEN }}@github.com/

      # See if we need to rerun go generate
      # TODO: consider implementing https://github.com/golang/go/issues/20520 to speed up process if possible
      - name: Try Go Generate
        working-directory: ${{matrix.package}}/
        run: |
          go generate ./...
        env:
          GOMEMLIMIT: 6GiB
          GOGC: -1

      - name: Verify Changed files
        uses: tj-actions/verify-changed-files@v11.1
        id: verify-changed-files
        with:
          files: |
            *.go

      - name: List all changed files tracked and untracked files
        if: steps.verify-changed-files.outputs.files_changed == 'true'
        run: |
          echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"

        # Fail if files need regeneration
        # TODO: this can run into a bit of a race condition if any other label is removed/added while this is run, look into fixing this by dispatching another workflow
      - name: Add Label
        if: ${{ !contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) && steps.verify-changed-files.outputs.files_changed == 'true' }}
        uses: ./.github/actions/add-label
        with:
          label: 'needs-go-generate-${{matrix.package}}'

      - name: Remove Label
        if: ${{ contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) && steps.verify-changed-files.outputs.files_changed != 'true' }}
        uses: ./.github/actions/remove-label
        with:
          label: 'needs-go-generate-${{matrix.package}}'

  remove-label-generation:
    name: Remove Generate Label From Unused Jobs
    runs-on: ubuntu-latest
    needs: [changes, pr_metadata]
    if: ${{ github.event_name != 'push' && needs.changes.outputs.unchanged_package_count_deps > 0 && contains(needs.pr_metadata.outputs.labels, 'needs-go-generate') }}
    strategy:
      fail-fast: false
      matrix:
        # only do on agents for now. Anything that relies on solidity in a package should do this
        package: ${{ fromJSON(needs.changes.outputs.unchanged_deps) }}
    steps:
      # needed for remove label action
      - uses: actions/checkout@v4
        if: ${{ contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) }}
        with:
          fetch-depth: 2
      - name: Remove Label
        if: ${{ contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) }}
        uses: ./.github/actions/remove-label
        with:
          label: 'needs-go-generate-${{matrix.package}}'

  # TODO: consider adding go work sync step to prevent https://github.com/golang/go/issues/65363
  check-generation:
    name: Go Generate (Module Changes)
    runs-on: ubuntu-latest
    needs: [changes, pr_metadata]
    if: ${{ github.event_name != 'push' && needs.changes.outputs.package_count_deps > 0 }}
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        # only do on agents for now. Anything that relies on solidity in a package should do this
        package: ${{ fromJSON(needs.changes.outputs.packages_deps) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2
          submodules: 'recursive'

      - name: Setup NodeJS
        uses: ./.github/actions/setup-nodejs

      # Foundry is required for flattening
      - name: Install Foundry
        uses: foundry-rs/foundry-toolchain@v1
        if: ${{ contains(matrix.package, 'agents') || contains(matrix.package, 'services/rfq') }}
        with:
          version: nightly

      - name: Install Node Dependencies
        run: yarn install --immutable
        if: ${{ contains(matrix.package, 'agents') || contains(matrix.package, 'services/rfq') }}

      # Generate flattened files
      - name: Run flattener
        run: npx lerna exec npm run build:go
        if: ${{ contains(matrix.package, 'agents') || contains(matrix.package, 'services/rfq') }}

      # Setup Go
      - uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Go modules cache
        uses: actions/cache@v4
        if: ${{ !contains(matrix.package, 'services/cctp-relayer') }}
        with:
          # see https://github.com/mvdan/github-actions-golang
          # also: https://glebbahmutov.com/blog/do-not-let-npm-cache-snowball/ w/ go build (workaround now is having a cache that just gets expired at night)
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
            ~/Library/Caches/go-build
            %LocalAppData%\go-build
          # use seperate cache for generate, builds less stuff
          # TODO: consider scoping to package
          key: ${{ runner.os }}-go-generate-${{matrix.package}}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-generate-${{matrix.package}}

      - name: Cache Linuxbrew
        uses: actions/cache@v4
        if: ${{ contains(matrix.package, 'scribe') }}
        with:
          path: |
            ~/.cache/Homebrew
            /home/linuxbrew/.linuxbrew/
          key: ${{ runner.os }}-linuxbrew-${{ hashFiles('**/Makefile') }}

      # TODO: remove
      - name: authenticate with github for private go modules
        if: ${{github.event.repository.private}}
        uses: fusion-engineering/setup-git-credentials@v2
        with:
          credentials: https://trajan0x:${{secrets.GIT_TOKEN }}@github.com/

      - name: setup env
        run: |
          echo "::set-env name=GOPATH::$(go env GOPATH)"
          echo "::add-path::$(go env GOPATH)/bin"
        shell: bash
        env:
          ACTIONS_ALLOW_UNSECURE_COMMANDS: true

      - name: Run Make Generate CI Deps (Scribe)
        working-directory: ${{matrix.package}}/
        if: ${{ contains(matrix.package, 'scribe') }}
        run: |
          export PATH=$PATH:$(go env GOPATH)/bin
          make generate-ci || exit 0

      - name: Run Make Generate CI (Scribe)
        working-directory: ${{matrix.package}}/
        if: ${{ contains(matrix.package, 'scribe') }}
        run: |
          make generate-ci

      # See if we need to rerun go generate
      # TODO: consider implementing https://github.com/golang/go/issues/20520 to speed up process if possible
      - name: Try Go Generate
        working-directory: ${{matrix.package}}/
        if: ${{ !contains(matrix.package, 'services/cctp-relayer') }}
        run: |
          go generate ./...
        env:
          GOMEMLIMIT: 6GiB
          GOGC: -1

      - name: Verify Changed files
        uses: tj-actions/verify-changed-files@v11
        id: verify-changed-files
        with:
          files: |
            *.go

      - name: List all changed files tracked and untracked files
        if: steps.verify-changed-files.outputs.files_changed == 'true'
        run: |
          echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"

        # Fail if files need regeneration
        # TODO: this can run into a bit of a race condition if any other label is removed/added while this is run, look into fixing this by dispatching another workflow
      - name: Add Label
        if: ${{ !contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) && steps.verify-changed-files.outputs.files_changed == 'true' }}
        uses: ./.github/actions/add-label
        with:
          label: 'needs-go-generate-${{matrix.package}}'

      - name: Remove Label
        if: ${{ contains(fromJson(needs.pr_metadata.outputs.labels), format('needs-go-generate-{0}', matrix.package)) && steps.verify-changed-files.outputs.files_changed != 'true' }}
        uses: ./.github/actions/remove-label
        with:
          label: 'needs-go-generate-${{matrix.package}}'