CI: try run kgw test

- main branch update
- new release
- auto trigger after 'CI' workflow success
- manual trigger

.github/workflows/ci.yaml

@@ -94,7 +94,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Cache Docker layers for kwild
+      - name: Cache Docker layers for kwild # both restore and save
         uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache-kwild
@@ -140,7 +140,6 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-kwild
           cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new
 
-      # maybe no need
       - name: Run acceptance test
         run: |
           testUserID=$(id -u)
@@ -150,7 +149,6 @@ jobs:
           KACT_LOG_LEVEL=warn task test:act:nb
 
       - name: Run integration test
-        # without kgw test, probably should disable this ?
         run: |
           testUserID=$(id -u)
           testGroupID=$(id -g)
@@ -169,4 +167,4 @@ jobs:
 
       - name: Show error log
         if: ${{ failure() }}
-        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilAct*/*.log  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilAct*/*.log  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log

.github/workflows/kgw-test-reuse.yaml

@@ -0,0 +1,154 @@
+name: Reusable kgw test workflow
+
+on:
+  workflow_call:
+    inputs:
+      kwil-db-branch:
+        required: true
+        type: string
+      kgw-branch:
+        required: true
+        type: string
+    secrets:
+      kgw-access-token:
+        required: true
+
+jobs:
+  run-kgw-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout kwil-db
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.kwil-db-branch }}
+          submodules: true
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22.x'
+          check-latest: true
+
+      - name: Install Taskfile
+        uses: arduino/setup-task@v2
+
+      - name: Init workspace
+        run: |
+          pwd
+          go work init && go work use . ./parse ./test ./core
+
+      - name: Generate go vendor
+        run: | # should build the vendor using `go work vendor`?
+          go version
+          task vendor
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers for kgw # both restore and save
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache-kgw
+          key: ${{ runner.os }}-buildx-kgw-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-kgw
+
+      - name: Pull kgw repo & create vendor
+        shell: bash
+        env:
+          GH_ACCESS_TOKEN: ${{ secrets.kgw-access-token }}
+        # vendor is used to bypass private repo issues;
+        # if kgw on main branch, we need to use go workspace, so that kgw
+        # always uses the latest version of kwil-db/core
+        run: |
+          git config --global url."https://${GH_ACCESS_TOKEN}:[email protected]/kwilteam/".insteadOf "https://github.com/kwilteam/"
+          rm -rf /tmp/kgw
+          git clone -b ${{ inputs.kgw-branch }}  https://github.com/kwilteam/kgw.git /tmp/kgw
+          cd /tmp/kgw
+          if [[ ${{ inputs.kgw-branch }} == release-* ]]; then
+            go mod vendor
+          else
+            # non release branch, use go workspace to always use the latest version of kwil-db/core
+            go work init
+            go work use . /home/runner/work/kwil-db/kwil-db/core
+            go work vendor
+          fi
+          cd -
+          rm -rf ~/.gitconfig
+
+      - name: Build kgw image
+        id: docker_build_kgw
+        uses: docker/build-push-action@v5
+        with:
+          context: /tmp/kgw
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          file: /tmp/kgw/Dockerfile
+          push: false
+          tags: kgw:latest
+          cache-from: type=local,src=/tmp/.buildx-cache-kgw
+          cache-to: type=local,dest=/tmp/.buildx-cache-kgw-new
+
+      - name: Cache Docker layers for kwild # both restore and save
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache-kwild
+          key: ${{ runner.os }}-buildx-kwild-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-kwild
+
+      - name: manual git tag
+        run: |
+          version=`echo ${{ github.sha }} | cut -c 1-7`
+          echo "GIT_TAG=$version" >> $GITHUB_ENV
+
+      - name: manual build time
+        run: |
+          build_time=`TZ=UTC date -u --date="@${SOURCE_DATE_EPOCH:-$(date +%s)}" +"%Y-%m-%dT%H:%M:%SZ"`
+          echo "BUILD_TIME=$build_time" >> $GITHUB_ENV
+
+      - name: Build kwild image
+        id: docker_build_kwild
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          build-args: |
+            git_commit=${{ github.sha }}
+            version=${{ env.GIT_TAG }}
+            build_time=${{ env.BUILD_TIME }}
+          file: ./build/package/docker/kwild.dockerfile
+          push: false
+          tags: kwild:latest
+          cache-from: type=local,src=/tmp/.buildx-cache-kwild
+          cache-to: type=local,dest=/tmp/.buildx-cache-kwild-new
+
+      - name: Build cli binaries
+        run: |
+          task build:cli
+
+      - name: Pull math extension docker image
+        run: |
+          docker pull kwilbrennan/extensions-math:multi-arch --platform linux/amd64
+
+      - name: Run kgw integration test
+        run: |
+          testUserID=$(id -u)
+          testGroupID=$(id -g)
+          cp test/integration/docker-compose.override.yml.example test/integration/docker-compose.override.yml
+          sed -i "s/\${UID}:\${GID}/${testUserID}:${testGroupID}/g" test/integration/docker-compose.override.yml
+          KIT_LOG_LEVEL=warn go test -count=1 -timeout 0 ./test/integration -run ^TestKGW$ -v
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache-kgw
+          mv /tmp/.buildx-cache-kgw-new /tmp/.buildx-cache-kgw
+
+      - name: Prune Docker
+        if: ${{ always() }}
+        run: docker rm $(docker ps -a -q) -f ; docker network prune -f ; docker volume prune -f || true
+
+      - name: Show error log
+        if: ${{ failure() }}
+        run: grep -C 20 -s -i -r -e 'kwild version' -e 'error' -e 'warn'  /tmp/TestKwilInt*/*.log /tmp/TestKwilInt*/*/*.log

.github/workflows/main.yaml

@@ -0,0 +1,15 @@
+name: main
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  kgw-test:
+    uses: ./.github/workflows/kgw-test-reuse.yaml
+    with:
+      kwil-db-branch: main
+      kgw-branch: main
+    secrets:
+      kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }}

.github/workflows/manual-test-kgw.yaml

@@ -0,0 +1,23 @@
+# manual-test-kgw.yaml run kgw-test with manual inputs: kwil-db-branch and kgw-branch.
+name: manual-test-kgw
+
+on:
+  workflow_dispatch:
+    inputs:
+      kwil-db-branch:
+        description: 'kwil-db branch to be tested'
+        required: true
+        type: string
+      kgw-branch:
+        description: 'kgw branch to be tested'
+        required: true
+        type: string
+
+jobs:
+  kgw-test:
+    uses: ./.github/workflows/kgw-test-reuse.yaml
+    with:
+      kwil-db-branch: ${{ github.event.inputs.kwil-db-branch }}
+      kgw-branch: ${{ github.event.inputs.kgw-branch }}
+    secrets:
+      kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }}

.github/workflows/release.yaml

@@ -1,11 +1,34 @@
 name: Release
 
 on:
-  workflow_dispatch:
   release:
-    types: [published, edited]
+    types: [unpublished]
 
 jobs:
+  parse-version:
+    name: Run KGW test on correspond version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get kgw version # minor version
+        id: versions
+        shell: bash
+        run: |
+          version=`echo ${{ github.event.release.tag_name }} | sed 's/^v//'` | cut -d '.' -f 2
+          kgw_version=`v7="v2"; v6="v1"; eval echo \${v$version}`
+          echo "kdb_version=v0.$version" >> $GITHUB_OUTPUT
+          echo "kgw_version=v0.$kgw_version" >> $GITHUB_OUTPUT
+          echo "kdb_version=v0.$version <> kgw_version=v0.$kgw_version"
+
+  call-kgw-test:
+    runs-on: ubuntu-latest
+    needs: parse-version
+    uses: ./.github/workflows/kgw-test-reuse.yaml
+    with:
+      kwil-db-branch: release-${{ needs.parse-version.outputs.kdb_version }}
+      kgw-branch: release-${{ needs.parse-version.outputs.kgw_version }}
+    secrets:
+      kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }}
+
   build-push-image:
     name: Build & push image
     if: false # temporary disable
@@ -105,6 +128,7 @@ jobs:
 
   deploy-to-eks:
     name: Deploy to k8s cluster
+    if: false # temporary disable
     runs-on: ubuntu-latest
     needs: build-push-image
 

.github/workflows/trigger-test-kgw.yaml

@@ -0,0 +1,21 @@
+# trigger-test-kgw will run jobs when the CI workflow is successful
+name: trigger-test-kgw
+
+on:
+  workflow_run:
+    workflows:
+      - CI
+    types:
+      - completed
+
+jobs:
+  on-ci-success:
+    # run when `CI` workflow is successful, not sure if this will work on forked PRs
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: ./.github/workflows/kgw-test-reuse.yaml
+    with:
+      # use the branch from the `CI` workflow
+      kwil-db-branch:  ${{ github.event.workflow_run.head_branch }}
+      kgw-branch: main
+    secrets:
+      kgw-access-token: ${{ secrets.KGW_MACH_SECRET_FOR_KWILDB }}