Branch2

.github/workflows/build_image.yml

@@ -42,6 +42,10 @@ on:
         required: false
         type: string
         default: 'image.tar'
+      enable_failpoint:
+        required: false
+        type: boolean
+        default: false
 
 
 jobs:
@@ -57,6 +61,11 @@ jobs:
         with:
           ref: ${{ inputs.branch }}
 
+      - name: Enable FailPoint
+        if: inputs.enable_failpoint
+        run: |
+          make failpoint-enable
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 

.github/workflows/new_endtoend_test.yml

@@ -108,10 +108,33 @@ jobs:
           $WESCALE_CI_IMAGE \
           /vt/examples/wesql-server/init_single_node_cluster.sh
 
+      - name: Wait for MySQL ports
+        run: |
+          timeout=300  # 5 minutes timeout
+          ports=(3306 15306)
+
+          for port in "${ports[@]}"; do
+            echo "Waiting for MySQL port $port..."
+            start_time=$(date +%s)
+            while ! nc -z localhost $port; do
+              current_time=$(date +%s)
+              elapsed=$((current_time - start_time))
+
+              if [ $elapsed -ge $timeout ]; then
+                echo "Timeout waiting for MySQL port $port"
+                exit 1
+              fi
+
+              echo "Port $port not ready. Retrying in 5 seconds..."
+              sleep 5
+            done
+            echo "MySQL port $port is ready!"
+          done
+
       - name: Run EndToEnd test
         run: |
           cd endtoend
-          go test ./... -v
+          go test ./wasm -v
 
       - name: Print Wescale logs
         run: |

.github/workflows/new_endtoend_test_for_branch.yml

@@ -0,0 +1,170 @@
+name: "New EndToEnd Test For Branch"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+      - 'release-*'
+    tags:
+      - '*'
+  workflow_dispatch:
+
+env:
+  IMAGE_NAME: wescale_ci_image
+  REGISTRY: ghcr.io
+  IMAGE_TAG: test-${{ github.sha }}
+  MYSQL_VERSION: 8.0.32
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-endtoend
+  cancel-in-progress: true
+
+jobs:
+  build-image:
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/build_image.yml
+    with:
+      branch: ${{ github.ref }}
+      image_name: ${{ github.repository_owner }}/wescale_ci_image
+      tags: test-${{ github.sha }}
+      platforms: linux/amd64
+      want_push: false
+      want_load: true
+      want_artifact: true
+      artifact_name: 'image.tar'
+      enable_failpoint: true
+
+  setup:
+    name: "New EndToEnd Test For Branch"
+    needs: build-image
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check if workflow needs to be skipped
+        id: skip-workflow
+        run: |
+          skip='false'
+          if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
+            skip='true'
+          fi
+          echo Skip ${skip}
+          echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
+
+      - name: Checkout code
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: actions/checkout@v3
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: Download Docker image
+        uses: actions/download-artifact@v3
+        with:
+          name: image.tar
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load < /tmp/image.tar
+          echo "Verifying image loaded:"
+          docker images
+
+      - name: Set up cluster
+        run: |
+          MYSQL_IMG="mysql/mysql-server:${{ env.MYSQL_VERSION }}"
+          WESCALE_CI_IMAGE="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}"
+
+          docker network create wescale-network
+
+          # target cluster
+          docker run -itd --network wescale-network --name mysql-server \
+          -p 3306:3306 \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_ROOT_HOST=% \
+          -e MYSQL_LOG_CONSOLE=true \
+          $MYSQL_IMG \
+          --bind-address=0.0.0.0 \
+          --port=3306 \
+          --log-bin=binlog \
+          --gtid_mode=ON \
+          --enforce_gtid_consistency=ON \
+          --log_replica_updates=ON \
+          --binlog_format=ROW
+
+          docker run -itd --network wescale-network --name wescale \
+          -p 15306:15306 \
+          -w /vt/examples/wesql-server \
+          -e MYSQL_ROOT_USER=root \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_PORT=3306 \
+          -e MYSQL_HOST=mysql-server \
+          -e CONFIG_PATH=/vt/config/wescale/default \
+          $WESCALE_CI_IMAGE \
+          /vt/examples/wesql-server/init_single_node_cluster.sh
+
+          # source cluster
+          docker run -itd --network wescale-network --name mysql-server3307 \
+          -p 3307:3307 \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_ROOT_HOST=% \
+          -e MYSQL_LOG_CONSOLE=true \
+          $MYSQL_IMG \
+          --bind-address=0.0.0.0 \
+          --port=3307 \
+          --log-bin=binlog \
+          --gtid_mode=ON \
+          --enforce_gtid_consistency=ON \
+          --log_replica_updates=ON \
+          --binlog_format=ROW
+
+          docker run -itd --network wescale-network --name wescale15307 \
+          -p 15307:15307 \
+          -w /vt/examples/wesql-server \
+          -e MYSQL_ROOT_USER=root \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_PORT=3307 \
+          -e MYSQL_HOST=mysql-server3307 \
+          -e VTGATE_MYSQL_PORT=15307 \
+          -e CONFIG_PATH=/vt/config/wescale/default \
+          $WESCALE_CI_IMAGE \
+          /vt/examples/wesql-server/init_single_node_cluster.sh
+
+      - name: Wait for MySQL ports
+        run: |
+          timeout=300  # 5 minutes timeout
+          ports=(3306 3307 15306 15307)
+
+          for port in "${ports[@]}"; do
+            echo "Waiting for MySQL port $port..."
+            start_time=$(date +%s)
+            while ! nc -z localhost $port; do
+              current_time=$(date +%s)
+              elapsed=$((current_time - start_time))
+
+              if [ $elapsed -ge $timeout ]; then
+                echo "Timeout waiting for MySQL port $port"
+                exit 1
+              fi
+
+              echo "Port $port not ready. Retrying in 5 seconds..."
+              sleep 5
+            done
+            echo "MySQL port $port is ready!"
+          done
+
+      - name: Run EndToEnd test
+        run: |
+          cd endtoend
+          go test ./branch -v --source_host_to_target=wescale15307
+
+      - name: Print Wescale logs
+        run: |
+          docker logs wescale

.gitignore

@@ -84,7 +84,6 @@ examples/mysql/vtdataroot
 /bin/
 /vtdataroot/
 venv
-.run/
 
 users.json
 .scannerwork

.run/vtgate.run.xml

@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="vtgate" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="wescale" />
+    <working_directory value="$PROJECT_DIR$/examples/wesql-server" />
+    <parameters value="--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global --alsologtostderr --log_dir $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate --log_queries_to_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate/vtgate_querylog.txt --port 15001 --grpc_port 15991 --mysql_server_port 15306 --mysql_server_socket_path /tmp/mysql.sock --cell zone1 --cells_to_watch zone1 --service_map grpc-vtgateservice --pid_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate/vtgate.pid --config_path $PROJECT_DIR$/config/wescale/default" />
+    <kind value="PACKAGE" />
+    <package value="vitess.io/vitess/go/cmd/vtgate" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$" />
+    <method v="2" />
+  </configuration>
+</component>

.run/vttablet.run.xml

@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="vttablet" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="wescale" />
+    <working_directory value="$PROJECT_DIR$/examples/wesql-server" />
+    <parameters value="--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global --alsologtostderr --log_dir $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet --log_queries_to_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/vttablet_0000000000_querylog.txt --tablet-path zone1-0000000000 --tablet_hostname 127.0.0.1 --init_tablet_type replica --enable_replication_reporter --backup_storage_implementation file --file_backup_storage_root $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/backups --port 15100 --db_port 3306 --db_host 127.0.0.1 --db_allprivs_user root --db_allprivs_password passwd --db_dba_user root --db_dba_password passwd --db_app_user root --db_app_password passwd --db_filtered_user root --db_filtered_password passwd --grpc_port 16100 --service_map grpc-queryservice,grpc-tabletmanager,grpc-updatestream --pid_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/vttablet.pid --vtctld_addr http://127.0.0.1:15000/ --disable_active_reparents --config_path $PROJECT_DIR$/config/wescale/default" />
+    <kind value="PACKAGE" />
+    <package value="vitess.io/vitess/go/cmd/vttablet" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$" />
+    <method v="2" />
+  </configuration>
+</component>