Branch2 (#577)

* feat: demo Signed-off-by: newborn22 <[email protected]> * fix: fix s3 cmd Signed-off-by: newborn22 <[email protected]> * fix: fix bucket var name Signed-off-by: newborn22 <[email protected]> * fix: fix s3 uri Signed-off-by: newborn22 <[email protected]> * fix: update api; uncomment s3 rb cmd Signed-off-by: newborn22 <[email protected]> * fix: add inputs for wescale image tag Signed-off-by: newborn22 <[email protected]> * fix: modify var ref Signed-off-by: newborn22 <[email protected]> * fix: generate yaml file Signed-off-by: newborn22 <[email protected]> * fix: fix var name error Signed-off-by: newborn22 <[email protected]> * fix: fix file path Signed-off-by: newborn22 <[email protected]> * fix: fix sed cmd Signed-off-by: newborn22 <[email protected]> * fix: fix var name Signed-off-by: newborn22 <[email protected]> * fix: create cluster Signed-off-by: newborn22 <[email protected]> * fix: fix configmap cmd Signed-off-by: newborn22 <[email protected]> * fix: add debug log print Signed-off-by: newborn22 <[email protected]> * fix: add debug log Signed-off-by: newborn22 <[email protected]> * fix: add debug log Signed-off-by: newborn22 <[email protected]> * fix: add log Signed-off-by: newborn22 <[email protected]> * fix: change pvc size Signed-off-by: newborn22 <[email protected]> * fix: add log Signed-off-by: newborn22 <[email protected]> * fix: add lg Signed-off-by: newborn22 <[email protected]> * fix: pvc size Signed-off-by: newborn22 <[email protected]> * fix: add log Signed-off-by: newborn22 <[email protected]> * fix: add lpg Signed-off-by: newborn22 <[email protected]> * fix: reduce cpu resource Signed-off-by: newborn22 <[email protected]> * fix: fix log Signed-off-by: newborn22 <[email protected]> * fix: add log Signed-off-by: newborn22 <[email protected]> * fix: log Signed-off-by: newborn22 <[email protected]> * fix:log Signed-off-by: newborn22 <[email protected]> * fix: log Signed-off-by: newborn22 <[email protected]> * fix: add log Signed-off-by: newborn22 <[email protected]> * fix: log Signed-off-by: newborn22 <[email protected]> * fix: log Signed-off-by: newborn22 <[email protected]> * fix: minikube Signed-off-by: newborn22 <[email protected]> * fix: add ngrok ssh Signed-off-by: newborn22 <[email protected]> * fix: replace ngrok with tmate Signed-off-by: newborn22 <[email protected]> * fix: wait for ssh login Signed-off-by: newborn22 <[email protected]> * fix: fix synx error Signed-off-by: newborn22 <[email protected]> * fix: change region to us Signed-off-by: newborn22 <[email protected]> * fix: debug image Signed-off-by: newborn22 <[email protected]> * fix: add env Signed-off-by: newborn22 <[email protected]> * fix: change os Signed-off-by: newborn22 <[email protected]> * fix: apt to brew Signed-off-by: newborn22 <[email protected]> * fix: brew install kind Signed-off-by: newborn22 <[email protected]> * fix: kind create cluster Signed-off-by: newborn22 <[email protected]> * fix: brew install docker, kubectl Signed-off-by: newborn22 <[email protected]> * fix: docker run wesql Signed-off-by: newborn22 <[email protected]> * fix: format Signed-off-by: newborn22 <[email protected]> * fix: change region Signed-off-by: newborn22 <[email protected]> * fix: single Signed-off-by: newborn22 <[email protected]> * fix: format Signed-off-by: newborn22 <[email protected]> * fix: fix bug Signed-off-by: newborn22 <[email protected]> * fix:fix Signed-off-by: newborn22 <[email protected]> * fix: format Signed-off-by: newborn22 <[email protected]> * fix: fix Signed-off-by: newborn22 <[email protected]> * fix: fix Signed-off-by: newborn22 <[email protected]> * feat: add wescale_wesql_performance_test2.yml Signed-off-by: earayu <[email protected]> * feat: add wescale_wesql_performance_test2.yml Signed-off-by: earayu <[email protected]> * feat: test minikube * feat: test minikube * feat: test minikube * feat: test minikube * feat: test minikube * feat: test minikube * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: init k8s * feat: update github actions * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * fix: build img Signed-off-by: newborn22 <[email protected]> * feat: add docker cluster * fix: img Signed-off-by: newborn22 <[email protected]> * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * fix: img Signed-off-by: newborn22 <[email protected]> * fix: ssh Signed-off-by: newborn22 <[email protected]> * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * feat: add docker cluster * fix: build img Signed-off-by: newborn22 <[email protected]> * feat: update * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_docker.yml * feat: update wescale_wesql_performance_kind.yml * feat: update wescale_wesql_performance_kind.yml * chore: disable github actions * feat: update branch2 * feat: update branch2 * chore: delete ci * doc: add pseudo * feat: dev * fix: spi * feat: branch create * feat: add MysqlService * feat: add MysqlService * feat: refactor branch code * feat: refactor branch * fix: move table to sidecar db * fix: remove useless code * fix: add branch schema type * fix: fix and refactor * fix: mock util * fix: fix testcase * fix: fix * feat: get branch diff tool func * fix: refactor buildTableInfosQuerySQL * fix: fix getTableInfos and its testcase * fix: fix GetBranchSchema and test case * fix: branch create design done; sub func need complete * fix: branch create done * fix: rename table; add type filed * fix: BranchDiff * fix: comment code * fix: modify table definiton * fix: BranchPrepareMergeBack design * fix: rename merge to merge back * fix: modify table definition; design branch prepare merge back * fix: prepapre merge back almost done * fix: code almost done * fix: fix branch table * fix: CR; table struct modified * fix: SelectOrInsertBranchMeta * fix: Simplified the function parameters * fix: Simplified the function parameters * fix: option type * fix: exec with db name; create table one by one * fix: common mysql service * fix: add ut * feat: add e2e test for branch * feat: add new_endtoend_test_for_branch.yml * feat: add new_endtoend_test_for_branch.yml * chore: remove useless go module * fix: fix sql synx error * fix: testcase todo * feat: add sql synx * fix: branch primitive framework * fix: branch params set and validate * fix: create branch; validate branch meta * fix: branch diff * fix: prepare mergeback; merge back * fix: reduce redundant code * fix: clean up cmd * fix: branch output * fix: validate ip * fix: comment * fix: add token to non_reserved_keyword * fix: ban diff merge option * fix: remove ip check; remove wrong testcase * fix: fix ut * fix: fix testcase * fix: wait cluster start up * fix: add log * fix: debug tmate * fix: vtgate mysql port 15307 * fix: vtgate mysql port 15307 * fix: new branch e2e, remove tmate debug * chore: add .run * fix: fix vtgate/vttablet debug xml * fix: build primitive * fix: create and clean up pass * fix: check table exists * fix: basic branch e2e done * test: failpoint test done * fix: config cluster info in github ci by config file * fix: fix go test cmd * fix: fix source host and port in github ci * fix: allow native passwd auth * fix: add flag source_host_to_target * fix: source_host_to_target * fix: enable failpoint when building image --------- Signed-off-by: newborn22 <[email protected]> Signed-off-by: earayu <[email protected]> Co-authored-by: newborn22 <[email protected]>
wesql · Dec 6, 2024 · 0ad0ac2 · 0ad0ac2
1 parent ace782e
commit 0ad0ac2
Show file tree

Hide file tree

Showing 51 changed files with 15,089 additions and 9,975 deletions.
diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml
@@ -42,6 +42,10 @@ on:
         required: false
         type: string
         default: 'image.tar'
+      enable_failpoint:
+        required: false
+        type: boolean
+        default: false
 
 
 jobs:
@@ -57,6 +61,11 @@ jobs:
         with:
           ref: ${{ inputs.branch }}
 
+      - name: Enable FailPoint
+        if: inputs.enable_failpoint
+        run: |
+          make failpoint-enable
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 

diff --git a/.github/workflows/new_endtoend_test.yml b/.github/workflows/new_endtoend_test.yml
@@ -108,10 +108,33 @@ jobs:
           $WESCALE_CI_IMAGE \
           /vt/examples/wesql-server/init_single_node_cluster.sh
 
+      - name: Wait for MySQL ports
+        run: |
+          timeout=300  # 5 minutes timeout
+          ports=(3306 15306)
+          
+          for port in "${ports[@]}"; do
+            echo "Waiting for MySQL port $port..."
+            start_time=$(date +%s)
+            while ! nc -z localhost $port; do
+              current_time=$(date +%s)
+              elapsed=$((current_time - start_time))
+          
+              if [ $elapsed -ge $timeout ]; then
+                echo "Timeout waiting for MySQL port $port"
+                exit 1
+              fi
+          
+              echo "Port $port not ready. Retrying in 5 seconds..."
+              sleep 5
+            done
+            echo "MySQL port $port is ready!"
+          done
+
       - name: Run EndToEnd test
         run: |
           cd endtoend
-          go test ./... -v
+          go test ./wasm -v
 
       - name: Print Wescale logs
         run: |

diff --git a/.github/workflows/new_endtoend_test_for_branch.yml b/.github/workflows/new_endtoend_test_for_branch.yml
@@ -0,0 +1,170 @@
+name: "New EndToEnd Test For Branch"
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+      - 'release-*'
+    tags:
+      - '*'
+  workflow_dispatch:
+
+env:
+  IMAGE_NAME: wescale_ci_image
+  REGISTRY: ghcr.io
+  IMAGE_TAG: test-${{ github.sha }}
+  MYSQL_VERSION: 8.0.32
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-endtoend
+  cancel-in-progress: true
+
+jobs:
+  build-image:
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/build_image.yml
+    with:
+      branch: ${{ github.ref }}
+      image_name: ${{ github.repository_owner }}/wescale_ci_image
+      tags: test-${{ github.sha }}
+      platforms: linux/amd64
+      want_push: false
+      want_load: true
+      want_artifact: true
+      artifact_name: 'image.tar'
+      enable_failpoint: true
+
+  setup:
+    name: "New EndToEnd Test For Branch"
+    needs: build-image
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check if workflow needs to be skipped
+        id: skip-workflow
+        run: |
+          skip='false'
+          if [[ "${{github.event.pull_request}}" ==  "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
+            skip='true'
+          fi
+          echo Skip ${skip}
+          echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
+
+      - name: Checkout code
+        if: steps.skip-workflow.outputs.skip-workflow == 'false'
+        uses: actions/checkout@v3
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: Download Docker image
+        uses: actions/download-artifact@v3
+        with:
+          name: image.tar
+          path: /tmp
+
+      - name: Load Docker image
+        run: |
+          docker load < /tmp/image.tar
+          echo "Verifying image loaded:"
+          docker images
+
+      - name: Set up cluster
+        run: |
+          MYSQL_IMG="mysql/mysql-server:${{ env.MYSQL_VERSION }}"
+          WESCALE_CI_IMAGE="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}"
+          
+          docker network create wescale-network
+          
+          # target cluster
+          docker run -itd --network wescale-network --name mysql-server \
+          -p 3306:3306 \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_ROOT_HOST=% \
+          -e MYSQL_LOG_CONSOLE=true \
+          $MYSQL_IMG \
+          --bind-address=0.0.0.0 \
+          --port=3306 \
+          --log-bin=binlog \
+          --gtid_mode=ON \
+          --enforce_gtid_consistency=ON \
+          --log_replica_updates=ON \
+          --binlog_format=ROW
+          
+          docker run -itd --network wescale-network --name wescale \
+          -p 15306:15306 \
+          -w /vt/examples/wesql-server \
+          -e MYSQL_ROOT_USER=root \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_PORT=3306 \
+          -e MYSQL_HOST=mysql-server \
+          -e CONFIG_PATH=/vt/config/wescale/default \
+          $WESCALE_CI_IMAGE \
+          /vt/examples/wesql-server/init_single_node_cluster.sh
+          
+          # source cluster
+          docker run -itd --network wescale-network --name mysql-server3307 \
+          -p 3307:3307 \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_ROOT_HOST=% \
+          -e MYSQL_LOG_CONSOLE=true \
+          $MYSQL_IMG \
+          --bind-address=0.0.0.0 \
+          --port=3307 \
+          --log-bin=binlog \
+          --gtid_mode=ON \
+          --enforce_gtid_consistency=ON \
+          --log_replica_updates=ON \
+          --binlog_format=ROW
+          
+          docker run -itd --network wescale-network --name wescale15307 \
+          -p 15307:15307 \
+          -w /vt/examples/wesql-server \
+          -e MYSQL_ROOT_USER=root \
+          -e MYSQL_ROOT_PASSWORD=passwd \
+          -e MYSQL_PORT=3307 \
+          -e MYSQL_HOST=mysql-server3307 \
+          -e VTGATE_MYSQL_PORT=15307 \
+          -e CONFIG_PATH=/vt/config/wescale/default \
+          $WESCALE_CI_IMAGE \
+          /vt/examples/wesql-server/init_single_node_cluster.sh
+
+      - name: Wait for MySQL ports
+        run: |
+          timeout=300  # 5 minutes timeout
+          ports=(3306 3307 15306 15307)
+          
+          for port in "${ports[@]}"; do
+            echo "Waiting for MySQL port $port..."
+            start_time=$(date +%s)
+            while ! nc -z localhost $port; do
+              current_time=$(date +%s)
+              elapsed=$((current_time - start_time))
+          
+              if [ $elapsed -ge $timeout ]; then
+                echo "Timeout waiting for MySQL port $port"
+                exit 1
+              fi
+          
+              echo "Port $port not ready. Retrying in 5 seconds..."
+              sleep 5
+            done
+            echo "MySQL port $port is ready!"
+          done
+
+      - name: Run EndToEnd test
+        run: |
+          cd endtoend
+          go test ./branch -v --source_host_to_target=wescale15307
+
+      - name: Print Wescale logs
+        run: |
+          docker logs wescale
diff --git a/.gitignore b/.gitignore
@@ -84,7 +84,6 @@ examples/mysql/vtdataroot
 /bin/
 /vtdataroot/
 venv
-.run/
 
 users.json
 .scannerwork

diff --git a/.run/vtgate.run.xml b/.run/vtgate.run.xml
@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="vtgate" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="wescale" />
+    <working_directory value="$PROJECT_DIR$/examples/wesql-server" />
+    <parameters value="--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global --alsologtostderr --log_dir $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate --log_queries_to_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate/vtgate_querylog.txt --port 15001 --grpc_port 15991 --mysql_server_port 15306 --mysql_server_socket_path /tmp/mysql.sock --cell zone1 --cells_to_watch zone1 --service_map grpc-vtgateservice --pid_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vtgate/vtgate.pid --config_path $PROJECT_DIR$/config/wescale/default" />
+    <kind value="PACKAGE" />
+    <package value="vitess.io/vitess/go/cmd/vtgate" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$" />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/.run/vttablet.run.xml b/.run/vttablet.run.xml
@@ -0,0 +1,12 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="vttablet" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="wescale" />
+    <working_directory value="$PROJECT_DIR$/examples/wesql-server" />
+    <parameters value="--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global --alsologtostderr --log_dir $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet --log_queries_to_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/vttablet_0000000000_querylog.txt --tablet-path zone1-0000000000 --tablet_hostname 127.0.0.1 --init_tablet_type replica --enable_replication_reporter --backup_storage_implementation file --file_backup_storage_root $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/backups --port 15100 --db_port 3306 --db_host 127.0.0.1 --db_allprivs_user root --db_allprivs_password passwd --db_dba_user root --db_dba_password passwd --db_app_user root --db_app_password passwd --db_filtered_user root --db_filtered_password passwd --grpc_port 16100 --service_map grpc-queryservice,grpc-tabletmanager,grpc-updatestream --pid_file $PROJECT_DIR$/examples/wesql-server/vtdataroot/vttablet/vttablet.pid --vtctld_addr http://127.0.0.1:15000/ --disable_active_reparents --config_path $PROJECT_DIR$/config/wescale/default" />
+    <kind value="PACKAGE" />
+    <package value="vitess.io/vitess/go/cmd/vttablet" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$" />
+    <method v="2" />
+  </configuration>
+</component>
-Original file line number
+Diff line change
@@ Expand Up / @@ -84,7 +84,6 @@ examples/mysql/vtdataroot @@
     /bin/
     /vtdataroot/
     venv
-    .run/
     users.json
     .scannerwork
@@ Expand Down @@