Merge remote-tracking branch 'origin/main' into avoid-flaky-topo-conc…

…urrency-test

.github/workflows/cluster_endtoend_onlineddl_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_schemadiff_vrepl.yml

@@ -149,6 +149,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_basic.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_cellalias.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_migrate.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_v2.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml

@@ -166,6 +166,10 @@ jobs:
         binlog-transaction-compression=ON
         EOF
         
+        cat <<-EOF>>./config/mycnf/mysql8026.cnf
+        binlog-row-value-options=PARTIAL_JSON
+        EOF
+        
         # run the tests however you normally do, then produce a JUnit XML file
         eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/codecov.yml

@@ -111,7 +111,7 @@ jobs:
 
     - name: Upload coverage reports to codecov.io
       if: steps.changes.outputs.changed_files == 'true'
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7
       with:
         fail_ci_if_error: true
         verbose: true

.github/workflows/unit_test_evalengine_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_evalengine_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="1"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql57.yml

@@ -163,6 +163,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql57"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql80.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql80"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

.github/workflows/unit_test_mysql84.yml

@@ -153,6 +153,9 @@ jobs:
 
         export NOVTADMINBUILD=1
         export VTEVALENGINETEST="0"
+        # We sometimes need to alter the behavior based on the platform we're
+        # testing, e.g. MySQL 5.7 vs 8.0.
+        export CI_DB_PLATFORM="mysql84"
         
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 

changelog/22.0/22.0.0/summary.md

@@ -8,6 +8,7 @@
   - **[RPC Changes](#rpc-changes)**
   - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)**
   - **[VTOrc Config File Changes](#vtorc-config-file-changes)**
+  - **[Support for More Efficient JSON Replication](#efficient-json-replication)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
   - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)**
@@ -59,6 +60,12 @@ The following fields can be dynamically changed -
 
 To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release.
 
+### <a id="efficient-json-replication"/>Support for More Efficient JSON Replication</a>
+
+In [#7345](https://github.com/vitessio/vitess/pull/17345) we added support for [`--binlog-row-value-options=PARTIAL_JSON`](https://dev.mysql.com/doc/refman/en/replication-options-binary-log.html#sysvar_binlog_row_value_options). You can read more about [this feature added to MySQL 8.0 here](https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/).
+
+If you are using MySQL 8.0 or later and using JSON columns, you can now enable this MySQL feature across your Vitess cluster(s) to lower the disk space needed for binary logs and improve the CPU and memory usage in both `mysqld` (standard intrashard MySQL replication) and `vttablet` ([VReplication](https://vitess.io/docs/reference/vreplication/vreplication/)) without losing any capabilities or features.
+
 
 ## <a id="minor-changes"/>Minor Changes</a>
 

examples/backups/create_commerce_schema.sql

@@ -1,6 +1,6 @@
 create table if not exists product(
-    sku varbinary(128),
-    description varbinary(128),
+    sku varchar(128),
+    description varchar(128),
     price bigint,
     primary key(sku)
 ) ENGINE=InnoDB;

examples/backups/create_customer_schema.sql

@@ -1,13 +1,13 @@
 create table if not exists customer(
     customer_id bigint not null,
-    email varbinary(128),
+    email varchar(128),
     primary key(customer_id)
 ) ENGINE=InnoDB;
 
 create table if not exists corder(
     order_id bigint not null,
     customer_id bigint,
-    sku varbinary(128),
+    sku varchar(128),
     price bigint,
     primary key(order_id)
 ) ENGINE=InnoDB;

examples/compose/default_vschema.json

@@ -1,8 +1,8 @@
 {
   "sharded": false,
   "vindexes": {
-    "hash": {
-      "type": "hash"
+    "xxhash": {
+      "type": "xxhash"
     }
   }
-}
+}

examples/compose/lookup_keyspace_vschema.json

@@ -5,22 +5,22 @@
 			"column_vindexes": [
 				{
 					"column": "id",
-					"name": "hash"
+					"name": "xxhash"
 				}
 			]
 		},
 		"tokens_token_lookup": {
 			"column_vindexes": [
 				{
 					"column": "id",
-					"name": "hash"
+					"name": "xxhash"
 				}
 			]
 		}
 	},
 	"vindexes": {
-		"hash": {
-			"type": "hash"
+		"xxhash": {
+			"type": "xxhash"
 		}
 	}
 }

examples/compose/test_keyspace_vschema.json

@@ -5,7 +5,7 @@
 			"column_vindexes": [
 				{
 					"column": "page",
-					"name": "hash"
+					"name": "xxhash"
 				},
 				{
 					"column": "message",
@@ -17,7 +17,7 @@
 			"column_vindexes": [
 				{
 					"column": "page",
-					"name": "hash"
+					"name": "xxhash"
 				},
 				{
 					"column": "token",
@@ -27,8 +27,8 @@
 		}
 	},
 	"vindexes": {
-		"hash": {
-			"type": "hash"
+		"xxhash": {
+			"type": "xxhash"
 		},
 		"messages_message_lookup": {
 			"type": "lookup_hash",

examples/compose/vtcompose/base_vschema.json

@@ -1,10 +1,9 @@
 {
   "sharded": true,
   "vindexes": {
-    "hash": {
-      "type": "hash"
+    "xxhash": {
+      "type": "xxhash"
     }
   },
-  "tables": {
-  }
-}
+  "tables": {}
+}