Draft: Tablet schema initialization refactoring

.github/workflows/unit_test_mariadb103.yml

@@ -104,4 +104,8 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
+
+        # mariadb103
+        export MYSQL_FLAVOR="MariaDB103"
+
         eatmydata -- make unit_test

.github/workflows/unit_test_mysql57.yml

@@ -89,4 +89,5 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
+
         eatmydata -- make unit_test

.github/workflows/unit_test_mysql80.yml

@@ -107,4 +107,5 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
+
         eatmydata -- make unit_test

config/embed.go

@@ -28,3 +28,6 @@ var MycnfMySQL57 string
 
 //go:embed mycnf/mysql80.cnf
 var MycnfMySQL80 string
+
+//go:embed init_maria_db.sql
+var DefaultInitMariaDB string

config/init_db.sql

@@ -11,6 +11,12 @@
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
+# We need to ensure that super_read_only is disabled so that we can execute
+# these commands. Note that disabling it does NOT disable read_only.
+# We save the current value so that we only re-enable it at the end if it was
+# enabled before.
+SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
+SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -101,3 +107,8 @@ FLUSH PRIVILEGES;
 
 RESET SLAVE ALL;
 RESET MASTER;
+
+# add custom sql here
+
+# We need to set super_read_only back to what it was before
+SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF');

config/init_maria_db.sql

@@ -0,0 +1,104 @@
+# This file is executed immediately after mysql_install_db,
+# to initialize a fresh data directory.
+
+###############################################################################
+# WARNING: This sql is *NOT* safe for production use,
+#          as it contains default well-known users and passwords.
+#          Care should be taken to change these users and passwords
+#          for production.
+###############################################################################
+
+###############################################################################
+# Equivalent of mysql_secure_installation
+###############################################################################
+# Changes during the init db should not make it to the binlog.
+# They could potentially create errant transactions on replicas.
+SET sql_log_bin = 0;
+# Remove anonymous users.
+DELETE FROM mysql.user WHERE User = '';
+
+# Disable remote root access (only allow UNIX socket).
+DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+
+# Remove test database.
+DROP DATABASE IF EXISTS test;
+
+###############################################################################
+# Vitess defaults
+###############################################################################
+
+# Vitess-internal database.
+CREATE DATABASE IF NOT EXISTS _vt;
+# Note that definitions of local_metadata and shard_metadata should be the same
+# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
+CREATE TABLE IF NOT EXISTS _vt.local_metadata (
+  name VARCHAR(255) NOT NULL,
+  value VARCHAR(255) NOT NULL,
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
+  ) ENGINE=InnoDB;
+CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
+  name VARCHAR(255) NOT NULL,
+  value MEDIUMBLOB NOT NULL,
+  db_name VARBINARY(255) NOT NULL,
+  PRIMARY KEY (db_name, name)
+  ) ENGINE=InnoDB;
+
+# Admin user with all privileges.
+CREATE USER 'vt_dba'@'localhost';
+GRANT ALL ON *.* TO 'vt_dba'@'localhost';
+GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';
+
+# User for app traffic, with global read-write access.
+CREATE USER 'vt_app'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_app'@'localhost';
+
+# User for app debug traffic, with global read access.
+CREATE USER 'vt_appdebug'@'localhost';
+GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';
+
+# User for administrative operations that need to be executed as non-SUPER.
+# Same permissions as vt_app here.
+CREATE USER 'vt_allprivs'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_allprivs'@'localhost';
+
+# User for slave replication connections.
+CREATE USER 'vt_repl'@'%';
+GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
+
+# User for Vitess VReplication (base vstreamers and vplayer).
+CREATE USER 'vt_filtered'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+  REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+  SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_filtered'@'localhost';
+
+# User for general MySQL monitoring.
+CREATE USER 'vt_monitoring'@'localhost';
+GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
+  ON *.* TO 'vt_monitoring'@'localhost';
+GRANT SELECT, UPDATE, DELETE, DROP
+  ON performance_schema.* TO 'vt_monitoring'@'localhost';
+
+# User for Orchestrator (https://github.com/openark/orchestrator).
+CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';
+GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD
+  ON *.* TO 'orc_client_user'@'%';
+GRANT SELECT
+  ON _vt.* TO 'orc_client_user'@'%';
+
+FLUSH PRIVILEGES;
+
+RESET SLAVE ALL;
+RESET MASTER;
+
+# add custom sql here

config/mycnf/mysql57.cnf

@@ -31,4 +31,5 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 # a primary that becomes unresponsive.
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
+super-read-only
 

config/mycnf/mysql80.cnf

@@ -27,4 +27,4 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 # tells the server it's fine if they are not understood.
 loose_rpl_semi_sync_master_timeout = 1000000000000000000
 loose_rpl_semi_sync_master_wait_no_slave = 1
-
+super-read-only

go/cmd/vtbackup/vtbackup.go

@@ -269,6 +269,12 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ResetReplication(ctx); err != nil {
 			return fmt.Errorf("can't reset replication: %v", err)
 		}
+		// We need to switch off super-read-only before we create database.
+		_ = mysqld.SetSuperReadOnly(false)
+		defer func() {
+			_ = mysqld.SetSuperReadOnly(true)
+		}()
+
 		cmds := mysqlctl.CreateReparentJournal()
 		cmds = append(cmds, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", sqlescape.EscapeID(dbName)))
 		if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil {

go/flags/endtoend/vtctld.txt

@@ -103,6 +103,7 @@ Usage of vtctld:
       --hot_row_protection_concurrent_transactions int                   Number of concurrent transactions let through to the txpool/MySQL for the same hot row. Should be > 1 to have enough 'ready' transactions in MySQL and benefit from a pipelining effect. (default 5)
       --hot_row_protection_max_global_queue_size int                     Global queue limit across all row (ranges). Useful to prevent that the queue can grow unbounded. (default 1000)
       --hot_row_protection_max_queue_size int                            Maximum number of BeginExecute RPCs which will be queued for the same row (range). (default 20)
+      --init_populate_metadata                                           (init parameter) populate metadata tables even if restore_from_backup is disabled. If restore_from_backup is enabled, metadata tables are always populated regardless of this flag.
       --jaeger-agent-host string                                         host and port to send spans to. if empty, no tracing will be done
       --keep_logs duration                                               keep logs for this long (using ctime) (zero to keep forever) (default 0s)
       --keep_logs_by_mtime duration                                      keep logs for this long (using mtime) (zero to keep forever) (default 0s)
@@ -190,11 +191,13 @@ Usage of vtctld:
       --security_policy string                                           the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
       --service_map StringList                                           comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice
       --serving_state_grace_period duration                              how long to pause after broadcasting health to vtgate, before enforcing a new serving state (default 0s)
+      --set_super_read_only_after_schema_initializer                     Set super_read_only in mysql to true after we are done with all schema initialization during vttablet initialization. (default true)
       --shutdown_grace_period float                                      how long to wait (in seconds) for queries and transactions to complete during graceful shutdown.
       --sql-max-length-errors int                                        truncate queries in error logs to the given length (default unlimited)
       --sql-max-length-ui int                                            truncate queries in debug UIs to the given length (default 512) (default 512)
       --srv_topo_cache_refresh duration                                  how frequently to refresh the topology for cached entries (default 1s)
       --srv_topo_cache_ttl duration                                      how long to use cached entries for topology (default 1s)
+      --srv_topo_no_cache_for_get                                        Always query topo when getting current value of the key
       --srv_topo_timeout duration                                        topo server timeout (default 5s)
       --stats_backend string                                             The name of the registered push-based monitoring/stats backend to use
       --stats_combine_dimensions string                                  List of dimensions to be combined into a single "all" value in exported stats vars

go/flags/endtoend/vtexplain.txt

@@ -104,6 +104,7 @@ Usage of vtexplain:
       --hot_row_protection_concurrent_transactions int                   Number of concurrent transactions let through to the txpool/MySQL for the same hot row. Should be > 1 to have enough 'ready' transactions in MySQL and benefit from a pipelining effect. (default 5)
       --hot_row_protection_max_global_queue_size int                     Global queue limit across all row (ranges). Useful to prevent that the queue can grow unbounded. (default 1000)
       --hot_row_protection_max_queue_size int                            Maximum number of BeginExecute RPCs which will be queued for the same row (range). (default 20)
+      --init_populate_metadata                                           (init parameter) populate metadata tables even if restore_from_backup is disabled. If restore_from_backup is enabled, metadata tables are always populated regardless of this flag.
       --keep_logs duration                                               keep logs for this long (using ctime) (zero to keep forever) (default 0s)
       --keep_logs_by_mtime duration                                      keep logs for this long (using mtime) (zero to keep forever) (default 0s)
       --ks-shard-map string                                              JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace
@@ -212,6 +213,7 @@ Usage of vtexplain:
       --security_policy string                                           the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
       --service_map StringList                                           comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice
       --serving_state_grace_period duration                              how long to pause after broadcasting health to vtgate, before enforcing a new serving state (default 0s)
+      --set_super_read_only_after_schema_initializer                     Set super_read_only in mysql to true after we are done with all schema initialization during vttablet initialization. (default true)
       --shards int                                                       Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored. (default 2)
       --shutdown_grace_period float                                      how long to wait (in seconds) for queries and transactions to complete during graceful shutdown.
       --sql string                                                       A list of semicolon-delimited SQL commands to analyze
@@ -220,6 +222,7 @@ Usage of vtexplain:
       --sql-max-length-ui int                                            truncate queries in debug UIs to the given length (default 512) (default 512)
       --srv_topo_cache_refresh duration                                  how frequently to refresh the topology for cached entries (default 1s)
       --srv_topo_cache_ttl duration                                      how long to use cached entries for topology (default 1s)
+      --srv_topo_no_cache_for_get                                        Always query topo when getting current value of the key
       --srv_topo_timeout duration                                        topo server timeout (default 5s)
       --stats_backend string                                             The name of the registered push-based monitoring/stats backend to use
       --stats_combine_dimensions string                                  List of dimensions to be combined into a single "all" value in exported stats vars

go/flags/endtoend/vtgate.txt

@@ -144,6 +144,7 @@ Usage of vtgate:
       --sql-max-length-ui int                                            truncate queries in debug UIs to the given length (default 512) (default 512)
       --srv_topo_cache_refresh duration                                  how frequently to refresh the topology for cached entries (default 1s)
       --srv_topo_cache_ttl duration                                      how long to use cached entries for topology (default 1s)
+      --srv_topo_no_cache_for_get                                        Always query topo when getting current value of the key
       --srv_topo_timeout duration                                        topo server timeout (default 5s)
       --stats_backend string                                             The name of the registered push-based monitoring/stats backend to use
       --stats_combine_dimensions string                                  List of dimensions to be combined into a single "all" value in exported stats vars

go/flags/endtoend/vttablet.txt

@@ -365,12 +365,14 @@ Usage of vttablet:
       --security_policy string                                           the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
       --service_map StringList                                           comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice
       --serving_state_grace_period duration                              how long to pause after broadcasting health to vtgate, before enforcing a new serving state (default 0s)
+      --set_super_read_only_after_schema_initializer                     Set super_read_only in mysql to true after we are done with all schema initialization during vttablet initialization. (default true)
       --shard_sync_retry_delay duration                                  delay between retries of updates to keep the tablet and its shard record in sync (default 30s)
       --shutdown_grace_period float                                      how long to wait (in seconds) for queries and transactions to complete during graceful shutdown.
       --sql-max-length-errors int                                        truncate queries in error logs to the given length (default unlimited)
       --sql-max-length-ui int                                            truncate queries in debug UIs to the given length (default 512) (default 512)
       --srv_topo_cache_refresh duration                                  how frequently to refresh the topology for cached entries (default 1s)
       --srv_topo_cache_ttl duration                                      how long to use cached entries for topology (default 1s)
+      --srv_topo_no_cache_for_get                                        Always query topo when getting current value of the key
       --srv_topo_timeout duration                                        topo server timeout (default 5s)
       --stats_backend string                                             The name of the registered push-based monitoring/stats backend to use
       --stats_combine_dimensions string                                  List of dimensions to be combined into a single "all" value in exported stats vars
@@ -448,7 +450,7 @@ Usage of vttablet:
       --tx_throttler_config string                                       The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n")
       --tx_throttler_healthcheck_cells StringList                        A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler.
       --unhealthy_threshold duration                                     replication lag after which a replica is considered unhealthy (default 2h0m0s)
-      --use_super_read_only                                              Set super_read_only flag when performing planned failover.
+      --use_super_read_only                                              Set super_read_only flag when performing planned failover. (default true)
   -v, --v Level                                                          log level for V logs
       --version                                                          print binary version
       --vmodule moduleSpec                                               comma-separated list of pattern=N settings for file-filtered logging

go/mysql/endtoend/main_test.go

@@ -244,6 +244,9 @@ ssl-key=%v/server-key.pem
 		// json.NewEncoder(os.Stdout).Encode(connParams)
 		// time.Sleep(10 * time.Minute)
 
+		// set super-read-only to 'Off' since all our instance now
+		// get started with super-read-only
+		cluster.UnsetReadOnly("")
 		return m.Run()
 	}()
 	os.Exit(exitCode)