From 3eced6b61f9f5e51b45b50be2642d1a8f707fd28 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 13 Apr 2022 23:20:54 -0400 Subject: [PATCH 1/6] Update the embedded config data in mysqlctl Signed-off-by: Matt Lord --- Makefile | 3 +-- go/vt/mysqlctl/rice-box.go | 50 +++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index a5bc241de61..883afd8d213 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,7 @@ export CGO_CFLAGS := -O1 # regenerate rice-box.go when any of the .cnf files change embed_config: - cd go/vt/mysqlctl && go run github.com/GeertJohan/go.rice/rice embed-go - cd go/vt/mysqlctl && go build . + cd go/vt/mysqlctl && go run github.com/GeertJohan/go.rice/rice embed-go && go build . # build the vitess binaries with dynamic dependency on libc build-dyn: diff --git a/go/vt/mysqlctl/rice-box.go b/go/vt/mysqlctl/rice-box.go index 282bab5b23e..68eeef3c170 100644 --- a/go/vt/mysqlctl/rice-box.go +++ b/go/vt/mysqlctl/rice-box.go @@ -11,103 +11,103 @@ func init() { // define files file2 := &embedded.EmbeddedFile{ Filename: "gomysql.pc.tmpl", - FileModTime: time.Unix(1539121419, 0), + FileModTime: time.Unix(1625867173, 0), Content: string("Name: GoMysql\nDescription: Flags for using mysql C client in go\n"), } file3 := &embedded.EmbeddedFile{ Filename: "init_db.sql", - FileModTime: time.Unix(1621959949, 0), + FileModTime: time.Unix(1649906373, 0), Content: string("# This file is executed immediately after mysql_install_db,\n# to initialize a fresh data directory.\n\n###############################################################################\n# WARNING: This sql is *NOT* safe for production use,\n# as it contains default well-known users and passwords.\n# Care should be taken to change these users and passwords\n# for production.\n###############################################################################\n\n###############################################################################\n# Equivalent of mysql_secure_installation\n###############################################################################\n\n# Changes during the init db should not make it to the binlog.\n# They could potentially create errant transactions on replicas.\nSET sql_log_bin = 0;\n# Remove anonymous users.\nDELETE FROM mysql.user WHERE User = '';\n\n# Disable remote root access (only allow UNIX socket).\nDELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';\n\n# Remove test database.\nDROP DATABASE IF EXISTS test;\n\n###############################################################################\n# Vitess defaults\n###############################################################################\n\n# Vitess-internal database.\nCREATE DATABASE IF NOT EXISTS _vt;\n# Note that definitions of local_metadata and shard_metadata should be the same\n# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.\nCREATE TABLE IF NOT EXISTS _vt.local_metadata (\n name VARCHAR(255) NOT NULL,\n value VARCHAR(255) NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\nCREATE TABLE IF NOT EXISTS _vt.shard_metadata (\n name VARCHAR(255) NOT NULL,\n value MEDIUMBLOB NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\n\n# Admin user with all privileges.\nCREATE USER 'vt_dba'@'localhost';\nGRANT ALL ON *.* TO 'vt_dba'@'localhost';\nGRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';\n\n# User for app traffic, with global read-write access.\nCREATE USER 'vt_app'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_app'@'localhost';\n\n# User for app debug traffic, with global read access.\nCREATE USER 'vt_appdebug'@'localhost';\nGRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';\n\n# User for administrative operations that need to be executed as non-SUPER.\n# Same permissions as vt_app here.\nCREATE USER 'vt_allprivs'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_allprivs'@'localhost';\n\n# User for slave replication connections.\nCREATE USER 'vt_repl'@'%';\nGRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';\n\n# User for Vitess filtered replication (binlog player).\n# Same permissions as vt_app.\nCREATE USER 'vt_filtered'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_filtered'@'localhost';\n\n# User for general MySQL monitoring.\nCREATE USER 'vt_monitoring'@'localhost';\nGRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD\n ON *.* TO 'vt_monitoring'@'localhost';\nGRANT SELECT, UPDATE, DELETE, DROP\n ON performance_schema.* TO 'vt_monitoring'@'localhost';\n\n# User for Orchestrator (https://github.com/openark/orchestrator).\nCREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';\nGRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD\n ON *.* TO 'orc_client_user'@'%';\nGRANT SELECT\n ON _vt.* TO 'orc_client_user'@'%';\n\nFLUSH PRIVILEGES;\n\nRESET SLAVE ALL;\nRESET MASTER;\n"), } file5 := &embedded.EmbeddedFile{ Filename: "mycnf/default.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# Global configuration that is auto-included for all MySQL/MariaDB versions\n\ndatadir = {{.DataDir}}\ninnodb_data_home_dir = {{.InnodbDataHomeDir}}\ninnodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}\nlog-error = {{.ErrorLogPath}}\nlog-bin = {{.BinLogPath}}\nrelay-log = {{.RelayLogPath}}\nrelay-log-index = {{.RelayLogIndexPath}}\npid-file = {{.PidFile}}\nport = {{.MysqlPort}}\n\n{{if .SecureFilePriv}}\nsecure-file-priv = {{.SecureFilePriv}}\n{{end}}\n\n# all db instances should start in read-only mode - once the db is started and\n# fully functional, we'll push it into read-write mode\nread-only\nserver-id = {{.ServerID}}\n\n# all db instances should skip starting replication threads - that way we can do any\n# additional configuration (like enabling semi-sync) before we connect to\n# the source.\nskip_slave_start\nsocket = {{.SocketFile}}\ntmpdir = {{.TmpDir}}\n\nslow-query-log-file = {{.SlowLogPath}}\n\n# These are sensible defaults that apply to all MySQL/MariaDB versions\n\nlong_query_time = 2\nslow-query-log\nskip-name-resolve\nconnect_timeout = 30\ninnodb_lock_wait_timeout = 20\nmax_allowed_packet = 64M\nmax_connections = 500\n\n\n"), } file6 := &embedded.EmbeddedFile{ Filename: "mycnf/mariadb100.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MariaDB 10.0 is detected.\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\nslave_net_timeout = 60\n\n# MariaDB 10.0 is unstrict by default\nsql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION\n\n# enable strict mode so it's safe to compare sequence numbers across different server IDs.\ngtid_strict_mode = 1\ninnodb_stats_persistent = 0\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\nexpire_logs_days = 3\n\nsync_binlog = 1\nbinlog_format = ROW\nlog_slave_updates\nexpire_logs_days = 3\n\n# In MariaDB the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n"), } file7 := &embedded.EmbeddedFile{ Filename: "mycnf/mariadb101.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MariaDB 10.1 is detected.\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\nslave_net_timeout = 60\n\n# MariaDB 10.1 default is only no-engine-substitution and no-auto-create-user\nsql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_CREATE_USER\n\n# enable strict mode so it's safe to compare sequence numbers across different server IDs.\ngtid_strict_mode = 1\ninnodb_stats_persistent = 0\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\nexpire_logs_days = 3\n\nsync_binlog = 1\nbinlog_format = ROW\nlog_slave_updates\nexpire_logs_days = 3\n\n# In MariaDB the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n"), } file8 := &embedded.EmbeddedFile{ Filename: "mycnf/mariadb102.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MariaDB 10.2 is detected.\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# enable strict mode so it's safe to compare sequence numbers across different server IDs.\ngtid_strict_mode = 1\ninnodb_stats_persistent = 0\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\nexpire_logs_days = 3\n\nsync_binlog = 1\nbinlog_format = ROW\nlog_slave_updates\nexpire_logs_days = 3\n\n# In MariaDB the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n"), } file9 := &embedded.EmbeddedFile{ Filename: "mycnf/mariadb103.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MariaDB 10.3 is detected.\n\n# enable strict mode so it's safe to compare sequence numbers across different server IDs.\ngtid_strict_mode = 1\ninnodb_stats_persistent = 0\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\nexpire_logs_days = 3\n\nsync_binlog = 1\nbinlog_format = ROW\nlog_slave_updates\nexpire_logs_days = 3\n\n# In MariaDB the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n\n"), } filea := &embedded.EmbeddedFile{ Filename: "mycnf/mariadb104.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MariaDB 10.4 is detected.\n\n# enable strict mode so it's safe to compare sequence numbers across different server IDs.\ngtid_strict_mode = 1\ninnodb_stats_persistent = 0\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\nexpire_logs_days = 3\n\nsync_binlog = 1\nbinlog_format = ROW\nlog_slave_updates\nexpire_logs_days = 3\n\n# In MariaDB the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n\n"), } fileb := &embedded.EmbeddedFile{ Filename: "mycnf/mysql56.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This file is auto-included when MySQL 5.6 is detected.\n\n# MySQL 5.6 does not enable the binary log by default, and \n# the default for sync_binlog is unsafe. The format is TABLE, and\n# info repositories also default to file.\n\nsync_binlog = 1\ngtid_mode = ON\nbinlog_format = ROW\nlog_slave_updates\nenforce_gtid_consistency\nexpire_logs_days = 3\nmaster_info_repository = TABLE\nrelay_log_info_repository = TABLE\nrelay_log_purge = 1\nrelay_log_recovery = 1\nslave_net_timeout = 60\n\n# In MySQL 5.6 the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n# MySQL 5.6 is unstrict by default\nsql_mode = STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n"), } filec := &embedded.EmbeddedFile{ Filename: "mycnf/mysql57.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1647620676, 0), - Content: string("# This file is auto-included when MySQL 5.7 is detected.\n\n# MySQL 5.7 does not enable the binary log by default, and \n# info repositories default to file\n\ngtid_mode = ON\nlog_slave_updates\nenforce_gtid_consistency\nexpire_logs_days = 3\nmaster_info_repository = TABLE\nrelay_log_info_repository = TABLE\nrelay_log_purge = 1\nrelay_log_recovery = 1\n\n# In MySQL 5.7 the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n"), + Content: string("# This file is auto-included when MySQL 5.7 is detected.\n\n# MySQL 5.7 does not enable the binary log by default, and \n# info repositories default to file\n\ngtid_mode = ON\nlog_slave_updates\nenforce_gtid_consistency\nexpire_logs_days = 3\nmaster_info_repository = TABLE\nrelay_log_info_repository = TABLE\nrelay_log_purge = 1\nrelay_log_recovery = 1\n\n# we should never need super privileges\nsuper-read-only\n\n# In MySQL 5.7 the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n"), } filed := &embedded.EmbeddedFile{ Filename: "mycnf/mysql80.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1647620676, 0), - Content: string("# This file is auto-included when MySQL 8.0 is detected.\n\n# MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories\n# It does not enable GTIDs or enforced GTID consistency\n\ngtid_mode = ON\nenforce_gtid_consistency\nrelay_log_recovery = 1\nbinlog_expire_logs_seconds = 259200\n\n# disable mysqlx\nmysqlx = 0\n\n# 8.0 changes the default auth-plugin to caching_sha2_password\ndefault_authentication_plugin = mysql_native_password\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# MySQL 8.0 will not load plugins during --initialize\n# which makes these options unknown. Prefixing with --loose\n# tells the server it's fine if they are not understood.\nloose_rpl_semi_sync_master_timeout = 1000000000000000000\nloose_rpl_semi_sync_master_wait_no_slave = 1\n\n"), + Content: string("# This file is auto-included when MySQL 8.0 is detected.\n\n# MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories\n# It does not enable GTIDs or enforced GTID consistency\n\ngtid_mode = ON\nenforce_gtid_consistency\nrelay_log_recovery = 1\nbinlog_expire_logs_seconds = 259200\n\n# disable mysqlx\nmysqlx = 0\n\n# we should never need super privileges\nsuper-read-only\n\n# 8.0 changes the default auth-plugin to caching_sha2_password\ndefault_authentication_plugin = mysql_native_password\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# MySQL 8.0 will not load plugins during --initialize\n# which makes these options unknown. Prefixing with --loose\n# tells the server it's fine if they are not understood.\nloose_rpl_semi_sync_master_timeout = 1000000000000000000\nloose_rpl_semi_sync_master_wait_no_slave = 1\n\n"), } filee := &embedded.EmbeddedFile{ Filename: "mycnf/sbr.cnf", - FileModTime: time.Unix(1614196831, 0), + FileModTime: time.Unix(1625867173, 0), Content: string("# This file is used to allow legacy tests to pass\n# In theory it should not be required\nbinlog_format=statement\n"), } filef := &embedded.EmbeddedFile{ Filename: "mycnf/test-suite.cnf", - FileModTime: time.Unix(1628797699, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("# This sets some unsafe settings specifically for \n# the test-suite which is currently MySQL 5.7 based\n# In future it should be renamed testsuite.cnf\n\ninnodb_buffer_pool_size = 32M\ninnodb_flush_log_at_trx_commit = 0\ninnodb_log_buffer_size = 1M\ninnodb_log_file_size = 5M\n\n# Native AIO tends to run into aio-max-nr limit during test startup.\ninnodb_use_native_aio = 0\n\nkey_buffer_size = 2M\nsync_binlog=0\ninnodb_doublewrite=0\n\n# These two settings are required for the testsuite to pass, \n# but enabling them does not spark joy. They should be removed\n# in the future. See:\n# https://github.com/vitessio/vitess/issues/5396\n\nsql_mode = STRICT_TRANS_TABLES\n\n# set a short heartbeat interval in order to detect failures quickly\nslave_net_timeout = 4\n"), } fileh := &embedded.EmbeddedFile{ Filename: "orchestrator/default.json", - FileModTime: time.Unix(1621959949, 0), + FileModTime: time.Unix(1625867173, 0), Content: string("{\n \"Debug\": true,\n \"MySQLTopologyUser\": \"orc_client_user\",\n \"MySQLTopologyPassword\": \"orc_client_user_password\",\n \"MySQLReplicaUser\": \"vt_repl\",\n \"MySQLReplicaPassword\": \"\",\n \"RecoveryPeriodBlockSeconds\": 5\n}\n"), } filej := &embedded.EmbeddedFile{ Filename: "tablet/default.yaml", - FileModTime: time.Unix(1629488700, 0), + FileModTime: time.Unix(1636429180, 0), Content: string("tabletID: zone-1234\n\ninit:\n dbName: # init_db_name_override\n keyspace: # init_keyspace\n shard: # init_shard\n tabletType: # init_tablet_type\n timeoutSeconds: 60 # init_timeout\n\ndb:\n socket: # db_socket\n host: # db_host\n port: 0 # db_port\n charSet: # db_charset\n flags: 0 # db_flags\n flavor: # db_flavor\n sslCa: # db_ssl_ca\n sslCaPath: # db_ssl_ca_path\n sslCert: # db_ssl_cert\n sslKey: # db_ssl_key\n serverName: # db_server_name\n connectTimeoutMilliseconds: 0 # db_connect_timeout_ms\n app:\n user: vt_app # db_app_user\n password: # db_app_password\n useSsl: true # db_app_use_ssl\n preferTcp: false\n dba:\n user: vt_dba # db_dba_user\n password: # db_dba_password\n useSsl: true # db_dba_use_ssl\n preferTcp: false\n filtered:\n user: vt_filtered # db_filtered_user\n password: # db_filtered_password\n useSsl: true # db_filtered_use_ssl\n preferTcp: false\n repl:\n user: vt_repl # db_repl_user\n password: # db_repl_password\n useSsl: true # db_repl_use_ssl\n preferTcp: false\n appdebug:\n user: vt_appdebug # db_appdebug_user\n password: # db_appdebug_password\n useSsl: true # db_appdebug_use_ssl\n preferTcp: false\n allprivs:\n user: vt_allprivs # db_allprivs_user\n password: # db_allprivs_password\n useSsl: true # db_allprivs_use_ssl\n preferTcp: false\n\noltpReadPool:\n size: 16 # queryserver-config-pool-size\n timeoutSeconds: 0 # queryserver-config-query-pool-timeout\n idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout\n prefillParallelism: 0 # queryserver-config-pool-prefill-parallelism\n maxWaiters: 50000 # queryserver-config-query-pool-waiter-cap\n\nolapReadPool:\n size: 200 # queryserver-config-stream-pool-size\n timeoutSeconds: 0 # queryserver-config-query-pool-timeout\n idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout\n prefillParallelism: 0 # queryserver-config-stream-pool-prefill-parallelism\n maxWaiters: 0\n\ntxPool:\n size: 20 # queryserver-config-transaction-cap\n timeoutSeconds: 1 # queryserver-config-txpool-timeout\n idleTimeoutSeconds: 1800 # queryserver-config-idle-timeout\n prefillParallelism: 0 # queryserver-config-transaction-prefill-parallelism\n maxWaiters: 50000 # queryserver-config-txpool-waiter-cap\n\noltp:\n queryTimeoutSeconds: 30 # queryserver-config-query-timeout\n txTimeoutSeconds: 30 # queryserver-config-transaction-timeout\n maxRows: 10000 # queryserver-config-max-result-size\n warnRows: 0 # queryserver-config-warn-result-size\n\nhealthcheck:\n intervalSeconds: 20 # health_check_interval\n degradedThresholdSeconds: 30 # degraded_threshold\n unhealthyThresholdSeconds: 7200 # unhealthy_threshold\n\ngracePeriods:\n shutdownSeconds: 0 # shutdown_grace_period\n transitionSeconds: 0 # serving_state_grace_period\n\nreplicationTracker:\n mode: disable # enable_replication_reporter\n heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval\n\nhotRowProtection:\n mode: disable|dryRun|enable # enable_hot_row_protection, enable_hot_row_protection_dry_run\n # Recommended value: same as txPool.size.\n maxQueueSize: 20 # hot_row_protection_max_queue_size\n maxGlobalQueueSize: 1000 # hot_row_protection_max_global_queue_size\n maxConcurrency: 5 # hot_row_protection_concurrent_transactions\n\nconsolidator: enable|disable|notOnPrimary # enable-consolidator, enable-consolidator-replicas\npassthroughDML: false # queryserver-config-passthrough-dmls\nstreamBufferSize: 32768 # queryserver-config-stream-buffer-size\nqueryCacheSize: 5000 # queryserver-config-query-cache-size\nschemaReloadIntervalSeconds: 1800 # queryserver-config-schema-reload-time\nwatchReplication: false # watch_replication_stream\nterseErrors: false # queryserver-config-terse-errors\nmessagePostponeParallelism: 4 # queryserver-config-message-postpone-cap\ncacheResultFields: true # enable-query-plan-field-caching\n\n\n# The following flags are currently not supported.\n# enforce_strict_trans_tables\n# queryserver-config-strict-table-acl\n# queryserver-config-enable-table-acl-dry-run\n# queryserver-config-acl-exempt-acl\n# enable-tx-throttler\n# tx-throttler-config\n# tx-throttler-healthcheck-cells\n# enable_transaction_limit\n# enable_transaction_limit_dry_run\n# transaction_limit_per_user\n# transaction_limit_by_username\n# transaction_limit_by_principal\n# transaction_limit_by_component\n# transaction_limit_by_subcomponent\n"), } filek := &embedded.EmbeddedFile{ Filename: "zk-client-dev.json", - FileModTime: time.Unix(1539121419, 0), + FileModTime: time.Unix(1625867173, 0), Content: string("{\n \"local\": \"localhost:3863\",\n \"global\": \"localhost:3963\"\n}\n"), } filem := &embedded.EmbeddedFile{ Filename: "zkcfg/zoo.cfg", - FileModTime: time.Unix(1539121419, 0), + FileModTime: time.Unix(1625867173, 0), Content: string("tickTime=2000\ndataDir={{.DataDir}}\nclientPort={{.ClientPort}}\ninitLimit=5\nsyncLimit=2\nmaxClientCnxns=0\n{{range .Servers}}\nserver.{{.ServerId}}={{.Hostname}}:{{.LeaderPort}}:{{.ElectionPort}}\n{{end}}\n"), } @@ -115,7 +115,7 @@ func init() { // define dirs dir1 := &embedded.EmbeddedDir{ Filename: "", - DirModTime: time.Unix(1621959949, 0), + DirModTime: time.Unix(1649906373, 0), ChildFiles: []*embedded.EmbeddedFile{ file2, // "gomysql.pc.tmpl" file3, // "init_db.sql" @@ -125,7 +125,7 @@ func init() { } dir4 := &embedded.EmbeddedDir{ Filename: "mycnf", - DirModTime: time.Unix(1628797699, 0), + DirModTime: time.Unix(1647620676, 0), ChildFiles: []*embedded.EmbeddedFile{ file5, // "mycnf/default.cnf" file6, // "mycnf/mariadb100.cnf" @@ -143,7 +143,7 @@ func init() { } dirg := &embedded.EmbeddedDir{ Filename: "orchestrator", - DirModTime: time.Unix(1621959949, 0), + DirModTime: time.Unix(1625867173, 0), ChildFiles: []*embedded.EmbeddedFile{ fileh, // "orchestrator/default.json" @@ -151,7 +151,7 @@ func init() { } diri := &embedded.EmbeddedDir{ Filename: "tablet", - DirModTime: time.Unix(1629488700, 0), + DirModTime: time.Unix(1636429180, 0), ChildFiles: []*embedded.EmbeddedFile{ filej, // "tablet/default.yaml" @@ -159,7 +159,7 @@ func init() { } dirl := &embedded.EmbeddedDir{ Filename: "zkcfg", - DirModTime: time.Unix(1539121419, 0), + DirModTime: time.Unix(1625867173, 0), ChildFiles: []*embedded.EmbeddedFile{ filem, // "zkcfg/zoo.cfg" @@ -182,7 +182,7 @@ func init() { // register embeddedBox embedded.RegisterEmbeddedBox(`../../../config`, &embedded.EmbeddedBox{ Name: `../../../config`, - Time: time.Unix(1621959949, 0), + Time: time.Unix(1649906373, 0), Dirs: map[string]*embedded.EmbeddedDir{ "": dir1, "mycnf": dir4, From 4ee3a6fb41395d309201e446b05f1491353c7fc0 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 14 Apr 2022 00:51:37 -0400 Subject: [PATCH 2/6] Turn super_read_only off temporarily during init Signed-off-by: Matt Lord --- config/init_db.sql | 11 ++++++++++- go/test/endtoend/recovery/pitr/shardedpitr_test.go | 2 ++ .../endtoend/recovery/pitrtls/shardedpitr_tls_test.go | 2 ++ .../sharding/initialsharding/sharding_util.go | 4 ++++ go/vt/mysqlctl/rice-box.go | 4 ++-- 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/config/init_db.sql b/config/init_db.sql index cf7fdd63350..e19d2c4f9d3 100644 --- a/config/init_db.sql +++ b/config/init_db.sql @@ -12,6 +12,13 @@ # Equivalent of mysql_secure_installation ############################################################################### +# We need to ensure that super_read_only is disabled so that we can execute +# these commands. Note that disabling it does NOT disable read_only. +# We save the current value so that we only re-enable it at the end if it was +# enabled before. +SET @original_super_read_only=@@global.super_read_only; +SET GLOBAL super_read_only=OFF; + # Changes during the init db should not make it to the binlog. # They could potentially create errant transactions on replicas. SET sql_log_bin = 0; @@ -63,7 +70,6 @@ CREATE USER 'vt_appdebug'@'localhost'; GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost'; # User for administrative operations that need to be executed as non-SUPER. -# Same permissions as vt_app here. CREATE USER 'vt_allprivs'@'localhost'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, @@ -101,3 +107,6 @@ FLUSH PRIVILEGES; RESET SLAVE ALL; RESET MASTER; + +# We need to set super_read_only back to what it was before +SET @@global.super_read_only=@original_super_read_only; diff --git a/go/test/endtoend/recovery/pitr/shardedpitr_test.go b/go/test/endtoend/recovery/pitr/shardedpitr_test.go index d05b8bec28f..2adc24ec364 100644 --- a/go/test/endtoend/recovery/pitr/shardedpitr_test.go +++ b/go/test/endtoend/recovery/pitr/shardedpitr_test.go @@ -426,11 +426,13 @@ func initializeCluster(t *testing.T) { } queryCmds := []string{ + "SET GLOBAL super_read_only=OFF;", fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", mysqlUserName, mysqlPassword), fmt.Sprintf("GRANT ALL ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("GRANT GRANT OPTION ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("create database %s;", "vt_ks"), "FLUSH PRIVILEGES;", + "SET GLOBAL super_read_only=ON;", } for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { diff --git a/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go b/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go index c044732903c..b89fc62faf3 100644 --- a/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go +++ b/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go @@ -164,11 +164,13 @@ func initializeCluster(t *testing.T) { } queryCmds := []string{ + "SET GLOBAL super_read_only=OFF;", fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", mysqlUserName, mysqlPassword), fmt.Sprintf("GRANT ALL ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("GRANT GRANT OPTION ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("create database %s;", "vt_ks"), "FLUSH PRIVILEGES;", + "SET GLOBAL super_read_only=ON;", } for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { diff --git a/go/test/endtoend/sharding/initialsharding/sharding_util.go b/go/test/endtoend/sharding/initialsharding/sharding_util.go index 2bfe3b32a4a..8afdaafcd81 100644 --- a/go/test/endtoend/sharding/initialsharding/sharding_util.go +++ b/go/test/endtoend/sharding/initialsharding/sharding_util.go @@ -636,6 +636,7 @@ func writeInitDBFile() { sql := string(initDb) newInitDbFile = path.Join(ClusterInstance.TmpDirectory, "init_db_with_passwords.sql") sql = sql + GetPasswordUpdateSQL(ClusterInstance) + ` +SET GLOBAL super_read_only=OFF; # connecting through a port requires 127.0.0.1 # --host=localhost will connect through socket CREATE USER 'vt_dba'@'127.0.0.1' IDENTIFIED BY 'VtDbaPass'; @@ -664,6 +665,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'127.0.0.1'; FLUSH PRIVILEGES; +SET GLOBAL super_read_only=ON; ` os.WriteFile(newInitDbFile, []byte(sql), 0666) @@ -686,6 +688,7 @@ func WriteDbCredentialToTmp(tmpDir string) string { // GetPasswordUpdateSQL returns the sql for password update func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { pwdChangeCmd := ` + SET GLOBAL super_read_only=OFF; # Set real passwords for all users. UPDATE mysql.user SET %s = PASSWORD('RootPass') WHERE User = 'root' AND Host = 'localhost'; @@ -700,6 +703,7 @@ func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass') WHERE User = 'vt_filtered' AND Host = 'localhost'; FLUSH PRIVILEGES; + SET GLOBAL super_read_only=ON; ` pwdCol, _ := getPasswordField(localCluster) return fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) diff --git a/go/vt/mysqlctl/rice-box.go b/go/vt/mysqlctl/rice-box.go index 68eeef3c170..26c28c31f44 100644 --- a/go/vt/mysqlctl/rice-box.go +++ b/go/vt/mysqlctl/rice-box.go @@ -17,9 +17,9 @@ func init() { } file3 := &embedded.EmbeddedFile{ Filename: "init_db.sql", - FileModTime: time.Unix(1649906373, 0), + FileModTime: time.Unix(1649912666, 0), - Content: string("# This file is executed immediately after mysql_install_db,\n# to initialize a fresh data directory.\n\n###############################################################################\n# WARNING: This sql is *NOT* safe for production use,\n# as it contains default well-known users and passwords.\n# Care should be taken to change these users and passwords\n# for production.\n###############################################################################\n\n###############################################################################\n# Equivalent of mysql_secure_installation\n###############################################################################\n\n# Changes during the init db should not make it to the binlog.\n# They could potentially create errant transactions on replicas.\nSET sql_log_bin = 0;\n# Remove anonymous users.\nDELETE FROM mysql.user WHERE User = '';\n\n# Disable remote root access (only allow UNIX socket).\nDELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';\n\n# Remove test database.\nDROP DATABASE IF EXISTS test;\n\n###############################################################################\n# Vitess defaults\n###############################################################################\n\n# Vitess-internal database.\nCREATE DATABASE IF NOT EXISTS _vt;\n# Note that definitions of local_metadata and shard_metadata should be the same\n# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.\nCREATE TABLE IF NOT EXISTS _vt.local_metadata (\n name VARCHAR(255) NOT NULL,\n value VARCHAR(255) NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\nCREATE TABLE IF NOT EXISTS _vt.shard_metadata (\n name VARCHAR(255) NOT NULL,\n value MEDIUMBLOB NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\n\n# Admin user with all privileges.\nCREATE USER 'vt_dba'@'localhost';\nGRANT ALL ON *.* TO 'vt_dba'@'localhost';\nGRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';\n\n# User for app traffic, with global read-write access.\nCREATE USER 'vt_app'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_app'@'localhost';\n\n# User for app debug traffic, with global read access.\nCREATE USER 'vt_appdebug'@'localhost';\nGRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';\n\n# User for administrative operations that need to be executed as non-SUPER.\n# Same permissions as vt_app here.\nCREATE USER 'vt_allprivs'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_allprivs'@'localhost';\n\n# User for slave replication connections.\nCREATE USER 'vt_repl'@'%';\nGRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';\n\n# User for Vitess filtered replication (binlog player).\n# Same permissions as vt_app.\nCREATE USER 'vt_filtered'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_filtered'@'localhost';\n\n# User for general MySQL monitoring.\nCREATE USER 'vt_monitoring'@'localhost';\nGRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD\n ON *.* TO 'vt_monitoring'@'localhost';\nGRANT SELECT, UPDATE, DELETE, DROP\n ON performance_schema.* TO 'vt_monitoring'@'localhost';\n\n# User for Orchestrator (https://github.com/openark/orchestrator).\nCREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';\nGRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD\n ON *.* TO 'orc_client_user'@'%';\nGRANT SELECT\n ON _vt.* TO 'orc_client_user'@'%';\n\nFLUSH PRIVILEGES;\n\nRESET SLAVE ALL;\nRESET MASTER;\n"), + Content: string("# This file is executed immediately after mysql_install_db,\n# to initialize a fresh data directory.\n\n###############################################################################\n# WARNING: This sql is *NOT* safe for production use,\n# as it contains default well-known users and passwords.\n# Care should be taken to change these users and passwords\n# for production.\n###############################################################################\n\n###############################################################################\n# Equivalent of mysql_secure_installation\n###############################################################################\n\n# We need to ensure that super_read_only is disabled so that we can execute\n# these commands. Note that disabling it does NOT disable read_only.\n# We save the current value so that we only re-enable it at the end if it was\n# enabled before.\nSET @original_super_read_only=@@global.super_read_only;\nSET GLOBAL super_read_only=OFF;\n\n# Changes during the init db should not make it to the binlog.\n# They could potentially create errant transactions on replicas.\nSET sql_log_bin = 0;\n# Remove anonymous users.\nDELETE FROM mysql.user WHERE User = '';\n\n# Disable remote root access (only allow UNIX socket).\nDELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';\n\n# Remove test database.\nDROP DATABASE IF EXISTS test;\n\n###############################################################################\n# Vitess defaults\n###############################################################################\n\n# Vitess-internal database.\nCREATE DATABASE IF NOT EXISTS _vt;\n# Note that definitions of local_metadata and shard_metadata should be the same\n# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.\nCREATE TABLE IF NOT EXISTS _vt.local_metadata (\n name VARCHAR(255) NOT NULL,\n value VARCHAR(255) NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\nCREATE TABLE IF NOT EXISTS _vt.shard_metadata (\n name VARCHAR(255) NOT NULL,\n value MEDIUMBLOB NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\n\n# Admin user with all privileges.\nCREATE USER 'vt_dba'@'localhost';\nGRANT ALL ON *.* TO 'vt_dba'@'localhost';\nGRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';\n\n# User for app traffic, with global read-write access.\nCREATE USER 'vt_app'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_app'@'localhost';\n\n# User for app debug traffic, with global read access.\nCREATE USER 'vt_appdebug'@'localhost';\nGRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';\n\n# User for administrative operations that need to be executed as non-SUPER.\nCREATE USER 'vt_allprivs'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_allprivs'@'localhost';\n\n# User for slave replication connections.\nCREATE USER 'vt_repl'@'%';\nGRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';\n\n# User for Vitess filtered replication (binlog player).\nCREATE USER 'vt_filtered'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_filtered'@'localhost';\n\n# User for general MySQL monitoring.\nCREATE USER 'vt_monitoring'@'localhost';\nGRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD\n ON *.* TO 'vt_monitoring'@'localhost';\nGRANT SELECT, UPDATE, DELETE, DROP\n ON performance_schema.* TO 'vt_monitoring'@'localhost';\n\n# User for Orchestrator (https://github.com/openark/orchestrator).\nCREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';\nGRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD\n ON *.* TO 'orc_client_user'@'%';\nGRANT SELECT\n ON _vt.* TO 'orc_client_user'@'%';\n\nFLUSH PRIVILEGES;\n\nRESET SLAVE ALL;\nRESET MASTER;\n\n# We need to set super_read_only back to what it was before\nSET @@global.super_read_only=@original_super_read_only;\n"), } file5 := &embedded.EmbeddedFile{ Filename: "mycnf/default.cnf", From a03de1fc51bbb07fa1ecb833994343d2f134e779 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 20 May 2022 12:33:52 -0400 Subject: [PATCH 3/6] Add automatic read only handling for queries Signed-off-by: Matt Lord --- config/init_db.sql | 6 ++-- go/mysql/endtoend/client_test.go | 8 ++--- go/mysql/endtoend/query_benchmark_test.go | 8 ++--- go/mysql/endtoend/schema_change_test.go | 4 +-- go/mysql/query.go | 34 +++++++++++++++++-- go/test/endtoend/cluster/vttablet_process.go | 2 +- .../recovery/pitr/shardedpitr_test.go | 2 -- .../recovery/pitrtls/shardedpitr_tls_test.go | 2 -- go/test/endtoend/tabletmanager/tablet_test.go | 13 ++----- go/test/endtoend/topotest/consul/main_test.go | 2 +- go/test/endtoend/topotest/etcd2/main_test.go | 2 +- go/test/endtoend/topotest/zk2/main_test.go | 2 +- go/test/endtoend/utils/mysql.go | 7 +++- go/test/endtoend/utils/utils.go | 2 +- .../endtoend/vtgate/reservedconn/udv_test.go | 2 +- .../endtoend/vtgate/unsharded/main_test.go | 2 +- go/vt/mysqlctl/metadata_tables.go | 18 +++++----- go/vt/mysqlctl/rice-box.go | 20 +++++------ go/vt/mysqlctl/schema.go | 6 ++++ go/vt/vttablet/tabletmanager/rpc_query.go | 2 +- go/vt/vttablet/tabletserver/schema/engine.go | 9 +++-- go/vt/vttablet/tabletserver/twopc.go | 2 +- go/vt/vttest/local_cluster.go | 6 ++-- 23 files changed, 97 insertions(+), 64 deletions(-) diff --git a/config/init_db.sql b/config/init_db.sql index e19d2c4f9d3..8751566874e 100644 --- a/config/init_db.sql +++ b/config/init_db.sql @@ -16,8 +16,8 @@ # these commands. Note that disabling it does NOT disable read_only. # We save the current value so that we only re-enable it at the end if it was # enabled before. -SET @original_super_read_only=@@global.super_read_only; -SET GLOBAL super_read_only=OFF; +SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF'); +SET GLOBAL super_read_only='OFF'; # Changes during the init db should not make it to the binlog. # They could potentially create errant transactions on replicas. @@ -109,4 +109,4 @@ RESET SLAVE ALL; RESET MASTER; # We need to set super_read_only back to what it was before -SET @@global.super_read_only=@original_super_read_only; +SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF'); diff --git a/go/mysql/endtoend/client_test.go b/go/mysql/endtoend/client_test.go index 051fbc1e5c1..7a3f8d51317 100644 --- a/go/mysql/endtoend/client_test.go +++ b/go/mysql/endtoend/client_test.go @@ -161,7 +161,7 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { expectNoError(t, err) defer conn.Close() - qr, more, err := conn.ExecuteFetchMulti("select 1 from dual; set autocommit=1; select 1 from dual", 10, true) + qr, more, err := conn.ExecuteFetchMulti("select 1 from dual; set autocommit=1; select 1 from dual", 10, true, false) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(multi result)", more, true) assert.EqualValues(t, 1, len(qr.Rows)) @@ -176,12 +176,12 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { expectFlag(t, "ReadQueryResult(2)", more, false) assert.EqualValues(t, 1, len(qr.Rows)) - qr, more, err = conn.ExecuteFetchMulti("select 1 from dual", 10, true) + qr, more, err = conn.ExecuteFetchMulti("select 1 from dual", 10, true, false) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(single result)", more, false) assert.EqualValues(t, 1, len(qr.Rows)) - qr, more, err = conn.ExecuteFetchMulti("set autocommit=1", 10, true) + qr, more, err = conn.ExecuteFetchMulti("set autocommit=1", 10, true, false) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(no result)", more, false) assert.EqualValues(t, 0, len(qr.Rows)) @@ -208,7 +208,7 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { assert.EqualValues(t, 1, result.RowsAffected, "insert into returned RowsAffected") } - qr, more, err = conn.ExecuteFetchMulti("update a set name = concat(name, ' updated'); select * from a; select count(*) from a", 300, true) + qr, more, err = conn.ExecuteFetchMulti("update a set name = concat(name, ' updated'); select * from a; select count(*) from a", 300, true, true) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(multi result)", more, true) assert.EqualValues(t, 255, qr.RowsAffected) diff --git a/go/mysql/endtoend/query_benchmark_test.go b/go/mysql/endtoend/query_benchmark_test.go index 7552e470b53..521a691df1b 100644 --- a/go/mysql/endtoend/query_benchmark_test.go +++ b/go/mysql/endtoend/query_benchmark_test.go @@ -243,7 +243,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { for _, sleepDuration := range []time.Duration{0, 1 * time.Millisecond} { b.Run(fmt.Sprintf("Sleep %d ms", sleepDuration/time.Millisecond), func(b *testing.B) { for i := 0; i < b.N; i++ { - _, _, err := conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; insert into t(id) values (%d)", i), 1, false) + _, _, err := conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; insert into t(id) values (%d)", i), 1, false, true) if err != nil { b.Fatal(err) } @@ -252,7 +252,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; select * from t where id = %d", i), 1, false) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; select * from t where id = %d", i), 1, false, false) if err != nil { b.Fatal(err) } @@ -261,7 +261,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; update t set name = 'foo' where id = %d", i), 1, false) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; update t set name = 'foo' where id = %d", i), 1, false, true) if err != nil { b.Fatal(err) } @@ -270,7 +270,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; delete from t where id = %d", i), 1, false) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; delete from t where id = %d", i), 1, false, true) if err != nil { b.Fatal(err) } diff --git a/go/mysql/endtoend/schema_change_test.go b/go/mysql/endtoend/schema_change_test.go index 39e1275dc08..9407fd8f303 100644 --- a/go/mysql/endtoend/schema_change_test.go +++ b/go/mysql/endtoend/schema_change_test.go @@ -42,9 +42,9 @@ func TestChangeSchemaIsNoticed(t *testing.T) { require.NoError(t, err) defer conn.Close() - _, err = conn.ExecuteFetch(createDb, 1000, true) + _, err = conn.ExecuteFetchWithReadOnlyHandling(createDb, 1000, true) require.NoError(t, err) - _, err = conn.ExecuteFetch(mysql.CreateSchemaCopyTable, 1000, true) + _, err = conn.ExecuteFetchWithReadOnlyHandling(mysql.CreateSchemaCopyTable, 1000, true) require.NoError(t, err) tests := []struct { diff --git a/go/mysql/query.go b/go/mysql/query.go index 7271d4462a1..ba3fb9ee5b5 100644 --- a/go/mysql/query.go +++ b/go/mysql/query.go @@ -300,14 +300,26 @@ func (c *Conn) parseRow(data []byte, fields []*querypb.Field, reader func([]byte // 2. if the server closes the connection when a command is in flight, // readComQueryResponse will fail, and we'll return CRServerLost(2013). func (c *Conn) ExecuteFetch(query string, maxrows int, wantfields bool) (result *sqltypes.Result, err error) { - result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields) + result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields, false) + return result, err +} + +// ExecuteFetchWithReadOnlyHandling should be used if you are executing a write query +// on a tablet that may NOT be a primary and you want to execute it regardless of +// tablet type. This function will temporarily make the mysql instance read-write and +// re-enable read-only mode after the query is executed if needed. +func (c *Conn) ExecuteFetchWithReadOnlyHandling(query string, maxrows int, wantfields bool) (result *sqltypes.Result, err error) { + result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields, true) return result, err } // ExecuteFetchMulti is for fetching multiple results from a multi-statement result. // It returns an additional 'more' flag. If it is set, you must fetch the additional // results using ReadQueryResult. -func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool) (result *sqltypes.Result, more bool, err error) { +// +// Pass disableReadOnly as true if you are executing a write on a tablet/connection +// that may NOT be a primary and you want to execute it regardless of tablet type. +func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool, disableReadOnly bool) (result *sqltypes.Result, more bool, err error) { defer func() { if err != nil { if sqlerr, ok := err.(*SQLError); ok { @@ -316,6 +328,24 @@ func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool) (re } }() + // Note: MariaDB does not have super_read_only but support for it is EOL in v14.0+ + if disableReadOnly && !c.IsMariaDB() { + var res *sqltypes.Result + if err = c.WriteComQuery("SELECT @@global.super_read_only"); err != nil { + return nil, false, err + } + res, _, _, err := c.ReadQueryResult(maxrows, wantfields) + if err == nil && len(res.Rows) == 1 { + sro := res.Rows[0][0].ToString() + if sro == "1" || sro == "ON" { + defer c.WriteComQuery("SET @@global.super_read_only='ON'") + if err = c.WriteComQuery("SET @@global.super_read_only='OFF'"); err != nil { + return nil, false, err + } + } + } + } + // Send the query as a COM_QUERY packet. if err = c.WriteComQuery(query); err != nil { return nil, false, err diff --git a/go/test/endtoend/cluster/vttablet_process.go b/go/test/endtoend/cluster/vttablet_process.go index f016d829d49..ea3f5c420c6 100644 --- a/go/test/endtoend/cluster/vttablet_process.go +++ b/go/test/endtoend/cluster/vttablet_process.go @@ -425,7 +425,7 @@ func executeQuery(dbConn *mysql.Conn, query string) (*sqltypes.Result, error) { retryDelay := 1 * time.Second for i := 1; i <= retries; i++ { log.Infof("Executing query %s on %s (attempt %d of %d)", query, i, retries) - result, err = dbConn.ExecuteFetch(query, 10000, true) + result, err = dbConn.ExecuteFetchWithReadOnlyHandling(query, 10000, true) if err == nil { break } diff --git a/go/test/endtoend/recovery/pitr/shardedpitr_test.go b/go/test/endtoend/recovery/pitr/shardedpitr_test.go index 2adc24ec364..d05b8bec28f 100644 --- a/go/test/endtoend/recovery/pitr/shardedpitr_test.go +++ b/go/test/endtoend/recovery/pitr/shardedpitr_test.go @@ -426,13 +426,11 @@ func initializeCluster(t *testing.T) { } queryCmds := []string{ - "SET GLOBAL super_read_only=OFF;", fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", mysqlUserName, mysqlPassword), fmt.Sprintf("GRANT ALL ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("GRANT GRANT OPTION ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("create database %s;", "vt_ks"), "FLUSH PRIVILEGES;", - "SET GLOBAL super_read_only=ON;", } for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { diff --git a/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go b/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go index b89fc62faf3..c044732903c 100644 --- a/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go +++ b/go/test/endtoend/recovery/pitrtls/shardedpitr_tls_test.go @@ -164,13 +164,11 @@ func initializeCluster(t *testing.T) { } queryCmds := []string{ - "SET GLOBAL super_read_only=OFF;", fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", mysqlUserName, mysqlPassword), fmt.Sprintf("GRANT ALL ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("GRANT GRANT OPTION ON *.* TO '%s'@'%%';", mysqlUserName), fmt.Sprintf("create database %s;", "vt_ks"), "FLUSH PRIVILEGES;", - "SET GLOBAL super_read_only=ON;", } for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { diff --git a/go/test/endtoend/tabletmanager/tablet_test.go b/go/test/endtoend/tabletmanager/tablet_test.go index 99926b61bc2..7c787b7edac 100644 --- a/go/test/endtoend/tabletmanager/tablet_test.go +++ b/go/test/endtoend/tabletmanager/tablet_test.go @@ -20,7 +20,6 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/test/endtoend/cluster" @@ -42,17 +41,11 @@ func TestEnsureDB(t *testing.T) { err = clusterInstance.StartVttablet(tablet, "NOT_SERVING", false, cell, "dbtest", hostname, "0") require.NoError(t, err) - // Make it the primary. + // Make it the primary, which should create the DB err = clusterInstance.VtctlclientProcess.ExecuteCommand("TabletExternallyReparented", tablet.Alias) - require.EqualError(t, err, "exit status 1") - - // It is still NOT_SERVING because the db is read-only. - assert.Equal(t, "NOT_SERVING", tablet.VttabletProcess.GetTabletStatus()) - status := tablet.VttabletProcess.GetStatusDetails() - assert.Contains(t, status, "read-only") + require.NoError(t, err) - // Switch to read-write and verify that that we go serving. - _ = clusterInstance.VtctlclientProcess.ExecuteCommand("SetReadWrite", tablet.Alias) + // verify that that we are serving err = tablet.VttabletProcess.WaitForTabletStatus("SERVING") require.NoError(t, err) killTablets(t, tablet) diff --git a/go/test/endtoend/topotest/consul/main_test.go b/go/test/endtoend/topotest/consul/main_test.go index 1c278864ced..08941fe197a 100644 --- a/go/test/endtoend/topotest/consul/main_test.go +++ b/go/test/endtoend/topotest/consul/main_test.go @@ -150,7 +150,7 @@ func execute(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/topotest/etcd2/main_test.go b/go/test/endtoend/topotest/etcd2/main_test.go index db34bd2ee86..995729d41e7 100644 --- a/go/test/endtoend/topotest/etcd2/main_test.go +++ b/go/test/endtoend/topotest/etcd2/main_test.go @@ -118,7 +118,7 @@ func TestTopoDownServingQuery(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/topotest/zk2/main_test.go b/go/test/endtoend/topotest/zk2/main_test.go index 816bbc72d72..8e2b84233e7 100644 --- a/go/test/endtoend/topotest/zk2/main_test.go +++ b/go/test/endtoend/topotest/zk2/main_test.go @@ -119,7 +119,7 @@ func TestTopoDownServingQuery(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/utils/mysql.go b/go/test/endtoend/utils/mysql.go index 22b6d7ddc09..9972cad3835 100644 --- a/go/test/endtoend/utils/mysql.go +++ b/go/test/endtoend/utils/mysql.go @@ -97,7 +97,12 @@ func createInitSQLFile(mysqlDir, ksName string) (string, error) { } defer f.Close() - _, err = f.WriteString(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", ksName)) + sql := fmt.Sprintf(` + SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF'); + SET GLOBAL super_read_only='OFF'; + CREATE DATABASE IF NOT EXISTS %s; + SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF');`, ksName) + _, err = f.WriteString(sql) if err != nil { return "", err } diff --git a/go/test/endtoend/utils/utils.go b/go/test/endtoend/utils/utils.go index e1b729dd483..d244a3b3394 100644 --- a/go/test/endtoend/utils/utils.go +++ b/go/test/endtoend/utils/utils.go @@ -101,7 +101,7 @@ func AssertResultIsEmpty(t *testing.T, conn *mysql.Conn, pre string) { // The test fails if the query produces an error. func Exec(t testing.TB, conn *mysql.Conn, query string) *sqltypes.Result { t.Helper() - qr, err := conn.ExecuteFetch(query, 1000, true) + qr, err := conn.ExecuteFetchWithReadOnlyHandling(query, 1000, true) require.NoError(t, err, "for query: "+query) return qr } diff --git a/go/test/endtoend/vtgate/reservedconn/udv_test.go b/go/test/endtoend/vtgate/reservedconn/udv_test.go index 48e84330c24..fed64fa7c7a 100644 --- a/go/test/endtoend/vtgate/reservedconn/udv_test.go +++ b/go/test/endtoend/vtgate/reservedconn/udv_test.go @@ -159,7 +159,7 @@ func TestMysqlDumpInitialLog(t *testing.T) { for _, query := range queries { t.Run(query, func(t *testing.T) { - _, more, err := conn.ExecuteFetchMulti(query, 1000, true) + _, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) require.NoError(t, err) require.False(t, more) }) diff --git a/go/test/endtoend/vtgate/unsharded/main_test.go b/go/test/endtoend/vtgate/unsharded/main_test.go index 71327a930d0..360bd14f762 100644 --- a/go/test/endtoend/vtgate/unsharded/main_test.go +++ b/go/test/endtoend/vtgate/unsharded/main_test.go @@ -482,7 +482,7 @@ func TestRowCountExceeded(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/vt/mysqlctl/metadata_tables.go b/go/vt/mysqlctl/metadata_tables.go index 5ed7bb9cfe1..1fbc5df80cf 100644 --- a/go/vt/mysqlctl/metadata_tables.go +++ b/go/vt/mysqlctl/metadata_tables.go @@ -131,7 +131,7 @@ func (m *MetadataManager) UpsertLocalMetadata(mysqld MysqlDaemon, localMetadata } func createMetadataTables(conn *dbconnpool.DBConnection, dbName string) error { - if _, err := conn.ExecuteFetch("CREATE DATABASE IF NOT EXISTS _vt", 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling("CREATE DATABASE IF NOT EXISTS _vt", 0, false); err != nil { return err } @@ -147,12 +147,12 @@ func createMetadataTables(conn *dbconnpool.DBConnection, dbName string) error { } func createLocalMetadataTable(conn *dbconnpool.DBConnection, dbName string) error { - if _, err := conn.ExecuteFetch(sqlCreateLocalMetadataTable, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sqlCreateLocalMetadataTable, 0, false); err != nil { return err } for _, sql := range sqlAlterLocalMetadataTable { - if _, err := conn.ExecuteFetch(sql, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sql, 0, false); err != nil { // Ignore "Duplicate column name 'db_name'" errors which can happen on every restart. if merr, ok := err.(*mysql.SQLError); !ok || merr.Num != mysql.ERDupFieldName { log.Errorf("Error executing %v: %v", sql, err) @@ -162,7 +162,7 @@ func createLocalMetadataTable(conn *dbconnpool.DBConnection, dbName string) erro } sql := fmt.Sprintf(sqlUpdateLocalMetadataTable, dbName) - if _, err := conn.ExecuteFetch(sql, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sql, 0, false); err != nil { log.Errorf("Error executing %v: %v, continuing. Please check the data in _vt.local_metadata and take corrective action.", sql, err) } @@ -170,12 +170,12 @@ func createLocalMetadataTable(conn *dbconnpool.DBConnection, dbName string) erro } func createShardMetadataTable(conn *dbconnpool.DBConnection, dbName string) error { - if _, err := conn.ExecuteFetch(sqlCreateShardMetadataTable, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sqlCreateShardMetadataTable, 0, false); err != nil { return err } for _, sql := range sqlAlterShardMetadataTable { - if _, err := conn.ExecuteFetch(sql, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sql, 0, false); err != nil { // Ignore "Duplicate column name 'db_name'" errors which can happen on every restart. if merr, ok := err.(*mysql.SQLError); !ok || merr.Num != mysql.ERDupFieldName { log.Errorf("Error executing %v: %v", sql, err) @@ -185,7 +185,7 @@ func createShardMetadataTable(conn *dbconnpool.DBConnection, dbName string) erro } sql := fmt.Sprintf(sqlUpdateShardMetadataTable, dbName) - if _, err := conn.ExecuteFetch(sql, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(sql, 0, false); err != nil { log.Errorf("Error executing %v: %v, continuing. Please check the data in _vt.shard_metadata and take corrective action.", sql, err) } @@ -221,12 +221,12 @@ func upsertLocalMetadata(conn *dbconnpool.DBConnection, localMetadata map[string queryBuf.WriteString(") ON DUPLICATE KEY UPDATE value = ") valValue.EncodeSQL(&queryBuf) - if _, err := conn.ExecuteFetch(queryBuf.String(), 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(queryBuf.String(), 0, false); err != nil { return err } } - if _, err := conn.ExecuteFetch("COMMIT", 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling("COMMIT", 0, false); err != nil { return err } diff --git a/go/vt/mysqlctl/rice-box.go b/go/vt/mysqlctl/rice-box.go index 26c28c31f44..4b846aff78f 100644 --- a/go/vt/mysqlctl/rice-box.go +++ b/go/vt/mysqlctl/rice-box.go @@ -17,9 +17,9 @@ func init() { } file3 := &embedded.EmbeddedFile{ Filename: "init_db.sql", - FileModTime: time.Unix(1649912666, 0), + FileModTime: time.Unix(1653192319, 0), - Content: string("# This file is executed immediately after mysql_install_db,\n# to initialize a fresh data directory.\n\n###############################################################################\n# WARNING: This sql is *NOT* safe for production use,\n# as it contains default well-known users and passwords.\n# Care should be taken to change these users and passwords\n# for production.\n###############################################################################\n\n###############################################################################\n# Equivalent of mysql_secure_installation\n###############################################################################\n\n# We need to ensure that super_read_only is disabled so that we can execute\n# these commands. Note that disabling it does NOT disable read_only.\n# We save the current value so that we only re-enable it at the end if it was\n# enabled before.\nSET @original_super_read_only=@@global.super_read_only;\nSET GLOBAL super_read_only=OFF;\n\n# Changes during the init db should not make it to the binlog.\n# They could potentially create errant transactions on replicas.\nSET sql_log_bin = 0;\n# Remove anonymous users.\nDELETE FROM mysql.user WHERE User = '';\n\n# Disable remote root access (only allow UNIX socket).\nDELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';\n\n# Remove test database.\nDROP DATABASE IF EXISTS test;\n\n###############################################################################\n# Vitess defaults\n###############################################################################\n\n# Vitess-internal database.\nCREATE DATABASE IF NOT EXISTS _vt;\n# Note that definitions of local_metadata and shard_metadata should be the same\n# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.\nCREATE TABLE IF NOT EXISTS _vt.local_metadata (\n name VARCHAR(255) NOT NULL,\n value VARCHAR(255) NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\nCREATE TABLE IF NOT EXISTS _vt.shard_metadata (\n name VARCHAR(255) NOT NULL,\n value MEDIUMBLOB NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\n\n# Admin user with all privileges.\nCREATE USER 'vt_dba'@'localhost';\nGRANT ALL ON *.* TO 'vt_dba'@'localhost';\nGRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';\n\n# User for app traffic, with global read-write access.\nCREATE USER 'vt_app'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_app'@'localhost';\n\n# User for app debug traffic, with global read access.\nCREATE USER 'vt_appdebug'@'localhost';\nGRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';\n\n# User for administrative operations that need to be executed as non-SUPER.\nCREATE USER 'vt_allprivs'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_allprivs'@'localhost';\n\n# User for slave replication connections.\nCREATE USER 'vt_repl'@'%';\nGRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';\n\n# User for Vitess filtered replication (binlog player).\nCREATE USER 'vt_filtered'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_filtered'@'localhost';\n\n# User for general MySQL monitoring.\nCREATE USER 'vt_monitoring'@'localhost';\nGRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD\n ON *.* TO 'vt_monitoring'@'localhost';\nGRANT SELECT, UPDATE, DELETE, DROP\n ON performance_schema.* TO 'vt_monitoring'@'localhost';\n\n# User for Orchestrator (https://github.com/openark/orchestrator).\nCREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';\nGRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD\n ON *.* TO 'orc_client_user'@'%';\nGRANT SELECT\n ON _vt.* TO 'orc_client_user'@'%';\n\nFLUSH PRIVILEGES;\n\nRESET SLAVE ALL;\nRESET MASTER;\n\n# We need to set super_read_only back to what it was before\nSET @@global.super_read_only=@original_super_read_only;\n"), + Content: string("# This file is executed immediately after mysql_install_db,\n# to initialize a fresh data directory.\n\n###############################################################################\n# WARNING: This sql is *NOT* safe for production use,\n# as it contains default well-known users and passwords.\n# Care should be taken to change these users and passwords\n# for production.\n###############################################################################\n\n###############################################################################\n# Equivalent of mysql_secure_installation\n###############################################################################\n\n# We need to ensure that super_read_only is disabled so that we can execute\n# these commands. Note that disabling it does NOT disable read_only.\n# We save the current value so that we only re-enable it at the end if it was\n# enabled before.\nSET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');\nSET GLOBAL super_read_only='OFF';\n\n# Changes during the init db should not make it to the binlog.\n# They could potentially create errant transactions on replicas.\nSET sql_log_bin = 0;\n# Remove anonymous users.\nDELETE FROM mysql.user WHERE User = '';\n\n# Disable remote root access (only allow UNIX socket).\nDELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';\n\n# Remove test database.\nDROP DATABASE IF EXISTS test;\n\n###############################################################################\n# Vitess defaults\n###############################################################################\n\n# Vitess-internal database.\nCREATE DATABASE IF NOT EXISTS _vt;\n# Note that definitions of local_metadata and shard_metadata should be the same\n# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.\nCREATE TABLE IF NOT EXISTS _vt.local_metadata (\n name VARCHAR(255) NOT NULL,\n value VARCHAR(255) NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\nCREATE TABLE IF NOT EXISTS _vt.shard_metadata (\n name VARCHAR(255) NOT NULL,\n value MEDIUMBLOB NOT NULL,\n db_name VARBINARY(255) NOT NULL,\n PRIMARY KEY (db_name, name)\n ) ENGINE=InnoDB;\n\n# Admin user with all privileges.\nCREATE USER 'vt_dba'@'localhost';\nGRANT ALL ON *.* TO 'vt_dba'@'localhost';\nGRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';\n\n# User for app traffic, with global read-write access.\nCREATE USER 'vt_app'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_app'@'localhost';\n\n# User for app debug traffic, with global read access.\nCREATE USER 'vt_appdebug'@'localhost';\nGRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';\n\n# User for administrative operations that need to be executed as non-SUPER.\nCREATE USER 'vt_allprivs'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_allprivs'@'localhost';\n\n# User for slave replication connections.\nCREATE USER 'vt_repl'@'%';\nGRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';\n\n# User for Vitess VReplication (base vstreamers and vplayer).\nCREATE USER 'vt_filtered'@'localhost';\nGRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,\n REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,\n LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,\n SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER\n ON *.* TO 'vt_filtered'@'localhost';\n\n# User for general MySQL monitoring.\nCREATE USER 'vt_monitoring'@'localhost';\nGRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD\n ON *.* TO 'vt_monitoring'@'localhost';\nGRANT SELECT, UPDATE, DELETE, DROP\n ON performance_schema.* TO 'vt_monitoring'@'localhost';\n\n# User for Orchestrator (https://github.com/openark/orchestrator).\nCREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';\nGRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD\n ON *.* TO 'orc_client_user'@'%';\nGRANT SELECT\n ON _vt.* TO 'orc_client_user'@'%';\n\nFLUSH PRIVILEGES;\n\nRESET SLAVE ALL;\nRESET MASTER;\n\n# We need to set super_read_only back to what it was before\nSET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF');\n"), } file5 := &embedded.EmbeddedFile{ Filename: "mycnf/default.cnf", @@ -65,13 +65,13 @@ func init() { } filec := &embedded.EmbeddedFile{ Filename: "mycnf/mysql57.cnf", - FileModTime: time.Unix(1647620676, 0), + FileModTime: time.Unix(1650553622, 0), Content: string("# This file is auto-included when MySQL 5.7 is detected.\n\n# MySQL 5.7 does not enable the binary log by default, and \n# info repositories default to file\n\ngtid_mode = ON\nlog_slave_updates\nenforce_gtid_consistency\nexpire_logs_days = 3\nmaster_info_repository = TABLE\nrelay_log_info_repository = TABLE\nrelay_log_purge = 1\nrelay_log_recovery = 1\n\n# we should never need super privileges\nsuper-read-only\n\n# In MySQL 5.7 the default charset is latin1\n\ncharacter_set_server = utf8\ncollation_server = utf8_general_ci\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# When semi-sync is enabled, don't allow fallback to async\n# if you get no ack, or have no replicas. This is necessary to\n# prevent alternate futures when doing a failover in response to\n# a primary that becomes unresponsive.\nrpl_semi_sync_master_timeout = 1000000000000000000\nrpl_semi_sync_master_wait_no_slave = 1\n\n"), } filed := &embedded.EmbeddedFile{ Filename: "mycnf/mysql80.cnf", - FileModTime: time.Unix(1647620676, 0), + FileModTime: time.Unix(1650553622, 0), Content: string("# This file is auto-included when MySQL 8.0 is detected.\n\n# MySQL 8.0 enables binlog by default with sync_binlog and TABLE info repositories\n# It does not enable GTIDs or enforced GTID consistency\n\ngtid_mode = ON\nenforce_gtid_consistency\nrelay_log_recovery = 1\nbinlog_expire_logs_seconds = 259200\n\n# disable mysqlx\nmysqlx = 0\n\n# we should never need super privileges\nsuper-read-only\n\n# 8.0 changes the default auth-plugin to caching_sha2_password\ndefault_authentication_plugin = mysql_native_password\n\n# Semi-sync replication is required for automated unplanned failover\n# (when the primary goes away). Here we just load the plugin so it's\n# available if desired, but it's disabled at startup.\n#\n# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync\n# at the proper time when replication is set up, or when a primary is\n# promoted or demoted.\nplugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so\n\n# MySQL 8.0 will not load plugins during --initialize\n# which makes these options unknown. Prefixing with --loose\n# tells the server it's fine if they are not understood.\nloose_rpl_semi_sync_master_timeout = 1000000000000000000\nloose_rpl_semi_sync_master_wait_no_slave = 1\n\n"), } @@ -107,15 +107,15 @@ func init() { } filem := &embedded.EmbeddedFile{ Filename: "zkcfg/zoo.cfg", - FileModTime: time.Unix(1625867173, 0), + FileModTime: time.Unix(1653065346, 0), - Content: string("tickTime=2000\ndataDir={{.DataDir}}\nclientPort={{.ClientPort}}\ninitLimit=5\nsyncLimit=2\nmaxClientCnxns=0\n{{range .Servers}}\nserver.{{.ServerId}}={{.Hostname}}:{{.LeaderPort}}:{{.ElectionPort}}\n{{end}}\n"), + Content: string("tickTime=2000\ndataDir={{.DataDir}}\nclientPort={{.ClientPort}}\ninitLimit=5\nsyncLimit=2\nmaxClientCnxns=0\n# enable commands like ruok by default\n4lw.commands.whitelist=*\n{{range .Servers}}\nserver.{{.ServerId}}={{.Hostname}}:{{.LeaderPort}}:{{.ElectionPort}}\n{{end}}\n"), } // define dirs dir1 := &embedded.EmbeddedDir{ Filename: "", - DirModTime: time.Unix(1649906373, 0), + DirModTime: time.Unix(1653192220, 0), ChildFiles: []*embedded.EmbeddedFile{ file2, // "gomysql.pc.tmpl" file3, // "init_db.sql" @@ -125,7 +125,7 @@ func init() { } dir4 := &embedded.EmbeddedDir{ Filename: "mycnf", - DirModTime: time.Unix(1647620676, 0), + DirModTime: time.Unix(1650553622, 0), ChildFiles: []*embedded.EmbeddedFile{ file5, // "mycnf/default.cnf" file6, // "mycnf/mariadb100.cnf" @@ -159,7 +159,7 @@ func init() { } dirl := &embedded.EmbeddedDir{ Filename: "zkcfg", - DirModTime: time.Unix(1625867173, 0), + DirModTime: time.Unix(1653065346, 0), ChildFiles: []*embedded.EmbeddedFile{ filem, // "zkcfg/zoo.cfg" @@ -182,7 +182,7 @@ func init() { // register embeddedBox embedded.RegisterEmbeddedBox(`../../../config`, &embedded.EmbeddedBox{ Name: `../../../config`, - Time: time.Unix(1649906373, 0), + Time: time.Unix(1653192220, 0), Dirs: map[string]*embedded.EmbeddedDir{ "": dir1, "mycnf": dir4, diff --git a/go/vt/mysqlctl/schema.go b/go/vt/mysqlctl/schema.go index e5135bbf00f..ac8eedc127f 100644 --- a/go/vt/mysqlctl/schema.go +++ b/go/vt/mysqlctl/schema.go @@ -48,6 +48,12 @@ func (mysqld *Mysqld) executeSchemaCommands(sql string) error { return err } + if !mysqld.capabilities.isMariaDB() { + srosql := "SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');\n" + srosql += "SET GLOBAL super_read_only='OFF';\n" + sql = srosql + sql + "\n;SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF');\n" + } + return mysqld.executeMysqlScript(params, strings.NewReader(sql)) } diff --git a/go/vt/vttablet/tabletmanager/rpc_query.go b/go/vt/vttablet/tabletmanager/rpc_query.go index 4265f57f2ab..bc7b0efaf6e 100644 --- a/go/vt/vttablet/tabletmanager/rpc_query.go +++ b/go/vt/vttablet/tabletmanager/rpc_query.go @@ -63,7 +63,7 @@ func (tm *TabletManager) ExecuteFetchAsDba(ctx context.Context, query []byte, db } } // run the query - result, err := conn.ExecuteFetch(string(query), maxrows, true /*wantFields*/) + result, err := conn.ExecuteFetchWithReadOnlyHandling(string(query), maxrows, true /*wantFields*/) // re-enable binlogs if necessary if disableBinlogs && !conn.IsClosed() { diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index cbfc8f3eb28..f72099c9480 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -129,7 +129,9 @@ func (se *Engine) InitDBConfig(cp dbconfigs.Connector) { // This function can be called before opening the Engine. func (se *Engine) EnsureConnectionAndDB(tabletType topodatapb.TabletType) error { ctx := tabletenv.LocalContext() - conn, err := dbconnpool.NewDBConnection(ctx, se.env.Config().DB.AppWithDB()) + // We need the SUPER privilege in order to ensure we can create the sidecar + // DB even when super_read_only is enabled + conn, err := dbconnpool.NewDBConnection(ctx, se.env.Config().DB.DbaWithDB()) if err == nil { conn.Close() se.dbCreationFailed = false @@ -144,14 +146,15 @@ func (se *Engine) EnsureConnectionAndDB(tabletType topodatapb.TabletType) error // We are primary and db is not found. Let's create it. // We use allprivs instead of DBA because we want db create to fail if we're read-only. - conn, err = dbconnpool.NewDBConnection(ctx, se.env.Config().DB.AllPrivsConnector()) + // We need the SUPER privilege in order to manage super_read_only when enabled. + conn, err = dbconnpool.NewDBConnection(ctx, se.env.Config().DB.DbaConnector()) if err != nil { return err } defer conn.Close() dbname := se.env.Config().DB.DBName - _, err = conn.ExecuteFetch(fmt.Sprintf("create database if not exists `%s`", dbname), 1, false) + _, err = conn.ExecuteFetchWithReadOnlyHandling(fmt.Sprintf("create database if not exists `%s`", dbname), 1, false) if err != nil { if !se.dbCreationFailed { // This is the first failure. diff --git a/go/vt/vttablet/tabletserver/twopc.go b/go/vt/vttablet/tabletserver/twopc.go index 95bbecb7ea4..32481352434 100644 --- a/go/vt/vttablet/tabletserver/twopc.go +++ b/go/vt/vttablet/tabletserver/twopc.go @@ -192,7 +192,7 @@ func (tpc *TwoPC) Open(dbconfigs *dbconfigs.DBConfigs) error { fmt.Sprintf(sqlCreateTableDTParticipant, dbname), } for _, s := range statements { - if _, err := conn.ExecuteFetch(s, 0, false); err != nil { + if _, err := conn.ExecuteFetchWithReadOnlyHandling(s, 0, false); err != nil { return err } } diff --git a/go/vt/vttest/local_cluster.go b/go/vt/vttest/local_cluster.go index 3bf63d76a92..9a995f2239c 100644 --- a/go/vt/vttest/local_cluster.go +++ b/go/vt/vttest/local_cluster.go @@ -516,13 +516,13 @@ func (db *LocalCluster) Execute(sql []string, dbname string) error { for _, cmd := range sql { log.Infof("Execute(%s): \"%s\"", dbname, cmd) - _, err := conn.ExecuteFetch(cmd, 0, false) + _, err := conn.ExecuteFetchWithReadOnlyHandling(cmd, 0, false) if err != nil { return err } } - _, err = conn.ExecuteFetch("COMMIT", 0, false) + _, err = conn.ExecuteFetchWithReadOnlyHandling("COMMIT", 0, false) return err } @@ -537,7 +537,7 @@ func (db *LocalCluster) Query(sql, dbname string, limit int) (*sqltypes.Result, } defer conn.Close() - return conn.ExecuteFetch(sql, limit, false) + return conn.ExecuteFetchWithReadOnlyHandling(sql, limit, false) } // JSONConfig returns a key/value object with the configuration From ef3ad650e459c9ffd65d4b266749e4441af3db39 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 22 May 2022 01:16:25 -0400 Subject: [PATCH 4/6] Move the SRO handling out of ExecuteFetchMulti Signed-off-by: Matt Lord --- go/mysql/query.go | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/go/mysql/query.go b/go/mysql/query.go index ba3fb9ee5b5..14ea65a5307 100644 --- a/go/mysql/query.go +++ b/go/mysql/query.go @@ -300,7 +300,7 @@ func (c *Conn) parseRow(data []byte, fields []*querypb.Field, reader func([]byte // 2. if the server closes the connection when a command is in flight, // readComQueryResponse will fail, and we'll return CRServerLost(2013). func (c *Conn) ExecuteFetch(query string, maxrows int, wantfields bool) (result *sqltypes.Result, err error) { - result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields, false) + result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields) return result, err } @@ -309,17 +309,31 @@ func (c *Conn) ExecuteFetch(query string, maxrows int, wantfields bool) (result // tablet type. This function will temporarily make the mysql instance read-write and // re-enable read-only mode after the query is executed if needed. func (c *Conn) ExecuteFetchWithReadOnlyHandling(query string, maxrows int, wantfields bool) (result *sqltypes.Result, err error) { - result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields, true) + // Note: MariaDB does not have super_read_only but support for it is EOL in v14.0+ + if !c.IsMariaDB() { + if err := c.WriteComQuery("SELECT @@global.super_read_only"); err != nil { + return nil, err + } + res, _, _, err := c.ReadQueryResult(1, false) + if err == nil && len(res.Rows) == 1 { + sro := res.Rows[0][0].ToString() + if sro == "1" || sro == "ON" { + defer c.WriteComQuery("SET GLOBAL super_read_only='ON'") + if err = c.WriteComQuery("SET GLOBAL super_read_only='OFF'"); err != nil { + return nil, err + } + } + } + } + + result, _, err = c.ExecuteFetchMulti(query, maxrows, wantfields) return result, err } // ExecuteFetchMulti is for fetching multiple results from a multi-statement result. // It returns an additional 'more' flag. If it is set, you must fetch the additional // results using ReadQueryResult. -// -// Pass disableReadOnly as true if you are executing a write on a tablet/connection -// that may NOT be a primary and you want to execute it regardless of tablet type. -func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool, disableReadOnly bool) (result *sqltypes.Result, more bool, err error) { +func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool) (result *sqltypes.Result, more bool, err error) { defer func() { if err != nil { if sqlerr, ok := err.(*SQLError); ok { @@ -328,24 +342,6 @@ func (c *Conn) ExecuteFetchMulti(query string, maxrows int, wantfields bool, dis } }() - // Note: MariaDB does not have super_read_only but support for it is EOL in v14.0+ - if disableReadOnly && !c.IsMariaDB() { - var res *sqltypes.Result - if err = c.WriteComQuery("SELECT @@global.super_read_only"); err != nil { - return nil, false, err - } - res, _, _, err := c.ReadQueryResult(maxrows, wantfields) - if err == nil && len(res.Rows) == 1 { - sro := res.Rows[0][0].ToString() - if sro == "1" || sro == "ON" { - defer c.WriteComQuery("SET @@global.super_read_only='ON'") - if err = c.WriteComQuery("SET @@global.super_read_only='OFF'"); err != nil { - return nil, false, err - } - } - } - } - // Send the query as a COM_QUERY packet. if err = c.WriteComQuery(query); err != nil { return nil, false, err From 5caa037c4e1f978a1afc5b153b1d2bcff722d437 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 22 May 2022 01:24:28 -0400 Subject: [PATCH 5/6] Properly manage SRO in legacy sharding utils Signed-off-by: Matt Lord --- go/test/endtoend/sharding/initialsharding/sharding_util.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/sharding/initialsharding/sharding_util.go b/go/test/endtoend/sharding/initialsharding/sharding_util.go index 8afdaafcd81..8cdaa4a5c81 100644 --- a/go/test/endtoend/sharding/initialsharding/sharding_util.go +++ b/go/test/endtoend/sharding/initialsharding/sharding_util.go @@ -636,6 +636,7 @@ func writeInitDBFile() { sql := string(initDb) newInitDbFile = path.Join(ClusterInstance.TmpDirectory, "init_db_with_passwords.sql") sql = sql + GetPasswordUpdateSQL(ClusterInstance) + ` +SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF'); SET GLOBAL super_read_only=OFF; # connecting through a port requires 127.0.0.1 # --host=localhost will connect through socket @@ -665,7 +666,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER ON *.* TO 'vt_filtered'@'127.0.0.1'; FLUSH PRIVILEGES; -SET GLOBAL super_read_only=ON; +SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF'); ` os.WriteFile(newInitDbFile, []byte(sql), 0666) @@ -688,6 +689,7 @@ func WriteDbCredentialToTmp(tmpDir string) string { // GetPasswordUpdateSQL returns the sql for password update func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { pwdChangeCmd := ` + SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF'); SET GLOBAL super_read_only=OFF; # Set real passwords for all users. UPDATE mysql.user SET %s = PASSWORD('RootPass') @@ -703,7 +705,7 @@ func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass') WHERE User = 'vt_filtered' AND Host = 'localhost'; FLUSH PRIVILEGES; - SET GLOBAL super_read_only=ON; + SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF'); ` pwdCol, _ := getPasswordField(localCluster) return fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) From ec030ae5f3119e779cf7f06998451272a123548b Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 22 May 2022 01:33:37 -0400 Subject: [PATCH 6/6] Add backup handling Restore test files Signed-off-by: Matt Lord --- go/cmd/vtbackup/vtbackup.go | 4 ++-- go/mysql/endtoend/client_test.go | 8 ++++---- go/mysql/endtoend/query_benchmark_test.go | 8 ++++---- go/mysql/endtoend/schema_change_test.go | 4 ++-- go/test/endtoend/topotest/consul/main_test.go | 2 +- go/test/endtoend/topotest/etcd2/main_test.go | 2 +- go/test/endtoend/topotest/zk2/main_test.go | 2 +- go/test/endtoend/utils/utils.go | 2 +- .../endtoend/vtgate/reservedconn/udv_test.go | 2 +- .../endtoend/vtgate/unsharded/main_test.go | 2 +- go/vt/mysqlctl/query.go | 19 +++++++++++++++++++ 11 files changed, 37 insertions(+), 18 deletions(-) diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go index b7dedd2eb20..08501aca9ff 100644 --- a/go/cmd/vtbackup/vtbackup.go +++ b/go/cmd/vtbackup/vtbackup.go @@ -271,13 +271,13 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back } cmds := mysqlctl.CreateReparentJournal() cmds = append(cmds, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", sqlescape.EscapeID(dbName))) - if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil { + if err := mysqld.ExecuteSuperQueryListWithReadOnlyHandling(ctx, cmds); err != nil { return fmt.Errorf("can't initialize database: %v", err) } // Execute Alter commands on reparent_journal and ignore errors cmds = mysqlctl.AlterReparentJournal() - _ = mysqld.ExecuteSuperQueryList(ctx, cmds) + _ = mysqld.ExecuteSuperQueryListWithReadOnlyHandling(ctx, cmds) backupParams.BackupTime = time.Now() // Now we're ready to take the backup. diff --git a/go/mysql/endtoend/client_test.go b/go/mysql/endtoend/client_test.go index 7a3f8d51317..051fbc1e5c1 100644 --- a/go/mysql/endtoend/client_test.go +++ b/go/mysql/endtoend/client_test.go @@ -161,7 +161,7 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { expectNoError(t, err) defer conn.Close() - qr, more, err := conn.ExecuteFetchMulti("select 1 from dual; set autocommit=1; select 1 from dual", 10, true, false) + qr, more, err := conn.ExecuteFetchMulti("select 1 from dual; set autocommit=1; select 1 from dual", 10, true) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(multi result)", more, true) assert.EqualValues(t, 1, len(qr.Rows)) @@ -176,12 +176,12 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { expectFlag(t, "ReadQueryResult(2)", more, false) assert.EqualValues(t, 1, len(qr.Rows)) - qr, more, err = conn.ExecuteFetchMulti("select 1 from dual", 10, true, false) + qr, more, err = conn.ExecuteFetchMulti("select 1 from dual", 10, true) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(single result)", more, false) assert.EqualValues(t, 1, len(qr.Rows)) - qr, more, err = conn.ExecuteFetchMulti("set autocommit=1", 10, true, false) + qr, more, err = conn.ExecuteFetchMulti("set autocommit=1", 10, true) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(no result)", more, false) assert.EqualValues(t, 0, len(qr.Rows)) @@ -208,7 +208,7 @@ func doTestMultiResult(t *testing.T, disableClientDeprecateEOF bool) { assert.EqualValues(t, 1, result.RowsAffected, "insert into returned RowsAffected") } - qr, more, err = conn.ExecuteFetchMulti("update a set name = concat(name, ' updated'); select * from a; select count(*) from a", 300, true, true) + qr, more, err = conn.ExecuteFetchMulti("update a set name = concat(name, ' updated'); select * from a; select count(*) from a", 300, true) expectNoError(t, err) expectFlag(t, "ExecuteMultiFetch(multi result)", more, true) assert.EqualValues(t, 255, qr.RowsAffected) diff --git a/go/mysql/endtoend/query_benchmark_test.go b/go/mysql/endtoend/query_benchmark_test.go index 521a691df1b..7552e470b53 100644 --- a/go/mysql/endtoend/query_benchmark_test.go +++ b/go/mysql/endtoend/query_benchmark_test.go @@ -243,7 +243,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { for _, sleepDuration := range []time.Duration{0, 1 * time.Millisecond} { b.Run(fmt.Sprintf("Sleep %d ms", sleepDuration/time.Millisecond), func(b *testing.B) { for i := 0; i < b.N; i++ { - _, _, err := conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; insert into t(id) values (%d)", i), 1, false, true) + _, _, err := conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; insert into t(id) values (%d)", i), 1, false) if err != nil { b.Fatal(err) } @@ -252,7 +252,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; select * from t where id = %d", i), 1, false, false) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; select * from t where id = %d", i), 1, false) if err != nil { b.Fatal(err) } @@ -261,7 +261,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; update t set name = 'foo' where id = %d", i), 1, false, true) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; update t set name = 'foo' where id = %d", i), 1, false) if err != nil { b.Fatal(err) } @@ -270,7 +270,7 @@ func BenchmarkSetVarsMultipleSetsInSameStmt(b *testing.B) { b.Fatal(err) } - _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; delete from t where id = %d", i), 1, false, true) + _, _, err = conn.ExecuteFetchMulti(fmt.Sprintf("set sql_mode = '', sql_safe_updates = 0 ; delete from t where id = %d", i), 1, false) if err != nil { b.Fatal(err) } diff --git a/go/mysql/endtoend/schema_change_test.go b/go/mysql/endtoend/schema_change_test.go index 9407fd8f303..39e1275dc08 100644 --- a/go/mysql/endtoend/schema_change_test.go +++ b/go/mysql/endtoend/schema_change_test.go @@ -42,9 +42,9 @@ func TestChangeSchemaIsNoticed(t *testing.T) { require.NoError(t, err) defer conn.Close() - _, err = conn.ExecuteFetchWithReadOnlyHandling(createDb, 1000, true) + _, err = conn.ExecuteFetch(createDb, 1000, true) require.NoError(t, err) - _, err = conn.ExecuteFetchWithReadOnlyHandling(mysql.CreateSchemaCopyTable, 1000, true) + _, err = conn.ExecuteFetch(mysql.CreateSchemaCopyTable, 1000, true) require.NoError(t, err) tests := []struct { diff --git a/go/test/endtoend/topotest/consul/main_test.go b/go/test/endtoend/topotest/consul/main_test.go index 08941fe197a..1c278864ced 100644 --- a/go/test/endtoend/topotest/consul/main_test.go +++ b/go/test/endtoend/topotest/consul/main_test.go @@ -150,7 +150,7 @@ func execute(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/topotest/etcd2/main_test.go b/go/test/endtoend/topotest/etcd2/main_test.go index 995729d41e7..db34bd2ee86 100644 --- a/go/test/endtoend/topotest/etcd2/main_test.go +++ b/go/test/endtoend/topotest/etcd2/main_test.go @@ -118,7 +118,7 @@ func TestTopoDownServingQuery(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/topotest/zk2/main_test.go b/go/test/endtoend/topotest/zk2/main_test.go index 8e2b84233e7..816bbc72d72 100644 --- a/go/test/endtoend/topotest/zk2/main_test.go +++ b/go/test/endtoend/topotest/zk2/main_test.go @@ -119,7 +119,7 @@ func TestTopoDownServingQuery(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/test/endtoend/utils/utils.go b/go/test/endtoend/utils/utils.go index d244a3b3394..e1b729dd483 100644 --- a/go/test/endtoend/utils/utils.go +++ b/go/test/endtoend/utils/utils.go @@ -101,7 +101,7 @@ func AssertResultIsEmpty(t *testing.T, conn *mysql.Conn, pre string) { // The test fails if the query produces an error. func Exec(t testing.TB, conn *mysql.Conn, query string) *sqltypes.Result { t.Helper() - qr, err := conn.ExecuteFetchWithReadOnlyHandling(query, 1000, true) + qr, err := conn.ExecuteFetch(query, 1000, true) require.NoError(t, err, "for query: "+query) return qr } diff --git a/go/test/endtoend/vtgate/reservedconn/udv_test.go b/go/test/endtoend/vtgate/reservedconn/udv_test.go index fed64fa7c7a..48e84330c24 100644 --- a/go/test/endtoend/vtgate/reservedconn/udv_test.go +++ b/go/test/endtoend/vtgate/reservedconn/udv_test.go @@ -159,7 +159,7 @@ func TestMysqlDumpInitialLog(t *testing.T) { for _, query := range queries { t.Run(query, func(t *testing.T) { - _, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) + _, more, err := conn.ExecuteFetchMulti(query, 1000, true) require.NoError(t, err) require.False(t, more) }) diff --git a/go/test/endtoend/vtgate/unsharded/main_test.go b/go/test/endtoend/vtgate/unsharded/main_test.go index 360bd14f762..71327a930d0 100644 --- a/go/test/endtoend/vtgate/unsharded/main_test.go +++ b/go/test/endtoend/vtgate/unsharded/main_test.go @@ -482,7 +482,7 @@ func TestRowCountExceeded(t *testing.T) { func execMulti(t *testing.T, conn *mysql.Conn, query string) []*sqltypes.Result { t.Helper() var res []*sqltypes.Result - qr, more, err := conn.ExecuteFetchMulti(query, 1000, true, true) + qr, more, err := conn.ExecuteFetchMulti(query, 1000, true) res = append(res, qr) require.NoError(t, err) for more == true { diff --git a/go/vt/mysqlctl/query.go b/go/vt/mysqlctl/query.go index fa6c842a37f..63281a32249 100644 --- a/go/vt/mysqlctl/query.go +++ b/go/vt/mysqlctl/query.go @@ -57,6 +57,25 @@ func (mysqld *Mysqld) ExecuteSuperQuery(ctx context.Context, query string) error return mysqld.ExecuteSuperQueryList(ctx, []string{query}) } +// ExecuteSuperQueryListWithReadOnlyHandling allows the user to execute queries as a super user +// even if the instance is in read-only mode. It will temporarily switch the instance to read-write +// before executing the queries and put the instance back in its original state afterward. +func (mysqld *Mysqld) ExecuteSuperQueryListWithReadOnlyHandling(ctx context.Context, queryList []string) error { + conn, err := getPoolReconnect(ctx, mysqld.dbaPool) + if err != nil { + return err + } + defer conn.Recycle() + + sroql := make([]string, 0, len(queryList)+3) + sroql[0] = "SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF')" + sroql[1] = "SET GLOBAL super_read_only='OFF'" + sroql = append(sroql, queryList...) + sroql[len(sroql)-1] = "SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF')" + + return mysqld.executeSuperQueryListConn(ctx, conn, sroql) +} + // ExecuteSuperQueryList alows the user to execute queries as a super user. func (mysqld *Mysqld) ExecuteSuperQueryList(ctx context.Context, queryList []string) error { conn, err := getPoolReconnect(ctx, mysqld.dbaPool)