diff --git a/changelog/17.0/17.0.0/summary.md b/changelog/17.0/17.0.0/summary.md
index 3abad32b54b..0d810bfdc08 100644
--- a/changelog/17.0/17.0.0/summary.md
+++ b/changelog/17.0/17.0.0/summary.md
@@ -22,6 +22,7 @@
- [Support for MySQL 8.0 `binlog_transaction_compression`](#binlog-compression)
- **[VTTablet](#vttablet)**
- [VTTablet: Initializing all replicas with super_read_only](#vttablet-initialization)
+ - [Vttablet Schema Reload Timeout](#vttablet-schema-reload-timeout)
- [Settings pool enabled](#settings-pool)
- **[VReplication](#VReplication)**
- [Support for the `noblob` binlog row image mode](#noblob)
@@ -316,6 +317,10 @@ This is even more important if you are running Vitess on the vitess-operator.
You must ensure your `init_db.sql` is up-to-date with the new default for `v17.0.0`.
The default file can be found in `./config/init_db.sql`.
+#### Vttablet Schema Reload Timeout
+
+A new flag, `--schema-change-reload-timeout` has been added to timeout the reload of the schema that Vttablet does periodically. This is required because sometimes this operation can get stuck after MySQL restarts, etc. More details available in the issue https://github.com/vitessio/vitess/issues/13001.
+
#### Settings Pool
This was introduced in v15 and it enables pooling the connection with modified connection settings.
To know more what it does read the [v15 release notes](https://github.com/vitessio/vitess/releases/tag/v15.0.0) or the [blog](https://vitess.io/blog/2023-03-27-connection-pooling-in-vitess/) or [docs](https://vitess.io/docs/17.0/reference/query-serving/reserved-conn/)
@@ -371,6 +376,8 @@ without any (known) limitations.
`schema_change_check_interval` now **only** accepts Go duration values. This affects `vtctld`.
* The flag `durability_policy` is no longer used by vtctld. Instead it reads the durability policies for all keyspaces from the topology server.
* The flag `use_super_read_only` is deprecated and will be removed in a later release. This affects `vttablet`.
+* The flag `queryserver-config-schema-change-signal-interval` is deprecated and will be removed in a later release. This affects `vttablet`.
+ Schema-tracking has been refactored in this release to not use polling anymore, therefore the signal interval isn't required anymore.
In `vttablet` various flags that took float values as seconds have updated to take the standard duration syntax as well.
Float-style parsing is now deprecated and will be removed in a later release.
diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt
index 158a6ca9922..6a4e858a711 100644
--- a/go/flags/endtoend/vttablet.txt
+++ b/go/flags/endtoend/vttablet.txt
@@ -233,7 +233,6 @@ Usage of vttablet:
--queryserver-config-query-pool-waiter-cap int query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection (default 5000)
--queryserver-config-query-timeout duration query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed. (default 30s)
--queryserver-config-schema-change-signal query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work (default true)
- --queryserver-config-schema-change-signal-interval duration query server schema change signal interval defines at which interval the query server shall send schema updates to vtgate. (default 5s)
--queryserver-config-schema-reload-time duration query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time. (default 30m0s)
--queryserver-config-stream-buffer-size int query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size. (default 32768)
--queryserver-config-stream-pool-size int query server stream connection pool size, stream pool is used by stream queries: queries that return results to client in a streaming fashion (default 200)
@@ -269,7 +268,7 @@ Usage of vttablet:
--s3_backup_storage_root string root prefix for all backup-related object names.
--s3_backup_tls_skip_verify_cert skip the 'certificate is valid' check for SSL connections.
--sanitize_log_messages Remove potentially sensitive information in tablet INFO, WARNING, and ERROR log messages such as query parameters.
- --schema-change-reload-timeout duration query server schema change signal reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up (default 30s)
+ --schema-change-reload-timeout duration query server schema change reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up (default 30s)
--schema-version-max-age-seconds int max age of schema version records to kept in memory by the vreplication historian
--security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only)
--service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice
diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go
index 7bd38e0665f..f43f63c0d53 100644
--- a/go/mysql/fakesqldb/server.go
+++ b/go/mysql/fakesqldb/server.go
@@ -124,6 +124,10 @@ type DB struct {
// if fakesqldb is asked to serve queries or query patterns that it has not been explicitly told about it will
// error out by default. However if you set this flag then any unmatched query results in an empty result
neverFail atomic.Bool
+
+ // lastError stores the last error in returning a query result.
+ lastErrorMu sync.Mutex
+ lastError error
}
// QueryHandler is the interface used by the DB to simulate executed queries
@@ -176,6 +180,7 @@ func New(t testing.TB) *DB {
connections: make(map[uint32]*mysql.Conn),
queryPatternUserCallback: make(map[*regexp.Regexp]func(string)),
patternData: make(map[string]exprResult),
+ lastErrorMu: sync.Mutex{},
}
db.Handler = db
@@ -245,6 +250,13 @@ func (db *DB) CloseAllConnections() {
}
}
+// LastError gives the last error the DB ran into
+func (db *DB) LastError() error {
+ db.lastErrorMu.Lock()
+ defer db.lastErrorMu.Unlock()
+ return db.lastError
+}
+
// WaitForClose should be used after CloseAllConnections() is closed and
// you want to provoke a MySQL client error with errno 2006.
//
@@ -342,7 +354,14 @@ func (db *DB) WarningCount(c *mysql.Conn) uint16 {
}
// HandleQuery is the default implementation of the QueryHandler interface
-func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.Result) error) error {
+func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.Result) error) (err error) {
+ defer func() {
+ if err != nil {
+ db.lastErrorMu.Lock()
+ db.lastError = err
+ db.lastErrorMu.Unlock()
+ }
+ }()
if db.allowAll.Load() {
return callback(&sqltypes.Result{})
}
@@ -413,7 +432,7 @@ func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.R
return callback(&sqltypes.Result{})
}
// Nothing matched.
- err := fmt.Errorf("fakesqldb:: query: '%s' is not supported on %v",
+ err = fmt.Errorf("fakesqldb:: query: '%s' is not supported on %v",
sqlparser.TruncateForUI(query), db.name)
log.Errorf("Query not found: %s", sqlparser.TruncateForUI(query))
diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go
index 16362bd24ce..6afffa344d7 100644
--- a/go/test/endtoend/cluster/cluster_process.go
+++ b/go/test/endtoend/cluster/cluster_process.go
@@ -929,6 +929,46 @@ func (cluster *LocalProcessCluster) StreamTabletHealth(ctx context.Context, vtta
return responses, nil
}
+// StreamTabletHealthUntil invokes a HealthStream on a local cluster Vttablet and
+// returns the responses. It waits until a certain condition is met. The amount of time to wait is an input that it takes.
+func (cluster *LocalProcessCluster) StreamTabletHealthUntil(ctx context.Context, vttablet *Vttablet, timeout time.Duration, condition func(shr *querypb.StreamHealthResponse) bool) error {
+ tablet, err := cluster.VtctlclientGetTablet(vttablet)
+ if err != nil {
+ return err
+ }
+
+ conn, err := tabletconn.GetDialer()(tablet, grpcclient.FailFast(false))
+ if err != nil {
+ return err
+ }
+
+ conditionSuccess := false
+ timeoutExceeded := false
+ go func() {
+ time.Sleep(timeout)
+ timeoutExceeded = true
+ }()
+
+ err = conn.StreamHealth(ctx, func(shr *querypb.StreamHealthResponse) error {
+ if condition(shr) {
+ conditionSuccess = true
+ }
+ if timeoutExceeded || conditionSuccess {
+ return io.EOF
+ }
+ return nil
+ })
+
+ if conditionSuccess {
+ return nil
+ }
+
+ if timeoutExceeded {
+ return errors.New("timeout exceed while waiting for the condition in StreamHealth")
+ }
+ return err
+}
+
func (cluster *LocalProcessCluster) VtctlclientGetTablet(tablet *Vttablet) (*topodatapb.Tablet, error) {
result, err := cluster.VtctlclientProcess.ExecuteCommandWithOutput("GetTablet", "--", tablet.Alias)
if err != nil {
diff --git a/go/test/endtoend/cluster/vtctldclient_process.go b/go/test/endtoend/cluster/vtctldclient_process.go
index c08c9f6f52a..b3c632a5afe 100644
--- a/go/test/endtoend/cluster/vtctldclient_process.go
+++ b/go/test/endtoend/cluster/vtctldclient_process.go
@@ -93,6 +93,18 @@ func VtctldClientProcessInstance(hostname string, grpcPort int, tmpDirectory str
return vtctldclient
}
+// PlannedReparentShard executes vtctlclient command to make specified tablet the primary for the shard.
+func (vtctldclient *VtctldClientProcess) PlannedReparentShard(Keyspace string, Shard string, alias string) (err error) {
+ output, err := vtctldclient.ExecuteCommandWithOutput(
+ "PlannedReparentShard",
+ fmt.Sprintf("%s/%s", Keyspace, Shard),
+ "--new-primary", alias)
+ if err != nil {
+ log.Errorf("error in PlannedReparentShard output %s, err %s", output, err.Error())
+ }
+ return err
+}
+
// CreateKeyspace executes the vtctl command to create a keyspace
func (vtctldclient *VtctldClientProcess) CreateKeyspace(keyspaceName string, sidecarDBName string) (err error) {
var output string
diff --git a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go
index 0d68c7a2521..fb6a64efef3 100644
--- a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go
+++ b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go
@@ -119,5 +119,5 @@ func TestTopoCustomRule(t *testing.T) {
// Reset the VtTabletExtraArgs
clusterInstance.VtTabletExtraArgs = []string{}
// Tear down custom processes
- killTablets(t, rTablet)
+ killTablets(rTablet)
}
diff --git a/go/test/endtoend/tabletmanager/main_test.go b/go/test/endtoend/tabletmanager/main_test.go
index 39f4830b33d..1d5992bd839 100644
--- a/go/test/endtoend/tabletmanager/main_test.go
+++ b/go/test/endtoend/tabletmanager/main_test.go
@@ -95,6 +95,7 @@ func TestMain(m *testing.M) {
// List of users authorized to execute vschema ddl operations
clusterInstance.VtGateExtraArgs = []string{
"--vschema_ddl_authorized_users=%",
+ "--enable-views",
"--discovery_low_replication_lag", tabletUnhealthyThreshold.String(),
}
// Set extra tablet args for lock timeout
diff --git a/go/test/endtoend/tabletmanager/tablet_health_test.go b/go/test/endtoend/tabletmanager/tablet_health_test.go
index aaa5719cdcd..17017e8b807 100644
--- a/go/test/endtoend/tabletmanager/tablet_health_test.go
+++ b/go/test/endtoend/tabletmanager/tablet_health_test.go
@@ -27,12 +27,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "k8s.io/utils/strings/slices"
"vitess.io/vitess/go/json2"
"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/test/endtoend/cluster"
"vitess.io/vitess/go/test/endtoend/utils"
-
querypb "vitess.io/vitess/go/vt/proto/query"
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
)
@@ -87,13 +87,15 @@ func TestTabletReshuffle(t *testing.T) {
err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", rTablet.Alias)
assert.Error(t, err, "cannot perform backup without my.cnf")
- killTablets(t, rTablet)
+ killTablets(rTablet)
}
func TestHealthCheck(t *testing.T) {
// Add one replica that starts not initialized
defer cluster.PanicHandler(t)
ctx := context.Background()
+ clusterInstance.DisableVTOrcRecoveries(t)
+ defer clusterInstance.EnableVTOrcRecoveries(t)
rTablet := clusterInstance.NewVttabletInstance("replica", 0, "")
@@ -192,7 +194,141 @@ func TestHealthCheck(t *testing.T) {
}
// Manual cleanup of processes
- killTablets(t, rTablet)
+ killTablets(rTablet)
+}
+
+// TestHealthCheckSchemaChangeSignal tests the tables and views, which report their schemas have changed in the output of a StreamHealth.
+func TestHealthCheckSchemaChangeSignal(t *testing.T) {
+ // Add one replica that starts not initialized
+ defer cluster.PanicHandler(t)
+ ctx := context.Background()
+
+ vtParams := clusterInstance.GetVTParams(keyspaceName)
+ conn, err := mysql.Connect(ctx, &vtParams)
+ require.NoError(t, err)
+ defer conn.Close()
+
+ // Make sure the primary is the primary when the test starts.
+ // This state should be ensured before we actually test anything.
+ checkTabletType(t, primaryTablet.Alias, "PRIMARY")
+
+ // Run a bunch of DDL queries and verify that the tables/views changed show up in the health stream.
+ // These tests are for the part where `--queryserver-enable-views` flag is not set.
+ verifyHealthStreamSchemaChangeSignals(t, conn, &primaryTablet, false)
+
+ // We start a new vttablet, this time with `--queryserver-enable-views` flag specified.
+ tempTablet := clusterInstance.NewVttabletInstance("replica", 0, "")
+ // Start Mysql Processes and return connection
+ _, err = cluster.StartMySQLAndGetConnection(ctx, tempTablet, username, clusterInstance.TmpDirectory)
+ require.NoError(t, err)
+ oldArgs := clusterInstance.VtTabletExtraArgs
+ clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, "--queryserver-enable-views")
+ defer func() {
+ clusterInstance.VtTabletExtraArgs = oldArgs
+ }()
+ // start vttablet process, should be in SERVING state as we already have a primary.
+ err = clusterInstance.StartVttablet(tempTablet, "SERVING", false, cell, keyspaceName, hostname, shardName)
+ require.NoError(t, err)
+
+ defer func() {
+ // Restore the primary tablet back to the original.
+ err = clusterInstance.VtctldClientProcess.PlannedReparentShard(keyspaceName, shardName, primaryTablet.Alias)
+ require.NoError(t, err)
+ // Manual cleanup of processes
+ killTablets(tempTablet)
+ }()
+
+ // Now we reparent the cluster to the new tablet we have.
+ err = clusterInstance.VtctldClientProcess.PlannedReparentShard(keyspaceName, shardName, tempTablet.Alias)
+ require.NoError(t, err)
+
+ checkTabletType(t, tempTablet.Alias, "PRIMARY")
+ // Run a bunch of DDL queries and verify that the tables/views changed show up in the health stream.
+ // These tests are for the part where `--queryserver-enable-views` flag is set.
+ verifyHealthStreamSchemaChangeSignals(t, conn, tempTablet, true)
+}
+
+func verifyHealthStreamSchemaChangeSignals(t *testing.T, vtgateConn *mysql.Conn, primaryTablet *cluster.Vttablet, viewsEnabled bool) {
+ var streamErr error
+ wg := sync.WaitGroup{}
+ wg.Add(1)
+ ranOnce := false
+ finished := false
+ ch := make(chan *querypb.StreamHealthResponse)
+ go func() {
+ defer wg.Done()
+ streamErr = clusterInstance.StreamTabletHealthUntil(context.Background(), primaryTablet, 30*time.Second, func(shr *querypb.StreamHealthResponse) bool {
+ ranOnce = true
+ // If we are finished, then close the channel and end the stream.
+ if finished {
+ close(ch)
+ return true
+ }
+ // Put the response in the channel.
+ ch <- shr
+ return false
+ })
+ }()
+ // The test becomes flaky if we run the DDL immediately after starting the above go routine because the client for the Stream
+ // sometimes isn't registered by the time DDL runs, and it misses the update we get. To prevent this situation, we wait for one Stream packet
+ // to have returned. Once we know we received a Stream packet, then we know that we are registered for the health stream and can execute the DDL.
+ for i := 0; i < 30; i++ {
+ if ranOnce {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+
+ verifyTableDDLSchemaChangeSignal(t, vtgateConn, ch, "CREATE TABLE `area` (`id` int NOT NULL, `country` varchar(30), PRIMARY KEY (`id`))", "area")
+ verifyTableDDLSchemaChangeSignal(t, vtgateConn, ch, "CREATE TABLE `area2` (`id` int NOT NULL, PRIMARY KEY (`id`))", "area2")
+ verifyViewDDLSchemaChangeSignal(t, vtgateConn, ch, "CREATE VIEW v2 as select * from t1", viewsEnabled)
+ verifyTableDDLSchemaChangeSignal(t, vtgateConn, ch, "ALTER TABLE `area` ADD COLUMN name varchar(30) NOT NULL", "area")
+ verifyTableDDLSchemaChangeSignal(t, vtgateConn, ch, "DROP TABLE `area2`", "area2")
+ verifyViewDDLSchemaChangeSignal(t, vtgateConn, ch, "ALTER VIEW v2 as select id from t1", viewsEnabled)
+ verifyViewDDLSchemaChangeSignal(t, vtgateConn, ch, "DROP VIEW v2", viewsEnabled)
+ verifyTableDDLSchemaChangeSignal(t, vtgateConn, ch, "DROP TABLE `area`", "area")
+
+ finished = true
+ wg.Wait()
+ require.NoError(t, streamErr)
+}
+
+func verifyTableDDLSchemaChangeSignal(t *testing.T, vtgateConn *mysql.Conn, ch chan *querypb.StreamHealthResponse, query string, table string) {
+ _, err := vtgateConn.ExecuteFetch(query, 10000, false)
+ require.NoError(t, err)
+
+ timeout := time.After(15 * time.Second)
+ for {
+ select {
+ case shr := <-ch:
+ if shr != nil && shr.RealtimeStats != nil && slices.Contains(shr.RealtimeStats.TableSchemaChanged, table) {
+ return
+ }
+ case <-timeout:
+ t.Errorf("didn't get the correct tables changed in stream response until timeout")
+ }
+ }
+}
+
+func verifyViewDDLSchemaChangeSignal(t *testing.T, vtgateConn *mysql.Conn, ch chan *querypb.StreamHealthResponse, query string, viewsEnabled bool) {
+ _, err := vtgateConn.ExecuteFetch(query, 10000, false)
+ require.NoError(t, err)
+
+ timeout := time.After(15 * time.Second)
+ for {
+ select {
+ case shr := <-ch:
+ listToUse := shr.RealtimeStats.TableSchemaChanged
+ if viewsEnabled {
+ listToUse = shr.RealtimeStats.ViewSchemaChanged
+ }
+ if shr != nil && shr.RealtimeStats != nil && slices.Contains(listToUse, "v2") {
+ return
+ }
+ case <-timeout:
+ t.Errorf("didn't get the correct views changed in stream response until timeout")
+ }
+ }
}
func checkHealth(t *testing.T, port int, shouldError bool) {
@@ -247,6 +383,8 @@ func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) {
//Wait if tablet is not in service state
defer cluster.PanicHandler(t)
+ clusterInstance.DisableVTOrcRecoveries(t)
+ defer clusterInstance.EnableVTOrcRecoveries(t)
err := rdonlyTablet.VttabletProcess.WaitForTabletStatus("SERVING")
require.NoError(t, err)
@@ -284,7 +422,7 @@ func TestHealthCheckDrainedStateDoesNotShutdownQueryService(t *testing.T) {
checkHealth(t, rdonlyTablet.HTTPPort, false)
}
-func killTablets(t *testing.T, tablets ...*cluster.Vttablet) {
+func killTablets(tablets ...*cluster.Vttablet) {
var wg sync.WaitGroup
for _, tablet := range tablets {
wg.Add(1)
@@ -292,6 +430,7 @@ func killTablets(t *testing.T, tablets ...*cluster.Vttablet) {
defer wg.Done()
_ = tablet.VttabletProcess.TearDown()
_ = tablet.MysqlctlProcess.Stop()
+ _ = clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias)
}(tablet)
}
wg.Wait()
diff --git a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go
index a2e1e8bd987..2ad907ec7b8 100644
--- a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go
+++ b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go
@@ -57,7 +57,7 @@ func TestFallbackSecurityPolicy(t *testing.T) {
// Reset the VtTabletExtraArgs
clusterInstance.VtTabletExtraArgs = []string{}
// Tear down custom processes
- killTablets(t, mTablet)
+ killTablets(mTablet)
}
func assertNotAllowedURLTest(t *testing.T, url string) {
@@ -112,7 +112,7 @@ func TestDenyAllSecurityPolicy(t *testing.T) {
// Reset the VtTabletExtraArgs
clusterInstance.VtTabletExtraArgs = []string{}
// Tear down custom processes
- killTablets(t, mTablet)
+ killTablets(mTablet)
}
func TestReadOnlySecurityPolicy(t *testing.T) {
@@ -144,5 +144,5 @@ func TestReadOnlySecurityPolicy(t *testing.T) {
// Reset the VtTabletExtraArgs
clusterInstance.VtTabletExtraArgs = []string{}
// Tear down custom processes
- killTablets(t, mTablet)
+ killTablets(mTablet)
}
diff --git a/go/test/endtoend/tabletmanager/tablet_test.go b/go/test/endtoend/tabletmanager/tablet_test.go
index 643785dcd89..97715d39a58 100644
--- a/go/test/endtoend/tabletmanager/tablet_test.go
+++ b/go/test/endtoend/tabletmanager/tablet_test.go
@@ -61,7 +61,7 @@ func TestEnsureDB(t *testing.T) {
require.NoError(t, err)
err = tablet.VttabletProcess.WaitForTabletStatus("SERVING")
require.NoError(t, err)
- killTablets(t, tablet)
+ killTablets(tablet)
}
// TestResetReplicationParameters tests that the RPC ResetReplicationParameters works as intended.
diff --git a/go/test/endtoend/vreplication/sidecardb_test.go b/go/test/endtoend/vreplication/sidecardb_test.go
index 56ca2d08acd..3b796da908c 100644
--- a/go/test/endtoend/vreplication/sidecardb_test.go
+++ b/go/test/endtoend/vreplication/sidecardb_test.go
@@ -38,7 +38,7 @@ var ddls1, ddls2 []string
func init() {
sidecarDBTables = []string{"copy_state", "dt_participant", "dt_state", "heartbeat", "post_copy_action", "redo_state",
- "redo_statement", "reparent_journal", "resharding_journal", "schema_migrations", "schema_version", "schemacopy",
+ "redo_statement", "reparent_journal", "resharding_journal", "schema_engine_tables", "schema_engine_views", "schema_migrations", "schema_version", "schemacopy",
"vdiff", "vdiff_log", "vdiff_table", "views", "vreplication", "vreplication_log"}
numSidecarDBTables = len(sidecarDBTables)
ddls1 = []string{
diff --git a/go/test/fuzzing/tabletserver_schema_fuzzer.go b/go/test/fuzzing/tabletserver_schema_fuzzer.go
index 939fbd4b0e8..67bb36e52ed 100644
--- a/go/test/fuzzing/tabletserver_schema_fuzzer.go
+++ b/go/test/fuzzing/tabletserver_schema_fuzzer.go
@@ -72,5 +72,5 @@ func newTestLoadTable(tableName, comment string, db *fakesqldb.DB) (*schema.Tabl
}
defer conn.Recycle()
- return schema.LoadTable(conn, "fakesqldb", tableName, comment)
+ return schema.LoadTable(conn, "fakesqldb", tableName, "BASE_TABLE", comment)
}
diff --git a/go/vt/sidecardb/schema/schemaengine/schema_engine_tables.sql b/go/vt/sidecardb/schema/schemaengine/schema_engine_tables.sql
new file mode 100644
index 00000000000..5c5869c850b
--- /dev/null
+++ b/go/vt/sidecardb/schema/schemaengine/schema_engine_tables.sql
@@ -0,0 +1,24 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+CREATE TABLE IF NOT EXISTS schema_engine_tables
+(
+ TABLE_SCHEMA varchar(64) NOT NULL,
+ TABLE_NAME varchar(64) NOT NULL,
+ CREATE_STATEMENT longtext,
+ CREATE_TIME BIGINT,
+ PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
+) engine = InnoDB
diff --git a/go/vt/sidecardb/schema/schemaengine/schema_engine_views.sql b/go/vt/sidecardb/schema/schemaengine/schema_engine_views.sql
new file mode 100644
index 00000000000..43e0fe2f247
--- /dev/null
+++ b/go/vt/sidecardb/schema/schemaengine/schema_engine_views.sql
@@ -0,0 +1,24 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+CREATE TABLE IF NOT EXISTS schema_engine_views
+(
+ TABLE_SCHEMA varchar(64) NOT NULL,
+ TABLE_NAME varchar(64) NOT NULL,
+ CREATE_STATEMENT longtext,
+ VIEW_DEFINITION longtext NOT NULL,
+ PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
+) engine = InnoDB
diff --git a/go/vt/vttablet/endtoend/framework/server.go b/go/vt/vttablet/endtoend/framework/server.go
index 2fca66f6d93..169055faba3 100644
--- a/go/vt/vttablet/endtoend/framework/server.go
+++ b/go/vt/vttablet/endtoend/framework/server.go
@@ -119,7 +119,6 @@ func StartServer(connParams, connAppDebugParams mysql.ConnParams, dbName string)
config.HotRowProtection.Mode = tabletenv.Enable
config.TrackSchemaVersions = true
_ = config.GracePeriods.ShutdownSeconds.Set("2s")
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("2100ms")
config.SignalWhenSchemaChange = true
_ = config.Healthcheck.IntervalSeconds.Set("100ms")
_ = config.Oltp.TxTimeoutSeconds.Set("5s")
diff --git a/go/vt/vttablet/endtoend/healthstream_test.go b/go/vt/vttablet/endtoend/healthstream_test.go
index ad6f0884270..1afe1238913 100644
--- a/go/vt/vttablet/endtoend/healthstream_test.go
+++ b/go/vt/vttablet/endtoend/healthstream_test.go
@@ -21,8 +21,8 @@ import (
"time"
"github.com/stretchr/testify/assert"
+ "golang.org/x/exp/slices"
- "vitess.io/vitess/go/test/utils"
querypb "vitess.io/vitess/go/vt/proto/query"
"vitess.io/vitess/go/vt/vttablet/endtoend/framework"
)
@@ -31,37 +31,37 @@ func TestSchemaChange(t *testing.T) {
client := framework.NewClient()
tcs := []struct {
- tName string
- response []string
- ddl string
+ tName string
+ expectedChange string
+ ddl string
}{
{
"create table 1",
- []string{"vitess_sc1"},
+ "vitess_sc1",
"create table vitess_sc1(id bigint primary key)",
}, {
"create table 2",
- []string{"vitess_sc2"},
+ "vitess_sc2",
"create table vitess_sc2(id bigint primary key)",
}, {
"add column 1",
- []string{"vitess_sc1"},
+ "vitess_sc1",
"alter table vitess_sc1 add column newCol varchar(50)",
}, {
"add column 2",
- []string{"vitess_sc2"},
+ "vitess_sc2",
"alter table vitess_sc2 add column newCol varchar(50)",
}, {
"remove column",
- []string{"vitess_sc1"},
+ "vitess_sc1",
"alter table vitess_sc1 drop column newCol",
}, {
"drop table 2",
- []string{"vitess_sc2"},
+ "vitess_sc2",
"drop table vitess_sc2",
}, {
"drop table 1",
- []string{"vitess_sc1"},
+ "vitess_sc1",
"drop table vitess_sc1",
},
}
@@ -76,24 +76,21 @@ func TestSchemaChange(t *testing.T) {
})
}(ch)
- select {
- case <-ch: // get the schema notification
- case <-time.After(3 * time.Second):
- // We might not see the initial changes
- // as the health stream ticker would have started very early on and
- // this test client might not be even registered.
- }
-
for _, tc := range tcs {
t.Run(tc.tName, func(t *testing.T) {
_, err := client.Execute(tc.ddl, nil)
assert.NoError(t, err)
- select {
- case res := <-ch: // get the schema notification
- utils.MustMatch(t, tc.response, res, "")
- case <-time.After(5 * time.Second):
- t.Errorf("timed out")
- return
+ timeout := time.After(5 * time.Second)
+ for {
+ select {
+ case res := <-ch: // get the schema notification
+ if slices.Contains(res, tc.expectedChange) {
+ return
+ }
+ case <-timeout:
+ t.Errorf("timed out waiting for a schema notification")
+ return
+ }
}
})
}
diff --git a/go/vt/vttablet/endtoend/streamtimeout/healthstream_test.go b/go/vt/vttablet/endtoend/streamtimeout/healthstream_test.go
index c4f47a15857..1f5d2f56cf6 100644
--- a/go/vt/vttablet/endtoend/streamtimeout/healthstream_test.go
+++ b/go/vt/vttablet/endtoend/streamtimeout/healthstream_test.go
@@ -22,8 +22,8 @@ import (
"time"
"github.com/stretchr/testify/require"
+ "golang.org/x/exp/slices"
- "vitess.io/vitess/go/test/utils"
querypb "vitess.io/vitess/go/vt/proto/query"
"vitess.io/vitess/go/vt/vttablet/endtoend/framework"
)
@@ -93,10 +93,16 @@ loop:
// wait for the health_streamer to complete retrying the notification.
reloadTimeout := config.SchemaChangeReloadTimeout
retryEstimatedTime := reloadTimeout + reloadInterval + reloadEstimatedTime
- select {
- case res := <-ch: // get the schema notification
- utils.MustMatch(t, []string{tableName}, res, "unexpected result from schema reload response")
- case <-time.After(retryEstimatedTime):
- t.Errorf("timed out even after the mysql hang was no longer simulated")
+ timeout := time.After(retryEstimatedTime)
+ for {
+ select {
+ case res := <-ch: // get the schema notification
+ if slices.Contains(res, tableName) {
+ return
+ }
+ case <-timeout:
+ t.Errorf("timed out even after the mysql hang was no longer simulated")
+ return
+ }
}
}
diff --git a/go/vt/vttablet/endtoend/streamtimeout/main_test.go b/go/vt/vttablet/endtoend/streamtimeout/main_test.go
index 30496e5c4d6..406a70fe3a7 100644
--- a/go/vt/vttablet/endtoend/streamtimeout/main_test.go
+++ b/go/vt/vttablet/endtoend/streamtimeout/main_test.go
@@ -83,7 +83,7 @@ func TestMain(m *testing.M) {
connParams := cluster.MySQLConnParams()
connAppDebugParams := cluster.MySQLAppDebugConnParams()
config = tabletenv.NewDefaultConfig()
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("2100ms")
+ _ = config.SchemaReloadIntervalSeconds.Set("2100ms")
config.SchemaChangeReloadTimeout = 10 * time.Second
config.SignalWhenSchemaChange = true
err = framework.StartCustomServer(connParams, connAppDebugParams, cluster.DbName(), config)
diff --git a/go/vt/vttablet/endtoend/views_test.go b/go/vt/vttablet/endtoend/views_test.go
index 9ba27d6ffae..e3e911e2c43 100644
--- a/go/vt/vttablet/endtoend/views_test.go
+++ b/go/vt/vttablet/endtoend/views_test.go
@@ -171,16 +171,22 @@ func TestDropViewDDL(t *testing.T) {
_, err = client.Execute("create view vitess_view2 as select * from vitess_a", nil)
require.NoError(t, err)
+ // validate both the views are stored in _vt.views.
+ waitForResult(t, client, 2, 1*time.Minute)
+
// drop vitess_view1, should PASS
_, err = client.Execute("drop view vitess_view1", nil)
require.NoError(t, err)
- // drop three views, only vitess_view2 exists. This should FAIL but drops the existing view.
+ // drop three views, only vitess_view2 exists.
+ // In MySQL 5.7, this would drop vitess_view2, but that behaviour has changed
+ // in MySQL 8.0, and not the view isn't dropped. CI is running 8.0, so the remaining test is
+ // written with those expectations.
_, err = client.Execute("drop view vitess_view1, vitess_view2, vitess_view3", nil)
require.ErrorContains(t, err, "Unknown table 'vttest.vitess_view1,vttest.vitess_view3'")
// validate ZERO rows in _vt.views.
- waitForResult(t, client, 0, 1*time.Minute)
+ waitForResult(t, client, 1, 1*time.Minute)
// create a view.
_, err = client.Execute("create view vitess_view1 as select * from vitess_a", nil)
@@ -298,6 +304,7 @@ func waitForResult(t *testing.T, client *framework.QueryClient, rowCount int, ti
select {
case <-wait:
t.Errorf("all views are not dropped within the time")
+ return
case <-time.After(1 * time.Second):
qr, err := client.Execute(qSelAllRows, nil)
require.NoError(t, err)
diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go
index a1d86cb30c9..645a99cfc2b 100644
--- a/go/vt/vttablet/endtoend/vstreamer_test.go
+++ b/go/vt/vttablet/endtoend/vstreamer_test.go
@@ -202,6 +202,13 @@ func TestSchemaVersioning(t *testing.T) {
log.Infof("Received event %v", event)
evs = append(evs, event)
}
+ // Ignore unrelated events.
+ if len(evs) == 3 &&
+ evs[0].Type == binlogdatapb.VEventType_BEGIN &&
+ evs[1].Type == binlogdatapb.VEventType_GTID &&
+ evs[2].Type == binlogdatapb.VEventType_COMMIT {
+ return nil
+ }
select {
case eventCh <- evs:
case <-ctx.Done():
@@ -267,6 +274,13 @@ func TestSchemaVersioning(t *testing.T) {
log.Infof("Received event %v", event)
evs = append(evs, event)
}
+ // Ignore unrelated events.
+ if len(evs) == 3 &&
+ evs[0].Type == binlogdatapb.VEventType_BEGIN &&
+ evs[1].Type == binlogdatapb.VEventType_GTID &&
+ evs[2].Type == binlogdatapb.VEventType_COMMIT {
+ return nil
+ }
select {
case eventCh <- evs:
case <-ctx.Done():
diff --git a/go/vt/vttablet/tabletserver/binlog_watcher.go b/go/vt/vttablet/tabletserver/binlog_watcher.go
index 6c713791e6f..8ae0b874119 100644
--- a/go/vt/vttablet/tabletserver/binlog_watcher.go
+++ b/go/vt/vttablet/tabletserver/binlog_watcher.go
@@ -17,11 +17,10 @@ limitations under the License.
package tabletserver
import (
+ "context"
"sync"
"time"
- "context"
-
"vitess.io/vitess/go/vt/log"
"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go
index e10564b1c83..fd641f1c8df 100644
--- a/go/vt/vttablet/tabletserver/health_streamer.go
+++ b/go/vt/vttablet/tabletserver/health_streamer.go
@@ -28,6 +28,7 @@ import (
"github.com/spf13/pflag"
"vitess.io/vitess/go/vt/vttablet/tabletserver/planbuilder"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
"vitess.io/vitess/go/vt/servenv"
"vitess.io/vitess/go/vt/sidecardb"
@@ -39,7 +40,6 @@ import (
"vitess.io/vitess/go/vt/dbconfigs"
"vitess.io/vitess/go/mysql"
- "vitess.io/vitess/go/timer"
"vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
"google.golang.org/protobuf/proto"
@@ -84,10 +84,12 @@ type healthStreamer struct {
cancel context.CancelFunc
clients map[chan *querypb.StreamHealthResponse]struct{}
state *querypb.StreamHealthResponse
+ // isServingPrimary stores if this tablet is currently the serving primary or not.
+ isServingPrimary bool
+ se *schema.Engine
history *history.History
- ticks *timer.Timer
dbConfig dbconfigs.Connector
conns *connpool.Pool
signalWhenSchemaChange bool
@@ -96,12 +98,9 @@ type healthStreamer struct {
viewsEnabled bool
}
-func newHealthStreamer(env tabletenv.Env, alias *topodatapb.TabletAlias) *healthStreamer {
- var newTimer *timer.Timer
+func newHealthStreamer(env tabletenv.Env, alias *topodatapb.TabletAlias, engine *schema.Engine) *healthStreamer {
var pool *connpool.Pool
if env.Config().SignalWhenSchemaChange {
- reloadTime := env.Config().SignalSchemaChangeReloadIntervalSeconds.Get()
- newTimer = timer.NewTimer(reloadTime)
// We need one connection for the reloader.
pool = connpool.NewPool(env, "", tabletenv.ConnPoolConfig{
Size: 1,
@@ -122,11 +121,11 @@ func newHealthStreamer(env tabletenv.Env, alias *topodatapb.TabletAlias) *health
},
history: history.New(5),
- ticks: newTimer,
conns: pool,
signalWhenSchemaChange: env.Config().SignalWhenSchemaChange,
reloadTimeout: env.Config().SchemaChangeReloadTimeout,
viewsEnabled: env.Config().EnableViews,
+ se: engine,
}
hs.unhealthyThreshold.Store(env.Config().Healthcheck.UnhealthyThresholdSeconds.Get().Nanoseconds())
return hs
@@ -148,14 +147,7 @@ func (hs *healthStreamer) Open() {
if hs.conns != nil {
// if we don't have a live conns object, it means we are not configured to signal when the schema changes
hs.conns.Open(hs.dbConfig, hs.dbConfig, hs.dbConfig)
- hs.ticks.Start(func() {
- if err := hs.reload(); err != nil {
- log.Errorf("periodic schema reload failed in health stream: %v", err)
- }
- })
-
}
-
}
func (hs *healthStreamer) Close() {
@@ -163,10 +155,7 @@ func (hs *healthStreamer) Close() {
defer hs.mu.Unlock()
if hs.cancel != nil {
- if hs.ticks != nil {
- hs.ticks.Stop()
- hs.conns.Close()
- }
+ hs.se.UnregisterNotifier("healthStreamer")
hs.cancel()
hs.cancel = nil
}
@@ -179,11 +168,6 @@ func (hs *healthStreamer) Stream(ctx context.Context, callback func(*querypb.Str
}
defer hs.unregister(ch)
- // trigger the initial schema reload
- if hs.signalWhenSchemaChange {
- hs.ticks.Trigger()
- }
-
for {
select {
case <-ctx.Done():
@@ -328,12 +312,38 @@ func (hs *healthStreamer) SetUnhealthyThreshold(v time.Duration) {
}
}
-// reload reloads the schema from the underlying mysql
-func (hs *healthStreamer) reload() error {
+// MakePrimary tells the healthstreamer that the current tablet is now the primary,
+// so it can read and write to the MySQL instance for schema-tracking.
+func (hs *healthStreamer) MakePrimary(serving bool) {
+ hs.mu.Lock()
+ defer hs.mu.Unlock()
+ hs.isServingPrimary = serving
+ // We register for notifications from the schema Engine only when schema tracking is enabled,
+ // and we are going to a serving primary state.
+ if serving && hs.signalWhenSchemaChange {
+ hs.se.RegisterNotifier("healthStreamer", func(full map[string]*schema.Table, created, altered, dropped []*schema.Table) {
+ if err := hs.reload(full, created, altered, dropped); err != nil {
+ log.Errorf("periodic schema reload failed in health stream: %v", err)
+ }
+ }, false)
+ }
+}
+
+// MakeNonPrimary tells the healthstreamer that the current tablet is now not a primary.
+func (hs *healthStreamer) MakeNonPrimary() {
+ hs.mu.Lock()
+ defer hs.mu.Unlock()
+ hs.isServingPrimary = false
+}
+
+// reload reloads the schema from the underlying mysql for the tables that we get the alert on.
+func (hs *healthStreamer) reload(full map[string]*schema.Table, created, altered, dropped []*schema.Table) error {
hs.mu.Lock()
defer hs.mu.Unlock()
- // Schema Reload to happen only on primary.
- if hs.state.Target.TabletType != topodatapb.TabletType_PRIMARY {
+ // Schema Reload to happen only on primary when it is serving.
+ // We can be in a state when the primary is not serving after we have run DemotePrimary. In that case,
+ // we don't want to run any queries in MySQL, so we shouldn't reload anything in the healthStreamer.
+ if !hs.isServingPrimary {
return nil
}
@@ -347,12 +357,28 @@ func (hs *healthStreamer) reload() error {
}
defer conn.Recycle()
- tables, err := hs.getChangedTableNames(ctx, conn)
+ // We create lists to store the tables that have schema changes.
+ var tables []string
+ var views []string
+
+ // Range over the tables that are created/altered and split them up based on their type.
+ for _, table := range append(append(dropped, created...), altered...) {
+ tableName := table.Name.String()
+ if table.Type == schema.View && hs.viewsEnabled {
+ views = append(views, tableName)
+ } else {
+ tables = append(tables, tableName)
+ }
+ }
+
+ // Reload the tables and views.
+ // This stores the data that is used by VTGates upto v17. So, we can remove this reload of
+ // tables and views in v19.
+ err = hs.reloadTables(ctx, conn, tables)
if err != nil {
return err
}
-
- views, err := hs.getChangedViewNames(ctx, conn)
+ err = hs.reloadViews(ctx, conn, views)
if err != nil {
if len(tables) == 0 {
return err
@@ -377,65 +403,42 @@ func (hs *healthStreamer) reload() error {
return nil
}
-func (hs *healthStreamer) getChangedTableNames(ctx context.Context, conn *connpool.DBConn) ([]string, error) {
- var tables []string
- var tableNames []string
-
- callback := func(qr *sqltypes.Result) error {
- for _, row := range qr.Rows {
- table := row[0].ToString()
- tables = append(tables, table)
-
- escapedTblName := sqlparser.String(sqlparser.NewStrLiteral(table))
- tableNames = append(tableNames, escapedTblName)
- }
-
+func (hs *healthStreamer) reloadTables(ctx context.Context, conn *connpool.DBConn, tableNames []string) error {
+ if len(tableNames) == 0 {
return nil
}
- alloc := func() *sqltypes.Result { return &sqltypes.Result{} }
- bufferSize := 1000
-
- schemaChangeQuery := sqlparser.BuildParsedQuery(mysql.DetectSchemaChange, sidecardb.GetIdentifier()).Query
- // If views are enabled, then views are tracked/handled separately and schema change does not need to track them.
- if hs.viewsEnabled {
- schemaChangeQuery = sqlparser.BuildParsedQuery(mysql.DetectSchemaChangeOnlyBaseTable, sidecardb.GetIdentifier()).Query
- }
- err := conn.Stream(ctx, schemaChangeQuery, callback, alloc, bufferSize, 0)
- if err != nil {
- return nil, err
- }
-
- // If no change detected, then return
- if len(tables) == 0 {
- return nil, nil
+ var escapedTableNames []string
+ for _, tableName := range tableNames {
+ escapedTblName := sqlparser.String(sqlparser.NewStrLiteral(tableName))
+ escapedTableNames = append(escapedTableNames, escapedTblName)
}
- tableNamePredicate := fmt.Sprintf("table_name IN (%s)", strings.Join(tableNames, ", "))
+ tableNamePredicate := fmt.Sprintf("table_name IN (%s)", strings.Join(escapedTableNames, ", "))
del := fmt.Sprintf("%s AND %s", sqlparser.BuildParsedQuery(mysql.ClearSchemaCopy, sidecardb.GetIdentifier()).Query, tableNamePredicate)
upd := fmt.Sprintf("%s AND %s", sqlparser.BuildParsedQuery(mysql.InsertIntoSchemaCopy, sidecardb.GetIdentifier()).Query, tableNamePredicate)
// Reload the schema in a transaction.
- _, err = conn.Exec(ctx, "begin", 1, false)
+ _, err := conn.Exec(ctx, "begin", 1, false)
if err != nil {
- return nil, err
+ return err
}
defer conn.Exec(ctx, "rollback", 1, false)
_, err = conn.Exec(ctx, del, 1, false)
if err != nil {
- return nil, err
+ return err
}
_, err = conn.Exec(ctx, upd, 1, false)
if err != nil {
- return nil, err
+ return err
}
_, err = conn.Exec(ctx, "commit", 1, false)
if err != nil {
- return nil, err
+ return err
}
- return tables, nil
+ return nil
}
type viewDefAndStmt struct {
@@ -443,63 +446,42 @@ type viewDefAndStmt struct {
stmt string
}
-func (hs *healthStreamer) getChangedViewNames(ctx context.Context, conn *connpool.DBConn) (views []string, err error) {
- if !hs.viewsEnabled {
- return nil, nil
- }
-
- /* Retrieve changed views */
- callback := func(qr *sqltypes.Result) error {
- for _, row := range qr.Rows {
- view := row[0].ToString()
- views = append(views, view)
- }
- return nil
- }
- alloc := func() *sqltypes.Result { return &sqltypes.Result{} }
- bufferSize := 1000
-
- viewChangeQuery := sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query
- err = conn.Stream(ctx, viewChangeQuery, callback, alloc, bufferSize, 0)
- if err != nil {
- return
- }
-
- // If no change detected, then return
+func (hs *healthStreamer) reloadViews(ctx context.Context, conn *connpool.DBConn, views []string) error {
if len(views) == 0 {
- return
+ return nil
}
/* Retrieve changed views definition */
viewsDefStmt := map[string]*viewDefAndStmt{}
- callback = func(qr *sqltypes.Result) error {
+ callback := func(qr *sqltypes.Result) error {
for _, row := range qr.Rows {
viewsDefStmt[row[0].ToString()] = &viewDefAndStmt{def: row[1].ToString()}
}
return nil
}
+ alloc := func() *sqltypes.Result { return &sqltypes.Result{} }
+ bufferSize := 1000
- var viewsBV *querypb.BindVariable
- viewsBV, err = sqltypes.BuildBindVariable(views)
+ viewsBV, err := sqltypes.BuildBindVariable(views)
if err != nil {
- return
+ return err
}
bv := map[string]*querypb.BindVariable{"tableNames": viewsBV}
err = hs.getViewDefinition(ctx, conn, bv, callback, alloc, bufferSize)
if err != nil {
- return
+ return err
}
/* Retrieve create statement for views */
viewsDefStmt, err = hs.getCreateViewStatement(ctx, conn, viewsDefStmt)
if err != nil {
- return
+ return err
}
/* update the views copy table */
err = hs.updateViewsTable(ctx, conn, bv, viewsDefStmt)
- return
+ return err
}
func (hs *healthStreamer) getViewDefinition(ctx context.Context, conn *connpool.DBConn, bv map[string]*querypb.BindVariable, callback func(qr *sqltypes.Result) error, alloc func() *sqltypes.Result, bufferSize int) error {
diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go
index 95cd4b06e6c..f3310abfd8b 100644
--- a/go/vt/vttablet/tabletserver/health_streamer_test.go
+++ b/go/vt/vttablet/tabletserver/health_streamer_test.go
@@ -26,6 +26,7 @@ import (
"time"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"vitess.io/vitess/go/mysql"
@@ -35,6 +36,7 @@ import (
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
"vitess.io/vitess/go/vt/sidecardb"
"vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
)
@@ -48,7 +50,7 @@ func TestHealthStreamerClosed(t *testing.T) {
Uid: 1,
}
blpFunc = testBlpFunc
- hs := newHealthStreamer(env, alias)
+ hs := newHealthStreamer(env, alias, &schema.Engine{})
err := hs.Stream(context.Background(), func(shr *querypb.StreamHealthResponse) error {
return nil
})
@@ -61,6 +63,39 @@ func newConfig(db *fakesqldb.DB) *tabletenv.TabletConfig {
return cfg
}
+// TestNotServingPrimaryNoWrite makes sure that the health-streamer doesn't write anything to the database when
+// the state is not serving primary.
+func TestNotServingPrimaryNoWrite(t *testing.T) {
+ db := fakesqldb.New(t)
+ defer db.Close()
+ config := newConfig(db)
+ config.SignalWhenSchemaChange = true
+
+ env := tabletenv.NewEnv(config, "TestNotServingPrimary")
+ alias := &topodatapb.TabletAlias{
+ Cell: "cell",
+ Uid: 1,
+ }
+ // Create a new health streamer and set it to a serving primary state
+ hs := newHealthStreamer(env, alias, &schema.Engine{})
+ hs.isServingPrimary = true
+ hs.InitDBConfig(&querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}, config.DB.DbaWithDB())
+ hs.Open()
+ defer hs.Close()
+ target := &querypb.Target{}
+ hs.InitDBConfig(target, db.ConnParams())
+
+ // Let's say the tablet goes to a non-serving primary state.
+ hs.MakePrimary(false)
+
+ // A reload now should not write anything to the database. If any write happens it will error out since we have not
+ // added any query to the database to expect.
+ t1 := schema.NewTable("t1", schema.NoType)
+ err := hs.reload(map[string]*schema.Table{"t1": t1}, []*schema.Table{t1}, nil, nil)
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+}
+
func TestHealthStreamerBroadcast(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
@@ -73,7 +108,7 @@ func TestHealthStreamerBroadcast(t *testing.T) {
Uid: 1,
}
blpFunc = testBlpFunc
- hs := newHealthStreamer(env, alias)
+ hs := newHealthStreamer(env, alias, &schema.Engine{})
hs.InitDBConfig(&querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}, config.DB.DbaWithDB())
hs.Open()
defer hs.Close()
@@ -159,273 +194,302 @@ func TestHealthStreamerBroadcast(t *testing.T) {
}
func TestReloadSchema(t *testing.T) {
- db := fakesqldb.New(t)
- defer db.Close()
- config := newConfig(db)
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("100ms")
- config.SignalWhenSchemaChange = true
-
- env := tabletenv.NewEnv(config, "ReplTrackerTest")
- alias := &topodatapb.TabletAlias{
- Cell: "cell",
- Uid: 1,
+ testcases := []struct {
+ name string
+ enableSchemaChange bool
+ }{
+ {
+ name: "Schema Change Enabled",
+ enableSchemaChange: true,
+ }, {
+ name: "Schema Change Disabled",
+ enableSchemaChange: false,
+ },
}
- blpFunc = testBlpFunc
- hs := newHealthStreamer(env, alias)
-
- target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- configs := config.DB
- db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.ClearSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
- db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.InsertIntoSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
- db.AddQuery("begin", &sqltypes.Result{})
- db.AddQuery("commit", &sqltypes.Result{})
- db.AddQuery("rollback", &sqltypes.Result{})
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectSchemaChange, sidecardb.GetIdentifier()).Query,
- sqltypes.MakeTestResult(
- sqltypes.MakeTestFields(
- "table_name",
- "varchar",
- ),
- "product",
- "users",
- ))
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, &sqltypes.Result{})
-
- hs.InitDBConfig(target, configs.DbaWithDB())
- hs.Open()
- defer hs.Close()
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- hs.Stream(ctx, func(response *querypb.StreamHealthResponse) error {
- if response.RealtimeStats.TableSchemaChanged != nil {
- assert.Equal(t, []string{"product", "users"}, response.RealtimeStats.TableSchemaChanged)
- wg.Done()
+ for _, testcase := range testcases {
+ t.Run(testcase.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ defer db.Close()
+ config := newConfig(db)
+ config.SignalWhenSchemaChange = testcase.enableSchemaChange
+ _ = config.SchemaReloadIntervalSeconds.Set("100ms")
+
+ env := tabletenv.NewEnv(config, "ReplTrackerTest")
+ alias := &topodatapb.TabletAlias{
+ Cell: "cell",
+ Uid: 1,
}
- return nil
- })
- }()
-
- c := make(chan struct{})
- go func() {
- defer close(c)
- wg.Wait()
- }()
- select {
- case <-c:
- case <-time.After(1 * time.Second):
- t.Errorf("timed out")
- }
-}
-
-func TestDoesNotReloadSchema(t *testing.T) {
- db := fakesqldb.New(t)
- defer db.Close()
- config := newConfig(db)
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("100ms")
- config.SignalWhenSchemaChange = false
-
- env := tabletenv.NewEnv(config, "ReplTrackerTest")
- alias := &topodatapb.TabletAlias{
- Cell: "cell",
- Uid: 1,
- }
- blpFunc = testBlpFunc
- hs := newHealthStreamer(env, alias)
-
- target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- configs := config.DB
-
- hs.InitDBConfig(target, configs.DbaWithDB())
- hs.Open()
- defer hs.Close()
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- hs.Stream(ctx, func(response *querypb.StreamHealthResponse) error {
- if response.RealtimeStats.TableSchemaChanged != nil {
- wg.Done()
+ blpFunc = testBlpFunc
+ se := schema.NewEngine(env)
+ hs := newHealthStreamer(env, alias, se)
+
+ target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
+ configs := config.DB
+
+ db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.ClearSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
+ db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.InsertIntoSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
+ db.AddQueryPattern("SELECT UNIX_TIMESTAMP()"+".*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "UNIX_TIMESTAMP(now())",
+ "varchar",
+ ),
+ "1684759138",
+ ))
+ db.AddQuery("begin", &sqltypes.Result{})
+ db.AddQuery("commit", &sqltypes.Result{})
+ db.AddQuery("rollback", &sqltypes.Result{})
+ // Add the query pattern for the query that schema.Engine uses to get the tables.
+ db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*",
+ sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT | SUM(i.file_size) | SUM(i.allocated_size)",
+ "varchar|varchar|int64|varchar|int64|int64",
+ ),
+ "product|BASE TABLE|1684735966||114688|114688",
+ "users|BASE TABLE|1684735966||114688|114688",
+ ))
+ db.AddQueryPattern("SELECT COLUMN_NAME as column_name.*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "column_name",
+ "varchar",
+ ),
+ "id",
+ ))
+ db.AddQueryPattern("SELECT `id` FROM `fakesqldb`.*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "id",
+ "int64",
+ ),
+ ))
+ db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("Variable_name|Value", "varchar|int32"),
+ "Innodb_rows_read|50"))
+ db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("table_name | column_name", "varchar|varchar"),
+ "product|id",
+ "users|id",
+ ))
+
+ hs.InitDBConfig(target, configs.DbaWithDB())
+ se.InitDBConfig(configs.DbaWithDB())
+ hs.Open()
+ defer hs.Close()
+ err := se.Open()
+ require.NoError(t, err)
+ defer se.Close()
+ // Start schema notifications.
+ hs.MakePrimary(true)
+
+ // Update the query pattern for the query that schema.Engine uses to get the tables so that it runs a reload again.
+ // If we don't change the t.create_time to a value greater than before, then the schema engine doesn't reload the database.
+ db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*",
+ sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT | SUM(i.file_size) | SUM(i.allocated_size)",
+ "varchar|varchar|int64|varchar|int64|int64",
+ ),
+ "product|BASE TABLE|1684735967||114688|114688",
+ "users|BASE TABLE|1684735967||114688|114688",
+ ))
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ hs.Stream(ctx, func(response *querypb.StreamHealthResponse) error {
+ if response.RealtimeStats.TableSchemaChanged != nil {
+ assert.Equal(t, []string{"product", "users"}, response.RealtimeStats.TableSchemaChanged)
+ wg.Done()
+ }
+ return nil
+ })
+ }()
+
+ c := make(chan struct{})
+ go func() {
+ defer close(c)
+ wg.Wait()
+ }()
+ timeout := false
+ select {
+ case <-c:
+ case <-time.After(1 * time.Second):
+ timeout = true
}
- return nil
- })
- }()
-
- c := make(chan struct{})
- go func() {
- defer close(c)
- wg.Wait()
- }()
- timeout := false
-
- // here we will wait for a second, to make sure that we are not signaling a changed schema.
- select {
- case <-c:
- case <-time.After(1 * time.Second):
- timeout = true
+ require.Equal(t, testcase.enableSchemaChange, !timeout, "If schema change tracking is enabled, then we shouldn't time out, otherwise we should")
+ })
}
-
- assert.True(t, timeout, "should have timed out")
}
-func TestInitialReloadSchema(t *testing.T) {
+// TestReloadView tests that the health streamer tracks view changes correctly
+func TestReloadView(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
config := newConfig(db)
- // Setting the signal schema change reload interval to one minute
- // that way we can test the initial reload trigger.
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("1m")
config.SignalWhenSchemaChange = true
+ _ = config.SchemaReloadIntervalSeconds.Set("100ms")
+ config.EnableViews = true
- env := tabletenv.NewEnv(config, "ReplTrackerTest")
- alias := &topodatapb.TabletAlias{
- Cell: "cell",
- Uid: 1,
- }
- blpFunc = testBlpFunc
- hs := newHealthStreamer(env, alias)
+ env := tabletenv.NewEnv(config, "TestReloadView")
+ alias := &topodatapb.TabletAlias{Cell: "cell", Uid: 1}
+ se := schema.NewEngine(env)
+ hs := newHealthStreamer(env, alias, se)
target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
configs := config.DB
db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.ClearSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
db.AddQueryPattern(sqlparser.BuildParsedQuery(mysql.InsertIntoSchemaCopy, sidecardb.GetIdentifier()).Query+".*", &sqltypes.Result{})
+ db.AddQueryPattern("SELECT UNIX_TIMESTAMP()"+".*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "UNIX_TIMESTAMP(now())",
+ "varchar",
+ ),
+ "1684759138",
+ ))
db.AddQuery("begin", &sqltypes.Result{})
db.AddQuery("commit", &sqltypes.Result{})
db.AddQuery("rollback", &sqltypes.Result{})
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectSchemaChange, sidecardb.GetIdentifier()).Query,
+ // Add the query pattern for the query that schema.Engine uses to get the tables.
+ db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*",
sqltypes.MakeTestResult(
sqltypes.MakeTestFields(
- "table_name",
- "varchar",
+ "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT | SUM(i.file_size) | SUM(i.allocated_size)",
+ "varchar|varchar|int64|varchar|int64|int64",
),
- "product",
- "users",
))
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, &sqltypes.Result{})
-
- hs.InitDBConfig(target, configs.DbaWithDB())
- hs.Open()
- defer hs.Close()
- var wg sync.WaitGroup
- wg.Add(1)
- go func() {
- hs.Stream(ctx, func(response *querypb.StreamHealthResponse) error {
- if response.RealtimeStats.TableSchemaChanged != nil {
- assert.Equal(t, []string{"product", "users"}, response.RealtimeStats.TableSchemaChanged)
- wg.Done()
- }
- return nil
- })
- }()
-
- c := make(chan struct{})
- go func() {
- defer close(c)
- wg.Wait()
- }()
- select {
- case <-c:
- case <-time.After(1 * time.Second):
- // should not timeout despite SignalSchemaChangeReloadIntervalSeconds being set to 1 minute
- t.Errorf("timed out")
- }
-}
-
-// TestReloadView tests that the health streamer tracks view changes correctly
-func TestReloadView(t *testing.T) {
- db := fakesqldb.New(t)
- defer db.Close()
- config := newConfig(db)
- _ = config.SignalSchemaChangeReloadIntervalSeconds.Set("100ms")
- config.EnableViews = true
-
- env := tabletenv.NewEnv(config, "TestReloadView")
- alias := &topodatapb.TabletAlias{Cell: "cell", Uid: 1}
- hs := newHealthStreamer(env, alias)
-
- target := &querypb.Target{TabletType: topodatapb.TabletType_PRIMARY}
- configs := config.DB
-
- db.AddQuery("begin", &sqltypes.Result{})
- db.AddQuery("commit", &sqltypes.Result{})
- db.AddQuery("rollback", &sqltypes.Result{})
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectSchemaChangeOnlyBaseTable, sidecardb.GetIdentifier()).Query, &sqltypes.Result{})
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, &sqltypes.Result{})
+ db.AddQueryPattern("SELECT COLUMN_NAME as column_name.*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "column_name",
+ "varchar",
+ ),
+ "id",
+ ))
+ db.AddQueryPattern("SELECT `id` FROM `fakesqldb`.*", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(
+ "id",
+ "int64",
+ ),
+ ))
+ db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("Variable_name|Value", "varchar|int32"),
+ "Innodb_rows_read|50"))
+ db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("table_name | column_name", "varchar|varchar"),
+ ))
+ db.AddQueryPattern(".*SELECT table_name, view_definition.*schema_engine_views.*", &sqltypes.Result{})
+ db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.schema_engine_tables", &sqltypes.Result{})
hs.InitDBConfig(target, configs.DbaWithDB())
+ se.InitDBConfig(configs.DbaWithDB())
hs.Open()
defer hs.Close()
+ err := se.Open()
+ require.NoError(t, err)
+ se.MakePrimary(true)
+ defer se.Close()
+ // Start schema notifications.
+ hs.MakePrimary(true)
showCreateViewFields := sqltypes.MakeTestFields(
"View|Create View|character_set_client|collation_connection",
"varchar|text|varchar|varchar")
+ showTableSizesFields := sqltypes.MakeTestFields(
+ "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT | SUM(i.file_size) | SUM(i.allocated_size)",
+ "varchar|varchar|int64|varchar|int64|int64",
+ )
+
tcases := []struct {
- tbl *sqltypes.Result
- def *sqltypes.Result
- stmt []*sqltypes.Result
- expTbl []string
- expDefQuery string
- expStmtQuery []string
- expClearQuery string
- expInsertQuery []string
- }{{
- // view_a and view_b added.
- tbl: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
- "view_a", "view_b"),
- def: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
- "view_a|def_a", "view_b|def_b"),
- stmt: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_a|create_view_a|utf8|utf8_general_ci"),
- sqltypes.MakeTestResult(showCreateViewFields, "view_b|create_view_b|utf8|utf8_general_ci")},
- expTbl: []string{"view_a", "view_b"},
- expDefQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_a', 'view_b')",
- expStmtQuery: []string{"show create table view_a", "show create table view_b"},
- expClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_a', 'view_b')",
- expInsertQuery: []string{
- "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_a', 'create_view_a', 'def_a')",
- "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_b', 'create_view_b', 'def_b')",
+ detectViewChangeOutput *sqltypes.Result
+ showTablesWithSizesOutput *sqltypes.Result
+
+ expCreateStmtQuery []string
+ createStmtOutput []*sqltypes.Result
+
+ expGetViewDefinitionsQuery string
+ viewDefinitionsOutput *sqltypes.Result
+
+ expClearQuery string
+ expHsClearQuery string
+ expInsertQuery []string
+ expViewsChanged []string
+ }{
+ {
+ // view_a and view_b added.
+ detectViewChangeOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
+ "view_a", "view_b"),
+ showTablesWithSizesOutput: sqltypes.MakeTestResult(showTableSizesFields, "view_a|VIEW|12345678||123|123", "view_b|VIEW|12345678||123|123"),
+ viewDefinitionsOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
+ "view_a|def_a", "view_b|def_b"),
+ createStmtOutput: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_a|create_view_a|utf8|utf8_general_ci"),
+ sqltypes.MakeTestResult(showCreateViewFields, "view_b|create_view_b|utf8|utf8_general_ci")},
+ expViewsChanged: []string{"view_a", "view_b"},
+ expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_a', 'view_b')",
+ expCreateStmtQuery: []string{"show create table view_a", "show create table view_b"},
+ expClearQuery: "delete from _vt.schema_engine_views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_a', 'view_b')",
+ expHsClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_a', 'view_b')",
+ expInsertQuery: []string{
+ "insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_a', 'create_view_a', 'def_a')",
+ "insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_b', 'create_view_b', 'def_b')",
+ "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_a', 'create_view_a', 'def_a')",
+ "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_b', 'create_view_b', 'def_b')",
+ },
},
- }, {
- // view_b modified
- tbl: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
- "view_b"),
- def: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
- "view_b|def_mod_b"),
- stmt: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_b|create_view_mod_b|utf8|utf8_general_ci")},
- expTbl: []string{"view_b"},
- expDefQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_b')",
- expStmtQuery: []string{"show create table view_b"},
- expClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_b')",
- expInsertQuery: []string{
- "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_b', 'create_view_mod_b', 'def_mod_b')",
+ {
+ // view_b modified
+ showTablesWithSizesOutput: sqltypes.MakeTestResult(showTableSizesFields, "view_a|VIEW|12345678||123|123", "view_b|VIEW|12345678||123|123"),
+ detectViewChangeOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
+ "view_b"),
+ viewDefinitionsOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
+ "view_b|def_mod_b"),
+ createStmtOutput: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_b|create_view_mod_b|utf8|utf8_general_ci")},
+ expViewsChanged: []string{"view_b"},
+ expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_b')",
+ expCreateStmtQuery: []string{"show create table view_b"},
+ expHsClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_b')",
+ expClearQuery: "delete from _vt.schema_engine_views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_b')",
+ expInsertQuery: []string{
+ "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_b', 'create_view_mod_b', 'def_mod_b')",
+ "insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_b', 'create_view_mod_b', 'def_mod_b')",
+ },
},
- }, {
- // view_a modified, view_b deleted and view_c added.
- tbl: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
- "view_a", "view_b", "view_c"),
- def: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
- "view_a|def_mod_a", "view_c|def_c"),
- stmt: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_a|create_view_mod_a|utf8|utf8_general_ci"),
- sqltypes.MakeTestResult(showCreateViewFields, "view_c|create_view_c|utf8|utf8_general_ci")},
- expTbl: []string{"view_a", "view_b", "view_c"},
- expDefQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_a', 'view_b', 'view_c')",
- expStmtQuery: []string{"show create table view_a", "show create table view_c"},
- expClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_a', 'view_b', 'view_c')",
- expInsertQuery: []string{
- "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_a', 'create_view_mod_a', 'def_mod_a')",
- "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_c', 'create_view_c', 'def_c')",
+ {
+ // view_a modified, view_b deleted and view_c added.
+ showTablesWithSizesOutput: sqltypes.MakeTestResult(showTableSizesFields, "view_c|VIEW|98732432||123|123", "view_a|VIEW|12345678||123|123"),
+ detectViewChangeOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
+ "view_a", "view_b", "view_c"),
+ viewDefinitionsOutput: sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|text"),
+ "view_a|def_mod_a", "view_c|def_c"),
+ createStmtOutput: []*sqltypes.Result{sqltypes.MakeTestResult(showCreateViewFields, "view_a|create_view_mod_a|utf8|utf8_general_ci"),
+ sqltypes.MakeTestResult(showCreateViewFields, "view_c|create_view_c|utf8|utf8_general_ci")},
+ expViewsChanged: []string{"view_a", "view_b", "view_c"},
+ expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_b', 'view_c', 'view_a')",
+ expCreateStmtQuery: []string{"show create table view_a", "show create table view_c"},
+ expClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_b', 'view_c', 'view_a')",
+ expHsClearQuery: "delete from _vt.schema_engine_views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_b', 'view_c', 'view_a')",
+ expInsertQuery: []string{
+ "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_a', 'create_view_mod_a', 'def_mod_a')",
+ "insert into _vt.views(table_schema, table_name, create_statement, view_definition) values (database(), 'view_c', 'create_view_c', 'def_c')",
+ "insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_a', 'create_view_mod_a', 'def_mod_a')",
+ "insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_c', 'create_view_c', 'def_c')",
+ },
},
- }}
+ }
// setting first test case result.
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, tcases[0].tbl)
- db.AddQuery(tcases[0].expDefQuery, tcases[0].def)
- for idx, stmt := range tcases[0].stmt {
- db.AddQuery(tcases[0].expStmtQuery[idx], stmt)
+ db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*", tcases[0].showTablesWithSizesOutput)
+ db.AddQueryPattern(".*SELECT table_name, view_definition.*schema_engine_views.*", tcases[0].detectViewChangeOutput)
+
+ db.AddQuery(tcases[0].expGetViewDefinitionsQuery, tcases[0].viewDefinitionsOutput)
+ for idx := range tcases[0].expCreateStmtQuery {
+ db.AddQuery(tcases[0].expCreateStmtQuery[idx], tcases[0].createStmtOutput[idx])
+ }
+ for idx := range tcases[0].expInsertQuery {
db.AddQuery(tcases[0].expInsertQuery[idx], &sqltypes.Result{})
}
db.AddQuery(tcases[0].expClearQuery, &sqltypes.Result{})
+ db.AddQuery(tcases[0].expHsClearQuery, &sqltypes.Result{})
var tcCount atomic.Int32
ch := make(chan struct{})
@@ -434,10 +498,11 @@ func TestReloadView(t *testing.T) {
hs.Stream(ctx, func(response *querypb.StreamHealthResponse) error {
if response.RealtimeStats.ViewSchemaChanged != nil {
sort.Strings(response.RealtimeStats.ViewSchemaChanged)
- assert.Equal(t, tcases[tcCount.Load()].expTbl, response.RealtimeStats.ViewSchemaChanged)
+ assert.Equal(t, tcases[tcCount.Load()].expViewsChanged, response.RealtimeStats.ViewSchemaChanged)
tcCount.Add(1)
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, &sqltypes.Result{})
+ db.AddQueryPattern(".*SELECT table_name, view_definition.*schema_engine_views.*", &sqltypes.Result{})
ch <- struct{}{}
+ require.NoError(t, db.LastError())
}
return nil
})
@@ -450,18 +515,21 @@ func TestReloadView(t *testing.T) {
return
}
idx := tcCount.Load()
- db.AddQuery(tcases[idx].expDefQuery, tcases[idx].def)
- for i, stmt := range tcases[idx].stmt {
- db.AddQuery(tcases[idx].expStmtQuery[i], stmt)
+ db.AddQuery(tcases[idx].expGetViewDefinitionsQuery, tcases[idx].viewDefinitionsOutput)
+ for i := range tcases[idx].expCreateStmtQuery {
+ db.AddQuery(tcases[idx].expCreateStmtQuery[i], tcases[idx].createStmtOutput[i])
+ }
+ for i := range tcases[idx].expInsertQuery {
db.AddQuery(tcases[idx].expInsertQuery[i], &sqltypes.Result{})
}
db.AddQuery(tcases[idx].expClearQuery, &sqltypes.Result{})
- db.AddQuery(sqlparser.BuildParsedQuery(mysql.DetectViewChange, sidecardb.GetIdentifier()).Query, tcases[idx].tbl)
+ db.AddQuery(tcases[idx].expHsClearQuery, &sqltypes.Result{})
+ db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*", tcases[idx].showTablesWithSizesOutput)
+ db.AddQueryPattern(".*SELECT table_name, view_definition.*schema_engine_views.*", tcases[idx].detectViewChangeOutput)
case <-time.After(10 * time.Second):
t.Fatalf("timed out")
}
}
-
}
func testStream(hs *healthStreamer) (<-chan *querypb.StreamHealthResponse, context.CancelFunc) {
diff --git a/go/vt/vttablet/tabletserver/messager/engine.go b/go/vt/vttablet/tabletserver/messager/engine.go
index 2d7fdf2bb82..d9072c83fb5 100644
--- a/go/vt/vttablet/tabletserver/messager/engine.go
+++ b/go/vt/vttablet/tabletserver/messager/engine.go
@@ -83,7 +83,7 @@ func (me *Engine) Open() {
log.Info("Messager: opening")
// Unlock before invoking RegisterNotifier because it
// obtains the same lock.
- me.se.RegisterNotifier("messages", me.schemaChanged)
+ me.se.RegisterNotifier("messages", me.schemaChanged, true)
}
// Close closes the Engine service.
@@ -137,10 +137,11 @@ func (me *Engine) Subscribe(ctx context.Context, name string, send func(*sqltype
return mm.Subscribe(ctx, send), nil
}
-func (me *Engine) schemaChanged(tables map[string]*schema.Table, created, altered, dropped []string) {
+func (me *Engine) schemaChanged(tables map[string]*schema.Table, created, altered, dropped []*schema.Table) {
me.mu.Lock()
defer me.mu.Unlock()
- for _, name := range append(dropped, altered...) {
+ for _, table := range append(dropped, altered...) {
+ name := table.Name.String()
mm := me.managers[name]
if mm == nil {
continue
@@ -150,8 +151,8 @@ func (me *Engine) schemaChanged(tables map[string]*schema.Table, created, altere
delete(me.managers, name)
}
- for _, name := range append(created, altered...) {
- t := tables[name]
+ for _, t := range append(created, altered...) {
+ name := t.Name.String()
if t.Type != schema.Message {
continue
}
diff --git a/go/vt/vttablet/tabletserver/messager/engine_test.go b/go/vt/vttablet/tabletserver/messager/engine_test.go
index 31d91b1c66e..e134a6fbe21 100644
--- a/go/vt/vttablet/tabletserver/messager/engine_test.go
+++ b/go/vt/vttablet/tabletserver/messager/engine_test.go
@@ -24,68 +24,76 @@ import (
"vitess.io/vitess/go/mysql/fakesqldb"
"vitess.io/vitess/go/sqltypes"
vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
+ "vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vterrors"
"vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
)
-var meTable = &schema.Table{
- Type: schema.Message,
- MessageInfo: newMMTable().MessageInfo,
-}
+var (
+ meTableT1 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: schema.Message,
+ MessageInfo: newMMTable().MessageInfo,
+ }
+ meTableT2 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: schema.Message,
+ MessageInfo: newMMTable().MessageInfo,
+ }
+ meTableT3 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t3"),
+ Type: schema.Message,
+ MessageInfo: newMMTable().MessageInfo,
+ }
+ meTableT4 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t4"),
+ Type: schema.Message,
+ MessageInfo: newMMTable().MessageInfo,
+ }
+
+ tableT2 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: schema.NoType,
+ }
+ tableT4 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t4"),
+ Type: schema.NoType,
+ }
+ tableT5 = &schema.Table{
+ Name: sqlparser.NewIdentifierCS("t5"),
+ Type: schema.NoType,
+ }
+)
func TestEngineSchemaChanged(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
engine := newTestEngine(db)
defer engine.Close()
- tables := map[string]*schema.Table{
- "t1": meTable,
- "t2": {
- Type: schema.NoType,
- },
- }
- engine.schemaChanged(tables, []string{"t1", "t2"}, nil, nil)
+
+ engine.schemaChanged(nil, []*schema.Table{meTableT1, tableT2}, nil, nil)
got := extractManagerNames(engine.managers)
want := map[string]bool{"t1": true}
if !reflect.DeepEqual(got, want) {
t.Errorf("got: %+v, want %+v", got, want)
}
- tables = map[string]*schema.Table{
- "t1": meTable,
- "t2": {
- Type: schema.NoType,
- },
- "t3": meTable,
- }
- engine.schemaChanged(tables, []string{"t3"}, nil, nil)
+
+ engine.schemaChanged(nil, []*schema.Table{meTableT3}, nil, nil)
got = extractManagerNames(engine.managers)
want = map[string]bool{"t1": true, "t3": true}
if !reflect.DeepEqual(got, want) {
t.Errorf("got: %+v, want %+v", got, want)
}
- tables = map[string]*schema.Table{
- "t1": meTable,
- "t2": {
- Type: schema.NoType,
- },
- "t4": meTable,
- }
- engine.schemaChanged(tables, []string{"t4"}, nil, []string{"t3", "t5"})
+
+ engine.schemaChanged(nil, []*schema.Table{meTableT4}, nil, []*schema.Table{meTableT3, tableT5})
got = extractManagerNames(engine.managers)
want = map[string]bool{"t1": true, "t4": true}
if !reflect.DeepEqual(got, want) {
t.Errorf("got: %+v, want %+v", got, want)
}
// Test update
- tables = map[string]*schema.Table{
- "t1": meTable,
- "t2": meTable,
- "t4": {
- Type: schema.NoType,
- },
- }
- engine.schemaChanged(tables, nil, []string{"t2", "t4"}, nil)
+ engine.schemaChanged(nil, nil, []*schema.Table{meTableT2, tableT4}, nil)
got = extractManagerNames(engine.managers)
want = map[string]bool{"t1": true, "t2": true}
if !reflect.DeepEqual(got, want) {
@@ -105,11 +113,7 @@ func TestSubscribe(t *testing.T) {
db := fakesqldb.New(t)
defer db.Close()
engine := newTestEngine(db)
- tables := map[string]*schema.Table{
- "t1": meTable,
- "t2": meTable,
- }
- engine.schemaChanged(tables, []string{"t1", "t2"}, nil, nil)
+ engine.schemaChanged(nil, []*schema.Table{meTableT1, meTableT2}, nil, nil)
f1, ch1 := newEngineReceiver()
f2, ch2 := newEngineReceiver()
// Each receiver is subscribed to different managers.
@@ -142,9 +146,7 @@ func TestEngineGenerate(t *testing.T) {
defer db.Close()
engine := newTestEngine(db)
defer engine.Close()
- engine.schemaChanged(map[string]*schema.Table{
- "t1": meTable,
- }, []string{"t1"}, nil, nil)
+ engine.schemaChanged(nil, []*schema.Table{meTableT1}, nil, nil)
if _, err := engine.GetGenerator("t1"); err != nil {
t.Error(err)
diff --git a/go/vt/vttablet/tabletserver/query_engine.go b/go/vt/vttablet/tabletserver/query_engine.go
index 476eb4c9f4f..5722b95003f 100644
--- a/go/vt/vttablet/tabletserver/query_engine.go
+++ b/go/vt/vttablet/tabletserver/query_engine.go
@@ -299,7 +299,7 @@ func (qe *QueryEngine) Open() error {
}
qe.streamConns.Open(qe.env.Config().DB.AppWithDB(), qe.env.Config().DB.DbaWithDB(), qe.env.Config().DB.AppDebugWithDB())
- qe.se.RegisterNotifier("qe", qe.schemaChanged)
+ qe.se.RegisterNotifier("qe", qe.schemaChanged, true)
qe.isOpen = true
return nil
}
@@ -436,7 +436,7 @@ func (qe *QueryEngine) IsMySQLReachable() error {
return nil
}
-func (qe *QueryEngine) schemaChanged(tables map[string]*schema.Table, created, altered, dropped []string) {
+func (qe *QueryEngine) schemaChanged(tables map[string]*schema.Table, created, altered, dropped []*schema.Table) {
qe.mu.Lock()
defer qe.mu.Unlock()
qe.tables = tables
diff --git a/go/vt/vttablet/tabletserver/schema/db.go b/go/vt/vttablet/tabletserver/schema/db.go
new file mode 100644
index 00000000000..89d794e9ffa
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/schema/db.go
@@ -0,0 +1,374 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package schema
+
+import (
+ "context"
+
+ "vitess.io/vitess/go/sqltypes"
+ querypb "vitess.io/vitess/go/vt/proto/query"
+ "vitess.io/vitess/go/vt/sidecardb"
+ "vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
+)
+
+const (
+ // insertTableIntoSchemaEngineTables inserts a record in the datastore for the schema-engine tables.
+ insertTableIntoSchemaEngineTables = `INSERT INTO %s.schema_engine_tables(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME)
+values (database(), :table_name, :create_statement, :create_time)`
+
+ // deleteFromSchemaEngineTablesTable removes the tables from the table that have been modified.
+ deleteFromSchemaEngineTablesTable = `DELETE FROM %s.schema_engine_tables WHERE TABLE_SCHEMA = database() AND TABLE_NAME IN ::tableNames`
+
+ // readTableCreateTimes reads the tables create times
+ readTableCreateTimes = `SELECT TABLE_NAME, CREATE_TIME FROM %s.schema_engine_tables`
+
+ // detectViewChange query detects if there is any view change from previous copy.
+ detectViewChange = `
+SELECT distinct table_name
+FROM (
+ SELECT table_name, view_definition
+ FROM information_schema.views
+ WHERE table_schema = database()
+
+ UNION ALL
+
+ SELECT table_name, view_definition
+ FROM %s.schema_engine_views
+ WHERE table_schema = database()
+) _inner
+GROUP BY table_name, view_definition
+HAVING COUNT(*) = 1
+`
+
+ // insertViewIntoSchemaEngineViews using information_schema.views.
+ insertViewIntoSchemaEngineViews = `INSERT INTO %s.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION)
+values (database(), :view_name, :create_statement, :view_definition)`
+
+ // deleteFromSchemaEngineViewsTable removes the views from the table that have been modified.
+ deleteFromSchemaEngineViewsTable = `DELETE FROM %s.schema_engine_views WHERE TABLE_SCHEMA = database() AND TABLE_NAME IN ::viewNames`
+
+ // fetchViewDefinitions retrieves view definition from information_schema.views table.
+ fetchViewDefinitions = `select table_name, view_definition from information_schema.views
+where table_schema = database() and table_name in ::viewNames`
+
+ // fetchCreateStatement retrieves create statement.
+ fetchCreateStatement = `show create table %s`
+)
+
+// reloadTablesDataInDB reloads teh tables information we have stored in our database we use for schema-tracking.
+func reloadTablesDataInDB(ctx context.Context, conn *connpool.DBConn, tables []*Table, droppedTables []string) error {
+ // No need to do anything if we have no tables to refresh or drop.
+ if len(tables) == 0 && len(droppedTables) == 0 {
+ return nil
+ }
+
+ // Delete all the tables that are dropped or modified.
+ tableNamesToDelete := droppedTables
+ for _, table := range tables {
+ tableNamesToDelete = append(tableNamesToDelete, table.Name.String())
+ }
+ tablesBV, err := sqltypes.BuildBindVariable(tableNamesToDelete)
+ if err != nil {
+ return err
+ }
+ bv := map[string]*querypb.BindVariable{"tableNames": tablesBV}
+
+ // Get the create statements for all the tables that are modified.
+ var createStatements []string
+ for _, table := range tables {
+ cs, err := getCreateStatement(ctx, conn, sqlparser.String(table.Name))
+ if err != nil {
+ return err
+ }
+ createStatements = append(createStatements, cs)
+ }
+
+ // Generate the queries to delete and insert table data.
+ clearTableParsedQuery, err := generateFullQuery(deleteFromSchemaEngineTablesTable)
+ if err != nil {
+ return err
+ }
+ clearTableQuery, err := clearTableParsedQuery.GenerateQuery(bv, nil)
+ if err != nil {
+ return err
+ }
+
+ insertTablesParsedQuery, err := generateFullQuery(insertTableIntoSchemaEngineTables)
+ if err != nil {
+ return err
+ }
+
+ // Reload the tables in a transaction.
+ _, err = conn.Exec(ctx, "begin", 1, false)
+ if err != nil {
+ return err
+ }
+ defer conn.Exec(ctx, "rollback", 1, false)
+
+ _, err = conn.Exec(ctx, clearTableQuery, 1, false)
+ if err != nil {
+ return err
+ }
+
+ for idx, table := range tables {
+ bv["table_name"] = sqltypes.StringBindVariable(table.Name.String())
+ bv["create_statement"] = sqltypes.StringBindVariable(createStatements[idx])
+ bv["create_time"] = sqltypes.Int64BindVariable(table.CreateTime)
+ insertTableQuery, err := insertTablesParsedQuery.GenerateQuery(bv, nil)
+ if err != nil {
+ return err
+ }
+ _, err = conn.Exec(ctx, insertTableQuery, 1, false)
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = conn.Exec(ctx, "commit", 1, false)
+ return err
+}
+
+// generateFullQuery generates the full query from the query as a string.
+func generateFullQuery(query string) (*sqlparser.ParsedQuery, error) {
+ stmt, err := sqlparser.Parse(
+ sqlparser.BuildParsedQuery(query, sidecardb.GetIdentifier()).Query)
+ if err != nil {
+ return nil, err
+ }
+ buf := sqlparser.NewTrackedBuffer(nil)
+ stmt.Format(buf)
+ return buf.ParsedQuery(), nil
+}
+
+// reloadViewsDataInDB reloads teh views information we have stored in our database we use for schema-tracking.
+func reloadViewsDataInDB(ctx context.Context, conn *connpool.DBConn, views []*Table, droppedViews []string) error {
+ // No need to do anything if we have no views to refresh or drop.
+ if len(views) == 0 && len(droppedViews) == 0 {
+ return nil
+ }
+
+ // Delete all the views that are dropped or modified.
+ viewNamesToDelete := droppedViews
+ for _, view := range views {
+ viewNamesToDelete = append(viewNamesToDelete, view.Name.String())
+ }
+ viewsBV, err := sqltypes.BuildBindVariable(viewNamesToDelete)
+ if err != nil {
+ return err
+ }
+ bv := map[string]*querypb.BindVariable{"viewNames": viewsBV}
+
+ // Get the create statements for all the views that are modified.
+ var createStatements []string
+ for _, view := range views {
+ cs, err := getCreateStatement(ctx, conn, sqlparser.String(view.Name))
+ if err != nil {
+ return err
+ }
+ createStatements = append(createStatements, cs)
+ }
+
+ // Get the view definitions for all the views that are modified.
+ // We only need to run this if we have any views to reload.
+ viewDefinitions := make(map[string]string)
+ if len(views) > 0 {
+ err = getViewDefinition(ctx, conn, bv,
+ func(qr *sqltypes.Result) error {
+ for _, row := range qr.Rows {
+ viewDefinitions[row[0].ToString()] = row[1].ToString()
+ }
+ return nil
+ },
+ func() *sqltypes.Result { return &sqltypes.Result{} },
+ 1000,
+ )
+ if err != nil {
+ return err
+ }
+ }
+
+ // Generate the queries to delete and insert view data.
+ clearViewParsedQuery, err := generateFullQuery(deleteFromSchemaEngineViewsTable)
+ if err != nil {
+ return err
+ }
+ clearViewQuery, err := clearViewParsedQuery.GenerateQuery(bv, nil)
+ if err != nil {
+ return err
+ }
+
+ insertViewsParsedQuery, err := generateFullQuery(insertViewIntoSchemaEngineViews)
+ if err != nil {
+ return err
+ }
+
+ // Reload the views in a transaction.
+ _, err = conn.Exec(ctx, "begin", 1, false)
+ if err != nil {
+ return err
+ }
+ defer conn.Exec(ctx, "rollback", 1, false)
+
+ _, err = conn.Exec(ctx, clearViewQuery, 1, false)
+ if err != nil {
+ return err
+ }
+
+ for idx, view := range views {
+ bv["view_name"] = sqltypes.StringBindVariable(view.Name.String())
+ bv["create_statement"] = sqltypes.StringBindVariable(createStatements[idx])
+ bv["view_definition"] = sqltypes.StringBindVariable(viewDefinitions[view.Name.String()])
+ insertViewQuery, err := insertViewsParsedQuery.GenerateQuery(bv, nil)
+ if err != nil {
+ return err
+ }
+ _, err = conn.Exec(ctx, insertViewQuery, 1, false)
+ if err != nil {
+ return err
+ }
+ }
+
+ _, err = conn.Exec(ctx, "commit", 1, false)
+ return err
+}
+
+// getViewDefinition gets the viewDefinition for the given views.
+func getViewDefinition(ctx context.Context, conn *connpool.DBConn, bv map[string]*querypb.BindVariable, callback func(qr *sqltypes.Result) error, alloc func() *sqltypes.Result, bufferSize int) error {
+ viewsDefParsedQuery, err := generateFullQuery(fetchViewDefinitions)
+ if err != nil {
+ return err
+ }
+ viewsDefQuery, err := viewsDefParsedQuery.GenerateQuery(bv, nil)
+ if err != nil {
+ return err
+ }
+ return conn.Stream(ctx, viewsDefQuery, callback, alloc, bufferSize, 0)
+}
+
+// getCreateStatement gets the create-statement for the given view/table.
+func getCreateStatement(ctx context.Context, conn *connpool.DBConn, tableName string) (string, error) {
+ res, err := conn.Exec(ctx, sqlparser.BuildParsedQuery(fetchCreateStatement, tableName).Query, 1, false)
+ if err != nil {
+ return "", err
+ }
+ return res.Rows[0][1].ToString(), nil
+}
+
+// getChangedViewNames gets the list of views that have their definitions changed.
+func getChangedViewNames(ctx context.Context, conn *connpool.DBConn, isServingPrimary bool) (map[string]any, error) {
+ /* Retrieve changed views */
+ views := make(map[string]any)
+ if !isServingPrimary {
+ return views, nil
+ }
+ callback := func(qr *sqltypes.Result) error {
+ for _, row := range qr.Rows {
+ view := row[0].ToString()
+ views[view] = true
+ }
+ return nil
+ }
+ alloc := func() *sqltypes.Result { return &sqltypes.Result{} }
+ bufferSize := 1000
+
+ viewChangeQuery := sqlparser.BuildParsedQuery(detectViewChange, sidecardb.GetIdentifier()).Query
+ err := conn.Stream(ctx, viewChangeQuery, callback, alloc, bufferSize, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ return views, nil
+}
+
+// getMismatchedTableNames gets the tables that do not align with the tables information we have in the cache.
+func (se *Engine) getMismatchedTableNames(ctx context.Context, conn *connpool.DBConn, isServingPrimary bool) (map[string]any, error) {
+ tablesMismatched := make(map[string]any)
+ if !isServingPrimary {
+ return tablesMismatched, nil
+ }
+ tablesFound := make(map[string]bool)
+ callback := func(qr *sqltypes.Result) error {
+ // For each row we check 2 things —
+ // 1. If a table exists in our database, but not in the cache, then it could have been dropped.
+ // 2. If the table's create time in our database doesn't match that in our cache, then it could have been altered.
+ for _, row := range qr.Rows {
+ tableName := row[0].ToString()
+ createTime, _ := row[1].ToInt64()
+ tablesFound[tableName] = true
+ table, isFound := se.tables[tableName]
+ if !isFound || table.CreateTime != createTime {
+ tablesMismatched[tableName] = true
+ }
+ }
+ return nil
+ }
+ alloc := func() *sqltypes.Result { return &sqltypes.Result{} }
+ bufferSize := 1000
+ readTableCreateTimesQuery := sqlparser.BuildParsedQuery(readTableCreateTimes, sidecardb.GetIdentifier()).Query
+ err := conn.Stream(ctx, readTableCreateTimesQuery, callback, alloc, bufferSize, 0)
+ if err != nil {
+ return nil, err
+ }
+
+ // Finally, we also check for tables that exist only in the cache, because these tables would have been created.
+ for tableName := range se.tables {
+ if se.tables[tableName].Type == View {
+ continue
+ }
+ // Explicitly ignore dual because schema-engine stores this in its list of tables.
+ if !tablesFound[tableName] && tableName != "dual" {
+ tablesMismatched[tableName] = true
+ }
+ }
+
+ return tablesMismatched, nil
+}
+
+// reloadDataInDB reloads the schema tracking data in the database
+func reloadDataInDB(ctx context.Context, conn *connpool.DBConn, altered []*Table, created []*Table, dropped []*Table) error {
+ // tablesToReload and viewsToReload stores the tables and views that need reloading and storing in our MySQL database.
+ var tablesToReload, viewsToReload []*Table
+ // droppedTables, droppedViews stores the list of tables and views we need to delete, respectively.
+ var droppedTables []string
+ var droppedViews []string
+
+ for _, table := range append(created, altered...) {
+ if table.Type == View {
+ viewsToReload = append(viewsToReload, table)
+ } else {
+ tablesToReload = append(tablesToReload, table)
+ }
+ }
+
+ for _, table := range dropped {
+ tableName := table.Name.String()
+ if table.Type == View {
+ droppedViews = append(droppedViews, tableName)
+ } else {
+ droppedTables = append(droppedTables, tableName)
+ }
+ }
+
+ if err := reloadTablesDataInDB(ctx, conn, tablesToReload, droppedTables); err != nil {
+ return err
+ }
+ if err := reloadViewsDataInDB(ctx, conn, viewsToReload, droppedViews); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/go/vt/vttablet/tabletserver/schema/db_test.go b/go/vt/vttablet/tabletserver/schema/db_test.go
new file mode 100644
index 00000000000..74f4b781566
--- /dev/null
+++ b/go/vt/vttablet/tabletserver/schema/db_test.go
@@ -0,0 +1,896 @@
+/*
+Copyright 2023 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package schema
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "golang.org/x/exp/maps"
+
+ "vitess.io/vitess/go/mysql/fakesqldb"
+ "vitess.io/vitess/go/sqltypes"
+ querypb "vitess.io/vitess/go/vt/proto/query"
+ "vitess.io/vitess/go/vt/sidecardb"
+ "vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
+)
+
+var (
+ tablesBV, _ = sqltypes.BuildBindVariable([]string{"t1", "lead"})
+)
+
+func TestGenerateFullQuery(t *testing.T) {
+ tests := []struct {
+ name string
+ query string
+ bv map[string]*querypb.BindVariable
+ wantQuery string
+ wantErr string
+ }{
+ {
+ name: "No bind variables",
+ query: "select TABLE_NAME, CREATE_TIME from schema_engine_tables",
+ }, {
+ name: "List bind variables",
+ query: "DELETE FROM %s.schema_engine_tables WHERE TABLE_SCHEMA = database() AND TABLE_NAME IN ::tableNames",
+ bv: map[string]*querypb.BindVariable{
+ "tableNames": tablesBV,
+ },
+ wantQuery: "delete from _vt.schema_engine_tables where TABLE_SCHEMA = database() and TABLE_NAME in ('t1', 'lead')",
+ }, {
+ name: "Multiple bind variables",
+ query: "INSERT INTO %s.schema_engine_tables(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), :table_name, :create_statement, :create_time)",
+ bv: map[string]*querypb.BindVariable{
+ "table_name": sqltypes.StringBindVariable("lead"),
+ "create_statement": sqltypes.StringBindVariable("create table `lead`"),
+ "create_time": sqltypes.Int64BindVariable(1),
+ },
+ wantQuery: "insert into _vt.schema_engine_tables(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 'lead', 'create table `lead`', 1)",
+ }, {
+ name: "parser error",
+ query: "insert syntax error",
+ wantErr: "syntax error at position 20 near 'error'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if tt.wantQuery == "" {
+ tt.wantQuery = tt.query
+ }
+
+ got, err := generateFullQuery(tt.query)
+ if tt.wantErr != "" {
+ require.EqualError(t, err, tt.wantErr)
+ return
+ }
+ require.NoError(t, err)
+ finalQuery, err := got.GenerateQuery(tt.bv, nil)
+ require.NoError(t, err)
+ require.Equal(t, tt.wantQuery, finalQuery)
+ })
+ }
+}
+
+func TestGetCreateStatement(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ // Success view
+ createStatement := "CREATE ALGORITHM=UNDEFINED DEFINER=`msandbox`@`localhost` SQL SECURITY DEFINER VIEW `lead` AS select `area`.`id` AS `id` from `area`"
+ db.AddQuery("show create table `lead`", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"),
+ fmt.Sprintf("lead|%v|utf8mb4|utf8mb4_0900_ai_ci", createStatement),
+ ))
+ got, err := getCreateStatement(context.Background(), conn, "`lead`")
+ require.NoError(t, err)
+ require.Equal(t, createStatement, got)
+ require.NoError(t, db.LastError())
+
+ // Success table
+ createStatement = "CREATE TABLE `area` (\n `id` int NOT NULL,\n `name` varchar(30) DEFAULT NULL,\n `zipcode` int DEFAULT NULL,\n `country` int DEFAULT NULL,\n `x` varchar(30) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_as_cs DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"
+ db.AddQuery("show create table area", sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields(" Table | Create Table", "varchar|varchar"),
+ fmt.Sprintf("area|%v", createStatement),
+ ))
+ got, err = getCreateStatement(context.Background(), conn, "area")
+ require.NoError(t, err)
+ require.Equal(t, createStatement, got)
+ require.NoError(t, db.LastError())
+
+ // Failure
+ errMessage := "ERROR 1146 (42S02): Table 'ks.v1' doesn't exist"
+ db.AddRejectedQuery("show create table v1", errors.New(errMessage))
+ got, err = getCreateStatement(context.Background(), conn, "v1")
+ require.ErrorContains(t, err, errMessage)
+ require.Equal(t, "", got)
+}
+
+func TestGetChangedViewNames(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ // Success
+ query := fmt.Sprintf(detectViewChange, sidecardb.GetIdentifier())
+ db.AddQuery(query, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("table_name", "varchar"),
+ "lead",
+ "v1",
+ "v2",
+ ))
+ got, err := getChangedViewNames(context.Background(), conn, true)
+ require.NoError(t, err)
+ require.Len(t, got, 3)
+ require.ElementsMatch(t, maps.Keys(got), []string{"v1", "v2", "lead"})
+ require.NoError(t, db.LastError())
+
+ // Not serving primary
+ got, err = getChangedViewNames(context.Background(), conn, false)
+ require.NoError(t, err)
+ require.Len(t, got, 0)
+ require.NoError(t, db.LastError())
+
+ // Failure
+ errMessage := "ERROR 1146 (42S02): Table '_vt.schema_engine_views' doesn't exist"
+ db.AddRejectedQuery(query, errors.New(errMessage))
+ got, err = getChangedViewNames(context.Background(), conn, true)
+ require.ErrorContains(t, err, errMessage)
+ require.Nil(t, got)
+}
+
+func TestGetViewDefinition(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ viewsBV, err := sqltypes.BuildBindVariable([]string{"v1", "lead"})
+ require.NoError(t, err)
+ bv := map[string]*querypb.BindVariable{"viewNames": viewsBV}
+
+ // Success
+ query := "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v1', 'lead')"
+ db.AddQuery(query, sqltypes.MakeTestResult(
+ sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
+ "v1|create_view_v1",
+ "lead|create_view_lead",
+ ))
+ got, err := collectGetViewDefinitions(conn, bv)
+ require.NoError(t, err)
+ require.Len(t, got, 2)
+ require.ElementsMatch(t, maps.Keys(got), []string{"v1", "lead"})
+ require.Equal(t, "create_view_v1", got["v1"])
+ require.Equal(t, "create_view_lead", got["lead"])
+ require.NoError(t, db.LastError())
+
+ // Failure
+ errMessage := "some error in MySQL"
+ db.AddRejectedQuery(query, errors.New(errMessage))
+ got, err = collectGetViewDefinitions(conn, bv)
+ require.ErrorContains(t, err, errMessage)
+ require.Len(t, got, 0)
+
+ // Failure empty bv
+ bv = nil
+ got, err = collectGetViewDefinitions(conn, bv)
+ require.EqualError(t, err, "missing bind var viewNames")
+ require.Len(t, got, 0)
+}
+
+func collectGetViewDefinitions(conn *connpool.DBConn, bv map[string]*querypb.BindVariable) (map[string]string, error) {
+ viewDefinitions := make(map[string]string)
+ err := getViewDefinition(context.Background(), conn, bv, func(qr *sqltypes.Result) error {
+ for _, row := range qr.Rows {
+ viewDefinitions[row[0].ToString()] = row[1].ToString()
+ }
+ return nil
+ }, func() *sqltypes.Result {
+ return &sqltypes.Result{}
+ }, 1000)
+ return viewDefinitions, err
+}
+
+func TestGetMismatchedTableNames(t *testing.T) {
+ queryFields := sqltypes.MakeTestFields("TABLE_NAME|CREATE_TIME", "varchar|int64")
+
+ testCases := []struct {
+ name string
+ tables map[string]*Table
+ dbData *sqltypes.Result
+ dbError string
+ isServingPrimary bool
+ expectedTableNames []string
+ expectedError string
+ }{
+ {
+ name: "Table create time differs",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t1|2341"),
+ isServingPrimary: true,
+ expectedTableNames: []string{"t1"},
+ }, {
+ name: "Table got deleted",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t1|31234",
+ "t2|2341"),
+ isServingPrimary: true,
+ expectedTableNames: []string{"t2"},
+ }, {
+ name: "Table got created",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ }, "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t1|31234"),
+ isServingPrimary: true,
+ expectedTableNames: []string{"t2"},
+ }, {
+ name: "Dual gets ignored",
+ tables: map[string]*Table{
+ "dual": NewTable("dual", NoType),
+ "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t2|31234"),
+ isServingPrimary: true,
+ expectedTableNames: []string{},
+ }, {
+ name: "All problems",
+ tables: map[string]*Table{
+ "dual": NewTable("dual", NoType),
+ "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t3|31234",
+ "t1|1342"),
+ isServingPrimary: true,
+ expectedTableNames: []string{"t1", "t2", "t3"},
+ }, {
+ name: "Not serving primary",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbData: sqltypes.MakeTestResult(queryFields,
+ "t1|2341"),
+ isServingPrimary: false,
+ expectedTableNames: []string{},
+ }, {
+ name: "Error in query",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 31234,
+ },
+ },
+ dbError: "some error in MySQL",
+ dbData: nil,
+ isServingPrimary: true,
+ expectedError: "some error in MySQL",
+ },
+ }
+
+ query := fmt.Sprintf(readTableCreateTimes, sidecardb.GetIdentifier())
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ if tc.dbError != "" {
+ db.AddRejectedQuery(query, errors.New(tc.dbError))
+ } else {
+ db.AddQuery(query, tc.dbData)
+ }
+ se := &Engine{
+ tables: tc.tables,
+ }
+ mismatchedTableNames, err := se.getMismatchedTableNames(context.Background(), conn, tc.isServingPrimary)
+ if tc.expectedError != "" {
+ require.ErrorContains(t, err, tc.expectedError)
+ } else {
+ require.ElementsMatch(t, maps.Keys(mismatchedTableNames), tc.expectedTableNames)
+ require.NoError(t, db.LastError())
+ }
+ })
+ }
+}
+
+func TestReloadTablesInDB(t *testing.T) {
+ showCreateTableFields := sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar")
+ errMessage := "some error in MySQL"
+ testCases := []struct {
+ name string
+ tablesToReload []*Table
+ tablesToDelete []string
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ expectedError string
+ }{
+ {
+ name: "Only tables to delete",
+ tablesToDelete: []string{"t1", "lead"},
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1', 'lead')": {},
+ },
+ }, {
+ name: "Only tables to reload",
+ tablesToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1', 'lead')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_table_lead"),
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": {},
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 'lead', 'create_table_lead', 1234)": {},
+ },
+ }, {
+ name: "Reload and Delete",
+ tablesToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ tablesToDelete: []string{"t2", "from"},
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t2', 'from', 't1', 'lead')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_table_lead"),
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": {},
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 'lead', 'create_table_lead', 1234)": {},
+ },
+ }, {
+ name: "Error In Insert",
+ tablesToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ },
+ queriesToReject: map[string]error{
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": errors.New(errMessage),
+ },
+ expectedError: errMessage,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ // Add queries with the expected results and errors.
+ for query, result := range tc.expectedQueries {
+ db.AddQuery(query, result)
+ }
+ for query, errorToThrow := range tc.queriesToReject {
+ db.AddRejectedQuery(query, errorToThrow)
+ }
+
+ err = reloadTablesDataInDB(context.Background(), conn, tc.tablesToReload, tc.tablesToDelete)
+ if tc.expectedError != "" {
+ require.ErrorContains(t, err, tc.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ })
+ }
+}
+
+func TestReloadViewsInDB(t *testing.T) {
+ showCreateTableFields := sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar")
+ getViewDefinitionsFields := sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar")
+ errMessage := "some error in MySQL"
+ testCases := []struct {
+ name string
+ viewsToReload []*Table
+ viewsToDelete []string
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ expectedError string
+ }{
+ {
+ name: "Only views to delete",
+ viewsToDelete: []string{"v1", "lead"},
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1', 'lead')": {},
+ },
+ }, {
+ name: "Only views to reload",
+ viewsToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1', 'lead')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v1', 'lead')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "lead|select_lead",
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateTableFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_view_lead|utf8mb4|utf8mb4_0900_ai_ci"),
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": {},
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'lead', 'create_view_lead', 'select_lead')": {},
+ },
+ }, {
+ name: "Reload and delete",
+ viewsToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ viewsToDelete: []string{"v2", "from"},
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "lead|select_lead",
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateTableFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_view_lead|utf8mb4|utf8mb4_0900_ai_ci"),
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": {},
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'lead', 'create_view_lead', 'select_lead')": {},
+ },
+ }, {
+ name: "Error In Insert",
+ viewsToReload: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v1')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateTableFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ },
+ queriesToReject: map[string]error{
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": errors.New(errMessage),
+ },
+ expectedError: errMessage,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ // Add queries with the expected results and errors.
+ for query, result := range tc.expectedQueries {
+ db.AddQuery(query, result)
+ }
+ for query, errorToThrow := range tc.queriesToReject {
+ db.AddRejectedQuery(query, errorToThrow)
+ }
+
+ err = reloadViewsDataInDB(context.Background(), conn, tc.viewsToReload, tc.viewsToDelete)
+ if tc.expectedError != "" {
+ require.ErrorContains(t, err, tc.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ })
+ }
+}
+
+func TestReloadDataInDB(t *testing.T) {
+ showCreateViewFields := sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar")
+ showCreateTableFields := sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar")
+ getViewDefinitionsFields := sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar")
+ errMessage := "some error in MySQL"
+ testCases := []struct {
+ name string
+ altered []*Table
+ created []*Table
+ dropped []*Table
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ expectedError string
+ }{
+ {
+ name: "Only views to delete",
+ dropped: []*Table{
+ NewTable("v1", View),
+ NewTable("lead", View),
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1', 'lead')": {},
+ },
+ }, {
+ name: "Only views to reload",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1', 'lead')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v1', 'lead')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "lead|select_lead",
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateViewFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateViewFields,
+ "lead|create_view_lead|utf8mb4|utf8mb4_0900_ai_ci"),
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": {},
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'lead', 'create_view_lead', 'select_lead')": {},
+ },
+ }, {
+ name: "Reload and delete views",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ dropped: []*Table{
+ NewTable("v2", View),
+ NewTable("from", View),
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "lead|select_lead",
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateViewFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateViewFields,
+ "lead|create_view_lead|utf8mb4|utf8mb4_0900_ai_ci"),
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": {},
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'lead', 'create_view_lead', 'select_lead')": {},
+ },
+ }, {
+ name: "Error In Inserting View Data",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v1')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v1')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateViewFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ },
+ queriesToReject: map[string]error{
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": errors.New(errMessage),
+ },
+ expectedError: errMessage,
+ }, {
+ name: "Only tables to delete",
+ dropped: []*Table{
+ NewTable("t1", NoType),
+ NewTable("lead", NoType),
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1', 'lead')": {},
+ },
+ }, {
+ name: "Only tables to reload",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1', 'lead')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_table_lead"),
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": {},
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 'lead', 'create_table_lead', 1234)": {},
+ },
+ }, {
+ name: "Reload and delete tables",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ dropped: []*Table{
+ NewTable("t2", NoType),
+ NewTable("from", NoType),
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t2', 'from', 't1', 'lead')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateTableFields,
+ "lead|create_table_lead"),
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": {},
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 'lead', 'create_table_lead', 1234)": {},
+ },
+ }, {
+ name: "Error In Inserting Table Data",
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t1')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ },
+ queriesToReject: map[string]error{
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": errors.New(errMessage),
+ },
+ expectedError: errMessage,
+ }, {
+ name: "Reload and delete all",
+ created: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ altered: []*Table{
+ {
+ Name: sqlparser.NewIdentifierCS("lead"),
+ Type: View,
+ CreateTime: 1234,
+ }, {
+ Name: sqlparser.NewIdentifierCS("where"),
+ Type: NoType,
+ CreateTime: 1234,
+ },
+ },
+ dropped: []*Table{
+ NewTable("v2", View),
+ NewTable("from", View),
+ NewTable("t2", NoType),
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ "begin": {},
+ "commit": {},
+ "rollback": {},
+ "delete from _vt.schema_engine_views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": {},
+ "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v2', 'from', 'v1', 'lead')": sqltypes.MakeTestResult(
+ getViewDefinitionsFields,
+ "lead|select_lead",
+ "v1|select_v1"),
+ "show create table v1": sqltypes.MakeTestResult(showCreateViewFields,
+ "v1|create_view_v1|utf8mb4|utf8mb4_0900_ai_ci"),
+ "show create table `lead`": sqltypes.MakeTestResult(showCreateViewFields,
+ "lead|create_view_lead|utf8mb4|utf8mb4_0900_ai_ci"),
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'v1', 'create_view_v1', 'select_v1')": {},
+ "insert into _vt.schema_engine_views(table_schema, table_name, create_statement, view_definition) values (database(), 'lead', 'create_view_lead', 'select_lead')": {},
+ "delete from _vt.schema_engine_tables where table_schema = database() and table_name in ('t2', 't1', 'where')": {},
+ "show create table t1": sqltypes.MakeTestResult(showCreateTableFields,
+ "t1|create_table_t1"),
+ "show create table `where`": sqltypes.MakeTestResult(showCreateTableFields,
+ "where|create_table_where"),
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 't1', 'create_table_t1', 1234)": {},
+ "insert into _vt.schema_engine_tables(table_schema, table_name, create_statement, create_time) values (database(), 'where', 'create_table_where', 1234)": {},
+ },
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ // Add queries with the expected results and errors.
+ for query, result := range tc.expectedQueries {
+ db.AddQuery(query, result)
+ }
+ for query, errorToThrow := range tc.queriesToReject {
+ db.AddRejectedQuery(query, errorToThrow)
+ }
+
+ err = reloadDataInDB(context.Background(), conn, tc.altered, tc.created, tc.dropped)
+ if tc.expectedError != "" {
+ require.ErrorContains(t, err, tc.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ })
+ }
+}
diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go
index 5242f17812e..1f5a7218085 100644
--- a/go/vt/vttablet/tabletserver/schema/engine.go
+++ b/go/vt/vttablet/tabletserver/schema/engine.go
@@ -25,6 +25,8 @@ import (
"sync"
"time"
+ "golang.org/x/exp/maps"
+
"vitess.io/vitess/go/sqltypes"
"vitess.io/vitess/go/vt/sidecardb"
@@ -51,7 +53,7 @@ import (
const maxTableCount = 10000
-type notifier func(full map[string]*Table, created, altered, dropped []string)
+type notifier func(full map[string]*Table, created, altered, dropped []*Table)
// Engine stores the schema info and performs operations that
// keep itself up-to-date.
@@ -64,19 +66,24 @@ type Engine struct {
isOpen bool
tables map[string]*Table
lastChange int64
- reloadTime time.Duration
//the position at which the schema was last loaded. it is only used in conjunction with ReloadAt
reloadAtPos mysql.Position
notifierMu sync.Mutex
notifiers map[string]notifier
+ // isServingPrimary stores if this tablet is currently the serving primary or not.
+ isServingPrimary bool
+ // schemaCopy stores if the user has requested signals on schema changes. If they have, then we
+ // also track the underlying schema and make a copy of it in our MySQL instance.
+ schemaCopy bool
// SkipMetaCheck skips the metadata about the database and table information
SkipMetaCheck bool
historian *historian
- conns *connpool.Pool
- ticks *timer.Timer
+ conns *connpool.Pool
+ ticks *timer.Timer
+ reloadTimeout time.Duration
// dbCreationFailed is for preventing log spam.
dbCreationFailed bool
@@ -98,15 +105,15 @@ func NewEngine(env tabletenv.Env) *Engine {
Size: 3,
IdleTimeoutSeconds: env.Config().OltpReadPool.IdleTimeoutSeconds,
}),
- ticks: timer.NewTimer(reloadTime),
- reloadTime: reloadTime,
+ ticks: timer.NewTimer(reloadTime),
}
+ se.schemaCopy = env.Config().SignalWhenSchemaChange
_ = env.Exporter().NewGaugeDurationFunc("SchemaReloadTime", "vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.", se.ticks.Interval)
se.tableFileSizeGauge = env.Exporter().NewGaugesWithSingleLabel("TableFileSize", "tracks table file size", "Table")
se.tableAllocatedSizeGauge = env.Exporter().NewGaugesWithSingleLabel("TableAllocatedSize", "tracks table allocated size", "Table")
se.innoDbReadRowsCounter = env.Exporter().NewCounter("InnodbRowsRead", "number of rows read by mysql")
se.SchemaReloadTimings = env.Exporter().NewTimings("SchemaReload", "time taken to reload the schema", "type")
-
+ se.reloadTimeout = env.Config().SchemaChangeReloadTimeout
env.Exporter().HandleFunc("/debug/schema", se.handleDebugSchema)
env.Exporter().HandleFunc("/schemaz", func(w http.ResponseWriter, r *http.Request) {
// Ensure schema engine is Open. If vttablet came up in a non_serving role,
@@ -241,7 +248,7 @@ func (se *Engine) Open() error {
}()
se.tables = map[string]*Table{
- "dual": NewTable("dual"),
+ "dual": NewTable("dual", NoType),
}
se.notifiers = make(map[string]notifier)
@@ -318,6 +325,7 @@ func (se *Engine) MakeNonPrimary() {
// This function is tested through endtoend test.
se.mu.Lock()
defer se.mu.Unlock()
+ se.isServingPrimary = false
for _, t := range se.tables {
if t.SequenceInfo != nil {
t.SequenceInfo.Lock()
@@ -328,6 +336,14 @@ func (se *Engine) MakeNonPrimary() {
}
}
+// MakePrimary tells the schema engine that the current tablet is now the primary,
+// so it can read and write to the MySQL instance for schema-tracking.
+func (se *Engine) MakePrimary(serving bool) {
+ se.mu.Lock()
+ defer se.mu.Unlock()
+ se.isServingPrimary = serving
+}
+
// EnableHistorian forces tracking to be on or off.
// Only used for testing.
func (se *Engine) EnableHistorian(enabled bool) error {
@@ -382,6 +398,15 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
se.SchemaReloadTimings.Record("SchemaReload", start)
}()
+ // if this flag is set, then we don't need table meta information
+ if se.SkipMetaCheck {
+ return nil
+ }
+
+ // add a timeout to prevent unbounded waits
+ ctx, cancel := context.WithTimeout(ctx, se.reloadTimeout)
+ defer cancel()
+
conn, err := se.conns.Get(ctx, nil)
if err != nil {
return err
@@ -393,20 +418,26 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
if err != nil {
return err
}
- // if this flag is set, then we don't need table meta information
- if se.SkipMetaCheck {
- return nil
+
+ tableData, err := getTableData(ctx, conn, includeStats)
+ if err != nil {
+ return vterrors.Wrapf(err, "in Engine.reload(), reading tables")
}
+ // On the primary tablet, we also check the data we have stored in our schema tables to see what all needs reloading.
+ shouldUseDatabase := se.isServingPrimary && se.schemaCopy
- var showTablesQuery string
- if includeStats {
- showTablesQuery = conn.BaseShowTablesWithSizes()
- } else {
- showTablesQuery = conn.BaseShowTables()
+ // changedViews are the views that have changed. We can't use the same createTime logic for views because, MySQL
+ // doesn't update the create_time field for views when they are altered. This is annoying, but something we have to work around.
+ changedViews, err := getChangedViewNames(ctx, conn, shouldUseDatabase)
+ if err != nil {
+ return err
}
- tableData, err := conn.Exec(ctx, showTablesQuery, maxTableCount, false)
+ // mismatchTables stores the tables whose createTime in our cache doesn't match the createTime stored in the database.
+ // This can happen if a primary crashed right after a DML succeeded, before it could reload its state. If all the replicas
+ // are able to reload their cache before one of them is promoted, then the database information would be out of sync.
+ mismatchTables, err := se.getMismatchedTableNames(ctx, conn, shouldUseDatabase)
if err != nil {
- return vterrors.Wrapf(err, "in Engine.reload(), reading tables")
+ return err
}
err = se.updateInnoDBRowsRead(ctx, conn)
@@ -420,7 +451,7 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
// changedTables keeps track of tables that have changed so we can reload their pk info.
changedTables := make(map[string]*Table)
// created and altered contain the names of created and altered tables for broadcast.
- var created, altered []string
+ var created, altered []*Table
for _, row := range tableData.Rows {
tableName := row[0].ToString()
curTables[tableName] = true
@@ -445,8 +476,18 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
// renamed to the table being altered. `se.lastChange` is updated every time the schema is reloaded (default: 30m).
// Online DDL can take hours. So it is possible that the `create_time` of the temporary table is before se.lastChange. Hence,
// #1 will not identify the renamed table as a changed one.
+ //
+ // 3. A table's create_time in our database doesn't match the create_time in the cache. This can happen if a primary crashed right after a DML succeeded,
+ // before it could reload its state. If all the replicas are able to reload their cache before one of them is promoted,
+ // then the database information would be out of sync. We check this by consulting the mismatchTables map.
+ //
+ // 4. A view's definition has changed. We can't use the same createTime logic for views because, MySQL
+ // doesn't update the create_time field for views when they are altered. This is annoying, but something we have to work around.
+ // We check this by consulting the changedViews map.
tbl, isInTablesMap := se.tables[tableName]
- if isInTablesMap && createTime == tbl.CreateTime && createTime < se.lastChange {
+ _, isInChangedViewMap := changedViews[tableName]
+ _, isInMismatchTableMap := mismatchTables[tableName]
+ if isInTablesMap && createTime == tbl.CreateTime && createTime < se.lastChange && !isInChangedViewMap && !isInMismatchTableMap {
if includeStats {
tbl.FileSize = fileSize
tbl.AllocatedSize = allocatedSize
@@ -455,7 +496,7 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
}
log.V(2).Infof("Reading schema for table: %s", tableName)
- table, err := LoadTable(conn, se.cp.DBName(), tableName, row[3].ToString())
+ table, err := LoadTable(conn, se.cp.DBName(), tableName, row[1].String(), row[3].ToString())
if err != nil {
rec.RecordError(vterrors.Wrapf(err, "in Engine.reload(), reading table %s", tableName))
continue
@@ -467,45 +508,91 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error {
table.CreateTime = createTime
changedTables[tableName] = table
if isInTablesMap {
- altered = append(altered, tableName)
+ altered = append(altered, table)
} else {
- created = append(created, tableName)
+ created = append(created, table)
}
}
if rec.HasErrors() {
return rec.Error()
}
- // Compute and handle dropped tables.
- var dropped []string
- for tableName := range se.tables {
- if !curTables[tableName] {
- dropped = append(dropped, tableName)
- delete(se.tables, tableName)
- // We can't actually delete the label from the stats, but we can set it to 0.
- // Many monitoring tools will drop zero-valued metrics.
- se.tableFileSizeGauge.Reset(tableName)
- se.tableAllocatedSizeGauge.Reset(tableName)
- }
- }
+ dropped := se.getDroppedTables(curTables, changedViews, mismatchTables)
// Populate PKColumns for changed tables.
if err := se.populatePrimaryKeys(ctx, conn, changedTables); err != nil {
return err
}
+ // If this tablet is the primary and schema tracking is required, we should reload the information in our database.
+ if shouldUseDatabase {
+ // If reloadDataInDB succeeds, then we don't want to prevent sending the broadcast notification.
+ // So, we do this step in the end when we can receive no more errors that fail the reload operation.
+ err = reloadDataInDB(ctx, conn, altered, created, dropped)
+ if err != nil {
+ log.Errorf("error in updating schema information in Engine.reload() - %v", err)
+ }
+ }
+
// Update se.tables
for k, t := range changedTables {
se.tables[k] = t
}
se.lastChange = curTime
if len(created) > 0 || len(altered) > 0 || len(dropped) > 0 {
- log.Infof("schema engine created %v, altered %v, dropped %v", created, altered, dropped)
+ log.Infof("schema engine created %v, altered %v, dropped %v", extractNamesFromTablesList(created), extractNamesFromTablesList(altered), extractNamesFromTablesList(dropped))
}
se.broadcast(created, altered, dropped)
return nil
}
+func (se *Engine) getDroppedTables(curTables map[string]bool, changedViews map[string]any, mismatchTables map[string]any) []*Table {
+ // Compute and handle dropped tables.
+ dropped := make(map[string]*Table)
+ for tableName, table := range se.tables {
+ if !curTables[tableName] {
+ dropped[tableName] = table
+ delete(se.tables, tableName)
+ // We can't actually delete the label from the stats, but we can set it to 0.
+ // Many monitoring tools will drop zero-valued metrics.
+ se.tableFileSizeGauge.Reset(tableName)
+ se.tableAllocatedSizeGauge.Reset(tableName)
+ }
+ }
+
+ // If we have a view that has changed, but doesn't exist in the current list of tables,
+ // then it was dropped before, and we were unable to update our database. So, we need to signal its
+ // drop again.
+ for viewName := range changedViews {
+ _, alreadyExists := dropped[viewName]
+ if !curTables[viewName] && !alreadyExists {
+ dropped[viewName] = NewTable(viewName, View)
+ }
+ }
+
+ // If we have a table that has a mismatch, but doesn't exist in the current list of tables,
+ // then it was dropped before, and we were unable to update our database. So, we need to signal its
+ // drop again.
+ for tableName := range mismatchTables {
+ _, alreadyExists := dropped[tableName]
+ if !curTables[tableName] && !alreadyExists {
+ dropped[tableName] = NewTable(tableName, NoType)
+ }
+ }
+
+ return maps.Values(dropped)
+}
+
+func getTableData(ctx context.Context, conn *connpool.DBConn, includeStats bool) (*sqltypes.Result, error) {
+ var showTablesQuery string
+ if includeStats {
+ showTablesQuery = conn.BaseShowTablesWithSizes()
+ } else {
+ showTablesQuery = conn.BaseShowTables()
+ }
+ return conn.Exec(ctx, showTablesQuery, maxTableCount, false)
+}
+
func (se *Engine) updateInnoDBRowsRead(ctx context.Context, conn *connpool.DBConn) error {
readRowsData, err := conn.Exec(ctx, mysql.ShowRowsRead, 10, false)
if err != nil {
@@ -599,7 +686,7 @@ func (se *Engine) GetTableForPos(tableName sqlparser.IdentifierCS, gtid string)
// It also causes an immediate notification to the caller. The notified
// function must not change the map or its contents. The only exception
// is the sequence table where the values can be changed using the lock.
-func (se *Engine) RegisterNotifier(name string, f notifier) {
+func (se *Engine) RegisterNotifier(name string, f notifier, runNotifier bool) {
if !se.isOpen {
return
}
@@ -608,11 +695,13 @@ func (se *Engine) RegisterNotifier(name string, f notifier) {
defer se.notifierMu.Unlock()
se.notifiers[name] = f
- var created []string
- for tableName := range se.tables {
- created = append(created, tableName)
+ var created []*Table
+ for _, table := range se.tables {
+ created = append(created, table)
+ }
+ if runNotifier {
+ f(se.tables, created, nil, nil)
}
- f(se.tables, created, nil, nil)
}
// UnregisterNotifier unregisters the notifier function.
@@ -632,7 +721,7 @@ func (se *Engine) UnregisterNotifier(name string) {
}
// broadcast must be called while holding a lock on se.mu.
-func (se *Engine) broadcast(created, altered, dropped []string) {
+func (se *Engine) broadcast(created, altered, dropped []*Table) {
if !se.isOpen {
return
}
@@ -724,3 +813,11 @@ func (se *Engine) SetTableForTests(table *Table) {
func (se *Engine) GetDBConnector() dbconfigs.Connector {
return se.cp
}
+
+func extractNamesFromTablesList(tables []*Table) []string {
+ var tableNames []string
+ for _, table := range tables {
+ tableNames = append(tableNames, table.Name.String())
+ }
+ return tableNames
+}
diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go
index e14896ce3e5..c2d2237b3a4 100644
--- a/go/vt/vttablet/tabletserver/schema/engine_test.go
+++ b/go/vt/vttablet/tabletserver/schema/engine_test.go
@@ -18,6 +18,7 @@ package schema
import (
"context"
+ "errors"
"expvar"
"fmt"
"net/http"
@@ -27,16 +28,18 @@ import (
"testing"
"time"
- "vitess.io/vitess/go/test/utils"
-
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"vitess.io/vitess/go/mysql"
"vitess.io/vitess/go/mysql/fakesqldb"
"vitess.io/vitess/go/sqltypes"
+ "vitess.io/vitess/go/stats"
+ "vitess.io/vitess/go/test/utils"
"vitess.io/vitess/go/vt/dbconfigs"
+ "vitess.io/vitess/go/vt/sidecardb"
"vitess.io/vitess/go/vt/sqlparser"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
"vitess.io/vitess/go/vt/vttablet/tabletserver/schema/schematest"
"vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv"
@@ -148,21 +151,21 @@ func TestOpenAndReload(t *testing.T) {
AddFakeInnoDBReadRowsResult(db, secondReadRowsValue)
firstTime := true
- notifier := func(full map[string]*Table, created, altered, dropped []string) {
+ notifier := func(full map[string]*Table, created, altered, dropped []*Table) {
if firstTime {
firstTime = false
- sort.Strings(created)
- assert.Equal(t, []string{"dual", "msg", "seq", "test_table_01", "test_table_02", "test_table_03"}, created)
- assert.Equal(t, []string(nil), altered)
- assert.Equal(t, []string(nil), dropped)
+ createTables := extractNamesFromTablesList(created)
+ sort.Strings(createTables)
+ assert.Equal(t, []string{"dual", "msg", "seq", "test_table_01", "test_table_02", "test_table_03"}, createTables)
+ assert.Equal(t, []*Table(nil), altered)
+ assert.Equal(t, []*Table(nil), dropped)
} else {
- assert.Equal(t, []string{"test_table_04"}, created)
- assert.Equal(t, []string{"test_table_03"}, altered)
- sort.Strings(dropped)
- assert.Equal(t, []string{"msg"}, dropped)
+ assert.Equal(t, []string{"test_table_04"}, extractNamesFromTablesList(created))
+ assert.Equal(t, []string{"test_table_03"}, extractNamesFromTablesList(altered))
+ assert.Equal(t, []string{"msg"}, extractNamesFromTablesList(dropped))
}
}
- se.RegisterNotifier("test", notifier)
+ se.RegisterNotifier("test", notifier, true)
err := se.Reload(context.Background())
require.NoError(t, err)
@@ -658,3 +661,568 @@ func AddFakeInnoDBReadRowsResult(db *fakesqldb.DB, value int) *fakesqldb.Expecte
fmt.Sprintf("Innodb_rows_read|%d", value),
))
}
+
+// TestEngineMysqlTime tests the functionality of Engine.mysqlTime function
+func TestEngineMysqlTime(t *testing.T) {
+ tests := []struct {
+ name string
+ timeStampResult []string
+ timeStampErr error
+ wantTime int64
+ wantErr string
+ }{
+ {
+ name: "Success",
+ timeStampResult: []string{"1685115631"},
+ wantTime: 1685115631,
+ }, {
+ name: "Error in result",
+ timeStampErr: errors.New("some error in MySQL"),
+ wantErr: "some error in MySQL",
+ }, {
+ name: "Error in parsing",
+ timeStampResult: []string{"16851r15631"},
+ wantErr: "could not parse time",
+ }, {
+ name: "More than 1 result",
+ timeStampResult: []string{"1685115631", "3241241"},
+ wantErr: "could not get MySQL time",
+ }, {
+ name: "Null result",
+ timeStampResult: []string{"null"},
+ wantErr: "unexpected result for MySQL time",
+ },
+ }
+
+ query := "SELECT UNIX_TIMESTAMP()"
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ se := &Engine{}
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ if tt.timeStampErr != nil {
+ db.AddRejectedQuery(query, tt.timeStampErr)
+ } else {
+ db.AddQuery(query, sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), tt.timeStampResult...))
+ }
+
+ gotTime, err := se.mysqlTime(context.Background(), conn)
+ if tt.wantErr != "" {
+ require.ErrorContains(t, err, tt.wantErr)
+ return
+ }
+ require.EqualValues(t, tt.wantTime, gotTime)
+ require.NoError(t, db.LastError())
+ })
+ }
+}
+
+// TestEnginePopulatePrimaryKeys tests the functionality of Engine.populatePrimaryKeys function
+func TestEnginePopulatePrimaryKeys(t *testing.T) {
+ tests := []struct {
+ name string
+ tables map[string]*Table
+ pkIndexes map[string]int
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ expectedError string
+ }{
+ {
+ name: "Success",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Fields: []*querypb.Field{
+ {
+ Name: "col1",
+ }, {
+ Name: "col2",
+ },
+ },
+ Type: NoType,
+ }, "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Fields: []*querypb.Field{
+ {
+ Name: "id",
+ },
+ },
+ Type: NoType,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ mysql.BaseShowPrimary: sqltypes.MakeTestResult(mysql.ShowPrimaryFields,
+ "t1|col2",
+ "t2|id"),
+ },
+ pkIndexes: map[string]int{
+ "t1": 1,
+ "t2": 0,
+ },
+ }, {
+ name: "Error in finding column",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Fields: []*querypb.Field{
+ {
+ Name: "col1",
+ }, {
+ Name: "col2",
+ },
+ },
+ Type: NoType,
+ },
+ },
+ expectedQueries: map[string]*sqltypes.Result{
+ mysql.BaseShowPrimary: sqltypes.MakeTestResult(mysql.ShowPrimaryFields,
+ "t1|col5"),
+ },
+ expectedError: "column col5 is listed as primary key, but not present in table t1",
+ }, {
+ name: "Error in query",
+ tables: map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Fields: []*querypb.Field{
+ {
+ Name: "col1",
+ }, {
+ Name: "col2",
+ },
+ },
+ Type: NoType,
+ },
+ },
+ queriesToReject: map[string]error{
+ mysql.BaseShowPrimary: errors.New("some error in MySQL"),
+ },
+ expectedError: "could not get table primary key info",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+ se := &Engine{}
+
+ for query, result := range tt.expectedQueries {
+ db.AddQuery(query, result)
+ }
+ for query, errToThrow := range tt.queriesToReject {
+ db.AddRejectedQuery(query, errToThrow)
+ }
+
+ err = se.populatePrimaryKeys(context.Background(), conn, tt.tables)
+ if tt.expectedError != "" {
+ require.ErrorContains(t, err, tt.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ for table, index := range tt.pkIndexes {
+ require.Equal(t, index, tt.tables[table].PKColumns[0])
+ }
+ })
+ }
+}
+
+// TestEngineUpdateInnoDBRowsRead tests the functionality of Engine.updateInnoDBRowsRead function
+func TestEngineUpdateInnoDBRowsRead(t *testing.T) {
+ showRowsReadFields := sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64")
+ tests := []struct {
+ name string
+ innoDbReadRowsCounter int
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ expectedError string
+ }{
+ {
+ name: "Success",
+ expectedQueries: map[string]*sqltypes.Result{
+ mysql.ShowRowsRead: sqltypes.MakeTestResult(showRowsReadFields,
+ "Innodb_rows_read|35"),
+ },
+ innoDbReadRowsCounter: 35,
+ }, {
+ name: "Unexpected result",
+ expectedQueries: map[string]*sqltypes.Result{
+ mysql.ShowRowsRead: sqltypes.MakeTestResult(showRowsReadFields,
+ "Innodb_rows_read|35",
+ "Innodb_rows_read|37"),
+ },
+ innoDbReadRowsCounter: 0,
+ }, {
+ name: "Error in query",
+ queriesToReject: map[string]error{
+ mysql.ShowRowsRead: errors.New("some error in MySQL"),
+ },
+ expectedError: "some error in MySQL",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+ se := &Engine{}
+ se.innoDbReadRowsCounter = stats.NewCounter("TestEngineUpdateInnoDBRowsRead-"+tt.name, "")
+
+ for query, result := range tt.expectedQueries {
+ db.AddQuery(query, result)
+ }
+ for query, errToThrow := range tt.queriesToReject {
+ db.AddRejectedQuery(query, errToThrow)
+ }
+
+ err = se.updateInnoDBRowsRead(context.Background(), conn)
+ if tt.expectedError != "" {
+ require.ErrorContains(t, err, tt.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ require.EqualValues(t, tt.innoDbReadRowsCounter, se.innoDbReadRowsCounter.Get())
+ })
+ }
+}
+
+// TestEngineGetTableData tests the functionality of getTableData function
+func TestEngineGetTableData(t *testing.T) {
+ db := fakesqldb.New(t)
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ tests := []struct {
+ name string
+ expectedQueries map[string]*sqltypes.Result
+ queriesToReject map[string]error
+ includeStats bool
+ expectedError string
+ }{
+ {
+ name: "Success",
+ expectedQueries: map[string]*sqltypes.Result{
+ conn.BaseShowTables(): {},
+ },
+ includeStats: false,
+ }, {
+ name: "Success with include stats",
+ expectedQueries: map[string]*sqltypes.Result{
+ conn.BaseShowTablesWithSizes(): {},
+ },
+ includeStats: true,
+ }, {
+ name: "Error in query",
+ queriesToReject: map[string]error{
+ conn.BaseShowTables(): errors.New("some error in MySQL"),
+ },
+ expectedError: "some error in MySQL",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db.ClearQueryPattern()
+
+ for query, result := range tt.expectedQueries {
+ db.AddQuery(query, result)
+ defer db.DeleteQuery(query)
+ }
+ for query, errToThrow := range tt.queriesToReject {
+ db.AddRejectedQuery(query, errToThrow)
+ defer db.DeleteRejectedQuery(query)
+ }
+
+ _, err = getTableData(context.Background(), conn, tt.includeStats)
+ if tt.expectedError != "" {
+ require.ErrorContains(t, err, tt.expectedError)
+ return
+ }
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+ })
+ }
+}
+
+// TestEngineGetDroppedTables tests the functionality of Engine.getDroppedTables function
+func TestEngineGetDroppedTables(t *testing.T) {
+ tests := []struct {
+ name string
+ tables map[string]*Table
+ curTables map[string]bool
+ changedViews map[string]any
+ mismatchTables map[string]any
+ wantDroppedTables []*Table
+ }{
+ {
+ name: "No mismatched tables or changed views",
+ tables: map[string]*Table{
+ "t1": NewTable("t1", NoType),
+ "t2": NewTable("t2", NoType),
+ "t3": NewTable("t3", NoType),
+ },
+ curTables: map[string]bool{
+ "t4": true,
+ "t2": true,
+ },
+ wantDroppedTables: []*Table{
+ NewTable("t1", NoType),
+ NewTable("t3", NoType),
+ },
+ }, {
+ name: "Mismatched tables having a dropped table",
+ tables: map[string]*Table{
+ "t1": NewTable("t1", NoType),
+ "t2": NewTable("t2", NoType),
+ "t3": NewTable("t3", NoType),
+ "v2": NewTable("v2", View),
+ },
+ curTables: map[string]bool{
+ "t4": true,
+ "t2": true,
+ },
+ mismatchTables: map[string]any{
+ "t5": true,
+ "v2": true,
+ },
+ wantDroppedTables: []*Table{
+ NewTable("t1", NoType),
+ NewTable("t3", NoType),
+ NewTable("t5", NoType),
+ NewTable("v2", View),
+ },
+ }, {
+ name: "Changed views having a dropped view",
+ tables: map[string]*Table{
+ "t1": NewTable("t1", NoType),
+ "t2": NewTable("t2", NoType),
+ "t3": NewTable("t3", NoType),
+ "v2": NewTable("v2", NoType),
+ },
+ curTables: map[string]bool{
+ "t4": true,
+ "t2": true,
+ },
+ changedViews: map[string]any{
+ "v1": true,
+ "v2": true,
+ },
+ wantDroppedTables: []*Table{
+ NewTable("t1", NoType),
+ NewTable("t3", NoType),
+ NewTable("v1", View),
+ NewTable("v2", NoType),
+ },
+ }, {
+ name: "Both have dropped tables",
+ tables: map[string]*Table{
+ "t1": NewTable("t1", NoType),
+ "t2": NewTable("t2", NoType),
+ "t3": NewTable("t3", NoType),
+ "v2": NewTable("v2", NoType),
+ "v3": NewTable("v3", View),
+ },
+ curTables: map[string]bool{
+ "t4": true,
+ "t2": true,
+ },
+ changedViews: map[string]any{
+ "v1": true,
+ "v2": true,
+ },
+ mismatchTables: map[string]any{
+ "t5": true,
+ "v3": true,
+ },
+ wantDroppedTables: []*Table{
+ NewTable("t1", NoType),
+ NewTable("t3", NoType),
+ NewTable("t5", NoType),
+ NewTable("v1", View),
+ NewTable("v3", View),
+ NewTable("v2", NoType),
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ se := &Engine{
+ tables: tt.tables,
+ }
+ se.tableFileSizeGauge = stats.NewGaugesWithSingleLabel("TestEngineGetDroppedTables-"+tt.name, "", "Table")
+ se.tableAllocatedSizeGauge = stats.NewGaugesWithSingleLabel("TestEngineGetDroppedTables-allocated-"+tt.name, "", "Table")
+ gotDroppedTables := se.getDroppedTables(tt.curTables, tt.changedViews, tt.mismatchTables)
+ require.ElementsMatch(t, gotDroppedTables, tt.wantDroppedTables)
+ })
+ }
+}
+
+// TestEngineReload tests the entire functioning of engine.Reload testing all the queries that we end up running against MySQL
+// while simulating the responses and verifies the final list of created, altered and dropped tables.
+func TestEngineReload(t *testing.T) {
+ db := fakesqldb.New(t)
+ cfg := tabletenv.NewDefaultConfig()
+ cfg.DB = newDBConfigs(db)
+ cfg.SignalWhenSchemaChange = true
+ conn, err := connpool.NewDBConnNoPool(context.Background(), db.ConnParams(), nil, nil)
+ require.NoError(t, err)
+
+ se := newEngine(10, 10*time.Second, 10*time.Second, 0, db)
+ se.conns.Open(se.cp, se.cp, se.cp)
+ se.isOpen = true
+ se.notifiers = make(map[string]notifier)
+ se.MakePrimary(true)
+
+ // If we have to skip the meta check, then there is nothing to do
+ se.SkipMetaCheck = true
+ err = se.reload(context.Background(), false)
+ require.NoError(t, err)
+
+ se.SkipMetaCheck = false
+ se.lastChange = 987654321
+
+ // Initial tables in the schema engine
+ se.tables = map[string]*Table{
+ "t1": {
+ Name: sqlparser.NewIdentifierCS("t1"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "t2": {
+ Name: sqlparser.NewIdentifierCS("t2"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "t4": {
+ Name: sqlparser.NewIdentifierCS("t4"),
+ Type: NoType,
+ CreateTime: 123456789,
+ },
+ "v1": {
+ Name: sqlparser.NewIdentifierCS("v1"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ "v2": {
+ Name: sqlparser.NewIdentifierCS("v2"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ "v4": {
+ Name: sqlparser.NewIdentifierCS("v4"),
+ Type: View,
+ CreateTime: 123456789,
+ },
+ }
+ // MySQL unix timestamp query.
+ db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), "987654326"))
+ // Table t2 is updated, t3 is created and t4 is deleted.
+ // View v2 is updated, v3 is created and v4 is deleted.
+ db.AddQuery(conn.BaseShowTables(), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|table_type|unix_timestamp(create_time)|table_comment",
+ "varchar|varchar|int64|varchar"),
+ "t1|BASE_TABLE|123456789|",
+ "t2|BASE_TABLE|123456790|",
+ "t3|BASE_TABLE|123456789|",
+ "v1|VIEW|123456789|",
+ "v2|VIEW|123456789|",
+ "v3|VIEW|123456789|",
+ ))
+
+ // Detecting view changes.
+ // According to the database, v2, v3, v4, and v5 require updating.
+ db.AddQuery(fmt.Sprintf(detectViewChange, sidecardb.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"),
+ "v2",
+ "v3",
+ "v4",
+ "v5",
+ ))
+
+ // Finding mismatches in the tables.
+ // t5 exists in the database.
+ db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.schema_engine_tables", sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"),
+ "t1|123456789",
+ "t2|123456789",
+ "t4|123456789",
+ "t5|123456789",
+ ))
+
+ // Read Innodb_rows_read.
+ db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64"),
+ "Innodb_rows_read|35"))
+
+ // Queries to load the tables' information.
+ for _, tableName := range []string{"t2", "t3", "v2", "v3"} {
+ db.AddQuery(fmt.Sprintf(`SELECT COLUMN_NAME as column_name
+ FROM INFORMATION_SCHEMA.COLUMNS
+ WHERE TABLE_SCHEMA = 'fakesqldb' AND TABLE_NAME = '%s'
+ ORDER BY ORDINAL_POSITION`, tableName),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"),
+ "col1"))
+ db.AddQuery(fmt.Sprintf("SELECT `col1` FROM `fakesqldb`.`%v` WHERE 1 != 1", tableName), sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar")))
+ }
+
+ // Primary key information.
+ db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(mysql.ShowPrimaryFields,
+ "t1|col1",
+ "t2|col1",
+ "t3|col1",
+ ))
+
+ // Queries for reloading the tables' information.
+ {
+ for _, tableName := range []string{"t2", "t3"} {
+ db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar"),
+ fmt.Sprintf("%v|create_table_%v", tableName, tableName)))
+ }
+ db.AddQuery("begin", &sqltypes.Result{})
+ db.AddQuery("commit", &sqltypes.Result{})
+ db.AddQuery("rollback", &sqltypes.Result{})
+ // We are adding both the variants of the delete statements that we can see in the test, since the deleted tables are initially stored as a map, the order is not defined.
+ db.AddQuery("delete from _vt.schema_engine_tables where TABLE_SCHEMA = database() and TABLE_NAME in ('t5', 't4', 't3', 't2')", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.schema_engine_tables where TABLE_SCHEMA = database() and TABLE_NAME in ('t4', 't5', 't3', 't2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.schema_engine_tables(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't2', 'create_table_t2', 123456790)", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.schema_engine_tables(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't3', 'create_table_t3', 123456789)", &sqltypes.Result{})
+ }
+
+ // Queries for reloading the views' information.
+ {
+ for _, tableName := range []string{"v2", "v3"} {
+ db.AddQuery(fmt.Sprintf(`show create table %s`, tableName),
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"),
+ fmt.Sprintf("%v|create_table_%v|utf8mb4|utf8mb4_0900_ai_ci", tableName, tableName)))
+ }
+ // We are adding both the variants of the select statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
+ db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v4', 'v5', 'v3', 'v2')",
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
+ "v2|select_v2",
+ "v3|select_v3",
+ ))
+ db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v5', 'v4', 'v3', 'v2')",
+ sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"),
+ "v2|select_v2",
+ "v3|select_v3",
+ ))
+
+ // We are adding both the variants of the delete statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined.
+ db.AddQuery("delete from _vt.schema_engine_views where TABLE_SCHEMA = database() and TABLE_NAME in ('v4', 'v5', 'v3', 'v2')", &sqltypes.Result{})
+ db.AddQuery("delete from _vt.schema_engine_views where TABLE_SCHEMA = database() and TABLE_NAME in ('v5', 'v4', 'v3', 'v2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v2', 'create_table_v2', 'select_v2')", &sqltypes.Result{})
+ db.AddQuery("insert into _vt.schema_engine_views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v3', 'create_table_v3', 'select_v3')", &sqltypes.Result{})
+ }
+
+ // Verify the list of created, altered and dropped tables seen.
+ se.RegisterNotifier("test", func(full map[string]*Table, created, altered, dropped []*Table) {
+ require.ElementsMatch(t, extractNamesFromTablesList(created), []string{"t3", "v3"})
+ require.ElementsMatch(t, extractNamesFromTablesList(altered), []string{"t2", "v2"})
+ require.ElementsMatch(t, extractNamesFromTablesList(dropped), []string{"t4", "v4", "t5", "v5"})
+ }, false)
+
+ // Run the reload.
+ err = se.reload(context.Background(), false)
+ require.NoError(t, err)
+ require.NoError(t, db.LastError())
+}
diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go
index 457129314cf..0670de79cb7 100644
--- a/go/vt/vttablet/tabletserver/schema/load_table.go
+++ b/go/vt/vttablet/tabletserver/schema/load_table.go
@@ -33,8 +33,8 @@ import (
)
// LoadTable creates a Table from the schema info in the database.
-func LoadTable(conn *connpool.DBConn, databaseName, tableName string, comment string) (*Table, error) {
- ta := NewTable(tableName)
+func LoadTable(conn *connpool.DBConn, databaseName, tableName, tableType string, comment string) (*Table, error) {
+ ta := NewTable(tableName, NoType)
sqlTableName := sqlparser.String(ta.Name)
if err := fetchColumns(ta, conn, databaseName, sqlTableName); err != nil {
return nil, err
@@ -48,6 +48,8 @@ func LoadTable(conn *connpool.DBConn, databaseName, tableName string, comment st
return nil, err
}
ta.Type = Message
+ case strings.Contains(tableType, "VIEW"):
+ ta.Type = View
}
return ta, nil
}
diff --git a/go/vt/vttablet/tabletserver/schema/load_table_test.go b/go/vt/vttablet/tabletserver/schema/load_table_test.go
index 9362431816e..24dff88793e 100644
--- a/go/vt/vttablet/tabletserver/schema/load_table_test.go
+++ b/go/vt/vttablet/tabletserver/schema/load_table_test.go
@@ -40,11 +40,32 @@ func TestLoadTable(t *testing.T) {
defer db.Close()
mockLoadTableQueries(db)
table, err := newTestLoadTable("USER_TABLE", "test table", db)
- if err != nil {
- t.Fatal(err)
+ require.NoError(t, err)
+ want := &Table{
+ Name: sqlparser.NewIdentifierCS("test_table"),
+ Fields: []*querypb.Field{{
+ Name: "pk",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "name",
+ Type: sqltypes.Int32,
+ }, {
+ Name: "addr",
+ Type: sqltypes.Int32,
+ }},
}
+ assert.Equal(t, want, table)
+}
+
+func TestLoadView(t *testing.T) {
+ db := fakesqldb.New(t)
+ defer db.Close()
+ mockLoadTableQueries(db)
+ table, err := newTestLoadTable("VIEW", "test table", db)
+ require.NoError(t, err)
want := &Table{
Name: sqlparser.NewIdentifierCS("test_table"),
+ Type: View,
Fields: []*querypb.Field{{
Name: "pk",
Type: sqltypes.Int32,
@@ -64,9 +85,7 @@ func TestLoadTableSequence(t *testing.T) {
defer db.Close()
mockLoadTableQueries(db)
table, err := newTestLoadTable("USER_TABLE", "vitess_sequence", db)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
want := &Table{
Name: sqlparser.NewIdentifierCS("test_table"),
Type: Sequence,
@@ -84,9 +103,7 @@ func TestLoadTableMessage(t *testing.T) {
defer db.Close()
mockMessageTableQueries(db)
table, err := newTestLoadTable("USER_TABLE", "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30", db)
- if err != nil {
- t.Fatal(err)
- }
+ require.NoError(t, err)
want := &Table{
Name: sqlparser.NewIdentifierCS("test_table"),
Type: Message,
@@ -218,7 +235,7 @@ func newTestLoadTable(tableType string, comment string, db *fakesqldb.DB) (*Tabl
}
defer conn.Recycle()
- return LoadTable(conn, "fakesqldb", "test_table", comment)
+ return LoadTable(conn, "fakesqldb", "test_table", tableType, comment)
}
func mockLoadTableQueries(db *fakesqldb.DB) {
diff --git a/go/vt/vttablet/tabletserver/schema/schema.go b/go/vt/vttablet/tabletserver/schema/schema.go
index 6dd2a3fef6d..cd23b57607a 100644
--- a/go/vt/vttablet/tabletserver/schema/schema.go
+++ b/go/vt/vttablet/tabletserver/schema/schema.go
@@ -30,6 +30,7 @@ const (
NoType = iota
Sequence
Message
+ View
)
// TypeNames allows to fetch a the type name for a table.
@@ -38,6 +39,7 @@ var TypeNames = []string{
"none",
"sequence",
"message",
+ "view",
}
// Table contains info about a table.
@@ -107,9 +109,10 @@ type MessageInfo struct {
}
// NewTable creates a new Table.
-func NewTable(name string) *Table {
+func NewTable(name string, tableType int) *Table {
return &Table{
Name: sqlparser.NewIdentifierCS(name),
+ Type: tableType,
}
}
diff --git a/go/vt/vttablet/tabletserver/state_manager.go b/go/vt/vttablet/tabletserver/state_manager.go
index e3a72edeabb..8453e685b5b 100644
--- a/go/vt/vttablet/tabletserver/state_manager.go
+++ b/go/vt/vttablet/tabletserver/state_manager.go
@@ -141,6 +141,7 @@ type (
EnsureConnectionAndDB(topodatapb.TabletType) error
Open() error
MakeNonPrimary()
+ MakePrimary(bool)
Close()
}
@@ -445,6 +446,11 @@ func (sm *stateManager) servePrimary() error {
return err
}
+ // We have to make the health streamer read to process updates from schema engine
+ // before we mark schema engine capable of running queries against the database. This is required
+ // to ensure that we don't miss any updates from the schema engine.
+ sm.hs.MakePrimary(true)
+ sm.se.MakePrimary(true)
sm.rt.MakePrimary()
sm.tracker.Open()
// We instantly kill all stateful queries to allow for
@@ -469,6 +475,8 @@ func (sm *stateManager) unservePrimary() error {
return err
}
+ sm.se.MakePrimary(false)
+ sm.hs.MakePrimary(false)
sm.rt.MakePrimary()
sm.setState(topodatapb.TabletType_PRIMARY, StateNotServing)
return nil
@@ -485,6 +493,7 @@ func (sm *stateManager) serveNonPrimary(wantTabletType topodatapb.TabletType) er
sm.messager.Close()
sm.tracker.Close()
sm.se.MakeNonPrimary()
+ sm.hs.MakeNonPrimary()
if err := sm.connect(wantTabletType); err != nil {
return err
@@ -502,6 +511,7 @@ func (sm *stateManager) unserveNonPrimary(wantTabletType topodatapb.TabletType)
sm.unserveCommon()
sm.se.MakeNonPrimary()
+ sm.hs.MakeNonPrimary()
if err := sm.connect(wantTabletType); err != nil {
return err
diff --git a/go/vt/vttablet/tabletserver/state_manager_test.go b/go/vt/vttablet/tabletserver/state_manager_test.go
index e06ce4126a1..b4793915c00 100644
--- a/go/vt/vttablet/tabletserver/state_manager_test.go
+++ b/go/vt/vttablet/tabletserver/state_manager_test.go
@@ -30,6 +30,7 @@ import (
"github.com/stretchr/testify/require"
"vitess.io/vitess/go/mysql/fakesqldb"
+ "vitess.io/vitess/go/vt/vttablet/tabletserver/schema"
"vitess.io/vitess/go/vt/log"
querypb "vitess.io/vitess/go/vt/proto/query"
@@ -705,7 +706,7 @@ func newTestStateManager(t *testing.T) *stateManager {
statelessql: NewQueryList("stateless"),
statefulql: NewQueryList("stateful"),
olapql: NewQueryList("olap"),
- hs: newHealthStreamer(env, &topodatapb.TabletAlias{}),
+ hs: newHealthStreamer(env, &topodatapb.TabletAlias{}, schema.NewEngine(env)),
se: &testSchemaEngine{},
rt: &testReplTracker{lag: 1 * time.Second},
vstreamer: &testSubcomponent{},
@@ -790,6 +791,10 @@ func (te *testSchemaEngine) MakeNonPrimary() {
te.nonPrimary = true
}
+func (te *testSchemaEngine) MakePrimary(serving bool) {
+ te.nonPrimary = false
+}
+
func (te *testSchemaEngine) Close() {
te.order = order.Add(1)
te.state = testStateClosed
diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go
index 3839e970ea6..c52754b1569 100644
--- a/go/vt/vttablet/tabletserver/tabletenv/config.go
+++ b/go/vt/vttablet/tabletserver/tabletenv/config.go
@@ -125,8 +125,9 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
currentConfig.SchemaReloadIntervalSeconds = defaultConfig.SchemaReloadIntervalSeconds.Clone()
fs.Var(¤tConfig.SchemaReloadIntervalSeconds, currentConfig.SchemaReloadIntervalSeconds.Name(), "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
currentConfig.SignalSchemaChangeReloadIntervalSeconds = defaultConfig.SignalSchemaChangeReloadIntervalSeconds.Clone()
- fs.Var(¤tConfig.SignalSchemaChangeReloadIntervalSeconds, currentConfig.SignalSchemaChangeReloadIntervalSeconds.Name(), "query server schema change signal interval defines at which interval the query server shall send schema updates to vtgate.")
- fs.DurationVar(¤tConfig.SchemaChangeReloadTimeout, "schema-change-reload-timeout", defaultConfig.SchemaChangeReloadTimeout, "query server schema change signal reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up")
+ fs.Var(¤tConfig.SignalSchemaChangeReloadIntervalSeconds, "queryserver-config-schema-change-signal-interval", "query server schema change signal interval defines at which interval the query server shall send schema updates to vtgate.")
+ _ = fs.MarkDeprecated("queryserver-config-schema-change-signal-interval", "We no longer poll for finding schema changes.")
+ fs.DurationVar(¤tConfig.SchemaChangeReloadTimeout, "schema-change-reload-timeout", defaultConfig.SchemaChangeReloadTimeout, "query server schema change reload timeout, this is how long to wait for the signaled schema reload operation to complete before giving up")
fs.BoolVar(¤tConfig.SignalWhenSchemaChange, "queryserver-config-schema-change-signal", defaultConfig.SignalWhenSchemaChange, "query server schema signal, will signal connected vtgates that schema has changed whenever this is detected. VTGates will need to have -schema_change_signal enabled for this to work")
currentConfig.Olap.TxTimeoutSeconds = defaultConfig.Olap.TxTimeoutSeconds.Clone()
fs.Var(¤tConfig.Olap.TxTimeoutSeconds, defaultConfig.Olap.TxTimeoutSeconds.Name(), "query server transaction timeout (in seconds), after which a transaction in an OLAP session will be killed")
@@ -377,10 +378,6 @@ func (cfg *TabletConfig) MarshalJSON() ([]byte, error) {
tmp.SchemaReloadIntervalSeconds = d.String()
}
- if d := cfg.SignalSchemaChangeReloadIntervalSeconds.Get(); d != 0 {
- tmp.SignalSchemaChangeReloadIntervalSeconds = d.String()
- }
-
if d := cfg.SchemaChangeReloadTimeout; d != 0 {
tmp.SchemaChangeReloadTimeout = d.String()
}
@@ -785,12 +782,11 @@ var defaultConfig = TabletConfig{
// memory copies. so with the encoding overhead, this seems to work
// great (the overhead makes the final packets on the wire about twice
// bigger than this).
- StreamBufferSize: 32 * 1024,
- QueryCacheSize: int(cache.DefaultConfig.MaxEntries),
- QueryCacheMemory: cache.DefaultConfig.MaxMemoryUsage,
- QueryCacheLFU: cache.DefaultConfig.LFU,
- SchemaReloadIntervalSeconds: flagutil.NewDeprecatedFloat64Seconds("queryserver-config-schema-reload-time", 30*time.Minute),
- SignalSchemaChangeReloadIntervalSeconds: flagutil.NewDeprecatedFloat64Seconds("queryserver-config-schema-change-signal-interval", 5*time.Second),
+ StreamBufferSize: 32 * 1024,
+ QueryCacheSize: int(cache.DefaultConfig.MaxEntries),
+ QueryCacheMemory: cache.DefaultConfig.MaxMemoryUsage,
+ QueryCacheLFU: cache.DefaultConfig.LFU,
+ SchemaReloadIntervalSeconds: flagutil.NewDeprecatedFloat64Seconds("queryserver-config-schema-reload-time", 30*time.Minute),
// SchemaChangeReloadTimeout is used for the signal reload operation where we have to query mysqld.
// The queries during the signal reload operation are typically expected to have low load,
// but in busy systems with many tables, some queries may take longer than anticipated.
diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go
index 5abd936c2a9..84848517adc 100644
--- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go
+++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go
@@ -163,7 +163,6 @@ rowStreamer:
maxMySQLReplLagSecs: 43200
schemaChangeReloadTimeout: 30s
schemaReloadIntervalSeconds: 30m0s
-signalSchemaChangeReloadIntervalSeconds: 5s
signalWhenSchemaChange: true
streamBufferSize: 32768
txPool:
diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go
index ff641fe7198..b1934b9e557 100644
--- a/go/vt/vttablet/tabletserver/tabletserver.go
+++ b/go/vt/vttablet/tabletserver/tabletserver.go
@@ -175,8 +175,8 @@ func NewTabletServer(name string, config *tabletenv.TabletConfig, topoServer *to
tsv.statelessql = NewQueryList("oltp-stateless")
tsv.statefulql = NewQueryList("oltp-stateful")
tsv.olapql = NewQueryList("olap")
- tsv.hs = newHealthStreamer(tsv, alias)
tsv.se = schema.NewEngine(tsv)
+ tsv.hs = newHealthStreamer(tsv, alias, tsv.se)
tsv.rt = repltracker.NewReplTracker(tsv, alias)
tsv.lagThrottler = throttle.NewThrottler(tsv, srvTopoServer, topoServer, alias.Cell, tsv.rt.HeartbeatWriter(), tabletTypeFunc)
tsv.vstreamer = vstreamer.NewEngine(tsv, srvTopoServer, tsv.se, tsv.lagThrottler, alias.Cell)
@@ -426,9 +426,9 @@ func (tsv *TabletServer) ReloadSchema(ctx context.Context) error {
// changes to finish being applied.
func (tsv *TabletServer) WaitForSchemaReset(timeout time.Duration) {
onSchemaChange := make(chan struct{}, 1)
- tsv.se.RegisterNotifier("_tsv_wait", func(_ map[string]*schema.Table, _, _, _ []string) {
+ tsv.se.RegisterNotifier("_tsv_wait", func(_ map[string]*schema.Table, _, _, _ []*schema.Table) {
onSchemaChange <- struct{}{}
- })
+ }, true)
defer tsv.se.UnregisterNotifier("_tsv_wait")
after := time.NewTimer(timeout)