Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

schema : make update to partition tables when 'set tiflash replica' #5267

Merged
merged 8 commits into from
Jul 7, 2022
Merged

schema : make update to partition tables when 'set tiflash replica' #5267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5d93249
fix set tiflash replica bug
hongyunyan Jul 1, 2022
53f0cc0
update schema
hongyunyan Jul 1, 2022
42e24a8
do rename
hongyunyan Jul 1, 2022
323f274
update code
hongyunyan Jul 4, 2022
8749320
add test for tiflash replica
hongyunyan Jul 7, 2022
405c593
move the function
hongyunyan Jul 7, 2022
d16c7ce
update test
hongyunyan Jul 7, 2022
f268e64
Merge branch 'master' into hongyunyan_fix_replica_ddl
ti-chi-bot Jul 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions dbms/src/Debug/DBGInvoker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ DBGInvoker::DBGInvoker()
regSchemalessFunc("mapped_database", dbgFuncMappedDatabase);
regSchemalessFunc("mapped_table", dbgFuncMappedTable);
regSchemafulFunc("query_mapped", dbgFuncQueryMapped);
regSchemalessFunc("get_tiflash_replica_count", dbgFuncGetTiflashReplicaCount);
regSchemalessFunc("get_partition_tables_tiflash_replica_count", dbgFuncGetPartitionTablesTiflashReplicaCount);

regSchemalessFunc("search_log_for_key", dbgFuncSearchLogForKey);
regSchemalessFunc("tidb_dag", dbgFuncTiDBQueryFromNaturalDag);
Expand Down
2 changes: 2 additions & 0 deletions dbms/src/Debug/dbgFuncSchema.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <Storages/IManageableStorage.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiDB.h>
#include <TiDB/Schema/SchemaNameMapper.h>
#include <TiDB/Schema/SchemaSyncService.h>
#include <TiDB/Schema/SchemaSyncer.h>
#include <fmt/core.h>
Expand Down Expand Up @@ -137,4 +138,5 @@ void dbgFuncIsTombstone(Context & context, const ASTs & args, DBGInvoker::Printe
output(fmt_buf.toString());
}


} // namespace DB
1 change: 0 additions & 1 deletion dbms/src/Debug/dbgFuncSchema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,4 @@ void dbgFuncResetSchemas(Context & context, const ASTs & args, DBGInvoker::Print
// Usage:
// ./storage-client.sh "DBGInvoke is_tombstone(db_name, table_name)"
void dbgFuncIsTombstone(Context & context, const ASTs & args, DBGInvoker::Printer output);

} // namespace DB
53 changes: 53 additions & 0 deletions dbms/src/Debug/dbgFuncSchemaName.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,4 +128,57 @@ BlockInputStreamPtr dbgFuncQueryMapped(Context & context, const ASTs & args)
return executeQuery(query, context, true).in;
}


void dbgFuncGetTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
if (args.empty() || args.size() != 2)
throw Exception("Args not matched, should be: database-name[, table-name]", ErrorCodes::BAD_ARGUMENTS);

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
FmtBuffer fmt_buf;

const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
auto mapped = mappedTable(context, database_name, table_name);
auto storage = context.getTable(mapped->first, mapped->second);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (!managed_storage)
throw Exception(database_name + "." + table_name + " is not ManageableStorage", ErrorCodes::BAD_ARGUMENTS);

fmt_buf.append((std::to_string(managed_storage->getTableInfo().replica_info.count)));

output(fmt_buf.toString());
}

void dbgFuncGetPartitionTablesTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
if (args.empty() || args.size() != 2)
throw Exception("Args not matched, should be: database-name[, table-name]", ErrorCodes::BAD_ARGUMENTS);

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
FmtBuffer fmt_buf;

const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
auto mapped = mappedTable(context, database_name, table_name);
auto storage = context.getTable(mapped->first, mapped->second);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (!managed_storage)
throw Exception(database_name + "." + table_name + " is not ManageableStorage", ErrorCodes::BAD_ARGUMENTS);

auto table_info = managed_storage->getTableInfo();

if (!table_info.isLogicalPartitionTable())
throw Exception(database_name + "." + table_name + " is not logical partition table", ErrorCodes::BAD_ARGUMENTS);

SchemaNameMapper name_mapper;
for (const auto & part_def : table_info.partition.definitions)
{
auto paritition_table_info = table_info.producePartitionTableInfo(part_def.id, name_mapper);
auto partition_storage = context.getTMTContext().getStorages().get(paritition_table_info->id);
fmt_buf.append((std::to_string(partition_storage->getTableInfo().replica_info.count)));
fmt_buf.append("/");
}

output(fmt_buf.toString());
}

} // namespace DB
10 changes: 10 additions & 0 deletions dbms/src/Debug/dbgFuncSchemaName.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,14 @@ void dbgFuncMappedTable(Context & context, const ASTs & args, DBGInvoker::Printe
// ./storage-client.sh "DBGInvoke query_mapped('select * from $d.$t', database_name[, table_name])"
BlockInputStreamPtr dbgFuncQueryMapped(Context & context, const ASTs & args);

// Get table's tiflash replica counts with mapped table name
// Usage:
// ./storage-client.sh "DBGInvoke get_tiflash_replica_count(db_name, table_name)"
void dbgFuncGetTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output);

// Get the logical table's partition tables' tiflash replica counts with mapped table name
// Usage:
// ./storage-client.sh "DBGInvoke get_partition_tables_tiflash_replica_count(db_name, table_name)"
void dbgFuncGetPartitionTablesTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output);

} // namespace DB
74 changes: 47 additions & 27 deletions dbms/src/TiDB/Schema/SchemaBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ inline SchemaChanges detectSchemaChanges(
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
LOG_FMT_INFO(log, "Altering table {}", name_mapper.debugCanonicalName(*db_info, *table_info));

Expand Down Expand Up @@ -394,7 +394,7 @@ void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(DBInfoPtr db_inf
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterTable(DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyAlterTable(const DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -413,7 +413,7 @@ void SchemaBuilder<Getter, NameMapper>::applyAlterTable(DBInfoPtr db_info, Table
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterLogicalTable(DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyAlterLogicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
// Alter logical table first.
applyAlterPhysicalTable(db_info, table_info, storage);
Expand Down Expand Up @@ -569,7 +569,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDiff(const SchemaDiff & diff)
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -593,7 +593,7 @@ void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(const TiDB::DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
const auto & orig_table_info = storage->getTableInfo();
if (!orig_table_info.isLogicalPartitionTable())
Expand Down Expand Up @@ -659,7 +659,7 @@ void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenameTable(DBInfoPtr new_db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyRenameTable(const DBInfoPtr & new_db_info, TableID table_id)
{
auto new_table_info = getter.getTableInfo(new_db_info->id, table_id);
if (new_table_info == nullptr)
Expand All @@ -679,9 +679,9 @@ void SchemaBuilder<Getter, NameMapper>::applyRenameTable(DBInfoPtr new_db_info,

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenameLogicalTable(
DBInfoPtr new_db_info,
TableInfoPtr new_table_info,
ManageableStoragePtr storage)
const DBInfoPtr & new_db_info,
const TableInfoPtr & new_table_info,
const ManageableStoragePtr & storage)
{
applyRenamePhysicalTable(new_db_info, *new_table_info, storage);

Expand All @@ -703,9 +703,9 @@ void SchemaBuilder<Getter, NameMapper>::applyRenameLogicalTable(

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenamePhysicalTable(
DBInfoPtr new_db_info,
TableInfo & new_table_info,
ManageableStoragePtr storage)
const DBInfoPtr & new_db_info,
const TableInfo & new_table_info,
const ManageableStoragePtr & storage)
{
const auto old_mapped_db_name = storage->getDatabaseName();
const auto new_mapped_db_name = name_mapper.mapDatabaseName(*new_db_info);
Expand Down Expand Up @@ -908,7 +908,7 @@ String createDatabaseStmt(Context & context, const DBInfo & db_info, const Schem
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateSchema(TiDB::DBInfoPtr db_info)
void SchemaBuilder<Getter, NameMapper>::applyCreateSchema(const TiDB::DBInfoPtr & db_info)
{
GET_METRIC(tiflash_schema_internal_ddl_count, type_create_db).Increment();
LOG_FMT_INFO(log, "Creating database {}", name_mapper.debugDatabaseName(*db_info));
Expand Down Expand Up @@ -1047,7 +1047,7 @@ String createTableStmt(
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(DBInfoPtr db_info, TableInfoPtr table_info)
void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info)
{
GET_METRIC(tiflash_schema_internal_ddl_count, type_create_table).Increment();
LOG_FMT_INFO(log, "Creating table {}", name_mapper.debugCanonicalName(*db_info, *table_info));
Expand Down Expand Up @@ -1109,7 +1109,7 @@ void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateTable(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyCreateTable(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -1123,7 +1123,7 @@ void SchemaBuilder<Getter, NameMapper>::applyCreateTable(TiDB::DBInfoPtr db_info
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateLogicalTable(TiDB::DBInfoPtr db_info, TableInfoPtr table_info)
void SchemaBuilder<Getter, NameMapper>::applyCreateLogicalTable(const TiDB::DBInfoPtr & db_info, const TableInfoPtr & table_info)
{
if (table_info->isLogicalPartitionTable())
{
Expand Down Expand Up @@ -1169,7 +1169,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDropPhysicalTable(const String & db
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyDropTable(DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyDropTable(const DBInfoPtr & db_info, TableID table_id)
{
auto & tmt_context = context.getTMTContext();
auto * storage = tmt_context.getStorages().get(table_id).get();
Expand All @@ -1193,13 +1193,14 @@ void SchemaBuilder<Getter, NameMapper>::applyDropTable(DBInfoPtr db_info, TableI
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto latest_table_info = getter.getTableInfo(db_info->id, table_id);
if (unlikely(latest_table_info == nullptr))
{
throw TiFlashException(fmt::format("miss table in TiKV : {}", table_id), Errors::DDL::StaleSchema);
}

auto & tmt_context = context.getTMTContext();
auto storage = tmt_context.getStorages().get(latest_table_info->id);
if (unlikely(storage == nullptr))
Expand All @@ -1208,18 +1209,37 @@ void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(TiDB::DBInfoPtr d
Errors::DDL::MissingTable);
}

auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (unlikely(!managed_storage))
throw Exception(fmt::format("{} is not a ManageableStorage", name_mapper.debugCanonicalName(*db_info, *latest_table_info)));
applySetTiFlashReplicaOnLogicalTable(db_info, latest_table_info, storage);
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplicaOnLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
applySetTiFlashReplicaOnPhysicalTable(db_info, table_info, storage);

if (table_info->isLogicalPartitionTable())
{
auto & tmt_context = context.getTMTContext();

applySetTiFlashReplica(db_info, latest_table_info, managed_storage);
for (const auto & part_def : table_info->partition.definitions)
{
auto new_part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper);
auto part_storage = tmt_context.getStorages().get(new_part_table_info->id);
if (unlikely(part_storage == nullptr))
{
throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*db_info, *new_part_table_info)),
Errors::DDL::MissingTable);
}
applySetTiFlashReplicaOnPhysicalTable(db_info, new_part_table_info, part_storage);
}
}
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(
TiDB::DBInfoPtr db_info,
TiDB::TableInfoPtr latest_table_info,
ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplicaOnPhysicalTable(
const TiDB::DBInfoPtr & db_info,
const TiDB::TableInfoPtr & latest_table_info,
const ManageableStoragePtr & storage)
{
if (storage->getTableInfo().replica_info.count == latest_table_info->replica_info.count)
return;
Expand Down Expand Up @@ -1306,7 +1326,7 @@ void SchemaBuilder<Getter, NameMapper>::syncAllSchema()
/// Rename if needed.
applyRenameLogicalTable(db, table, storage);
/// Update replica info if needed.
applySetTiFlashReplica(db, table, storage);
applySetTiFlashReplicaOnLogicalTable(db, table, storage);
/// Alter if needed.
applyAlterLogicalTable(db, table, storage);
LOG_FMT_DEBUG(log, "Table {} synced during sync all schemas", name_mapper.debugCanonicalName(*db, *table));
Expand Down
31 changes: 16 additions & 15 deletions dbms/src/TiDB/Schema/SchemaBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,39 +55,40 @@ struct SchemaBuilder

bool applyCreateSchema(DatabaseID schema_id);

void applyCreateSchema(TiDB::DBInfoPtr db_info);
void applyCreateSchema(const TiDB::DBInfoPtr & db_info);

void applyCreateTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyCreateTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyCreateLogicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info);
void applyCreateLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);

void applyCreatePhysicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info);
void applyCreatePhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);

void applyDropTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyDropTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

/// Parameter schema_name should be mapped.
void applyDropPhysicalTable(const String & db_name, TableID table_id);

void applyPartitionDiff(TiDB::DBInfoPtr db_info, TableID table_id);
void applyPartitionDiff(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyPartitionDiff(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyPartitionDiff(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyAlterTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyAlterTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyAlterLogicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyAlterLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyAlterPhysicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyAlterPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyRenameTable(TiDB::DBInfoPtr new_db_info, TiDB::TableID table_id);
void applyRenameTable(const TiDB::DBInfoPtr & new_db_info, TiDB::TableID table_id);

void applyRenameLogicalTable(TiDB::DBInfoPtr new_db_info, TiDB::TableInfoPtr new_table_info, ManageableStoragePtr storage);
void applyRenameLogicalTable(const TiDB::DBInfoPtr & new_db_info, const TiDB::TableInfoPtr & new_table_info, const ManageableStoragePtr & storage);

void applyRenamePhysicalTable(TiDB::DBInfoPtr new_db_info, TiDB::TableInfo & new_table_info, ManageableStoragePtr storage);
void applyRenamePhysicalTable(const TiDB::DBInfoPtr & new_db_info, const TiDB::TableInfo & new_table_info, const ManageableStoragePtr & storage);

void applyExchangeTablePartition(const SchemaDiff & diff);

void applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TableID table_id);
void applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applySetTiFlashReplica(const TiDB::DBInfoPtr & db_info, TableID table_id);
void applySetTiFlashReplicaOnLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);
void applySetTiFlashReplicaOnPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);
};

} // namespace DB
Loading