Refactor: add new module schema

dbms/CMakeLists.txt

@@ -90,6 +90,7 @@ add_headers_and_sources(dbms src/Storages/Page/V2/mvcc)
 add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
 add_headers_and_sources(dbms src/Storages/Page/V2/gc)
 add_headers_and_sources(dbms src/WindowFunctions)
+add_headers_and_sources(dbms src/TiDB/Schema)
 if (ENABLE_V3_PAGESTORAGE)
     add_headers_and_sources(dbms src/Storages/Page/V3)
     add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)

dbms/src/Databases/test/gtest_database.cpp

@@ -25,11 +25,11 @@
 #include <Storages/IManageableStorage.h>
 #include <Storages/IStorage.h>
 #include <Storages/MutableSupport.h>
-#include <Storages/Transaction/SchemaNameMapper.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/Transaction/TMTStorages.h>
 #include <Storages/registerStorages.h>
 #include <TestUtils/TiFlashTestBasic.h>
+#include <TiDB/Schema/SchemaNameMapper.h>
 #include <common/ThreadPool.h>
 #include <common/logger_useful.h>
 

dbms/src/Debug/MockSchemaGetter.h

@@ -15,14 +15,13 @@
 #pragma once
 
 #include <Debug/MockTiDB.h>
-#include <Storages/Transaction/SchemaGetter.h>
+#include <TiDB/Schema/SchemaGetter.h>
 
 namespace DB
 {
 
 struct MockSchemaGetter
 {
-
     TiDB::DBInfoPtr getDatabase(DatabaseID db_id) { return MockTiDB::instance().getDBInfoByID(db_id); }
 
     Int64 getVersion() { return MockTiDB::instance().getVersion(); }

dbms/src/Debug/MockSchemaNameMapper.h

@@ -14,7 +14,7 @@
 
 #pragma once
 
-#include <Storages/Transaction/SchemaNameMapper.h>
+#include <TiDB/Schema/SchemaNameMapper.h>
 
 namespace DB
 {

dbms/src/Debug/MockTiDB.h

@@ -15,10 +15,10 @@
 #pragma once
 
 #include <Storages/ColumnsDescription.h>
-#include <Storages/Transaction/SchemaGetter.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TiDB.h>
 #include <Storages/Transaction/Types.h>
+#include <TiDB/Schema/SchemaGetter.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 
 #include <atomic>
 

dbms/src/Debug/dbgFuncCoprocessor.cpp

@@ -336,7 +336,7 @@ void dbgFuncTiDBQueryFromNaturalDag(Context & context, const ASTs & args, DBGInv
     if (args.size() != 1)
         throw Exception("Args not matched, should be: json_dag_path", ErrorCodes::BAD_ARGUMENTS);
 
-    String json_dag_path = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
+    auto json_dag_path = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
     auto dag = NaturalDag(json_dag_path, &Poco::Logger::get("MockDAG"));
     dag.init();
     dag.build(context);
@@ -431,7 +431,7 @@ BlockInputStreamPtr dbgFuncTiDBQuery(Context & context, const ASTs & args)
     if (args.empty() || args.size() > 3)
         throw Exception("Args not matched, should be: query[, region-id, dag_prop_string]", ErrorCodes::BAD_ARGUMENTS);
 
-    String query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
+    auto query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
     RegionID region_id = InvalidRegionID;
     if (args.size() >= 2)
         region_id = safeGet<RegionID>(typeid_cast<const ASTLiteral &>(*args[1]).value);
@@ -464,8 +464,8 @@ BlockInputStreamPtr dbgFuncMockTiDBQuery(Context & context, const ASTs & args)
     if (args.size() < 2 || args.size() > 4)
         throw Exception("Args not matched, should be: query, region-id[, start-ts, dag_prop_string]", ErrorCodes::BAD_ARGUMENTS);
 
-    String query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
-    RegionID region_id = safeGet<RegionID>(typeid_cast<const ASTLiteral &>(*args[1]).value);
+    auto query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
+    auto region_id = safeGet<RegionID>(typeid_cast<const ASTLiteral &>(*args[1]).value);
     Timestamp start_ts = DEFAULT_MAX_READ_TSO;
     if (args.size() >= 3)
         start_ts = safeGet<Timestamp>(typeid_cast<const ASTLiteral &>(*args[2]).value);
@@ -671,14 +671,14 @@ const ASTTablesInSelectQueryElement * getJoin(ASTSelectQuery & ast_query)
     if (!ast_query.tables)
         return nullptr;
 
-    const ASTTablesInSelectQuery & tables_in_select_query = static_cast<const ASTTablesInSelectQuery &>(*ast_query.tables);
+    const auto & tables_in_select_query = static_cast<const ASTTablesInSelectQuery &>(*ast_query.tables);
     if (tables_in_select_query.children.empty())
         return nullptr;
 
     const ASTTablesInSelectQueryElement * joined_table = nullptr;
     for (const auto & child : tables_in_select_query.children)
     {
-        const ASTTablesInSelectQueryElement & tables_element = static_cast<const ASTTablesInSelectQueryElement &>(*child);
+        const auto & tables_element = static_cast<const ASTTablesInSelectQueryElement &>(*child);
         if (tables_element.table_join)
         {
             if (!joined_table)
@@ -737,7 +737,7 @@ std::pair<ExecutorPtr, bool> compileQueryBlock(
             bool append_pk_column = false;
             for (const auto & expr : ast_query.select_expression_list->children)
             {
-                if (ASTIdentifier * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
+                if (auto * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
                 {
                     if (identifier->getColumnName() == MutableSupport::tidb_pk_column_name)
                     {
@@ -756,7 +756,7 @@ std::pair<ExecutorPtr, bool> compileQueryBlock(
         String right_table_alias;
         {
             String database_name, table_name;
-            const ASTTableExpression & table_to_join = static_cast<const ASTTableExpression &>(*joined_table->table_expression);
+            const auto & table_to_join = static_cast<const ASTTableExpression &>(*joined_table->table_expression);
             if (table_to_join.database_and_table_name)
             {
                 auto identifier = static_cast<const ASTIdentifier &>(*table_to_join.database_and_table_name);
@@ -788,7 +788,7 @@ std::pair<ExecutorPtr, bool> compileQueryBlock(
         bool right_append_pk_column = false;
         for (const auto & expr : ast_query.select_expression_list->children)
         {
-            if (ASTIdentifier * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
+            if (auto * identifier = typeid_cast<ASTIdentifier *>(expr.get()))
             {
                 auto names = splitQualifiedName(identifier->getColumnName());
                 if (names.second == MutableSupport::tidb_pk_column_name)
@@ -831,7 +831,7 @@ std::pair<ExecutorPtr, bool> compileQueryBlock(
     bool has_agg_func = false;
     for (const auto & child : ast_query.select_expression_list->children)
     {
-        const ASTFunction * func = typeid_cast<const ASTFunction *>(child.get());
+        const auto * func = typeid_cast<const ASTFunction *>(child.get());
         if (func && AggregateFunctionFactory::instance().isAggregateFunctionName(func->name))
         {
             has_agg_func = true;

dbms/src/Debug/dbgFuncMockTiDBTable.cpp

@@ -24,8 +24,8 @@
 #include <Parsers/ParserRenameQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Storages/Transaction/KVStore.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 #include <fmt/core.h>
 
 namespace DB

dbms/src/Debug/dbgFuncSchema.cpp

@@ -22,10 +22,10 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Storages/IManageableStorage.h>
-#include <Storages/Transaction/SchemaSyncService.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/Transaction/TiDB.h>
+#include <TiDB/Schema/SchemaSyncService.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 #include <fmt/core.h>
 
 #include <ext/singleton.h>

dbms/src/Debug/dbgFuncSchemaName.cpp

@@ -20,9 +20,9 @@
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Storages/IManageableStorage.h>
-#include <Storages/Transaction/SchemaNameMapper.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
+#include <TiDB/Schema/SchemaNameMapper.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 #include <fmt/core.h>
 
 #include <boost/algorithm/string/replace.hpp>
@@ -97,7 +97,7 @@ BlockInputStreamPtr dbgFuncQueryMapped(Context & context, const ASTs & args)
     if (args.size() < 2 || args.size() > 3)
         throw Exception("Args not matched, should be: query, database-name[, table-name]", ErrorCodes::BAD_ARGUMENTS);
 
-    String query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
+    auto query = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[0]).value);
     const String & database_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
 
     if (args.size() == 3)

dbms/src/Flash/BatchCoprocessorHandler.cpp

@@ -18,8 +18,8 @@
 #include <Flash/Coprocessor/InterpreterDAG.h>
 #include <Flash/ServiceUtils.h>
 #include <Storages/IStorage.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 
 #include <ext/scope_guard.h>
 

dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp

@@ -33,7 +33,7 @@
 #include <Storages/MutableSupport.h>
 #include <Storages/Transaction/KVStore.h>
 #include <Storages/Transaction/LockException.h>
-#include <Storages/Transaction/SchemaSyncer.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-parameter"
@@ -186,7 +186,7 @@ void setQuotaAndLimitsOnTableScan(Context & context, DAGPipeline & pipeline)
     QuotaForIntervals & quota = context.getQuota();
 
     pipeline.transform([&](auto & stream) {
-        if (IProfilingBlockInputStream * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
+        if (auto * p_stream = dynamic_cast<IProfilingBlockInputStream *>(stream.get()))
         {
             p_stream->setLimits(limits);
             p_stream->setQuota(quota);

dbms/src/Flash/CoprocessorHandler.cpp

@@ -22,8 +22,8 @@
 #include <Storages/IStorage.h>
 #include <Storages/Transaction/LockException.h>
 #include <Storages/Transaction/RegionException.h>
-#include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
+#include <TiDB/Schema/SchemaSyncer.h>
 
 #include <ext/scope_guard.h>
 

dbms/src/Interpreters/Context.cpp

@@ -59,9 +59,9 @@
 #include <Storages/PathCapacityMetrics.h>
 #include <Storages/PathPool.h>
 #include <Storages/Transaction/BackgroundService.h>
-#include <Storages/Transaction/SchemaSyncService.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <TableFunctions/TableFunctionFactory.h>
+#include <TiDB/Schema/SchemaSyncService.h>
 #include <common/logger_useful.h>
 #include <fiu.h>
 #include <fmt/core.h>
@@ -207,6 +207,7 @@ struct ContextShared
 
     explicit ContextShared(std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory_)
         : runtime_components_factory(std::move(runtime_components_factory_))
+        , storage_run_mode(PageStorageRunMode::MIX_MODE)
     {
         /// TODO: make it singleton (?)
         static std::atomic<size_t> num_calls{0};
@@ -714,7 +715,7 @@ Dependencies Context::getDependencies(const String & database_name, const String
         checkDatabaseAccessRightsImpl(db);
     }
 
-    ViewDependencies::const_iterator iter = shared->view_dependencies.find(DatabaseAndTableName(db, table_name));
+    auto iter = shared->view_dependencies.find(DatabaseAndTableName(db, table_name));
     if (iter == shared->view_dependencies.end())
         return {};
 
@@ -728,7 +729,7 @@ bool Context::isTableExist(const String & database_name, const String & table_na
     String db = resolveDatabase(database_name, current_database);
     checkDatabaseAccessRightsImpl(db);
 
-    Databases::const_iterator it = shared->databases.find(db);
+    auto it = shared->databases.find(db);
     return shared->databases.end() != it
         && it->second->isTableExist(*this, table_name);
 }
@@ -754,7 +755,7 @@ void Context::assertTableExists(const String & database_name, const String & tab
     String db = resolveDatabase(database_name, current_database);
     checkDatabaseAccessRightsImpl(db);
 
-    Databases::const_iterator it = shared->databases.find(db);
+    auto it = shared->databases.find(db);
     if (shared->databases.end() == it)
         throw Exception(fmt::format("Database {} doesn't exist", backQuoteIfNeed(db)), ErrorCodes::UNKNOWN_DATABASE);
 
@@ -771,7 +772,7 @@ void Context::assertTableDoesntExist(const String & database_name, const String
     if (check_database_access_rights)
         checkDatabaseAccessRightsImpl(db);
 
-    Databases::const_iterator it = shared->databases.find(db);
+    auto it = shared->databases.find(db);
     if (shared->databases.end() != it && it->second->isTableExist(*this, table_name))
         throw Exception(fmt::format("Table {}.{} already exists.", backQuoteIfNeed(db), backQuoteIfNeed(table_name)), ErrorCodes::TABLE_ALREADY_EXISTS);
 }
@@ -826,7 +827,7 @@ Tables Context::getExternalTables() const
 
 StoragePtr Context::tryGetExternalTable(const String & table_name) const
 {
-    TableAndCreateASTs::const_iterator jt = external_tables.find(table_name);
+    auto jt = external_tables.find(table_name);
     if (external_tables.end() == jt)
         return StoragePtr();
 
@@ -864,7 +865,7 @@ StoragePtr Context::getTableImpl(const String & database_name, const String & ta
     String db = resolveDatabase(database_name, current_database);
     checkDatabaseAccessRightsImpl(db);
 
-    Databases::const_iterator it = shared->databases.find(db);
+    auto it = shared->databases.find(db);
     if (shared->databases.end() == it)
     {
         if (exception)
@@ -894,7 +895,7 @@ void Context::addExternalTable(const String & table_name, const StoragePtr & sto
 
 StoragePtr Context::tryRemoveExternalTable(const String & table_name)
 {
-    TableAndCreateASTs::const_iterator it = external_tables.find(table_name);
+    auto it = external_tables.find(table_name);
 
     if (external_tables.end() == it)
         return StoragePtr();
@@ -954,7 +955,7 @@ std::unique_ptr<DDLGuard> Context::getDDLGuardIfTableDoesntExist(const String &
 {
     auto lock = getLock();
 
-    Databases::const_iterator it = shared->databases.find(database);
+    auto it = shared->databases.find(database);
     if (shared->databases.end() != it && it->second->isTableExist(*this, table))
         return {};
 
@@ -993,7 +994,7 @@ ASTPtr Context::getCreateTableQuery(const String & database_name, const String &
 
 ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const
 {
-    TableAndCreateASTs::const_iterator jt = external_tables.find(table_name);
+    auto jt = external_tables.find(table_name);
     if (external_tables.end() == jt)
         throw Exception(fmt::format("Temporary table {} doesn't exist", backQuoteIfNeed(table_name)), ErrorCodes::UNKNOWN_TABLE);
 
@@ -1088,7 +1089,7 @@ void Context::setCurrentQueryId(const String & query_id)
                 UInt64 a;
                 UInt64 b;
             };
-        } random;
+        } random{};
 
         {
             auto lock = getLock();