Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor: add new module schema #4932

Merged
merged 19 commits into from
May 30, 2022
Merged

Refactor: add new module schema #4932

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
bf149f8
move schemabuilder
hzh0425 May 19, 2022
393d810
move schemaNameMapped and getter
hzh0425 May 19, 2022
b3ad4e6
move schemaSyncService
hzh0425 May 19, 2022
9f027a8
move schemaSyncService
hzh0425 May 19, 2022
c22bc15
move schemaSyncer
hzh0425 May 19, 2022
9fd8016
code format
hzh0425 May 19, 2022
4c8c3ca
Rename module to TiDB
hzh0425 May 23, 2022
189b930
Merge branch 'master' of https://github.com/pingcap/tiflash into feat…
hzh0425 May 23, 2022
4216969
clang format
hzh0425 May 23, 2022
ed50feb
rename module again
hzh0425 May 23, 2022
bc3231f
Merge branch 'master' into feature/add_module_schema
hzh0425 May 23, 2022
f4b1967
code format
hzh0425 May 24, 2022
da7d487
code format
hzh0425 May 24, 2022
5c8ddd5
format
hzh0425 May 24, 2022
a64e3d4
code format
hzh0425 May 24, 2022
3fe2cc1
review
hzh0425 May 24, 2022
9ac0e19
Code review
hzh0425 May 25, 2022
0161857
Merge branch 'master' into feature/add_module_schema
Lloyd-Pottiger May 30, 2022
f804461
Merge branch 'master' into feature/add_module_schema
ti-chi-bot May 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dbms/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ add_headers_and_sources(dbms src/Storages/Page/V2/mvcc)
add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
add_headers_and_sources(dbms src/Storages/Page/V2/gc)
add_headers_and_sources(dbms src/WindowFunctions)
add_headers_and_sources(dbms src/TIDB/Schema)
JaySon-Huang marked this conversation as resolved.
Show resolved Hide resolved
if (ENABLE_V3_PAGESTORAGE)
add_headers_and_sources(dbms src/Storages/Page/V3)
add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Databases/test/gtest_database.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
#include <Storages/IManageableStorage.h>
#include <Storages/IStorage.h>
#include <Storages/MutableSupport.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TMTStorages.h>
#include <Storages/registerStorages.h>
#include <TIDB/Schema/SchemaNameMapper.h>
#include <TestUtils/TiFlashTestBasic.h>
#include <common/ThreadPool.h>
#include <common/logger_useful.h>
Expand Down
3 changes: 1 addition & 2 deletions dbms/src/Debug/MockSchemaGetter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,13 @@
#pragma once

#include <Debug/MockTiDB.h>
#include <Storages/Transaction/SchemaGetter.h>
#include <TIDB/Schema/SchemaGetter.h>

namespace DB
{

struct MockSchemaGetter
{

TiDB::DBInfoPtr getDatabase(DatabaseID db_id) { return MockTiDB::instance().getDBInfoByID(db_id); }

Int64 getVersion() { return MockTiDB::instance().getVersion(); }
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Debug/MockSchemaNameMapper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

#pragma once

#include <Storages/Transaction/SchemaNameMapper.h>
#include <TIDB/Schema/SchemaNameMapper.h>

namespace DB
{
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Debug/MockTiDB.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
#pragma once

#include <Storages/ColumnsDescription.h>
#include <Storages/Transaction/SchemaGetter.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/Types.h>
#include <TIDB/Schema/SchemaGetter.h>
#include <TIDB/Schema/SchemaSyncer.h>

#include <atomic>

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Debug/dbgFuncMockTiDBTable.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
#include <Parsers/ParserRenameQuery.h>
#include <Parsers/parseQuery.h>
#include <Storages/Transaction/KVStore.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <fmt/core.h>

namespace DB
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Debug/dbgFuncSchema.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@
#include <Parsers/ASTLiteral.h>
#include <Parsers/ParserCreateQuery.h>
#include <Storages/IManageableStorage.h>
#include <Storages/Transaction/SchemaSyncService.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiDB.h>
#include <TIDB/Schema/SchemaSyncService.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <fmt/core.h>

#include <ext/singleton.h>
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Debug/dbgFuncSchemaName.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Storages/IManageableStorage.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaNameMapper.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <fmt/core.h>

#include <boost/algorithm/string/replace.hpp>
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Flash/BatchCoprocessorHandler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
#include <Flash/Coprocessor/InterpreterDAG.h>
#include <Flash/ServiceUtils.h>
#include <Storages/IStorage.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaSyncer.h>

#include <ext/scope_guard.h>

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#include <Storages/MutableSupport.h>
#include <Storages/Transaction/KVStore.h>
#include <Storages/Transaction/LockException.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <TIDB/Schema/SchemaSyncer.h>

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Flash/CoprocessorHandler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
#include <Storages/IStorage.h>
#include <Storages/Transaction/LockException.h>
#include <Storages/Transaction/RegionException.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaSyncer.h>

#include <ext/scope_guard.h>

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Interpreters/Context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@
#include <Storages/PathCapacityMetrics.h>
#include <Storages/PathPool.h>
#include <Storages/Transaction/BackgroundService.h>
#include <Storages/Transaction/SchemaSyncService.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaSyncService.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <common/logger_useful.h>
#include <fiu.h>
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Interpreters/IDAsPathUpgrader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@
#include <Poco/FileStream.h>
#include <Storages/MutableSupport.h>
#include <Storages/PathPool.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/TiDBSchemaSyncer.h>
#include <TIDB/Schema/SchemaNameMapper.h>
#include <TIDB/Schema/TiDBSchemaSyncer.h>
#include <common/logger_useful.h>
#include <fmt/core.h>

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Interpreters/InterpreterSelectQuery.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@
#include <Storages/RegionQueryInfo.h>
#include <Storages/Transaction/LearnerRead.h>
#include <Storages/Transaction/RegionRangeKeys.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/StorageEngineType.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiKVRange.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <TableFunctions/ITableFunction.h>
#include <TableFunctions/TableFunctionFactory.h>

Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Interpreters/loadMetadata.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@
#include <Parsers/parseQuery.h>
#include <Poco/DirectoryIterator.h>
#include <Poco/FileStream.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaNameMapper.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <common/ThreadPool.h>

#include <future>
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Server/Server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@
#include <Storages/Transaction/FileEncryption.h>
#include <Storages/Transaction/KVStore.h>
#include <Storages/Transaction/ProxyFFI.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/registerStorages.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <TableFunctions/registerTableFunctions.h>
#include <WindowFunctions/registerWindowFunctions.h>
#include <common/ErrorHandlers.h>
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/StorageDeltaMerge.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@
#include <Storages/StorageDeltaMerge.h>
#include <Storages/StorageDeltaMergeHelpers.h>
#include <Storages/Transaction/Region.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiKVRecordFormat.h>
#include <Storages/Transaction/TypeMapping.h>
#include <TIDB/Schema/SchemaNameMapper.h>
#include <common/ThreadPool.h>
#include <common/config_common.h>
#include <common/logger_useful.h>
Expand Down
15 changes: 8 additions & 7 deletions dbms/src/Storages/System/StorageSystemDTSegments.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@
#include <Storages/MutableSupport.h>
#include <Storages/StorageDeltaMerge.h>
#include <Storages/System/StorageSystemDTSegments.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/Types.h>
#include <TIDB/Schema/SchemaNameMapper.h>

namespace DB
{
StorageSystemDTSegments::StorageSystemDTSegments(const std::string & name_) : name(name_)
StorageSystemDTSegments::StorageSystemDTSegments(const std::string & name_)
: name(name_)
{
setColumns(ColumnsDescription({
{"database", std::make_shared<DataTypeString>()},
Expand Down Expand Up @@ -61,11 +62,11 @@ StorageSystemDTSegments::StorageSystemDTSegments(const std::string & name_) : na
}

BlockInputStreams StorageSystemDTSegments::read(const Names & column_names,
const SelectQueryInfo &,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
const SelectQueryInfo &,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/System/StorageSystemDTTables.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#include <Storages/MutableSupport.h>
#include <Storages/StorageDeltaMerge.h>
#include <Storages/System/StorageSystemDTTables.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <TIDB/Schema/SchemaNameMapper.h>

namespace DB
{
Expand Down
15 changes: 8 additions & 7 deletions dbms/src/Storages/System/StorageSystemDatabases.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,17 @@
#include <Databases/IDatabase.h>
#include <Interpreters/Context.h>
#include <Storages/System/StorageSystemDatabases.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/Types.h>
#include <TIDB/Schema/SchemaNameMapper.h>


namespace DB
{


StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) : name(name_)
StorageSystemDatabases::StorageSystemDatabases(const std::string & name_)
: name(name_)
{
setColumns(ColumnsDescription({
{"name", std::make_shared<DataTypeString>()},
Expand All @@ -45,11 +46,11 @@ StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) : name


BlockInputStreams StorageSystemDatabases::read(const Names & column_names,
const SelectQueryInfo &,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
const SelectQueryInfo &,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
check(column_names);
processed_stage = QueryProcessingStage::FetchColumns;
Expand Down
21 changes: 12 additions & 9 deletions dbms/src/Storages/System/StorageSystemTables.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@
#include <Storages/IManageableStorage.h>
#include <Storages/MutableSupport.h>
#include <Storages/System/StorageSystemTables.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/Types.h>
#include <Storages/VirtualColumnUtils.h>
#include <TIDB/Schema/SchemaNameMapper.h>

namespace DB
{
Expand Down Expand Up @@ -71,7 +71,8 @@ NameAndTypePair tryGetColumn(const ColumnsWithTypeAndName & columns, const Strin
struct VirtualColumnsProcessor
{
explicit VirtualColumnsProcessor(const ColumnsWithTypeAndName & all_virtual_columns_)
: all_virtual_columns(all_virtual_columns_), virtual_columns_mask(all_virtual_columns_.size(), 0)
: all_virtual_columns(all_virtual_columns_)
, virtual_columns_mask(all_virtual_columns_.size(), 0)
{}

/// Separates real and virtual column names, returns real ones
Expand Down Expand Up @@ -131,7 +132,8 @@ struct VirtualColumnsProcessor
} // namespace


StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_)
StorageSystemTables::StorageSystemTables(const std::string & name_)
: name(name_)
{
setColumns(ColumnsDescription({
{"database", std::make_shared<DataTypeString>()},
Expand All @@ -147,7 +149,8 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_
}));

virtual_columns = {{std::make_shared<DataTypeDateTime>(), "metadata_modification_time"},
{std::make_shared<DataTypeString>(), "create_table_query"}, {std::make_shared<DataTypeString>(), "engine_full"}};
{std::make_shared<DataTypeString>(), "create_table_query"},
{std::make_shared<DataTypeString>(), "engine_full"}};
}


Expand All @@ -164,11 +167,11 @@ static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & cont


BlockInputStreams StorageSystemTables::read(const Names & column_names,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum & processed_stage,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
processed_stage = QueryProcessingStage::FetchColumns;

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/ApplySnapshot.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
#include <Storages/Transaction/Region.h>
#include <Storages/Transaction/RegionTable.h>
#include <Storages/Transaction/SSTReader.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <TIDB/Schema/SchemaSyncer.h>

#include <ext/scope_guard.h>

Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/PartitionStreams.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@
#include <Storages/Transaction/Region.h>
#include <Storages/Transaction/RegionBlockReader.h>
#include <Storages/Transaction/RegionTable.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiKVRange.h>
#include <Storages/Transaction/Utils.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <common/logger_useful.h>

namespace DB
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/RegionTable.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
#include <Storages/Transaction/Region.h>
#include <Storages/Transaction/RegionManager.h>
#include <Storages/Transaction/RegionTable.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiKVRange.h>
#include <TIDB/Schema/SchemaSyncer.h>

namespace DB
{
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Storages/Transaction/TMTContext.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
#include <Storages/Transaction/KVStore.h>
#include <Storages/Transaction/RegionExecutionResult.h>
#include <Storages/Transaction/RegionRangeKeys.h>
#include <Storages/Transaction/SchemaSyncer.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiDBSchemaSyncer.h>
#include <TIDB/Schema/SchemaSyncer.h>
#include <TIDB/Schema/TiDBSchemaSyncer.h>
#include <pingcap/pd/MockPDClient.h>

namespace DB
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/TiDB.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
#include <Poco/StringTokenizer.h>
#include <Storages/MutableSupport.h>
#include <Storages/Transaction/Collator.h>
#include <Storages/Transaction/SchemaNameMapper.h>
#include <Storages/Transaction/TiDB.h>
#include <TIDB/Schema/SchemaNameMapper.h>

namespace DB
{
Expand Down
Loading