Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

*: Add some comments about decoding #5158

Merged
merged 3 commits into from
Jun 18, 2022
Merged

*: Add some comments about decoding #5158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dfe31a2
Add some comments about decoding
JaySon-Huang Jun 16, 2022
bfc5cd6
add analyze
JaySon-Huang Jun 18, 2022
64d1213
fix
JaySon-Huang Jun 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions dbms/src/Debug/dbgFuncMockRaftSnapshot.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)
{
const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
TableID table_id = RegionBench::getTableID(context, database_name, table_name, "");
MockTiDB::TablePtr table = MockTiDB::instance().getTableByName(database_name, table_name);
auto & table_info = table->table_info;
Expand All @@ -70,8 +70,8 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)

if (!is_common_handle)
{
HandleID start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
HandleID end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
region = RegionBench::createRegion(table_id, region_id, start, end);
}
else
Expand Down Expand Up @@ -105,8 +105,8 @@ RegionPtr GenDbgRegionSnapshotWithData(Context & context, const ASTs & args)
for (auto it = args_begin; it != args_end; it += len)
{
HandleID handle_id = is_common_handle ? 0 : static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[0]).value));
Timestamp tso = static_cast<Timestamp>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[1]).value));
UInt8 del = static_cast<UInt8>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[2]).value));
auto tso = static_cast<Timestamp>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[1]).value));
auto del = static_cast<UInt8>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*it[2]).value));
{
std::vector<Field> fields;

Expand Down Expand Up @@ -168,7 +168,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotWithData(Context & context, const AST
// DBGInvoke region_snapshot(region-id, start-key, end-key, database-name, table-name[, partition-id])
void MockRaftCommand::dbgFuncRegionSnapshot(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value));
bool has_partition_id = false;
size_t args_size = args.size();
if (dynamic_cast<ASTLiteral *>(args[args_size - 1].get()) != nullptr)
Expand Down Expand Up @@ -214,15 +214,15 @@ void MockRaftCommand::dbgFuncRegionSnapshot(Context & context, const ASTs & args
}
else
{
HandleID start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[1]).value));
HandleID end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[1]).value));
auto end = static_cast<HandleID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
start_key = RecordKVFormat::genKey(table_id, start);
end_key = RecordKVFormat::genKey(table_id, end);
}
region_info.set_start_key(start_key.toString());
region_info.set_end_key(end_key.toString());
*region_info.add_peers() = createPeer(1, true);
*region_info.add_peers() = createPeer(2, true);
*region_info.add_peers() = tests::createPeer(1, true);
*region_info.add_peers() = tests::createPeer(2, true);
auto peer_id = 1;
auto start_decoded_key = RecordKVFormat::decodeTiKVKey(start_key);
auto end_decoded_key = RecordKVFormat::decodeTiKVKey(end_key);
Expand Down Expand Up @@ -432,9 +432,9 @@ void MockRaftCommand::dbgFuncIngestSST(Context & context, const ASTs & args, DBG
{
const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
RegionID start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
RegionID end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
MockTiDB::TablePtr table = MockTiDB::instance().getTableByName(database_name, table_name);

const auto & table_info = RegionBench::getTableInfo(context, database_name, table_name);
Expand Down Expand Up @@ -555,7 +555,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotApplyBlock(Context & context, const A
throw Exception("Args not matched, should be: region-id", ErrorCodes::BAD_ARGUMENTS);
}

RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto [region, block_cache] = GLOBAL_REGION_MAP.popRegionCache("__snap_" + std::to_string(region_id));
auto & tmt = context.getTMTContext();
context.getTMTContext().getKVStore()->checkAndApplySnapshot<RegionPtrWithBlock>({region, std::move(block_cache)}, tmt);
Expand All @@ -577,12 +577,12 @@ void MockRaftCommand::dbgFuncRegionSnapshotPreHandleDTFiles(Context & context, c

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
RegionID start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
RegionID end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto start_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[3]).value));
auto end_handle = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[4]).value));

const String schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[5]).value);
String handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[6]).value);
const auto schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[5]).value);
auto handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[6]).value);

UInt64 test_fields = 1;
if (args.size() > 7)
Expand Down Expand Up @@ -677,10 +677,10 @@ void MockRaftCommand::dbgFuncRegionSnapshotPreHandleDTFilesWithHandles(Context &

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[2]).value));

const String schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[3]).value);
String handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[4]).value);
const auto schema_str = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[3]).value);
auto handle_pk_name = safeGet<String>(typeid_cast<const ASTLiteral &>(*args[4]).value);

std::vector<UInt64> handles;
for (size_t i = 5; i < args.size(); ++i)
Expand Down Expand Up @@ -770,7 +770,7 @@ void MockRaftCommand::dbgFuncRegionSnapshotApplyDTFiles(Context & context, const
if (args.size() != 1)
throw Exception("Args not matched, should be: region-id", ErrorCodes::BAD_ARGUMENTS);

RegionID region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
auto region_id = static_cast<RegionID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args.front()).value));
const auto region_name = "__snap_snap_" + std::to_string(region_id);
auto [new_region, ingest_ids] = GLOBAL_REGION_MAP.popRegionSnap(region_name);
auto & tmt = context.getTMTContext();
Expand Down
4 changes: 4 additions & 0 deletions dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,20 +306,23 @@ try
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 128,
.padded_size = 0,
.tag = 0,
.offset = 1024,
.checksum = 0x4567,
});
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
});
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = file_id2,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
Expand Down Expand Up @@ -402,6 +405,7 @@ try
blob_store.blob_stats.restoreByEntry(PageEntryV3{
.file_id = id,
.size = 1024,
.padded_size = 0,
.tag = 0,
.offset = 0,
.checksum = 0x4567,
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,11 @@ struct DecodingStorageSchemaSnapshot
{
auto pk_pos_iter = pk_pos_map.begin();
size_t column_pos_in_block = 0;
for (auto iter = sorted_column_id_with_pos.begin(); iter != sorted_column_id_with_pos.end(); iter++)
for (auto & column_id_with_pos : sorted_column_id_with_pos)
{
if (pk_pos_iter == pk_pos_map.end())
break;
if (pk_pos_iter->first == iter->first)
if (pk_pos_iter->first == column_id_with_pos.first)
{
pk_pos_iter->second = column_pos_in_block;
pk_pos_iter++;
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/Transaction/PartitionStreams.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

#include <Common/Allocator.h>
#include <Common/Exception.h>
#include <Common/FailPoint.h>
#include <Common/TiFlashMetrics.h>
#include <Common/setThreadName.h>
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/Transaction/RegionBlockReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d
}
else
{
// For common handle, sometimes we need to decode the value from encoded key instead of encoded value
auto * raw_extra_column = const_cast<IColumn *>((block.getByPosition(extra_handle_column_pos)).column.get());
raw_extra_column->insertData(pk->data(), pk->size());
/// decode key and insert pk columns if needed
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/RegionBlockReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class Block;
class RegionBlockReader : private boost::noncopyable
{
public:
RegionBlockReader(DecodingStorageSchemaSnapshotConstPtr schema_snapshot_);
explicit RegionBlockReader(DecodingStorageSchemaSnapshotConstPtr schema_snapshot_);

/// Read `data_list` as a block.
///
Expand Down
43 changes: 26 additions & 17 deletions dbms/src/Storages/Transaction/RowCodec.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ bool appendRowV2ToBlock(
ColumnID pk_handle_id,
bool force_decode)
{
UInt8 row_flag = readLittleEndian<UInt8>(&raw_value[1]);
auto row_flag = readLittleEndian<UInt8>(&raw_value[1]);
bool is_big = row_flag & RowV2::BigRowMask;
return is_big ? appendRowV2ToBlockImpl<true>(raw_value, column_ids_iter, column_ids_iter_end, block, block_column_pos, column_infos, pk_handle_id, force_decode)
: appendRowV2ToBlockImpl<false>(raw_value, column_ids_iter, column_ids_iter_end, block, block_column_pos, column_infos, pk_handle_id, force_decode);
Expand Down Expand Up @@ -360,9 +360,10 @@ bool appendRowV2ToBlockImpl(
decodeUInts<ColumnID, typename RowV2::Types<is_big>::ColumnIDType>(cursor, raw_value, num_null_columns, null_column_ids);
decodeUInts<size_t, typename RowV2::Types<is_big>::ValueOffsetType>(cursor, raw_value, num_not_null_columns, value_offsets);
size_t values_start_pos = cursor;
size_t id_not_null = 0, id_null = 0;
size_t idx_not_null = 0;
size_t idx_null = 0;
// Merge ordered not null/null columns to keep order.
while (id_not_null < not_null_column_ids.size() || id_null < null_column_ids.size())
while (idx_not_null < not_null_column_ids.size() || idx_null < null_column_ids.size())
{
if (column_ids_iter == column_ids_iter_end)
{
Expand All @@ -371,24 +372,31 @@ bool appendRowV2ToBlockImpl(
}

bool is_null;
if (id_not_null < not_null_column_ids.size() && id_null < null_column_ids.size())
is_null = not_null_column_ids[id_not_null] > null_column_ids[id_null];
if (idx_not_null < not_null_column_ids.size() && idx_null < null_column_ids.size())
is_null = not_null_column_ids[idx_not_null] > null_column_ids[idx_null];
else
is_null = id_null < null_column_ids.size();
is_null = idx_null < null_column_ids.size();

auto next_datum_column_id = is_null ? null_column_ids[id_null] : not_null_column_ids[id_not_null];
auto next_datum_column_id = is_null ? null_column_ids[idx_null] : not_null_column_ids[idx_not_null];
if (column_ids_iter->first > next_datum_column_id)
{
// extra column
// The next column id to read is bigger than the column id of next datum in encoded row.
// It means this is the datum of extra column. May happen when reading after dropping
// a column.
if (!force_decode)
return false;
// Ignore the extra column and continue to parse other datum
if (is_null)
id_null++;
idx_null++;
else
id_not_null++;
idx_not_null++;
}
else if (column_ids_iter->first < next_datum_column_id)
{
// The next column id to read is less than the column id of next datum in encoded row.
// It means this is the datum of missing column. May happen when reading after adding
// a column.
// Fill with default value and continue to read data for next column id.
const auto & column_info = column_infos[column_ids_iter->second];
if (!addDefaultValueToColumnIfPossible(column_info, block, block_column_pos, force_decode))
return false;
Expand All @@ -397,23 +405,24 @@ bool appendRowV2ToBlockImpl(
}
else
{
// if pk_handle_id is a valid column id, then it means the table's pk_is_handle is true
// If pk_handle_id is a valid column id, then it means the table's pk_is_handle is true
// we can just ignore the pk value encoded in value part
if (unlikely(column_ids_iter->first == pk_handle_id))
{
column_ids_iter++;
block_column_pos++;
if (is_null)
{
id_null++;
idx_null++;
}
else
{
id_not_null++;
idx_not_null++;
}
continue;
}

// Parse the datum.
auto * raw_column = const_cast<IColumn *>((block.getByPosition(block_column_pos)).column.get());
const auto & column_info = column_infos[column_ids_iter->second];
if (is_null)
Expand All @@ -432,15 +441,15 @@ bool appendRowV2ToBlockImpl(
}
// ColumnNullable::insertDefault just insert a null value
raw_column->insertDefault();
id_null++;
idx_null++;
}
else
{
size_t start = id_not_null ? value_offsets[id_not_null - 1] : 0;
size_t length = value_offsets[id_not_null] - start;
size_t start = idx_not_null ? value_offsets[idx_not_null - 1] : 0;
size_t length = value_offsets[idx_not_null] - start;
if (!raw_column->decodeTiDBRowV2Datum(values_start_pos + start, raw_value, length, force_decode))
return false;
id_not_null++;
idx_not_null++;
}
column_ids_iter++;
block_column_pos++;
Expand Down
Loading