From f4d4ea7d92ada6355cd56a6335fcc624c394650b Mon Sep 17 00:00:00 2001 From: Neha Deodhar Date: Tue, 28 Jan 2020 10:42:59 -0800 Subject: [PATCH] #3365: [Colocation] Use 4 byte PG table ID as dockey prefix Summary: Currently, colocated system tables use 16 byte cotable ID as doc key prefix. For colocated user tables, we want to optimize the storage space and use 4 byte PG table ID instead. This diff includes: 1) User colocated tables will use 4 byte PG table ID. This uses a new ValueTypeChar for PG table OIDs. 2) System tables will continue to use 16 byte cotable ID for backward compatibility. Test Plan: Jenkins Added pgtable ID tests to doc_key-test. Reviewers: mikhail, jason Reviewed By: jason Subscribers: yql, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D7817 --- src/yb/common/schema.cc | 28 ++++- src/yb/common/schema.h | 51 +++++++-- src/yb/docdb/doc_key-test.cc | 62 ++++++++--- src/yb/docdb/doc_key.cc | 127 ++++++++++++++++++++-- src/yb/docdb/doc_key.h | 48 +++++++- src/yb/docdb/doc_pgsql_scanspec.cc | 2 +- src/yb/docdb/doc_rowwise_iterator.cc | 31 ++++-- src/yb/docdb/docdb.cc | 11 ++ src/yb/docdb/docdb_test_util.cc | 3 +- src/yb/docdb/pgsql_operation.cc | 8 +- src/yb/docdb/primitive_value.cc | 15 +++ src/yb/docdb/primitive_value.h | 1 + src/yb/docdb/value_type.h | 1 + src/yb/master/catalog_manager.cc | 3 +- src/yb/master/master-path-handlers.cc | 4 - src/yb/master/master.proto | 3 +- src/yb/tablet/tablet-split-test.cc | 2 +- src/yb/tablet/tablet_metadata.cc | 27 +++-- src/yb/tserver/remote_bootstrap_client.cc | 2 +- 19 files changed, 352 insertions(+), 77 deletions(-) diff --git a/src/yb/common/schema.cc b/src/yb/common/schema.cc index 3ed6c7e08974..89a6b860e9a1 100644 --- a/src/yb/common/schema.cc +++ b/src/yb/common/schema.cc @@ -228,6 +228,10 @@ void Schema::CopyFrom(const Schema& other) { has_statics_ = other.has_statics_; table_properties_ = other.table_properties_; cotable_id_ = other.cotable_id_; + pgtable_id_ = other.pgtable_id_; + + // Schema cannot have both, cotable ID and pgtable ID. + DCHECK(cotable_id_.IsNil() || pgtable_id_ == 0); } void Schema::swap(Schema& other) { @@ -242,18 +246,24 @@ void Schema::swap(Schema& other) { std::swap(has_statics_, other.has_statics_); std::swap(table_properties_, other.table_properties_); std::swap(cotable_id_, other.cotable_id_); + std::swap(pgtable_id_, other.pgtable_id_); + + // Schema cannot have both, cotable ID or pgtable ID. + DCHECK(cotable_id_.IsNil() || pgtable_id_ == 0); } Status Schema::Reset(const vector& cols, const vector& ids, int key_columns, const TableProperties& table_properties, - const Uuid& cotable_id) { + const Uuid& cotable_id, + const PgTableOid pgtable_id) { cols_ = cols; num_key_columns_ = key_columns; num_hash_key_columns_ = 0; table_properties_ = table_properties; cotable_id_ = cotable_id; + pgtable_id_ = pgtable_id; // Determine whether any column is nullable or static, and count number of hash columns. has_nullables_ = false; @@ -285,6 +295,11 @@ Status Schema::Reset(const vector& cols, "The number of ids does not match with the number of columns"); } + if (PREDICT_FALSE(!cotable_id.IsNil() && pgtable_id > 0)) { + return STATUS(InvalidArgument, + "Bad schema", "Cannot have both cotable ID and pgtable ID"); + } + // Verify that the key columns are not nullable nor static for (int i = 0; i < key_columns; ++i) { if (PREDICT_FALSE(cols_[i].is_nullable())) { @@ -359,7 +374,7 @@ Status Schema::CreateProjectionByNames(const std::vector& col_name } cols.push_back(column(idx)); } - return out->Reset(cols, ids, num_key_columns, TableProperties(), cotable_id_); + return out->Reset(cols, ids, num_key_columns, TableProperties(), cotable_id_, pgtable_id_); } Status Schema::CreateProjectionByIdsIgnoreMissing(const std::vector& col_ids, @@ -374,7 +389,7 @@ Status Schema::CreateProjectionByIdsIgnoreMissing(const std::vector& c cols.push_back(column(idx)); filtered_col_ids.push_back(id); } - return out->Reset(cols, filtered_col_ids, 0, TableProperties(), cotable_id_); + return out->Reset(cols, filtered_col_ids, 0, TableProperties(), cotable_id_, pgtable_id_); } Schema Schema::CopyWithColumnIds() const { @@ -383,12 +398,12 @@ Schema Schema::CopyWithColumnIds() const { for (int32_t i = 0; i < num_columns(); i++) { ids.push_back(ColumnId(kFirstColumnId + i)); } - return Schema(cols_, ids, num_key_columns_, table_properties_, cotable_id_); + return Schema(cols_, ids, num_key_columns_, table_properties_, cotable_id_, pgtable_id_); } Schema Schema::CopyWithoutColumnIds() const { CHECK(has_column_ids()); - return Schema(cols_, num_key_columns_, table_properties_, cotable_id_); + return Schema(cols_, num_key_columns_, table_properties_, cotable_id_, pgtable_id_); } Status Schema::VerifyProjectionCompatibility(const Schema& projection) const { @@ -462,7 +477,8 @@ string Schema::ToString() const { JoinStrings(col_strs, ",\n\t"), "\n]\nproperties: ", tablet_properties_pb.ShortDebugString(), - cotable_id_.IsNil() ? "" : ("\ncotable_id: " + cotable_id_.ToString())); + cotable_id_.IsNil() ? "" : ("\ncotable_id: " + cotable_id_.ToString()), + pgtable_id_ == 0 ? "" : ("\npgtable_id: " + std::to_string(pgtable_id_))); } Status Schema::DecodeRowKey(Slice encoded_key, diff --git a/src/yb/common/schema.h b/src/yb/common/schema.h index aff506f1518b..c8eaf59a6b0b 100644 --- a/src/yb/common/schema.h +++ b/src/yb/common/schema.h @@ -482,6 +482,8 @@ class TableProperties { bool is_ysql_catalog_table_ = false; }; +typedef uint32_t PgTableOid; + // The schema for a set of rows. // // A Schema is simply a set of columns, along with information about @@ -507,7 +509,8 @@ class Schema { NameToIndexMap::key_equal(), NameToIndexMapAllocator(&name_to_index_bytes_)), has_nullables_(false), - cotable_id_(boost::uuids::nil_uuid()) { + cotable_id_(boost::uuids::nil_uuid()), + pgtable_id_(0) { } Schema(const Schema& other); @@ -526,14 +529,15 @@ class Schema { Schema(const vector& cols, int key_columns, const TableProperties& table_properties = TableProperties(), - const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid())) + const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid()), + const PgTableOid pgtable_id = 0) : name_to_index_bytes_(0), // TODO: C++11 provides a single-arg constructor name_to_index_(10, NameToIndexMap::hasher(), NameToIndexMap::key_equal(), NameToIndexMapAllocator(&name_to_index_bytes_)) { - CHECK_OK(Reset(cols, key_columns, table_properties, cotable_id)); + CHECK_OK(Reset(cols, key_columns, table_properties, cotable_id, pgtable_id)); } // Construct a schema with the given information. @@ -546,14 +550,15 @@ class Schema { const vector& ids, int key_columns, const TableProperties& table_properties = TableProperties(), - const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid())) + const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid()), + const PgTableOid pgtable_id = 0) : name_to_index_bytes_(0), // TODO: C++11 provides a single-arg constructor name_to_index_(10, NameToIndexMap::hasher(), NameToIndexMap::key_equal(), NameToIndexMapAllocator(&name_to_index_bytes_)) { - CHECK_OK(Reset(cols, ids, key_columns, table_properties, cotable_id)); + CHECK_OK(Reset(cols, ids, key_columns, table_properties, cotable_id, pgtable_id)); } // Reset this Schema object to the given schema. @@ -561,9 +566,10 @@ class Schema { // state and may not be used. CHECKED_STATUS Reset(const vector& cols, int key_columns, const TableProperties& table_properties = TableProperties(), - const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid())) { + const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid()), + const PgTableOid pgtable_id = 0) { std::vector ids; - return Reset(cols, ids, key_columns, table_properties, cotable_id); + return Reset(cols, ids, key_columns, table_properties, cotable_id, pgtable_id); } // Reset this Schema object to the given schema. @@ -573,7 +579,8 @@ class Schema { const vector& ids, int key_columns, const TableProperties& table_properties = TableProperties(), - const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid())); + const Uuid& cotable_id = Uuid(boost::uuids::nil_uuid()), + const PgTableOid pgtable_id = 0); // Return the number of bytes needed to represent a single row of this schema. // @@ -762,10 +769,34 @@ class Schema { return cotable_id_; } + bool has_cotable_id() const { + return !cotable_id_.IsNil(); + } + void set_cotable_id(const Uuid& cotable_id) { + if (!cotable_id.IsNil()) { + DCHECK_EQ(pgtable_id_, 0); + } cotable_id_ = cotable_id; } + // Gets and sets the PG table OID of the non-primary table this schema belongs to in a tablet + // with colocated tables. + const PgTableOid pgtable_id() const { + return pgtable_id_; + } + + bool has_pgtable_id() const { + return pgtable_id_ > 0; + } + + void set_pgtable_id(const PgTableOid pgtable_id) { + if (pgtable_id > 0) { + DCHECK(cotable_id_.IsNil()); + } + pgtable_id_ = pgtable_id; + } + // Extract a given column from a row where the type is // known at compile-time. The type is checked with a debug // assertion -- but if the wrong type is used and these assertions @@ -1090,6 +1121,10 @@ class Schema { // primary or single-tenant table. Uuid cotable_id_; + // PG table OID of the non-primary table this schema belongs to in a tablet with colocated + // tables. Nil for the primary or single-tenant table. + PgTableOid pgtable_id_; + // NOTE: if you add more members, make sure to add the appropriate // code to swap() and CopyFrom() as well to prevent subtle bugs. }; diff --git a/src/yb/docdb/doc_key-test.cc b/src/yb/docdb/doc_key-test.cc index 29e950931ece..2efd5695c221 100644 --- a/src/yb/docdb/doc_key-test.cc +++ b/src/yb/docdb/doc_key-test.cc @@ -61,16 +61,25 @@ class DocKeyTest : public YBTest { vector sub_doc_keys; Uuid cotable_id; EXPECT_OK(cotable_id.FromHexString("0123456789abcdef0123456789abcdef")); - for (bool has_cotable_id : {false, true}) { + + std::vector> table_id_pairs; + table_id_pairs.emplace_back(cotable_id, 0); + table_id_pairs.emplace_back(Uuid(boost::uuids::nil_uuid()), 9911); + table_id_pairs.emplace_back(Uuid(boost::uuids::nil_uuid()), 0); + + for (const auto& table_id_pair : table_id_pairs) { for (int num_hash_keys = 0; num_hash_keys <= kMaxNumHashKeys; ++num_hash_keys) { for (int num_range_keys = 0; num_range_keys <= kMaxNumRangeKeys; ++num_range_keys) { for (int num_sub_keys = 0; num_sub_keys <= kMaxNumSubKeys; ++num_sub_keys) { for (int has_hybrid_time = 0; has_hybrid_time <= 1; ++has_hybrid_time) { SubDocKey sub_doc_key; - if (has_cotable_id) { + if (!table_id_pair.first.IsNil()) { sub_doc_key.doc_key().set_cotable_id(cotable_id); + } else if (table_id_pair.second > 0) { + sub_doc_key.doc_key().set_pgtable_id(table_id_pair.second); } + if (num_hash_keys > 0) { sub_doc_key.doc_key().set_hash(kAsciiFriendlyHash); } @@ -464,13 +473,19 @@ TEST_F(DocKeyTest, TestDecodePrefixLengths) { size_t expected_hash_enc_size = 0; const DocKey& doc_key = sub_doc_key.doc_key(); DocKey hash_only_key; - if (!doc_key.hashed_group().empty() || doc_key.has_cotable_id()) { + if (!doc_key.hashed_group().empty() || doc_key.has_cotable_id() || doc_key.has_pgtable_id()) { if (doc_key.has_cotable_id()) { if (doc_key.hashed_group().empty()) { hash_only_key = DocKey(doc_key.cotable_id()); } else { hash_only_key = DocKey(doc_key.cotable_id(), doc_key.hash(), doc_key.hashed_group()); } + } else if (doc_key.has_pgtable_id()) { + if (doc_key.hashed_group().empty()) { + hash_only_key = DocKey(doc_key.pgtable_id()); + } else { + hash_only_key = DocKey(doc_key.pgtable_id(), doc_key.hash(), doc_key.hashed_group()); + } } else { hash_only_key = DocKey(doc_key.hash(), doc_key.hashed_group()); } @@ -484,14 +499,16 @@ TEST_F(DocKeyTest, TestDecodePrefixLengths) { SubDocKey cur_key; boost::container::small_vector prefix_lengths; std::vector expected_prefix_lengths; - if (doc_key.has_hash() || doc_key.has_cotable_id()) { - if (doc_key.has_hash() && doc_key.has_cotable_id()) { - cur_key.doc_key() = DocKey(doc_key.cotable_id(), doc_key.hash(), doc_key.hashed_group()); - } else if (doc_key.has_hash()) { + if (doc_key.has_hash() || doc_key.has_cotable_id() || doc_key.has_pgtable_id()) { + if (doc_key.has_hash()) { cur_key.doc_key() = DocKey(doc_key.hash(), doc_key.hashed_group()); - } else { - cur_key.doc_key() = DocKey(doc_key.cotable_id()); } + if (doc_key.has_cotable_id()) { + cur_key.doc_key().set_cotable_id(doc_key.cotable_id()); + } else if (doc_key.has_pgtable_id()) { + cur_key.doc_key().set_pgtable_id(doc_key.pgtable_id()); + } + // Subtract one to avoid counting the final kGroupEnd, unless this is the entire key. if (doc_key.range_group().empty()) { expected_prefix_lengths.push_back(cur_key.Encode().size()); @@ -567,20 +584,31 @@ TEST_F(DocKeyTest, TestEnumerateIntents) { std::vector expected_intents; SubDocKey current_expected_intent; - if (sub_doc_key.doc_key().has_cotable_id()) { - DocKey cotable_id_only_doc_key; - cotable_id_only_doc_key.set_cotable_id(sub_doc_key.doc_key().cotable_id()); - current_expected_intent = SubDocKey(cotable_id_only_doc_key); + if (sub_doc_key.doc_key().has_cotable_id() || sub_doc_key.doc_key().has_pgtable_id()) { + DocKey table_id_only_doc_key; + if (sub_doc_key.doc_key().has_cotable_id()) { + table_id_only_doc_key.set_cotable_id(sub_doc_key.doc_key().cotable_id()); + } else { + table_id_only_doc_key.set_pgtable_id(sub_doc_key.doc_key().pgtable_id()); + } + current_expected_intent = SubDocKey(table_id_only_doc_key); expected_intents.push_back(current_expected_intent); } else { expected_intents.push_back(SubDocKey()); } if (!sub_doc_key.doc_key().hashed_group().empty()) { - current_expected_intent = SubDocKey(DocKey( - sub_doc_key.doc_key().cotable_id(), - sub_doc_key.doc_key().hash(), - sub_doc_key.doc_key().hashed_group())); + if (sub_doc_key.doc_key().has_cotable_id()) { + current_expected_intent = SubDocKey(DocKey( + sub_doc_key.doc_key().cotable_id(), + sub_doc_key.doc_key().hash(), + sub_doc_key.doc_key().hashed_group())); + } else { + current_expected_intent = SubDocKey(DocKey( + sub_doc_key.doc_key().pgtable_id(), + sub_doc_key.doc_key().hash(), + sub_doc_key.doc_key().hashed_group())); + } expected_intents.push_back(current_expected_intent); } diff --git a/src/yb/docdb/doc_key.cc b/src/yb/docdb/doc_key.cc index 7aa1c2f55061..978c0b968c98 100644 --- a/src/yb/docdb/doc_key.cc +++ b/src/yb/docdb/doc_key.cc @@ -116,11 +116,12 @@ Status ConsumePrimitiveValuesFromKey(Slice* slice, std::vector* // DocKey // ------------------------------------------------------------------------------------------------ -DocKey::DocKey() : cotable_id_(boost::uuids::nil_uuid()), hash_present_(false) { +DocKey::DocKey() : cotable_id_(boost::uuids::nil_uuid()), pgtable_id_(0), hash_present_(false) { } DocKey::DocKey(std::vector range_components) : cotable_id_(boost::uuids::nil_uuid()), + pgtable_id_(0), hash_present_(false), range_group_(std::move(range_components)) { } @@ -129,6 +130,7 @@ DocKey::DocKey(DocKeyHash hash, std::vector hashed_components, std::vector range_components) : cotable_id_(boost::uuids::nil_uuid()), + pgtable_id_(0), hash_present_(true), hash_(hash), hashed_group_(std::move(hashed_components)), @@ -140,6 +142,19 @@ DocKey::DocKey(const Uuid& cotable_id, std::vector hashed_components, std::vector range_components) : cotable_id_(cotable_id), + pgtable_id_(0), + hash_present_(true), + hash_(hash), + hashed_group_(std::move(hashed_components)), + range_group_(std::move(range_components)) { +} + +DocKey::DocKey(const PgTableOid pgtable_id, + DocKeyHash hash, + std::vector hashed_components, + std::vector range_components) + : cotable_id_(boost::uuids::nil_uuid()), + pgtable_id_(pgtable_id), hash_present_(true), hash_(hash), hashed_group_(std::move(hashed_components)), @@ -148,23 +163,34 @@ DocKey::DocKey(const Uuid& cotable_id, DocKey::DocKey(const Uuid& cotable_id) : cotable_id_(cotable_id), + pgtable_id_(0), + hash_present_(false), + hash_(0) { +} + +DocKey::DocKey(const PgTableOid pgtable_id) + : cotable_id_(boost::uuids::nil_uuid()), + pgtable_id_(pgtable_id), hash_present_(false), hash_(0) { } DocKey::DocKey(const Schema& schema) : cotable_id_(schema.cotable_id()), + pgtable_id_(schema.pgtable_id()), hash_present_(false) { } DocKey::DocKey(const Schema& schema, DocKeyHash hash) : cotable_id_(schema.cotable_id()), + pgtable_id_(schema.pgtable_id()), hash_present_(true), hash_(hash) { } DocKey::DocKey(const Schema& schema, std::vector range_components) : cotable_id_(schema.cotable_id()), + pgtable_id_(schema.pgtable_id()), hash_present_(false), range_group_(std::move(range_components)) { } @@ -173,6 +199,7 @@ DocKey::DocKey(const Schema& schema, DocKeyHash hash, std::vector hashed_components, std::vector range_components) : cotable_id_(schema.cotable_id()), + pgtable_id_(schema.pgtable_id()), hash_present_(true), hash_(hash), hashed_group_(std::move(hashed_components)), @@ -204,8 +231,12 @@ RefCntPrefix DocKey::EncodeAsRefCntPrefix() const { } void DocKey::AppendTo(KeyBytes* out) const { - DocKeyEncoder(out).CotableId(cotable_id_).Hash(hash_present_, hash_, hashed_group_). - Range(range_group_); + auto encoder = DocKeyEncoder(out); + if (!cotable_id_.IsNil()) { + encoder.CotableId(cotable_id_).Hash(hash_present_, hash_, hashed_group_).Range(range_group_); + } else { + encoder.PgtableId(pgtable_id_).Hash(hash_present_, hash_, hashed_group_).Range(range_group_); + } } void DocKey::Clear() { @@ -241,6 +272,8 @@ class DecodeDocKeyCallback { void SetCoTableId(const Uuid cotable_id) const {} + void SetPgTableId(const PgTableOid pgtable_id) const {} + private: boost::container::small_vector_base* out_; }; @@ -259,6 +292,8 @@ class DummyCallback { void SetCoTableId(const Uuid cotable_id) const {} + void SetPgTableId(const PgTableOid pgtable_id) const {} + PrimitiveValue* AddSubkey() const { return nullptr; } @@ -281,6 +316,8 @@ class EncodedSizesCallback { void SetCoTableId(const Uuid cotable_id) const {} + void SetPgTableId(const PgTableOid pgtable_id) const {} + PrimitiveValue* AddSubkey() const { return nullptr; } @@ -309,6 +346,7 @@ yb::Status DocKey::PartiallyDecode(Slice *slice, Result DocKey::DecodeHash(const Slice& slice) { DocKeyDecoder decoder(slice); RETURN_NOT_OK(decoder.DecodeCotableId()); + RETURN_NOT_OK(decoder.DecodePgtableId()); uint16_t hash; RETURN_NOT_OK(decoder.DecodeHashCode(&hash)); return hash; @@ -356,6 +394,10 @@ class DocKey::DecodeFromCallback { key_->cotable_id_ = cotable_id; } + void SetPgTableId(const PgTableOid pgtable_id) const { + key_->pgtable_id_ = pgtable_id; + } + private: DocKey* key_; }; @@ -381,8 +423,11 @@ yb::Status DocKey::DoDecode(DocKeyDecoder* decoder, AllowSpecial allow_special, const Callback& callback) { Uuid cotable_id; + PgTableOid pgtable_id; if (VERIFY_RESULT(decoder->DecodeCotableId(&cotable_id))) { callback.SetCoTableId(cotable_id); + } else if (VERIFY_RESULT(decoder->DecodePgtableId(&pgtable_id))) { + callback.SetPgTableId(pgtable_id); } uint16_t hash_code; @@ -428,7 +473,12 @@ string DocKey::ToString() const { result += "CoTableId="; result += cotable_id_.ToString(); result += ", "; + } else if (pgtable_id_ > 0) { + result += "PgTableId="; + result += std::to_string(pgtable_id_); + result += ", "; } + if (hash_present_) { result += StringPrintf("0x%04x", hash_); result += ", "; @@ -443,6 +493,7 @@ string DocKey::ToString() const { bool DocKey::operator ==(const DocKey& other) const { return cotable_id_ == other.cotable_id_ && + pgtable_id_ == other.pgtable_id_ && HashedComponentsEqual(other) && range_group_ == other.range_group_; } @@ -466,6 +517,9 @@ int DocKey::CompareTo(const DocKey& other) const { int result = CompareUsingLessThan(cotable_id_, other.cotable_id_); if (result != 0) return result; + result = CompareUsingLessThan(pgtable_id_, other.pgtable_id_); + if (result != 0) return result; + result = CompareUsingLessThan(hash_present_, other.hash_present_); if (result != 0) return result; @@ -936,14 +990,30 @@ const rocksdb::FilterPolicy::KeyTransformer* DocDbAwareFilterPolicy::GetKeyTrans return &HashedComponentsExtractor::GetInstance(); } -DocKeyEncoderAfterCotableIdStep DocKeyEncoder::CotableId(const Uuid& cotable_id) { +DocKeyEncoderAfterTableIdStep DocKeyEncoder::CotableId(const Uuid& cotable_id) { if (!cotable_id.IsNil()) { std::string bytes; cotable_id.EncodeToComparable(&bytes); out_->AppendValueType(ValueType::kTableId); out_->AppendRawBytes(bytes); } - return DocKeyEncoderAfterCotableIdStep(out_); + return DocKeyEncoderAfterTableIdStep(out_); +} + +DocKeyEncoderAfterTableIdStep DocKeyEncoder::PgtableId(const PgTableOid pgtable_id) { + if (pgtable_id > 0) { + out_->AppendValueType(ValueType::kPgTableOid); + out_->AppendUInt32(pgtable_id); + } + return DocKeyEncoderAfterTableIdStep(out_); +} + +DocKeyEncoderAfterTableIdStep DocKeyEncoder::Schema(const class Schema& schema) { + if (schema.pgtable_id() > 0) { + return PgtableId(schema.pgtable_id()); + } else { + return CotableId(schema.cotable_id()); + } } Result DocKeyDecoder::DecodeCotableId(Uuid* uuid) { @@ -966,6 +1036,29 @@ Result DocKeyDecoder::DecodeCotableId(Uuid* uuid) { return true; } +Result DocKeyDecoder::DecodePgtableId(PgTableOid* pgtable_id) { + if (input_.empty() || input_[0] != ValueTypeAsChar::kPgTableOid) { + return false; + } + + input_.consume_byte(); + + if (input_.size() < sizeof(PgTableOid)) { + return STATUS_FORMAT( + Corruption, "Not enough bytes for pgtable id: $0", input_.ToDebugHexString()); + } + + static_assert( + sizeof(PgTableOid) == sizeof(uint32_t), + "It looks like the pgtable ID's size has changed -- need to update encoder/decoder."); + if (pgtable_id) { + *pgtable_id = BigEndian::Load32(input_.data()); + } + input_.remove_prefix(sizeof(PgTableOid)); + + return true; +} + Result DocKeyDecoder::DecodeHashCode(uint16_t* out, AllowSpecial allow_special) { if (input_.empty()) { return false; @@ -1034,6 +1127,7 @@ Result DocKeyDecoder::HasPrimitiveValue() { Status DocKeyDecoder::DecodeToRangeGroup() { RETURN_NOT_OK(DecodeCotableId()); + RETURN_NOT_OK(DecodePgtableId()); if (VERIFY_RESULT(DecodeHashCode())) { while (VERIFY_RESULT(HasPrimitiveValue())) { RETURN_NOT_OK(DecodePrimitiveValue()); @@ -1064,6 +1158,8 @@ Result HashedComponentsEqual(const Slice& lhs, const Slice& rhs) { DocKeyDecoder rhs_decoder(rhs); RETURN_NOT_OK(lhs_decoder.DecodeCotableId()); RETURN_NOT_OK(rhs_decoder.DecodeCotableId()); + RETURN_NOT_OK(lhs_decoder.DecodePgtableId()); + RETURN_NOT_OK(rhs_decoder.DecodePgtableId()); bool hash_present = VERIFY_RESULT(lhs_decoder.DecodeHashCode(AllowSpecial::kTrue)); RETURN_NOT_OK(rhs_decoder.DecodeHashCode(AllowSpecial::kTrue)); @@ -1102,9 +1198,10 @@ Result HashedComponentsEqual(const Slice& lhs, const Slice& rhs) { } bool DocKeyBelongsTo(Slice doc_key, const Schema& schema) { - bool has_table_id = !doc_key.empty() && doc_key[0] == ValueTypeAsChar::kTableId; + bool has_table_id = !doc_key.empty() && + (doc_key[0] == ValueTypeAsChar::kTableId || doc_key[0] == ValueTypeAsChar::kPgTableOid); - if (schema.cotable_id().IsNil()) { + if (schema.cotable_id().IsNil() && schema.pgtable_id() == 0) { return !has_table_id; } @@ -1112,11 +1209,19 @@ bool DocKeyBelongsTo(Slice doc_key, const Schema& schema) { return false; } - doc_key.consume_byte(); + if (doc_key[0] == ValueTypeAsChar::kTableId) { + doc_key.consume_byte(); - uint8_t bytes[kUuidSize]; - schema.cotable_id().EncodeToComparable(bytes); - return doc_key.starts_with(Slice(bytes, kUuidSize)); + uint8_t bytes[kUuidSize]; + schema.cotable_id().EncodeToComparable(bytes); + return doc_key.starts_with(Slice(bytes, kUuidSize)); + } else { + DCHECK(doc_key[0] == ValueTypeAsChar::kPgTableOid); + doc_key.consume_byte(); + char buf[sizeof(PgTableOid)]; + BigEndian::Store32(buf, schema.pgtable_id()); + return doc_key.starts_with(Slice(buf, sizeof(PgTableOid))); + } } const KeyBounds KeyBounds::kNoBounds; diff --git a/src/yb/docdb/doc_key.h b/src/yb/docdb/doc_key.h index 21c138dd83cb..43b4246d0745 100644 --- a/src/yb/docdb/doc_key.h +++ b/src/yb/docdb/doc_key.h @@ -95,8 +95,15 @@ class DocKey { std::vector hashed_components, std::vector range_components = std::vector()); + DocKey(PgTableOid pgtable_id, + DocKeyHash hash, + std::vector hashed_components, + std::vector range_components = std::vector()); + explicit DocKey(const Uuid& cotable_id); + explicit DocKey(PgTableOid pgtable_id); + // Constructors to create a DocKey for the given schema to support co-located tables. explicit DocKey(const Schema& schema); DocKey(const Schema& schema, DocKeyHash hash); @@ -130,6 +137,14 @@ class DocKey { return !cotable_id_.IsNil(); } + const PgTableOid pgtable_id() const { + return pgtable_id_; + } + + bool has_pgtable_id() const { + return pgtable_id_ > 0; + } + DocKeyHash hash() const { return hash_; } @@ -223,13 +238,28 @@ class DocKey { } bool BelongsTo(const Schema& schema) const { - return cotable_id_ == schema.cotable_id(); + if (!cotable_id_.IsNil()) { + return cotable_id_ == schema.cotable_id(); + } else if (pgtable_id_ > 0) { + return pgtable_id_ == schema.pgtable_id(); + } + return schema.cotable_id().IsNil() && schema.pgtable_id() == 0; } void set_cotable_id(const Uuid& cotable_id) { + if (!cotable_id.IsNil()) { + DCHECK_EQ(pgtable_id_, 0); + } cotable_id_ = cotable_id; } + void set_pgtable_id(const PgTableOid pgtable_id) { + if (pgtable_id > 0) { + DCHECK(cotable_id_.IsNil()); + } + pgtable_id_ = pgtable_id; + } + void set_hash(DocKeyHash hash) { hash_ = hash; hash_present_ = true; @@ -257,6 +287,10 @@ class DocKey { // primary or single-tenant table. Uuid cotable_id_; + // Postgres table OID of the non-primary table this DocKey belongs to in colocated tables. + // 0 for primary or single tenant table. + PgTableOid pgtable_id_; + // TODO: can we get rid of this field and just use !hashed_group_.empty() instead? bool hash_present_; @@ -286,9 +320,9 @@ class DocKeyEncoderAfterHashStep { KeyBytes* out_; }; -class DocKeyEncoderAfterCotableIdStep { +class DocKeyEncoderAfterTableIdStep { public: - explicit DocKeyEncoderAfterCotableIdStep(KeyBytes* out) : out_(out) { + explicit DocKeyEncoderAfterTableIdStep(KeyBytes* out) : out_(out) { } template @@ -331,7 +365,11 @@ class DocKeyEncoder { public: explicit DocKeyEncoder(KeyBytes* out) : out_(out) {} - DocKeyEncoderAfterCotableIdStep CotableId(const Uuid& cotable_id); + DocKeyEncoderAfterTableIdStep CotableId(const Uuid& cotable_id); + + DocKeyEncoderAfterTableIdStep PgtableId(const PgTableOid pgtable_id); + + DocKeyEncoderAfterTableIdStep Schema(const Schema& schema); private: KeyBytes* out_; @@ -342,6 +380,8 @@ class DocKeyDecoder { explicit DocKeyDecoder(const Slice& input) : input_(input) {} Result DecodeCotableId(Uuid* uuid = nullptr); + Result DecodePgtableId(PgTableOid* pgtable_id = nullptr); + Result HasPrimitiveValue(); Result DecodeHashCode( diff --git a/src/yb/docdb/doc_pgsql_scanspec.cc b/src/yb/docdb/doc_pgsql_scanspec.cc index a238eb300333..9184acc39053 100644 --- a/src/yb/docdb/doc_pgsql_scanspec.cc +++ b/src/yb/docdb/doc_pgsql_scanspec.cc @@ -210,7 +210,7 @@ void DocPgsqlScanSpec::InitRangeOptions(const PgsqlConditionPB& condition) { KeyBytes DocPgsqlScanSpec::bound_key(const Schema& schema, const bool lower_bound) const { KeyBytes result; - auto encoder = DocKeyEncoder(&result).CotableId(schema.cotable_id()); + auto encoder = DocKeyEncoder(&result).Schema(schema); // If no hashed_component use hash lower/upper bounds if set. if (hashed_components_->empty()) { diff --git a/src/yb/docdb/doc_rowwise_iterator.cc b/src/yb/docdb/doc_rowwise_iterator.cc index 4d1dd6e4c15d..6f53df112670 100644 --- a/src/yb/docdb/doc_rowwise_iterator.cc +++ b/src/yb/docdb/doc_rowwise_iterator.cc @@ -498,7 +498,7 @@ Status DocRowwiseIterator::Init() { doc_db_, BloomFilterMode::DONT_USE_BLOOM_FILTER, boost::none /* user_key_for_filter */, query_id, txn_op_context_, deadline_, read_time_); - DocKeyEncoder(&iter_key_).CotableId(schema_.cotable_id()); + DocKeyEncoder(&iter_key_).Schema(schema_); row_key_ = iter_key_; row_hash_key_ = row_key_; VLOG(3) << __PRETTY_FUNCTION__ << " Seeking to " << row_key_; @@ -792,6 +792,7 @@ Status DocRowwiseIterator::DoNextRow(const Schema& projection, QLTableRow* table DocKeyDecoder decoder(row_key_); RETURN_NOT_OK(decoder.DecodeCotableId()); + RETURN_NOT_OK(decoder.DecodePgtableId()); bool has_hash_components = VERIFY_RESULT(decoder.DecodeHashCode()); // Populate the key column values from the doc key. The key column values in doc key were @@ -851,26 +852,36 @@ CHECKED_STATUS DocRowwiseIterator::GetNextReadSubDocKey(SubDocKey* sub_doc_key) } Result DocRowwiseIterator::GetTupleId() const { - // Return tuple id without cotable id if any. + // Return tuple id without cotable id / pgtable id if any. Slice tuple_id = row_key_; if (tuple_id.starts_with(ValueTypeAsChar::kTableId)) { tuple_id.remove_prefix(1 + kUuidSize); + } else if (tuple_id.starts_with(ValueTypeAsChar::kPgTableOid)) { + tuple_id.remove_prefix(1 + sizeof(PgTableOid)); } return tuple_id; } Result DocRowwiseIterator::SeekTuple(const Slice& tuple_id) { - // If cotable id is present in the table schema, we need to prepend it in the tuple key to seek. - if (!schema_.cotable_id().IsNil()) { + // If cotable id / pgtable id is present in the table schema, then + // we need to prepend it in the tuple key to seek. + if (schema_.has_cotable_id() || schema_.has_pgtable_id()) { + uint32_t size = schema_.has_pgtable_id() ? sizeof(PgTableOid) : kUuidSize; if (!tuple_key_) { - std::string bytes; - schema_.cotable_id().EncodeToComparable(&bytes); tuple_key_.emplace(); - tuple_key_->Reserve(1 + kUuidSize + tuple_id.size()); - tuple_key_->AppendValueType(ValueType::kTableId); - tuple_key_->AppendRawBytes(bytes); + tuple_key_->Reserve(1 + size + tuple_id.size()); + + if (schema_.has_cotable_id()) { + std::string bytes; + schema_.cotable_id().EncodeToComparable(&bytes); + tuple_key_->AppendValueType(ValueType::kTableId); + tuple_key_->AppendRawBytes(bytes); + } else { + tuple_key_->AppendValueType(ValueType::kPgTableOid); + tuple_key_->AppendUInt32(schema_.pgtable_id()); + } } else { - tuple_key_->Truncate(1 + kUuidSize); + tuple_key_->Truncate(1 + size); } tuple_key_->AppendRawBytes(tuple_id); db_iter_->Seek(*tuple_key_); diff --git a/src/yb/docdb/docdb.cc b/src/yb/docdb/docdb.cc index 33a1147e9d65..7f5a04c555c7 100644 --- a/src/yb/docdb/docdb.cc +++ b/src/yb/docdb/docdb.cc @@ -444,6 +444,7 @@ Status EnumerateWeakIntents( } const bool has_cotable_id = *key.cdata() == ValueTypeAsChar::kTableId; + const bool has_pgtable_id = *key.cdata() == ValueTypeAsChar::kPgTableOid; { bool is_table_root_key = false; if (has_cotable_id) { @@ -456,6 +457,16 @@ Status EnumerateWeakIntents( } encoded_key_buffer->AppendRawBytes(key.cdata(), kUuidSize + 1); is_table_root_key = key[kUuidSize + 1] == ValueTypeAsChar::kGroupEnd; + } else if (has_pgtable_id) { + const auto kMinExpectedSize = sizeof(PgTableOid) + 2; + if (key.size() < kMinExpectedSize) { + return STATUS_FORMAT( + Corruption, + "Expected an encoded SubDocKey starting with a pgtable id to be at least $0 bytes long", + kMinExpectedSize); + } + encoded_key_buffer->AppendRawBytes(key.cdata(), sizeof(PgTableOid) + 1); + is_table_root_key = key[sizeof(PgTableOid) + 1] == ValueTypeAsChar::kGroupEnd; } else { is_table_root_key = *key.cdata() == ValueTypeAsChar::kGroupEnd; } diff --git a/src/yb/docdb/docdb_test_util.cc b/src/yb/docdb/docdb_test_util.cc index 4064d687c2da..a252ca94d5fe 100644 --- a/src/yb/docdb/docdb_test_util.cc +++ b/src/yb/docdb/docdb_test_util.cc @@ -238,7 +238,8 @@ vector GenRandomPrimitiveValues(RandomNumberGenerator* rng, int } DocKey CreateMinimalDocKey(RandomNumberGenerator* rng, UseHash use_hash) { - return use_hash ? DocKey(static_cast((*rng)()), {}, {}) : DocKey(); + return use_hash ? DocKey(static_cast((*rng)()), std::vector(), + std::vector()) : DocKey(); } DocKey GenRandomDocKey(RandomNumberGenerator* rng, UseHash use_hash) { diff --git a/src/yb/docdb/pgsql_operation.cc b/src/yb/docdb/pgsql_operation.cc index 1382073f1cb2..3d000cfced35 100644 --- a/src/yb/docdb/pgsql_operation.cc +++ b/src/yb/docdb/pgsql_operation.cc @@ -315,10 +315,12 @@ Status PgsqlWriteOperation::PopulateResultSet(const QLTableRow::SharedPtr& table for (const PgsqlExpressionPB& expr : request_.targets()) { if (expr.has_column_id()) { if (expr.column_id() == static_cast(PgSystemAttrNum::kYBTupleId)) { - // Strip cotable id from the serialized DocKey before returning it as ybctid. + // Strip cotable id / pgtable id from the serialized DocKey before returning it as ybctid. Slice tuple_id = encoded_doc_key_.as_slice(); if (tuple_id.starts_with(ValueTypeAsChar::kTableId)) { tuple_id.remove_prefix(1 + kUuidSize); + } else if (tuple_id.starts_with(ValueTypeAsChar::kPgTableOid)) { + tuple_id.remove_prefix(1 + sizeof(PgTableOid)); } rsrow->rscol(rscol_index)->set_binary_value(tuple_id.data(), tuple_id.size()); } else { @@ -565,14 +567,14 @@ Status PgsqlReadOperation::GetIntents(const Schema& schema, KeyValueWriteBatchPB // Empty components mean that we don't have primary key at all, but request // could still contain hash_code as part of tablet routing. // So we should ignore it. - DocKey doc_key(schema.cotable_id()); + DocKey doc_key(schema); pair->set_key(doc_key.Encode().data()); } else { std::vector hashed_components; RETURN_NOT_OK(InitKeyColumnPrimitiveValues( request_.partition_column_values(), schema, 0 /* start_idx */, &hashed_components)); - DocKey doc_key(schema.cotable_id(), request_.hash_code(), hashed_components); + DocKey doc_key(schema, request_.hash_code(), hashed_components); pair->set_key(doc_key.Encode().data()); } diff --git a/src/yb/docdb/primitive_value.cc b/src/yb/docdb/primitive_value.cc index 472d1427d662..f973f60273c7 100644 --- a/src/yb/docdb/primitive_value.cc +++ b/src/yb/docdb/primitive_value.cc @@ -18,6 +18,7 @@ #include #include "yb/common/jsonb.h" +#include "yb/common/schema.h" #include "yb/common/ql_value.h" #include "yb/docdb/doc_kv_util.h" @@ -210,6 +211,8 @@ string PrimitiveValue::ToString() const { return "[]"; case ValueType::kTableId: return Format("TableId($0)", uuid_val_.ToString()); + case ValueType::kPgTableOid: + return Format("PgTableOid($0)", uint32_val_); case ValueType::kTransactionId: return Substitute("TransactionId($0)", uuid_val_.ToString()); case ValueType::kWriteId: @@ -272,6 +275,7 @@ void PrimitiveValue::AppendToKey(KeyBytes* key_bytes) const { key_bytes->AppendInt32(int32_val_); return; + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32: key_bytes->AppendUInt32(uint32_val_); return; @@ -446,6 +450,7 @@ string PrimitiveValue::ToValue() const { AppendBigEndianUInt32(int32_val_, &result); return result; + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32Descending: FALLTHROUGH_INTENDED; case ValueType::kUInt32: AppendBigEndianUInt32(uint32_val_, &result); @@ -723,6 +728,7 @@ Status PrimitiveValue::DecodeKey(rocksdb::Slice* slice, PrimitiveValue* out) { type_ref = value_type; return Status::OK(); + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32Descending: FALLTHROUGH_INTENDED; case ValueType::kUInt32: if (slice->size() < sizeof(uint32_t)) { @@ -1021,6 +1027,7 @@ Status PrimitiveValue::DecodeFromValue(const rocksdb::Slice& rocksdb_slice) { int32_val_ = BigEndian::Load32(slice.data()); return Status::OK(); + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32: FALLTHROUGH_INTENDED; case ValueType::kUInt32Descending: if (slice.size() != sizeof(uint32_t)) { @@ -1269,6 +1276,12 @@ PrimitiveValue PrimitiveValue::TableId(Uuid table_id) { return primitive_value; } +PrimitiveValue PrimitiveValue::PgTableOid(const yb::PgTableOid pgtable_id) { + PrimitiveValue primitive_value(pgtable_id); + primitive_value.type_ = ValueType::kPgTableOid; + return primitive_value; +} + PrimitiveValue PrimitiveValue::Jsonb(const std::string& json) { PrimitiveValue primitive_value; primitive_value.type_ = ValueType::kJsonb; @@ -1310,6 +1323,7 @@ bool PrimitiveValue::operator==(const PrimitiveValue& other) const { case ValueType::kWriteId: FALLTHROUGH_INTENDED; case ValueType::kInt32: return int32_val_ == other.int32_val_; + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32Descending: FALLTHROUGH_INTENDED; case ValueType::kUInt32: return uint32_val_ == other.uint32_val_; @@ -1392,6 +1406,7 @@ int PrimitiveValue::CompareTo(const PrimitiveValue& other) const { return CompareUsingLessThan(int32_val_, other.int32_val_); case ValueType::kUInt32Descending: return CompareUsingLessThan(other.uint32_val_, uint32_val_); + case ValueType::kPgTableOid: FALLTHROUGH_INTENDED; case ValueType::kUInt32: return CompareUsingLessThan(uint32_val_, other.uint32_val_); case ValueType::kUInt64Descending: diff --git a/src/yb/docdb/primitive_value.h b/src/yb/docdb/primitive_value.h index 678d29834249..efaca7bc9cbd 100644 --- a/src/yb/docdb/primitive_value.h +++ b/src/yb/docdb/primitive_value.h @@ -246,6 +246,7 @@ class PrimitiveValue { static PrimitiveValue UInt64(uint64_t v, SortOrder sort_order = SortOrder::kAscending); static PrimitiveValue TransactionId(Uuid transaction_id); static PrimitiveValue TableId(Uuid table_id); + static PrimitiveValue PgTableOid(const PgTableOid pgtable_id); static PrimitiveValue Jsonb(const std::string& json); KeyBytes ToKeyBytes() const; diff --git a/src/yb/docdb/value_type.h b/src/yb/docdb/value_type.h index fd2474df280e..131eeea3b283 100644 --- a/src/yb/docdb/value_type.h +++ b/src/yb/docdb/value_type.h @@ -127,6 +127,7 @@ namespace docdb { ((kWriteId, 'w')) /* ASCII code 119 */ \ ((kTransactionId, 'x')) /* ASCII code 120 */ \ ((kTableId, 'y')) /* ASCII code 121 */ \ + ((kPgTableOid, 'z')) /* ASCII code 122 */ \ \ ((kObject, '{')) /* ASCII code 123 */ \ \ diff --git a/src/yb/master/catalog_manager.cc b/src/yb/master/catalog_manager.cc index 99f9950626b5..6f23b73438d2 100644 --- a/src/yb/master/catalog_manager.cc +++ b/src/yb/master/catalog_manager.cc @@ -3757,7 +3757,8 @@ bool CatalogManager::IsUserCreatedTable(const TableInfo& table) const { bool CatalogManager::IsUserCreatedTableUnlocked(const TableInfo& table) const { if (table.GetTableType() == PGSQL_TABLE_TYPE || table.GetTableType() == YQL_TABLE_TYPE) { if (!IsSystemTableUnlocked(table) && !IsSequencesSystemTable(table) && - GetNamespaceNameUnlocked(table.namespace_id()) != kSystemNamespaceName) { + GetNamespaceNameUnlocked(table.namespace_id()) != kSystemNamespaceName && + !IsColocatedParentTable(table)) { return true; } } diff --git a/src/yb/master/master-path-handlers.cc b/src/yb/master/master-path-handlers.cc index c916fbf69bcd..0b74403f4905 100644 --- a/src/yb/master/master-path-handlers.cc +++ b/src/yb/master/master-path-handlers.cc @@ -681,10 +681,6 @@ void MasterPathHandlers::HandleCatalogManager(const Webserver::WebRequest& req, continue; } - if (master_->catalog_manager()->IsColocatedParentTable(*table)) { - continue; - } - table_cat = kUserTable; string keyspace = master_->catalog_manager()->GetNamespaceName(table->namespace_id()); bool is_platform = keyspace.compare(kSystemPlatformNamespace) == 0; diff --git a/src/yb/master/master.proto b/src/yb/master/master.proto index 1261e98ae4b5..2b439df0347d 100644 --- a/src/yb/master/master.proto +++ b/src/yb/master/master.proto @@ -817,7 +817,8 @@ message CreateTableRequestPB { optional bool is_pg_catalog_table = 14 [ default = false ]; // Is this a sys catalog table? optional bool is_pg_shared_table = 15 [ default = false ]; // Is this a shared table? - optional bool colocated = 17 [ default = true ]; // Is this a colocated table? + // Is this a colocated table? This field is only applicable for a colocated database. + optional bool colocated = 17 [ default = true ]; } message CreateTableResponsePB { diff --git a/src/yb/tablet/tablet-split-test.cc b/src/yb/tablet/tablet-split-test.cc index 1f270224881a..ececc2d3f931 100644 --- a/src/yb/tablet/tablet-split-test.cc +++ b/src/yb/tablet/tablet-split-test.cc @@ -125,7 +125,7 @@ TEST_F(TabletSplitTest, v) { LOG(INFO) << "Split hash code: " << split_hash_code; const auto partition_key = PartitionSchema::EncodeMultiColumnHashValue(split_hash_code); docdb::KeyBytes encoded_doc_key; - docdb::DocKeyEncoderAfterCotableIdStep(&encoded_doc_key).Hash( + docdb::DocKeyEncoderAfterTableIdStep(&encoded_doc_key).Hash( split_hash_code, std::vector()); partition.TEST_set_partition_key_end(partition_key); key_bounds.upper = encoded_doc_key; diff --git a/src/yb/tablet/tablet_metadata.cc b/src/yb/tablet/tablet_metadata.cc index 36d130fa7f02..e19e2efa8c5a 100644 --- a/src/yb/tablet/tablet_metadata.cc +++ b/src/yb/tablet/tablet_metadata.cc @@ -40,6 +40,7 @@ #include #include "yb/rocksdb/db.h" #include "yb/rocksdb/options.h" +#include "yb/common/entity_ids.h" #include "yb/common/wire_protocol.h" #include "yb/consensus/opid_util.h" #include "yb/docdb/docdb_rocksdb_util.h" @@ -171,11 +172,16 @@ Status KvStoreInfo::LoadTablesFromPB( auto table_info = std::make_unique(); RETURN_NOT_OK(table_info->LoadFromPB(table_pb)); if (table_info->table_id != primary_table_id) { - Uuid cotable_id; - CHECK_OK(cotable_id.FromHexString(table_info->table_id)); - // TODO(#79): when adding for multiple KV-stores per Raft group support - check if we need - // to set cotable ID. - table_info->schema.set_cotable_id(cotable_id); + if (table_pb.schema().table_properties().is_ysql_catalog_table()) { + Uuid cotable_id; + CHECK_OK(cotable_id.FromHexString(table_info->table_id)); + // TODO(#79): when adding for multiple KV-stores per Raft group support - check if we need + // to set cotable ID. + table_info->schema.set_cotable_id(cotable_id); + } else { + auto pgtable_id = VERIFY_RESULT(GetPgsqlTableOid(table_info->table_id)); + table_info->schema.set_pgtable_id(pgtable_id); + } } tables[table_info->table_id] = std::move(table_info); } @@ -674,9 +680,14 @@ void RaftGroupMetadata::AddTable(const std::string& table_id, schema_version, partition_schema)); if (table_id != primary_table_id_) { - Uuid cotable_id; - CHECK_OK(cotable_id.FromHexString(table_id)); - new_table_info->schema.set_cotable_id(cotable_id); + if (schema.table_properties().is_ysql_catalog_table()) { + Uuid cotable_id; + CHECK_OK(cotable_id.FromHexString(table_id)); + new_table_info->schema.set_cotable_id(cotable_id); + } else { + auto result = CHECK_RESULT(GetPgsqlTableOid(table_id)); + new_table_info->schema.set_pgtable_id(result); + } } std::lock_guard lock(data_mutex_); auto& tables = kv_store_.tables; diff --git a/src/yb/tserver/remote_bootstrap_client.cc b/src/yb/tserver/remote_bootstrap_client.cc index eddba269fb95..b32903d68465 100644 --- a/src/yb/tserver/remote_bootstrap_client.cc +++ b/src/yb/tserver/remote_bootstrap_client.cc @@ -325,7 +325,7 @@ Status RemoteBootstrapClient::Start(const string& bootstrap_peer_uuid, // Create the superblock on disk. if (ts_manager != nullptr && !colocated) { // TODO: GetAndRegisterDataAndWalDir is in charge of load balancing the disk storage. - // Collocated tables are mostly for small clusters, single nodes. But we still need to make + // Colocated tables are mostly for small clusters, single nodes. But we still need to make // sure that if the node has more than one data disk, we are balancing the storage correctly. ts_manager->GetAndRegisterDataAndWalDir(&fs_manager(), table_id,