Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix broken unit test for --gtest_filter='*StorageDeltaMergeTest*:*RegionKVStoreTest*' #4903

Merged
merged 3 commits into from
May 30, 2022
Merged

Fix broken unit test for --gtest_filter='*StorageDeltaMergeTest*:*RegionKVStoreTest*' #4903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
47153fc
Fix fail for --gtest_filter='*StorageDeltaMergeTest*:RegionKVStoreTes…
JaySon-Huang May 13, 2022
270514a
enable tests & clenup config for mock test
JaySon-Huang May 16, 2022
4988be5
Remove IStorage object from TiFlash context after dropped
JaySon-Huang May 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include <Storages/StorageDeltaMerge.h>
#include <Storages/StorageDeltaMergeHelpers.h>
#include <Storages/Transaction/RegionRangeKeys.h>
#include <Storages/Transaction/TiDB.h>
#include <Storages/Transaction/TiKVRange.h>
#include <Storages/Transaction/TiKVRecordFormat.h>
#include <TestUtils/FunctionTestUtils.h>
Expand Down Expand Up @@ -168,6 +169,8 @@ try
}
EXPECT_EQ(total_segment_rows, num_rows_read);
storage->drop();
// remove the storage from TiFlash context manually
storage->removeFromTMTContext();
}
CATCH

Expand Down Expand Up @@ -252,6 +255,8 @@ try
ASSERT_EQ(storage->getDatabaseName(), new_db_name);

storage->drop();
// remove the storage from TiFlash context manually
storage->removeFromTMTContext();
}
CATCH

Expand Down Expand Up @@ -315,6 +320,8 @@ try
ASSERT_EQ(sort_desc.front().nulls_direction, sort_desc2.front().nulls_direction);

storage->drop();
// remove the storage from TiFlash context manually
storage->removeFromTMTContext();
}
CATCH

Expand Down Expand Up @@ -609,6 +616,8 @@ try
sample.insert(DB::tests::createColumn<String>(
Strings(100, "a"),
"col2"));
constexpr TiDB::TableID table_id = 1;
const String table_name = fmt::format("t_{}", table_id);

Context ctx = DMTestEnv::getContext();
std::shared_ptr<StorageDeltaMerge> storage;
Expand All @@ -631,12 +640,11 @@ try
path.remove(true);

// primary_expr_ast
const String table_name = "t_1233";
ASTPtr astptr(new ASTIdentifier(table_name, ASTIdentifier::Kind::Table));
astptr->children.emplace_back(new ASTIdentifier("col1"));

TiDB::TableInfo tidb_table_info;
tidb_table_info.id = 1;
tidb_table_info.id = table_id;

storage = StorageDeltaMerge::create("TiFlash",
/* db_name= */ "default",
Expand Down Expand Up @@ -692,15 +700,17 @@ try
{
Field res;
c->get(i, res);
ASSERT(!res.isNull());
ASSERT(res.get<Int64>() == 1);
ASSERT_TRUE(!res.isNull());
ASSERT_EQ(res.get<Int64>(), table_id);
}
}
}
}
in->readSuffix();
ASSERT_EQ(num_rows_read, sample.rows());
storage->drop();
// remove the storage from TiFlash context manually
storage->removeFromTMTContext();
}
CATCH

Expand Down Expand Up @@ -848,6 +858,8 @@ try
ASSERT_LT(read_data(), num_rows_write);
}
storage->drop();
// remove the storage from TiFlash context manually
storage->removeFromTMTContext();
}
CATCH

Expand Down
23 changes: 20 additions & 3 deletions dbms/src/Storages/Transaction/tests/gtest_kvstore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@ RegionPtr makeRegion(UInt64 id, const std::string start_key, const std::string e
class RegionKVStoreTest : public ::testing::Test
{
public:
RegionKVStoreTest()
= default;
RegionKVStoreTest() = default;

static void SetUpTestCase() {}
static void testBasic();
Expand Down Expand Up @@ -1372,12 +1371,30 @@ void RegionKVStoreTest::testBasic()
}
}

TEST_F(RegionKVStoreTest, run)
TEST_F(RegionKVStoreTest, Basic)
try
{
testBasic();
}
CATCH

TEST_F(RegionKVStoreTest, KVStore)
try
{
testKVStore();
}
CATCH

TEST_F(RegionKVStoreTest, Region)
try
{
testRegion();
}
CATCH

TEST_F(RegionKVStoreTest, ReadIndex)
try
{
testReadIndex();
}
CATCH
Expand Down
13 changes: 2 additions & 11 deletions tests/docker/config/tics_dt.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,16 @@
# limitations under the License.

tmp_path = "/tmp/tiflash/data/tmp"
display_name = "TiFlash"
# specify paths used for store data, multiple path should be seperated by comma
path = "/tmp/tiflash/data/db"
capacity = "107374182400"
# multi-paths example
# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2"
# capacity = "0,0,0"
capacity = "107374182400" # 100GB
mark_cache_size = 5368709120
minmax_index_cache_size = 5368709120
tcp_port = 9000
http_port = 8123

[logger]
count = 10
errorlog = "/tmp/tiflash/log/error.log"
size = "1000M"
log = "/tmp/tiflash/log/server.log"
level = "trace"
[application]
runAsDaemon = true
[raft]
# specify which storage engine we use. tmt or dt
storage_engine = "dt"
69 changes: 0 additions & 69 deletions tests/docker/config/tiflash_dt_async_grpc.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,71 +13,15 @@
# limitations under the License.

tmp_path = "/tmp/tiflash/data/tmp"
display_name = "TiFlash"

## Deprecated storage path setting style. Check [storage] section for new style.
path = "/tmp/tiflash/data/db"
capacity = "10737418240"
## Deprecated storage path setting style of multi-disks. Check [storage] section for new style.
# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2"
# capacity = "0"

mark_cache_size = 5368709120
minmax_index_cache_size = 5368709120
tcp_port = 9000
http_port = 8123

## Storage paths settings.
# [storage]
## The storage format version in storage engine. Valid values: 1, 2 (experimental).
## format_version = 1

## If there are multiple SSD disks on the machine,
## specify the path list on `storage.main.dir` can improve TiFlash performance.

## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
## on the machine,
## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
## can improve TiFlash performance.

# [storage.main]
## The path to store main data.
# e.g.
# dir = [ "/data0/tiflash" ]
# or
# dir = [ "/data0/tiflash", "/data1/tiflash" ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
## Note that we don't support human-readable big numbers(like "10GB") yet.
## Please set in the specified number of bytes.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

# [storage.latest]
## The path(s) to store latest data.
## If not set, it will be the same with `storage.main.dir`.
# dir = [ ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

# [storage.raft]
## The path(s) to store Raft data.
## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore".
# dir = [ ]

# [storage.io_rate_limit]
## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled.
# max_bytes_per_sec = 268435456
## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec,
## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks.
# max_read_bytes_per_sec = 0
# max_write_bytes_per_sec = 0

[flash]
tidb_status_addr = "tidb0:10080"
service_addr = "0.0.0.0:3930"
Expand All @@ -100,22 +44,9 @@ size = "1000M"
log = "/tmp/tiflash/log/server.log"
level = "trace"

[application]
runAsDaemon = true

[raft]
pd_addr = "pd0:2379"
ignore_databases = "system,default"
# specify which storage engine we use. tmt or dt
storage_engine = "dt"
# Deprecated Raft data storage path setting style. Check [storage.raft] section for new style.
# If it is not set, it will be the first path of "path" appended with "/kvstore".
# kvstore_path = ""

[raft.snapshot]
# The way to apply snapshot data
# The value is one of "block" / "file1" / "file2".
# method = "file1"

[profiles]
[profiles.default]
Expand Down
69 changes: 0 additions & 69 deletions tests/docker/config/tiflash_dt_disable_local_tunnel.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,71 +13,15 @@
# limitations under the License.

tmp_path = "/tmp/tiflash/data/tmp"
display_name = "TiFlash"

## Deprecated storage path setting style. Check [storage] section for new style.
path = "/tmp/tiflash/data/db"
capacity = "10737418240"
## Deprecated storage path setting style of multi-disks. Check [storage] section for new style.
# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2"
# capacity = "0"

mark_cache_size = 5368709120
minmax_index_cache_size = 5368709120
tcp_port = 9000
http_port = 8123

## Storage paths settings.
# [storage]
## The storage format version in storage engine. Valid values: 1, 2 (experimental).
## format_version = 1

## If there are multiple SSD disks on the machine,
## specify the path list on `storage.main.dir` can improve TiFlash performance.

## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs)
## on the machine,
## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics)
## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics)
## can improve TiFlash performance.

# [storage.main]
## The path to store main data.
# e.g.
# dir = [ "/data0/tiflash" ]
# or
# dir = [ "/data0/tiflash", "/data1/tiflash" ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
## Note that we don't support human-readable big numbers(like "10GB") yet.
## Please set in the specified number of bytes.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

# [storage.latest]
## The path(s) to store latest data.
## If not set, it will be the same with `storage.main.dir`.
# dir = [ ]

## Store capacity of each path, i.e. max data size allowed.
## If it is not set, or is set to 0s, the actual disk capacity is used.
# e.g.
# capacity = [ 10737418240, 10737418240 ]

# [storage.raft]
## The path(s) to store Raft data.
## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore".
# dir = [ ]

# [storage.io_rate_limit]
## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled.
# max_bytes_per_sec = 268435456
## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec,
## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks.
# max_read_bytes_per_sec = 0
# max_write_bytes_per_sec = 0

[flash]
tidb_status_addr = "tidb0:10080"
service_addr = "0.0.0.0:3930"
Expand All @@ -100,22 +44,9 @@ size = "1000M"
log = "/tmp/tiflash/log/server.log"
level = "trace"

[application]
runAsDaemon = true

[raft]
pd_addr = "pd0:2379"
ignore_databases = "system,default"
# specify which storage engine we use. tmt or dt
storage_engine = "dt"
# Deprecated Raft data storage path setting style. Check [storage.raft] section for new style.
# If it is not set, it will be the first path of "path" appended with "/kvstore".
# kvstore_path = ""

[raft.snapshot]
# The way to apply snapshot data
# The value is one of "block" / "file1" / "file2".
# method = "file1"

[profiles]
[profiles.default]
Expand Down