From 3842074a38a2835e0084f876d48fdecc87c8881a Mon Sep 17 00:00:00 2001 From: tabokie Date: Mon, 14 Aug 2023 14:12:40 +0200 Subject: [PATCH] fix build and test Signed-off-by: tabokie --- db/db_bloom_filter_test.cc | 22 ++++++++++++++-------- db/db_compaction_test.cc | 8 -------- db/db_impl/db_impl_compaction_flush.cc | 11 +++++------ db/db_impl/db_impl_open.cc | 12 +++++------- db/version_set.cc | 5 +---- db/version_set_test.cc | 10 ++++++++++ 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc index f75a190f23e9..9451d1686425 100644 --- a/db/db_bloom_filter_test.cc +++ b/db/db_bloom_filter_test.cc @@ -2067,9 +2067,11 @@ TEST_F(DBBloomFilterTest, PrefixScan) { } TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { + const int kNumKeysPerFlush = 1000; + Options options = CurrentOptions(); - options.write_buffer_size = 64 * 1024; - options.arena_block_size = 4 * 1024; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFlush)); options.target_file_size_base = 64 * 1024; options.level0_file_num_compaction_trigger = 2; options.level0_slowdown_writes_trigger = 2; @@ -2083,7 +2085,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { options.level_compaction_dynamic_level_bytes = true; BlockBasedTableOptions bbto; bbto.cache_index_and_filter_blocks = true; - bbto.filter_policy.reset(NewBloomFilterPolicy(10, true)); + bbto.filter_policy.reset(NewBloomFilterPolicy(10)); bbto.whole_key_filtering = true; options.table_factory.reset(NewBlockBasedTableFactory(bbto)); options.optimize_filters_for_hits = true; @@ -2101,12 +2103,17 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { for (int i = 0; i < numkeys; i += 2) { keys.push_back(i); } - RandomShuffle(std::begin(keys), std::end(keys)); + RandomShuffle(std::begin(keys), std::end(keys), /*seed*/ 42); int num_inserted = 0; for (int key : keys) { ASSERT_OK(Put(1, Key(key), "val")); - if (++num_inserted % 1000 == 0) { - ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + num_inserted++; + // The write after each `kNumKeysPerFlush` keys triggers a flush. Always + // wait for that flush and any follow-on compactions for deterministic LSM + // shape. + if (num_inserted > kNumKeysPerFlush && + num_inserted % kNumKeysPerFlush == 1) { + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1])); ASSERT_OK(dbfull()->TEST_WaitForCompact()); } } @@ -2225,8 +2232,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) { BottommostLevelCompaction::kSkip; compact_options.change_level = true; compact_options.target_level = 7; - ASSERT_TRUE(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr) - .IsNotSupported()); + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); ASSERT_GE(trivial_move, 1); ASSERT_EQ(non_trivial_move, 0); diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index 12fe18bc947a..b7f56310e15d 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -5718,20 +5718,12 @@ TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) { ASSERT_EQ("0,1", FilesPerLevel(0)); } } else { -<<<<<<< HEAD - // Just trivial move from level 0 -> 1 - ASSERT_EQ("0," + ToString(kSstNum), FilesPerLevel(0)); -||||||| parent of 43e9a60bb (Always allow L0->L1 trivial move during manual compaction (#11375)) - // Just trivial move from level 0 -> 1 - ASSERT_EQ("0," + std::to_string(kSstNum), FilesPerLevel(0)); -======= // Just trivial move from level 0 -> 1/base if (dynamic_level) { ASSERT_EQ("0,0,0,0,0,0," + std::to_string(kSstNum), FilesPerLevel(0)); } else { ASSERT_EQ("0," + std::to_string(kSstNum), FilesPerLevel(0)); } ->>>>>>> 43e9a60bb (Always allow L0->L1 trivial move during manual compaction (#11375)) } } diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 334bbbf99dac..56fd4d26d606 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1120,8 +1120,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, s = RunManualCompaction( cfd, first_overlapped_level, first_overlapped_level, options, begin, end, exclusive, true /* disallow_trivial_move */, - std::numeric_limits::max() /* max_file_num_to_ignore */, - trim_ts); + std::numeric_limits::max() /* max_file_num_to_ignore */); final_output_level = first_overlapped_level; } else { assert(cfd->ioptions()->compaction_style == kCompactionStyleLevel); @@ -1166,9 +1165,8 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, // files down. s = RunManualCompaction( cfd, level, output_level, options, begin, end, exclusive, - !trim_ts.empty() /* disallow_trivial_move */, + false /* disallow_trivial_move */, std::numeric_limits::max() /* max_file_num_to_ignore */, - trim_ts, output_level == ColumnFamilyData::kCompactToBaseLevel ? &base_level : nullptr); @@ -1202,7 +1200,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options, s = RunManualCompaction( cfd, final_output_level, final_output_level, options, begin, end, exclusive, true /* disallow_trivial_move */, - next_file_number /* max_file_num_to_ignore */, trim_ts); + next_file_number /* max_file_num_to_ignore */); } } } @@ -1941,7 +1939,8 @@ Status DBImpl::RunManualCompaction( assert(!exclusive || !manual_conflict); if (!scheduled) { // There is a conflicting compaction - if (manual_compaction_paused_ > 0 || manual.canceled == true) { + if (manual_compaction_paused_ > 0 || + (manual.canceled != nullptr && *manual.canceled == true)) { // Stop waiting since it was canceled. Pretend the error came from // compaction so the below cleanup/error handling code can process it. manual.done = true; diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 4558ac9d201a..66b9b7485b21 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -545,8 +545,7 @@ Status DBImpl::Recover( // allow_ingest_behind does not support Level Compaction, // and per_key_placement can have infinite compaction loop for Level // Compaction. Adjust to_level here just to be safe. - if (cfd->ioptions()->allow_ingest_behind || - cfd->ioptions()->preclude_last_level_data_seconds > 0) { + if (cfd->ioptions()->allow_ingest_behind) { to_level -= 1; } // Whether this column family has a level trivially moved @@ -600,9 +599,9 @@ Status DBImpl::Recover( f->temperature, // this can be different from // `last_level_temperature` f->oldest_blob_file_number, f->oldest_ancester_time, - f->file_creation_time, f->epoch_number, - f->file_checksum, f->file_checksum_func_name, - f->unique_id, f->compensated_range_deletion_size); + f->file_creation_time, f->file_checksum, + f->file_checksum_func_name, f->min_timestamp, + f->max_timestamp); ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] Moving #%" PRIu64 " from from_level-%d to from_level-%d %" PRIu64 @@ -610,14 +609,13 @@ Status DBImpl::Recover( cfd->GetName().c_str(), f->fd.GetNumber(), from_level, to_level, f->fd.GetFileSize()); } - recovery_ctx->UpdateVersionEdits(cfd, edit); } --to_level; } } } } - s = SetupDBId(read_only, recovery_ctx); + s = SetDBId(read_only); ROCKS_LOG_INFO(immutable_db_options_.info_log, "DB ID: %s\n", db_id_.c_str()); if (s.ok() && !read_only) { s = DeleteUnreferencedSstFiles(); diff --git a/db/version_set.cc b/db/version_set.cc index 0f833573c31b..d85aec213d2e 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -3816,9 +3816,7 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions, cur_level_size = static_cast( cur_level_size / options.max_bytes_for_level_multiplier); if (lowest_unnecessary_level_ == -1 && - cur_level_size <= base_bytes_min && - (ioptions.preclude_last_level_data_seconds == 0 || - i < num_levels_ - 2)) { + cur_level_size <= base_bytes_min) { // When per_key_placement is enabled, the penultimate level is // necessary. lowest_unnecessary_level_ = i; @@ -3833,7 +3831,6 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions, // which can less than base_bytes_min AND necessary, // or there is some unnecessary level. assert(first_non_empty_level == num_levels_ - 1 || - ioptions.preclude_last_level_data_seconds > 0 || lowest_unnecessary_level_ != -1); // Case 1. If we make target size of last level to be max_level_size, // target size of the first non-empty level would be smaller than diff --git a/db/version_set_test.cc b/db/version_set_test.cc index d4f280f1da15..bbc90a44bab7 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -204,6 +204,16 @@ class VersionStorageInfoTestBase : public testing::Test { } return result; } + + void UpdateVersionStorageInfo() { + vstorage_.UpdateFilesByCompactionPri(ioptions_, mutable_cf_options_); + vstorage_.UpdateNumNonEmptyLevels(); + vstorage_.GenerateFileIndexer(); + vstorage_.GenerateLevelFilesBrief(); + vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_); + vstorage_.GenerateLevel0NonOverlapping(); + vstorage_.SetFinalized(); + } }; class VersionStorageInfoTest : public VersionStorageInfoTestBase {