Skip to content

Commit

Permalink
raftstore: optimize region destroy (tikv#13384)
Browse files Browse the repository at this point in the history
close tikv#12421

Optimize the performance of merging empty regions

Signed-off-by: tabokie <[email protected]>
  • Loading branch information
tabokie committed Sep 6, 2022
1 parent c518118 commit a3680a9
Show file tree
Hide file tree
Showing 6 changed files with 238 additions and 245 deletions.
4 changes: 0 additions & 4 deletions components/engine_panic/src/misc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ impl MiscExt for PanicEngine {
panic!()
}

fn roughly_cleanup_ranges(&self, ranges: &[(Vec<u8>, Vec<u8>)]) -> Result<()> {
panic!()
}

fn path(&self) -> &str {
panic!()
}
Expand Down
81 changes: 26 additions & 55 deletions components/engine_rocks/src/misc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ use engine_traits::{
CFNamesExt, DeleteStrategy, ImportExt, IterOptions, Iterable, Iterator, MiscExt, Mutable,
Range, Result, SstWriter, SstWriterBuilder, WriteBatch, WriteBatchExt, ALL_CFS,
};
use rocksdb::Range as RocksRange;
use tikv_util::{box_try, keybuilder::KeyBuilder};

use crate::{
Expand All @@ -28,17 +27,6 @@ impl RocksEngine {
) -> Result<()> {
let mut ranges = ranges.to_owned();
ranges.sort_by(|a, b| a.start_key.cmp(b.start_key));
let max_end_key = ranges
.iter()
.fold(ranges[0].end_key, |x, y| std::cmp::max(x, y.end_key));
let start = KeyBuilder::from_slice(ranges[0].start_key, 0, 0);
let end = KeyBuilder::from_slice(max_end_key, 0, 0);
let mut opts = IterOptions::new(Some(start), Some(end), false);
if self.is_titan() {
// Cause DeleteFilesInRange may expose old blob index keys, setting key only for Titan
// to avoid referring to missing blob files.
opts.set_key_only(true);
}

let mut writer_wrapper: Option<RocksSstWriter> = None;
let mut data: Vec<Vec<u8>> = vec![];
Expand All @@ -54,7 +42,17 @@ impl RocksEngine {
}
last_end_key = Some(r.end_key.to_owned());

let mut it = self.iterator_cf_opt(cf, opts.clone())?;
let mut opts = IterOptions::new(
Some(KeyBuilder::from_slice(r.start_key, 0, 0)),
Some(KeyBuilder::from_slice(r.end_key, 0, 0)),
false,
);
if self.is_titan() {
// Cause DeleteFilesInRange may expose old blob index keys, setting key only for
// Titan to avoid referring to missing blob files.
opts.set_key_only(true);
}
let mut it = self.iterator_cf_opt(cf, opts)?;
let mut it_valid = it.seek(r.start_key.into())?;
while it_valid {
if it.key() >= r.end_key {
Expand Down Expand Up @@ -225,28 +223,6 @@ impl MiscExt for RocksEngine {
Ok(used_size)
}

fn roughly_cleanup_ranges(&self, ranges: &[(Vec<u8>, Vec<u8>)]) -> Result<()> {
let db = self.as_inner();
let mut delete_ranges = Vec::new();
for &(ref start, ref end) in ranges {
if start == end {
continue;
}
assert!(start < end);
delete_ranges.push(RocksRange::new(start, end));
}
if delete_ranges.is_empty() {
return Ok(());
}

for cf in db.cf_names() {
let handle = util::get_cf_handle(db, cf)?;
db.delete_files_in_ranges_cf(handle, &delete_ranges, /* include_end */ false)?;
}

Ok(())
}

fn path(&self) -> &str {
self.as_inner().path()
}
Expand Down Expand Up @@ -364,13 +340,9 @@ mod tests {
}
}

fn test_delete_all_in_range(
strategy: DeleteStrategy,
origin_keys: &[Vec<u8>],
ranges: &[Range<'_>],
) {
fn test_delete_ranges(strategy: DeleteStrategy, origin_keys: &[Vec<u8>], ranges: &[Range<'_>]) {
let path = Builder::new()
.prefix("engine_delete_all_in_range")
.prefix("engine_delete_ranges")
.tempdir()
.unwrap();
let path_str = path.path().to_str().unwrap();
Expand Down Expand Up @@ -406,8 +378,7 @@ mod tests {
wb.write().unwrap();
check_data(&db, ALL_CFS, kvs.as_slice());

// Delete all in ranges.
db.delete_all_in_range(strategy, ranges).unwrap();
db.delete_ranges_cfs(strategy, ranges).unwrap();

let mut kvs_left: Vec<_> = kvs;
for r in ranges {
Expand All @@ -429,25 +400,25 @@ mod tests {
b"k4".to_vec(),
];
// Single range.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByRange,
&data,
&[Range::new(b"k1", b"k4")],
);
// Two ranges without overlap.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByRange,
&data,
&[Range::new(b"k0", b"k1"), Range::new(b"k3", b"k4")],
);
// Two ranges with overlap.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByRange,
&data,
&[Range::new(b"k1", b"k3"), Range::new(b"k2", b"k4")],
);
// One range contains the other range.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByRange,
&data,
&[Range::new(b"k1", b"k4"), Range::new(b"k2", b"k3")],
Expand All @@ -464,25 +435,25 @@ mod tests {
b"k4".to_vec(),
];
// Single range.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByKey,
&data,
&[Range::new(b"k1", b"k4")],
);
// Two ranges without overlap.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByKey,
&data,
&[Range::new(b"k0", b"k1"), Range::new(b"k3", b"k4")],
);
// Two ranges with overlap.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByKey,
&data,
&[Range::new(b"k1", b"k3"), Range::new(b"k2", b"k4")],
);
// One range contains the other range.
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByKey,
&data,
&[Range::new(b"k1", b"k4"), Range::new(b"k2", b"k3")],
Expand All @@ -501,7 +472,7 @@ mod tests {
for i in 1000..5000 {
data.push(i.to_string().as_bytes().to_vec());
}
test_delete_all_in_range(
test_delete_ranges(
DeleteStrategy::DeleteByWriter { sst_path },
&data,
&[
Expand Down Expand Up @@ -550,9 +521,9 @@ mod tests {
}
check_data(&db, ALL_CFS, kvs.as_slice());

db.delete_all_in_range(DeleteStrategy::DeleteFiles, &[Range::new(b"k2", b"k4")])
db.delete_ranges_cfs(DeleteStrategy::DeleteFiles, &[Range::new(b"k2", b"k4")])
.unwrap();
db.delete_all_in_range(DeleteStrategy::DeleteBlobs, &[Range::new(b"k2", b"k4")])
db.delete_ranges_cfs(DeleteStrategy::DeleteBlobs, &[Range::new(b"k2", b"k4")])
.unwrap();
check_data(&db, ALL_CFS, kvs_left.as_slice());
}
Expand Down Expand Up @@ -598,7 +569,7 @@ mod tests {
check_data(&db, &[cf], kvs.as_slice());

// Delete all in ["k2", "k4").
db.delete_all_in_range(
db.delete_ranges_cfs(
DeleteStrategy::DeleteByRange,
&[Range::new(b"kabcdefg2", b"kabcdefg4")],
)
Expand Down
27 changes: 13 additions & 14 deletions components/engine_traits/src/misc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,18 @@ use crate::{

#[derive(Clone, Debug)]
pub enum DeleteStrategy {
/// Delete the SST files that are fullly fit in range. However, the SST files that are partially
/// overlapped with the range will not be touched.
/// Delete the SST files that are fullly fit in range. However, the SST
/// files that are partially overlapped with the range will not be
/// touched.
///
/// Note:
/// - After this operation, some keys in the range might still exist in
/// the database.
/// - After this operation, some keys in the range might be removed from
/// existing snapshot, so you shouldn't expect to be able to read data
/// from the range using existing snapshots any more.
///
/// Ref: <https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys>
DeleteFiles,
/// Delete the data stored in Titan.
DeleteBlobs,
Expand All @@ -30,7 +40,7 @@ pub trait MiscExt: CFNamesExt + FlowControlFactorsExt {

fn flush_cf(&self, cf: &str, sync: bool) -> Result<()>;

fn delete_all_in_range(&self, strategy: DeleteStrategy, ranges: &[Range<'_>]) -> Result<()> {
fn delete_ranges_cfs(&self, strategy: DeleteStrategy, ranges: &[Range<'_>]) -> Result<()> {
for cf in self.cf_names() {
self.delete_ranges_cf(cf, strategy.clone(), ranges)?;
}
Expand All @@ -56,17 +66,6 @@ pub trait MiscExt: CFNamesExt + FlowControlFactorsExt {
///
fn get_engine_used_size(&self) -> Result<u64>;

/// Roughly deletes files in multiple ranges.
///
/// Note:
/// - After this operation, some keys in the range might still exist in the database.
/// - After this operation, some keys in the range might be removed from existing snapshot,
/// so you shouldn't expect to be able to read data from the range using existing snapshots
/// any more.
///
/// Ref: <https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys>
fn roughly_cleanup_ranges(&self, ranges: &[(Vec<u8>, Vec<u8>)]) -> Result<()>;

/// The path to the directory on the filesystem where the database is stored
fn path(&self) -> &str;

Expand Down
10 changes: 8 additions & 2 deletions components/raftstore/src/store/fsm/store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1221,8 +1221,14 @@ impl<EK: KvEngine, ER: RaftEngine, T> RaftPollerBuilder<EK, ER, T> {
last_start_key = keys::enc_end_key(region);
}
ranges.push((last_start_key, keys::DATA_MAX_KEY.to_vec()));
let ranges: Vec<_> = ranges
.iter()
.map(|(start, end)| Range::new(start, end))
.collect();

self.engines.kv.roughly_cleanup_ranges(&ranges)?;
self.engines
.kv
.delete_ranges_cfs(DeleteStrategy::DeleteFiles, &ranges)?;

info!(
"cleans up garbage data";
Expand Down Expand Up @@ -2779,7 +2785,7 @@ impl<'a, EK: KvEngine, ER: RaftEngine, T: Transport> StoreFsmDelegate<'a, EK, ER
}
drop(meta);

if let Err(e) = self.ctx.engines.kv.delete_all_in_range(
if let Err(e) = self.ctx.engines.kv.delete_ranges_cfs(
DeleteStrategy::DeleteByKey,
&[Range::new(&start_key, &end_key)],
) {
Expand Down
Loading

0 comments on commit a3680a9

Please sign in to comment.