Adapt RocksDB 7.6 (#95)

build.sh

@@ -36,7 +36,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}
 # Note: if you don't have a good reason, please do not set -DPORTABLE=ON
 #
 # This one is set here on purpose of compatibility with github action runtime processor
-rocksdb_version="7.5.3"
+rocksdb_version="7.6.0"
 cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \
     mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
     -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \

c.h

@@ -1302,6 +1302,17 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_blob_file_starting_level(
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_cache(
     rocksdb_options_t* opt, rocksdb_cache_t* blob_cache);
 
+enum {
+  rocksdb_prepopulate_blob_disable = 0,
+  rocksdb_prepopulate_blob_flush_only = 1
+};
+
+extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prepopulate_blob_cache(
+    rocksdb_options_t* opt, int val);
+
+extern ROCKSDB_LIBRARY_API int rocksdb_options_get_prepopulate_blob_cache(
+    rocksdb_options_t* opt);
+
 /* returns a pointer to a malloc()-ed, null terminated string */
 extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string(
     rocksdb_options_t* opt);
@@ -2061,13 +2072,15 @@ extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete(
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_with_ts(
     rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
     const char* ts, size_t tslen, char** errptr);
+extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_delete_range(
+    rocksdb_sstfilewriter_t* writer, const char* begin_key, size_t begin_keylen,
+    const char* end_key, size_t end_keylen, char** errptr);
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish(
     rocksdb_sstfilewriter_t* writer, char** errptr);
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_file_size(
     rocksdb_sstfilewriter_t* writer, uint64_t* file_size);
 extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy(
     rocksdb_sstfilewriter_t* writer);
-
 extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t*
 rocksdb_ingestexternalfileoptions_create(void);
 extern ROCKSDB_LIBRARY_API void

options.go

@@ -87,6 +87,17 @@ const (
 	SkipAnyCorruptedRecordsRecovery = WALRecoveryMode(3)
 )
 
+// PrepopulateBlob represents strategy for prepopulate warm/hot blobs which are already in memory into
+// blob cache at the time of flush.
+type PrepopulateBlob int
+
+const (
+	// PrepopulateBlobDisable disables prepopulate blob cache.
+	PrepopulateBlobDisable = PrepopulateBlob(0)
+	// PrepopulateBlobFlushOnly prepopulates blobs during flush only.
+	PrepopulateBlobFlushOnly = PrepopulateBlob(1)
+)
+
 // Options represent all of the available options when opening a database with Open.
 type Options struct {
 	c   *C.rocksdb_options_t
@@ -2111,6 +2122,29 @@ func (opts *Options) SetBlobCache(cache *Cache) {
 	C.rocksdb_options_set_blob_cache(opts.c, cache.c)
 }
 
+// SetPrepopulateBlobCache sets strategy for prepopulate blob caching strategy.
+//
+// If enabled, prepopulate warm/hot blobs which are already in memory into
+// blob cache at the time of flush. On a flush, the blob that is in memory (in
+// memtables) get flushed to the device. If using Direct IO, additional IO is
+// incurred to read this blob back into memory again, which is avoided by
+// enabling this option. This further helps if the workload exhibits high
+// temporal locality, where most of the reads go to recently written data.
+// This also helps in case of the remote file system since it involves network
+// traffic and higher latencies.
+//
+// Default: disabled
+//
+// Dynamically changeable through this API
+func (opts *Options) SetPrepopulateBlobCache(strategy PrepopulateBlob) {
+	C.rocksdb_options_set_prepopulate_blob_cache(opts.c, C.int(strategy))
+}
+
+// GetPrepopulateBlobCache gets prepopulate blob caching strategy
+func (opts *Options) GetPrepopulateBlobCache() PrepopulateBlob {
+	return PrepopulateBlob(C.rocksdb_options_get_prepopulate_blob_cache(opts.c))
+}
+
 // SetMaxWriteBufferNumberToMaintain sets total maximum number of write buffers
 // to maintain in memory including copies of buffers that have already been flushed.
 // Unlike max_write_buffer_number, this parameter does not affect flushing.

options_compaction_test.go

@@ -12,9 +12,9 @@ func TestOptionCompactions(t *testing.T) {
 
 	co.SetFullHistoryTsLow([]byte{1, 2, 3})
 
-	require.EqualValues(t, true, co.GetExclusiveManualCompaction())
-	co.SetExclusiveManualCompaction(false)
 	require.EqualValues(t, false, co.GetExclusiveManualCompaction())
+	co.SetExclusiveManualCompaction(true)
+	require.EqualValues(t, true, co.GetExclusiveManualCompaction())
 
 	require.EqualValues(t, KIfHaveCompactionFilter, co.BottommostLevelCompaction())
 	co.SetBottommostLevelCompaction(KForce)

options_test.go

@@ -369,6 +369,10 @@ func TestOptions(t *testing.T) {
 	opts.SetBlobFileStartingLevel(1)
 	require.Equal(t, 1, opts.GetBlobFileStartingLevel())
 
+	require.Equal(t, PrepopulateBlobDisable, opts.GetPrepopulateBlobCache())
+	opts.SetPrepopulateBlobCache(PrepopulateBlobFlushOnly)
+	require.Equal(t, PrepopulateBlobFlushOnly, opts.GetPrepopulateBlobCache())
+
 	// cloning
 	cl := opts.Clone()
 	require.EqualValues(t, 5, cl.GetTableCacheNumshardbits())

sst_file_writer.go

@@ -108,6 +108,16 @@ func (w *SSTFileWriter) DeleteWithTS(key, ts []byte) (err error) {
 	return
 }
 
+// DeleteRange deletes keys that are between [startKey, endKey)
+func (w *SSTFileWriter) DeleteRange(startKey, endKey []byte) (err error) {
+	cStartKey := byteToChar(startKey)
+	cEndKey := byteToChar(endKey)
+	var cErr *C.char
+	C.rocksdb_sstfilewriter_delete_range(w.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey)), &cErr)
+	err = fromCError(cErr)
+	return
+}
+
 // FileSize returns size of currently opened file.
 func (w *SSTFileWriter) FileSize() (size uint64) {
 	C.rocksdb_sstfilewriter_file_size(w.c, (*C.uint64_t)(&size))