Adapt RocksDB 7.10.2

build.sh

@@ -3,30 +3,31 @@ DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 INSTALL_PREFIX=$1
 
-export CFLAGS='-fPIC -O3 -pipe' 
-export CXXFLAGS='-fPIC -O3 -pipe -Wno-maybe-uninitialized'
-
 BUILD_PATH=/tmp/build
 mkdir -p $BUILD_PATH
 
 CMAKE_REQUIRED_PARAMS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}"
 
+snappy_version="1.1.10"
+cd $BUILD_PATH && wget https://github.com/google/snappy/archive/${snappy_version}.tar.gz && tar xzf ${snappy_version}.tar.gz && cd snappy-${snappy_version} && \
+    mkdir -p build_place && cd build_place && \
+    CXXFLAGS='-fPIC -O3 -pipe -Wno-uninitialized -Werror,-Wno-sign-compare' cmake $CMAKE_REQUIRED_PARAMS -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF .. && make install/strip -j16 && \
+    cd $BUILD_PATH && rm -rf *
+
+export CFLAGS='-fPIC -O3 -pipe' 
+export CXXFLAGS='-fPIC -O3 -pipe -Wno-uninitialized'
+
 zlib_version="1.2.11"
 cd $BUILD_PATH && wget https://github.com/madler/zlib/archive/v${zlib_version}.tar.gz && tar xzf v${zlib_version}.tar.gz && cd zlib-${zlib_version} && \
     ./configure --prefix=$INSTALL_PREFIX --static && make -j16 install && \
     cd $BUILD_PATH && rm -rf *
 
-snappy_version="1.1.8"
-cd $BUILD_PATH && wget https://github.com/google/snappy/archive/${snappy_version}.tar.gz && tar xzf ${snappy_version}.tar.gz && cd snappy-${snappy_version} && \
-    mkdir -p build_place && cd build_place && cmake $CMAKE_REQUIRED_PARAMS -DSNAPPY_BUILD_TESTS=OFF .. && make install/strip -j16 && \
-    cd $BUILD_PATH && rm -rf *
-
-lz4_version="1.9.3"
+lz4_version="1.9.4"
 cd $BUILD_PATH && wget https://github.com/lz4/lz4/archive/v${lz4_version}.tar.gz && tar xzf v${lz4_version}.tar.gz && cd lz4-${lz4_version}/build/cmake && \
     cmake $CMAKE_REQUIRED_PARAMS -DLZ4_BUILD_LEGACY_LZ4C=OFF -DBUILD_SHARED_LIBS=OFF -DLZ4_POSITION_INDEPENDENT_LIB=ON && make -j16 install && \
     cd $BUILD_PATH && rm -rf *
 
-zstd_version="1.5.0"
+zstd_version="1.5.4"
 cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz && tar xzf v${zstd_version}.tar.gz && \
     cd zstd-${zstd_version}/build/cmake && mkdir -p build_place && cd build_place && \
     cmake $CMAKE_REQUIRED_PARAMS -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_CONTRIB=OFF -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_TESTS=OFF \
@@ -36,7 +37,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}
 # Note: if you don't have a good reason, please do not set -DPORTABLE=ON
 #
 # This one is set here on purpose of compatibility with github action runtime processor
-rocksdb_version="7.9.2"
+rocksdb_version="7.10.2"
 cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \
     mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
     -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \

c.h

@@ -986,6 +986,10 @@ rocksdb_block_based_options_set_partition_filters(
     rocksdb_block_based_table_options_t* options,
     unsigned char partition_filters);
 extern ROCKSDB_LIBRARY_API void
+rocksdb_block_based_options_set_optimize_filters_for_memory(
+    rocksdb_block_based_table_options_t* options,
+    unsigned char optimize_filters_for_memory);
+extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_use_delta_encoding(
     rocksdb_block_based_table_options_t* options,
     unsigned char use_delta_encoding);
@@ -1890,6 +1894,10 @@ extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout(
     rocksdb_readoptions_t*, uint64_t microseconds);
 extern ROCKSDB_LIBRARY_API uint64_t
 rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*);
+extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io(
+    rocksdb_readoptions_t*, unsigned char);
+extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io(
+    rocksdb_readoptions_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp(
     rocksdb_readoptions_t*, const char* ts, size_t tslen);
 extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts(

options_block_based_table.go

@@ -238,6 +238,43 @@ func (opts *BlockBasedTableOptions) SetPartitionFilters(value bool) {
 	C.rocksdb_block_based_options_set_partition_filters(opts.c, boolToChar(value))
 }
 
+// SetOptimizeFiltersForMemory to generate Bloom/Ribbon filters that minimize memory
+// internal fragmentation.
+//
+// When false, malloc_usable_size is not available, or format_version < 5,
+// filters are generated without regard to internal fragmentation when
+// loaded into memory (historical behavior). When true (and
+// malloc_usable_size is available and format_version >= 5), then
+// filters are generated to "round up" and "round down" their sizes to
+// minimize internal fragmentation when loaded into memory, assuming the
+// reading DB has the same memory allocation characteristics as the
+// generating DB. This option does not break forward or backward
+// compatibility.
+//
+// While individual filters will vary in bits/key and false positive rate
+// when setting is true, the implementation attempts to maintain a weighted
+// average FP rate for filters consistent with this option set to false.
+//
+// With Jemalloc for example, this setting is expected to save about 10% of
+// the memory footprint and block cache charge of filters, while increasing
+// disk usage of filters by about 1-2% due to encoding efficiency losses
+// with variance in bits/key.
+//
+// NOTE: Because some memory counted by block cache might be unmapped pages
+// within internal fragmentation, this option can increase observed RSS
+// memory usage. With cache_index_and_filter_blocks=true, this option makes
+// the block cache better at using space it is allowed. (These issues
+// should not arise with partitioned filters.)
+//
+// NOTE: Do not set to true if you do not trust malloc_usable_size. With
+// this option, RocksDB might access an allocated memory object beyond its
+// original size if malloc_usable_size says it is safe to do so. While this
+// can be considered bad practice, it should not produce undefined behavior
+// unless malloc_usable_size is buggy or broken.
+func (opts *BlockBasedTableOptions) SetOptimizeFiltersForMemory(value bool) {
+	C.rocksdb_block_based_options_set_optimize_filters_for_memory(opts.c, boolToChar(value))
+}
+
 // SetUseDeltaEncoding uses delta encoding to compress keys in blocks.
 // ReadOptions::pin_data requires this option to be disabled.
 //

options_block_based_table_test.go

@@ -0,0 +1,13 @@
+package grocksdb
+
+import (
+	"testing"
+)
+
+func TestBBT(t *testing.T) {
+	b := NewDefaultBlockBasedTableOptions()
+	defer b.Destroy()
+
+	b.SetBlockSize(123)
+	b.SetOptimizeFiltersForMemory(true)
+}

options_read.go

@@ -286,6 +286,24 @@ func (opts *ReadOptions) SetIOTimeout(microseconds uint64) {
 	C.rocksdb_readoptions_set_io_timeout(opts.c, C.uint64_t(microseconds))
 }
 
+// SetAsyncIO toggles async_io flag.
+//
+// If async_io is enabled, RocksDB will prefetch some of data asynchronously.
+// RocksDB apply it if reads are sequential and its internal automatic
+// prefetching.
+//
+// Default: false
+//
+// Note: Experimental
+func (opts *ReadOptions) SetAsyncIO(value bool) {
+	C.rocksdb_readoptions_set_async_io(opts.c, boolToChar(value))
+}
+
+// IsAsyncIO checks if async_io flag is on.
+func (opts *ReadOptions) IsAsyncIO() bool {
+	return charToBool(C.rocksdb_readoptions_get_async_io(opts.c))
+}
+
 // GetIOTimeout gets timeout in microseconds to be passed to the underlying FileSystem for
 // reads. As opposed to deadline, this determines the timeout for each
 // individual file read request. If a MultiGet/Get/Seek/Next etc call

options_read_test.go

@@ -68,4 +68,8 @@ func TestReadOptions(t *testing.T) {
 	require.EqualValues(t, 0, ro.GetIOTimeout())
 	ro.SetIOTimeout(1212)
 	require.EqualValues(t, 1212, ro.GetIOTimeout())
+
+	require.False(t, ro.IsAsyncIO())
+	ro.SetAsyncIO(true)
+	require.True(t, ro.IsAsyncIO())
 }

testing_darwin_arm64.go

@@ -0,0 +1,8 @@
+//go:build testing
+
+package grocksdb
+
+// #cgo CFLAGS: -I${SRCDIR}/dist/darwin_arm64/include
+// #cgo CXXFLAGS: -I${SRCDIR}/dist/darwin_arm64/include
+// #cgo LDFLAGS: -L${SRCDIR}/dist/darwin_arm64/lib -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy
+import "C"