diff --git a/build.sh b/build.sh index 2018225..2116426 100644 --- a/build.sh +++ b/build.sh @@ -3,30 +3,31 @@ DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" INSTALL_PREFIX=$1 -export CFLAGS='-fPIC -O3 -pipe' -export CXXFLAGS='-fPIC -O3 -pipe -Wno-maybe-uninitialized' - BUILD_PATH=/tmp/build mkdir -p $BUILD_PATH CMAKE_REQUIRED_PARAMS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}" +snappy_version="1.1.10" +cd $BUILD_PATH && wget https://github.com/google/snappy/archive/${snappy_version}.tar.gz && tar xzf ${snappy_version}.tar.gz && cd snappy-${snappy_version} && \ + mkdir -p build_place && cd build_place && \ + CXXFLAGS='-fPIC -O3 -pipe -Wno-uninitialized -Werror,-Wno-sign-compare' cmake $CMAKE_REQUIRED_PARAMS -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF .. && make install/strip -j16 && \ + cd $BUILD_PATH && rm -rf * + +export CFLAGS='-fPIC -O3 -pipe' +export CXXFLAGS='-fPIC -O3 -pipe -Wno-uninitialized' + zlib_version="1.2.11" cd $BUILD_PATH && wget https://github.com/madler/zlib/archive/v${zlib_version}.tar.gz && tar xzf v${zlib_version}.tar.gz && cd zlib-${zlib_version} && \ ./configure --prefix=$INSTALL_PREFIX --static && make -j16 install && \ cd $BUILD_PATH && rm -rf * -snappy_version="1.1.8" -cd $BUILD_PATH && wget https://github.com/google/snappy/archive/${snappy_version}.tar.gz && tar xzf ${snappy_version}.tar.gz && cd snappy-${snappy_version} && \ - mkdir -p build_place && cd build_place && cmake $CMAKE_REQUIRED_PARAMS -DSNAPPY_BUILD_TESTS=OFF .. && make install/strip -j16 && \ - cd $BUILD_PATH && rm -rf * - -lz4_version="1.9.3" +lz4_version="1.9.4" cd $BUILD_PATH && wget https://github.com/lz4/lz4/archive/v${lz4_version}.tar.gz && tar xzf v${lz4_version}.tar.gz && cd lz4-${lz4_version}/build/cmake && \ cmake $CMAKE_REQUIRED_PARAMS -DLZ4_BUILD_LEGACY_LZ4C=OFF -DBUILD_SHARED_LIBS=OFF -DLZ4_POSITION_INDEPENDENT_LIB=ON && make -j16 install && \ cd $BUILD_PATH && rm -rf * -zstd_version="1.5.0" +zstd_version="1.5.4" cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz && tar xzf v${zstd_version}.tar.gz && \ cd zstd-${zstd_version}/build/cmake && mkdir -p build_place && cd build_place && \ cmake $CMAKE_REQUIRED_PARAMS -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_CONTRIB=OFF -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_TESTS=OFF \ @@ -36,7 +37,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version} # Note: if you don't have a good reason, please do not set -DPORTABLE=ON # # This one is set here on purpose of compatibility with github action runtime processor -rocksdb_version="7.9.2" +rocksdb_version="7.10.2" cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \ mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \ -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \ diff --git a/c.h b/c.h index 1639f3c..11e4d26 100644 --- a/c.h +++ b/c.h @@ -986,6 +986,10 @@ rocksdb_block_based_options_set_partition_filters( rocksdb_block_based_table_options_t* options, unsigned char partition_filters); extern ROCKSDB_LIBRARY_API void +rocksdb_block_based_options_set_optimize_filters_for_memory( + rocksdb_block_based_table_options_t* options, + unsigned char optimize_filters_for_memory); +extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_use_delta_encoding( rocksdb_block_based_table_options_t* options, unsigned char use_delta_encoding); @@ -1890,6 +1894,10 @@ extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_io_timeout( rocksdb_readoptions_t*, uint64_t microseconds); extern ROCKSDB_LIBRARY_API uint64_t rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t*); +extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_async_io( + rocksdb_readoptions_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char rocksdb_readoptions_get_async_io( + rocksdb_readoptions_t*); extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_timestamp( rocksdb_readoptions_t*, const char* ts, size_t tslen); extern ROCKSDB_LIBRARY_API void rocksdb_readoptions_set_iter_start_ts( diff --git a/options_block_based_table.go b/options_block_based_table.go index ecf1961..35e79ec 100644 --- a/options_block_based_table.go +++ b/options_block_based_table.go @@ -238,6 +238,43 @@ func (opts *BlockBasedTableOptions) SetPartitionFilters(value bool) { C.rocksdb_block_based_options_set_partition_filters(opts.c, boolToChar(value)) } +// SetOptimizeFiltersForMemory to generate Bloom/Ribbon filters that minimize memory +// internal fragmentation. +// +// When false, malloc_usable_size is not available, or format_version < 5, +// filters are generated without regard to internal fragmentation when +// loaded into memory (historical behavior). When true (and +// malloc_usable_size is available and format_version >= 5), then +// filters are generated to "round up" and "round down" their sizes to +// minimize internal fragmentation when loaded into memory, assuming the +// reading DB has the same memory allocation characteristics as the +// generating DB. This option does not break forward or backward +// compatibility. +// +// While individual filters will vary in bits/key and false positive rate +// when setting is true, the implementation attempts to maintain a weighted +// average FP rate for filters consistent with this option set to false. +// +// With Jemalloc for example, this setting is expected to save about 10% of +// the memory footprint and block cache charge of filters, while increasing +// disk usage of filters by about 1-2% due to encoding efficiency losses +// with variance in bits/key. +// +// NOTE: Because some memory counted by block cache might be unmapped pages +// within internal fragmentation, this option can increase observed RSS +// memory usage. With cache_index_and_filter_blocks=true, this option makes +// the block cache better at using space it is allowed. (These issues +// should not arise with partitioned filters.) +// +// NOTE: Do not set to true if you do not trust malloc_usable_size. With +// this option, RocksDB might access an allocated memory object beyond its +// original size if malloc_usable_size says it is safe to do so. While this +// can be considered bad practice, it should not produce undefined behavior +// unless malloc_usable_size is buggy or broken. +func (opts *BlockBasedTableOptions) SetOptimizeFiltersForMemory(value bool) { + C.rocksdb_block_based_options_set_optimize_filters_for_memory(opts.c, boolToChar(value)) +} + // SetUseDeltaEncoding uses delta encoding to compress keys in blocks. // ReadOptions::pin_data requires this option to be disabled. // diff --git a/options_block_based_table_test.go b/options_block_based_table_test.go new file mode 100644 index 0000000..a1b54ba --- /dev/null +++ b/options_block_based_table_test.go @@ -0,0 +1,13 @@ +package grocksdb + +import ( + "testing" +) + +func TestBBT(t *testing.T) { + b := NewDefaultBlockBasedTableOptions() + defer b.Destroy() + + b.SetBlockSize(123) + b.SetOptimizeFiltersForMemory(true) +} diff --git a/options_read.go b/options_read.go index 0322535..e81a23b 100644 --- a/options_read.go +++ b/options_read.go @@ -286,6 +286,24 @@ func (opts *ReadOptions) SetIOTimeout(microseconds uint64) { C.rocksdb_readoptions_set_io_timeout(opts.c, C.uint64_t(microseconds)) } +// SetAsyncIO toggles async_io flag. +// +// If async_io is enabled, RocksDB will prefetch some of data asynchronously. +// RocksDB apply it if reads are sequential and its internal automatic +// prefetching. +// +// Default: false +// +// Note: Experimental +func (opts *ReadOptions) SetAsyncIO(value bool) { + C.rocksdb_readoptions_set_async_io(opts.c, boolToChar(value)) +} + +// IsAsyncIO checks if async_io flag is on. +func (opts *ReadOptions) IsAsyncIO() bool { + return charToBool(C.rocksdb_readoptions_get_async_io(opts.c)) +} + // GetIOTimeout gets timeout in microseconds to be passed to the underlying FileSystem for // reads. As opposed to deadline, this determines the timeout for each // individual file read request. If a MultiGet/Get/Seek/Next etc call diff --git a/options_read_test.go b/options_read_test.go index 00adc6e..1cb86ad 100644 --- a/options_read_test.go +++ b/options_read_test.go @@ -68,4 +68,8 @@ func TestReadOptions(t *testing.T) { require.EqualValues(t, 0, ro.GetIOTimeout()) ro.SetIOTimeout(1212) require.EqualValues(t, 1212, ro.GetIOTimeout()) + + require.False(t, ro.IsAsyncIO()) + ro.SetAsyncIO(true) + require.True(t, ro.IsAsyncIO()) } diff --git a/testing_darwin_arm64.go b/testing_darwin_arm64.go new file mode 100644 index 0000000..e7738e5 --- /dev/null +++ b/testing_darwin_arm64.go @@ -0,0 +1,8 @@ +//go:build testing + +package grocksdb + +// #cgo CFLAGS: -I${SRCDIR}/dist/darwin_arm64/include +// #cgo CXXFLAGS: -I${SRCDIR}/dist/darwin_arm64/include +// #cgo LDFLAGS: -L${SRCDIR}/dist/darwin_arm64/lib -lrocksdb -pthread -lstdc++ -ldl -lm -lzstd -llz4 -lz -lsnappy +import "C"