replace gxhash with foldhash (#624)

Co-authored-by: zhangli20 <[email protected]>
kwai · Oct 24, 2024 · 716c876 · 716c876
1 parent 99b6c9b
commit 716c876
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 16 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/native-engine/datafusion-ext-plans/Cargo.toml b/native-engine/datafusion-ext-plans/Cargo.toml
@@ -22,8 +22,8 @@ datafusion-ext-commons = { workspace = true }
 datafusion-ext-exprs = { workspace = true }
 datafusion-ext-functions = { workspace = true }
 derivative = "2.2.0"
+foldhash = "0.1.3"
 futures = "0.3"
-gxhash = "3.4.1"
 hashbrown = "0.14.5"
 itertools = "0.13.0"
 jni = "0.20.0"

diff --git a/native-engine/datafusion-ext-plans/src/agg/acc.rs b/native-engine/datafusion-ext-plans/src/agg/acc.rs
@@ -14,6 +14,7 @@
 
 use std::{
     any::Any,
+    hash::BuildHasher,
     io::{Cursor, Read, Write},
     mem::{size_of, size_of_val},
 };
@@ -932,7 +933,9 @@ impl InternalSet {
 #[inline]
 pub fn acc_hash(value: impl AsRef<[u8]>) -> u64 {
     const ACC_HASH_SEED: u32 = 0x7BCB48DA;
-    gxhash::gxhash64(value.as_ref(), ACC_HASH_SEED as i64)
+    const HASHER: foldhash::fast::FixedState =
+        foldhash::fast::FixedState::with_seed(ACC_HASH_SEED as u64);
+    HASHER.hash_one(value.as_ref())
 }
 
 impl AggDynSet {

diff --git a/native-engine/datafusion-ext-plans/src/agg/agg_hash_map.rs b/native-engine/datafusion-ext-plans/src/agg/agg_hash_map.rs
@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::simd::{cmp::SimdPartialEq, Simd};
+use std::{
+    hash::BuildHasher,
+    simd::{cmp::SimdPartialEq, Simd},
+};
 
 use datafusion::common::Result;
 use datafusion_ext_commons::{
@@ -236,5 +239,7 @@ impl AggHashMap {
 pub fn agg_hash(value: impl AsRef<[u8]>) -> u32 {
     // 32-bits non-zero hash
     const AGG_HASH_SEED_HASHING: i64 = 0x3F6F1B93;
-    gxhash::gxhash32(value.as_ref(), AGG_HASH_SEED_HASHING) | 0x80000000
+    const HASHER: foldhash::fast::FixedState =
+        foldhash::fast::FixedState::with_seed(AGG_HASH_SEED_HASHING as u64);
+    HASHER.hash_one(value.as_ref()) as u32 | 0x80000000
 }
diff --git a/native-engine/datafusion-ext-plans/src/agg/agg_table.rs b/native-engine/datafusion-ext-plans/src/agg/agg_table.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::{
+    hash::BuildHasher,
     io::{Cursor, Read, Write},
     sync::{Arc, Weak},
 };
@@ -819,5 +820,8 @@ impl<'a> KeyForRadixTournamentTree for RecordsSpillCursor<'a> {
 #[inline]
 fn bucket_id(key: impl AsRef<[u8]>) -> u16 {
     const AGG_HASH_SEED_HASHING: i64 = 0xC732BD66;
-    (gxhash::gxhash32(key.as_ref(), AGG_HASH_SEED_HASHING) % NUM_SPILL_BUCKETS as u32) as u16
+    const HASHER: foldhash::fast::FixedState =
+        foldhash::fast::FixedState::with_seed(AGG_HASH_SEED_HASHING as u64);
+    let hash = HASHER.hash_one(key.as_ref()) as u32;
+    (hash % NUM_SPILL_BUCKETS as u32) as u16
 }
diff --git a/native-engine/datafusion-ext-plans/src/joins/join_hash_map.rs b/native-engine/datafusion-ext-plans/src/joins/join_hash_map.rs
@@ -14,6 +14,7 @@
 
 use std::{
     fmt::{Debug, Formatter},
+    hash::{BuildHasher, Hasher},
     io::Cursor,
     mem::MaybeUninit,
     simd::{cmp::SimdPartialEq, Simd},
@@ -434,8 +435,13 @@ pub fn join_hash_map_schema(data_schema: &SchemaRef) -> SchemaRef {
 #[inline]
 pub fn join_create_hashes(num_rows: usize, key_columns: &[ArrayRef]) -> Vec<u32> {
     const JOIN_HASH_RANDOM_SEED: u32 = 0x1E39FA04;
+    const HASHER: foldhash::fast::FixedState =
+        foldhash::fast::FixedState::with_seed(JOIN_HASH_RANDOM_SEED as u64);
     let mut hashes = create_hashes(num_rows, key_columns, JOIN_HASH_RANDOM_SEED, |v, h| {
-        gxhash::gxhash32(v, h as i64)
+        let mut hasher = HASHER.build_hasher();
+        hasher.write_u32(h);
+        hasher.write(v);
+        hasher.finish() as u32
     });
 
     // use 31-bit non-zero hash