Skip to content

Commit

Permalink
Use unsafe-free murmur3 in bloom filter (#2577)
Browse files Browse the repository at this point in the history 
Use misuses of unsafe-free murmur3 in bloom filter, and converge on a single library for murmur3 hashes.
  • Loading branch information
@vdarulis
vdarulis authored Sep 1, 2020
1 parent bf81bda commit edd2dc8
Show file tree
Hide file tree
Showing 10 changed files with 30 additions and 23 deletions.
9 changes: 4 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,19 @@ require (
github.com/influxdata/influxdb v1.7.7
github.com/jhump/protoreflect v1.6.1
github.com/json-iterator/go v1.1.9
github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db
github.com/leanovate/gopter v0.2.8
github.com/lib/pq v1.6.0 // indirect
github.com/lightstep/lightstep-tracer-go v0.18.1
github.com/m3db/bitset v2.0.0+incompatible
github.com/m3db/bloom v3.0.0+incompatible
github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9
github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a
github.com/m3db/m3x v0.0.0-20190408051622-ebf3c7b94afd // indirect
github.com/m3db/prometheus_client_golang v0.8.1
github.com/m3db/prometheus_client_model v0.0.0-20180517145114-8b2299a4bf7d
github.com/m3db/prometheus_common v0.0.0-20180517030744-25aaa3dff79b
github.com/m3db/prometheus_procfs v0.8.1
github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f
github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e // indirect
github.com/m3db/stackmurmur3/v2 v2.0.2
github.com/m3db/tools v0.0.0-20181008195521-c6ded3f34878
github.com/m3dbx/pilosa v0.0.0-20200602205121-7f389745e9ab
github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de
Expand Down Expand Up @@ -97,9 +97,8 @@ require (
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.6.1
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
github.com/stretchr/testify v1.4.0
github.com/stretchr/testify v1.6.1
github.com/subosito/gotenv v1.2.1-0.20190917103637-de67a6614a4d // indirect
github.com/twmb/murmur3 v1.1.3
github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553
github.com/uber-go/atomic v0.0.0-00010101000000-000000000000 // indirect
github.com/uber-go/tally v3.3.13+incompatible
Expand Down
24 changes: 16 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,8 +435,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db h1:OIgdqANJZZBFAwpmDekbBGzwhdFptLf56fuVpMwv0Qg=
github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8=
github.com/leanovate/gopter v0.2.8 h1:eFPtJ3aa5zLfbxGROSNY75T9Dume60CWBAqoWQ3h/ig=
github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8=
github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc=
github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo=
Expand All @@ -446,8 +446,10 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/m3db/bitset v2.0.0+incompatible h1:wMgri1Z2QSwJ8K/7ZuV7vE4feLOT7EofVC8RakIOybI=
github.com/m3db/bitset v2.0.0+incompatible/go.mod h1:X8CCqZmZxs2O6d4qHhiqtAKCin4G5mScPhiwX9rsc5c=
github.com/m3db/bloom v3.0.0+incompatible h1:cZzeDfFmgJ076MBv0zM8ZHS/soNoll7F+r5hudgxPCk=
github.com/m3db/bloom v3.0.0+incompatible/go.mod h1:W6XzpFw4t+CIYq+NGyp5c2394YsUc1P1+W/KAWty2lU=
github.com/m3db/bloom v3.0.1+incompatible h1:EILDlnoiPKJAGtg3RCm7Zf5VIpwh+hgcHokoZNzeJy8=
github.com/m3db/bloom v3.0.1+incompatible/go.mod h1:W6XzpFw4t+CIYq+NGyp5c2394YsUc1P1+W/KAWty2lU=
github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9 h1:H5Iznc9FI44Sekos8STE+Hj2cPohzaYfWTUHf77q6RY=
github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9/go.mod h1:JDmGHlO6ygyY1V9eOHtXiNl3+axznDTrBqwWEeWALlQ=
github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a h1:CwsSHIJLeCESKdZ844jXg/3rQD3yA5azuVlJBp5w8U8=
github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a/go.mod h1:Pk9AtZeKuCO2xcAth0gxwzRNFv4lV26GPSx4I6A7DQ8=
github.com/m3db/m3x v0.0.0-20190408051622-ebf3c7b94afd h1:wzLBtXzxZM9b6IXwLSRE5crynocLTCuRDpGDaOJzyuI=
Expand All @@ -462,8 +464,10 @@ github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK
github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo=
github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f h1:+FWCOZjB96lBc0L7lvz6/TW1gF5fEVd6JQNQG59CpZE=
github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f/go.mod h1:VJQWii2dj/AE1cRKup4MXL4TKbbs5J4wZnNe5I1CZaw=
github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e h1:B4xyv9ezWgbe30jxYzXxR45iE0Os7oZchmmyuQhGRVg=
github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e/go.mod h1:hkR/F91aDUfPxDv5Jw55ifIXV6ZW1jzNHW0d8GP02rw=
github.com/m3db/stackmurmur3 v1.0.1 h1:ASTdJ6Bd2m34dgsnOBZHrwWdKGcpaavP3MokiKL/ERs=
github.com/m3db/stackmurmur3 v1.0.1/go.mod h1:hkR/F91aDUfPxDv5Jw55ifIXV6ZW1jzNHW0d8GP02rw=
github.com/m3db/stackmurmur3/v2 v2.0.2 h1:q/kOlC12PwOib9XziHUTM/jsCQULHnRxBz/H4hT/Aro=
github.com/m3db/stackmurmur3/v2 v2.0.2/go.mod h1:HnguA9wfEbTqYdTT/5klUlhxSDPRMGaWyvLzugxW+MA=
github.com/m3db/thrift v0.0.0-20190820191926-05b5a2227fe4 h1:1x3mMuURd3wqKJ2qVjhRYOAmL9g4EA9JTagWB/y/3xo=
github.com/m3db/thrift v0.0.0-20190820191926-05b5a2227fe4/go.mod h1:xVfRinGzD3cYDRvMjy6RkIwM+iNL2KHNLZjT0VpVZT8=
github.com/m3db/tools v0.0.0-20181008195521-c6ded3f34878 h1:kww0LtVVfGrXR7Ofpbi/9bvc2EGYMQC0LCH/gQXoolE=
Expand All @@ -474,6 +478,8 @@ github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de h1:C4DpCfTNzJf5RhJqxO
github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de/go.mod h1:DOTAUfV4bzK6Nrb0dboT/oCG0DnQuX+/n0jfZPh6xxI=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.2 h1:znVR8Q4g7/WlcvsxLBRWvo+vtFJUAbDn3w+Yak2xVMI=
github.com/magiconair/properties v1.8.2/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand Down Expand Up @@ -689,6 +695,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
Expand Down Expand Up @@ -737,8 +745,8 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twmb/murmur3 v1.1.4 h1:NnlAxelwOgdQDmYuV0T/K+tpDQ/8wdsDVOGmvUqBOCw=
github.com/twmb/murmur3 v1.1.4/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 h1:DRC1ubdb3ZmyyIeCSTxjZIQAnpLPfKVgYrLETQuOPjo=
github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553/go.mod h1:Rj7Csq/tZ/egz+Ltc2IVpsA5309AmSMEswjkTZmq2Xc=
github.com/uber-go/tally v3.3.13+incompatible h1:5ic2UsDwjcWsw9jvEdWEE2XsmGCLMTt5Ukg4d74fed4=
Expand Down
2 changes: 1 addition & 1 deletion src/aggregator/hash/hash.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
// were added.
package hash

import "github.com/twmb/murmur3"
import "github.com/m3db/stackmurmur3/v2"

// Hash128 is a 128-bit hash of an ID consisting of two unsigned 64-bit ints.
type Hash128 [2]uint64
Expand Down
2 changes: 1 addition & 1 deletion src/aggregator/sharding/hash.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

"github.com/m3db/m3/src/metrics/metric/id"

"github.com/twmb/murmur3"
murmur3 "github.com/m3db/stackmurmur3/v2"
)

const (
Expand Down
4 changes: 2 additions & 2 deletions src/dbnode/client/connection_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ import (
"github.com/m3db/m3/src/dbnode/generated/thrift/rpc"
"github.com/m3db/m3/src/dbnode/topology"
xclose "github.com/m3db/m3/src/x/close"
"github.com/m3db/stackmurmur3/v2"

"github.com/twmb/murmur3"
"github.com/uber/tchannel-go/thrift"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -80,7 +80,7 @@ type healthCheckFn func(client rpc.TChanNode, opts Options) error
type sleepFn func(t time.Duration)

func newConnectionPool(host topology.Host, opts Options) connectionPool {
seed := int64(murmur3.Sum32([]byte(host.Address())))
seed := int64(murmur3.StringSum32(host.Address()))

p := &connPool{
opts: opts,
Expand Down
2 changes: 1 addition & 1 deletion src/dbnode/persist/fs/bloom_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ package fs
import (
"os"

"github.com/m3db/bloom"
"github.com/m3db/bloom/v4"
"github.com/m3db/m3/src/dbnode/digest"
"github.com/m3db/m3/src/x/mmap"
)
Expand Down
2 changes: 1 addition & 1 deletion src/dbnode/persist/fs/read_write_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"testing"
"time"

"github.com/m3db/bloom"
"github.com/m3db/bloom/v4"
"github.com/m3db/m3/src/dbnode/digest"
"github.com/m3db/m3/src/dbnode/persist"
"github.com/m3db/m3/src/x/checked"
Expand Down
2 changes: 1 addition & 1 deletion src/dbnode/persist/fs/write.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"sort"
"time"

"github.com/m3db/bloom"
"github.com/m3db/bloom/v4"
"github.com/m3db/m3/src/dbnode/digest"
"github.com/m3db/m3/src/dbnode/persist"
"github.com/m3db/m3/src/dbnode/persist/fs/msgpack"
Expand Down
4 changes: 2 additions & 2 deletions src/dbnode/sharding/shardset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (

"github.com/m3db/m3/src/cluster/shard"
"github.com/m3db/m3/src/x/ident"
twmbmurmur3 "github.com/twmb/murmur3"
"github.com/m3db/stackmurmur3/v2"
)

var (
Expand Down Expand Up @@ -160,6 +160,6 @@ func NewHashGenWithSeed(seed uint32) HashGen {
// NewHashFn generates a HashFN based on murmur32 with a given seed
func NewHashFn(length int, seed uint32) HashFn {
return func(id ident.ID) uint32 {
return twmbmurmur3.SeedSum32(seed, id.Bytes()) % uint32(length)
return murmur3.SeedSum32(seed, id.Bytes()) % uint32(length)
}
}
2 changes: 1 addition & 1 deletion src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ import (
"github.com/leanovate/gopter"
"github.com/leanovate/gopter/gen"
"github.com/leanovate/gopter/prop"
"github.com/twmb/murmur3"
murmur3 "github.com/m3db/stackmurmur3/v2"
"github.com/stretchr/testify/require"
)

Expand Down

0 comments on commit edd2dc8

Please sign in to comment.