diff --git a/cuckoofilter.go b/cuckoofilter.go index 983a048..3f47a96 100644 --- a/cuckoofilter.go +++ b/cuckoofilter.go @@ -44,14 +44,14 @@ func getTable(tableType uint) interface{} { } } -type VictimCache struct { +type victimCache struct { index uint tag uint32 used bool } type Filter struct { - victim VictimCache + victim victimCache numItems uint table Table bitsPerItem uint @@ -100,7 +100,11 @@ func (f *Filter) altIndex(index uint, tag uint32) uint { } func (f *Filter) Size() uint { - return f.numItems + var c uint + if f.victim.used { + c = 1 + } + return f.numItems + c } func (f *Filter) LoadFactor() float64 { return 1.0 * float64(f.Size()) / float64(f.table.SizeInTags()) @@ -164,9 +168,6 @@ func (f *Filter) Contain(key []byte) bool { f.generateIndexTagHash(key, &i1, &tag) i2 = f.altIndex(i1, tag) - if i1 != f.altIndex(i2, tag) { - panic("hash err") - } found = f.victim.used && tag == f.victim.tag && (i1 == f.victim.index || i2 == f.victim.index) if found || f.table.FindTagInBuckets(i1, i2, tag) { @@ -282,7 +283,7 @@ func Decode(bytes []byte) (*Filter, error) { table: table, numItems: numItems, bitsPerItem: table.BitsPerItem(), - victim: VictimCache{ + victim: victimCache{ index: curIndex, tag: curTag, used: used, diff --git a/cuckoofilter_test.go b/cuckoofilter_test.go index 5ec3504..41fdaf7 100644 --- a/cuckoofilter_test.go +++ b/cuckoofilter_test.go @@ -7,6 +7,7 @@ package cuckoo import ( "crypto/rand" + "fmt" "io" "reflect" "testing" @@ -14,94 +15,73 @@ import ( const size = 100000 -func TestInsertion_Single(t *testing.T) { - var insertNum uint = 5000 - var hash [32]byte - cf := NewFilter(4, 8, size, TableTypeSingle) - a := make([][]byte, insertNum) - for i := uint(0); i < insertNum; i++ { - _, _ = io.ReadFull(rand.Reader, hash[:]) - tmp := make([]byte, 32) - copy(tmp, hash[:]) - a = append(a, tmp) - cf.Add(hash[:]) - } - - count := cf.numItems - if count != insertNum { - t.Errorf("Expected count = %d, instead count = %d", insertNum, count) - } - - for _, v := range a { - cf.Delete(v) - } +var testBucketSize = []uint{2, 4, 8} +var testFingerprintSize = []uint{2, 4, 5, 6, 7, 8, 9, 10, 12, 13, 16, 17, 32} +var testTableType = []uint{TableTypeSingle, TableTypePacked} - count = cf.numItems - if count != 0 { - t.Errorf("Expected count = 0, instead count == %d", count) - } -} - -func TestInsertion_Packed(t *testing.T) { - var insertNum uint = 5000 +func TestFilter(t *testing.T) { + var insertNum uint = 50000 var hash [32]byte - cf := NewFilter(4, 9, size, TableTypePacked) - a := make([][]byte, insertNum) - for i := uint(0); i < insertNum; i++ { - _, _ = io.ReadFull(rand.Reader, hash[:]) - tmp := make([]byte, 32) - copy(tmp, hash[:]) - a = append(a, tmp) - cf.Add(hash[:]) - } - count := cf.numItems - if count != insertNum { - t.Errorf("Expected count = %d, instead count = %d", insertNum, count) + for _, b := range testBucketSize { + for _, f := range testFingerprintSize { + for _, table := range testTableType { + if f == 2 && table == TableTypePacked { + continue + } + if table == TableTypePacked { + b = 4 + } + cf := NewFilter(b, f, 8190, table) + //fmt.Println(cf.Info()) + a := make([][]byte, 0) + for i := uint(0); i < insertNum; i++ { + _, _ = io.ReadFull(rand.Reader, hash[:]) + if cf.AddUnique(hash[:]) { + tmp := make([]byte, 32) + copy(tmp, hash[:]) + a = append(a, tmp) + } + } + + count := cf.Size() + if count != uint(len(a)) { + t.Errorf("Expected count = %d, instead count = %d, b %v f %v", uint(len(a)), count, b, f) + } + + for _, v := range a { + if !cf.Contain(v) { + t.Errorf("Expected contain, instead not contain") + } + } + + for _, v := range a { + cf.Delete(v) + } + + count = cf.Size() + if count != 0 { + t.Errorf("Expected count = 0, instead count == %d", count) + } + + bytes := cf.Encode() + ncf, err := Decode(bytes) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if !reflect.DeepEqual(cf, ncf) { + t.Errorf("Expected %v, got %v", cf, ncf) + } + + cf.Info() + cf.BitsPerItem() + cf.SizeInBytes() + cf.LoadFactor() + fmt.Printf("Filter bucketSize %v fingerprintSize %v tableType %v falsePositive Rate %v \n", b, f, table, cf.FalsePositiveRate()) + } + } } - for _, v := range a { - cf.Delete(v) - } - - count = cf.numItems - if count != 0 { - t.Errorf("Expected count = 0, instead count == %d", count) - } -} - -func TestEncodeDecode_Single(t *testing.T) { - cf := NewFilter(4, 8, size, TableTypeSingle) - var hash [32]byte - for i := 0; i < 5; i++ { - _, _ = io.ReadFull(rand.Reader, hash[:]) - cf.Add(hash[:]) - } - bytes := cf.Encode() - ncf, err := Decode(bytes) - if err != nil { - t.Errorf("Expected no error, got %v", err) - } - if !reflect.DeepEqual(cf, ncf) { - t.Errorf("Expected %v, got %v", cf, ncf) - } -} - -func TestEncodeDecode_Packed(t *testing.T) { - cf := NewFilter(4, 9, size, TableTypePacked) - var hash [32]byte - for i := 0; i < 5; i++ { - _, _ = io.ReadFull(rand.Reader, hash[:]) - cf.Add(hash[:]) - } - bytes := cf.Encode() - ncf, err := Decode(bytes) - if err != nil { - t.Errorf("Expected no error, got %v", err) - } - if !reflect.DeepEqual(cf, ncf) { - t.Errorf("Expected %v, got %v", cf, ncf) - } } func BenchmarkFilterSingle_Reset(b *testing.B) { diff --git a/packedtable.go b/packedtable.go index 65a675f..96e25fd 100644 --- a/packedtable.go +++ b/packedtable.go @@ -439,12 +439,6 @@ func (p *PackedTable) FindTagInBuckets(i1, i2 uint, tag uint32) bool { (tags1[3] == tag) || (tags2[0] == tag) || (tags2[1] == tag) || (tags2[2] == tag) || (tags2[3] == tag) } -func (p *PackedTable) FindTagInBucket(i uint, tag uint32) bool { - var tags [tagsPerPTable]uint32 - p.ReadBucket(i, &tags) - - return (tags[0] == tag) || (tags[1] == tag) || (tags[2] == tag) || (tags[3] == tag) -} func (p *PackedTable) DeleteTagFromBucket(i uint, tag uint32) bool { var tags [tagsPerPTable]uint32 diff --git a/permencoding.go b/permencoding.go index 06b9be7..e85f1db 100644 --- a/permencoding.go +++ b/permencoding.go @@ -7,11 +7,8 @@ package cuckoo import ( "encoding/binary" - "fmt" ) -const DEBUG = false - type PermEncoding struct { nEnts uint DecTable []uint16 @@ -49,14 +46,12 @@ func (p *PermEncoding) Decode(codeword uint16, lowBits *[tagsPerPTable]uint8) { p.unpack(p.DecTable[codeword], lowBits) } func (p *PermEncoding) Encode(lowBits [tagsPerPTable]uint8) uint16 { - if DEBUG { - fmt.Printf("Perm.encode\n") - for i := 0; i < tagsPerPTable; i++ { - fmt.Printf("encode lowBits[%d]=%x\n", i, lowBits[i]) - } - fmt.Printf("pack(lowBits) = %x\n", p.pack(lowBits)) - fmt.Printf("enc_table[%x]=%x\n", p.pack(lowBits), p.EncTable[p.pack(lowBits)]) - } + //fmt.Printf("Perm.encode\n") + //for i := 0; i < tagsPerPTable; i++ { + // fmt.Printf("encode lowBits[%d]=%x\n", i, lowBits[i]) + //} + //fmt.Printf("pack(lowBits) = %x\n", p.pack(lowBits)) + //fmt.Printf("enc_table[%x]=%x\n", p.pack(lowBits), p.EncTable[p.pack(lowBits)]) return p.EncTable[p.pack(lowBits)] } @@ -69,10 +64,8 @@ func (p *PermEncoding) genTables(base, k int, dst [tagsPerPTable]uint8, idx *uin } else { p.DecTable[*idx] = p.pack(dst) p.EncTable[p.pack(dst)] = *idx - if DEBUG { - fmt.Printf("enc_table[%04x]=%04x\t%x %x %x %x\n", p.pack(dst), *idx, dst[0], - dst[1], dst[2], dst[3]) - } + //fmt.Printf("enc_table[%04x]=%04x\t%x %x %x %x\n", p.pack(dst), *idx, dst[0], + // dst[1], dst[2], dst[3]) *idx++ } } diff --git a/singletable.go b/singletable.go index f4ec09d..e58c7d0 100644 --- a/singletable.go +++ b/singletable.go @@ -189,23 +189,10 @@ func (t *SingleTable) FindTagInBuckets(i1, i2 uint, tag uint32) bool { return false } -func (t *SingleTable) FindTagInBucket(i uint, tag uint32) bool { - var j uint - for j = 0; j < t.kTagsPerBucket; j++ { - if t.ReadTag(i, j) == tag { - return true - } - } - return false -} - func (t *SingleTable) DeleteTagFromBucket(i uint, tag uint32) bool { var j uint for j = 0; j < t.kTagsPerBucket; j++ { if t.ReadTag(i, j) == tag { - if t.FindTagInBucket(i, tag) != true { - panic("not exist") - } t.WriteTag(i, j, 0) return true } @@ -229,16 +216,6 @@ func (t *SingleTable) InsertTagToBucket(i uint, tag uint32, kickOut bool, oldTag return false } -func (t *SingleTable) NumTagsInBucket(i uint) uint { - var j, num uint - for j = 0; j < t.kTagsPerBucket; j++ { - if t.ReadTag(i, j) != 0 { - num++ - } - } - return num -} - func (t *SingleTable) Reset() { for i := range t.bucket { t.bucket[i] = 0