filter

Author: alex cai

README.md

@@ -13,6 +13,16 @@
 ## Example
 
 ```go
-
 ```
+## 冲突的数量
+
+### 只使用一个hash函数（fnv.New32a）
+
+使用2MB的空间，即1.67kw左右的表示空间: 
+
+- 元素个数1000w，其hash冲突率约为6.7%
+- 元素个数700w，其hash冲突率约为1.3%
+- 元素个数600w或者低于600w时，其hash冲突率约为0%
+
+每次用完得alg.Reset()
 

filter.go

@@ -2,51 +2,157 @@ package bitFilter
 
 import ()
 
+const (
+	// 2MB的存储空间. 1MB=2^20, 一个uint64长度为8个字节，就是2^3
+	// 总共的位数：8*1024*1024*2 = 16 777 216
+	Size2MB uint64 = 1 << (21 - 3)
+
+	// 更大的空间
+	Size8MB   uint64 = 1 << (23 - 3)
+	Size32MB  uint64 = 1 << (25 - 3)
+	Size128MB uint64 = 1 << (27 - 3)
+	Size512MB uint64 = 1 << (29 - 3)
+	Size2GB   uint64 = 1 << (31 - 3)
+
+	// 8*1024*2 = 16 384
+	Size2KB uint64 = 1 << (11 - 3)
+)
+
 type TBits struct {
-	bytes     []byte          // 字符数组
-	bits      int             // 位空间，1个字节8个位
-	collision map[uint32]bool // 冲突的key值（理论上冲突的key值应该很小才有效）
+	buckets []uint64 // 数组
+
+	// 冲突的key值（理论上冲突的key值应该很小才有效）
+	// 下标是key，值是hashKey
+	collision map[uint64]uint32
 }
 
 var (
 	// 位的掩码
-	bitMask = [8]uint8{1, 2, 4, 8, 16, 32, 64, 128}
+	bitMask [64]uint64
 )
 
-func New(spaceBytes int) *TBits {
+func init() {
+	var i uint
+	for i = 0; i < 64; i++ {
+		bitMask[i] = 1 << i
+	}
+}
+
+func New(len uint64) *TBits {
 	return &TBits{
-		bytes:     make([]byte, spaceBytes),
-		bits:      spaceBytes << 3,
-		collision: make(map[uint32]bool),
+		buckets:   make([]uint64, len),
+		collision: make(map[uint64]uint32),
 	}
 }
 
-func (b *TBits) AddAllHashKeys(hashKeys []uint32) {
+// 复制一个结构
+func (b *TBits) Copy() *TBits {
+	new := &TBits{
+		buckets:   b.buckets[:],
+		collision: make(map[uint64]uint32),
+	}
+	for key, hashKey := range b.collision {
+		new.collision[key] = hashKey
+	}
+
+	return new
+}
+
+// 初始化所有hash key值
+func (b *TBits) InitAllHashKeys(hashKeys []uint32) {
 	var bucket uint32
 	for _, key := range hashKeys {
-		bucket = key >> 3 // 确定在那个位上
+		bucket = key >> 6 // 确定在那个位上
 		//println(bucket, key&7)
-		b.bytes[bucket] |= bitMask[key&7]
+		b.buckets[bucket] |= bitMask[key&63] // 63 = 1<<6 - 1
 	}
 }
 
-func (b *TBits) AddCollisionKeys(keys []uint32) {
-	for _, key := range keys {
-		b.collision[key] = true
+// 初始化冲突的key值
+func (b *TBits) InitCollisionKeys(keys []uint64, hashKeys []uint32) {
+	for index, key := range keys {
+		b.collision[key] = hashKeys[index]
 	}
 }
 
 // Filter 过滤器，如果元素存在，则返回true，否则返回
-func (b *TBits) Filter(key, hashKey uint32) (isExist bool) {
+func (b *TBits) Filter(key uint64, hashKey uint32) (isExist bool) {
 	if _, isExist = b.collision[key]; isExist {
 		return isExist
 	}
 
-	bucket := hashKey >> 3
-	if b.bytes[bucket]&bitMask[hashKey&7] == 0 {
-		//println("==")
+	bucket := hashKey >> 6
+	if b.buckets[bucket]&bitMask[hashKey&63] == 0 {
 		return false
 	}
 
 	return true
 }
+
+// Add 新增key
+// 如果如果hashKey已经存在，但是又没有冲突的话，则返回false，这时需要先添加原来的冲突
+func (b *TBits) Add(key uint64, hashKey uint32) (ok bool) {
+	var (
+		bucket  uint32 = hashKey >> 6        // 确定在那个位上
+		keyMask        = bitMask[hashKey&63] // 63 = 1<<6 - 1
+	)
+	if b.buckets[bucket]&keyMask == keyMask {
+		// 判断该haskkey是否已经冲突
+		if isExist := b.hasHashKeyInCollision(hashKey); !isExist {
+			// 需要添加原来的key到冲突key map才可以继续
+			return false
+		}
+
+		// 原来已经冲突了，这是直接增加一个冲突即可
+		b.collision[key] = hashKey
+	} else {
+		// 该key完全是新的
+		b.buckets[bucket] |= keyMask
+	}
+
+	return true
+}
+
+// Remove 减少key
+func (b *TBits) Remove(key uint64, hashKey uint32) {
+	var oldKeys []uint64
+	if _, isExist := b.collision[key]; isExist {
+		for k, hk := range b.collision {
+			if hk == hashKey {
+				oldKeys = append(oldKeys, k)
+			}
+		}
+
+		if len(oldKeys) > 2 {
+			// 有三个或者三个以上冲突
+			delete(b.collision, key)
+		} else {
+			// 只有两个冲突，则全部删除
+			// 冲突的key，至少也会有两个
+			for _, k := range oldKeys {
+				delete(b.collision, k)
+			}
+		}
+	} else {
+		var (
+			bucket  uint32 = hashKey >> 6        // 确定在那个位上
+			keyMask        = bitMask[hashKey&63] // 63 = 1<<6 - 1
+		)
+		b.buckets[bucket] &= ^keyMask
+	}
+}
+
+// AddCollisionKey 增加冲突key
+func (b *TBits) AddCollisionKey(key uint64, hashKey uint32) {
+	b.collision[key] = hashKey
+}
+
+func (b *TBits) hasHashKeyInCollision(hashKey uint32) (isExist bool) {
+	for _, hk := range b.collision {
+		if hk == hashKey {
+			return true
+		}
+	}
+
+	return isExist
+}

filter_test.go

@@ -1,41 +1,49 @@
 package bitFilter
 
 import (
+	"fmt"
+	//"hash"
+	"encoding/binary"
+	"hash/fnv"
 	"testing"
 )
 
 var bits *TBits
 var maxMask uint32
 
-func init() {
-	bits = New(1024)
-	maxMask = 1024<<3 - 1
-}
-
 func TestFilter(t *testing.T) {
-	if len(bits.bytes) != 1024 {
-		t.Fatalf("len bytes: %d != 2014", len(bits.bytes))
+	bits = New(128)
+	maxMask = 128<<6 - 1
+
+	if len(bits.buckets) != 128 {
+		t.Fatalf("len buckets: %d != 128", len(bits.buckets))
 	}
 
-	bits.AddCollisionKeys([]uint32{0, 1, 2, 3, 4, 8192, 8193, 8194, 8195, 8196})
-	bits.AddAllHashKeys([]uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
+	keys := []uint64{0, 1, 2, 3, 4, 8192, 8193, 8194, 8195, 8196, 16385}
+	collHashKeys := []uint32{0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 1}
+	hashKeys := []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+	bits.InitCollisionKeys(keys, collHashKeys)
+	bits.InitAllHashKeys(hashKeys)
 
-	if len(bits.collision) != 10 {
-		t.Fatalf("collision len != 10")
+	if len(bits.collision) != len(collHashKeys) {
+		t.Fatalf("collision len != %d", len(collHashKeys))
 	}
 
-	if bits.bytes[0] != 255 {
-		t.Fatalf("bytes[0]:%d != 255", bits.bytes[0])
+	if bits.buckets[0] != 1023 {
+		t.Fatalf("buckets[0]:%d != 1023", bits.buckets[0])
 	}
 
-	if bits.bytes[1] != 3 {
-		t.Fatalf("bytes[1]:%d != 3", bits.bytes[1])
+	if bits.buckets[1] != 0 {
+		t.Fatalf("buckets[1]:%d != 0", bits.buckets[1])
 	}
 
-	var i, hashKey uint32
-	var exist bool
+	var (
+		i       uint64
+		hashKey uint32
+		exist   bool
+	)
 	for i = 0; i < 10000; i++ {
-		hashKey = i & maxMask
+		hashKey = uint32(i) & maxMask
 		//println(i, hashKey)
 		exist = bits.Filter(i, hashKey)
 		if hashKey < 10 {
@@ -49,3 +57,103 @@ func TestFilter(t *testing.T) {
 		}
 	}
 }
+
+func TestNew(t *testing.T) {
+	newBits := bits.Copy()
+
+	if len(newBits.collision) != 11 {
+		t.Fatalf("collision len != 11")
+	}
+
+	if newBits.buckets[0] != 1023 {
+		t.Fatalf("buckets[0]:%d != 1023", newBits.buckets[0])
+	}
+
+	if newBits.buckets[1] != 0 {
+		t.Fatalf("buckets[1]:%d != 0", newBits.buckets[1])
+	}
+
+	newBits.Remove(8192, 0)
+
+	if len(newBits.collision) != 9 {
+		t.Fatalf("collision len:9 != %d", len(newBits.collision))
+	}
+
+	if newBits.buckets[0] != 1023 {
+		t.Fatalf("buckets[0]:%d != 1023", newBits.buckets[0])
+	}
+
+	if isExist := newBits.Filter(8192, 0); !isExist {
+		// 8192这个值虽然remove了，但是0这个值还没remove
+		// 所以这里还是存在的，这是误判
+		t.Fatalf("error")
+	}
+
+	newBits.Remove(0, 0)
+	if isExist := newBits.Filter(0, 0); isExist {
+		t.Fatalf("error")
+	}
+
+	if newBits.buckets[0] != 1022 {
+		t.Fatalf("buckets[0]:%d != 1022", newBits.buckets[0])
+	}
+
+	newBits.Remove(16385, 1)
+	if len(newBits.collision) != 8 {
+		t.Fatalf("collision len:8 != %d", len(newBits.collision))
+	}
+
+	if isExist := newBits.Filter(1, 1); !isExist {
+		t.Fatalf("error")
+	}
+
+	if newBits.buckets[0] != 1022 {
+		t.Fatalf("buckets[0]:%d != 1022", newBits.buckets[0])
+	}
+
+}
+
+func TestRand(t *testing.T) {
+	bits = New(Size2MB)
+	maxMask = uint32(Size2MB<<6 - 1)
+
+	var exist = make(map[uint32]bool)
+	var errorCount int
+	var projectId, posId, n1, n2 uint32
+	var b1 = make([]byte, 4)
+	var b2 = make([]byte, 4)
+	alg := fnv.New32a()
+	n1, n2 = 68000, 100
+	for projectId = 1; projectId < n1; projectId++ {
+		binary.BigEndian.PutUint32(b1, projectId)
+		for posId = 1; posId < n2; posId++ {
+			binary.BigEndian.PutUint32(b2, posId)
+			alg.Reset()
+			alg.Write(append(b1, b2...))
+			hashKey := alg.Sum32()
+			hashKey = hashKey<<6 - 1
+			if _, ok := exist[hashKey]; ok {
+				errorCount++
+			} else {
+				exist[hashKey] = true
+			}
+		}
+	}
+
+	fmt.Printf("Error Count: %d in %d*%d\nError Rate: %f\n", errorCount, n1, n2, float32(errorCount)/(float32(n1)*float32(n2)))
+
+	projectId, posId = 32314, 23413
+	binary.BigEndian.PutUint32(b1, projectId)
+	binary.BigEndian.PutUint32(b2, posId)
+	alg.Reset()
+	alg.Write(append(b1, b2...))
+	fmt.Printf("fnv.New32a: %d, %d: %d\n", projectId, posId, alg.Sum32()&maxMask)
+}
+
+func BenchmarkFilter(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_ = bits.Filter(1234, 43)
+		}
+	})
+}