diff --git a/db/engine/put_get_test.go b/db/engine/put_get_test.go index bb04dad..85e09f9 100644 --- a/db/engine/put_get_test.go +++ b/db/engine/put_get_test.go @@ -1,38 +1,51 @@ package engine import ( - "fmt" + "os" + "testing" + "github.com/ByteStorage/FlyDB/config" "github.com/ByteStorage/FlyDB/lib/randkv" "github.com/stretchr/testify/assert" - "os" - "testing" - "time" ) -func TestPutAndGet(t *testing.T) { +func BenchmarkPut(b *testing.B) { opts := config.DefaultOptions dir, _ := os.MkdirTemp("", "flydb-benchmark") opts.DirPath = dir opts.DataFileSize = 64 * 1024 * 1024 db, err := NewDB(opts) defer db.Clean() - assert.Nil(t, err) - assert.NotNil(t, db) + assert.Nil(b, err) + assert.NotNil(b, db) - start := time.Now() - for n := 0; n < 500000; n++ { + n := 0 + for b.Loop() { err = db.Put(randkv.GetTestKey(n), randkv.RandomValue(24)) - assert.Nil(t, err) + assert.Nil(b, err) + n++ } - end := time.Now() - fmt.Println("put time: ", end.Sub(start).String()) +} + +func BenchmarkGet(b *testing.B) { + opts := config.DefaultOptions + dir, _ := os.MkdirTemp("", "flydb-benchmark") + opts.DirPath = dir + opts.DataFileSize = 64 * 1024 * 1024 + db, err := NewDB(opts) + defer db.Clean() + assert.Nil(b, err) + assert.NotNil(b, db) + + for n := 0; n < b.N; n++ { + err = db.Put(randkv.GetTestKey(n), randkv.RandomValue(24)) + assert.Nil(b, err) + } + + b.ResetTimer() - start = time.Now() - for n := 0; n < 500000; n++ { + for n := 0; n < b.N; n++ { _, err = db.Get(randkv.GetTestKey(n)) - assert.Nil(t, err) + assert.Nil(b, err) } - end = time.Now() - fmt.Println("get time: ", end.Sub(start).String()) } diff --git a/db/index/art.go b/db/index/art.go index 15be4ad..78176bb 100644 --- a/db/index/art.go +++ b/db/index/art.go @@ -2,25 +2,26 @@ package index import ( "bytes" - "github.com/ByteStorage/FlyDB/db/data" - art "github.com/plar/go-adaptive-radix-tree" "sort" "sync" + + "github.com/ByteStorage/FlyDB/db/data" + art "github.com/Clement-Jean/go-art" ) // Adaptive Radix Tree Index // The following link is the ART library written by go. // If you need to know more about it, please go to the corresponding warehouse. -// https://github.com/plar/go-adaptive-radix-tree +// https://github.com/Clement-Jean/go-art type AdaptiveRadixTree struct { - tree art.Tree + tree art.Tree[[]byte, *data.LogRecordPst] lock *sync.RWMutex } // NewART Initializes the adaptive radix tree index func NewART() *AdaptiveRadixTree { return &AdaptiveRadixTree{ - tree: art.New(), + tree: art.NewAlphaSortedTree[[]byte, *data.LogRecordPst](), lock: new(sync.RWMutex), } } @@ -39,21 +40,19 @@ func (artree *AdaptiveRadixTree) Get(key []byte) *data.LogRecordPst { if !found { return nil } - return value.(*data.LogRecordPst) + return value } func (artree *AdaptiveRadixTree) Delete(key []byte) bool { artree.lock.Lock() defer artree.lock.Unlock() - _, deleted := artree.tree.Delete(key) - return deleted + return artree.tree.Delete(key) } func (artree *AdaptiveRadixTree) Size() int { artree.lock.RLock() defer artree.lock.RUnlock() - size := artree.tree.Size() - return size + return artree.tree.Size() } func (artree *AdaptiveRadixTree) Iterator(reverse bool) Iterator { @@ -69,7 +68,7 @@ type ARTreeIterator struct { values []*Item // Key + Location index information } -func NewARTreeIterator(tree art.Tree, reverse bool) *ARTreeIterator { +func NewARTreeIterator(tree art.Tree[[]byte, *data.LogRecordPst], reverse bool) *ARTreeIterator { // Estimate the expected slice capacity based on tree size expectedSize := tree.Size() @@ -80,26 +79,32 @@ func NewARTreeIterator(tree art.Tree, reverse bool) *ARTreeIterator { values := make([]*Item, 0, expectedSize) // Store all the data in an array - saveToValues := func(node art.Node) bool { - item := &Item{ - key: node.Key(), - pst: node.Value().(*data.LogRecordPst), + if reverse { + // Reverse the values slice if reverse is true + for key, value := range tree.Backward() { + item := &Item{ + key: key, + pst: value, + } + mutex.Lock() + + // Append item to values slice + values = append(values, item) + + mutex.Unlock() } - mutex.Lock() - - // Append item to values slice - values = append(values, item) - - mutex.Unlock() + } else { + for key, value := range tree.All() { + item := &Item{ + key: key, + pst: value, + } + mutex.Lock() - return true - } - tree.ForEach(saveToValues) + // Append item to values slice + values = append(values, item) - // Reverse the values slice if reverse is true - if reverse { - for i, j := 0, len(values)-1; i < j; i, j = i+1, j-1 { - values[i], values[j] = values[j], values[i] + mutex.Unlock() } } diff --git a/db/index/art_with_bloom.go b/db/index/art_with_bloom.go index 4e20db4..a1290c1 100644 --- a/db/index/art_with_bloom.go +++ b/db/index/art_with_bloom.go @@ -1,18 +1,19 @@ package index import ( + "sync" + "github.com/ByteStorage/FlyDB/db/data" "github.com/ByteStorage/FlyDB/lib/bloom" - art "github.com/plar/go-adaptive-radix-tree" - "sync" + art "github.com/Clement-Jean/go-art" ) // AdaptiveRadixTreeWithBloom Adaptive Radix Tree Index // The following link is the ART library written by go. // If you need to know more about it, please go to the corresponding warehouse. -// https://github.com/plar/go-adaptive-radix-tree +// https://github.com/Clement-Jean/go-art type AdaptiveRadixTreeWithBloom struct { - tree art.Tree + tree art.Tree[[]byte, *data.LogRecordPst] lock *sync.RWMutex filter *bloom.Filter } @@ -20,7 +21,7 @@ type AdaptiveRadixTreeWithBloom struct { // NewARTWithBloom Initializes the adaptive radix tree index func NewARTWithBloom() *AdaptiveRadixTreeWithBloom { return &AdaptiveRadixTreeWithBloom{ - tree: art.New(), + tree: art.NewAlphaSortedTree[[]byte, *data.LogRecordPst](), lock: new(sync.RWMutex), filter: bloom.NewBloomFilter(1000, 0.01), } @@ -44,7 +45,7 @@ func (artree *AdaptiveRadixTreeWithBloom) Get(key []byte) *data.LogRecordPst { if !found { return nil } - return value.(*data.LogRecordPst) + return value } func (artree *AdaptiveRadixTreeWithBloom) Delete(key []byte) bool { @@ -53,15 +54,13 @@ func (artree *AdaptiveRadixTreeWithBloom) Delete(key []byte) bool { } artree.lock.Lock() defer artree.lock.Unlock() - _, deleted := artree.tree.Delete(key) - return deleted + return artree.tree.Delete(key) } func (artree *AdaptiveRadixTreeWithBloom) Size() int { artree.lock.RLock() defer artree.lock.RUnlock() - size := artree.tree.Size() - return size + return artree.tree.Size() } func (artree *AdaptiveRadixTreeWithBloom) Iterator(reverse bool) Iterator { diff --git a/go.mod b/go.mod index 8f28b68..fd3d2c8 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/ByteStorage/FlyDB -go 1.18 +go 1.24.0 require ( + github.com/Clement-Jean/go-art v0.0.0-20250326081316-f1b2fb5e334c github.com/bits-and-blooms/bitset v1.8.0 github.com/boltdb/bolt v1.3.1 github.com/bwmarrin/snowflake v0.3.0 @@ -17,13 +18,14 @@ require ( github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 github.com/klauspost/reedsolomon v1.11.7 github.com/pkg/errors v0.9.1 - github.com/plar/go-adaptive-radix-tree v1.0.5 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.8.2 github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c go.etcd.io/bbolt v1.3.7 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 + golang.org/x/net v0.8.0 + golang.org/x/sys v0.6.0 google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.31.0 ) @@ -47,14 +49,11 @@ require ( github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/panjf2000/ants v1.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/text v0.23.0 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 8c568fd..373c68c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Clement-Jean/go-art v0.0.0-20250326081316-f1b2fb5e334c h1:rk07r6OAsuDIri5WashmUSWcIqzr0Ljp0emVJrlyqKQ= +github.com/Clement-Jean/go-art v0.0.0-20250326081316-f1b2fb5e334c/go.mod h1:2I5OXlf3ctXzpdEfEidXwYYgT28ByMMzprFAW/HseNQ= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= @@ -11,6 +13,7 @@ github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -75,6 +78,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= @@ -140,16 +144,12 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= -github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/plar/go-adaptive-radix-tree v1.0.5 h1:rHR89qy/6c24TBAHullFMrJsU9hGlKmPibdBGU6/gbM= -github.com/plar/go-adaptive-radix-tree v1.0.5/go.mod h1:15VOUO7R9MhJL8HOJdpydR0rvanrtRE6fA6XSa/tqWE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -185,7 +185,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= @@ -196,6 +195,7 @@ go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -233,8 +233,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=