From c0a463ad0b8a74f9d079c4aea7aa75b92af252e6 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Thu, 10 Aug 2023 18:18:48 +0800 Subject: [PATCH 1/7] add batch_write_and_read_test.go --- structure/batch_write_and_read_test.go | 107 +++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 structure/batch_write_and_read_test.go diff --git a/structure/batch_write_and_read_test.go b/structure/batch_write_and_read_test.go new file mode 100644 index 0000000..3666247 --- /dev/null +++ b/structure/batch_write_and_read_test.go @@ -0,0 +1,107 @@ +package structure + +import ( + "github.com/ByteStorage/FlyDB/config" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func initStringCur() (*StringStructure, *config.Options) { + dir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + opts := config.DefaultOptions + opts.DirPath = dir + "/flydb" + str, _ := NewStringStructure(opts) + return str, &opts +} + +func initHashCur() (*HashStructure, *config.Options) { + dir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + opts := config.DefaultOptions + opts.DirPath = dir + "/flydb" + hash, _ := NewHashStructure(opts) + return hash, &opts +} + +func initListCur() (*ListStructure, *config.Options) { + dir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + opts := config.DefaultOptions + opts.DirPath = dir + "/flydb" + list, _ := NewListStructure(opts) + return list, &opts +} + +func initSetCur() (*SetStructure, *config.Options) { + dir, err := os.UserHomeDir() + if err != nil { + panic(err) + } + opts := config.DefaultOptions + opts.DirPath = dir + "/flydb" + str, _ := NewSetStructure(opts) + return str, &opts +} + +func TestBatchWrite(t *testing.T) { + dbs, _ := initStringCur() + defer dbs.Clean() + dbh, _ := initHashCur() + defer dbh.Clean() + dbl, _ := initListCur() + defer dbl.Stop() + dbset, _ := initSetCur() + defer dbset.Stop() + err = dbs.Set("1", "1", 0) + assert.Nil(t, err) + _, err = dbh.HSet("2", "1", "1", 0) + assert.Nil(t, err) + err = dbl.LPush("3", "1") + assert.Nil(t, err) + err = dbset.SAdd("4", "1") + assert.Nil(t, err) +} + +func TestBatchWriteAndRead(t *testing.T) { + dbs, _ := initStringCur() + defer dbs.Clean() + dbh, _ := initHashCur() + defer dbh.Clean() + dbl, _ := initListCur() + defer dbl.Stop() + dbset, _ := initSetCur() + defer dbset.Stop() + + err := dbs.Set("1", "1", 0) + assert.Nil(t, err) + val, err := dbs.Get("1") + assert.Nil(t, err) + assert.Equal(t, "1", val) + + err = dbl.LPush("3", "1") + assert.Nil(t, err) + val, err = dbl.LPop("3") + assert.Nil(t, err) + assert.Equal(t, "1", val) + + err = dbset.SAdd("4", "1") + assert.Nil(t, err) + assert.Equal(t, "1", val) + members, err := dbset.SMembers("4") + assert.Nil(t, err) + assert.Equal(t, "1", members[0]) + + _, err = dbh.HSet("2", "1", "1", 0) + assert.Nil(t, err) + val, err = dbh.HGet("2", "1") + assert.Nil(t, err) + assert.Equal(t, "1", val) +} From 03f23cb4c41223e0044cb44a69528354fd5dfe50 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 17:52:15 +0800 Subject: [PATCH 2/7] add common method to convert regx #280 --- structure/batch_write_and_read_test.go | 107 ------------------------- structure/types.go | 14 ++++ 2 files changed, 14 insertions(+), 107 deletions(-) delete mode 100644 structure/batch_write_and_read_test.go diff --git a/structure/batch_write_and_read_test.go b/structure/batch_write_and_read_test.go deleted file mode 100644 index 3666247..0000000 --- a/structure/batch_write_and_read_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package structure - -import ( - "github.com/ByteStorage/FlyDB/config" - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func initStringCur() (*StringStructure, *config.Options) { - dir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - opts := config.DefaultOptions - opts.DirPath = dir + "/flydb" - str, _ := NewStringStructure(opts) - return str, &opts -} - -func initHashCur() (*HashStructure, *config.Options) { - dir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - opts := config.DefaultOptions - opts.DirPath = dir + "/flydb" - hash, _ := NewHashStructure(opts) - return hash, &opts -} - -func initListCur() (*ListStructure, *config.Options) { - dir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - opts := config.DefaultOptions - opts.DirPath = dir + "/flydb" - list, _ := NewListStructure(opts) - return list, &opts -} - -func initSetCur() (*SetStructure, *config.Options) { - dir, err := os.UserHomeDir() - if err != nil { - panic(err) - } - opts := config.DefaultOptions - opts.DirPath = dir + "/flydb" - str, _ := NewSetStructure(opts) - return str, &opts -} - -func TestBatchWrite(t *testing.T) { - dbs, _ := initStringCur() - defer dbs.Clean() - dbh, _ := initHashCur() - defer dbh.Clean() - dbl, _ := initListCur() - defer dbl.Stop() - dbset, _ := initSetCur() - defer dbset.Stop() - err = dbs.Set("1", "1", 0) - assert.Nil(t, err) - _, err = dbh.HSet("2", "1", "1", 0) - assert.Nil(t, err) - err = dbl.LPush("3", "1") - assert.Nil(t, err) - err = dbset.SAdd("4", "1") - assert.Nil(t, err) -} - -func TestBatchWriteAndRead(t *testing.T) { - dbs, _ := initStringCur() - defer dbs.Clean() - dbh, _ := initHashCur() - defer dbh.Clean() - dbl, _ := initListCur() - defer dbl.Stop() - dbset, _ := initSetCur() - defer dbset.Stop() - - err := dbs.Set("1", "1", 0) - assert.Nil(t, err) - val, err := dbs.Get("1") - assert.Nil(t, err) - assert.Equal(t, "1", val) - - err = dbl.LPush("3", "1") - assert.Nil(t, err) - val, err = dbl.LPop("3") - assert.Nil(t, err) - assert.Equal(t, "1", val) - - err = dbset.SAdd("4", "1") - assert.Nil(t, err) - assert.Equal(t, "1", val) - members, err := dbset.SMembers("4") - assert.Nil(t, err) - assert.Equal(t, "1", members[0]) - - _, err = dbh.HSet("2", "1", "1", 0) - assert.Nil(t, err) - val, err = dbh.HGet("2", "1") - assert.Nil(t, err) - assert.Equal(t, "1", val) -} diff --git a/structure/types.go b/structure/types.go index 4537851..ea7431e 100644 --- a/structure/types.go +++ b/structure/types.go @@ -2,7 +2,9 @@ package structure import ( "errors" + "regexp" "strconv" + "strings" ) type DataStructure = byte @@ -189,3 +191,15 @@ func convertToFloat(Value interface{}) (float64, error) { return 0, errors.New("unsupported type") } } + +func convertToRegexp(pattern string) string { + // 转义正则特殊字符 + re := regexp.MustCompile(`([\[\]\(\)\{\}\^\$\.\+])`) + pattern = re.ReplaceAllString(pattern, `\$1`) + + // 将 * 和 ? 转换为对应的正则表达式 + pattern = strings.ReplaceAll(pattern, "*", ".*") + pattern = strings.ReplaceAll(pattern, "?", ".") + + return "^" + pattern + "$" // 添加 ^ 和 $ 以确保完全匹配 +} From 46cb9d67a3aac49cb8dbf976de9754641a8be018 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 17:52:56 +0800 Subject: [PATCH 3/7] add hash method to match regx #280 --- structure/hash.go | 35 ++++++++++++++++++++++++++++------- structure/hash_test.go | 16 ++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/structure/hash.go b/structure/hash.go index 91fd786..0706bfc 100644 --- a/structure/hash.go +++ b/structure/hash.go @@ -7,6 +7,7 @@ import ( "github.com/ByteStorage/FlyDB/config" "github.com/ByteStorage/FlyDB/engine" _const "github.com/ByteStorage/FlyDB/lib/const" + "regexp" "strconv" "time" ) @@ -1158,21 +1159,41 @@ func (hs *HashStructure) HTypes(k string, f interface{}) (string, error) { // // []string: A list of field names in the hash. // error: An error if occurred during the operation, or nil on success. -func (hs *HashStructure) Keys() []string { +func (hs *HashStructure) Keys(regx string) ([]string, error) { + toRegexp := convertToRegexp(regx) + compile, err := regexp.Compile(toRegexp) + if err != nil { + return nil, err + } // Get all the keys from the database - byte_keys := hs.db.GetListKeys() + byteKeys := hs.db.GetListKeys() // Create a new slice of strings keys := make([]string, 0) - for _, key := range byte_keys { - // Check if the key has the identifier suffix - if !keysIdentify(key) { - keys = append(keys, string(key)) + for _, key := range byteKeys { + if compile.MatchString(string(key)) { + // Check if the key has the identifier suffix + if !keysIdentify(key) { + fields := hs.GetFields(string(key)) + ok := true + for _, field := range fields { + _, err := hs.HGet(string(key), field) + if err != nil { + ok = false + break + } + } + if !ok { + continue + } + keys = append(keys, string(key)) + } } + } - return keys + return keys, nil } // GetFields returns a list of all field names in the hash stored at the specified key. diff --git a/structure/hash_test.go b/structure/hash_test.go index a1089ed..5032e0f 100644 --- a/structure/hash_test.go +++ b/structure/hash_test.go @@ -553,6 +553,22 @@ func TestHashStructure_GetFields(t *testing.T) { fields2 := hash.GetFields("qqqqqq") assert.Equal(t, fields2, []string{"!qqq!1", "!qqq!2", "!qqq!3"}) + keys, err := hash.Keys("*") + assert.Nil(t, err) + assert.Equal(t, keys, []string{"qqqqqq", "qweqwe"}) + + keys, err = hash.Keys("q*") + assert.Nil(t, err) + assert.Equal(t, keys, []string{"qqqqqq", "qweqwe"}) + + keys, err = hash.Keys("qq*") + assert.Nil(t, err) + assert.Equal(t, keys, []string{"qqqqqq"}) + + keys, err = hash.Keys("q?*") + assert.Nil(t, err) + assert.Equal(t, keys, []string{"qqqqqq", "qweqwe"}) + } func TestHashStructure_HGetAllFieldAndValue(t *testing.T) { From b9ac10b9ea360fb17c55d13f969154865e7c6b3a Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 17:53:11 +0800 Subject: [PATCH 4/7] add list method to match regx #280 --- structure/list.go | 25 +++++++++++++++++++++---- structure/list_test.go | 24 ++++++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/structure/list.go b/structure/list.go index 03280d1..882df07 100644 --- a/structure/list.go +++ b/structure/list.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "reflect" + "regexp" "time" "encoding/gob" @@ -432,11 +433,27 @@ func (l *ListStructure) LIndex(key string, index int) (interface{}, error) { } // Keys returns all the keys of the list structure. -func (l *ListStructure) Keys() ([]string, error) { +func (l *ListStructure) Keys(regx string) ([]string, error) { + toRegexp := convertToRegexp(regx) + compile, err := regexp.Compile(toRegexp) + if err != nil { + return nil, err + } var keys []string - byte_keys := l.db.GetListKeys() - for _, key := range byte_keys { - keys = append(keys, string(key)) + byteKeys := l.db.GetListKeys() + for _, key := range byteKeys { + // match prefix and key + if compile.MatchString(string(key)) { + // check if deleted + db, _, err := l.getListFromDB(string(key), true) + if err != nil { + return nil, err + } + if db.Length == 0 { + continue + } + keys = append(keys, string(key)) + } } return keys, nil } diff --git a/structure/list_test.go b/structure/list_test.go index 0330d4d..a78e71b 100644 --- a/structure/list_test.go +++ b/structure/list_test.go @@ -363,16 +363,32 @@ func TestListStructure_Keys(t *testing.T) { list, _ := initList() defer list.db.Clean() - listErr = list.LPush("1", randkv.RandomValue(100), 0) + listErr = list.LPush("111", randkv.RandomValue(100), 0) assert.Nil(t, listErr) - listErr = list.LPush("2", randkv.RandomValue(100), 0) + listErr = list.LPush("122", randkv.RandomValue(100), 0) assert.Nil(t, listErr) - listErr = list.LPush("q", randkv.RandomValue(100), 0) + listErr = list.LPush("123", randkv.RandomValue(100), 0) assert.Nil(t, listErr) - keys, err := list.Keys() + keys, err := list.Keys("*") + assert.Nil(t, err) + assert.Equal(t, len(keys), 3) + + keys, err = list.Keys("1*") + assert.Nil(t, err) + assert.Equal(t, len(keys), 3) + + keys, err = list.Keys("12*") + assert.Nil(t, err) + assert.Equal(t, len(keys), 2) + + keys, err = list.Keys("123") + assert.Nil(t, err) + assert.Equal(t, len(keys), 1) + + keys, err = list.Keys("1?*") assert.Nil(t, err) assert.Equal(t, len(keys), 3) } From cff35bc77e840efb4ba00e66f1c847dbbb0c3387 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 17:53:25 +0800 Subject: [PATCH 5/7] add set method to match regx #280 --- structure/set.go | 20 ++++++++++++++++---- structure/set_test.go | 22 +++++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/structure/set.go b/structure/set.go index d076dfa..04cf7f7 100644 --- a/structure/set.go +++ b/structure/set.go @@ -6,6 +6,7 @@ import ( "github.com/ByteStorage/FlyDB/engine" _const "github.com/ByteStorage/FlyDB/lib/const" "github.com/ByteStorage/FlyDB/lib/encoding" + "regexp" ) type SetStructure struct { @@ -224,11 +225,22 @@ func (s *SetStructure) SDiff(keys ...string) ([]string, error) { } // Keys returns all the keys of the set structure -func (s *SetStructure) Keys() ([]string, error) { +func (s *SetStructure) Keys(regx string) ([]string, error) { + toRegexp := convertToRegexp(regx) + compile, err := regexp.Compile(toRegexp) + if err != nil { + return nil, err + } var keys []string - byte_keys := s.db.GetListKeys() - for _, key := range byte_keys { - keys = append(keys, string(key)) + byteKeys := s.db.GetListKeys() + for _, key := range byteKeys { + if compile.MatchString(string(key)) { + // check if deleted + if !s.exists(string(key)) { + continue + } + keys = append(keys, string(key)) + } } return keys, nil } diff --git a/structure/set_test.go b/structure/set_test.go index d49b187..ed22403 100644 --- a/structure/set_test.go +++ b/structure/set_test.go @@ -1062,10 +1062,10 @@ func TestSetStructure_Keys(t *testing.T) { set, _ := initTestSetDb() defer set.db.Clean() - err = set.SAdd("testKey1", "non1") + err = set.SAdd("testKey11", "non1") assert.Nil(t, err) - err = set.SAdd("testKey2", "non1") + err = set.SAdd("testKey12", "non1") assert.Nil(t, err) err = set.SAdd("testKey3", "non1") @@ -1074,8 +1074,24 @@ func TestSetStructure_Keys(t *testing.T) { err = set.SAdd("testKey4", "non1") assert.Nil(t, err) - keys, err := set.Keys() + keys, err := set.Keys("*") assert.Nil(t, err) assert.Equal(t, 4, len(keys)) + keys, err = set.Keys("testKey*") + assert.Nil(t, err) + assert.Equal(t, 4, len(keys)) + + keys, err = set.Keys("testKey1*") + assert.Nil(t, err) + assert.Equal(t, 2, len(keys)) + + keys, err = set.Keys("testKey?*") + assert.Nil(t, err) + assert.Equal(t, 4, len(keys)) + + keys, err = set.Keys("testKey111?*") + assert.Nil(t, err) + assert.Equal(t, 0, len(keys)) + } From 3eea5344dcf64b54f459c8b0c5cd4f8e29be07f2 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 17:53:42 +0800 Subject: [PATCH 6/7] add string method to match regx #280 --- structure/string.go | 27 ++++++++++++++++++--------- structure/string_test.go | 20 ++++++++++++++++---- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/structure/string.go b/structure/string.go index cbb977f..2b01dee 100644 --- a/structure/string.go +++ b/structure/string.go @@ -7,6 +7,7 @@ import ( "github.com/ByteStorage/FlyDB/config" "github.com/ByteStorage/FlyDB/engine" _const "github.com/ByteStorage/FlyDB/lib/const" + "regexp" "strconv" "time" ) @@ -311,20 +312,28 @@ func (s *StringStructure) DecrBy(key string, amount int, ttl int64) error { } // Keys returns all keys matching pattern -func (s *StringStructure) Keys() ([]string, error) { +func (s *StringStructure) Keys(regx string) ([]string, error) { + toRegexp := convertToRegexp(regx) + compile, err := regexp.Compile(toRegexp) + if err != nil { + return nil, err + } var keys []string byteKeys := s.db.GetListKeys() for _, key := range byteKeys { - // check if key is expired - _, err := s.Get(string(key)) - if err != nil { - if errors.Is(err, _const.ErrKeyIsExpired) { - continue - } else { - return nil, err + if compile.MatchString(string(key)) { + // check if key is expired + _, err := s.Get(string(key)) + if err != nil { + if errors.Is(err, _const.ErrKeyIsExpired) || errors.Is(err, _const.ErrKeyNotFound) { + continue + } else { + return nil, err + } } + keys = append(keys, string(key)) } - keys = append(keys, string(key)) + } return keys, nil } diff --git a/structure/string_test.go b/structure/string_test.go index 45827ce..8453058 100644 --- a/structure/string_test.go +++ b/structure/string_test.go @@ -425,13 +425,13 @@ func TestStringStructure_Keys(t *testing.T) { str, _ := initdb() defer str.db.Clean() - err = str.Set("1", randkv.RandomValue(100), 1) + err = str.Set("11", randkv.RandomValue(100), 1) assert.Nil(t, err) - err = str.Set("2", randkv.RandomValue(100), 2) + err = str.Set("21", randkv.RandomValue(100), 2) assert.Nil(t, err) - err = str.Set("3", randkv.RandomValue(100), 0) + err = str.Set("31", randkv.RandomValue(100), 0) assert.Nil(t, err) err = str.Set("hhh", randkv.RandomValue(100), 0) @@ -442,9 +442,21 @@ func TestStringStructure_Keys(t *testing.T) { time.Sleep(2 * time.Second) - keys, err := str.Keys() + keys, err := str.Keys("*") assert.Nil(t, err) assert.Equal(t, len(keys), 3) + + keys, err = str.Keys("1*") + assert.Nil(t, err) + assert.Equal(t, len(keys), 0) + + keys, err = str.Keys("3*") + assert.Nil(t, err) + assert.Equal(t, len(keys), 1) + + keys, err = str.Keys("*1") + assert.Nil(t, err) + assert.Equal(t, len(keys), 1) } func TestStringStructure_TTL(t *testing.T) { From 255d213e6ff53f9aa4005479e15946fe6a4faac5 Mon Sep 17 00:00:00 2001 From: sjcsjc123 <1401189096@qq.com> Date: Fri, 11 Aug 2023 18:00:36 +0800 Subject: [PATCH 7/7] add string method to match regx #280 --- structure/list.go | 2 +- structure/string.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/structure/list.go b/structure/list.go index 882df07..8097f7f 100644 --- a/structure/list.go +++ b/structure/list.go @@ -447,7 +447,7 @@ func (l *ListStructure) Keys(regx string) ([]string, error) { // check if deleted db, _, err := l.getListFromDB(string(key), true) if err != nil { - return nil, err + continue } if db.Length == 0 { continue diff --git a/structure/string.go b/structure/string.go index 2b01dee..0006730 100644 --- a/structure/string.go +++ b/structure/string.go @@ -325,11 +325,7 @@ func (s *StringStructure) Keys(regx string) ([]string, error) { // check if key is expired _, err := s.Get(string(key)) if err != nil { - if errors.Is(err, _const.ErrKeyIsExpired) || errors.Is(err, _const.ErrKeyNotFound) { - continue - } else { - return nil, err - } + continue } keys = append(keys, string(key)) }