Implement Redis backend for cache (#54)

Author: man90es

README.md

@@ -58,6 +58,10 @@ Available flags:
 - `-verbose`
 	- Allows to put the app into verbose mode and print out additional logs to stdout
 	- Default value: none, no additional output is produced
+- `-redis`
+	- Specifies Redis server's address
+	- Type: string
+	- Default value: none
 
 You can use them like this:
 ```bash

cache/cache.go

@@ -1,12 +1,14 @@
 package cache
 
 import (
+	"context"
+	"encoding/json"
 	"strings"
 	"time"
 
 	goCache "github.com/patrickmn/go-cache"
+	"github.com/redis/go-redis/v9"
 	"github.com/spf13/viper"
-	messagebus "github.com/vardius/message-bus"
 	"golang.org/x/exp/maps"
 
 	"bdo-rest-api/models"
@@ -19,60 +21,65 @@ type CacheEntry[T any] struct {
 	Status int       `json:"status"`
 }
 
-type cache[T any] struct {
-	Bus           messagebus.MessageBus
-	internalCache *goCache.Cache
+type Cache[T any] interface {
+	AddRecord(keys []string, data T, status int, taskId string) (date string, expires string)
+	GetRecord(keys []string) (data T, status int, date string, expires string, found bool)
+	GetItemCount() int
+	GetKeys() []string
+	GetValues() []CacheEntry[T]
 }
 
 func joinKeys(keys []string) string {
 	return strings.Join(keys, ",")
 }
 
-func newCache[T any]() *cache[T] {
-	cacheTTL := viper.GetDuration("cachettl")
+type memoryCache[T any] struct {
+	internalCache *goCache.Cache
+	ttl           time.Duration
+}
+
+func newMemoryCache[T any]() *memoryCache[T] {
+	ttl := viper.GetDuration("cachettl")
 
-	return &cache[T]{
-		Bus:           messagebus.New(100), // Idk what buffer size is optimal
-		internalCache: goCache.New(cacheTTL, min(time.Hour, cacheTTL)),
+	return &memoryCache[T]{
+		internalCache: goCache.New(ttl, min(time.Hour, ttl)),
+		ttl:           ttl,
 	}
 }
 
-func (c *cache[T]) AddRecord(keys []string, data T, status int, taskId string) (date string, expires string) {
-	cacheTTL := viper.GetDuration("cachettl")
+func (c *memoryCache[T]) AddRecord(keys []string, data T, status int, taskId string) (date, expires string) {
 	entry := CacheEntry[T]{
 		Data:   data,
 		Date:   time.Now(),
 		Status: status,
 	}
 
-	c.internalCache.Add(joinKeys(keys), entry, cacheTTL)
-	c.Bus.Publish(taskId, entry)
+	c.internalCache.Add(joinKeys(keys), entry, c.ttl)
 
-	return utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(cacheTTL))
+	return utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(c.ttl))
 }
 
-func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, expires string, found bool) {
-	cacheTTL := viper.GetDuration("cachettl")
-	anyEntry, found := c.internalCache.Get(joinKeys(keys))
+func (c *memoryCache[T]) GetRecord(keys []string) (data T, status int, date, expires string, found bool) {
+	anyEntry, exp, found := c.internalCache.GetWithExpiration(joinKeys(keys))
 
 	if !found {
 		return
 	}
 
 	entry := anyEntry.(CacheEntry[T])
 
-	return entry.Data, entry.Status, utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(cacheTTL)), found
+	return entry.Data, entry.Status, utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(exp), true
 }
 
-func (c *cache[T]) GetItemCount() int {
+func (c *memoryCache[T]) GetItemCount() int {
 	return c.internalCache.ItemCount()
 }
 
-func (c *cache[T]) GetKeys() []string {
+func (c *memoryCache[T]) GetKeys() []string {
 	return maps.Keys(c.internalCache.Items())
 }
 
-func (c *cache[T]) GetValues() []CacheEntry[T] {
+func (c *memoryCache[T]) GetValues() []CacheEntry[T] {
 	items := c.internalCache.Items()
 	result := make([]CacheEntry[T], 0, len(items))
 
@@ -83,7 +90,107 @@ func (c *cache[T]) GetValues() []CacheEntry[T] {
 	return result
 }
 
-var GuildProfiles = newCache[models.GuildProfile]()
-var GuildSearch = newCache[[]models.GuildProfile]()
-var Profiles = newCache[models.Profile]()
-var ProfileSearch = newCache[[]models.Profile]()
+type redisCache[T any] struct {
+	client    *redis.Client
+	ctx       context.Context
+	namespace string
+	ttl       time.Duration
+}
+
+func newRedisCache[T any](client *redis.Client, namespace string) *redisCache[T] {
+	return &redisCache[T]{
+		client:    client,
+		ctx:       context.Background(),
+		namespace: namespace + ":",
+		ttl:       viper.GetDuration("cachettl"),
+	}
+}
+
+func (c *redisCache[T]) AddRecord(keys []string, data T, status int, taskId string) (date, expires string) {
+	entry := CacheEntry[T]{
+		Data:   data,
+		Date:   time.Now(),
+		Status: status,
+	}
+
+	b, _ := json.Marshal(entry)
+	c.client.Set(c.ctx, c.namespace+joinKeys(keys), b, c.ttl)
+
+	return utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(c.ttl))
+}
+
+func (c *redisCache[T]) GetRecord(keys []string) (data T, status int, date string, expires string, found bool) {
+	val, err := c.client.Get(c.ctx, c.namespace+joinKeys(keys)).Bytes()
+	if err != nil {
+		return
+	}
+
+	var entry CacheEntry[T]
+	if err := json.Unmarshal(val, &entry); err != nil {
+		return
+	}
+
+	ttl := c.client.TTL(c.ctx, c.namespace+joinKeys(keys)).Val()
+
+	return entry.Data, entry.Status, utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(time.Now().Add(ttl)), true
+}
+
+func (c *redisCache[T]) GetItemCount() int {
+	keys, err := c.client.Keys(c.ctx, c.namespace+"*").Result()
+	if err != nil {
+		return 0
+	}
+	return len(keys)
+}
+
+func (c *redisCache[T]) GetKeys() []string {
+	keys, _ := c.client.Keys(c.ctx, c.namespace+"*").Result()
+
+	// Remove namespace from keys
+	for i, k := range keys {
+		keys[i] = strings.TrimPrefix(k, c.namespace)
+	}
+
+	return keys
+}
+
+func (c *redisCache[T]) GetValues() []CacheEntry[T] {
+	keys, _ := c.client.Keys(c.ctx, c.namespace+"*").Result()
+	result := make([]CacheEntry[T], 0, len(keys))
+
+	for _, k := range keys {
+		val, err := c.client.Get(c.ctx, k).Bytes()
+		if err != nil {
+			continue
+		}
+
+		var entry CacheEntry[T]
+		if err := json.Unmarshal(val, &entry); err != nil {
+			continue
+		}
+		result = append(result, entry)
+	}
+
+	return result
+}
+
+var (
+	GuildProfiles Cache[models.GuildProfile]
+	GuildSearch   Cache[[]models.GuildProfile]
+	Profiles      Cache[models.Profile]
+	ProfileSearch Cache[[]models.Profile]
+)
+
+func InitCache() {
+	if redisClient, err := newRedisClient(viper.GetString("redis")); err == nil {
+		GuildProfiles = newRedisCache[models.GuildProfile](redisClient, "gpc")
+		GuildSearch = newRedisCache[[]models.GuildProfile](redisClient, "gsc")
+		Profiles = newRedisCache[models.Profile](redisClient, "pc")
+		ProfileSearch = newRedisCache[[]models.Profile](redisClient, "psc")
+	} else {
+		GuildProfiles = newMemoryCache[models.GuildProfile]()
+		GuildSearch = newMemoryCache[[]models.GuildProfile]()
+		Profiles = newMemoryCache[models.Profile]()
+		ProfileSearch = newMemoryCache[[]models.Profile]()
+	}
+}

cache/cache_test.go

@@ -7,58 +7,124 @@ import (
 	"github.com/spf13/viper"
 )
 
+// Simple helper type for testing
+type testStruct struct {
+	Value string
+}
+
 func init() {
-	viper.Set("cachettl", time.Second)
+	// Ensure TTL is something predictable
+	viper.Set("cachettl", time.Second*10)
+}
+
+func TestJoinKeys(t *testing.T) {
+	keys := []string{"a", "b", "c"}
+	expected := "a,b,c"
+
+	if got := joinKeys(keys); got != expected {
+		t.Fatalf("joinKeys() = %s; want %s", got, expected)
+	}
 }
 
-func TestCache(t *testing.T) {
-	// Create a cache instance for testing
-	testCache := newCache[string]()
+func TestMemoryCacheAddAndGetRecord(t *testing.T) {
+	c := newMemoryCache[testStruct]()
 
-	// Test AddRecord and GetRecord
+	data := testStruct{Value: "hello"}
 	keys := []string{"key1", "key2"}
-	data := "test data"
 	status := 200
-	taskId := "task-id"
 
-	date, expires := testCache.AddRecord(keys, data, status, taskId)
+	_, _ = c.AddRecord(keys, data, status, "task123")
+
+	gotData, gotStatus, _, _, found := c.GetRecord(keys)
+	if !found {
+		t.Fatal("Expected record to be found, got not found")
+	}
+
+	if gotData.Value != "hello" {
+		t.Fatalf("Expected data 'hello', got '%s'", gotData.Value)
+	}
 
-	// Validate AddRecord results
-	if date == "" || expires == "" {
-		t.Error("AddRecord should return non-empty date and expires values")
+	if gotStatus != status {
+		t.Fatalf("Expected status %d, got %d", status, gotStatus)
 	}
+}
 
-	// Test GetRecord for an existing record
-	returnedData, returnedStatus, returnedDate, returnedExpires, found := testCache.GetRecord(keys)
+func TestMemoryCacheMissingRecord(t *testing.T) {
+	c := newMemoryCache[testStruct]()
 
-	if !found {
-		t.Error("GetRecord should find the record")
+	_, _, _, _, found := c.GetRecord([]string{"does", "not", "exist"})
+	if found {
+		t.Fatal("Expected record NOT to be found")
+	}
+}
+
+func TestMemoryCacheItemCount(t *testing.T) {
+	c := newMemoryCache[testStruct]()
+
+	if c.GetItemCount() != 0 {
+		t.Fatal("Expected empty cache")
+	}
+
+	c.AddRecord([]string{"a"}, testStruct{"x"}, 200, "task1")
+	c.AddRecord([]string{"b"}, testStruct{"y"}, 200, "task2")
+
+	if c.GetItemCount() != 2 {
+		t.Fatalf("Expected 2 items, got %d", c.GetItemCount())
+	}
+}
+
+func TestMemoryCacheGetKeys(t *testing.T) {
+	c := newMemoryCache[testStruct]()
+
+	c.AddRecord([]string{"k1"}, testStruct{"v1"}, 200, "task1")
+	c.AddRecord([]string{"k2"}, testStruct{"v2"}, 200, "task2")
+
+	keys := c.GetKeys()
+
+	if len(keys) != 2 {
+		t.Fatalf("Expected 2 keys, got %d", len(keys))
 	}
 
-	// Validate GetRecord results
-	if returnedData != data || returnedStatus != status || returnedDate == "" || returnedExpires == "" {
-		t.Error("GetRecord returned unexpected values")
+	// NOTE: key order in map is not stable
+	found1, found2 := false, false
+	for _, k := range keys {
+		if k == "k1" {
+			found1 = true
+		}
+		if k == "k2" {
+			found2 = true
+		}
 	}
 
-	// Test GetItemCount
-	itemCount := testCache.GetItemCount()
-	if itemCount != 1 {
-		t.Errorf("GetItemCount should return 1, but got %d", itemCount)
+	if !found1 || !found2 {
+		t.Fatalf("Expected keys k1 and k2, got %v", keys)
 	}
+}
 
-	// Sleep for a while to allow the cache entry to expire
-	time.Sleep(2 * time.Second)
+func TestMemoryCacheGetValues(t *testing.T) {
+	c := newMemoryCache[testStruct]()
 
-	// Test GetRecord for an expired record
-	_, _, _, _, found = testCache.GetRecord(keys)
+	c.AddRecord([]string{"a"}, testStruct{"aaa"}, 200, "task1")
+	c.AddRecord([]string{"b"}, testStruct{"bbb"}, 200, "task2")
 
-	if found {
-		t.Error("GetRecord should not find an expired record")
+	values := c.GetValues()
+
+	if len(values) != 2 {
+		t.Fatalf("Expected 2 values, got %d", len(values))
+	}
+
+	// Verify content
+	foundA, foundB := false, false
+	for _, v := range values {
+		if v.Data.Value == "aaa" {
+			foundA = true
+		}
+		if v.Data.Value == "bbb" {
+			foundB = true
+		}
 	}
 
-	// Test GetItemCount after expiration
-	itemCount = testCache.GetItemCount()
-	if itemCount != 0 {
-		t.Errorf("GetItemCount should return 0 after expiration, but got %d", itemCount)
+	if !foundA || !foundB {
+		t.Fatalf("Missing expected values, got %v", values)
 	}
 }