Refactoring & bug fixes (#39)

Author: man90es

README.md

@@ -23,27 +23,35 @@ If you host the API yourself, either via Docker or natively, you can control som
 
 Available flags:
 - `-cachettl`
-	- Allows to specify cache TTL in minutes
+	- Specifies cache TTL in minutes
 	- Type: unsigned integer
 	- Default value: `180`
 - `-maintenancettl`
-	- Allows to limit how frequently scraper can check for maintenance end in minutes
+	- Limits how frequently scraper can check for maintenance end in minutes
+	- Type: unsigned integer
+	- Default value: `5`
+- `-maxtasksperclient`
+	- Limits the number of concurrent scraping tasks that can be executed per client
 	- Type: unsigned integer
 	- Default value: `5`
 - `-port`
-	- Allows to specify API server's port
+	- Specifies API server's port
 	- Type: unsigned integer
 	- Default value: `8001`
 	- Also available as `PORT` environment variable (doesn't work in Docker)
 - `-proxy`
-	- Allows to specify a list of proxies to make requests to BDO servers through
+	- Specifies a list of proxies to make requests to BDO servers through
 	- Type: string, space-separated list of IP addresses or URLs
 	- Default value: none, requests are made directly
 	- Also available as `PROXY` environment variable
 - `-ratelimit`
 	- Sets the maximum number of requests per minute per IP address
 	- Type: unsigned integer
 	- Default value: 512
+- `taskretries`
+	- Specifies the number of retries for a scraping task
+	- Type: unsigned integer
+	- Default value: `3`
 - `-verbose`
 	- Allows to put the app into verbose mode and print out additional logs to stdout
 	- Default value: none, no additional output is produced

cache/cache.go

@@ -1,16 +1,14 @@
 package cache
 
 import (
-	"net/http"
 	"strings"
-	"sync"
 	"time"
 
 	goCache "github.com/patrickmn/go-cache"
+	"github.com/spf13/viper"
 	messagebus "github.com/vardius/message-bus"
 	"golang.org/x/exp/maps"
 
-	"bdo-rest-api/config"
 	"bdo-rest-api/models"
 	"bdo-rest-api/utils"
 )
@@ -31,7 +29,7 @@ func joinKeys(keys []string) string {
 }
 
 func newCache[T any]() *cache[T] {
-	cacheTTL := config.GetCacheTTL()
+	cacheTTL := viper.GetDuration("cachettl")
 
 	return &cache[T]{
 		Bus:           messagebus.New(100), // Idk what buffer size is optimal
@@ -40,32 +38,21 @@ func newCache[T any]() *cache[T] {
 }
 
 func (c *cache[T]) AddRecord(keys []string, data T, status int, taskId string) (date string, expires string) {
-	ttl := config.GetCacheTTL()
+	cacheTTL := viper.GetDuration("cachettl")
 	entry := CacheEntry[T]{
 		Data:   data,
 		Date:   time.Now(),
 		Status: status,
 	}
 
-	c.internalCache.Add(joinKeys(keys), entry, ttl)
+	c.internalCache.Add(joinKeys(keys), entry, cacheTTL)
 	c.Bus.Publish(taskId, entry)
 
-	return utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(ttl))
-}
-
-func (c *cache[T]) SignalBypassCache(status int, taskId string) {
-	var data T
-	entry := CacheEntry[T]{
-		Data:   data,
-		Date:   time.Now(),
-		Status: status,
-	}
-
-	c.Bus.Publish(taskId, entry)
+	return utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(cacheTTL))
 }
 
 func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, expires string, found bool) {
-	cacheTTL := config.GetCacheTTL()
+	cacheTTL := viper.GetDuration("cachettl")
 	anyEntry, found := c.internalCache.Get(joinKeys(keys))
 
 	if !found {
@@ -77,30 +64,6 @@ func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, ex
 	return entry.Data, entry.Status, utils.FormatDateForHeaders(entry.Date), utils.FormatDateForHeaders(entry.Date.Add(cacheTTL)), found
 }
 
-func (c *cache[T]) WaitForRecord(taskId string) (data T, status int, date string, expires string) {
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	c.Bus.Subscribe(taskId, func(entry CacheEntry[T]) {
-		data = entry.Data
-		status = entry.Status
-		date = utils.FormatDateForHeaders(entry.Date)
-
-		if entry.Status == http.StatusInternalServerError {
-			expires = date
-		} else if entry.Status == http.StatusServiceUnavailable {
-			expires = utils.FormatDateForHeaders(entry.Date.Add(config.GetMaintenanceStatusTTL()))
-		} else {
-			expires = utils.FormatDateForHeaders(entry.Date.Add(config.GetCacheTTL()))
-		}
-
-		wg.Done()
-	})
-
-	wg.Wait()
-	return
-}
-
 func (c *cache[T]) GetItemCount() int {
 	return c.internalCache.ItemCount()
 }

cache/cache_test.go

@@ -4,11 +4,11 @@ import (
 	"testing"
 	"time"
 
-	"bdo-rest-api/config"
+	"github.com/spf13/viper"
 )
 
 func init() {
-	config.SetCacheTTL(time.Second)
+	viper.Set("cachettl", time.Second)
 }
 
 func TestCache(t *testing.T) {

config/config.go

@@ -1,26 +1,32 @@
 # Broken Stuff
-BDO website, where all the data is taken from, has a number of bugs, which affects by this scraper by design and, sadly, not much can be done about it.
+BDO website, where all the data is taken from, has a number of bugs, which affects by this API by design and, sadly, not much can be done about it.
 
 ## List of known bugs
-This is a list of bugs that either used to occur or still occur that I am aware of:
+This is a list of bugs that either used to occur that I'm aware of. They may be still occuring or be fixed.
 
 ### 🐞 Data is not updated immediately after it is updated in game
-This is a common problem. The website's lag is around a few hours, and you can only wait. This API introduces additional lag that depends on the cache TTL parameter.
+The website's lag is around a few hours, and you can only wait. This API introduces additional lag that depends on the cache TTL parameter.
 
 ### 🐞 Members who left the guild remain on the guild members' list for some time
-This is a common problem. I believe maintenances remove "ghost members" from guilds. If you don't feel like waiting, request profiles of those players. Guild membership status in player profiles is more reliable, unless it's set to private.
+I believe maintenances remove "ghost members" from guilds. If you don't feel like waiting, request profiles of those players. Guild membership status in player profiles is more reliable, unless it's set to private.
 
 ### 🐞 Profile of a different guild is returned instead of the one requested
-This is an uncommon problem. Always check if the guild profile in response has the same name you specified. Also see next bug.
+Always check if the guild profile in response has the same name you specified. Also see next bug.
 
 ### 🐞 Guild profile returned as "Not Found" although the guild exists in the game
-This is an uncommon problem. You can get some information like creation date, guild master's name and population by searching for guild instead of requesting its profile. It's not much, but better than nothing.
+You can get some information like creation date, guild master's name and population by searching for guild instead of requesting its profile. It's not much, but better than nothing.
 
 ### 🐞 Player profile returned as "Not Found" although that player exists in the game
-This is an uncommon problem. There are no known workarounds. [See issue #5](https://github.com/man90es/BDO-REST-API/issues/5).
+There are no known workarounds. [See issue #5](https://github.com/man90es/BDO-REST-API/issues/5).
 
 ### 🐞 Players whose family name is longer than 16 characters aren't searchable
-This is a rare problem. Family names longer than 16 characters aren't officially allowed in BDO, but there may be a bug that allows players to take them. [See issue #13](https://github.com/man90es/BDO-REST-API/issues/13). This API won't support them unless there will be many reports of players with long family names.
+Family names longer than 16 characters aren't officially allowed in BDO, but there may be a bug that allows players to take them. [See issue #13](https://github.com/man90es/BDO-REST-API/issues/13). This API won't support them unless there will be many reports of players with long family names.
+
+### 🐞 Some guild members are missing when requesting a guild profile
+BDO website hides some guild members on its website based on unknown conditions. Supposedly, all hidden members are alt accounts with zero progress on the account.
+
+### 🐞 The number of guild members when searching for a guild doesn't match the number of guild members when requesting the guild profile
+See the previous bug on the list. The number when searching is the correct one.
 
 ## Contribute to this list
 If you found a bug on the original BDO website that affects this API and is not listed in this file, you can contribute by either: