Skip to content

Commit d6b1e28

Browse files
bzneinbznein
committed
backend: replace ratelimiter with x/time/rate.
Removes the ratelimiter package and replaces its usages with x/time/rate.
1 parent 16a748c commit d6b1e28

File tree

12 files changed

+647
-203
lines changed

12 files changed

+647
-203
lines changed

‎backend/backend.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ import (
5151
"github.com/BitBoxSwiss/bitbox-wallet-app/util/logging"
5252
"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable"
5353
"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable/action"
54-
"github.com/BitBoxSwiss/bitbox-wallet-app/util/ratelimit"
5554
"github.com/BitBoxSwiss/bitbox-wallet-app/util/socksproxy"
5655
"github.com/btcsuite/btcd/chaincfg"
5756
"github.com/ethereum/go-ethereum/params"
@@ -279,7 +278,7 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe
279278
backend.notifier = notifier
280279
backend.socksProxy = backendProxy
281280
backend.httpClient = hclient
282-
backend.etherScanHTTPClient = ratelimit.FromTransport(hclient.Transport, etherscan.CallInterval)
281+
backend.etherScanHTTPClient = hclient
283282

284283
ratesCache := filepath.Join(arguments.CacheDirectoryPath(), "exchangerates")
285284
if err := os.MkdirAll(ratesCache, 0700); err != nil {

‎backend/coins/eth/etherscan/etherscan.go

+25-18
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,13 @@ import (
3535
"github.com/ethereum/go-ethereum/common"
3636
"github.com/ethereum/go-ethereum/common/hexutil"
3737
"github.com/ethereum/go-ethereum/core/types"
38+
"golang.org/x/time/rate"
3839
)
3940

40-
// CallInterval is the duration between etherscan requests.
41+
// CallsPerSec is thenumber of etherscanr equests allowed
42+
// per second.
4143
// Etherscan rate limits to one request per 0.2 seconds.
42-
var CallInterval = 260 * time.Millisecond
44+
var CallsPerSec = 3.8
4345

4446
const apiKey = "X3AFAGQT2QCAFTFPIH9VJY88H9PIQ2UWP7"
4547

@@ -50,17 +52,22 @@ const ERC20GasErr = "insufficient funds for gas * price + value"
5052
type EtherScan struct {
5153
url string
5254
httpClient *http.Client
55+
limiter *rate.Limiter
5356
}
5457

5558
// NewEtherScan creates a new instance of EtherScan.
5659
func NewEtherScan(url string, httpClient *http.Client) *EtherScan {
5760
return &EtherScan{
5861
url: url,
5962
httpClient: httpClient,
63+
limiter: rate.NewLimiter(rate.Limit(CallsPerSec), 1),
6064
}
6165
}
6266

63-
func (etherScan *EtherScan) call(params url.Values, result interface{}) error {
67+
func (etherScan *EtherScan) call(ctx context.Context, params url.Values, result interface{}) error {
68+
if err := etherScan.limiter.Wait(ctx); err != nil {
69+
return errp.WithStack(err)
70+
}
6471
params.Set("apikey", apiKey)
6572
response, err := etherScan.httpClient.Get(etherScan.url + "?" + params.Encode())
6673
if err != nil {
@@ -323,7 +330,7 @@ func (etherScan *EtherScan) Transactions(
323330
result := struct {
324331
Result []*Transaction
325332
}{}
326-
if err := etherScan.call(params, &result); err != nil {
333+
if err := etherScan.call(context.TODO(), params, &result); err != nil {
327334
return nil, err
328335
}
329336
isERC20 := erc20Token != nil
@@ -338,7 +345,7 @@ func (etherScan *EtherScan) Transactions(
338345
resultInternal := struct {
339346
Result []*Transaction
340347
}{}
341-
if err := etherScan.call(params, &resultInternal); err != nil {
348+
if err := etherScan.call(context.TODO(), params, &resultInternal); err != nil {
342349
return nil, err
343350
}
344351
var err error
@@ -353,7 +360,7 @@ func (etherScan *EtherScan) Transactions(
353360

354361
// ----- RPC node proxy methods follow
355362

356-
func (etherScan *EtherScan) rpcCall(params url.Values, result interface{}) error {
363+
func (etherScan *EtherScan) rpcCall(ctx context.Context, params url.Values, result interface{}) error {
357364
params.Set("module", "proxy")
358365

359366
var wrapped struct {
@@ -364,7 +371,7 @@ func (etherScan *EtherScan) rpcCall(params url.Values, result interface{}) error
364371
} `json:"error"`
365372
Result *json.RawMessage `json:"result"`
366373
}
367-
if err := etherScan.call(params, &wrapped); err != nil {
374+
if err := etherScan.call(ctx, params, &wrapped); err != nil {
368375
return err
369376
}
370377
if wrapped.Error != nil {
@@ -389,7 +396,7 @@ func (etherScan *EtherScan) TransactionReceiptWithBlockNumber(
389396
params.Set("action", "eth_getTransactionReceipt")
390397
params.Set("txhash", hash.Hex())
391398
var result *rpcclient.RPCTransactionReceipt
392-
if err := etherScan.rpcCall(params, &result); err != nil {
399+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
393400
return nil, err
394401
}
395402
return result, nil
@@ -402,7 +409,7 @@ func (etherScan *EtherScan) TransactionByHash(
402409
params.Set("action", "eth_getTransactionByHash")
403410
params.Set("txhash", hash.Hex())
404411
var result rpcclient.RPCTransaction
405-
if err := etherScan.rpcCall(params, &result); err != nil {
412+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
406413
return nil, false, err
407414
}
408415
return &result.Transaction, result.BlockNumber == nil, nil
@@ -415,7 +422,7 @@ func (etherScan *EtherScan) BlockNumber(ctx context.Context) (*big.Int, error) {
415422
params.Set("tag", "latest")
416423
params.Set("boolean", "false")
417424
var header *types.Header
418-
if err := etherScan.rpcCall(params, &header); err != nil {
425+
if err := etherScan.rpcCall(ctx, params, &header); err != nil {
419426
return nil, err
420427
}
421428
return header.Number, nil
@@ -434,7 +441,7 @@ func (etherScan *EtherScan) Balance(ctx context.Context, account common.Address)
434441
params.Set("action", "balance")
435442
params.Set("address", account.Hex())
436443
params.Set("tag", "latest")
437-
if err := etherScan.call(params, &result); err != nil {
444+
if err := etherScan.call(ctx, params, &result); err != nil {
438445
return nil, err
439446
}
440447
if result.Status != "1" {
@@ -461,7 +468,7 @@ func (etherScan *EtherScan) ERC20Balance(account common.Address, erc20Token *erc
461468
params.Set("address", account.Hex())
462469
params.Set("contractaddress", erc20Token.ContractAddress().Hex())
463470
params.Set("tag", "latest")
464-
if err := etherScan.call(params, &result); err != nil {
471+
if err := etherScan.call(context.TODO(), params, &result); err != nil {
465472
return nil, err
466473
}
467474
if result.Status != "1" {
@@ -485,7 +492,7 @@ func (etherScan *EtherScan) CallContract(ctx context.Context, msg ethereum.CallM
485492
panic("not implemented")
486493
}
487494
var result hexutil.Bytes
488-
if err := etherScan.rpcCall(params, &result); err != nil {
495+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
489496
return nil, err
490497
}
491498
return result, nil
@@ -515,7 +522,7 @@ func (etherScan *EtherScan) EstimateGas(ctx context.Context, msg ethereum.CallMs
515522
callMsgParams(&params, msg)
516523

517524
var result hexutil.Uint64
518-
if err := etherScan.rpcCall(params, &result); err != nil {
525+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
519526
return 0, err
520527
}
521528
return uint64(result), nil
@@ -528,7 +535,7 @@ func (etherScan *EtherScan) PendingNonceAt(ctx context.Context, account common.A
528535
params.Set("address", account.Hex())
529536
params.Set("tag", "pending")
530537
var result hexutil.Uint64
531-
if err := etherScan.rpcCall(params, &result); err != nil {
538+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
532539
return 0, err
533540
}
534541
return uint64(result), nil
@@ -544,15 +551,15 @@ func (etherScan *EtherScan) SendTransaction(ctx context.Context, tx *types.Trans
544551
params := url.Values{}
545552
params.Set("action", "eth_sendRawTransaction")
546553
params.Set("hex", hexutil.Encode(encodedTx))
547-
return etherScan.rpcCall(params, nil)
554+
return etherScan.rpcCall(ctx, params, nil)
548555
}
549556

550557
// SuggestGasPrice implements rpc.Interface.
551558
func (etherScan *EtherScan) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
552559
params := url.Values{}
553560
params.Set("action", "eth_gasPrice")
554561
var result hexutil.Big
555-
if err := etherScan.rpcCall(params, &result); err != nil {
562+
if err := etherScan.rpcCall(ctx, params, &result); err != nil {
556563
return nil, err
557564
}
558565
return (*big.Int)(&result), nil
@@ -581,7 +588,7 @@ func (etherScan *EtherScan) FeeTargets(ctx context.Context) ([]*ethtypes.FeeTarg
581588
params := url.Values{}
582589
params.Set("module", "gastracker")
583590
params.Set("action", "gasoracle")
584-
if err := etherScan.call(params, &result); err != nil {
591+
if err := etherScan.call(ctx, params, &result); err != nil {
585592
return nil, err
586593
}
587594
// Convert string fields to int64

‎backend/rates/gecko.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,21 @@ const (
1616
maxGeckoRange = 364 * 24 * time.Hour
1717
)
1818

19-
// apiRateLimit specifies the minimal interval between equally spaced API calls
19+
// apiRateLimit specifies the maximum number of API calls per second
2020
// to one of the supported exchange rates providers.
21-
func apiRateLimit(baseURL string) time.Duration {
21+
func apiRateLimit(baseURL string) float64 {
2222
switch baseURL {
2323
default:
24-
return time.Second // arbitrary; localhost, staging, etc.
24+
return 1 // arbitrary; localhost, staging, etc.
2525
case coingeckoAPIV3:
2626
// API calls. From https://www.coingecko.com/en/api:
2727
// > Generous rate limits with up to 100 requests/minute
2828
// We use slightly lower value.
29-
return 2 * time.Second
29+
return 0.5
3030
case shiftGeckoMirrorAPIV3:
3131
// Avoid zero to prevent unexpected panics like in time.NewTicker
3232
// and leave some room to breathe.
33-
return 10 * time.Millisecond
33+
return 100
3434
}
3535
}
3636

‎backend/rates/history.go

+28-28
Original file line numberDiff line numberDiff line change
@@ -278,38 +278,38 @@ func (updater *RateUpdater) fetchGeckoMarketRange(ctx context.Context, coin, fia
278278
}
279279

280280
// Make the call, abiding the upstream rate limits.
281-
msg := fmt.Sprintf("fetch coingecko coin=%s fiat=%s start=%s", coin, fiat, timeRange.start)
282281
var jsonBody struct{ Prices [][2]float64 } // [timestamp in milliseconds, value]
283-
callErr := updater.geckoLimiter.Call(ctx, msg, func() error {
284-
param := url.Values{
285-
"from": {strconv.FormatInt(timeRange.start.Unix(), 10)},
286-
"to": {strconv.FormatInt(timeRange.end().Unix(), 10)},
287-
"vs_currency": {gfiat},
288-
}
289-
endpoint := fmt.Sprintf("%s/coins/%s/market_chart/range?%s", updater.coingeckoURL, gcoin, param.Encode())
290-
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
291-
if err != nil {
292-
return err
293-
}
282+
if err := updater.geckoLimiter.Wait(ctx); err != nil {
283+
return nil, err
284+
}
294285

295-
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
296-
defer cancel()
297-
res, err := updater.httpClient.Do(req.WithContext(ctx))
298-
if err != nil {
299-
return err
300-
}
301-
defer res.Body.Close() //nolint:errcheck
302-
if res.StatusCode != http.StatusOK {
303-
return fmt.Errorf("fetchGeckoMarketRange: bad response code %d", res.StatusCode)
304-
}
305-
// 1Mb is more than enough for a single response, but make sure initial
306-
// download with empty cache fits here. See maxGeckoRange.
307-
return json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&jsonBody)
308-
})
309-
if callErr != nil {
310-
return nil, callErr
286+
param := url.Values{
287+
"from": {strconv.FormatInt(timeRange.start.Unix(), 10)},
288+
"to": {strconv.FormatInt(timeRange.end().Unix(), 10)},
289+
"vs_currency": {gfiat},
290+
}
291+
endpoint := fmt.Sprintf("%s/coins/%s/market_chart/range?%s", updater.coingeckoURL, gcoin, param.Encode())
292+
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
293+
if err != nil {
294+
return nil, err
295+
}
296+
297+
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
298+
defer cancel()
299+
res, err := updater.httpClient.Do(req.WithContext(ctx))
300+
if err != nil {
301+
return nil, err
302+
}
303+
defer res.Body.Close() //nolint:errcheck
304+
if res.StatusCode != http.StatusOK {
305+
return nil, fmt.Errorf("fetchGeckoMarketRange: bad response code %d", res.StatusCode)
311306
}
312307

308+
// 1Mb is more than enough for a single response, but make sure initial
309+
// download with empty cache fits here. See maxGeckoRange
310+
if err := json.NewDecoder(io.LimitReader(res.Body, 1<<20)).Decode(&jsonBody); err != nil {
311+
return nil, err
312+
}
313313
// Transform the response into a usable result.
314314
rates := make([]exchangeRate, len(jsonBody.Prices))
315315
for i, v := range jsonBody.Prices {

‎backend/rates/rates.go

+38-31
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,12 @@ import (
2727
"sync"
2828
"time"
2929

30+
"golang.org/x/time/rate"
31+
3032
"github.com/BitBoxSwiss/bitbox-wallet-app/util/errp"
3133
"github.com/BitBoxSwiss/bitbox-wallet-app/util/logging"
3234
"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable"
3335
"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable/action"
34-
"github.com/BitBoxSwiss/bitbox-wallet-app/util/ratelimit"
3536
"github.com/sirupsen/logrus"
3637
"go.etcd.io/bbolt"
3738
)
@@ -119,15 +120,15 @@ type RateUpdater struct {
119120
// See https://www.coingecko.com/en/api for details.
120121
coingeckoURL string
121122
// All requests to coingeckoURL are rate-limited using geckoLimiter.
122-
geckoLimiter *ratelimit.LimitedCall
123+
geckoLimiter *rate.Limiter
123124
}
124125

125126
// NewRateUpdater returns a new rates updater.
126127
// The dbdir argument is the location of a historical rates database cache.
127128
// The returned updater can function without a valid database cache but may be
128129
// impacted by rate limits. The database cache is transparent to the updater users.
129130
// To stay within acceptable rate limits defined by CoinGeckoRateLimit, callers can
130-
// use util/ratelimit package.
131+
// use https://pkg.go.dev/golang.org/x/time/rate package.
131132
//
132133
// Both Last and PriceAt of the newly created updater always return zero values
133134
// until data is fetched from the external APIs. To make the updater start fetching data
@@ -155,7 +156,7 @@ func NewRateUpdater(client *http.Client, dbdir string) *RateUpdater {
155156
log: log,
156157
httpClient: client,
157158
coingeckoURL: apiURL,
158-
geckoLimiter: ratelimit.NewLimitedCall(apiRateLimit(apiURL)),
159+
geckoLimiter: rate.NewLimiter(rate.Limit(apiRateLimit(apiURL)), 1),
159160
}
160161
}
161162

@@ -275,36 +276,42 @@ func (updater *RateUpdater) updateLast(ctx context.Context) {
275276
}
276277

277278
var geckoRates map[string]map[string]float64
278-
callErr := updater.geckoLimiter.Call(ctx, "updateLast", func() error {
279-
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
280-
defer cancel()
281-
res, err := updater.httpClient.Do(req.WithContext(ctx))
282-
if err != nil {
283-
return errp.WithStack(err)
284-
}
285-
defer res.Body.Close() //nolint:errcheck
286-
if res.StatusCode != http.StatusOK {
287-
return errp.Newf("bad response code %d", res.StatusCode)
288-
}
289-
const max = 10240
290-
responseBody, err := io.ReadAll(io.LimitReader(res.Body, max+1))
291-
if err != nil {
292-
return errp.WithStack(err)
293-
}
294-
if len(responseBody) > max {
295-
return errp.Newf("rates response too long (> %d bytes)", max)
296-
}
297-
if err := json.Unmarshal(responseBody, &geckoRates); err != nil {
298-
return errp.WithMessage(err,
299-
fmt.Sprintf("could not parse rates response: %s", string(responseBody)))
300-
}
301-
return nil
302-
})
303-
if callErr != nil {
304-
updater.log.WithError(callErr).Errorf("updateLast")
279+
if err := updater.geckoLimiter.Wait(ctx); err != nil {
280+
updater.log.WithError(err).Error("could not wait for rate limiter")
281+
updater.last = nil
282+
return
283+
}
284+
285+
res, err := updater.httpClient.Do(req.WithContext(ctx))
286+
if err != nil {
287+
updater.log.WithError(err).Error("could not make request")
288+
updater.last = nil
289+
return
290+
}
291+
defer res.Body.Close() //nolint:errcheck
292+
if res.StatusCode != http.StatusOK {
293+
updater.log.Errorf("bad response code %d", res.StatusCode)
305294
updater.last = nil
306295
return
307296
}
297+
const max = 10240
298+
responseBody, err := io.ReadAll(io.LimitReader(res.Body, max+1))
299+
if err != nil {
300+
updater.log.WithError(err).Error("could not read response")
301+
updater.last = nil
302+
return
303+
}
304+
if len(responseBody) > max {
305+
updater.last = nil
306+
updater.log.Errorf("rates response too long (> %d bytes)", max)
307+
return
308+
}
309+
if err := json.Unmarshal(responseBody, &geckoRates); err != nil {
310+
updater.last = nil
311+
updater.log.Errorf("could not parse rates response: %s", string(responseBody))
312+
return
313+
}
314+
308315
// Convert the map with coingecko coin/fiat codes to a map of coin/fiat units.
309316
rates := map[string]map[string]float64{}
310317
for coin, val := range geckoRates {

0 commit comments

Comments
 (0)