chore: improvements on all tokens list

Instead of using all tokens (atm there are ~15k unique tokens that the app operates with) in all parts of the app, a new
token list, containing only tokens of interest, is being used. There is also a new list containing all tokens introduced that
will be used in places where we really need all tokens, like drop-down list for Swap.

Author: saledjenic

src/app/modules/main/wallet_section/accounts/module.nim

@@ -100,6 +100,8 @@ method load*(self: Module) =
 
   self.events.on(SIGNAL_TOKENS_MARKET_VALUES_UPDATED) do(e:Args):
     self.refreshAllWalletAccountsBalances()
+  self.events.on(SIGNAL_TOKENS_PRICES_UPDATED) do(e:Args):
+    self.refreshAllWalletAccountsBalances()
   self.events.on(SIGNAL_CURRENCY_FORMATS_UPDATED) do(e:Args):
     self.refreshAllWalletAccountsBalances()
   self.events.on(SIGNAL_WALLET_ACCOUNT_TOKENS_REBUILT) do(e:Args):

src/app/modules/main/wallet_section/all_tokens/controller.nim

@@ -70,8 +70,14 @@ proc getHistoricalDataForToken*(self: Controller, tokenKey: string, currency: st
 proc getAllTokenLists*(self: Controller): var seq[TokenListItem] =
   return self.tokenService.getAllTokenLists()
 
-proc getAllTokenGroups*(self: Controller): var seq[TokenGroupItem] =
-  return self.tokenService.getAllTokenGroups()
+proc buildGroupsForChain*(self: Controller, chainId: int) =
+  self.tokenService.buildGroupsForChain(chainId)
+
+proc getGroupsForChain*(self: Controller): var seq[TokenGroupItem] =
+  return self.tokenService.getGroupsForChain()
+
+proc getGroupsOfInterest*(self: Controller): var seq[TokenGroupItem] =
+  return self.tokenService.getGroupsOfInterest()
 
 proc getTokenDetails*(self: Controller, tokenKey: string): TokenDetailsItem =
   return self.tokenService.getTokenDetails(tokenKey)

src/app/modules/main/wallet_section/all_tokens/io_interface.nim

@@ -60,6 +60,9 @@ method getTokensModelDataSource*(self: AccessInterface): TokensModelDataSource {
 method getTokenGroupsModelDataSource*(self: AccessInterface): TokenGroupsModelDataSource {.base.} =
   raise newException(ValueError, "No implementation available")
 
+method getTokenGroupsForChainModelDataSource*(self: AccessInterface): TokenGroupsModelDataSource {.base.} =
+  raise newException(ValueError, "No implementation available")
+
 method getTokenMarketValuesDataSource*(self: AccessInterface): TokenMarketValuesDataSource {.base.} =
   raise newException(ValueError, "No implementation available")
 
@@ -75,6 +78,9 @@ method getTokenPreferencesJson*(self: AccessInterface): string {.base.} =
 method viewDidLoad*(self: AccessInterface) {.base.} =
   raise newException(ValueError, "No implementation available")
 
+method buildGroupsForChain*(self: AccessInterface, chainId: int) {.base.} =
+  raise newException(ValueError, "No implementation available")
+
 method filterChanged*(self: AccessInterface, addresses: seq[string]) {.base.} =
   raise newException(ValueError, "No implementation available")
 

src/app/modules/main/wallet_section/all_tokens/module.nim

@@ -53,8 +53,6 @@ method load*(self: Module) =
   self.events.on(SIGNAL_TOKENS_LIST_UPDATED) do(e: Args):
     self.view.modelsUpdated()
     self.view.emitTokenListUpdatedAtSignal()
-  self.events.on(SIGNAL_TOKENS_DETAILS_ABOUT_TO_BE_UPDATED) do(e: Args):
-    self.view.tokensDetailsAboutToUpdate()
   self.events.on(SIGNAL_TOKENS_DETAILS_UPDATED) do(e: Args):
     self.view.tokensDetailsUpdated()
   self.events.on(SIGNAL_TOKENS_MARKET_VALUES_ABOUT_TO_BE_UPDATED) do(e: Args):
@@ -103,7 +101,17 @@ method getTokenListsModelDataSource*(self: Module): TokenListsModelDataSource =
 
 method getTokenGroupsModelDataSource*(self: Module): TokenGroupsModelDataSource =
   return (
-    getAllTokenGroups: proc(): var seq[TokenGroupItem] = self.controller.getAllTokenGroups(),
+    getAllTokenGroups: proc(): var seq[TokenGroupItem] = self.controller.getGroupsOfInterest(),
+    getTokenDetails: proc(tokenKey: string): TokenDetailsItem = self.controller.getTokenDetails(tokenKey),
+    getTokenPreferences: proc(groupKey: string): TokenPreferencesItem = self.controller.getTokenPreferences(groupKey),
+    getCommunityTokenDescription: proc(chainId: int, address: string): string = self.controller.getCommunityTokenDescription(chainId, address),
+    getTokensDetailsLoading: proc(): bool = self.controller.getTokensDetailsLoading(),
+    getTokensMarketValuesLoading: proc(): bool = self.controller.getTokensMarketValuesLoading(),
+  )
+
+method getTokenGroupsForChainModelDataSource*(self: Module): TokenGroupsModelDataSource =
+  return (
+    getAllTokenGroups: proc(): var seq[TokenGroupItem] = self.controller.getGroupsForChain(),
     getTokenDetails: proc(tokenKey: string): TokenDetailsItem = self.controller.getTokenDetails(tokenKey),
     getTokenPreferences: proc(groupKey: string): TokenPreferencesItem = self.controller.getTokenPreferences(groupKey),
     getCommunityTokenDescription: proc(chainId: int, address: string): string = self.controller.getCommunityTokenDescription(chainId, address),
@@ -119,6 +127,9 @@ method getTokenMarketValuesDataSource*(self: Module): TokenMarketValuesDataSourc
     getTokensMarketValuesLoading: proc(): bool = self.controller.getTokensMarketValuesLoading(),
   )
 
+method buildGroupsForChain*(self: Module, chainId: int) =
+  self.controller.buildGroupsForChain(chainId)
+
 method filterChanged*(self: Module, addresses: seq[string]) =
   if addresses == self.addresses:
       return

src/app/modules/main/wallet_section/all_tokens/token_groups_model.nim

@@ -30,18 +30,21 @@ QtObject:
     tokensModel: TokensModel
     marketValuesDelegate: io_interface.TokenMarketValuesDataSource
     tokenMarketDetails: seq[MarketDetailsItem]
+    allowEmptyMarketDetails: bool
 
   proc setup(self: TokenGroupsModel)
   proc delete(self: TokenGroupsModel)
   proc newTokenGroupsModel*(
     delegate: io_interface.TokenGroupsModelDataSource,
-    marketValuesDelegate: io_interface.TokenMarketValuesDataSource
+    marketValuesDelegate: io_interface.TokenMarketValuesDataSource,
+    allowEmptyMarketDetails: bool
     ): TokenGroupsModel =
     new(result, delete)
     result.setup
     result.delegate = delegate
     result.marketValuesDelegate = marketValuesDelegate
     result.tokenMarketDetails = @[]
+    result.allowEmptyMarketDetails = allowEmptyMarketDetails
 
   method rowCount(self: TokenGroupsModel, index: QModelIndex = nil): int =
     return self.delegate.getAllTokenGroups().len
@@ -81,7 +84,7 @@ QtObject:
     if not index.isValid:
       return
     if index.row < 0 or index.row >= self.delegate.getAllTokenGroups().len or
-      index.row >= self.tokenMarketDetails.len:
+      (not self.allowEmptyMarketDetails and index.row >= self.tokenMarketDetails.len):
       return
 
     let item = self.delegate.getAllTokenGroups()[index.row]
@@ -122,6 +125,8 @@ QtObject:
         let tokenKey = item.tokens[0].key
         return newQVariant(self.delegate.getTokenDetails(tokenKey).description)
       of ModelRole.MarketDetails:
+        if self.allowEmptyMarketDetails:
+          return newQVariant("")
         return newQVariant(self.tokenMarketDetails[index.row])
       of ModelRole.DetailsLoading:
         return newQVariant(self.delegate.getTokensDetailsLoading())
@@ -134,6 +139,11 @@ QtObject:
 
   proc modelsUpdated*(self: TokenGroupsModel) =
     self.beginResetModel()
+    defer: self.endResetModel()
+
+    if self.allowEmptyMarketDetails:
+      return
+
     self.tokenMarketDetails = @[]
     let tokensList = self.delegate.getAllTokenGroups()
     let currencyFormat = self.marketValuesDelegate.getCurrentCurrencyFormat()
@@ -145,9 +155,10 @@ QtObject:
         self.marketValuesDelegate.getMarketValuesForToken(tokenKey),
         currencyFormat)
       self.tokenMarketDetails.add(item)
-    self.endResetModel()
 
   proc tokensMarketValuesUpdated*(self: TokenGroupsModel) =
+    if self.allowEmptyMarketDetails:
+      return
     if not self.delegate.getTokensMarketValuesLoading():
       for marketDetails in self.tokenMarketDetails:
         marketDetails.updateTokenPrice(self.marketValuesDelegate.getPriceForToken(marketDetails.tokenKey))
@@ -162,15 +173,6 @@ QtObject:
       defer: lastindex.delete
       self.dataChanged(index, lastindex, @[ModelRole.MarketDetailsLoading.int])
 
-  proc tokensDetailsAboutToUpdate*(self: TokenGroupsModel) =
-    let tokenGroupsListLength = self.delegate.getAllTokenGroups().len
-    if tokenGroupsListLength > 0:
-      let index = self.createIndex(0, 0, nil)
-      let lastindex = self.createIndex(tokenGroupsListLength-1, 0, nil)
-      defer: index.delete
-      defer: lastindex.delete
-      self.dataChanged(index, lastindex, @[ModelRole.Description.int, ModelRole.WebsiteUrl.int, ModelRole.DetailsLoading.int])
-
   proc tokensDetailsUpdated*(self: TokenGroupsModel) =
     let tokenGroupsListLength = self.delegate.getAllTokenGroups().len
     if tokenGroupsListLength > 0:
@@ -181,6 +183,8 @@ QtObject:
       self.dataChanged(index, lastindex, @[ModelRole.Description.int, ModelRole.WebsiteUrl.int, ModelRole.DetailsLoading.int])
 
   proc currencyFormatsUpdated*(self: TokenGroupsModel) =
+    if self.allowEmptyMarketDetails:
+      return
     let currencyFormat = self.marketValuesDelegate.getCurrentCurrencyFormat()
     for marketDetails in self.tokenMarketDetails:
         marketDetails.updateCurrencyFormat(currencyFormat)

src/app/modules/main/wallet_section/all_tokens/view.nim

@@ -9,7 +9,8 @@ QtObject:
       marketHistoryIsLoading: bool
 
       tokenListsModel: TokenListsModel
-      tokenGroupsModel: TokenGroupsModel
+      tokenGroupsModel: TokenGroupsModel # refers to tokens of interest for active networks mode
+      tokenGroupsForChainModel: TokenGroupsModel # refers to all tokens for a specific chain
 
   ## Forward declaration
   proc modelsUpdated*(self: View)
@@ -23,12 +24,22 @@ QtObject:
     result.tokenListsModel = newTokenListsModel(delegate.getTokenListsModelDataSource())
     result.tokenGroupsModel = newTokenGroupsModel(
       delegate.getTokenGroupsModelDataSource(),
-      delegate.getTokenMarketValuesDataSource())
+      delegate.getTokenMarketValuesDataSource(),
+      allowEmptyMarketDetails = false)
+    result.tokenGroupsForChainModel = newTokenGroupsModel(
+      delegate.getTokenGroupsForChainModelDataSource(),
+      delegate.getTokenMarketValuesDataSource(),
+      allowEmptyMarketDetails = true) # We allow empty market details for the tokenGroupsForChainModel, because it is
+      # used for the swap input panel where market details are not needed, also updating it would be a performance bottleneck.
 
   proc load*(self: View) =
     self.modelsUpdated()
     self.delegate.viewDidLoad()
 
+  proc buildGroupsForChain*(self: View, chainId: int) {.slot.} =
+    self.delegate.buildGroupsForChain(chainId)
+    self.tokenGroupsForChainModel.modelsUpdated()
+
   proc marketHistoryIsLoadingChanged*(self: View) {.signal.}
   proc getMarketHistoryIsLoading(self: View): QVariant {.slot.} =
     return newQVariant(self.marketHistoryIsLoading)
@@ -67,6 +78,13 @@ QtObject:
     read = getTokenListsModel
     notify = tokenListsModelChanged
 
+  proc tokenGroupsForChainModelChanged*(self: View) {.signal.}
+  proc getTokenGroupsForChainModel(self: View): QVariant {.slot.} =
+    return newQVariant(self.tokenGroupsForChainModel)
+  QtProperty[QVariant] tokenGroupsForChainModel:
+    read = getTokenGroupsForChainModel
+    notify = tokenGroupsForChainModelChanged
+
   proc tokenGroupsModelChanged*(self: View) {.signal.}
   proc getTokenGroupsModel(self: View): QVariant {.slot.} =
     return newQVariant(self.tokenGroupsModel)
@@ -77,16 +95,14 @@ QtObject:
   proc modelsUpdated*(self: View) =
     self.tokenListsModel.modelsUpdated()
     self.tokenGroupsModel.modelsUpdated()
+    self.tokenGroupsForChainModel.modelsUpdated()
 
   proc tokensMarketValuesUpdated*(self: View) =
     self.tokenGroupsModel.tokensMarketValuesUpdated()
 
   proc tokensMarketValuesAboutToUpdate*(self: View) =
     self.tokenGroupsModel.tokensMarketValuesAboutToUpdate()
 
-  proc tokensDetailsAboutToUpdate*(self: View) =
-    self.tokenGroupsModel.tokensDetailsAboutToUpdate()
-
   proc tokensDetailsUpdated*(self: View) =
     self.tokenGroupsModel.tokensDetailsUpdated()
 

src/app/modules/main/wallet_section/assets/controller.nim

@@ -29,9 +29,6 @@ proc newController*(
 proc delete*(self: Controller) =
   discard
 
-proc buildAllTokens*(self: Controller, addresses: seq[string]) =
-  self.walletAccountService.buildAllTokens(addresses, forceRefresh = false)
-
 proc init*(self: Controller) =
   let walletAddresses = self.walletAccountService.getWalletAddresses()
   discard

src/app/modules/main/wallet_section/assets/module.nim

@@ -75,4 +75,4 @@ method getGroupedAccountAssetsDataSource*(self: Module): GroupedAccountAssetsDat
   )
 
 method filterChanged*(self: Module, addresses: seq[string], chainIds: seq[int]) =
-  self.controller.buildAllTokens(addresses)
+  discard

src/app_service/service/token/service.nim

@@ -42,9 +42,10 @@ QtObject:
     # local storage, fulfilled by need, empty at the start
     tokensForBridgingViaHop: Table[string, bool] # [tokenKey, bool]
     # local storage
-    allTokensByKey: Table[string, TokenItem] # [tokenKey, TokenItem]
-    allGroupsByKey: Table[string, TokenGroupItem] # [tokenGroupKey, TokenGroupItem]
-    allGroups: seq[TokenGroupItem]
+    tokensOfInterestByKey: Table[string, TokenItem] # [tokenKey, TokenItem]
+    groupsOfInterestByKey: Table[string, TokenGroupItem] # [tokenGroupKey, TokenGroupItem]
+    groupsOfInterest: seq[TokenGroupItem] # refers to groups for tokens of interest
+    groupsForChain: seq[TokenGroupItem] # refers to groups for a specific chain
     allTokenLists: seq[TokenListItem] # TODO: remove this, fetch async when needed, don't store it
     tokenDetailsTable: Table[string, TokenDetailsItem] # [tokenKey, TokenDetailsItem]
     tokenMarketValuesTable: Table[string, TokenMarketValuesItem] # [tokenKey, TokenMarketValuesItem]
@@ -86,8 +87,8 @@ QtObject:
     result.networkService = networkService
     result.settingsService = settingsService
 
-    result.allTokensByKey = initTable[string, TokenItem]()
-    result.allGroupsByKey = initTable[string, TokenGroupItem]()
+    result.tokensOfInterestByKey = initTable[string, TokenItem]()
+    result.groupsOfInterestByKey = initTable[string, TokenGroupItem]()
     result.tokenDetailsTable = initTable[string, TokenDetailsItem]()
     result.tokenMarketValuesTable = initTable[string, TokenMarketValuesItem]()
     result.tokenPriceTable = initTable[string, float64]()

src/app_service/service/token/service_main.nim

@@ -8,22 +8,24 @@ proc rebuildMarketData*(self: Service) =
 proc refreshTokens(self: Service) =
   self.allTokenLists = getAllTokenLists()
 
-  self.allTokensByKey.clear()
-  self.allGroupsByKey.clear()
-  let allTokens = getTokensForActiveNetworksMode()
-  for token in allTokens:
+  # build groups of interest
+  self.tokensOfInterestByKey.clear()
+  self.groupsOfInterestByKey.clear()
+
+  var tokens = getTokensOfInterestForActiveNetworksMode()
+  for token in tokens:
     let groupKey = token.groupKey
-    self.allTokensByKey[token.key] = token
-    if not self.allGroupsByKey.hasKey(groupKey):
-      self.allGroupsByKey[groupKey] = TokenGroupItem(
+    self.tokensOfInterestByKey[token.key] = token
+    if not self.groupsOfInterestByKey.hasKey(groupKey):
+      self.groupsOfInterestByKey[groupKey] = TokenGroupItem(
         key: groupKey,
         name: token.name,
         symbol: token.symbol,
         decimals: token.decimals,
         logoUri: token.logoUri
       )
-    self.allGroupsByKey[groupKey].addToken(token)
-  self.allGroups = toSeq(self.allGroupsByKey.values)
+    self.groupsOfInterestByKey[groupKey].addToken(token)
+  self.groupsOfInterest = toSeq(self.groupsOfInterestByKey.values)
 
   self.rebuildMarketData()
   self.fetchTokensDetails() # TODO: if the only place where we can see these details is account's details page, we should fetch this on demand, no need to have local cache
@@ -37,7 +39,7 @@ proc init*(self: Service) =
     # this is the delay before the first call to the callback, this is an action that doesn't need to be called immediately, but it's pretty expensive in terms of time/performances
     # for example `wallet-tick-reload` event is emitted for every single chain-account pair, and at the app start can be more such signals received from the statusgo side if the balance have changed.
     # Means it the app contains more accounts the likelihood of having more `wallet-tick-reload` signals is higher, so we need to delay the rebuildMarketData call to avoid unnecessary calls.
-    delayMs = 1500,
+    delayMs = 1000,
     checkIntervalMs = 500)
   self.rebuildMarketDataDebouncer.registerCall0(callback = proc() = self.rebuildMarketDataInternal())
 
@@ -61,8 +63,30 @@ proc init*(self: Service) =
 proc getCurrency*(self: Service): string =
   return self.settingsService.getCurrency()
 
-proc getAllTokenGroups*(self: Service): var seq[TokenGroupItem] =
-  return self.allGroups
+proc getGroupsOfInterest*(self: Service): var seq[TokenGroupItem] =
+  return self.groupsOfInterest
+
+proc buildGroupsForChain*(self: Service, chainId: int) =
+  if chainId <= 0:
+    warn "invalid chainId", chainId = chainId
+    return
+  var allTokens = getTokensByChain(chainId)
+  var groupsByTokenKey = initTable[string, TokenGroupItem]()
+  for token in allTokens:
+    let groupKey = token.groupKey
+    if not groupsByTokenKey.hasKey(groupKey):
+      groupsByTokenKey[groupKey] = TokenGroupItem(
+        key: groupKey,
+        name: token.name,
+        symbol: token.symbol,
+        decimals: token.decimals,
+        logoUri: token.logoUri
+      )
+    groupsByTokenKey[groupKey].addToken(token)
+  self.groupsForChain = toSeq(groupsByTokenKey.values)
+
+proc getGroupsForChain*(self: Service): var seq[TokenGroupItem] =
+  return self.groupsForChain
 
 proc getAllTokenLists*(self: Service): var seq[TokenListItem] =
   return self.allTokenLists
@@ -76,22 +100,22 @@ proc getAllCommunityTokens*(self: Service): var seq[TokenItem] =
 proc getTokenByKey*(self: Service, key: string): TokenItem =
   if key.isEmptyOrWhitespace or not key.toLower.contains("-0x"):
     return nil
-  if self.allTokensByKey.hasKey(key):
-    return self.allTokensByKey[key]
+  if self.tokensOfInterestByKey.hasKey(key):
+    return self.tokensOfInterestByKey[key]
   let tokens = getTokensByKeys(@[key])
   if tokens.len > 0:
-    self.allTokensByKey[key] = tokens[0]
-    return self.allTokensByKey[key]
+    self.tokensOfInterestByKey[key] = tokens[0]
+    return self.tokensOfInterestByKey[key]
   return nil
 
 proc getTokenByChainAddress*(self: Service, chainId: int, address: string): TokenItem =
   let key = common_utils.createTokenKey(chainId, address)
   return self.getTokenByKey(key)
 
 proc getTokensByGroupKey*(self: Service, groupKey: string): seq[TokenItem] =
-  if not self.allGroupsByKey.hasKey(groupKey):
+  if not self.groupsOfInterestByKey.hasKey(groupKey):
     return @[]
-  return self.allGroupsByKey[groupKey].tokens
+  return self.groupsOfInterestByKey[groupKey].tokens
 
 proc getTokenByGroupKeyAndChainId*(self: Service, groupKey: string, chainId: int): TokenItem =
   let tokens = self.getTokensByGroupKey(groupKey)
@@ -146,9 +170,9 @@ proc updateTokenPrices*(self: Service, updatedPrices: Table[string, float64]) =
     self.events.emit(SIGNAL_TOKENS_PRICES_UPDATED, Args())
 
 proc addNewCommunityToken*(self: Service, token: TokenItem) =
-  if self.allGroupsByKey.hasKey(token.groupKey):
-    let tokens = self.allGroupsByKey[token.groupKey].tokens
+  if self.groupsOfInterestByKey.hasKey(token.groupKey):
+    let tokens = self.groupsOfInterestByKey[token.groupKey].tokens
     for t in tokens:
       if t.key == token.key:
         return
-  self.refreshTokens()
+  self.refreshTokens()