diff --git a/README.md b/README.md index 4ed352f..f669f67 100644 --- a/README.md +++ b/README.md @@ -52,4 +52,6 @@ SHERLOCK_V2_SHER_CLAIM=0x7289C61C75dCdB8Fe4DF0b937c08c9c40902BDd3 SHERLOCK_V2_PROTOCOL_MANAGER=0x3d0b8A0A10835Ab9b0f0BeB54C5400B8aAcaa1D3 SHERLOCK_V2_CORE_PATH=/home/evert/sherlock/sherlock-v2-core INDEXER_SLEEP_BETWEEN_CALL=0.1 +SENTRY_DSN= +SENTRY_ENVIRONMENT=production ``` diff --git a/alembic/versions/309b0f5de150_additional_apy_and_strategy_balance_.py b/alembic/versions/309b0f5de150_additional_apy_and_strategy_balance_.py new file mode 100644 index 0000000..fbbba74 --- /dev/null +++ b/alembic/versions/309b0f5de150_additional_apy_and_strategy_balance_.py @@ -0,0 +1,39 @@ +"""Additional APY and strategy balance model + +Revision ID: 309b0f5de150 +Revises: 266a0dc816d5 +Create Date: 2022-05-31 17:05:50.534901 + +""" +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "309b0f5de150" +down_revision = "266a0dc816d5" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "strategy_balances", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("address", sa.Text(), nullable=False), + sa.Column("value", sa.NUMERIC(precision=78), nullable=False), + sa.Column("timestamp", postgresql.TIMESTAMP(), nullable=False), + sa.Column("block", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.add_column("indexer_state", sa.Column("additional_apy", sa.Float(), server_default="0", nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("indexer_state", "additional_apy") + op.drop_table("strategy_balances") + # ### end Alembic commands ### diff --git a/app.py b/app.py index af0778f..288766b 100644 --- a/app.py +++ b/app.py @@ -3,6 +3,7 @@ import sys from threading import Thread +import sentry # noqa import settings from flask_app import app from indexer import Indexer diff --git a/indexer.py b/indexer.py index 390fd6b..759694c 100644 --- a/indexer.py +++ b/indexer.py @@ -8,6 +8,7 @@ from sqlalchemy.exc import IntegrityError from web3.constants import ADDRESS_ZERO +import sentry import settings from models import ( FundraisePositions, @@ -21,7 +22,10 @@ StatsAPY, StatsTVC, StatsTVL, + StrategyBalance, ) +from strategies.custom_yields import CUSTOM_YIELDS +from strategies.strategies import Strategies from utils import get_event_logs_in_range, requests_retry_session, time_delta_apy YEAR = Decimal(timedelta(days=365).total_seconds()) @@ -77,6 +81,8 @@ def __init__(self, blocks_per_call=None): self.index_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL, # 268 blocks is roughly every hour on current Ethereum mainnet self.reset_apy_calc: 268, + self.index_strategy_balances: settings.INDEXER_STATS_BLOCKS_PER_CALL, + self.calc_additional_apy: 268 * 6, # 6 hours } # Also get called after listening to events with `end_block` @@ -127,11 +133,41 @@ def calc_factors(self, session, indx, block): indx.block_last_updated = block indx.last_time = datetime.fromtimestamp(timestamp) - # Update APY only if relevant (skip negative APYs generated by payouts). + # Update APY only if relevant: + # - skip negative APYs generated by payouts + # - skip short term, very high APYs, generated by strategies (e.g. a loan is paid back in Maple) # Position balances are still being correctly kept up to date # using the balance factor which accounts for payouts. - if apy > 0.0: - indx.apy = apy + + if apy < 0: + logger.warning("APY %s is being skipped because is negative." % apy) + sentry.report_message( + "APY is being skipped because it is negative!", + "warning", + {"current_apy": float(apy * 100)}, + ) + return + + if apy > 0.15: + logger.warning("APY %s is being skipped because is higher than 15%%." % apy) + sentry.report_message( + "APY is being skipped because it higher than 15%!", + "warning", + {"current_apy": float(apy * 100)}, + ) + return + + if indx.apy != 0 and apy > indx.apy * 2: + logger.warning( + "APY %s is being skipped because it is 2 times higher than the previous APY of %s" % (apy, indx.apy) + ) + sentry.report_message( + "APY is 2 times higher than the previous APY!", + "warning", + {"current_apy": float(apy * 100), "previous_apy": float(indx.apy * 100)}, + ) + + indx.apy = apy def index_apy(self, session, indx, block): """Index current APY. @@ -142,7 +178,7 @@ def index_apy(self, session, indx, block): block: Current block """ timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"]) - apy = indx.apy + apy = indx.apy + indx.additional_apy StatsAPY.insert(session, block, timestamp, apy) @@ -239,6 +275,64 @@ def reset_apy_calc(self, session, indx, block): # Reset factor indx.balance_factor = Decimal(1) + def index_strategy_balances(self, session, indx, block): + """Index each strategy's current balances. + + Args: + session: DB session + indx: Indexer state + block: Block number + """ + timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"]) + + for strategy in Strategies.ALL: + balance = strategy.get_balance(block) + + if balance is not None: + # If strategy is deployed and active + StrategyBalance.insert(session, block, timestamp, strategy.address, balance) + + def calc_additional_apy(self, session, indx, block): + """Compute the additionl APY coming from custom yield strtegies. + (e.g. Maple, TrueFi) + + Args: + session: DB session + indx: Indexer state + block: Block number + """ + timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"]) + + additional_apy = 0.0 + for custom_yield in CUSTOM_YIELDS: + apy = custom_yield.get_apy(block, timestamp) + balance = custom_yield.strategy.get_balance(block) + + logger.info("Strategy %s has balance %s and APY %s" % (custom_yield.strategy, balance, apy)) + + # If strategy is deployed and active and the APY has been successfully fetched + if balance is not None and apy is not None: + TVL = session.query(StatsTVL).order_by(StatsTVL.timestamp.desc()).first() + + # TVL not yet computed. Can happen if the interval for computing the additional APY + # is shorter than the interval for computing the TVL. + if not TVL or TVL.value == 0: + return + + logger.info("Balance is %s and TVL value is %s" % (balance, str(TVL.value))) + + # Compute the additional APY generated by this strategy by multipliying the + # computed APY with the weights of this strategy in the entire TVL + strategy_weight = balance / (TVL.value) + logger.info("Strategy weight %s" % strategy_weight) + weighted_apy = float(strategy_weight) * apy + logger.info("Weghted APY %s" % weighted_apy) + + additional_apy += weighted_apy + + logger.info("Computed additional APY of %s" % additional_apy) + indx.additional_apy = additional_apy + class Transfer: def new(self, session, indx, block, args): if args["to"] == ADDRESS_ZERO: diff --git a/models/__init__.py b/models/__init__.py index b030587..1e6e880 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -9,6 +9,7 @@ from .stats_apy import StatsAPY from .stats_tvc import StatsTVC from .stats_tvl import StatsTVL +from .strategy_balance import StrategyBalance __all__ = [ Base, @@ -24,4 +25,5 @@ StatsTVC, ProtocolCoverage, StatsAPY, + StrategyBalance, ] diff --git a/models/indexer_state.py b/models/indexer_state.py index f3acd40..c8ab153 100644 --- a/models/indexer_state.py +++ b/models/indexer_state.py @@ -15,3 +15,4 @@ class IndexerState(Base): balance_factor = Column(NUMERIC(78, 70), nullable=False, default=1.0) apy = Column(Float, nullable=False, default=0.0) apy_50ms_factor = Column(NUMERIC(78, 70), nullable=False, default=0.0) # TODO: Remove unused column + additional_apy = Column(Float, nullable=False, default=0.0, server_default="0") diff --git a/models/strategy_balance.py b/models/strategy_balance.py new file mode 100644 index 0000000..a72a1f8 --- /dev/null +++ b/models/strategy_balance.py @@ -0,0 +1,37 @@ +import logging + +from sqlalchemy import Column, Integer, Text +from sqlalchemy.dialects.postgresql import NUMERIC, TIMESTAMP + +from models.base import Base + +logger = logging.getLogger(__name__) + + +class StrategyBalance(Base): + __tablename__ = "strategy_balances" + + id = Column(Integer, primary_key=True) + address = Column(Text, nullable=False) + value = Column(NUMERIC(78), nullable=False) + timestamp = Column(TIMESTAMP, nullable=False) + block = Column(Integer, nullable=False) + + @staticmethod + def insert(session, block, timestamp, address, value): + new_balance = StrategyBalance() + new_balance.address = address + new_balance.value = value + new_balance.block = block + new_balance.value = value + new_balance.timestamp = timestamp + + session.add(new_balance) + + def to_dict(self): + return { + "address": self.address, + "value": self.value, + "timestamp": int(self.timestamp.timestamp()), + "block": self.block, + } diff --git a/requirements.txt b/requirements.txt index 580fdbf..0e4f695 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,3 +61,4 @@ flake8==4.0.1 # https://github.com/PyCQA/flake8 flake8-isort==4.1.1 # https://github.com/gforcada/flake8-isort pre-commit==2.15.0 # https://github.com/pre-commit/pre-commit alembic==1.7.7 # https://alembic.sqlalchemy.org/en/latest/ +sentry-sdk[flask]==1.5.12 # https://github.com/getsentry/sentry-python diff --git a/sentry.py b/sentry.py new file mode 100644 index 0000000..6992c72 --- /dev/null +++ b/sentry.py @@ -0,0 +1,45 @@ +import sentry_sdk +from sentry_sdk.integrations.flask import FlaskIntegration + +import settings + +sentry_sdk.init( + dsn=settings.SENTRY_DSN, + environment=settings.SENTRY_ENVIRONMENT, + integrations=[ + FlaskIntegration(), + ], + # Set traces_sample_rate to 1.0 to capture 100% + # of transactions for performance monitoring. + # We recommend adjusting this value in production. + traces_sample_rate=0.01, + # By default the SDK will try to use the SENTRY_RELEASE + # environment variable, or infer a git commit + # SHA as release, however you may want to set + # something more human-readable. + # release="myapp@1.0.0", +) + + +def report_message(message: str, level: str = None, extra={}): + """Capture a message and send it to Sentry + + Available levels are: + - fatal + - critical + - error + - warning + - log + - info + - debug + + Args: + message (str): Message text + extra (dict): Dict of extra items to send with the message + """ + + with sentry_sdk.push_scope() as scope: + for key, value in extra.items(): + scope.set_extra(key, value) + + sentry_sdk.capture_message(message, level) diff --git a/settings.py b/settings.py index 98cdfdf..e5b2a46 100644 --- a/settings.py +++ b/settings.py @@ -58,6 +58,11 @@ address=SHERLOCK_PROTOCOL_MANAGER_ADDRESS, abi=SHERLOCK_PROTOCOL_MANAGER_ABI ) +with open( + os.path.join(REPO, "artifacts", "contracts", "strategy", "base", "BaseStrategy.sol", "BaseStrategy.json") +) as json_data: + STRATEGY_ABI = json.load(json_data)["abi"] + SHER_CLAIM_AT = SHER_CLAIM_WSS.functions.newEntryDeadline().call() + 60 * 60 * 24 * 7 * 26 # + 26 weeks INDEXER_BLOCKS_PER_CALL = 5 @@ -104,3 +109,8 @@ logger.addHandler(console_handler) logger.addHandler(file_handler) logger.addHandler(debug_file_handler) + +# SENTRY +# ------------------------------------------------------------------------------ +SENTRY_DSN = config("SENTRY_DSN") +SENTRY_ENVIRONMENT = config("SENTRY_ENVIRONMENT") diff --git a/strategies/__init__.py b/strategies/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/strategies/custom_yields.py b/strategies/custom_yields.py new file mode 100644 index 0000000..5a2f01a --- /dev/null +++ b/strategies/custom_yields.py @@ -0,0 +1,221 @@ +import json +import logging +from typing import List, Optional, Tuple + +from attr import define + +from settings import WEB3_WSS +from utils import requests_retry_session + +from .strategies import Strategies, Strategy + +logger = logging.getLogger(__name__) + + +@define +class CustomYield: + strategy: Strategy + + def get_apy(self, block: int, timestamp: int) -> Optional[float]: + """Fetch the APY at a given block/timestamp. + + Args: + block (int): Block number + timestamp (int): UNIX timestamp + + Raises: + NotImplementedError: This method must be overriden by children + + Returns: + float: APY in number format (e.g. 0.035 for 3.5%) + """ + raise NotImplementedError() + + +class MapleYield(CustomYield): + pool_address = "0x6f6c8013f639979c84b756c7fc1500eb5af18dc4" # Maven11 USDC Pool + + def get_apy(self, block: int, timestamp: int) -> Optional[float]: + try: + r = requests_retry_session() + + # Bypass CloudFlare until a more suitable adapter will be developed + r.headers.update( + { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", # noqa + "Origin": "https://app.maple.finance", + "apollographql-client-name": "WebApp", + "apollographql-client-version": "1.0", + "accept": "*/*", + "accept-language": "en-GB,en-US;q=0.9,en;q=0.8", + "cache-control": "no-cache", + "content-type": "application/json", + "pragma": "no-cache", + "sec-ch-ua": 'Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97', + "sec-ch-ua-mobile": "70", + "sec-ch-ua-platform": "macOS", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + } + ) + + res = r.post( + "https://api.maple.finance/v1/graphql", + r""" + { + "query":"query PoolData { results: pool(contractAddress: \"%s\") {\n lendingApy\n }\n}\n", + "variables":null, + "operationName":"PoolData" + } + """ + % self.pool_address, + ).json() + + # APY is returned as a string, for example "583" representing 5.83% + apy = int(res["data"]["results"]["lendingApy"]) / 100 / 100 + logger.debug("Maple APY is %s" % apy) + + return apy + + except Exception as e: + logger.exception(e) + return None + + +class TrueFiYield(CustomYield): + pool_address: str = "0xA991356d261fbaF194463aF6DF8f0464F8f1c742" # TrueFi V2 USDC Pool + + def get_curve_y_apy(self) -> float: + try: + return requests_retry_session().get("https://stats.curve.fi/raw-stats/apys.json").json()["apy"]["day"]["y"] + except Exception as e: + logger.exception(e) + return 0.0 + + def get_pool_values(self, block: int) -> Tuple[float, float]: + try: + POOL_WSS = WEB3_WSS.eth.contract( + address=self.pool_address, + abi=json.loads( + """ + [ + { + "inputs": [], + "name": "poolValue", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "strategyValue", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + } + ] + """ + ), + ) + pool_value = POOL_WSS.functions.poolValue().call(block_identifier=block) + strategy_value = POOL_WSS.functions.strategyValue().call(block_identifier=block) + + return (pool_value, strategy_value) + except Exception as e: + logger.exception(e) + return 0.0 + + def get_loans(self) -> List[Tuple[int, int]]: + """Fetch loans from the USDC pool, as a list of tuples of (loan amount, loan APY)""" + r = requests_retry_session() + r.headers.update( + { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", # noqa + "Origin": "https://app.truefi.io/", + "apollographql-client-name": "WebApp", + "apollographql-client-version": "1.0", + "accept": "*/*", + "accept-language": "en-GB,en-US;q=0.9,en;q=0.8", + "cache-control": "no-cache", + "content-type": "application/json", + "pragma": "no-cache", + "sec-ch-ua": 'Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97', + "sec-ch-ua-mobile": "70", + "sec-ch-ua-platform": "macOS", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + } + ) + + res = r.post( + "https://api.thegraph.com/subgraphs/name/mikemccready/truefi-legacy", + r""" + { + "query":"query Loans { loans(first: 1000, where: {status_in: [1, 2], poolAddress: \"%s\"} \n) {\n APY\n amount\n term\n }\n}\n", + "variables":null, + "operationName":"Loans" + } + """ # noqa + % self.pool_address, + ).json()["data"]["loans"] + + return [(int(item["amount"]), int(item["APY"]) / 10000) for item in res] + + def get_apy(self, block: int, timestamp: int) -> Optional[float]: + try: + (pool_value, strategy_value) = self.get_pool_values(block) + logger.debug("TrueFi Pool value: %s" % pool_value) + logger.debug("TrueFi Strategy value: %s" % strategy_value) + + crv_apy = self.get_curve_y_apy() + logger.debug("CRV Y-POOL APY: %s" % crv_apy) + + crv_weighted_apy = strategy_value * crv_apy + logger.debug("CRV Weighted APY: %s" % crv_weighted_apy) + + loans = self.get_loans() + sum = 0 + weighted_apy = 0 + for item in loans: + # item[0] = amount + # item[1] = APY + # TODO: Make use of the `term` to compute the actual value of this loan, + # but ALL subgraphs available for TrueFi return term = 0 for all loans + # They are fetching the `term` from each `LoanToken` contract. + weighted_apy += item[0] * item[1] + sum += item[0] + # weighted_apy /= sum + + weighted_apy = (weighted_apy + crv_weighted_apy) / pool_value + + logger.debug("TrueFi Weighted APY: %s" % weighted_apy) + + return weighted_apy + except Exception as e: + logger.exception(e) + return None + + +CUSTOM_YIELDS: List[CustomYield] = [MapleYield(Strategies.MAPLE), TrueFiYield(Strategies.TRUEFI)] + + +if __name__ == "__main__": + apy = MapleYield(Strategies.TRUEFI).get_apy(14878797, 1) + print("Maple APY %s" % apy) + + apy = TrueFiYield(Strategies.TRUEFI.address).get_apy(14878797, 1) + print("TrueFI APY %s" % apy) diff --git a/strategies/strategies.py b/strategies/strategies.py new file mode 100644 index 0000000..aab545c --- /dev/null +++ b/strategies/strategies.py @@ -0,0 +1,83 @@ +import logging +from typing import Optional + +from web3.contract import Contract +from web3.exceptions import BadFunctionCallOutput + +from settings import STRATEGY_ABI, WEB3_WSS + +logger = logging.getLogger(__name__) + + +class Strategy: + address: str + name: str + contract: Contract = None + + def __init__(self, address, name) -> None: + self.address = address + self.name = name + + self.connect() + + def connect(self) -> bool: + if self.contract: + return True + + try: + self.contract = WEB3_WSS.eth.contract(address=self.address, abi=STRATEGY_ABI) + return True + except Exception as e: + logger.exception(e) + return False + + def __str__(self) -> str: + return f"Strategy {self.name} @ {self.address}" + + def get_balance(self, block: int) -> Optional[int]: + if not self.connect(): + return None + + logger.debug("Fetching %s balance" % self) + + try: + balance = self.contract.functions.balanceOf().call(block_identifier=block) + logger.info("%s balance is %s" % (self, balance)) + + return balance + except BadFunctionCallOutput as e: + # Skip logging the entire stack as an error, when the exception is related + # to the strategy contract not being deployed yet. + logger.debug(e, exc_info=True) + logger.debug("%s is not yet deployed." % self) + + return None + except Exception as e: + logger.exception(e) + return None + + +class Strategies: + AAVE = Strategy(address="0xE3C37e951F1404b162DFA71A13F0c99c9798Db82", name="Aave") + COMPOUND = Strategy(address="0x8AEA96da625791103a29a16C06c5cfC8B25f6832", name="Compound") + EULER = Strategy(address="0x9a902e8Aae5f1aB423c7aFB29C0Af50e0d3Fea7e", name="Euler") + TRUEFI = Strategy(address="0x1eC37c35BeE1b8b18fC01740db7750Cf93943254", name="TrueFi") + MAPLE = Strategy(address="0xfa01268bd200d0D0f13A6F9758Ba3C09F928E2f7", name="Maple") + + ALL = [AAVE, COMPOUND, EULER, TRUEFI, MAPLE] + + @classmethod + def get(self, address): + """Fetch a strategy by its address.j + + Args: + address (_type_): Strategy address + + Returns: + Strategy: Strategy instance + """ + for item in self.ALL: + if item.address == address: + return item + + return None diff --git a/views/__init__.py b/views/__init__.py index 0323719..275b4f3 100644 --- a/views/__init__.py +++ b/views/__init__.py @@ -4,3 +4,4 @@ from .staking import * # noqa from .stats import * # noqa from .status import * # noqa +from .strategies import * # noqa diff --git a/views/staking.py b/views/staking.py index 2d1910d..619ce71 100644 --- a/views/staking.py +++ b/views/staking.py @@ -23,19 +23,23 @@ def staking_positions(user=None): # Compute USDC increment and updated balance apy = indexer_data.apy + additional_apy = indexer_data.additional_apy + expected_apy = apy + additional_apy for pos in positions: - position_apy = ( - 0.15 if (pos["id"] <= settings.LAST_POSITION_ID_FOR_15PERC_APY and pos["restake_count"] == 0) else apy - ) + if pos["id"] <= settings.LAST_POSITION_ID_FOR_15PERC_APY and pos["restake_count"] == 0: + position_apy = 0.15 + pos["usdc_increment"] = calculate_increment(pos["usdc"], position_apy) + else: + position_apy = expected_apy + pos["usdc_increment"] = calculate_increment(pos["usdc"], apy) - pos["usdc_increment"] = calculate_increment(pos["usdc"], position_apy) pos["usdc"] = round(pos["usdc"] * indexer_data.balance_factor) pos["usdc_apy"] = round(position_apy * 100, 6) return { "ok": True, "positions_usdc_last_updated": int(indexer_data.last_time.timestamp()), - "usdc_apy": round(apy * 100, 6), + "usdc_apy": round(expected_apy * 100, 6), "data": positions, } diff --git a/views/strategies.py b/views/strategies.py new file mode 100644 index 0000000..b4d238c --- /dev/null +++ b/views/strategies.py @@ -0,0 +1,23 @@ +from flask_app import app +from models import Session, StrategyBalance +from strategies.strategies import Strategies + + +@app.route("/strategies") +def strategies(): + with Session() as s: + strategies = ( + s.query(StrategyBalance) + .distinct(StrategyBalance.address) + .order_by(StrategyBalance.address, StrategyBalance.timestamp.desc()) + .all() + ) + + # Transform rows in list of dictionaries + data = [] + for strategy in strategies: + strat_obj = Strategies.get(strategy.address) + + data.append({**strategy.to_dict(), "name": strat_obj.name if strat_obj else "Unknown"}) + + return {"ok": True, "data": data}