Index strategies with custom yields

README.md

@@ -52,4 +52,6 @@ SHERLOCK_V2_SHER_CLAIM=0x7289C61C75dCdB8Fe4DF0b937c08c9c40902BDd3
 SHERLOCK_V2_PROTOCOL_MANAGER=0x3d0b8A0A10835Ab9b0f0BeB54C5400B8aAcaa1D3
 SHERLOCK_V2_CORE_PATH=/home/evert/sherlock/sherlock-v2-core
 INDEXER_SLEEP_BETWEEN_CALL=0.1
+SENTRY_DSN=
+SENTRY_ENVIRONMENT=production
 ```

alembic/versions/309b0f5de150_additional_apy_and_strategy_balance_.py

@@ -0,0 +1,39 @@
+"""Additional APY and strategy balance model
+
+Revision ID: 309b0f5de150
+Revises: 266a0dc816d5
+Create Date: 2022-05-31 17:05:50.534901
+
+"""
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "309b0f5de150"
+down_revision = "266a0dc816d5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "strategy_balances",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("address", sa.Text(), nullable=False),
+        sa.Column("value", sa.NUMERIC(precision=78), nullable=False),
+        sa.Column("timestamp", postgresql.TIMESTAMP(), nullable=False),
+        sa.Column("block", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.add_column("indexer_state", sa.Column("additional_apy", sa.Float(), server_default="0", nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("indexer_state", "additional_apy")
+    op.drop_table("strategy_balances")
+    # ### end Alembic commands ###

app.py

@@ -3,6 +3,7 @@
 import sys
 from threading import Thread
 
+import sentry  # noqa
 import settings
 from flask_app import app
 from indexer import Indexer

indexer.py

@@ -8,6 +8,7 @@
 from sqlalchemy.exc import IntegrityError
 from web3.constants import ADDRESS_ZERO
 
+import sentry
 import settings
 from models import (
     FundraisePositions,
@@ -21,7 +22,10 @@
     StatsAPY,
     StatsTVC,
     StatsTVL,
+    StrategyBalance,
 )
+from strategies.custom_yields import CUSTOM_YIELDS
+from strategies.strategies import Strategies
 from utils import get_event_logs_in_range, requests_retry_session, time_delta_apy
 
 YEAR = Decimal(timedelta(days=365).total_seconds())
@@ -77,6 +81,8 @@ def __init__(self, blocks_per_call=None):
             self.index_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             # 268 blocks is roughly every hour on current Ethereum mainnet
             self.reset_apy_calc: 268,
+            self.index_strategy_balances: settings.INDEXER_STATS_BLOCKS_PER_CALL,
+            self.calc_additional_apy: 268 * 6,  # 6 hours
         }
 
     # Also get called after listening to events with `end_block`
@@ -127,11 +133,41 @@ def calc_factors(self, session, indx, block):
         indx.block_last_updated = block
         indx.last_time = datetime.fromtimestamp(timestamp)
 
-        # Update APY only if relevant (skip negative APYs generated by payouts).
+        # Update APY only if relevant:
+        # - skip negative APYs generated by payouts
+        # - skip short term, very high APYs, generated by strategies (e.g. a loan is paid back in Maple)
         # Position balances are still being correctly kept up to date
         # using the balance factor which accounts for payouts.
-        if apy > 0.0:
-            indx.apy = apy
+
+        if apy < 0:
+            logger.warning("APY %s is being skipped because is negative." % apy)
+            sentry.report_message(
+                "APY is being skipped because it is negative!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if apy > 0.15:
+            logger.warning("APY %s is being skipped because is higher than 15%%." % apy)
+            sentry.report_message(
+                "APY is being skipped because it higher than 15%!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if indx.apy != 0 and apy > indx.apy * 2:
+            logger.warning(
+                "APY %s is being skipped because it is 2 times higher than the previous APY of %s" % (apy, indx.apy)
+            )
+            sentry.report_message(
+                "APY is 2 times higher than the previous APY!",
+                "warning",
+                {"current_apy": float(apy * 100), "previous_apy": float(indx.apy * 100)},
+            )
+
+        indx.apy = apy
 
     def index_apy(self, session, indx, block):
         """Index current APY.
@@ -142,7 +178,7 @@ def index_apy(self, session, indx, block):
             block: Current block
         """
         timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
-        apy = indx.apy
+        apy = indx.apy + indx.additional_apy
 
         StatsAPY.insert(session, block, timestamp, apy)
 
@@ -239,6 +275,64 @@ def reset_apy_calc(self, session, indx, block):
         # Reset factor
         indx.balance_factor = Decimal(1)
 
+    def index_strategy_balances(self, session, indx, block):
+        """Index each strategy's current balances.
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
+
+        for strategy in Strategies.ALL:
+            balance = strategy.get_balance(block)
+
+            if balance is not None:
+                # If strategy is deployed and active
+                StrategyBalance.insert(session, block, timestamp, strategy.address, balance)
+
+    def calc_additional_apy(self, session, indx, block):
+        """Compute the additionl APY coming from custom yield strtegies.
+        (e.g. Maple, TrueFi)
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
+
+        additional_apy = 0.0
+        for custom_yield in CUSTOM_YIELDS:
+            apy = custom_yield.get_apy(block, timestamp)
+            balance = custom_yield.strategy.get_balance(block)
+
+            logger.info("Strategy %s has balance %s and APY %s" % (custom_yield.strategy, balance, apy))
+
+            # If strategy is deployed and active and the APY has been successfully fetched
+            if balance is not None and apy is not None:
+                TVL = session.query(StatsTVL).order_by(StatsTVL.timestamp.desc()).first()
+
+                # TVL not yet computed. Can happen if the interval for computing the additional APY
+                # is shorter than the interval for computing the TVL.
+                if not TVL or TVL.value == 0:
+                    return
+
+                logger.info("Balance is %s and TVL value is %s" % (balance, str(TVL.value)))
+
+                # Compute the additional APY generated by this strategy by multipliying the
+                # computed APY with the weights of this strategy in the entire TVL
+                strategy_weight = balance / (TVL.value)
+                logger.info("Strategy weight %s" % strategy_weight)
+                weighted_apy = float(strategy_weight) * apy
+                logger.info("Weghted APY %s" % weighted_apy)
+
+                additional_apy += weighted_apy
+
+        logger.info("Computed additional APY of %s" % additional_apy)
+        indx.additional_apy = additional_apy
+
     class Transfer:
         def new(self, session, indx, block, args):
             if args["to"] == ADDRESS_ZERO:

models/__init__.py

@@ -9,6 +9,7 @@
 from .stats_apy import StatsAPY
 from .stats_tvc import StatsTVC
 from .stats_tvl import StatsTVL
+from .strategy_balance import StrategyBalance
 
 __all__ = [
     Base,
@@ -24,4 +25,5 @@
     StatsTVC,
     ProtocolCoverage,
     StatsAPY,
+    StrategyBalance,
 ]

models/indexer_state.py

@@ -15,3 +15,4 @@ class IndexerState(Base):
     balance_factor = Column(NUMERIC(78, 70), nullable=False, default=1.0)
     apy = Column(Float, nullable=False, default=0.0)
     apy_50ms_factor = Column(NUMERIC(78, 70), nullable=False, default=0.0)  # TODO: Remove unused column
+    additional_apy = Column(Float, nullable=False, default=0.0, server_default="0")

models/strategy_balance.py

@@ -0,0 +1,37 @@
+import logging
+
+from sqlalchemy import Column, Integer, Text
+from sqlalchemy.dialects.postgresql import NUMERIC, TIMESTAMP
+
+from models.base import Base
+
+logger = logging.getLogger(__name__)
+
+
+class StrategyBalance(Base):
+    __tablename__ = "strategy_balances"
+
+    id = Column(Integer, primary_key=True)
+    address = Column(Text, nullable=False)
+    value = Column(NUMERIC(78), nullable=False)
+    timestamp = Column(TIMESTAMP, nullable=False)
+    block = Column(Integer, nullable=False)
+
+    @staticmethod
+    def insert(session, block, timestamp, address, value):
+        new_balance = StrategyBalance()
+        new_balance.address = address
+        new_balance.value = value
+        new_balance.block = block
+        new_balance.value = value
+        new_balance.timestamp = timestamp
+
+        session.add(new_balance)
+
+    def to_dict(self):
+        return {
+            "address": self.address,
+            "value": self.value,
+            "timestamp": int(self.timestamp.timestamp()),
+            "block": self.block,
+        }

requirements.txt

@@ -61,3 +61,4 @@ flake8==4.0.1  # https://github.com/PyCQA/flake8
 flake8-isort==4.1.1  # https://github.com/gforcada/flake8-isort
 pre-commit==2.15.0  # https://github.com/pre-commit/pre-commit
 alembic==1.7.7  # https://alembic.sqlalchemy.org/en/latest/
+sentry-sdk[flask]==1.5.12  # https://github.com/getsentry/sentry-python

sentry.py

@@ -0,0 +1,45 @@
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+import settings
+
+sentry_sdk.init(
+    dsn=settings.SENTRY_DSN,
+    environment=settings.SENTRY_ENVIRONMENT,
+    integrations=[
+        FlaskIntegration(),
+    ],
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=0.01,
+    # By default the SDK will try to use the SENTRY_RELEASE
+    # environment variable, or infer a git commit
+    # SHA as release, however you may want to set
+    # something more human-readable.
+    # release="[email protected]",
+)
+
+
+def report_message(message: str, level: str = None, extra={}):
+    """Capture a message and send it to Sentry
+
+    Available levels are:
+        - fatal
+        - critical
+        - error
+        - warning
+        - log
+        - info
+        - debug
+
+    Args:
+        message (str): Message text
+        extra (dict): Dict of extra items to send with the message
+    """
+
+    with sentry_sdk.push_scope() as scope:
+        for key, value in extra.items():
+            scope.set_extra(key, value)
+
+        sentry_sdk.capture_message(message, level)

settings.py

@@ -58,6 +58,11 @@
     address=SHERLOCK_PROTOCOL_MANAGER_ADDRESS, abi=SHERLOCK_PROTOCOL_MANAGER_ABI
 )
 
+with open(
+    os.path.join(REPO, "artifacts", "contracts", "strategy", "base", "BaseStrategy.sol", "BaseStrategy.json")
+) as json_data:
+    STRATEGY_ABI = json.load(json_data)["abi"]
+
 SHER_CLAIM_AT = SHER_CLAIM_WSS.functions.newEntryDeadline().call() + 60 * 60 * 24 * 7 * 26  # + 26 weeks
 
 INDEXER_BLOCKS_PER_CALL = 5
@@ -104,3 +109,8 @@
 logger.addHandler(console_handler)
 logger.addHandler(file_handler)
 logger.addHandler(debug_file_handler)
+
+# SENTRY
+# ------------------------------------------------------------------------------
+SENTRY_DSN = config("SENTRY_DSN")
+SENTRY_ENVIRONMENT = config("SENTRY_ENVIRONMENT")