Feat/strategy apy v2

README.md

@@ -52,4 +52,6 @@ SHERLOCK_V2_SHER_CLAIM=0x7289C61C75dCdB8Fe4DF0b937c08c9c40902BDd3
 SHERLOCK_V2_PROTOCOL_MANAGER=0x3d0b8A0A10835Ab9b0f0BeB54C5400B8aAcaa1D3
 SHERLOCK_V2_CORE_PATH=/home/evert/sherlock/sherlock-v2-core
 INDEXER_SLEEP_BETWEEN_CALL=0.1
+SENTRY_DSN=
+SENTRY_ENVIRONMENT=production
 ```

alembic/versions/9563fd232d2f_add_additional_apy.py

@@ -0,0 +1,28 @@
+"""Add additional_apy
+
+Revision ID: 9563fd232d2f
+Revises: fe1a6b4ecd8a
+Create Date: 2022-08-29 13:49:43.314919
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "9563fd232d2f"
+down_revision = "fe1a6b4ecd8a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("indexer_state", sa.Column("additional_apy", sa.Float(), server_default="0", nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("indexer_state", "additional_apy")
+    # ### end Alembic commands ###

app.py

@@ -3,6 +3,7 @@
 import sys
 from threading import Thread
 
+import sentry  # noqa
 import settings
 from flask_app import app
 from indexer import Indexer

indexer.py

@@ -11,6 +11,7 @@
 from web3.constants import ADDRESS_ZERO
 from web3.exceptions import ContractLogicError
 
+import sentry
 import settings
 from models import (
     Claim,
@@ -29,6 +30,7 @@
     StrategyBalance,
 )
 from models.interval_function import IntervalFunction
+from strategies.custom_yields import CUSTOM_YIELDS, MapleYield
 from strategies.strategies import Strategies
 from utils import get_event_logs_in_range, get_premiums_apy, requests_retry_session, time_delta_apy
 
@@ -80,13 +82,18 @@ def __init__(self, blocks_per_call=None):
 
         # Order is important, because some functions might depends on the result of the previous ones.
         # - `index_apy` must have an up to date APY computed, so it must come after `calc_apy`
+        # -- NOTE: `calc_apy` can fail to store an up to date APY under specific conditions,
+        #          this would cause `index_apy` to use an old value.
+        # - `calc_additional_apy` must have TVL computed, so it must come after `calc_tvl`
         self.intervals = {
             self.calc_tvl: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.calc_tvc: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.calc_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
+            self.calc_additional_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.index_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.reset_balance_factor: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.index_strategy_balances: settings.INDEXER_STATS_BLOCKS_PER_CALL,
+            self.log_maple_apy: 240,  # 1 hour
         }
 
     def calc_balance_factor(self, session, indx, block):
@@ -140,7 +147,7 @@ def index_apy(self, session, indx, block):
             block: Current block
         """
         timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
-        apy = indx.apy
+        apy = indx.apy + indx.additional_apy
         premiums_apy = indx.premiums_apy
 
         StatsAPY.insert(session, block, timestamp, apy, premiums_apy)
@@ -251,12 +258,41 @@ def calc_apy(self, session, indx, block):
 
         # Update APY only if relevant, that means:
         # - skip negative APYs generated by payouts
+        # - skip short term, very high APYs, generated by strategies (e.g. a loan is paid back in Maple)
+        # -- skip very high APYs (15%)
+        # -- skip if apy is 2.5 times as high as previous apy
         # Position balances are still being correctly kept up to date
         # using the balance factor which accounts for payouts.
         if apy < 0:
             logger.warning("APY %s is being skipped because is negative." % apy)
-        else:
-            indx.apy = apy
+            sentry.report_message(
+                "APY is being skipped because it is negative!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if apy > 0.15:
+            logger.warning("APY %s is being skipped because is higher than 15%%." % apy)
+            sentry.report_message(
+                "APY is being skipped because it higher than 15%!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if indx.apy != 0 and apy > indx.apy * 2.5:
+            logger.warning(
+                "APY %s is being skipped because it is 2.5 times higher than the previous APY of %s" % (apy, indx.apy)
+            )
+            sentry.report_message(
+                "APY is 2.5 times higher than the previous APY!",
+                "warning",
+                {"current_apy": float(apy * 100), "previous_apy": float(indx.apy * 100)},
+            )
+            return
+
+        indx.apy = apy
 
         # Compute the APY coming from protocol premiums
         tvl = StatsTVL.get_current_tvl(session)
@@ -325,6 +361,59 @@ def index_strategy_balances(self, session, indx, block):
                 # If strategy is deployed and active
                 StrategyBalance.insert(session, block, timestamp, strategy.address, balance)
 
+    def calc_additional_apy(self, session, indx, block):
+        """Compute the additionl APY coming from custom yield strtegies.
+        (e.g. Maple, TrueFi)
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
+
+        additional_apy = 0.0
+        for custom_yield in CUSTOM_YIELDS:
+            apy = custom_yield.get_apy(block, timestamp)
+            balance = custom_yield.strategy.get_balance(block)
+
+            logger.info("Strategy %s has balance %s and APY %s" % (custom_yield.strategy, balance, apy))
+
+            # If strategy is deployed and active and the APY has been successfully fetched
+            if balance is not None and apy is not None:
+                TVL = session.query(StatsTVL).order_by(StatsTVL.timestamp.desc()).first()
+
+                logger.info("Balance is %s and TVL value is %s" % (balance, str(TVL.value)))
+
+                # Compute the additional APY generated by this strategy by multipliying the
+                # computed APY with the weights of this strategy in the entire TVL
+                strategy_weight = balance / (TVL.value)
+                logger.info("Strategy weight %s" % strategy_weight)
+                weighted_apy = float(strategy_weight) * apy
+                logger.info("Weghted APY %s" % weighted_apy)
+
+                additional_apy += weighted_apy
+
+        logger.info("Computed additional APY of %s" % additional_apy)
+        indx.additional_apy = additional_apy
+
+    def log_maple_apy(self, session, indx, block):
+        """Log Maple APY to a file in order to save historical data.
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        logger.info("Saving historical Maple APY")
+        timestamp = settings.WEB3_WSS.eth.get_block(block)["timestamp"]
+
+        apy = MapleYield(Strategies.MAPLE).get_apy(0, 0)
+        logger.info("Maple APY: %s" % apy)
+
+        with open("maple.csv", "a") as f:
+            f.write(f"{block},{timestamp},{apy}\n")
+
     class Transfer:
         def new(self, session, indx, block, tx_hash, args):
             if args["to"] == ADDRESS_ZERO:

models/indexer_state.py

@@ -16,3 +16,4 @@ class IndexerState(Base):
     apy = Column(Float, nullable=False, default=0.0)
     premiums_apy = Column(Float, nullable=False, default=0.0)
     apy_50ms_factor = Column(NUMERIC(78, 70), nullable=False, default=0.0)  # TODO: Remove unused column
+    additional_apy = Column(Float, nullable=False, default=0.0, server_default="0")

models/strategy_balance.py

@@ -21,7 +21,6 @@ class StrategyBalance(Base):
     def insert(session, block, timestamp, address, value):
         new_balance = StrategyBalance()
         new_balance.address = address
-        new_balance.value = value
         new_balance.block = block
         new_balance.value = value
         new_balance.timestamp = timestamp

requirements.txt

@@ -61,3 +61,4 @@ flake8==4.0.1  # https://github.com/PyCQA/flake8
 flake8-isort==4.1.1  # https://github.com/gforcada/flake8-isort
 pre-commit==2.15.0  # https://github.com/pre-commit/pre-commit
 alembic==1.7.7  # https://alembic.sqlalchemy.org/en/latest/
+sentry-sdk[flask]==1.5.12  # https://github.com/getsentry/sentry-python

sentry.py

@@ -0,0 +1,45 @@
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+import settings
+
+sentry_sdk.init(
+    dsn=settings.SENTRY_DSN,
+    environment=settings.SENTRY_ENVIRONMENT,
+    integrations=[
+        FlaskIntegration(),
+    ],
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=0.01,
+    # By default the SDK will try to use the SENTRY_RELEASE
+    # environment variable, or infer a git commit
+    # SHA as release, however you may want to set
+    # something more human-readable.
+    # release="[email protected]",
+)
+
+
+def report_message(message: str, level: str = None, extra={}):
+    """Capture a message and send it to Sentry
+
+    Available levels are:
+        - fatal
+        - critical
+        - error
+        - warning
+        - log
+        - info
+        - debug
+
+    Args:
+        message (str): Message text
+        extra (dict): Dict of extra items to send with the message
+    """
+
+    with sentry_sdk.push_scope() as scope:
+        for key, value in extra.items():
+            scope.set_extra(key, value)
+
+        sentry_sdk.capture_message(message, level)

settings.py

@@ -150,3 +150,8 @@
 logger.addHandler(console_handler)
 logger.addHandler(file_handler)
 logger.addHandler(debug_file_handler)
+
+# SENTRY
+# ------------------------------------------------------------------------------
+SENTRY_DSN = config("SENTRY_DSN")
+SENTRY_ENVIRONMENT = config("SENTRY_ENVIRONMENT")