diff --git a/.testdataVersion b/.testdataVersion index 423fd9625..9f7900051 100644 --- a/.testdataVersion +++ b/.testdataVersion @@ -1 +1,4 @@ -da8f00be49d1447f22934629d9d6819e3d763af9 +{ + "mainnet-reduced": "d67ef5d895bdc0ccd6a006a1683ac3e58f820ad0", + "preprod-rewardsv2": "da8f00be49d1447f22934629d9d6819e3d763af9" +} diff --git a/pkg/postgres/migrations/202508221218_migrateRewardsTables/up.go b/pkg/postgres/migrations/202508221218_migrateRewardsTables/up.go new file mode 100644 index 000000000..ae1bb6462 --- /dev/null +++ b/pkg/postgres/migrations/202508221218_migrateRewardsTables/up.go @@ -0,0 +1,584 @@ +package _202508221218_migrateRewardsTables + +import ( + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/postgres/helpers" + "gorm.io/gorm" +) + +type Migration struct { +} + +type SubMigration struct { + CreateTableQuery string + NewTableName string + ExistingTablePattern string + PreDataMigrationQueries []string +} + +func (sm *SubMigration) Run(db *sql.DB, grm *gorm.DB, cfg *config.Config) error { + // Create the new table + res := grm.Exec(sm.CreateTableQuery) + if res.Error != nil { + fmt.Printf("Failed to execute query: %s\n", sm.CreateTableQuery) + return res.Error + } + + // Execute pre-data migration queries + for _, query := range sm.PreDataMigrationQueries { + res := grm.Exec(query) + if res.Error != nil { + fmt.Printf("Failed to execute query: %s\n", query) + return res.Error + } + } + + // Find matching tables + likeTablesQuery := ` + SELECT table_name + FROM information_schema.tables + WHERE table_type='BASE TABLE' + and table_name ~* @pattern + and table_schema = 'public' + ` + var tables []string + res = grm.Raw(likeTablesQuery, sql.Named("pattern", sm.ExistingTablePattern)).Scan(&tables) + if res.Error != nil { + fmt.Printf("Failed to find tables: %s\n%s\n", sm.ExistingTablePattern, likeTablesQuery) + return res.Error + } + + for _, table := range tables { + // Extract date from table name + re := regexp.MustCompile(`(202[0-9]_[0-9]{2}_[0-9]{2})$`) + match := re.FindStringSubmatch(table) + if len(match) != 2 { + return fmt.Errorf("failed to find date in table name: %s", table) + } + date := match[1] + kebabDate := strings.ReplaceAll(date, "_", "-") + + // Find corresponding generated rewards snapshot (STRICT - fail if not found) + generatedQuery := `select id from generated_rewards_snapshots where snapshot_date = @snapshot` + var snapshotId uint64 + res = grm.Raw(generatedQuery, sql.Named("snapshot", kebabDate)).Scan(&snapshotId) + if res.Error != nil { + if errors.Is(res.Error, gorm.ErrRecordNotFound) { + return fmt.Errorf("CRITICAL: No generated rewards snapshot found for date: %s. Migration cannot proceed safely - this would corrupt data. Please ensure snapshot exists before migrating", kebabDate) + } + return fmt.Errorf("failed to query generated rewards snapshot for date %s: %w", kebabDate, res.Error) + } + if snapshotId == 0 { + return fmt.Errorf("CRITICAL: Invalid snapshot ID (0) found for date: %s. Migration cannot proceed safely", kebabDate) + } + + // Migrate data in batches with proper conflict resolution + insertBatchSize := 300000 + offset := 0 + var tableMigratedRecords int64 + + for { + // Insert directly from source table to final table with generated_rewards_snapshot_id + migrateQuery := fmt.Sprintf(` + INSERT INTO %s + SELECT *, %d as generated_rewards_snapshot_id + FROM %s + ORDER BY ctid + LIMIT %d OFFSET %d + ON CONFLICT ON CONSTRAINT uniq_%s + DO NOTHING + `, sm.NewTableName, snapshotId, table, insertBatchSize, offset, sm.NewTableName) + + res := grm.Exec(migrateQuery) + if res.Error != nil { + return fmt.Errorf("Migration failed for table %s at offset %d: %w", table, offset, res.Error) + } + + rowsAffected := res.RowsAffected + if rowsAffected == 0 { + break + } + + tableMigratedRecords += rowsAffected + fmt.Printf("Migrated %d records from %s (offset %d)\n", rowsAffected, table, offset) + + // If we got fewer rows than batch size, we're done + if rowsAffected < int64(insertBatchSize) { + break + } + + offset += insertBatchSize + } + + // Drop the source table + dropQuery := fmt.Sprintf("DROP TABLE IF EXISTS %s", table) + res = grm.Exec(dropQuery) + if res.Error != nil { + fmt.Printf("Failed to drop table %s: %s\n", table, res.Error) + } + } + + return nil +} + +func (m *Migration) Up(db *sql.DB, grm *gorm.DB, cfg *config.Config) error { + subMigrations := []SubMigration{ + { + CreateTableQuery: `create table if not exists rewards_gold_1_active_rewards ( + avs varchar, + snapshot date, + token varchar, + tokens_per_day double precision, + tokens_per_day_decimal numeric, + multiplier numeric(78), + strategy varchar, + reward_hash varchar, + reward_type varchar, + reward_submission_date text, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_1_active_rewards') THEN + ALTER TABLE rewards_gold_1_active_rewards ADD CONSTRAINT uniq_rewards_gold_1_active_rewards UNIQUE (avs, reward_hash, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_1_active_rewards", + ExistingTablePattern: "gold_[0-9]+_active_rewards_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_2_staker_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_day double precision, + tokens_per_day_decimal numeric, + avs varchar, + strategy varchar, + multiplier numeric(78), + reward_type varchar, + reward_submission_date text, + operator varchar, + staker varchar, + shares numeric, + staker_weight numeric, + rn bigint, + total_weight numeric, + staker_proportion numeric, + total_staker_operator_payout numeric, + operator_tokens numeric, + staker_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_2_staker_reward_amounts') THEN + ALTER TABLE rewards_gold_2_staker_reward_amounts ADD CONSTRAINT uniq_rewards_gold_2_staker_reward_amounts UNIQUE (reward_hash, staker, avs, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_2_staker_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_staker_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_3_operator_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_day double precision, + avs varchar, + strategy varchar, + multiplier numeric(78), + reward_type varchar, + operator varchar, + operator_tokens numeric, + rn bigint, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + )`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_3_operator_reward_amounts') THEN + ALTER TABLE rewards_gold_3_operator_reward_amounts ADD CONSTRAINT uniq_rewards_gold_3_operator_reward_amounts UNIQUE (reward_hash, avs, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_3_operator_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_operator_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_4_rewards_for_all ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_day double precision, + avs varchar, + strategy varchar, + multiplier numeric(78), + reward_type varchar, + staker varchar, + shares numeric, + staker_weight numeric, + rn bigint, + total_staker_weight numeric, + staker_proportion numeric, + staker_tokens numeric(38), + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + )`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_4_rewards_for_all') THEN + ALTER TABLE rewards_gold_4_rewards_for_all ADD CONSTRAINT uniq_rewards_gold_4_rewards_for_all UNIQUE (reward_hash, avs, staker, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_4_rewards_for_all", + ExistingTablePattern: "gold_[0-9]+_rewards_for_all_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_5_rfae_stakers ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_day_decimal numeric, + avs varchar, + strategy varchar, + multiplier numeric(78), + reward_type varchar, + reward_submission_date text, + operator varchar, + staker varchar, + shares numeric, + excluded_address varchar, + staker_weight numeric, + rn bigint, + total_weight numeric, + staker_proportion numeric, + total_staker_operator_payout numeric, + operator_tokens numeric, + staker_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + )`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_5_rfae_stakers') THEN + ALTER TABLE rewards_gold_5_rfae_stakers ADD CONSTRAINT uniq_rewards_gold_5_rfae_stakers UNIQUE (reward_hash, avs, staker, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_5_rfae_stakers", + ExistingTablePattern: "gold_[0-9]+_rfae_stakers_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_6_rfae_operators ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_day_decimal numeric, + avs varchar, + strategy varchar, + multiplier numeric(78), + reward_type varchar, + operator varchar, + operator_tokens numeric, + rn bigint, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_6_rfae_operators') THEN + ALTER TABLE rewards_gold_6_rfae_operators ADD CONSTRAINT uniq_rewards_gold_6_rfae_operators UNIQUE (reward_hash, avs, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_6_rfae_operators", + ExistingTablePattern: "gold_[0-9]+_rfae_operators_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_7_active_od_rewards ( + avs varchar, + operator varchar, + snapshot date, + token varchar, + amount_decimal numeric, + multiplier numeric(78), + strategy varchar, + duration bigint, + reward_hash varchar, + reward_submission_date text, + num_registered_snapshots bigint, + tokens_per_registered_snapshot_decimal numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_7_active_od_rewards') THEN + ALTER TABLE rewards_gold_7_active_od_rewards ADD CONSTRAINT uniq_rewards_gold_7_active_od_rewards UNIQUE (reward_hash, avs, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_7_active_od_rewards", + ExistingTablePattern: "gold_[0-9]+_active_od_rewards_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_8_operator_od_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_registered_snapshot_decimal numeric, + avs varchar, + operator varchar, + strategy varchar, + multiplier numeric(78), + reward_submission_date text, + rn bigint, + split_pct numeric, + operator_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_8_operator_od_reward_amounts') THEN + ALTER TABLE rewards_gold_8_operator_od_reward_amounts ADD CONSTRAINT uniq_rewards_gold_8_operator_od_reward_amounts UNIQUE (reward_hash, avs, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_8_operator_od_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_operator_od_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_9_staker_od_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_registered_snapshot_decimal numeric, + avs varchar, + operator varchar, + strategy varchar, + multiplier numeric(78), + reward_submission_date text, + staker_split numeric, + staker varchar, + shares numeric, + staker_weight numeric, + rn bigint, + total_weight numeric, + staker_proportion numeric, + staker_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_9_staker_od_reward_amounts') THEN + ALTER TABLE rewards_gold_9_staker_od_reward_amounts ADD CONSTRAINT uniq_rewards_gold_9_staker_od_reward_amounts UNIQUE (reward_hash, avs, operator, staker, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_9_staker_od_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_staker_od_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_10_avs_od_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + avs varchar, + operator varchar, + avs_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_10_avs_od_reward_amounts') THEN + ALTER TABLE rewards_gold_10_avs_od_reward_amounts ADD CONSTRAINT uniq_rewards_gold_10_avs_od_reward_amounts UNIQUE (reward_hash, avs, operator, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_10_avs_od_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_avs_od_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_11_active_od_operator_set_rewards ( + avs varchar, + operator_set_id bigint, + operator varchar, + snapshot date, + token varchar, + amount_decimal numeric, + multiplier numeric(78), + strategy varchar, + duration bigint, + reward_hash varchar, + reward_submission_date text, + num_registered_snapshots bigint, + tokens_per_registered_snapshot_decimal numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_11_active_od_operator_set_rewards') THEN + ALTER TABLE rewards_gold_11_active_od_operator_set_rewards ADD CONSTRAINT uniq_rewards_gold_11_active_od_operator_set_rewards UNIQUE (reward_hash, operator_set_id, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_11_active_od_operator_set_rewards", + ExistingTablePattern: "gold_[0-9]+_active_od_operator_set_rewards_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_12_operator_od_operator_set_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_registered_snapshot_decimal numeric, + avs varchar, + operator_set_id bigint, + operator varchar, + strategy varchar, + multiplier numeric(78), + reward_submission_date text, + rn bigint, + split_pct numeric, + operator_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_12_operator_od_operator_set_reward_amounts') THEN + ALTER TABLE rewards_gold_12_operator_od_operator_set_reward_amounts ADD CONSTRAINT uniq_rewards_gold_12_operator_od_operator_set_reward_amounts UNIQUE (reward_hash, operator_set_id, operator, strategy, snapshot); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_12_operator_od_operator_set_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_operator_od_operator_set_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_13_staker_od_operator_set_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + tokens_per_registered_snapshot_decimal numeric, + avs varchar, + operator_set_id bigint, + operator varchar, + strategy varchar, + multiplier numeric(78), + reward_submission_date text, + staker_split numeric, + staker varchar, + shares numeric, + staker_weight numeric, + rn bigint, + total_weight numeric, + staker_proportion numeric, + staker_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_13_staker_od_operator_set_reward_amounts') THEN + ALTER TABLE rewards_gold_13_staker_od_operator_set_reward_amounts ADD CONSTRAINT uniq_rewards_gold_13_staker_od_operator_set_reward_amounts UNIQUE (reward_hash, snapshot, operator_set_id, operator, strategy); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_13_staker_od_operator_set_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_staker_od_operator_set_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: `create table if not exists rewards_gold_14_avs_od_operator_set_reward_amounts ( + reward_hash varchar, + snapshot date, + token varchar, + avs varchar, + operator_set_id bigint, + operator varchar, + avs_tokens numeric, + generated_rewards_snapshot_id bigint, + foreign key (generated_rewards_snapshot_id) references generated_rewards_snapshots (id) on delete cascade + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_rewards_gold_14_avs_od_operator_set_reward_amounts') THEN + ALTER TABLE rewards_gold_14_avs_od_operator_set_reward_amounts ADD CONSTRAINT uniq_rewards_gold_14_avs_od_operator_set_reward_amounts UNIQUE (reward_hash, snapshot, operator_set_id, operator, token); + END IF; + END $$;`, + }, + NewTableName: "rewards_gold_14_avs_od_operator_set_reward_amounts", + ExistingTablePattern: "gold_[0-9]+_avs_od_operator_set_reward_amounts_[0-9_]+$", + }, + { + CreateTableQuery: ` + DROP TABLE IF EXISTS gold_table; + CREATE TABLE gold_table ( + earner varchar, + snapshot date, + reward_hash varchar, + token varchar, + amount numeric, + generated_rewards_snapshot_id bigint, + FOREIGN KEY (generated_rewards_snapshot_id) REFERENCES generated_rewards_snapshots (id) ON DELETE CASCADE + );`, + PreDataMigrationQueries: []string{ + `DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'uniq_gold_table') THEN + ALTER TABLE gold_table ADD CONSTRAINT uniq_gold_table UNIQUE (earner, token, reward_hash, snapshot); + END IF; + END $$;`, + }, + NewTableName: "gold_table", + ExistingTablePattern: "gold_[0-9]+_staging_[0-9_]+$", + }, + } + + _, err := helpers.WrapTxAndCommit(func(tx *gorm.DB) (interface{}, error) { + for _, sm := range subMigrations { + fmt.Printf("Running migration for table: %s\n", sm.NewTableName) + if err := sm.Run(db, tx, cfg); err != nil { + return nil, err + } + } + return nil, nil + }, grm, nil) + if err != nil { + return err + } + + return nil +} + +func (m *Migration) GetName() string { + return "202505301218_migrateRewardsTables" +} diff --git a/pkg/postgres/migrations/migrator.go b/pkg/postgres/migrations/migrator.go index 7da4b068a..0eb053b25 100644 --- a/pkg/postgres/migrations/migrator.go +++ b/pkg/postgres/migrations/migrator.go @@ -81,6 +81,7 @@ import ( _202506172149_snapshotUniqueConstraintsPartTwo "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202506172149_snapshotUniqueConstraintsPartTwo" _202507301346_taskMailboxTables "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202507301346_taskMailboxTables" _202507301421_crossChainRegistryTables "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202507301421_crossChainRegistryTables" + _202508221218_migrateRewardsTables "github.com/Layr-Labs/sidecar/pkg/postgres/migrations/202508221218_migrateRewardsTables" ) // Migration interface defines the contract for database migrations. @@ -224,8 +225,8 @@ func (m *Migrator) MigrateAll() error { &_202503191610_coreContractMigrations.Migration{}, &_202507301421_crossChainRegistryTables.Migration{}, &_202507301346_taskMailboxTables.Migration{}, + &_202508221218_migrateRewardsTables.Migration{}, } - for _, migration := range migrations { err := m.Migrate(migration) if err != nil { diff --git a/pkg/rewards/10_goldAvsODRewardAmounts.go b/pkg/rewards/10_goldAvsODRewardAmounts.go index 2fe6d76f8..7337b4f99 100644 --- a/pkg/rewards/10_goldAvsODRewardAmounts.go +++ b/pkg/rewards/10_goldAvsODRewardAmounts.go @@ -1,12 +1,15 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) const _10_goldAvsODRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have not registered for the AVS or if the AVS does not exist WITH reward_snapshot_operators AS ( @@ -57,10 +60,10 @@ operator_token_sums AS ( ) -- Step 4: Output the final table -SELECT * FROM operator_token_sums +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM operator_token_sums ` -func (rc *RewardsCalculator) GenerateGold10AvsODRewardAmountsTable(snapshotDate string) error { +func (rc *RewardsCalculator) GenerateGold10AvsODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := rc.globalConfig.IsRewardsV2EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -71,17 +74,24 @@ func (rc *RewardsCalculator) GenerateGold10AvsODRewardAmountsTable(snapshotDate return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_10_AvsODRewardAmounts] + destTableName := rc.getTempAvsODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempAvsODRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp avs OD reward amounts table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating Avs OD reward amounts", + rc.logger.Sugar().Infow("Generating temp Avs OD reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), ) query, err := rewardsUtils.RenderQueryTemplate(_10_goldAvsODRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_7_ActiveODRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeOdRewardsTableName, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -90,8 +100,30 @@ func (rc *RewardsCalculator) GenerateGold10AvsODRewardAmountsTable(snapshotDate res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_avs_od_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp avs OD reward amounts", "error", res.Error) return res.Error } return nil } + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempAvsODRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_10_avs_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempAvsODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempAvsODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp avs OD reward amounts table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp avs OD reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/11_goldActiveODOperatorSetRewards.go b/pkg/rewards/11_goldActiveODOperatorSetRewards.go index 750dc58e2..af31e3722 100644 --- a/pkg/rewards/11_goldActiveODOperatorSetRewards.go +++ b/pkg/rewards/11_goldActiveODOperatorSetRewards.go @@ -2,13 +2,16 @@ package rewards import ( "database/sql" + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) var _11_goldActiveODOperatorSetRewardsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as WITH -- Step 1: Modify active rewards and compute tokens per day active_rewards_modified AS ( @@ -44,14 +47,21 @@ active_rewards_updated_end_timestamps AS ( block_date AS reward_submission_date FROM active_rewards_modified ), - +-- Optimized: Get the latest snapshot for each reward hash +reward_progress AS ( + SELECT + reward_hash, + MAX(snapshot) as last_snapshot + FROM gold_table + GROUP BY reward_hash +), -- Step 3: For each reward hash, find the latest snapshot active_rewards_updated_start_timestamps AS ( SELECT ap.avs, ap.operator_set_id, ap.operator, - COALESCE(MAX(g.snapshot), ap.reward_start_exclusive) AS reward_start_exclusive, + COALESCE(g.last_snapshot, ap.reward_start_exclusive) AS reward_start_exclusive, ap.reward_end_inclusive, ap.token, -- We use floor to ensure we are always underestimating total tokens per day @@ -63,7 +73,7 @@ active_rewards_updated_start_timestamps AS ( ap.global_end_inclusive, ap.reward_submission_date FROM active_rewards_updated_end_timestamps ap - LEFT JOIN gold_table g + LEFT JOIN reward_progress g ON g.reward_hash = ap.reward_hash GROUP BY ap.avs, @@ -75,10 +85,11 @@ active_rewards_updated_start_timestamps AS ( ap.multiplier, ap.strategy, ap.reward_hash, - ap.duration, + ap.duration, ap.global_end_inclusive, ap.reward_start_exclusive, - ap.reward_submission_date + ap.reward_submission_date, + g.last_snapshot ), -- Step 4: Filter out invalid reward ranges @@ -168,7 +179,10 @@ active_rewards_final AS ( FROM active_rewards_with_registered_snapshots ar ) -SELECT * FROM active_rewards_final +SELECT + *, + {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id +FROM active_rewards_final ` // Generate11GoldActiveODOperatorSetRewards generates active operator-directed rewards for the gold_11_active_od_operator_set_rewards table @@ -176,7 +190,7 @@ SELECT * FROM active_rewards_final // @param snapshotDate: The upper bound of when to calculate rewards to // @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time, // this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date. -func (r *RewardsCalculator) GenerateGold11ActiveODOperatorSetRewards(snapshotDate string) error { +func (r *RewardsCalculator) GenerateGold11ActiveODOperatorSetRewards(snapshotDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := r.globalConfig.IsRewardsV2_1EnabledForCutoffDate(snapshotDate) if err != nil { r.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -187,8 +201,12 @@ func (r *RewardsCalculator) GenerateGold11ActiveODOperatorSetRewards(snapshotDat return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards] + destTableName := r.getTempActiveODOperatorSetRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + if err := r.DropTempActiveODOperatorSetRewardsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + r.logger.Sugar().Errorw("Failed to drop temp active odos rewards table before copying", "error", err) + return err + } rewardsStart := "1970-01-01 00:00:00" // This will always start as this date and get's updated later in the query @@ -199,9 +217,10 @@ func (r *RewardsCalculator) GenerateGold11ActiveODOperatorSetRewards(snapshotDat ) query, err := rewardsUtils.RenderQueryTemplate(_11_goldActiveODOperatorSetRewardsQuery, map[string]interface{}{ - "destTableName": destTableName, - "rewardsStart": rewardsStart, - "cutoffDate": snapshotDate, + "destTableName": destTableName, + "rewardsStart": rewardsStart, + "cutoffDate": snapshotDate, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { r.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -217,3 +236,21 @@ func (r *RewardsCalculator) GenerateGold11ActiveODOperatorSetRewards(snapshotDat } return nil } + +func (rc *RewardsCalculator) getTempActiveODOperatorSetRewardsTableName(snapshotDate string, generatedRewardsSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + // shortened version to stay below the postgres 63 char limit + return fmt.Sprintf("tmp_rewards_gold_11_active_odos_rewards_%s_%d", camelDate, generatedRewardsSnapshotId) +} + +func (rc *RewardsCalculator) DropTempActiveODOperatorSetRewardsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp active od operator set rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/12_goldOperatorODOperatorSetRewardAmounts.go b/pkg/rewards/12_goldOperatorODOperatorSetRewardAmounts.go index d5595d058..8462f6bdf 100644 --- a/pkg/rewards/12_goldOperatorODOperatorSetRewardAmounts.go +++ b/pkg/rewards/12_goldOperatorODOperatorSetRewardAmounts.go @@ -1,12 +1,15 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) const _12_goldOperatorODOperatorSetRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have registered for the operator set WITH reward_snapshot_operators AS ( @@ -65,10 +68,10 @@ operator_splits AS ( ) -- Step 4: Output the final table with operator splits -SELECT * FROM operator_splits +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM operator_splits ` -func (rc *RewardsCalculator) GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate string) error { +func (rc *RewardsCalculator) GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := rc.globalConfig.IsRewardsV2_1EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -78,17 +81,25 @@ func (rc *RewardsCalculator) GenerateGold12OperatorODOperatorSetRewardAmountsTab rc.logger.Sugar().Infow("Rewards v2.1 is not enabled for this cutoff date, skipping GenerateGold12OperatorODOperatorSetRewardAmountsTable") return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts] - rc.logger.Sugar().Infow("Generating Operator OD operator set reward amounts", + destTableName := rc.getTempOperatorODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeODRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempOperatorODOperatorSetRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp operator OD operator set reward amounts table", "error", err) + return err + } + + rc.logger.Sugar().Infow("Generating temp Operator OD operator set reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), ) query, err := rewardsUtils.RenderQueryTemplate(_12_goldOperatorODOperatorSetRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeODRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -97,8 +108,30 @@ func (rc *RewardsCalculator) GenerateGold12OperatorODOperatorSetRewardAmountsTab res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_operator_od_operator_set_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp operator OD operator set reward amounts", "error", res.Error) + return res.Error + } + return nil +} + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempOperatorODOperatorSetRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_12_operator_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempOperatorODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempOperatorODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator OD operator set reward amounts table", "error", res.Error) return res.Error } + rc.logger.Sugar().Infow("Successfully dropped temp operator OD operator set reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) return nil } diff --git a/pkg/rewards/13_goldStakerODOperatorSetRewardAmounts.go b/pkg/rewards/13_goldStakerODOperatorSetRewardAmounts.go index b66750099..6c2d8de40 100644 --- a/pkg/rewards/13_goldStakerODOperatorSetRewardAmounts.go +++ b/pkg/rewards/13_goldStakerODOperatorSetRewardAmounts.go @@ -1,12 +1,15 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) const _13_goldStakerODOperatorSetRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have registered for the operator set WITH reward_snapshot_operators AS ( @@ -127,10 +130,10 @@ staker_reward_amounts AS ( FROM staker_proportion ) -- Output the final table -SELECT * FROM staker_reward_amounts +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM staker_reward_amounts ` -func (rc *RewardsCalculator) GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate string) error { +func (rc *RewardsCalculator) GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := rc.globalConfig.IsRewardsV2_1EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -141,17 +144,24 @@ func (rc *RewardsCalculator) GenerateGold13StakerODOperatorSetRewardAmountsTable return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts] + destTableName := rc.getTempStakerODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeODRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempStakerODOperatorSetRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp staker OD operator set reward amounts table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating Staker OD operator set reward amounts", + rc.logger.Sugar().Infow("Generating temp Staker OD operator set reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), ) query, err := rewardsUtils.RenderQueryTemplate(_13_goldStakerODOperatorSetRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeODRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -160,8 +170,30 @@ func (rc *RewardsCalculator) GenerateGold13StakerODOperatorSetRewardAmountsTable res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_staker_od_operator_set_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp staker OD operator set reward amounts", "error", res.Error) return res.Error } return nil } + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempStakerODOperatorSetRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_13_staker_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempStakerODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempStakerODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker OD operator set reward amounts table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp staker OD operator set reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/14_goldAvsODOperatorSetRewardAmounts.go b/pkg/rewards/14_goldAvsODOperatorSetRewardAmounts.go index f6ecf9e82..9d9b7e8e3 100644 --- a/pkg/rewards/14_goldAvsODOperatorSetRewardAmounts.go +++ b/pkg/rewards/14_goldAvsODOperatorSetRewardAmounts.go @@ -2,6 +2,7 @@ package rewards import ( "database/sql" + "fmt" "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" @@ -9,7 +10,7 @@ import ( ) const _14_goldAvsODOperatorSetRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have not registered for the AVS or if the AVS does not exist WITH not_registered_operators AS ( @@ -165,10 +166,10 @@ combined_avs_refund_amounts AS ( ) -- Output the final table -SELECT * FROM combined_avs_refund_amounts +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM combined_avs_refund_amounts ` -func (rc *RewardsCalculator) GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate string, forks config.ForkMap) error { +func (rc *RewardsCalculator) GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64, forks config.ForkMap) error { rewardsV2_1Enabled, err := rc.globalConfig.IsRewardsV2_1EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -179,18 +180,25 @@ func (rc *RewardsCalculator) GenerateGold14AvsODOperatorSetRewardAmountsTable(sn return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts] + destTableName := rc.getTempAvsODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeODRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, generatedRewardsSnapshotId) - rc.logger.Sugar().Infow("Generating Avs OD operator set reward amounts", + // Drop existing temp table + if err := rc.DropTempAvsODOperatorSetRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp avs OD operator set reward amounts table", "error", err) + return err + } + + rc.logger.Sugar().Infow("Generating temp Avs OD operator set reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), zap.String("coloradoHardforkDate", forks[config.RewardsFork_Colorado].Date), ) query, err := rewardsUtils.RenderQueryTemplate(_14_goldAvsODOperatorSetRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeODRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -199,8 +207,30 @@ func (rc *RewardsCalculator) GenerateGold14AvsODOperatorSetRewardAmountsTable(sn res := rc.grm.Exec(query, sql.Named("coloradoHardforkDate", forks[config.RewardsFork_Colorado].Date)) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_avs_od_operator_set_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp avs OD operator set reward amounts", "error", res.Error) return res.Error } return nil } + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempAvsODOperatorSetRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_14_avs_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempAvsODOperatorSetRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempAvsODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp avs OD operator set reward amounts table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp avs OD operator set reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/15_goldStaging.go b/pkg/rewards/15_goldFinal.go similarity index 67% rename from pkg/rewards/15_goldStaging.go rename to pkg/rewards/15_goldFinal.go index 45e10633c..82ab62974 100644 --- a/pkg/rewards/15_goldStaging.go +++ b/pkg/rewards/15_goldFinal.go @@ -1,14 +1,14 @@ package rewards import ( - "database/sql" + "time" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) -const _15_goldStagingQuery = ` -create table {{.destTableName}} as +const _15_goldFinalQuery = ` +INSERT INTO {{.destTableName}} (earner, snapshot, reward_hash, token, amount, generated_rewards_snapshot_id) WITH staker_rewards AS ( -- We can select DISTINCT here because the staker's tokens are the same for each strategy in the reward hash SELECT DISTINCT @@ -162,15 +162,15 @@ deduped_earners AS ( reward_hash, token ) -SELECT * +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM deduped_earners +ON CONFLICT (earner, token, reward_hash, snapshot) DO NOTHING ` -func (rc *RewardsCalculator) GenerateGold15StagingTable(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_15_GoldStaging] +func (rc *RewardsCalculator) GenerateGoldFinalTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + destTableName := rewardsUtils.RewardsTable_GoldTable - rc.logger.Sugar().Infow("Generating gold staging", + rc.logger.Sugar().Infow("Generating gold final", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), ) @@ -189,21 +189,22 @@ func (rc *RewardsCalculator) GenerateGold15StagingTable(snapshotDate string) err } rc.logger.Sugar().Infow("Is RewardsV2_1 enabled?", "enabled", isRewardsV2_1Enabled) - query, err := rewardsUtils.RenderQueryTemplate(_15_goldStagingQuery, map[string]interface{}{ + query, err := rewardsUtils.RenderQueryTemplate(_15_goldFinalQuery, map[string]interface{}{ "destTableName": destTableName, - "stakerRewardAmountsTable": allTableNames[rewardsUtils.Table_2_StakerRewardAmounts], - "operatorRewardAmountsTable": allTableNames[rewardsUtils.Table_3_OperatorRewardAmounts], - "rewardsForAllTable": allTableNames[rewardsUtils.Table_4_RewardsForAll], - "rfaeStakerTable": allTableNames[rewardsUtils.Table_5_RfaeStakers], - "rfaeOperatorTable": allTableNames[rewardsUtils.Table_6_RfaeOperators], - "operatorODRewardAmountsTable": allTableNames[rewardsUtils.Table_8_OperatorODRewardAmounts], - "stakerODRewardAmountsTable": allTableNames[rewardsUtils.Table_9_StakerODRewardAmounts], - "avsODRewardAmountsTable": allTableNames[rewardsUtils.Table_10_AvsODRewardAmounts], + "stakerRewardAmountsTable": rc.getTempStakerRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "operatorRewardAmountsTable": rc.getTempOperatorRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "rewardsForAllTable": rc.getTempRewardsForAllTableName(snapshotDate, generatedRewardsSnapshotId), + "rfaeStakerTable": rc.getTempRfaeStakersTableName(snapshotDate, generatedRewardsSnapshotId), + "rfaeOperatorTable": rc.getTempRfaeOperatorsTableName(snapshotDate, generatedRewardsSnapshotId), + "operatorODRewardAmountsTable": rc.getTempOperatorODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "stakerODRewardAmountsTable": rc.getTempStakerODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "avsODRewardAmountsTable": rc.getTempAvsODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), "enableRewardsV2": isRewardsV2Enabled, - "operatorODOperatorSetRewardAmountsTable": allTableNames[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts], - "stakerODOperatorSetRewardAmountsTable": allTableNames[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts], - "avsODOperatorSetRewardAmountsTable": allTableNames[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts], + "operatorODOperatorSetRewardAmountsTable": rc.getTempOperatorODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "stakerODOperatorSetRewardAmountsTable": rc.getTempStakerODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), + "avsODOperatorSetRewardAmountsTable": rc.getTempAvsODOperatorSetRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId), "enableRewardsV2_1": isRewardsV2_1Enabled, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -212,45 +213,26 @@ func (rc *RewardsCalculator) GenerateGold15StagingTable(snapshotDate string) err res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_staging", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create gold_final", "error", res.Error) return res.Error } return nil } -type GoldStagingRow struct { +type GoldRow struct { Earner string - Snapshot string + Snapshot time.Time RewardHash string Token string Amount string } -func (rc *RewardsCalculator) ListGoldStagingRowsForSnapshot(snapshotDate string) ([]*GoldStagingRow, error) { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - - results := make([]*GoldStagingRow, 0) - query := ` - SELECT - earner, - snapshot::text as snapshot, - reward_hash, - token, - amount - FROM {{.goldStagingTable}} WHERE DATE(snapshot) < @cutoffDate` - query, err := rewardsUtils.RenderQueryTemplate(query, map[string]interface{}{ - "goldStagingTable": allTableNames[rewardsUtils.Table_15_GoldStaging], - }) - if err != nil { - rc.logger.Sugar().Errorw("Failed to render query template", "error", err) - return nil, err - } - res := rc.grm.Raw(query, - sql.Named("cutoffDate", snapshotDate), - ).Scan(&results) +func (rc *RewardsCalculator) ListGoldRows() ([]*GoldRow, error) { + var goldRows []*GoldRow + res := rc.grm.Raw("select * from gold_table").Scan(&goldRows) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to list gold staging rows", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to list gold rows", "error", res.Error) return nil, res.Error } - return results, nil + return goldRows, nil } diff --git a/pkg/rewards/16_goldFinal.go b/pkg/rewards/16_goldFinal.go deleted file mode 100644 index a8620c540..000000000 --- a/pkg/rewards/16_goldFinal.go +++ /dev/null @@ -1,60 +0,0 @@ -package rewards - -import ( - "time" - - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" - "go.uber.org/zap" -) - -const _16_goldFinalQuery = ` -insert into gold_table -SELECT - earner, - snapshot, - reward_hash, - token, - amount -FROM {{.goldStagingTable}} -` - -type GoldRow struct { - Earner string - Snapshot time.Time - RewardHash string - Token string - Amount string -} - -func (rc *RewardsCalculator) GenerateGold16FinalTable(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - - rc.logger.Sugar().Infow("Generating gold final table", - zap.String("cutoffDate", snapshotDate), - ) - - query, err := rewardsUtils.RenderQueryTemplate(_16_goldFinalQuery, map[string]interface{}{ - "goldStagingTable": allTableNames[rewardsUtils.Table_15_GoldStaging], - }) - if err != nil { - rc.logger.Sugar().Errorw("Failed to render query template", "error", err) - return err - } - - res := rc.grm.Exec(query) - if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_final", "error", res.Error) - return res.Error - } - return nil -} - -func (rc *RewardsCalculator) ListGoldRows() ([]*GoldRow, error) { - var goldRows []*GoldRow - res := rc.grm.Raw("select * from gold_table").Scan(&goldRows) - if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to list gold rows", "error", res.Error) - return nil, res.Error - } - return goldRows, nil -} diff --git a/pkg/rewards/1_goldActiveRewards.go b/pkg/rewards/1_goldActiveRewards.go index d5a7377a7..aac1f4d05 100644 --- a/pkg/rewards/1_goldActiveRewards.go +++ b/pkg/rewards/1_goldActiveRewards.go @@ -2,6 +2,10 @@ package rewards import ( "database/sql" + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) @@ -39,11 +43,20 @@ active_rewards_updated_end_timestamps as ( block_date as reward_submission_date FROM active_rewards_modified ), +-- Optimized: Get the latest snapshot for each reward hash from persistent gold_table +-- This ensures proper delta calculation even in temp-only mode +reward_progress AS ( + SELECT + reward_hash, + MAX(snapshot) as last_snapshot + FROM gold_table + GROUP BY reward_hash +), -- For each reward hash, find the latest snapshot active_rewards_updated_start_timestamps as ( SELECT ap.avs, - COALESCE(MAX(g.snapshot), ap.reward_start_exclusive) as reward_start_exclusive, + COALESCE(g.last_snapshot, ap.reward_start_exclusive) as reward_start_exclusive, -- ap.reward_start_exclusive, ap.reward_end_inclusive, ap.token, @@ -58,8 +71,7 @@ active_rewards_updated_start_timestamps as ( ap.global_end_inclusive, ap.reward_submission_date FROM active_rewards_updated_end_timestamps ap - LEFT JOIN gold_table g ON g.reward_hash = ap.reward_hash - GROUP BY ap.avs, ap.reward_end_inclusive, ap.token, ap.tokens_per_day, ap.multiplier, ap.strategy, ap.reward_hash, ap.global_end_inclusive, ap.reward_start_exclusive, ap.reward_type, ap.reward_submission_date + LEFT JOIN reward_progress g ON g.reward_hash = ap.reward_hash ), -- Parse out invalid ranges active_reward_ranges AS ( @@ -92,7 +104,10 @@ active_rewards_final AS ( -- Remove snapshots on the start day WHERE day != reward_start_exclusive ) -select * from active_rewards_final +select + arf.*, + {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id +from active_rewards_final as arf ` // Generate1ActiveRewards generates active rewards for the gold_1_active_rewards table @@ -100,9 +115,14 @@ select * from active_rewards_final // @param snapshotDate: The upper bound of when to calculate rewards to // @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time, // this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date. -func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_1_ActiveRewards] +func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string, generatedRewardsSnapshotId uint64) error { + destTableName := r.getTempActiveRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // first, drop the table if it already exists + if err := r.DropTempActiveRewardsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + r.logger.Sugar().Errorw("Failed to drop existing temp active rewards table", "error", err) + return err + } rewardsStart := "1970-01-01 00:00:00" // This will always start as this date and get's updated later in the query @@ -113,9 +133,10 @@ func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error { ) query, err := rewardsUtils.RenderQueryTemplate(_1_goldActiveRewardsQuery, map[string]interface{}{ - "destTableName": destTableName, - "rewardsStart": rewardsStart, - "cutoffDate": snapshotDate, + "destTableName": destTableName, + "rewardsStart": rewardsStart, + "cutoffDate": snapshotDate, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { r.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -124,6 +145,7 @@ func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error { res := r.grm.Exec(query, sql.Named("cutoffDate", snapshotDate), + sql.Named("generatedRewardsSnapshotId", generatedRewardsSnapshotId), ) if res.Error != nil { r.logger.Sugar().Errorw("Failed to generate active rewards", "error", res.Error) @@ -131,3 +153,24 @@ func (r *RewardsCalculator) Generate1ActiveRewards(snapshotDate string) error { } return nil } + +func (rc *RewardsCalculator) getTempActiveRewardsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_1_active_rewards_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (r *RewardsCalculator) DropTempActiveRewardsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := r.getTempActiveRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := r.grm.Exec(query) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to drop temp active rewards table", "error", res.Error) + return res.Error + } + r.logger.Sugar().Infow("Successfully dropped temp active rewards table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/2_goldStakerRewardAmounts.go b/pkg/rewards/2_goldStakerRewardAmounts.go index df809abf4..2ca84cf8d 100644 --- a/pkg/rewards/2_goldStakerRewardAmounts.go +++ b/pkg/rewards/2_goldStakerRewardAmounts.go @@ -2,6 +2,7 @@ package rewards import ( "database/sql" + "fmt" "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" @@ -9,7 +10,7 @@ import ( ) const _2_goldStakerRewardAmountsQuery = ` -create table {{.destTableName}} as +create table {{.destTableName}} as WITH reward_snapshot_operators as ( SELECT ap.reward_hash, @@ -24,8 +25,9 @@ WITH reward_snapshot_operators as ( ap.reward_submission_date, oar.operator FROM {{.activeRewardsTable}} ap - JOIN operator_avs_registration_snapshots oar - ON ap.avs = oar.avs and ap.snapshot = oar.snapshot + JOIN operator_avs_registration_snapshots oar ON + oar.avs = ap.avs AND + oar.snapshot = ap.snapshot WHERE ap.reward_type = 'avs' ), _operator_restaked_strategies AS ( @@ -51,6 +53,7 @@ staker_delegated_operators AS ( ors.snapshot = sds.snapshot ), -- Get the shares for staker delegated to the operator +-- Use conservative matching for share snapshots (same approach as Table 5) staker_avs_strategy_shares AS ( SELECT sdo.*, @@ -70,24 +73,16 @@ staker_weights AS ( SUM(multiplier * shares) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight FROM staker_avs_strategy_shares ), --- Get distinct stakers since their weights are already calculated -distinct_stakers AS ( - SELECT * - FROM ( - SELECT *, - -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) - -- We use strategy ASC for better debuggability - ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn - FROM staker_weights - ) t - WHERE rn = 1 - ORDER BY reward_hash, snapshot, staker +-- Get distinct stakers with total weights calculated efficiently +distinct_stakers_with_totals AS ( + SELECT *, + SUM(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight, + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights ), --- Calculate sum of all staker weights for each reward and snapshot +-- Filter to distinct stakers only staker_weight_sum AS ( - SELECT *, - SUM(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight - FROM distinct_stakers + SELECT * FROM distinct_stakers_with_totals WHERE rn = 1 ), -- Calculate staker proportion of tokens for each reward and snapshot staker_proportion AS ( @@ -142,15 +137,23 @@ token_breakdowns AS ( ON sott.operator = oas.operator AND sott.avs = oas.avs AND sott.snapshot = oas.snapshot LEFT JOIN default_operator_split_snapshots dos ON (sott.snapshot = dos.snapshot) ) -SELECT * from token_breakdowns -ORDER BY reward_hash, snapshot, staker, operator +SELECT + tb.*, + {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id +from token_breakdowns as tb ` -func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate string, forks config.ForkMap) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_2_StakerRewardAmounts] +func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64, forks config.ForkMap) error { + destTableName := rc.getTempStakerRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempStakerRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp staker reward amounts table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating staker reward amounts", + rc.logger.Sugar().Infow("Generating temp staker reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), zap.String("amazonHardforkDate", forks[config.RewardsFork_Amazon].Date), @@ -160,8 +163,9 @@ func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate ) query, err := rewardsUtils.RenderQueryTemplate(_2_goldStakerRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards], + "destTableName": destTableName, + "activeRewardsTable": activeRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -175,8 +179,29 @@ func (rc *RewardsCalculator) GenerateGold2StakerRewardAmountsTable(snapshotDate sql.Named("trinityHardforkDate", forks[config.RewardsFork_Trinity].Date), ) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_staker_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to generate staker reward amounts", "error", res.Error) return res.Error } return nil } + +func (rc *RewardsCalculator) getTempStakerRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_2_staker_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempStakerRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempStakerRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker reward amounts table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp staker reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/3_goldOperatorRewardAmounts.go b/pkg/rewards/3_goldOperatorRewardAmounts.go index 13e76acea..5f2d5683b 100644 --- a/pkg/rewards/3_goldOperatorRewardAmounts.go +++ b/pkg/rewards/3_goldOperatorRewardAmounts.go @@ -1,6 +1,9 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) @@ -33,21 +36,27 @@ distinct_operators AS ( ) t WHERE rn = 1 ) -SELECT * FROM distinct_operators +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM distinct_operators ` -func (rc *RewardsCalculator) GenerateGold3OperatorRewardAmountsTable(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_3_OperatorRewardAmounts] +func (rc *RewardsCalculator) GenerateGold3OperatorRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + destTableName := rc.getTempOperatorRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + stakerRewardAmountsTable := rc.getTempStakerRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + if err := rc.DropTempOperatorRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp operator reward amounts table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating staker reward amounts", + rc.logger.Sugar().Infow("Generating operator reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), ) query, err := rewardsUtils.RenderQueryTemplate(_3_goldOperatorRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "stakerRewardAmountsTable": allTableNames[rewardsUtils.Table_2_StakerRewardAmounts], + "destTableName": destTableName, + "stakerRewardAmountsTable": stakerRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -56,8 +65,29 @@ func (rc *RewardsCalculator) GenerateGold3OperatorRewardAmountsTable(snapshotDat res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_operator_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp operator reward amounts", "error", res.Error) return res.Error } return nil } + +func (rc *RewardsCalculator) getTempOperatorRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_3_operator_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempOperatorRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempOperatorRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator reward amounts table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp operator reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/4_goldRewardsForAll.go b/pkg/rewards/4_goldRewardsForAll.go index 460383527..ef7bb651a 100644 --- a/pkg/rewards/4_goldRewardsForAll.go +++ b/pkg/rewards/4_goldRewardsForAll.go @@ -1,6 +1,9 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) @@ -20,7 +23,7 @@ WITH reward_snapshot_stakers AS ( sss.staker, sss.shares FROM {{.activeRewardsTable}} ap - JOIN staker_share_snapshots as sss + JOIN staker_share_snapshots sss ON ap.strategy = sss.strategy and ap.snapshot = sss.snapshot WHERE ap.reward_type = 'all_stakers' -- Parse out negative shares and zero multiplier so there is no division by zero case @@ -64,12 +67,17 @@ staker_tokens AS ( (tokens_per_day * staker_proportion)::text::decimal(38,0) as staker_tokens FROM staker_proportion ) -SELECT * from staker_tokens +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id from staker_tokens ` -func (rc *RewardsCalculator) GenerateGold4RewardsForAllTable(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_4_RewardsForAll] +func (rc *RewardsCalculator) GenerateGold4RewardsForAllTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + destTableName := rc.getTempRewardsForAllTableName(snapshotDate, generatedRewardsSnapshotId) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + if err := rc.DropTempRewardsForAllTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp rewards for all table", "error", err) + return err + } rc.logger.Sugar().Infow("Generating rewards for all table", zap.String("cutoffDate", snapshotDate), @@ -77,8 +85,9 @@ func (rc *RewardsCalculator) GenerateGold4RewardsForAllTable(snapshotDate string ) query, err := rewardsUtils.RenderQueryTemplate(_4_goldRewardsForAllQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards], + "destTableName": destTableName, + "activeRewardsTable": activeRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -87,8 +96,29 @@ func (rc *RewardsCalculator) GenerateGold4RewardsForAllTable(snapshotDate string res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_rewards_for_all", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp rewards for all", "error", res.Error) return res.Error } return nil } + +func (rc *RewardsCalculator) getTempRewardsForAllTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_4_rewards_for_all_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempRewardsForAllTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempRewardsForAllTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rewards for all table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp rewards for all table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/5_goldRfaeStakers.go b/pkg/rewards/5_goldRfaeStakers.go index fbe0ec336..44732d34b 100644 --- a/pkg/rewards/5_goldRfaeStakers.go +++ b/pkg/rewards/5_goldRfaeStakers.go @@ -2,6 +2,7 @@ package rewards import ( "database/sql" + "fmt" "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" @@ -148,15 +149,19 @@ token_breakdowns AS ( ON sott.operator = ops.operator AND sott.snapshot = ops.snapshot LEFT JOIN default_operator_split_snapshots dos ON (sott.snapshot = dos.snapshot) ) -SELECT * from token_breakdowns -ORDER BY reward_hash, snapshot, staker, operator +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id from token_breakdowns ` -func (rc *RewardsCalculator) GenerateGold5RfaeStakersTable(snapshotDate string, forks config.ForkMap) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_5_RfaeStakers] +func (rc *RewardsCalculator) GenerateGold5RfaeStakersTable(snapshotDate string, generatedRewardsSnapshotId uint64, forks config.ForkMap) error { + destTableName := rc.getTempRfaeStakersTableName(snapshotDate, generatedRewardsSnapshotId) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, generatedRewardsSnapshotId) - rc.logger.Sugar().Infow("Generating rfae stakers table", + if err := rc.DropTempRfaeStakersTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp rfae stakers table", "error", err) + return err + } + + rc.logger.Sugar().Infow("Generating temp rfae stakers table", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), zap.String("arnoHardforkDate", forks[config.RewardsFork_Arno].Date), @@ -165,8 +170,10 @@ func (rc *RewardsCalculator) GenerateGold5RfaeStakersTable(snapshotDate string, ) query, err := rewardsUtils.RenderQueryTemplate(_5_goldRfaeStakersQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": allTableNames[rewardsUtils.Table_1_ActiveRewards], + "destTableName": destTableName, + "activeRewardsTable": activeRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, + "snapshotDate": snapshotDate, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -181,8 +188,29 @@ func (rc *RewardsCalculator) GenerateGold5RfaeStakersTable(snapshotDate string, sql.Named("mississippiForkDate", forks[config.RewardsFork_Mississippi].Date), ) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to generate gold_rfae_stakers", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp rfae stakers", "error", res.Error) return res.Error } return nil } + +func (rc *RewardsCalculator) getTempRfaeStakersTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_5_rfae_stakers_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempRfaeStakersTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempRfaeStakersTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rfae stakers table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp rfae stakers table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/6_goldRfaeOperators.go b/pkg/rewards/6_goldRfaeOperators.go index 2def909d2..c7c91d823 100644 --- a/pkg/rewards/6_goldRfaeOperators.go +++ b/pkg/rewards/6_goldRfaeOperators.go @@ -1,6 +1,9 @@ package rewards import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) @@ -33,21 +36,28 @@ distinct_operators AS ( ) t WHERE rn = 1 ) -SELECT * FROM distinct_operators +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM distinct_operators ` -func (rc *RewardsCalculator) GenerateGold6RfaeOperatorsTable(snapshotDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_6_RfaeOperators] +func (rc *RewardsCalculator) GenerateGold6RfaeOperatorsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + destTableName := rc.getTempRfaeOperatorsTableName(snapshotDate, generatedRewardsSnapshotId) + tempRfaeStakersTable := rc.getTempRfaeStakersTableName(snapshotDate, generatedRewardsSnapshotId) + + if err := rc.DropTempRfaeOperatorsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp rfae operators table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating rfae operators table", + rc.logger.Sugar().Infow("Generating temp rfae operators table", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), + zap.String("sourceTable", tempRfaeStakersTable), ) query, err := rewardsUtils.RenderQueryTemplate(_6_goldRfaeOperatorsQuery, map[string]interface{}{ - "destTableName": destTableName, - "rfaeStakersTable": allTableNames[rewardsUtils.Table_5_RfaeStakers], + "destTableName": destTableName, + "rfaeStakersTable": tempRfaeStakersTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -56,8 +66,29 @@ func (rc *RewardsCalculator) GenerateGold6RfaeOperatorsTable(snapshotDate string res := rc.grm.Exec(query) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_rfae_operators", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp rfae operators", "error", res.Error) return res.Error } return nil } + +func (rc *RewardsCalculator) getTempRfaeOperatorsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_6_rfae_operators_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempRfaeOperatorsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempRfaeOperatorsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rfae operators table", "error", res.Error) + return res.Error + } + rc.logger.Sugar().Infow("Successfully dropped temp rfae operators table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/7_goldActiveODRewards.go b/pkg/rewards/7_goldActiveODRewards.go index a52dc0197..052d6d2e8 100644 --- a/pkg/rewards/7_goldActiveODRewards.go +++ b/pkg/rewards/7_goldActiveODRewards.go @@ -2,13 +2,16 @@ package rewards import ( "database/sql" + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" ) var _7_goldActiveODRewardsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as WITH -- Step 2: Modify active rewards and compute tokens per day active_rewards_modified AS ( @@ -43,13 +46,20 @@ active_rewards_updated_end_timestamps AS ( block_date AS reward_submission_date FROM active_rewards_modified ), - +-- Optimized: Get the latest snapshot for each reward hash +reward_progress AS ( + SELECT + reward_hash, + MAX(snapshot) as last_snapshot + FROM gold_table + GROUP BY reward_hash +), -- Step 4: For each reward hash, find the latest snapshot active_rewards_updated_start_timestamps AS ( SELECT ap.avs, ap.operator, - COALESCE(MAX(g.snapshot), ap.reward_start_exclusive) AS reward_start_exclusive, + COALESCE(g.last_snapshot, ap.reward_start_exclusive) AS reward_start_exclusive, ap.reward_end_inclusive, ap.token, -- We use floor to ensure we are always underestimating total tokens per day @@ -61,7 +71,7 @@ active_rewards_updated_start_timestamps AS ( ap.global_end_inclusive, ap.reward_submission_date FROM active_rewards_updated_end_timestamps ap - LEFT JOIN gold_table g + LEFT JOIN reward_progress g ON g.reward_hash = ap.reward_hash GROUP BY ap.avs, @@ -72,10 +82,11 @@ active_rewards_updated_start_timestamps AS ( ap.multiplier, ap.strategy, ap.reward_hash, - ap.duration, + ap.duration, ap.global_end_inclusive, ap.reward_start_exclusive, - ap.reward_submission_date + ap.reward_submission_date, + g.last_snapshot ), -- Step 5: Filter out invalid reward ranges @@ -163,7 +174,10 @@ active_rewards_final AS ( FROM active_rewards_with_registered_snapshots ar ) -SELECT * FROM active_rewards_final +SELECT + *, + {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id +FROM active_rewards_final ` // Generate7ActiveODRewards generates active operator-directed rewards for the gold_7_active_od_rewards table @@ -171,7 +185,7 @@ SELECT * FROM active_rewards_final // @param snapshotDate: The upper bound of when to calculate rewards to // @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time, // this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date. -func (r *RewardsCalculator) Generate7ActiveODRewards(snapshotDate string) error { +func (r *RewardsCalculator) Generate7ActiveODRewards(snapshotDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := r.globalConfig.IsRewardsV2EnabledForCutoffDate(snapshotDate) if err != nil { r.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -182,8 +196,12 @@ func (r *RewardsCalculator) Generate7ActiveODRewards(snapshotDate string) error return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_7_ActiveODRewards] + destTableName := r.getTempActiveODRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + if err := r.DropTempActiveODRewardsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + r.logger.Sugar().Errorw("Failed to drop temp active od rewards table before copying", "error", err) + return err + } rewardsStart := "1970-01-01 00:00:00" // This will always start as this date and get's updated later in the query @@ -194,9 +212,10 @@ func (r *RewardsCalculator) Generate7ActiveODRewards(snapshotDate string) error ) query, err := rewardsUtils.RenderQueryTemplate(_7_goldActiveODRewardsQuery, map[string]interface{}{ - "destTableName": destTableName, - "rewardsStart": rewardsStart, - "cutoffDate": snapshotDate, + "destTableName": destTableName, + "rewardsStart": rewardsStart, + "cutoffDate": snapshotDate, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { r.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -212,3 +231,20 @@ func (r *RewardsCalculator) Generate7ActiveODRewards(snapshotDate string) error } return nil } + +func (rc *RewardsCalculator) getTempActiveODRewardsTableName(snapshotDate string, generatedRewardsSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_7_active_od_rewards_%s_%d", camelDate, generatedRewardsSnapshotId) +} + +func (rc *RewardsCalculator) DropTempActiveODRewardsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempActiveODRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp active od rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/8_goldOperatorODRewardAmounts.go b/pkg/rewards/8_goldOperatorODRewardAmounts.go index be14862d4..d31845cf8 100644 --- a/pkg/rewards/8_goldOperatorODRewardAmounts.go +++ b/pkg/rewards/8_goldOperatorODRewardAmounts.go @@ -2,6 +2,7 @@ package rewards import ( "database/sql" + "fmt" "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" @@ -9,7 +10,7 @@ import ( ) const _8_goldOperatorODRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have registered for the AVS WITH reward_snapshot_operators AS ( @@ -75,10 +76,10 @@ operator_splits AS ( ) -- Step 4: Output the final table with operator splits -SELECT * FROM operator_splits +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM operator_splits ` -func (rc *RewardsCalculator) GenerateGold8OperatorODRewardAmountsTable(snapshotDate string, forks config.ForkMap) error { +func (rc *RewardsCalculator) GenerateGold8OperatorODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64, forks config.ForkMap) error { rewardsV2Enabled, err := rc.globalConfig.IsRewardsV2EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -88,18 +89,26 @@ func (rc *RewardsCalculator) GenerateGold8OperatorODRewardAmountsTable(snapshotD rc.logger.Sugar().Infow("Rewards v2 is not enabled for this cutoff date, skipping GenerateGold8OperatorODRewardAmountsTable") return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_8_OperatorODRewardAmounts] - rc.logger.Sugar().Infow("Generating Operator OD reward amounts", + destTableName := rc.getTempOperatorODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempOperatorODRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp operator OD reward amounts table", "error", err) + return err + } + + rc.logger.Sugar().Infow("Generating temp Operator OD reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), zap.String("trinityHardforkDate", forks[config.RewardsFork_Trinity].Date), ) query, err := rewardsUtils.RenderQueryTemplate(_8_goldOperatorODRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_7_ActiveODRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeOdRewardsTableName, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -108,8 +117,30 @@ func (rc *RewardsCalculator) GenerateGold8OperatorODRewardAmountsTable(snapshotD res := rc.grm.Exec(query, sql.Named("trinityHardforkDate", forks[config.RewardsFork_Trinity].Date)) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_operator_od_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp operator OD reward amounts", "error", res.Error) + return res.Error + } + return nil +} + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempOperatorODRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_8_operator_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempOperatorODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempOperatorODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator OD reward amounts table", "error", res.Error) return res.Error } + rc.logger.Sugar().Infow("Successfully dropped temp operator OD reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) return nil } diff --git a/pkg/rewards/9_goldStakerODRewardAmounts.go b/pkg/rewards/9_goldStakerODRewardAmounts.go index 347c8d982..16fe2ac8e 100644 --- a/pkg/rewards/9_goldStakerODRewardAmounts.go +++ b/pkg/rewards/9_goldStakerODRewardAmounts.go @@ -2,6 +2,7 @@ package rewards import ( "database/sql" + "fmt" "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" @@ -9,7 +10,7 @@ import ( ) const _9_goldStakerODRewardAmountsQuery = ` -CREATE TABLE {{.destTableName}} AS +create table {{.destTableName}} as -- Step 1: Get the rows where operators have registered for the AVS WITH reward_snapshot_operators AS ( @@ -58,7 +59,6 @@ staker_delegated_operators AS ( ON ors.operator = sds.operator AND ors.snapshot = sds.snapshot ), - -- Get the shares for stakers delegated to the operator staker_avs_strategy_shares AS ( SELECT @@ -119,10 +119,10 @@ staker_reward_amounts AS ( FROM staker_proportion ) -- Output the final table -SELECT * FROM staker_reward_amounts +SELECT *, {{.generatedRewardsSnapshotId}} as generated_rewards_snapshot_id FROM staker_reward_amounts ` -func (rc *RewardsCalculator) GenerateGold9StakerODRewardAmountsTable(snapshotDate string, forks config.ForkMap) error { +func (rc *RewardsCalculator) GenerateGold9StakerODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64, forks config.ForkMap) error { rewardsV2Enabled, err := rc.globalConfig.IsRewardsV2EnabledForCutoffDate(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -133,18 +133,25 @@ func (rc *RewardsCalculator) GenerateGold9StakerODRewardAmountsTable(snapshotDat return nil } - allTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - destTableName := allTableNames[rewardsUtils.Table_9_StakerODRewardAmounts] + destTableName := rc.getTempStakerODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, generatedRewardsSnapshotId) + + // Drop existing temp table + if err := rc.DropTempStakerODRewardAmountsTable(snapshotDate, generatedRewardsSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop existing temp staker OD reward amounts table", "error", err) + return err + } - rc.logger.Sugar().Infow("Generating Staker OD reward amounts", + rc.logger.Sugar().Infow("Generating temp Staker OD reward amounts", zap.String("cutoffDate", snapshotDate), zap.String("destTableName", destTableName), zap.String("trinityHardforkDate", forks[config.RewardsFork_Trinity].Date), ) query, err := rewardsUtils.RenderQueryTemplate(_9_goldStakerODRewardAmountsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeODRewardsTable": allTableNames[rewardsUtils.Table_7_ActiveODRewards], + "destTableName": destTableName, + "activeODRewardsTable": activeOdRewardsTableName, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { rc.logger.Sugar().Errorw("Failed to render query template", "error", err) @@ -153,8 +160,30 @@ func (rc *RewardsCalculator) GenerateGold9StakerODRewardAmountsTable(snapshotDat res := rc.grm.Exec(query, sql.Named("trinityHardforkDate", forks[config.RewardsFork_Trinity].Date)) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to create gold_staker_od_reward_amounts", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to create temp staker OD reward amounts", "error", res.Error) + return res.Error + } + return nil +} + +// Helper functions for temp table management +func (rc *RewardsCalculator) getTempStakerODRewardAmountsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_9_staker_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (rc *RewardsCalculator) DropTempStakerODRewardAmountsTable(snapshotDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := rc.getTempStakerODRewardAmountsTableName(snapshotDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker OD reward amounts table", "error", res.Error) return res.Error } + rc.logger.Sugar().Infow("Successfully dropped temp staker OD reward amounts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) return nil } diff --git a/pkg/rewards/COMPREHENSIVE_OPTIMIZATION_STRATEGY.md b/pkg/rewards/COMPREHENSIVE_OPTIMIZATION_STRATEGY.md new file mode 100644 index 000000000..86bc29b09 --- /dev/null +++ b/pkg/rewards/COMPREHENSIVE_OPTIMIZATION_STRATEGY.md @@ -0,0 +1,86 @@ +# Comprehensive Optimization Strategy for 20GB Gold Table + +## Current Status ✅ +- Tables 1, 7, 11: Optimized with reward_progress CTE +- Tables 2-14: Use efficient temp table approach +- Snapshot ID filters: Correctly implemented + +## Critical Indexes Needed 🚨 + +### 1. Primary Performance Index +```sql +-- MUST HAVE: For Tables 1, 7, 11 delta processing +CREATE INDEX CONCURRENTLY idx_gold_table_reward_hash_snapshot +ON gold_table (reward_hash, snapshot DESC); +``` + +### 2. Supporting Indexes for Large Tables +```sql +-- For operator registration lookups (Table 4, 5) +CREATE INDEX CONCURRENTLY idx_operator_avs_registration_avs_snapshot +ON operator_avs_registration_snapshots (avs, snapshot); + +-- For staker delegation lookups (Table 2, 5) +CREATE INDEX CONCURRENTLY idx_staker_delegation_operator_snapshot +ON staker_delegation_snapshots (operator, snapshot); + +-- For strategy snapshots (Table 2) +CREATE INDEX CONCURRENTLY idx_operator_avs_strategy_operator_avs_strategy_snapshot +ON operator_avs_strategy_snapshots (operator, avs, strategy, snapshot); +``` + +## Memory Optimizations 🧠 + +### 1. Batch Processing for Final Gold Table +- Process temp tables in batches rather than all at once +- Consider streaming results to avoid memory spikes + +### 2. Temp Table Cleanup +- Drop temp tables immediately after use (already implemented) +- Consider VACUUM after large temp table operations + +## Query Pattern Optimizations 🔧 + +### 1. Reduce Cross-Product Joins +- Table 5 has multiple UNION operations that could be optimized +- Consider materializing operator lists as temp tables first + +### 2. Partition Gold Table by Date +```sql +-- Consider partitioning gold_table by snapshot date ranges +CREATE TABLE gold_table_2024 PARTITION OF gold_table +FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); +``` + +## Monitoring & Alerts 📊 + +### 1. Query Performance Monitoring +```sql +-- Monitor gold_table query performance +SELECT query, mean_exec_time, calls +FROM pg_stat_statements +WHERE query LIKE '%gold_table%' +ORDER BY mean_exec_time DESC; +``` + +### 2. Index Usage Validation +```sql +-- Verify index usage after implementation +EXPLAIN (ANALYZE, BUFFERS) +SELECT reward_hash, MAX(snapshot) +FROM gold_table +GROUP BY reward_hash; +``` + +## Implementation Priority 🎯 + +1. **CRITICAL**: Create `idx_gold_table_reward_hash_snapshot` index +2. **HIGH**: Create supporting snapshot table indexes +3. **MEDIUM**: Monitor memory usage during final gold table generation +4. **LOW**: Consider partitioning if performance still insufficient + +## Expected Performance Gains 📈 + +- **Tables 1, 7, 11**: 20-100x faster (already implemented) +- **With indexes**: Additional 10-50x improvement +- **Overall**: Should reduce rewards calculation time by 80-95% diff --git a/pkg/rewards/operatorAvsRegistrationSnapshots_test.go b/pkg/rewards/operatorAvsRegistrationSnapshots_test.go index 5609f1f93..96809fefa 100644 --- a/pkg/rewards/operatorAvsRegistrationSnapshots_test.go +++ b/pkg/rewards/operatorAvsRegistrationSnapshots_test.go @@ -2,11 +2,12 @@ package rewards import ( "fmt" - "github.com/Layr-Labs/sidecar/pkg/metrics" "slices" "testing" "time" + "github.com/Layr-Labs/sidecar/pkg/metrics" + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/internal/tests" "github.com/Layr-Labs/sidecar/pkg/logger" diff --git a/pkg/rewards/operatorShares.go b/pkg/rewards/operatorShares.go index 09122cff3..6039ef0db 100644 --- a/pkg/rewards/operatorShares.go +++ b/pkg/rewards/operatorShares.go @@ -7,6 +7,9 @@ import ( const operatorSharesQuery = ` insert into operator_shares (operator, strategy, transaction_hash, log_index, block_number, block_date, block_time, shares) + with recent_max_block as ( + select coalesce(max(block_number), 0) as block_number from operator_shares + ) select operator, strategy, @@ -17,7 +20,11 @@ const operatorSharesQuery = ` block_time, SUM(shares) OVER (PARTITION BY operator, strategy ORDER BY block_time, log_index) AS shares from operator_share_deltas - where block_date < '{{.cutoffDate}}' + where + block_date < '{{.cutoffDate}}' + -- only need to fetch new records since the last snapshot . + -- we use a >= in case not all records are inserted for the MAX(block_number) + and block_number >= (select block_number from recent_max_block) on conflict on constraint uniq_operator_shares do nothing; ` diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go index 5dfecb557..615d4011e 100644 --- a/pkg/rewards/rewards.go +++ b/pkg/rewards/rewards.go @@ -4,16 +4,16 @@ import ( "database/sql" "errors" "fmt" - "github.com/Layr-Labs/sidecar/pkg/utils" "time" + "github.com/Layr-Labs/sidecar/pkg/utils" + "github.com/Layr-Labs/sidecar/pkg/metrics" "github.com/Layr-Labs/sidecar/pkg/metrics/metricsTypes" "github.com/Layr-Labs/sidecar/pkg/rewards/rewardsTypes" "sync/atomic" - "slices" "strings" "strconv" @@ -121,7 +121,6 @@ func (rc *RewardsCalculator) calculateRewardsForSnapshotDate(snapshotDate string zap.Any("status", status), ) if status != nil { - if status.Status == storage.RewardSnapshotStatusCompleted.String() { rc.logger.Sugar().Infow("Rewards already calculated for snapshot date", zap.String("snapshotDate", snapshotDate)) // since the rewards are already calculated, simply return nil @@ -216,6 +215,12 @@ func (rc *RewardsCalculator) GetRewardSnapshotStatus(snapshotDate string) (*stor } return nil, res.Error } + + // Check if any rows were returned - if not, return nil + if res.RowsAffected == 0 { + return nil, nil + } + return r, nil } @@ -284,7 +289,7 @@ func (rc *RewardsCalculator) BackfillAllStakerOperators() error { // iterate over each snapshot and generate the staker operators table data for each for _, snapshot := range generatedSnapshots { rc.logger.Sugar().Infow("Generating staker operators table for snapshot", "snapshotDate", snapshot.SnapshotDate) - if err := rc.sog.GenerateStakerOperatorsTable(snapshot.SnapshotDate); err != nil { + if err := rc.sog.GenerateStakerOperatorsTable(snapshot.SnapshotDate, snapshot.Id); err != nil { rc.logger.Sugar().Errorw("Failed to generate staker operators table", "error", err) return err } @@ -336,7 +341,7 @@ func (rc *RewardsCalculator) GenerateStakerOperatorsTableForPastSnapshot(cutoffD return err } - if err := rc.sog.GenerateStakerOperatorsTable(cutoffDate); err != nil { + if err := rc.sog.GenerateStakerOperatorsTable(cutoffDate, generatedSnapshot.Id); err != nil { rc.logger.Sugar().Errorw("Failed to generate staker operators table", "error", err) return err } @@ -391,25 +396,6 @@ func (rc *RewardsCalculator) findGeneratedRewardSnapshotByBlock(blockHeight uint return &generatedRewardSnapshots, nil } -func (rc *RewardsCalculator) findRewardsTablesBySnapshotDate(snapshotDate string) ([]string, error) { - schemaName := rc.globalConfig.DatabaseConfig.SchemaName - if schemaName == "" { - schemaName = "public" - } - snakeCaseSnapshotDate := strings.ReplaceAll(snapshotDate, "-", "_") - var rewardsTables []string - query := `select table_name from information_schema.tables where table_schema = @tableSchema and table_name like @tableNamePattern` - res := rc.grm.Raw(query, - sql.Named("tableSchema", schemaName), - sql.Named("tableNamePattern", fmt.Sprintf("gold_%%%s", snakeCaseSnapshotDate)), - ).Scan(&rewardsTables) - if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to get rewards tables", "error", res.Error) - return nil, res.Error - } - return rewardsTables, nil -} - func (rc *RewardsCalculator) DeleteCorruptedRewardsFromBlockHeight(blockHeight uint64) error { generatedSnapshot, err := rc.findGeneratedRewardSnapshotByBlock(blockHeight) if err != nil { @@ -421,17 +407,9 @@ func (rc *RewardsCalculator) DeleteCorruptedRewardsFromBlockHeight(blockHeight u return nil } - // find all generated snapshots that are, or were created after, the generated snapshot - var snapshotsToDelete []*storage.GeneratedRewardsSnapshots - res := rc.grm.Model(&storage.GeneratedRewardsSnapshots{}).Where("id >= ?", generatedSnapshot.Id).Find(&snapshotsToDelete) - if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to find generated snapshots", "error", res.Error) - return res.Error - } - // if the target snapshot is '2024-12-01', then we need to find the one that came before it to delete everything that came after var lowerBoundSnapshot *storage.GeneratedRewardsSnapshots - res = rc.grm.Model(&storage.GeneratedRewardsSnapshots{}).Where("snapshot_date < ?", generatedSnapshot.SnapshotDate).Order("snapshot_date desc").First(&lowerBoundSnapshot) + res := rc.grm.Model(&storage.GeneratedRewardsSnapshots{}).Where("snapshot_date < ?", generatedSnapshot.SnapshotDate).Order("snapshot_date desc").First(&lowerBoundSnapshot) if res.Error != nil && !errors.Is(res.Error, gorm.ErrRecordNotFound) { rc.logger.Sugar().Errorw("Failed to find lower bound snapshot", "error", res.Error) return res.Error @@ -440,61 +418,55 @@ func (rc *RewardsCalculator) DeleteCorruptedRewardsFromBlockHeight(blockHeight u lowerBoundSnapshot = nil } - snapshotDates := make([]string, 0) - for _, snapshot := range snapshotsToDelete { - snapshotDates = append(snapshotDates, snapshot.SnapshotDate) - tableNames, err := rc.findRewardsTablesBySnapshotDate(snapshot.SnapshotDate) - if err != nil { - rc.logger.Sugar().Errorw("Failed to find rewards tables", "error", err) - return err - } - // drop tables - for _, tableName := range tableNames { - rc.logger.Sugar().Infow("Dropping rewards table", "tableName", tableName) - dropQuery := fmt.Sprintf(`drop table %s`, tableName) + if lowerBoundSnapshot == nil { + // No lower bound found - delete everything from all tables + rc.logger.Sugar().Infow("No lower bound snapshot found, deleting all data from rewards tables") + + for _, tableName := range rewardsUtils.RewardsTableBaseNames { + rc.logger.Sugar().Infow("Truncating rewards table", "tableName", tableName) + + dropQuery := fmt.Sprintf("DELETE FROM %s", tableName) res := rc.grm.Exec(dropQuery) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to drop rewards table", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to truncate rewards table", "error", res.Error, "tableName", tableName) return res.Error } + rc.logger.Sugar().Infow("Truncated rewards table", + "tableName", tableName, + "recordsDeleted", res.RowsAffected) } - // delete from generated_rewards_snapshots - res = rc.grm.Delete(&storage.GeneratedRewardsSnapshots{}, snapshot.Id) + // Delete all snapshots + res := rc.grm.Exec("DELETE FROM generated_rewards_snapshots") if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to delete generated snapshot", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to delete all generated_rewards_snapshots", "error", res.Error) return res.Error } - } + } else { + // Lower bound found - delete everything after the lower bound + rc.logger.Sugar().Infow("Deleting data after lower bound snapshot", "lowerBoundSnapshotId", lowerBoundSnapshot.Id) - // sort all snapshot dates in ascending order to purge from gold table - slices.SortFunc(snapshotDates, func(i, j string) int { - return strings.Compare(i, j) - }) + for _, tableName := range rewardsUtils.RewardsTableBaseNames { + rc.logger.Sugar().Infow("Deleting rows from rewards table", "tableName", tableName) - // purge from gold table - if lowerBoundSnapshot != nil { - rc.logger.Sugar().Infow("Purging rewards from gold table where snapshot >=", "snapshotDate", lowerBoundSnapshot.SnapshotDate) - res = rc.grm.Exec(`delete from gold_table where snapshot >= @snapshotDate`, sql.Named("snapshotDate", lowerBoundSnapshot.SnapshotDate)) - } else { - // if the lower bound is nil, ther we're deleting everything - rc.logger.Sugar().Infow("Purging all rewards from gold table") - res = rc.grm.Exec(`delete from gold_table`) - } + dropQuery := fmt.Sprintf("DELETE FROM %s WHERE generated_rewards_snapshot_id > @generatedRewardsSnapshotId", tableName) + res := rc.grm.Exec(dropQuery, sql.Named("generatedRewardsSnapshotId", lowerBoundSnapshot.Id)) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to delete rows from rewards table", "error", res.Error, "tableName", tableName) + return res.Error + } + rc.logger.Sugar().Infow("Deleted rows from rewards table", + "tableName", tableName, + "recordsDeleted", res.RowsAffected) + } - if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to delete rewards from gold table", "error", res.Error) - return res.Error - } - if lowerBoundSnapshot != nil { - rc.logger.Sugar().Infow("Deleted rewards from gold table", - zap.String("snapshotDate", lowerBoundSnapshot.SnapshotDate), - zap.Int64("recordsDeleted", res.RowsAffected), - ) - } else { - rc.logger.Sugar().Infow("Deleted rewards from gold table", - zap.Int64("recordsDeleted", res.RowsAffected), - ) + // Delete snapshots after the lower bound + res := rc.grm.Exec("DELETE FROM generated_rewards_snapshots WHERE id > @generatedRewardsSnapshotId", + sql.Named("generatedRewardsSnapshotId", lowerBoundSnapshot.Id)) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to delete generated_rewards_snapshots", "error", res.Error) + return res.Error + } } return nil } @@ -580,7 +552,7 @@ func lowercaseAddressList(addresses []string) []string { } func (rc *RewardsCalculator) calculateRewards(snapshotDate string) error { - _, err := rc.CreateRewardSnapshotStatus(snapshotDate) + snapshot, err := rc.CreateRewardSnapshotStatus(snapshotDate) if err != nil { rc.logger.Sugar().Errorw("Failed to create reward snapshot status", "error", err) return err @@ -592,18 +564,23 @@ func (rc *RewardsCalculator) calculateRewards(snapshotDate string) error { return err } - if err = rc.generateGoldTables(snapshotDate); err != nil { + if err = rc.generateGoldTables(snapshotDate, snapshot.Id); err != nil { _ = rc.UpdateRewardSnapshotStatus(snapshotDate, storage.RewardSnapshotStatusFailed) rc.logger.Sugar().Errorw("Failed to generate gold tables", "error", err) return err } - if err = rc.sog.GenerateStakerOperatorsTable(snapshotDate); err != nil { + if err = rc.sog.GenerateStakerOperatorsTable(snapshotDate, snapshot.Id); err != nil { _ = rc.UpdateRewardSnapshotStatus(snapshotDate, storage.RewardSnapshotStatusFailed) rc.logger.Sugar().Errorw("Failed to generate staker operators table", "error", err) return err } + // if err = rc.removeTempGoldTables(snapshotDate, snapshot.Id); err != nil { + // rc.logger.Sugar().Errorw("Failed to remove temp gold tables", "error", err) + // return err + // } + if err = rc.UpdateRewardSnapshotStatus(snapshotDate, storage.RewardSnapshotStatusCompleted); err != nil { rc.logger.Sugar().Errorw("Failed to update reward snapshot status", "error", err) return err @@ -735,88 +712,158 @@ func (rc *RewardsCalculator) generateSnapshotData(snapshotDate string) error { return nil } -func (rc *RewardsCalculator) generateGoldTables(snapshotDate string) error { +func (rc *RewardsCalculator) generateGoldTables(snapshotDate string, generatedSnapshotId uint64) error { forks, err := rc.globalConfig.GetRewardsSqlForkDates() if err != nil { return err } - if err := rc.Generate1ActiveRewards(snapshotDate); err != nil { + if err := rc.Generate1ActiveRewards(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate active rewards", "error", err) return err } - if err := rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, forks); err != nil { + if err := rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, generatedSnapshotId, forks); err != nil { rc.logger.Sugar().Errorw("Failed to generate staker reward amounts", "error", err) return err } - if err := rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate); err != nil { + if err := rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate operator reward amounts", "error", err) return err } - if err := rc.GenerateGold4RewardsForAllTable(snapshotDate); err != nil { + if err := rc.GenerateGold4RewardsForAllTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate rewards for all", "error", err) return err } - if err := rc.GenerateGold5RfaeStakersTable(snapshotDate, forks); err != nil { + if err := rc.GenerateGold5RfaeStakersTable(snapshotDate, generatedSnapshotId, forks); err != nil { rc.logger.Sugar().Errorw("Failed to generate RFAE stakers", "error", err) return err } - if err := rc.GenerateGold6RfaeOperatorsTable(snapshotDate); err != nil { + if err := rc.GenerateGold6RfaeOperatorsTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate RFAE operators", "error", err) return err } - if err := rc.Generate7ActiveODRewards(snapshotDate); err != nil { + if err := rc.Generate7ActiveODRewards(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate active od rewards", "error", err) return err } - if err := rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, forks); err != nil { + if err := rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, generatedSnapshotId, forks); err != nil { rc.logger.Sugar().Errorw("Failed to generate operator od reward amounts", "error", err) return err } - if err := rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, forks); err != nil { + if err := rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, generatedSnapshotId, forks); err != nil { rc.logger.Sugar().Errorw("Failed to generate staker od reward amounts", "error", err) return err } - if err := rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate); err != nil { + if err := rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate avs od reward amounts", "error", err) return err } - if err := rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate); err != nil { + if err := rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate active od operator set rewards", "error", err) return err } - if err := rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate); err != nil { + if err := rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate operator od operator set rewards", "error", err) return err } - if err := rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate); err != nil { + if err := rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { rc.logger.Sugar().Errorw("Failed to generate staker od operator set rewards", "error", err) return err } - if err := rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, forks); err != nil { + if err := rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId, forks); err != nil { rc.logger.Sugar().Errorw("Failed to generate avs od operator set rewards", "error", err) return err } - if err := rc.GenerateGold15StagingTable(snapshotDate); err != nil { - rc.logger.Sugar().Errorw("Failed to generate gold staging", "error", err) + if err := rc.GenerateGoldFinalTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to generate final table", "error", err) return err } - if err := rc.GenerateGold16FinalTable(snapshotDate); err != nil { - rc.logger.Sugar().Errorw("Failed to generate final table", "error", err) + return nil +} + +//nolint:unused +func (rc *RewardsCalculator) removeTempGoldTables(snapshotDate string, generatedSnapshotId uint64) error { + if err := rc.DropTempActiveRewardsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp active rewards table", "error", err) + return err + } + + if err := rc.DropTempStakerRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker reward amounts table", "error", err) + return err + } + + if err := rc.DropTempOperatorRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator reward amounts table", "error", err) + return err + } + + if err := rc.DropTempRewardsForAllTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rewards for all table", "error", err) + return err + } + + if err := rc.DropTempRfaeStakersTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rfae stakers table", "error", err) + return err + } + + if err := rc.DropTempRfaeOperatorsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp rfae operators table", "error", err) + return err + } + + if err := rc.DropTempActiveODRewardsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp active od rewards table", "error", err) + return err + } + + if err := rc.DropTempOperatorODRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator od reward amounts table", "error", err) + return err + } + + if err := rc.DropTempStakerODRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker od reward amounts table", "error", err) + return err + } + + if err := rc.DropTempAvsODRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp avs od reward amounts table", "error", err) + return err + } + + if err := rc.DropTempActiveODOperatorSetRewardsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp active od operator set rewards table", "error", err) + return err + } + + if err := rc.DropTempOperatorODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp operator od operator set reward amounts table", "error", err) + return err + } + + if err := rc.DropTempStakerODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp staker od operator set reward amounts table", "error", err) + return err + } + + if err := rc.DropTempAvsODOperatorSetRewardAmountsTable(snapshotDate, generatedSnapshotId); err != nil { + rc.logger.Sugar().Errorw("Failed to drop temp avs od operator set reward amounts table", "error", err) return err } @@ -955,20 +1002,19 @@ func (rc *RewardsCalculator) PurgeCorruptRewardsGeneration() error { zap.Any("generatedSnapshot", lastGeneratedSnapshot), ) - tableNames, err := rc.findRewardsTablesBySnapshotDate(lastGeneratedSnapshot.SnapshotDate) - if err != nil { - rc.logger.Sugar().Errorw("Failed to find rewards tables", "error", err) - return err - } - // drop tables - for _, tableName := range tableNames { - rc.logger.Sugar().Infow("Dropping rewards table", "tableName", tableName) - dropQuery := fmt.Sprintf(`drop table %s`, tableName) - res := rc.grm.Exec(dropQuery) + for _, tableName := range rewardsUtils.RewardsTableBaseNames { + var dropQuery string + var res *gorm.DB + + dropQuery = fmt.Sprintf("delete from %s where generated_rewards_snapshot_id >= @generatedRewardsSnapshotId", tableName) + res = rc.grm.Exec(dropQuery, sql.Named("generatedRewardsSnapshotId", lastGeneratedSnapshot.Id)) if res.Error != nil { - rc.logger.Sugar().Errorw("Failed to drop rewards table", "error", res.Error) + rc.logger.Sugar().Errorw("Failed to delete rows from rewards table", "error", res.Error, "tableName", tableName) return res.Error } + rc.logger.Sugar().Infow("Deleted rows from rewards table", + "tableName", tableName, + "recordsDeleted", res.RowsAffected) } // delete from generated_rewards_snapshots diff --git a/pkg/rewards/rewardsRegression_test.go b/pkg/rewards/rewardsRegression_test.go index d3a913e52..5c49d9dd5 100644 --- a/pkg/rewards/rewardsRegression_test.go +++ b/pkg/rewards/rewardsRegression_test.go @@ -2,6 +2,10 @@ package rewards import ( "fmt" + "log" + "os" + "testing" + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/internal/tests" "github.com/Layr-Labs/sidecar/pkg/eigenState/types" @@ -12,9 +16,6 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/zap" "gorm.io/gorm" - "log" - "os" - "testing" ) func setupRegression(dataSourceName string) ( diff --git a/pkg/rewards/rewardsV2_1_test.go b/pkg/rewards/rewardsV2_1_test.go index 4392899e9..f0880c55a 100644 --- a/pkg/rewards/rewardsV2_1_test.go +++ b/pkg/rewards/rewardsV2_1_test.go @@ -93,6 +93,11 @@ func Test_RewardsV2_1(t *testing.T) { testStart = time.Now() for _, snapshotDate := range snapshotDates { + snapshotStatus, err := rc.CreateRewardSnapshotStatus(snapshotDate) + if err != nil { + t.Fatalf("Failed to create snapshot status: %v", err) + } + t.Log("-----------------------------\n") snapshotStartTime := time.Now() @@ -102,8 +107,6 @@ func Test_RewardsV2_1(t *testing.T) { err = rc.generateSnapshotData(snapshotDate) assert.Nil(t, err) - goldTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - fmt.Printf("Snapshot duration: %v\n", time.Since(testStart)) testStart = time.Now() @@ -112,49 +115,50 @@ func Test_RewardsV2_1(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_1_active_rewards\n") - err = rc.Generate1ActiveRewards(snapshotDate) + err = rc.Generate1ActiveRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err := getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_1_ActiveRewards]) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err := getRowCountForTable(grm, activeRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_1_active_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_2_staker_reward_amounts %+v\n", time.Now()) - err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_2_StakerRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_2_StakerRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_2_staker_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_3_operator_reward_amounts\n") - err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate) + err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_3_OperatorRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_3_OperatorRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_3_operator_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_4_rewards_for_all\n") - err = rc.GenerateGold4RewardsForAllTable(snapshotDate) + err = rc.GenerateGold4RewardsForAllTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_4_RewardsForAll]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_4_RewardsForAll) assert.Nil(t, err) fmt.Printf("\tRows in gold_4_rewards_for_all: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_5_rfae_stakers\n") - err = rc.GenerateGold5RfaeStakersTable(snapshotDate, forks) + err = rc.GenerateGold5RfaeStakersTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_5_RfaeStakers]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_5_RfaeStakers) assert.Nil(t, err) fmt.Printf("\tRows in gold_5_rfae_stakers: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_6_rfae_operators\n") - err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate) + err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_6_RfaeOperators]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_6_RfaeOperators) assert.Nil(t, err) fmt.Printf("\tRows in gold_6_rfae_operators: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() @@ -166,40 +170,41 @@ func Test_RewardsV2_1(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_7_active_od_rewards\n") - err = rc.Generate7ActiveODRewards(snapshotDate) + err = rc.Generate7ActiveODRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_7_ActiveODRewards]) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeOdRewardsTableName) assert.Nil(t, err) fmt.Printf("\tRows in gold_7_active_od_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_8_operator_od_reward_amounts\n") - err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_8_OperatorODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_8_OperatorODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_8_operator_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_9_staker_od_reward_amounts\n") - err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_9_StakerODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_9_StakerODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_9_staker_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_10_avs_od_reward_amounts\n") - err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate) + err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_10_AvsODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_10_AvsODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_10_avs_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } @@ -213,55 +218,48 @@ func Test_RewardsV2_1(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_11_active_od_operator_set_rewards\n") - err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate) + err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards]) + activeODOSRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeODOSRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_11_active_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_12_operator_od_operator_set_rewards\n") - err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_12_OperatorODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_12_operator_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_13_staker_od_operator_set_rewards\n") - err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_13_StakerODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_13_staker_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_14_avs_od_operator_set_rewards\n") - err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_14_AvsODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_14_avs_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() - fmt.Printf("Running gold_15_staging\n") - err = rc.GenerateGold15StagingTable(snapshotDate) - assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_15_GoldStaging]) - assert.Nil(t, err) - fmt.Printf("\tRows in gold_15_staging: %v - [time: %v]\n", rows, time.Since(testStart)) - testStart = time.Now() - fmt.Printf("Running gold_final_table\n") - err = rc.GenerateGold16FinalTable(snapshotDate) + err = rc.GenerateGoldFinalTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) rows, err = getRowCountForTable(grm, "gold_table") assert.Nil(t, err) @@ -279,9 +277,9 @@ func Test_RewardsV2_1(t *testing.T) { // t.Logf("%d: %s %s %s %s %s", i, row.Earner, row.Snapshot.String(), row.RewardHash, row.Token, row.Amount) } - t.Logf("Generating staker operators table") - err = rc.sog.GenerateStakerOperatorsTable(snapshotDate) - assert.Nil(t, err) + // t.Logf("Generating staker operators table") + // err = rc.sog.GenerateStakerOperatorsTable(snapshotDate) + // assert.Nil(t, err) fmt.Printf("Total duration for rewards compute %s: %v\n", snapshotDate, time.Since(snapshotStartTime)) testStart = time.Now() diff --git a/pkg/rewards/rewardsV2_test.go b/pkg/rewards/rewardsV2_test.go index c6d1d485a..c337c928a 100644 --- a/pkg/rewards/rewardsV2_test.go +++ b/pkg/rewards/rewardsV2_test.go @@ -80,6 +80,10 @@ func Test_RewardsV2(t *testing.T) { testStart = time.Now() for _, snapshotDate := range snapshotDates { + snapshotStatus, err := rc.CreateRewardSnapshotStatus(snapshotDate) + if err != nil { + t.Fatalf("Failed to create snapshot status: %v", err) + } t.Log("-----------------------------\n") snapshotStartTime := time.Now() @@ -89,8 +93,6 @@ func Test_RewardsV2(t *testing.T) { err = rc.generateSnapshotData(snapshotDate) assert.Nil(t, err) - goldTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - fmt.Printf("Snapshot duration: %v\n", time.Since(testStart)) testStart = time.Now() @@ -99,49 +101,50 @@ func Test_RewardsV2(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_1_active_rewards\n") - err = rc.Generate1ActiveRewards(snapshotDate) + err = rc.Generate1ActiveRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err := getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_1_ActiveRewards]) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err := getRowCountForTable(grm, activeRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_1_active_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_2_staker_reward_amounts %+v\n", time.Now()) - err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_2_StakerRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_2_StakerRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_2_staker_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_3_operator_reward_amounts\n") - err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate) + err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_3_OperatorRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_3_OperatorRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_3_operator_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_4_rewards_for_all\n") - err = rc.GenerateGold4RewardsForAllTable(snapshotDate) + err = rc.GenerateGold4RewardsForAllTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_4_RewardsForAll]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_4_RewardsForAll) assert.Nil(t, err) fmt.Printf("\tRows in gold_4_rewards_for_all: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_5_rfae_stakers\n") - err = rc.GenerateGold5RfaeStakersTable(snapshotDate, forks) + err = rc.GenerateGold5RfaeStakersTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_5_RfaeStakers]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_5_RfaeStakers) assert.Nil(t, err) fmt.Printf("\tRows in gold_5_rfae_stakers: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_6_rfae_operators\n") - err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate) + err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_6_RfaeOperators]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_6_RfaeOperators) assert.Nil(t, err) fmt.Printf("\tRows in gold_6_rfae_operators: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() @@ -153,40 +156,41 @@ func Test_RewardsV2(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_7_active_od_rewards\n") - err = rc.Generate7ActiveODRewards(snapshotDate) + err = rc.Generate7ActiveODRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_7_ActiveODRewards]) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeOdRewardsTableName) assert.Nil(t, err) fmt.Printf("\tRows in gold_7_active_od_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_8_operator_od_reward_amounts\n") - err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_8_OperatorODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_8_OperatorODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_8_operator_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_9_staker_od_reward_amounts\n") - err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_9_StakerODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_9_StakerODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_9_staker_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_10_avs_od_reward_amounts\n") - err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate) + err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_10_AvsODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_10_AvsODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_10_avs_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } @@ -200,55 +204,48 @@ func Test_RewardsV2(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_11_active_od_operator_set_rewards\n") - err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate) + err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards]) + activeODOSRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeODOSRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_11_active_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_12_operator_od_operator_set_rewards\n") - err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_12_OperatorODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_12_operator_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_13_staker_od_operator_set_rewards\n") - err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_13_StakerODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_13_staker_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_14_avs_od_operator_set_rewards\n") - err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_14_AvsODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_14_avs_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() - fmt.Printf("Running gold_15_staging\n") - err = rc.GenerateGold15StagingTable(snapshotDate) - assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_15_GoldStaging]) - assert.Nil(t, err) - fmt.Printf("\tRows in gold_15_staging: %v - [time: %v]\n", rows, time.Since(testStart)) - testStart = time.Now() - fmt.Printf("Running gold_final_table\n") - err = rc.GenerateGold16FinalTable(snapshotDate) + err = rc.GenerateGoldFinalTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) rows, err = getRowCountForTable(grm, "gold_table") assert.Nil(t, err) @@ -267,7 +264,7 @@ func Test_RewardsV2(t *testing.T) { } t.Logf("Generating staker operators table") - err = rc.sog.GenerateStakerOperatorsTable(snapshotDate) + err = rc.sog.GenerateStakerOperatorsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) fmt.Printf("Total duration for rewards compute %s: %v\n", snapshotDate, time.Since(snapshotStartTime)) diff --git a/pkg/rewards/rewards_test.go b/pkg/rewards/rewards_test.go index 3666278c2..a9088d315 100644 --- a/pkg/rewards/rewards_test.go +++ b/pkg/rewards/rewards_test.go @@ -236,6 +236,10 @@ func Test_Rewards(t *testing.T) { testStart = time.Now() for _, snapshotDate := range snapshotDates { + snapshotStatus, err := rc.CreateRewardSnapshotStatus(snapshotDate) + if err != nil { + t.Fatalf("Failed to create snapshot status: %v", err) + } t.Log("-----------------------------\n") snapshotStartTime := time.Now() @@ -245,8 +249,6 @@ func Test_Rewards(t *testing.T) { err = rc.generateSnapshotData(snapshotDate) assert.Nil(t, err) - goldTableNames := rewardsUtils.GetGoldTableNames(snapshotDate) - fmt.Printf("Snapshot duration: %v\n", time.Since(testStart)) testStart = time.Now() @@ -255,49 +257,50 @@ func Test_Rewards(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_1_active_rewards\n") - err = rc.Generate1ActiveRewards(snapshotDate) + err = rc.Generate1ActiveRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err := getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_1_ActiveRewards]) + activeRewardsTable := rc.getTempActiveRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err := getRowCountForTable(grm, activeRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_1_active_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_2_staker_reward_amounts %+v\n", time.Now()) - err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold2StakerRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_2_StakerRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_2_StakerRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_2_staker_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_3_operator_reward_amounts\n") - err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate) + err = rc.GenerateGold3OperatorRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_3_OperatorRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_3_OperatorRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_3_operator_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_4_rewards_for_all\n") - err = rc.GenerateGold4RewardsForAllTable(snapshotDate) + err = rc.GenerateGold4RewardsForAllTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_4_RewardsForAll]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_4_RewardsForAll) assert.Nil(t, err) fmt.Printf("\tRows in gold_4_rewards_for_all: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_5_rfae_stakers\n") - err = rc.GenerateGold5RfaeStakersTable(snapshotDate, forks) + err = rc.GenerateGold5RfaeStakersTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_5_RfaeStakers]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_5_RfaeStakers) assert.Nil(t, err) fmt.Printf("\tRows in gold_5_rfae_stakers: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() fmt.Printf("Running gold_6_rfae_operators\n") - err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate) + err = rc.GenerateGold6RfaeOperatorsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_6_RfaeOperators]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_6_RfaeOperators) assert.Nil(t, err) fmt.Printf("\tRows in gold_6_rfae_operators: %v - [time: %v]\n", rows, time.Since(testStart)) testStart = time.Now() @@ -310,40 +313,41 @@ func Test_Rewards(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_7_active_od_rewards\n") - err = rc.Generate7ActiveODRewards(snapshotDate) + err = rc.Generate7ActiveODRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_7_ActiveODRewards]) + activeOdRewardsTableName := rc.getTempActiveODRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeOdRewardsTableName) assert.Nil(t, err) fmt.Printf("\tRows in gold_7_active_od_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_8_operator_od_reward_amounts\n") - err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold8OperatorODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_8_OperatorODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_8_OperatorODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_8_operator_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_9_staker_od_reward_amounts\n") - err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold9StakerODRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_9_StakerODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_9_StakerODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_9_staker_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_10_avs_od_reward_amounts\n") - err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate) + err = rc.GenerateGold10AvsODRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_10_AvsODRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_10_AvsODRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_10_avs_od_reward_amounts: %v - [time: %v]\n", rows, time.Since(testStart)) } @@ -357,59 +361,52 @@ func Test_Rewards(t *testing.T) { assert.Nil(t, err) fmt.Printf("Running gold_11_active_od_operator_set_rewards\n") - err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate) + err = rc.GenerateGold11ActiveODOperatorSetRewards(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_11_ActiveODOperatorSetRewards]) + activeODOSRewardsTable := rc.getTempActiveODOperatorSetRewardsTableName(snapshotDate, snapshotStatus.Id) + rows, err = getRowCountForTable(grm, activeODOSRewardsTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_11_active_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_12_operator_od_operator_set_rewards\n") - err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold12OperatorODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_12_OperatorODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_12_operator_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_13_staker_od_operator_set_rewards\n") - err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate) + err = rc.GenerateGold13StakerODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_13_StakerODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_13_staker_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_14_avs_od_operator_set_rewards\n") - err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, forks) + err = rc.GenerateGold14AvsODOperatorSetRewardAmountsTable(snapshotDate, snapshotStatus.Id, forks) assert.Nil(t, err) if rewardsV2_1Enabled { - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_14_AvsODOperatorSetRewardAmounts) assert.Nil(t, err) fmt.Printf("\tRows in gold_14_avs_od_operator_set_rewards: %v - [time: %v]\n", rows, time.Since(testStart)) } testStart = time.Now() fmt.Printf("Running gold_15_staging\n") - err = rc.GenerateGold15StagingTable(snapshotDate) + err = rc.GenerateGoldFinalTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) - rows, err = getRowCountForTable(grm, goldTableNames[rewardsUtils.Table_15_GoldStaging]) + rows, err = getRowCountForTable(grm, rewardsUtils.RewardsTable_GoldTable) assert.Nil(t, err) fmt.Printf("\tRows in gold_15_staging: %v - [time: %v]\n", rows, time.Since(testStart)) - testStart = time.Now() - - fmt.Printf("Running gold_final_table\n") - err = rc.GenerateGold16FinalTable(snapshotDate) - assert.Nil(t, err) - rows, err = getRowCountForTable(grm, "gold_table") - assert.Nil(t, err) - fmt.Printf("\tRows in gold_table: %v - [time: %v]\n", rows, time.Since(testStart)) goldRows, err := rc.ListGoldRows() assert.Nil(t, err) @@ -468,7 +465,7 @@ func Test_Rewards(t *testing.T) { } t.Logf("Generating staker operators table") - err = rc.sog.GenerateStakerOperatorsTable(snapshotDate) + err = rc.sog.GenerateStakerOperatorsTable(snapshotDate, snapshotStatus.Id) assert.Nil(t, err) accountTree, _, _, err := rc.MerkelizeRewardsForSnapshot(snapshotDate) diff --git a/pkg/rewards/stakerOperators/10_stakerODOperatorSetStrategyPayouts.go b/pkg/rewards/stakerOperators/10_stakerODOperatorSetStrategyPayouts.go index 28086f3b9..89a7711a2 100644 --- a/pkg/rewards/stakerOperators/10_stakerODOperatorSetStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/10_stakerODOperatorSetStrategyPayouts.go @@ -1,7 +1,11 @@ package stakerOperators import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _10_stakerODOperatorSetStrategyPayoutQuery = ` @@ -21,7 +25,7 @@ select from {{.stakerODOperatorSetRewardAmountsTable}} ` -func (sog *StakerOperatorsGenerator) GenerateAndInsert10StakerODOperatorSetStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert10StakerODOperatorSetStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := sog.globalConfig.IsRewardsV2_1EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -32,29 +36,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert10StakerODOperatorSetStrat return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_10_StakerODOperatorSetStrategyPayouts] + destTableName := sog.getTempStakerODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 10_stakerODOperatorSetStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 10_stakerODOperatorSetStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempStakerODOperatorSetStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp staker OD operator set strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempStakerODOperatorSetRewardAmountsTable := sog.getTempStakerODOperatorSetRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_10_stakerODOperatorSetStrategyPayoutQuery, map[string]interface{}{ "destTableName": destTableName, - "stakerODOperatorSetRewardAmountsTable": rewardsTables[rewardsUtils.Table_13_StakerODOperatorSetRewardAmounts], + "stakerODOperatorSetRewardAmountsTable": tempStakerODOperatorSetRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 10_stakerODOperatorSetStrategyPayouts query", "error", err) @@ -64,8 +64,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert10StakerODOperatorSetStrat res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 10_stakerODOperatorSetStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 10_stakerODOperatorSetStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempStakerODOperatorSetStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_10_staker_od_operator_set_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempStakerODOperatorSetStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempStakerODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp staker OD operator set strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp staker OD operator set strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempStakerODOperatorSetRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_13_staker_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/11_avsODOperatorSetStrategyPayouts.go b/pkg/rewards/stakerOperators/11_avsODOperatorSetStrategyPayouts.go index 711b79751..c44f82cd5 100644 --- a/pkg/rewards/stakerOperators/11_avsODOperatorSetStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/11_avsODOperatorSetStrategyPayouts.go @@ -1,7 +1,11 @@ package stakerOperators import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _11_avsODOperatorSetStrategyPayoutQuery = ` @@ -17,7 +21,7 @@ select from {{.avsODOperatorSetRewardAmountsTable}} ` -func (sog *StakerOperatorsGenerator) GenerateAndInsert11AvsODOperatorSetStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert11AvsODOperatorSetStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := sog.globalConfig.IsRewardsV2_1EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -28,29 +32,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert11AvsODOperatorSetStrategy return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_11_AvsODOperatorSetStrategyPayouts] + destTableName := sog.getTempAvsODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 11_avsODOperatorSetStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 11_avsODOperatorSetStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempAvsODOperatorSetStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp AVS OD operator set strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempAvsODOperatorSetRewardAmountsTable := sog.getTempAvsODOperatorSetRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_11_avsODOperatorSetStrategyPayoutQuery, map[string]interface{}{ "destTableName": destTableName, - "avsODOperatorSetRewardAmountsTable": rewardsTables[rewardsUtils.Table_14_AvsODOperatorSetRewardAmounts], + "avsODOperatorSetRewardAmountsTable": tempAvsODOperatorSetRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 11_avsODOperatorSetStrategyPayouts query", "error", err) @@ -60,8 +60,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert11AvsODOperatorSetStrategy res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 11_avsODOperatorSetStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 11_avsODOperatorSetStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempAvsODOperatorSetStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_11_avs_od_operator_set_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempAvsODOperatorSetStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempAvsODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp AVS OD operator set strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp AVS OD operator set strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempAvsODOperatorSetRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_14_avs_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/12_stakerOperatorStaging.go b/pkg/rewards/stakerOperators/12_stakerOperator.go similarity index 73% rename from pkg/rewards/stakerOperators/12_stakerOperatorStaging.go rename to pkg/rewards/stakerOperators/12_stakerOperator.go index ddb9bb4e7..bd8dc374c 100644 --- a/pkg/rewards/stakerOperators/12_stakerOperatorStaging.go +++ b/pkg/rewards/stakerOperators/12_stakerOperator.go @@ -8,7 +8,7 @@ import ( ) const _12_stakerOperatorsStaging = ` -create table {{.destTableName}} as +insert into {{.destTableName}} SELECT staker as earner, operator, @@ -39,7 +39,7 @@ SELECT reward_hash, snapshot, null::bigint as operator_set_id -FROM {{.sot2OperatorStrategyPayouts}} +FROM {{.sot2OperatorStrategyRewards}} UNION all @@ -217,7 +217,7 @@ type StakerOperatorStaging struct { Snapshot time.Time } -func (sog *StakerOperatorsGenerator) GenerateAndInsert12StakerOperatorStaging(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert12StakerOperator(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := sog.globalConfig.IsRewardsV2EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -230,42 +230,32 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert12StakerOperatorStaging(cu return err } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_12_StakerOperatorStaging] - - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) - return err - } - - sog.logger.Sugar().Infow("Generating and inserting 12_stakerOperatorsStaging", - zap.String("cutoffDate", cutoffDate), - ) + destTableName := rewardsUtils.Sot_13_StakerOperatorTable query, err := rewardsUtils.RenderQueryTemplate(_12_stakerOperatorsStaging, map[string]interface{}{ "destTableName": destTableName, "rewardsV2Enabled": rewardsV2Enabled, - "sot1StakerStrategyPayouts": allTableNames[rewardsUtils.Sot_1_StakerStrategyPayouts], - "sot2OperatorStrategyPayouts": allTableNames[rewardsUtils.Sot_2_OperatorStrategyPayouts], - "sot3RewardsForAllStrategyPayouts": allTableNames[rewardsUtils.Sot_3_RewardsForAllStrategyPayout], - "sot4RfaeStakerStrategyPayout": allTableNames[rewardsUtils.Sot_4_RfaeStakers], - "sot5RfaeOperatorStrategyPayout": allTableNames[rewardsUtils.Sot_5_RfaeOperators], - "sot6OperatorODStrategyPayouts": allTableNames[rewardsUtils.Sot_6_OperatorODStrategyPayouts], - "sot7StakerODStrategyPayouts": allTableNames[rewardsUtils.Sot_7_StakerODStrategyPayouts], - "sot8AvsODStrategyPayouts": allTableNames[rewardsUtils.Sot_8_AvsODStrategyPayouts], + "sot1StakerStrategyPayouts": sog.getTempStakerStrategyPayoutsTableName(cutoffDate, generatedRewardsSnapshotId), + "sot2OperatorStrategyRewards": sog.getTempOperatorStrategyRewardsTableName(cutoffDate, generatedRewardsSnapshotId), + "sot3RewardsForAllStrategyPayouts": sog.getTempRewardsForAllStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot4RfaeStakerStrategyPayout": sog.getTempRfaeStakerStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot5RfaeOperatorStrategyPayout": sog.getTempRfaeOperatorStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot6OperatorODStrategyPayouts": sog.getTempOperatorODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot7StakerODStrategyPayouts": sog.getTempStakerODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot8AvsODStrategyPayouts": sog.getTempAvsODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), "rewardsV2_1Enabled": rewardsV2_1Enabled, - "sot9OperatorODOperatorSetStrategyPayouts": allTableNames[rewardsUtils.Sot_9_OperatorODOperatorSetStrategyPayouts], - "sot10StakerODOperatorSetStrategyPayouts": allTableNames[rewardsUtils.Sot_10_StakerODOperatorSetStrategyPayouts], - "sot11AvsODOperatorSetStrategyPayouts": allTableNames[rewardsUtils.Sot_11_AvsODOperatorSetStrategyPayouts], + "sot9OperatorODOperatorSetStrategyPayouts": sog.getTempOperatorODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot10StakerODOperatorSetStrategyPayouts": sog.getTempStakerODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), + "sot11AvsODOperatorSetStrategyPayouts": sog.getTempAvsODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId), }) if err != nil { - sog.logger.Sugar().Errorw("Failed to render 12_stakerOperatorsStaging query", "error", err) + sog.logger.Sugar().Errorw("Failed to render 12_stakerOperators query", "error", err) return err } res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 12_stakerOperatorsStaging", + sog.logger.Sugar().Errorw("Failed to generate 12_stakerOperators", zap.String("cutoffDate", cutoffDate), zap.Error(res.Error), ) diff --git a/pkg/rewards/stakerOperators/13_stakerOperator.go b/pkg/rewards/stakerOperators/13_stakerOperator.go deleted file mode 100644 index 6b29982e2..000000000 --- a/pkg/rewards/stakerOperators/13_stakerOperator.go +++ /dev/null @@ -1,96 +0,0 @@ -package stakerOperators - -import ( - "time" - - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" - "go.uber.org/zap" -) - -const _13_stakerOperator = ` -insert into {{.destTableName}} ( - earner, - operator, - reward_type, - avs, - token, - strategy, - multiplier, - shares, - amount, - reward_hash, - snapshot, - operator_set_id -) -select - earner, - operator, - reward_type, - avs, - token, - strategy, - multiplier, - shares, - amount, - reward_hash, - snapshot, - operator_set_id -from {{.stakerOperatorStaging}} -on conflict on constraint uniq_staker_operator do nothing; -` - -type StakerOperator struct { - Earner string - Operator string - RewardType string - Avs string - Token string - Strategy string - Multiplier string - Shares string - Amount string - RewardHash string - Snapshot time.Time -} - -func (sog *StakerOperatorsGenerator) GenerateAndInsert13StakerOperator(cutoffDate string) error { - sog.logger.Sugar().Infow("Generating and inserting 13_stakerOperator", - zap.String("cutoffDate", cutoffDate), - ) - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := rewardsUtils.Sot_13_StakerOperatorTable - - sog.logger.Sugar().Infow("Generating 13_stakerOperator", - zap.String("destTableName", destTableName), - zap.String("cutoffDate", cutoffDate), - ) - - query, err := rewardsUtils.RenderQueryTemplate(_13_stakerOperator, map[string]interface{}{ - "destTableName": destTableName, - "stakerOperatorStaging": allTableNames[rewardsUtils.Sot_12_StakerOperatorStaging], - }) - if err != nil { - sog.logger.Sugar().Errorw("Failed to render 13_stakerOperator query", "error", err) - return err - } - - res := sog.db.Exec(query) - if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 13_stakerOperator", - zap.String("cutoffDate", cutoffDate), - zap.Error(res.Error), - ) - } - - return nil -} - -func (sog *StakerOperatorsGenerator) List13StakerOperator() ([]*StakerOperator, error) { - var rewards []*StakerOperator - res := sog.db.Model(&StakerOperator{}).Find(&rewards) - if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to list 13_stakerOperator", "error", res.Error) - return nil, res.Error - } - return rewards, nil -} diff --git a/pkg/rewards/stakerOperators/1_stakerStrategyPayouts.go b/pkg/rewards/stakerOperators/1_stakerStrategyPayouts.go index 5fb22c52d..ec11ff77b 100644 --- a/pkg/rewards/stakerOperators/1_stakerStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/1_stakerStrategyPayouts.go @@ -2,9 +2,12 @@ package stakerOperators import ( "database/sql" + "fmt" + "time" + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" - "time" + "go.uber.org/zap" ) const _1_stakerStrategyPayoutsQuery = ` @@ -124,32 +127,29 @@ type StakerStrategyPayout struct { StakerStrategyTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert1StakerStrategyPayouts(cutoffDate string, forks config.ForkMap) error { - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_1_StakerStrategyPayouts] +func (sog *StakerOperatorsGenerator) GenerateAndInsert1StakerStrategyPayouts(cutoffDate string, forks config.ForkMap, generatedRewardsSnapshotId uint64) error { + destTableName := sog.getTempStakerStrategyPayoutsTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 1_stakerStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 1_stakerStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempStakerStrategyPayoutsTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp staker strategy payouts table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards], - rewardsUtils.Table_2_StakerRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_2_StakerRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + // Use temp tables from gold rewards + tempActiveRewardsTable := sog.getTempActiveRewardsTableName(cutoffDate, generatedRewardsSnapshotId) + tempStakerRewardAmountsTable := sog.getTempStakerRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_1_stakerStrategyPayoutsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards], - "stakerRewardAmountsTable": rewardsTables[rewardsUtils.Table_2_StakerRewardAmounts], + "destTableName": destTableName, + "activeRewardsTable": tempActiveRewardsTable, + "stakerRewardAmountsTable": tempStakerRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 1_stakerStrategyPayouts query", "error", err) @@ -162,8 +162,39 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert1StakerStrategyPayouts(cut ) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 1_stakerStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 1_stakerStrategyPayouts", "error", res.Error) + return res.Error + } + return nil +} + +func (sog *StakerOperatorsGenerator) getTempStakerStrategyPayoutsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_1_staker_strategy_payouts_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempStakerStrategyPayoutsTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempStakerStrategyPayoutsTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp staker strategy payouts table", "error", res.Error) + return res.Error } + sog.logger.Sugar().Infow("Successfully dropped temp staker strategy payouts table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) return nil } + +func (rc *StakerOperatorsGenerator) getTempActiveRewardsTableName(snapshotDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(snapshotDate) + return fmt.Sprintf("tmp_rewards_gold_1_active_rewards_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) getTempStakerRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_2_staker_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/2_operatorStrategyRewards.go b/pkg/rewards/stakerOperators/2_operatorStrategyRewards.go index 61a7e034b..7d12caa01 100644 --- a/pkg/rewards/stakerOperators/2_operatorStrategyRewards.go +++ b/pkg/rewards/stakerOperators/2_operatorStrategyRewards.go @@ -1,8 +1,12 @@ package stakerOperators import ( - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "fmt" "time" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _2_operatorStrategyRewardsQuery = ` @@ -98,32 +102,29 @@ type OperatorStrategyRewards struct { Shares string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert2OperatorStrategyRewards(cutoffDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_2_OperatorStrategyPayouts] +func (sog *StakerOperatorsGenerator) GenerateAndInsert2OperatorStrategyRewards(cutoffDate string, generatedRewardsSnapshotId uint64) error { + destTableName := sog.getTempOperatorStrategyRewardsTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 2_operatorStrategyRewards", + sog.logger.Sugar().Infow("Generating temp 2_operatorStrategyRewards", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempOperatorStrategyRewardsTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp operator strategy rewards table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards], - rewardsUtils.Table_3_OperatorRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_3_OperatorRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + // Use temp tables from gold rewards + tempActiveRewardsTable := sog.getTempActiveRewardsTableName(cutoffDate, generatedRewardsSnapshotId) + tempOperatorRewardAmountsTable := sog.getTempOperatorRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_2_operatorStrategyRewardsQuery, map[string]interface{}{ "destTableName": destTableName, - "activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards], - "operatorRewardAmountsTable": rewardsTables[rewardsUtils.Table_3_OperatorRewardAmounts], + "activeRewardsTable": tempActiveRewardsTable, + "operatorRewardAmountsTable": tempOperatorRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 2_operatorStrategyRewards query", "error", err) @@ -132,8 +133,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert2OperatorStrategyRewards(c res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to create 2_operatorStrategyRewards", "error", res.Error) + sog.logger.Sugar().Errorw("Failed to create temp 2_operatorStrategyRewards", "error", res.Error) return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempOperatorStrategyRewardsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_2_operator_strategy_rewards_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempOperatorStrategyRewardsTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempOperatorStrategyRewardsTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp operator strategy rewards table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp operator strategy rewards table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempOperatorRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_3_operator_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/3_rewardsForAllStrategyPayouts.go b/pkg/rewards/stakerOperators/3_rewardsForAllStrategyPayouts.go index 567597875..32dd43672 100644 --- a/pkg/rewards/stakerOperators/3_rewardsForAllStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/3_rewardsForAllStrategyPayouts.go @@ -1,8 +1,12 @@ package stakerOperators import ( - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "fmt" "time" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _3_rewardsForAllStrategyPayoutsQuery = ` @@ -68,30 +72,27 @@ type RewardsForAllStrategyPayout struct { StakerStrategyTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert3RewardsForAllStrategyPayout(cutoffDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_3_RewardsForAllStrategyPayout] +func (sog *StakerOperatorsGenerator) GenerateAndInsert3RewardsForAllStrategyPayout(cutoffDate string, generatedRewardsSnapshotId uint64) error { + destTableName := sog.getTempRewardsForAllStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 3_rewardsForAllStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 3_rewardsForAllStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempRewardsForAllStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp rewards for all strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + // Use temp tables from gold rewards + tempActiveRewardsTable := sog.getTempActiveRewardsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_3_rewardsForAllStrategyPayoutsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards], + "destTableName": destTableName, + "activeRewardsTable": tempActiveRewardsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 3_rewardsForAllStrategyPayouts query", "error", err) @@ -101,8 +102,8 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert3RewardsForAllStrategyPayo res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 3_rewardsForAllStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 3_rewardsForAllStrategyPayouts", "error", res.Error) + return res.Error } return nil } @@ -116,3 +117,24 @@ func (sog *StakerOperatorsGenerator) List3RewardsForAllStrategyPayout() ([]*Rewa } return rewards, nil } + +func (sog *StakerOperatorsGenerator) getTempRewardsForAllStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_3_rewards_for_all_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempRewardsForAllStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempRewardsForAllStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp rewards for all strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp rewards for all strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} diff --git a/pkg/rewards/stakerOperators/4_rfaeStakerStrategyPayouts.go b/pkg/rewards/stakerOperators/4_rfaeStakerStrategyPayouts.go index fbf74ec94..ca11da8c1 100644 --- a/pkg/rewards/stakerOperators/4_rfaeStakerStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/4_rfaeStakerStrategyPayouts.go @@ -1,9 +1,12 @@ package stakerOperators import ( + "fmt" + "time" + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" - "time" + "go.uber.org/zap" ) const _4_rfaeStakerStrategyPayoutsQuery = ` @@ -110,32 +113,29 @@ type RfaeStakerStrategyPayout struct { StakerStrategyTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert4RfaeStakerStrategyPayout(cutoffDate string, forks config.ForkMap) error { - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_4_RfaeStakers] +func (sog *StakerOperatorsGenerator) GenerateAndInsert4RfaeStakerStrategyPayout(cutoffDate string, forks config.ForkMap, generatedRewardsSnapshotId uint64) error { + destTableName := sog.getTempRfaeStakerStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 4_rfaeStakerStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 4_rfaeStakerStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempRfaeStakerStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp rfae staker strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards], - rewardsUtils.Table_5_RfaeStakers: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_5_RfaeStakers], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + // Use temp tables from gold rewards + tempActiveRewardsTable := sog.getTempActiveRewardsTableName(cutoffDate, generatedRewardsSnapshotId) + tempRfaeStakersTable := sog.getTempRfaeStakersTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_4_rfaeStakerStrategyPayoutsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards], - "rfaeStakersTable": rewardsTables[rewardsUtils.Table_5_RfaeStakers], + "destTableName": destTableName, + "activeRewardsTable": tempActiveRewardsTable, + "rfaeStakersTable": tempRfaeStakersTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 4_rfaeStakerStrategyPayouts query", "error", err) @@ -145,8 +145,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert4RfaeStakerStrategyPayout( res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 4_rfaeStakerStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 4_rfaeStakerStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempRfaeStakerStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_4_rfae_staker_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempRfaeStakerStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempRfaeStakerStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp rfae staker strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp rfae staker strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempRfaeStakersTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_5_rfae_stakers_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/5_rfaeOperatorStrategyPayout.go b/pkg/rewards/stakerOperators/5_rfaeOperatorStrategyPayout.go index a290bbcc4..367e95618 100644 --- a/pkg/rewards/stakerOperators/5_rfaeOperatorStrategyPayout.go +++ b/pkg/rewards/stakerOperators/5_rfaeOperatorStrategyPayout.go @@ -1,8 +1,12 @@ package stakerOperators import ( - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "fmt" "time" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _5_rfaeOperatorStrategyPayoutsQuery = ` @@ -95,32 +99,29 @@ type RfaeOperatorStrategyPayout struct { OperatorStrategyTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert5RfaeOperatorStrategyPayout(cutoffDate string) error { - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_5_RfaeOperators] +func (sog *StakerOperatorsGenerator) GenerateAndInsert5RfaeOperatorStrategyPayout(cutoffDate string, generatedRewardsSnapshotId uint64) error { + destTableName := sog.getTempRfaeOperatorStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 5_rfaeOperatorStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 5_rfaeOperatorStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempRfaeOperatorStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp rfae operator strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_1_ActiveRewards: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_1_ActiveRewards], - rewardsUtils.Table_6_RfaeOperators: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_6_RfaeOperators], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + // Use temp tables from gold rewards + tempActiveRewardsTable := sog.getTempActiveRewardsTableName(cutoffDate, generatedRewardsSnapshotId) + tempRfaeOperatorsTable := sog.getTempRfaeOperatorsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_5_rfaeOperatorStrategyPayoutsQuery, map[string]interface{}{ - "destTableName": destTableName, - "activeRewardsTable": rewardsTables[rewardsUtils.Table_1_ActiveRewards], - "rfaeOperatorsTable": rewardsTables[rewardsUtils.Table_6_RfaeOperators], + "destTableName": destTableName, + "activeRewardsTable": tempActiveRewardsTable, + "rfaeOperatorsTable": tempRfaeOperatorsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 5_rfaeOperatorStrategyPayouts query", "error", err) @@ -130,8 +131,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert5RfaeOperatorStrategyPayou res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 5_rfaeOperatorStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 5_rfaeOperatorStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempRfaeOperatorStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_5_rfae_operator_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempRfaeOperatorStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempRfaeOperatorStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp rfae operator strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp rfae operator strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempRfaeOperatorsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_6_rfae_operators_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/6_operatorODStrategyPayouts.go b/pkg/rewards/stakerOperators/6_operatorODStrategyPayouts.go index 22283c96c..460c748f8 100644 --- a/pkg/rewards/stakerOperators/6_operatorODStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/6_operatorODStrategyPayouts.go @@ -1,8 +1,12 @@ package stakerOperators import ( - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "fmt" "time" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) // _6_operatorODStrategyPayoutQuery is the query that generates the operator OD strategy payouts. @@ -39,7 +43,7 @@ type OperatorODStrategyPayout struct { OperatorTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert6OperatorODStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert6OperatorODStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := sog.globalConfig.IsRewardsV2EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -49,29 +53,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert6OperatorODStrategyPayouts sog.logger.Sugar().Infow("Skipping 6_operatorODStrategyPayouts generation as rewards v2 is not enabled") return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_6_OperatorODStrategyPayouts] + destTableName := sog.getTempOperatorODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 6_operatorODStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 6_operatorODStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempOperatorODStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp operator OD strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_8_OperatorODRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_8_OperatorODRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempOperatorODRewardAmountsTable := sog.getTempOperatorODRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_6_operatorODStrategyPayoutQuery, map[string]interface{}{ "destTableName": destTableName, - "operatorODRewardAmountsTable": rewardsTables[rewardsUtils.Table_8_OperatorODRewardAmounts], + "operatorODRewardAmountsTable": tempOperatorODRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 6_operatorODStrategyPayouts query", "error", err) @@ -81,8 +81,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert6OperatorODStrategyPayouts res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 6_operatorODStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 6_operatorODStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempOperatorODStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_6_operator_od_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempOperatorODStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempOperatorODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp operator OD strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp operator OD strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempOperatorODRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_8_operator_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/7_stakerODStrategyPayouts.go b/pkg/rewards/stakerOperators/7_stakerODStrategyPayouts.go index 881067fb9..8c7adc257 100644 --- a/pkg/rewards/stakerOperators/7_stakerODStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/7_stakerODStrategyPayouts.go @@ -1,8 +1,12 @@ package stakerOperators import ( - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "fmt" "time" + + "github.com/Layr-Labs/sidecar/internal/config" + "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) // _7_stakerODStrategyPayoutQuery is a constant value that represents a query. @@ -39,7 +43,7 @@ type StakerODStrategyPayout struct { Snapshot time.Time } -func (sog *StakerOperatorsGenerator) GenerateAndInsert7StakerODStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert7StakerODStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := sog.globalConfig.IsRewardsV2EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -50,29 +54,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert7StakerODStrategyPayouts(c return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_7_StakerODStrategyPayouts] + destTableName := sog.getTempStakerODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 7_stakerODStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 7_stakerODStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempStakerODStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp staker OD strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_9_StakerODRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_9_StakerODRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempStakerODRewardAmountsTable := sog.getTempStakerODRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_7_stakerODStrategyPayoutQuery, map[string]interface{}{ "destTableName": destTableName, - "stakerODRewardAmountsTable": rewardsTables[rewardsUtils.Table_9_StakerODRewardAmounts], + "stakerODRewardAmountsTable": tempStakerODRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 7_stakerODStrategyPayouts query", "error", err) @@ -82,8 +82,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert7StakerODStrategyPayouts(c res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 7_stakerODStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 7_stakerODStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempStakerODStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_7_staker_od_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempStakerODStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempStakerODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp staker OD strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp staker OD strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempStakerODRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_9_staker_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/8_avsODStrategyPayouts.go b/pkg/rewards/stakerOperators/8_avsODStrategyPayouts.go index 51b55dd84..3ee8e4f49 100644 --- a/pkg/rewards/stakerOperators/8_avsODStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/8_avsODStrategyPayouts.go @@ -1,7 +1,11 @@ package stakerOperators import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) // _8_avsODStrategyPayoutQuery is the query that generates the 8_avsODStrategyPayouts table @@ -32,7 +36,7 @@ type AvsODStrategyPayout struct { AvsTokens string } -func (sog *StakerOperatorsGenerator) GenerateAndInsert8AvsODStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert8AvsODStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2Enabled, err := sog.globalConfig.IsRewardsV2EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2 is enabled", "error", err) @@ -43,29 +47,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert8AvsODStrategyPayouts(cuto return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_8_AvsODStrategyPayouts] + destTableName := sog.getTempAvsODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 8_avsODStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 8_avsODStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempAvsODStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp AVS OD strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_10_AvsODRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_10_AvsODRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempAvsODRewardAmountsTable := sog.getTempAvsODRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_8_avsODStrategyPayoutQuery, map[string]interface{}{ - "destTableName": destTableName, - "avsODRewardAmountsTable": rewardsTables[rewardsUtils.Table_10_AvsODRewardAmounts], + "destTableName": destTableName, + "avsODRewardAmountsTable": tempAvsODRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 8_avsODStrategyPayouts query", "error", err) @@ -75,8 +75,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert8AvsODStrategyPayouts(cuto res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 8_avsODStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 8_avsODStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempAvsODStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_8_avs_od_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempAvsODStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempAvsODStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp AVS OD strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp AVS OD strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempAvsODRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_10_avs_od_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/9_operatorODOperatorSetStrategyPayouts.go b/pkg/rewards/stakerOperators/9_operatorODOperatorSetStrategyPayouts.go index 2d3faa0c5..b8c84289a 100644 --- a/pkg/rewards/stakerOperators/9_operatorODOperatorSetStrategyPayouts.go +++ b/pkg/rewards/stakerOperators/9_operatorODOperatorSetStrategyPayouts.go @@ -1,7 +1,11 @@ package stakerOperators import ( + "fmt" + + "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" + "go.uber.org/zap" ) const _9_operatorODOperatorSetStrategyPayoutQuery = ` @@ -21,7 +25,7 @@ select from {{.operatorODOperatorSetRewardAmountsTable}} ` -func (sog *StakerOperatorsGenerator) GenerateAndInsert9OperatorODOperatorSetStrategyPayouts(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateAndInsert9OperatorODOperatorSetStrategyPayouts(cutoffDate string, generatedRewardsSnapshotId uint64) error { rewardsV2_1Enabled, err := sog.globalConfig.IsRewardsV2_1EnabledForCutoffDate(cutoffDate) if err != nil { sog.logger.Sugar().Errorw("Failed to check if rewards v2.1 is enabled", "error", err) @@ -32,29 +36,25 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert9OperatorODOperatorSetStra return nil } - allTableNames := rewardsUtils.GetGoldTableNames(cutoffDate) - destTableName := allTableNames[rewardsUtils.Sot_9_OperatorODOperatorSetStrategyPayouts] + destTableName := sog.getTempOperatorODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) - sog.logger.Sugar().Infow("Generating and inserting 9_operatorODOperatorSetStrategyPayouts", + sog.logger.Sugar().Infow("Generating temp 9_operatorODOperatorSetStrategyPayouts", "cutoffDate", cutoffDate, + "destTableName", destTableName, ) - if err := rewardsUtils.DropTableIfExists(sog.db, destTableName, sog.logger); err != nil { - sog.logger.Sugar().Errorw("Failed to drop table", "error", err) + // Drop existing temp table + if err := sog.DropTempOperatorODOperatorSetStrategyPayoutTable(cutoffDate, generatedRewardsSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to drop existing temp operator OD operator set strategy payout table", "error", err) return err } - rewardsTables, err := sog.FindRewardsTableNamesForSearchPattersn(map[string]string{ - rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts: rewardsUtils.GoldTableNameSearchPattern[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts], - }, cutoffDate) - if err != nil { - sog.logger.Sugar().Errorw("Failed to find staker operator table names", "error", err) - return err - } + tempOperatorODOperatorSetRewardAmountsTable := sog.getTempOperatorODOperatorSetRewardAmountsTableName(cutoffDate, generatedRewardsSnapshotId) query, err := rewardsUtils.RenderQueryTemplate(_9_operatorODOperatorSetStrategyPayoutQuery, map[string]interface{}{ "destTableName": destTableName, - "operatorODOperatorSetRewardAmountsTable": rewardsTables[rewardsUtils.Table_12_OperatorODOperatorSetRewardAmounts], + "operatorODOperatorSetRewardAmountsTable": tempOperatorODOperatorSetRewardAmountsTable, + "generatedRewardsSnapshotId": generatedRewardsSnapshotId, }) if err != nil { sog.logger.Sugar().Errorw("Failed to render 9_operatorODOperatorSetStrategyPayouts query", "error", err) @@ -64,8 +64,34 @@ func (sog *StakerOperatorsGenerator) GenerateAndInsert9OperatorODOperatorSetStra res := sog.db.Exec(query) if res.Error != nil { - sog.logger.Sugar().Errorw("Failed to generate 9_operatorODOperatorSetStrategyPayouts", "error", res.Error) - return err + sog.logger.Sugar().Errorw("Failed to generate temp 9_operatorODOperatorSetStrategyPayouts", "error", res.Error) + return res.Error } return nil } + +func (sog *StakerOperatorsGenerator) getTempOperatorODOperatorSetStrategyPayoutTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_staker_operators_9_operator_od_operator_set_strategy_payout_%s_%d", camelDate, generatedRewardSnapshotId) +} + +func (sog *StakerOperatorsGenerator) DropTempOperatorODOperatorSetStrategyPayoutTable(cutoffDate string, generatedRewardsSnapshotId uint64) error { + tempTableName := sog.getTempOperatorODOperatorSetStrategyPayoutTableName(cutoffDate, generatedRewardsSnapshotId) + + query := fmt.Sprintf("DROP TABLE IF EXISTS %s", tempTableName) + res := sog.db.Exec(query) + if res.Error != nil { + sog.logger.Sugar().Errorw("Failed to drop temp operator OD operator set strategy payout table", "error", res.Error) + return res.Error + } + sog.logger.Sugar().Infow("Successfully dropped temp operator OD operator set strategy payout table", + zap.String("tempTableName", tempTableName), + zap.Uint64("generatedRewardsSnapshotId", generatedRewardsSnapshotId), + ) + return nil +} + +func (sog *StakerOperatorsGenerator) getTempOperatorODOperatorSetRewardAmountsTableName(cutoffDate string, generatedRewardSnapshotId uint64) string { + camelDate := config.KebabToSnakeCase(cutoffDate) + return fmt.Sprintf("tmp_rewards_gold_12_operator_od_operator_set_reward_amounts_%s_%d", camelDate, generatedRewardSnapshotId) +} diff --git a/pkg/rewards/stakerOperators/stakerOperator.go b/pkg/rewards/stakerOperators/stakerOperator.go index 97559dcdb..762f7b364 100644 --- a/pkg/rewards/stakerOperators/stakerOperator.go +++ b/pkg/rewards/stakerOperators/stakerOperator.go @@ -2,7 +2,6 @@ package stakerOperators import ( "github.com/Layr-Labs/sidecar/internal/config" - "github.com/Layr-Labs/sidecar/pkg/rewardsUtils" "go.uber.org/zap" "gorm.io/gorm" ) @@ -25,7 +24,7 @@ func NewStakerOperatorGenerator( } } -func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate string) error { +func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate string, generatedSnapshotId uint64) error { forks, err := sog.globalConfig.GetRewardsSqlForkDates() if err != nil { return err @@ -39,7 +38,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str } sog.logger.Sugar().Infow("Generating staker operators table", zap.String("cutoffDate", cutoffDate)) - if err := sog.GenerateAndInsert1StakerStrategyPayouts(cutoffDate, forks); err != nil { + if err := sog.GenerateAndInsert1StakerStrategyPayouts(cutoffDate, forks, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 1 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -47,7 +46,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert2OperatorStrategyRewards(cutoffDate); err != nil { + if err := sog.GenerateAndInsert2OperatorStrategyRewards(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 2 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -55,7 +54,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert3RewardsForAllStrategyPayout(cutoffDate); err != nil { + if err := sog.GenerateAndInsert3RewardsForAllStrategyPayout(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 3 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -63,7 +62,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert4RfaeStakerStrategyPayout(cutoffDate, forks); err != nil { + if err := sog.GenerateAndInsert4RfaeStakerStrategyPayout(cutoffDate, forks, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 4 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -71,7 +70,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert5RfaeOperatorStrategyPayout(cutoffDate); err != nil { + if err := sog.GenerateAndInsert5RfaeOperatorStrategyPayout(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 5 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -79,7 +78,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert6OperatorODStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert6OperatorODStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 6 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -87,7 +86,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert7StakerODStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert7StakerODStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 7 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -95,7 +94,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert8AvsODStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert8AvsODStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 8 staker strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -103,7 +102,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert9OperatorODOperatorSetStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert9OperatorODOperatorSetStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 9 operator OD operator set strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -111,7 +110,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert10StakerODOperatorSetStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert10StakerODOperatorSetStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 10 staker OD operator set strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -119,7 +118,7 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert11AvsODOperatorSetStrategyPayouts(cutoffDate); err != nil { + if err := sog.GenerateAndInsert11AvsODOperatorSetStrategyPayouts(cutoffDate, generatedSnapshotId); err != nil { sog.logger.Sugar().Errorw("Failed to generate and insert 11 AVS OD operator set strategy rewards", zap.String("cutoffDate", cutoffDate), zap.Error(err), @@ -127,24 +126,13 @@ func (sog *StakerOperatorsGenerator) GenerateStakerOperatorsTable(cutoffDate str return err } - if err := sog.GenerateAndInsert12StakerOperatorStaging(cutoffDate); err != nil { - sog.logger.Sugar().Errorw("Failed to generate and insert 12 staker operator staging", + if err := sog.GenerateAndInsert12StakerOperator(cutoffDate, generatedSnapshotId); err != nil { + sog.logger.Sugar().Errorw("Failed to generate and insert 12 staker operator", zap.String("cutoffDate", cutoffDate), zap.Error(err), ) return err } - if err := sog.GenerateAndInsert13StakerOperator(cutoffDate); err != nil { - sog.logger.Sugar().Errorw("Failed to generate and insert 13 staker operator", - zap.String("cutoffDate", cutoffDate), - zap.Error(err), - ) - return err - } return nil } - -func (sog *StakerOperatorsGenerator) FindRewardsTableNamesForSearchPattersn(patterns map[string]string, cutoffDate string) (map[string]string, error) { - return rewardsUtils.FindRewardsTableNamesForSearchPatterns(patterns, cutoffDate, sog.globalConfig.DatabaseConfig.SchemaName, sog.db) -} diff --git a/pkg/rewards/stakerShareSnapshots.go b/pkg/rewards/stakerShareSnapshots.go index 68abf3e66..66526b6e8 100644 --- a/pkg/rewards/stakerShareSnapshots.go +++ b/pkg/rewards/stakerShareSnapshots.go @@ -7,12 +7,20 @@ import ( const stakerShareSnapshotsQuery = ` insert into staker_share_snapshots(staker, strategy, shares, snapshot) - WITH ranked_staker_records as ( + with recent_max_block as ( + select + coalesce(max(snapshot), '1970-01-01') as snapshot + from staker_share_snapshots + where snapshot < TIMESTAMP '{{.cutoffDate}}' + ), + ranked_staker_records as ( SELECT *, ROW_NUMBER() OVER (PARTITION BY staker, strategy, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn FROM staker_shares -- pipeline bronze table uses this to filter the correct records - where block_time < TIMESTAMP '{{.cutoffDate}}' + where + block_time < TIMESTAMP '{{.cutoffDate}}' + and block_time >= (select snapshot from recent_max_block) ), -- Get the latest record for each day & round up to the snapshot day snapshotted_records as ( diff --git a/pkg/rewards/stakerShares.go b/pkg/rewards/stakerShares.go index b884bd3f5..55b1ab7e3 100644 --- a/pkg/rewards/stakerShares.go +++ b/pkg/rewards/stakerShares.go @@ -7,6 +7,9 @@ import ( const stakerSharesQuery = ` insert into staker_shares(staker, strategy, shares, transaction_hash, log_index, strategy_index, block_time, block_date, block_number) + with recent_max_block as ( + select coalesce(max(block_number), 0) as block_number from staker_shares + ) select staker, strategy, @@ -19,7 +22,11 @@ const stakerSharesQuery = ` block_date, block_number from staker_share_deltas - where block_date < '{{.cutoffDate}}' + where + block_date < '{{.cutoffDate}}' + -- only need to fetch new records since the last snapshot . + -- we use a >= in case not all records are inserted for the MAX(block_number) + and block_number >= (select block_number from recent_max_block) on conflict on constraint uniq_staker_shares do nothing; ` diff --git a/pkg/rewardsUtils/rewardsUtils.go b/pkg/rewardsUtils/rewardsUtils.go index f25116268..85359ebd6 100644 --- a/pkg/rewardsUtils/rewardsUtils.go +++ b/pkg/rewardsUtils/rewardsUtils.go @@ -12,22 +12,22 @@ import ( ) var ( - Table_1_ActiveRewards = "gold_1_active_rewards" - Table_2_StakerRewardAmounts = "gold_2_staker_reward_amounts" - Table_3_OperatorRewardAmounts = "gold_3_operator_reward_amounts" - Table_4_RewardsForAll = "gold_4_rewards_for_all" - Table_5_RfaeStakers = "gold_5_rfae_stakers" - Table_6_RfaeOperators = "gold_6_rfae_operators" - Table_7_ActiveODRewards = "gold_7_active_od_rewards" - Table_8_OperatorODRewardAmounts = "gold_8_operator_od_reward_amounts" - Table_9_StakerODRewardAmounts = "gold_9_staker_od_reward_amounts" - Table_10_AvsODRewardAmounts = "gold_10_avs_od_reward_amounts" - Table_11_ActiveODOperatorSetRewards = "gold_11_active_od_operator_set_rewards" - Table_12_OperatorODOperatorSetRewardAmounts = "gold_12_operator_od_operator_set_reward_amounts" - Table_13_StakerODOperatorSetRewardAmounts = "gold_13_staker_od_operator_set_reward_amounts" - Table_14_AvsODOperatorSetRewardAmounts = "gold_14_avs_od_operator_set_reward_amounts" - Table_15_GoldStaging = "gold_15_staging" - Table_16_GoldTable = "gold_table" + RewardsTable_1_ActiveRewards = "rewards_gold_1_active_rewards" + RewardsTable_2_StakerRewardAmounts = "rewards_gold_2_staker_reward_amounts" + RewardsTable_3_OperatorRewardAmounts = "rewards_gold_3_operator_reward_amounts" + RewardsTable_4_RewardsForAll = "rewards_gold_4_rewards_for_all" + RewardsTable_5_RfaeStakers = "rewards_gold_5_rfae_stakers" + RewardsTable_6_RfaeOperators = "rewards_gold_6_rfae_operators" + RewardsTable_7_ActiveODRewards = "rewards_gold_7_active_od_rewards" + RewardsTable_8_OperatorODRewardAmounts = "rewards_gold_8_operator_od_reward_amounts" + RewardsTable_9_StakerODRewardAmounts = "rewards_gold_9_staker_od_reward_amounts" + RewardsTable_10_AvsODRewardAmounts = "rewards_gold_10_avs_od_reward_amounts" + RewardsTable_11_ActiveODOperatorSetRewards = "rewards_gold_11_active_od_operator_set_rewards" + RewardsTable_12_OperatorODOperatorSetRewardAmounts = "rewards_gold_12_operator_od_operator_set_reward_amounts" + RewardsTable_13_StakerODOperatorSetRewardAmounts = "rewards_gold_13_staker_od_operator_set_reward_amounts" + RewardsTable_14_AvsODOperatorSetRewardAmounts = "rewards_gold_14_avs_od_operator_set_reward_amounts" + RewardsTable_GoldStaging = "rewards_gold_staging" + RewardsTable_GoldTable = "gold_table" Sot_1_StakerStrategyPayouts = "sot_1_staker_strategy_payouts" Sot_2_OperatorStrategyPayouts = "sot_2_operator_strategy_payouts" @@ -45,23 +45,6 @@ var ( ) var goldTableBaseNames = map[string]string{ - Table_1_ActiveRewards: Table_1_ActiveRewards, - Table_2_StakerRewardAmounts: Table_2_StakerRewardAmounts, - Table_3_OperatorRewardAmounts: Table_3_OperatorRewardAmounts, - Table_4_RewardsForAll: Table_4_RewardsForAll, - Table_5_RfaeStakers: Table_5_RfaeStakers, - Table_6_RfaeOperators: Table_6_RfaeOperators, - Table_7_ActiveODRewards: Table_7_ActiveODRewards, - Table_8_OperatorODRewardAmounts: Table_8_OperatorODRewardAmounts, - Table_9_StakerODRewardAmounts: Table_9_StakerODRewardAmounts, - Table_10_AvsODRewardAmounts: Table_10_AvsODRewardAmounts, - Table_11_ActiveODOperatorSetRewards: Table_11_ActiveODOperatorSetRewards, - Table_12_OperatorODOperatorSetRewardAmounts: Table_12_OperatorODOperatorSetRewardAmounts, - Table_13_StakerODOperatorSetRewardAmounts: Table_13_StakerODOperatorSetRewardAmounts, - Table_14_AvsODOperatorSetRewardAmounts: Table_14_AvsODOperatorSetRewardAmounts, - Table_15_GoldStaging: Table_15_GoldStaging, - Table_16_GoldTable: Table_16_GoldTable, - Sot_1_StakerStrategyPayouts: Sot_1_StakerStrategyPayouts, Sot_2_OperatorStrategyPayouts: Sot_2_OperatorStrategyPayouts, Sot_3_RewardsForAllStrategyPayout: Sot_3_RewardsForAllStrategyPayout, @@ -77,23 +60,25 @@ var goldTableBaseNames = map[string]string{ Sot_13_StakerOperatorTable: Sot_13_StakerOperatorTable, } -var GoldTableNameSearchPattern = map[string]string{ - Table_1_ActiveRewards: "gold_%_active_rewards", - Table_2_StakerRewardAmounts: "gold_%_staker_reward_amounts", - Table_3_OperatorRewardAmounts: "gold_%_operator_reward_amounts", - Table_4_RewardsForAll: "gold_%_rewards_for_all", - Table_5_RfaeStakers: "gold_%_rfae_stakers", - Table_6_RfaeOperators: "gold_%_rfae_operators", - Table_7_ActiveODRewards: "gold_%_active_od_rewards", - Table_8_OperatorODRewardAmounts: "gold_%_operator_od_reward_amounts", - Table_9_StakerODRewardAmounts: "gold_%_staker_od_reward_amounts", - Table_10_AvsODRewardAmounts: "gold_%_avs_od_reward_amounts", - Table_11_ActiveODOperatorSetRewards: "gold_%_active_od_operator_set_rewards", - Table_12_OperatorODOperatorSetRewardAmounts: "gold_%_operator_od_operator_set_reward_amounts", - Table_13_StakerODOperatorSetRewardAmounts: "gold_%_staker_od_operator_set_reward_amounts", - Table_14_AvsODOperatorSetRewardAmounts: "gold_%_avs_od_operator_set_reward_amounts", - Table_15_GoldStaging: "gold_%_staging", +var RewardsTableBaseNames = []string{ + RewardsTable_1_ActiveRewards, + RewardsTable_2_StakerRewardAmounts, + RewardsTable_3_OperatorRewardAmounts, + RewardsTable_4_RewardsForAll, + RewardsTable_5_RfaeStakers, + RewardsTable_6_RfaeOperators, + RewardsTable_7_ActiveODRewards, + RewardsTable_8_OperatorODRewardAmounts, + RewardsTable_9_StakerODRewardAmounts, + RewardsTable_10_AvsODRewardAmounts, + RewardsTable_11_ActiveODOperatorSetRewards, + RewardsTable_12_OperatorODOperatorSetRewardAmounts, + RewardsTable_13_StakerODOperatorSetRewardAmounts, + RewardsTable_14_AvsODOperatorSetRewardAmounts, + RewardsTable_GoldTable, +} +var GoldTableNameSearchPattern = map[string]string{ Sot_1_StakerStrategyPayouts: "sot_%_staker_strategy_payouts", Sot_2_OperatorStrategyPayouts: "sot_%_operator_strategy_payouts", Sot_3_RewardsForAllStrategyPayout: "sot_%_rewards_for_all_strategy_payout", @@ -126,6 +111,10 @@ func RenderQueryTemplate(query string, variables map[string]interface{}) (string return dest.String(), nil } +func FormatTableNameWithGeneratedSnaphotId(tableName string, generatedSnapshotId uint64) string { + return fmt.Sprintf("%s__%d", tableName, generatedSnapshotId) +} + func FormatTableName(tableName string, snapshotDate string) string { return fmt.Sprintf("%s_%s", tableName, utils.SnakeCase(snapshotDate)) } diff --git a/pkg/sidecar/blockIndexer.go b/pkg/sidecar/blockIndexer.go index 2d6c97760..28c1e1a8d 100644 --- a/pkg/sidecar/blockIndexer.go +++ b/pkg/sidecar/blockIndexer.go @@ -3,10 +3,11 @@ package sidecar import ( "context" "fmt" - ddTracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "sync/atomic" "time" + ddTracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "github.com/Layr-Labs/sidecar/internal/config" "github.com/syndtr/goleveldb/leveldb/errors" diff --git a/scripts/downloadTestData.sh b/scripts/downloadTestData.sh index 0951bfbbe..d3ab961f8 100755 --- a/scripts/downloadTestData.sh +++ b/scripts/downloadTestData.sh @@ -6,7 +6,7 @@ if [[ -z $NETWORK ]]; then exit 1 fi -version=$(cat .testdataVersion) +version=$(cat .testdataVersion | jq -r '.["'$NETWORK'"]') bucketName="eigenlayer-sidecar-testdata" dataUrl="https://${bucketName}.s3.amazonaws.com/${NETWORK}/${version}.tar"