fetch_erc20_metadata.js

const { ethers } = require('ethers');
const fs = require('fs');
const path = require('path');

// ERC20 ABI for the functions we need
const ERC20_ABI = [
  'function name() view returns (string)',
  'function symbol() view returns (string)',
  'function decimals() view returns (uint8)'
];

// Multicall3 contract ABI with tryAggregate for handling individual failures
const MULTICALL3_ABI = [
  'function tryAggregate(bool requireSuccess, (address target, bytes callData)[]) view returns ((bool success, bytes returnData)[])'
];

// Standard Multicall3 address on Ethereum mainnet
const MULTICALL3_ADDRESS = '0xcA11bde05977b3631167028862bE2a173976CA11';

// Configuration
const BATCH_SIZE = 10; // Smaller batch size for reliability
const RETRY_DELAY = 100; // Delay between batches

// Fallback RPC URLs (will try in order)
const RPC_URLS = [
  process.env.RPC_URL || 'https://ethereum.publicnode.com',
  'https://cloudflare-eth.com',
  'https://eth.llamarpc.com'
];
const INPUT_FILE = path.join(__dirname, 'erc20_with_blocks.json');
const OUTPUT_FILE = path.join(__dirname, 'erc20_metadata.json');
const FAILED_FILE = path.join(__dirname, 'erc20_metadata_failed.json');

// Create interface for ERC20
const erc20Interface = new ethers.Interface(ERC20_ABI);

// Load already processed addresses from output file
function loadProcessedAddresses() {
  const processed = new Set();
  if (fs.existsSync(OUTPUT_FILE)) {
    const content = fs.readFileSync(OUTPUT_FILE, 'utf-8');
    const lines = content.trim().split('\n').filter(l => l);
    for (const line of lines) {
      try {
        const entry = JSON.parse(line);
        processed.add(entry.address.toLowerCase());
      } catch (e) { /* ignore */ }
    }
  }
  return processed;
}

// Append a batch of results to the output file
function appendResults(metadataList) {
  const jsonlContent = metadataList.map(m => JSON.stringify(m)).join('\n') + '\n';
  fs.appendFileSync(OUTPUT_FILE, jsonlContent);
}

// Append failed batch addresses to the failed file for later retry
function appendFailed(addresses, contractsMap) {
  const failedData = addresses.map(addr => ({
    address: addr,
    number: contractsMap[addr.toLowerCase()]
  }));
  const jsonlContent = failedData.map(m => JSON.stringify(m)).join('\n') + '\n';
  fs.appendFileSync(FAILED_FILE, jsonlContent);
}

async function fetchMetadataWithMulticall3(provider, addresses, contractsMap) {
  const multicall = new ethers.Contract(MULTICALL3_ADDRESS, MULTICALL3_ABI, provider);
  
  // Prepare calls for name, symbol, decimals
  const calls = [];
  for (const address of addresses) {
    calls.push({
      target: address,
      callData: erc20Interface.encodeFunctionData('name', [])
    });
  }
  for (const address of addresses) {
    calls.push({
      target: address,
      callData: erc20Interface.encodeFunctionData('symbol', [])
    });
  }
  for (const address of addresses) {
    calls.push({
      target: address,
      callData: erc20Interface.encodeFunctionData('decimals', [])
    });
  }

  try {
    // Use tryAggregate which handles individual call failures (false = don't require all to succeed)
    const results = await multicall.tryAggregate.staticCall(false, calls);

    const metadataList = [];
    for (let i = 0; i < addresses.length; i++) {
      const address = addresses[i];
      const baseIdx = i;
      const nameIdx = baseIdx;
      const symbolIdx = baseIdx + addresses.length;
      const decimalsIdx = baseIdx + addresses.length * 2;

      // Safely decode each result - handle failures gracefully
      let name = null, symbol = null, decimals = null;
      
      // name
      try {
        const nameResult = results[nameIdx];
        if (nameResult && nameResult.success && nameResult.returnData) {
          const decoded = erc20Interface.decodeFunctionResult('name', nameResult.returnData)[0];
          if (decoded && decoded.length > 0) name = decoded;
        }
      } catch (e) { /* name call failed, keep null */ }
      
      // symbol
      try {
        const symbolResult = results[symbolIdx];
        if (symbolResult && symbolResult.success && symbolResult.returnData) {
          const decoded = erc20Interface.decodeFunctionResult('symbol', symbolResult.returnData)[0];
          if (decoded && decoded.length > 0) symbol = decoded;
        }
      } catch (e) { /* symbol call failed, keep null */ }
      
      // decimals
      try {
        const decimalsResult = results[decimalsIdx];
        if (decimalsResult && decimalsResult.success && decimalsResult.returnData) {
          decimals = erc20Interface.decodeFunctionResult('decimals', decimalsResult.returnData)[0];
        }
      } catch (e) { /* decimals call failed, keep null */ }

      metadataList.push({
        address: address,
        number: contractsMap[address.toLowerCase()],
        name,
        symbol,
        decimals: decimals !== null ? Number(decimals) : null
      });
    }

    return metadataList;
  } catch (error) {
    console.error('Multicall3 tryAggregate error:', error.message);
    // Return no values for all addresses so they get rechecked on next run
    return [];
  }
}

async function main() {
  console.log('Starting ERC20 metadata fetch...');
  console.log(`Input file: ${INPUT_FILE}`);
  console.log(`Output file: ${OUTPUT_FILE}`);
  console.log(`Failed file: ${FAILED_FILE}`);

  // Load already processed addresses
  const processedAddresses = loadProcessedAddresses();
  console.log(`Already processed: ${processedAddresses.size} addresses`);

  // Read input file
  if (!fs.existsSync(INPUT_FILE)) {
    console.error(`Input file not found: ${INPUT_FILE}`);
    process.exit(1);
  }

  const fileContent = fs.readFileSync(INPUT_FILE, 'utf-8');
  const lines = fileContent.trim().split('\n');
  
  // Parse addresses from JSONL and create contracts map
  const contracts = lines.map(line => {
    try {
      return JSON.parse(line);
    } catch (e) {
      console.error('Error parsing line:', line);
      return null;
    }
  }).filter(c => c !== null);

  // Create address -> number mapping (keep first occurrence)
  const contractsMap = {};
  for (const contract of contracts) {
    const addr = contract.address.toLowerCase();
    if (!contractsMap[addr]) {
      contractsMap[addr] = contract.number;
    }
  }

  // Filter out already processed addresses
  const uniqueAddresses = Object.keys(contractsMap).filter(addr => !processedAddresses.has(addr));
  
  console.log(`Total contracts: ${contracts.length}`);
  console.log(`Unique addresses: ${uniqueAddresses.length}`);
  console.log(`Remaining to process: ${uniqueAddresses.length}`);

  if (uniqueAddresses.length === 0) {
    console.log('All contracts already processed!');
    return;
  }

  // Create provider with fallback URLs
  let provider;
  let connectedUrl = null;
  
  for (const url of RPC_URLS) {
    try {
      const testProvider = new ethers.JsonRpcProvider(url);
      const network = await testProvider.getNetwork();
      provider = testProvider;
      connectedUrl = url;
      console.log(`Connected to network: ${network.name} (chainId: ${network.chainId}) using ${url}`);
      break;
    } catch (error) {
      console.log(`Failed to connect to ${url}: ${error.message}`);
    }
  }
  
  if (!provider) {
    console.error('Failed to connect to any RPC endpoint');
    process.exit(1);
  }

  // Process in batches with retry logic
  let processed = 0;
  const maxRetries = 3;
  const retryDelay = 1000;
  const totalToProcess = uniqueAddresses.length;

  for (let i = 0; i < uniqueAddresses.length; i += BATCH_SIZE) {
    const batch = uniqueAddresses.slice(i, i + BATCH_SIZE);
    const batchNum = Math.floor(i / BATCH_SIZE) + 1;
    const totalBatches = Math.ceil(totalToProcess / BATCH_SIZE);
    
    console.log(`Processing batch ${batchNum}/${totalBatches} (${batch.length} contracts)...`);
    
    let success = false;
    for (let retry = 0; retry < maxRetries && !success; retry++) {
      try {
        const batchMetadata = await fetchMetadataWithMulticall3(provider, batch, contractsMap);
        if (batchMetadata.length > 0) {
          appendResults(batchMetadata);
        } else {
          appendFailed(batch, contractsMap);
        }
        processed += batch.length;
        success = true;
        
        console.log(`Progress: ${processed}/${totalToProcess} contracts processed`);
      } catch (error) {
        console.error(`Retry ${retry + 1}/${maxRetries} for batch ${batchNum}:`, error.message);
        if (retry < maxRetries - 1) {
          const backoffDelay = retryDelay * Math.pow(2, retry);
          console.log(`  Waiting ${backoffDelay}ms before retry...`);
          await new Promise(resolve => setTimeout(resolve, backoffDelay));
        }
      }
    }
    
    // Add delay between batches
    if (i + BATCH_SIZE < uniqueAddresses.length) {
      await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
    }
  }

  console.log(`\nCompleted! Total metadata fetched: ${processed}`);
  console.log(`Output written to: ${OUTPUT_FILE}`);
}

main().catch(console.error);