Merge pull request #110 from hirosystems/beta-next

release beta.8

Author: rafaelcr

package-lock.json

@@ -56,6 +56,7 @@
     "evt": "^1.11.2",
     "fastify": "^4.9.2",
     "fastify-metrics": "^9.2.4",
+    "json5": "^2.2.3",
     "node-pg-migrate": "^6.2.2",
     "p-queue": "^6.6.2",
     "pino": "^8.8.0",

package.json

@@ -39,6 +39,7 @@ export const Metadata = Type.Object({
   name: Type.Optional(Type.String()),
   description: Type.Optional(Type.String()),
   image: Type.Optional(Type.String({ format: 'uri' })),
+  cached_image: Type.Optional(Type.String({ format: 'uri' })),
   attributes: Type.Optional(Type.Array(MetadataAttribute)),
   properties: Type.Optional(MetadataProperties),
   localization: Type.Optional(MetadataLocalization),

src/api/types.ts

@@ -8,15 +8,16 @@ export function parseMetadataLocaleBundle(
   if (locale && locale.metadata) {
     response = {
       sip: locale.metadata.sip,
-      name: locale.metadata.name ?? undefined,
-      description: locale.metadata.description ?? undefined,
-      image: locale.metadata.image ?? undefined,
+      name: locale.metadata.name,
+      description: locale.metadata.description,
+      image: locale.metadata.image,
+      cached_image: locale.metadata.cached_image,
     };
     if (locale.attributes.length > 0) {
       response.attributes = locale.attributes.map(item => ({
         trait_type: item.trait_type,
         value: item.value as MetadataValueType,
-        display_type: item.display_type ?? undefined,
+        display_type: item.display_type,
       }));
     }
     if (locale.properties.length > 0) {

src/api/util/helpers.ts

@@ -73,6 +73,10 @@ interface Env {
    * header returned by the domain, if any.
    */
   METADATA_RATE_LIMITED_HOST_RETRY_AFTER: number;
+  /**
+   * Maximum number of HTTP redirections to follow when fetching metadata. Defaults to 5.
+   */
+  METADATA_FETCH_MAX_REDIRECTIONS: number;
 
   /** Whether or not the `JobQueue` will continue to try retryable failed jobs indefinitely. */
   JOB_QUEUE_STRICT_MODE: boolean;
@@ -235,6 +239,10 @@ export function getEnvVars(): Env {
         type: 'number',
         default: 3600, // 1 hour
       },
+      METADATA_FETCH_MAX_REDIRECTIONS: {
+        type: 'number',
+        default: 5,
+      },
       JOB_QUEUE_CONCURRENCY_LIMIT: {
         type: 'number',
         default: 5,

src/env.ts

@@ -17,13 +17,21 @@ export class SmartContractImportInterruptedError extends Error {
   }
 }
 
+class ApiBlockHeightNotReadyError extends Error {
+  constructor() {
+    super();
+    this.name = this.constructor.name;
+  }
+}
+
 /**
  * Imports token contracts and SIP-019 token metadata update notifications from the Stacks
  * Blockchain API database.
  */
 export class BlockchainImporter {
   private readonly db: PgStore;
   private readonly apiDb: PgBlockchainApiStore;
+  private apiBlockHeightRetryIntervalMs = 5000;
   private startingBlockHeight: number;
   private importInterruptWaiter: Waiter<void>;
   private importInterrupted = false;
@@ -46,6 +54,7 @@ export class BlockchainImporter {
   }
 
   async import() {
+    logger.info(`BlockchainImporter last imported block height: ${this.startingBlockHeight}`);
     while (!this.importFinished) {
       try {
         const apiBlockHeight = await this.getApiBlockHeight();
@@ -70,7 +79,10 @@ export class BlockchainImporter {
             error,
             'BlockchainImporter encountered a PG connection error during import, retrying...'
           );
-          await timeout(100);
+          await timeout(1000);
+        } else if (error instanceof ApiBlockHeightNotReadyError) {
+          logger.warn(`BlockchainImporter API block height too low, retrying...`);
+          await timeout(this.apiBlockHeightRetryIntervalMs);
         } else if (error instanceof SmartContractImportInterruptedError) {
           this.importInterruptWaiter.finish();
           throw error;
@@ -83,10 +95,9 @@ export class BlockchainImporter {
 
   private async getApiBlockHeight(): Promise<number> {
     const blockHeight = (await this.apiDb.getCurrentBlockHeight()) ?? 1;
+    logger.info(`BlockchainImporter API block height: ${blockHeight}`);
     if (this.startingBlockHeight > blockHeight) {
-      throw new Error(
-        `BlockchainImporter starting block height ${this.startingBlockHeight} is greater than the API's block height ${blockHeight}`
-      );
+      throw new ApiBlockHeightNotReadyError();
     }
     return blockHeight;
   }

src/token-processor/blockchain-api/blockchain-importer.ts

@@ -1,5 +1,5 @@
 import * as querystring from 'querystring';
-import { IncomingHttpHeaders } from 'http';
+import * as JSON5 from 'json5';
 import { Agent, errors, request } from 'undici';
 import {
   DbMetadataAttributeInsert,
@@ -31,6 +31,10 @@ const METADATA_FETCH_HTTP_AGENT = new Agent({
   headersTimeout: ENV.METADATA_FETCH_TIMEOUT_MS,
   bodyTimeout: ENV.METADATA_FETCH_TIMEOUT_MS,
   maxResponseSize: ENV.METADATA_MAX_PAYLOAD_BYTE_SIZE,
+  maxRedirections: ENV.METADATA_FETCH_MAX_REDIRECTIONS,
+  connect: {
+    rejectUnauthorized: false, // Ignore SSL cert errors.
+  },
 });
 
 /**
@@ -183,14 +187,12 @@ async function parseMetadataForInsertion(
 }
 
 /**
- * Fetches metadata while monitoring timeout and size limits. Throws if any is reached.
- * Taken from https://github.com/node-fetch/node-fetch/issues/1149#issuecomment-840416752
+ * Fetches metadata while monitoring timeout and size limits, hostname rate limits, etc. Throws if
+ * any is reached.
  * @param httpUrl - URL to fetch
  * @returns Response text
  */
-export async function performSizeAndTimeLimitedMetadataFetch(
-  httpUrl: URL
-): Promise<string | undefined> {
+export async function fetchMetadata(httpUrl: URL): Promise<string | undefined> {
   const url = httpUrl.toString();
   try {
     const result = await request(url, {
@@ -238,7 +240,7 @@ export async function getMetadataFromUri(token_uri: string): Promise<RawMetadata
       content = dataUrl.data;
     }
     try {
-      const result = JSON.parse(content);
+      const result = JSON5.parse(content);
       if (RawMetadataCType.Check(result)) {
         return result;
       }
@@ -258,8 +260,8 @@ export async function getMetadataFromUri(token_uri: string): Promise<RawMetadata
   // metadata as dead.
   do {
     try {
-      const text = await performSizeAndTimeLimitedMetadataFetch(httpUrl);
-      result = text ? JSON.parse(text) : undefined;
+      const text = await fetchMetadata(httpUrl);
+      result = text ? JSON5.parse(text) : undefined;
       break;
     } catch (error) {
       fetchImmediateRetryCount++;

src/token-processor/util/metadata-helpers.ts

@@ -6,7 +6,7 @@ import {
   BlockchainDbSmartContract,
 } from '../src/pg/blockchain-api/pg-blockchain-api-store';
 import { DbSipNumber, DbSmartContractInsert, DbTokenType } from '../src/pg/types';
-import { MockPgBlockchainApiStore, SIP_009_ABI, SIP_010_ABI, SIP_013_ABI } from './helpers';
+import { MockPgBlockchainApiStore, SIP_009_ABI, SIP_010_ABI, SIP_013_ABI, sleep } from './helpers';
 import { BlockchainImporter } from '../src/token-processor/blockchain-api/blockchain-importer';
 import { cvToHex, tupleCV, bufferCV, listCV, uintCV } from '@stacks/transactions';
 
@@ -125,4 +125,24 @@ describe('BlockchainImporter', () => {
     expect(jobs2[0].token_id).toBe(1);
     expect(jobs2[1].token_id).toBe(2);
   });
+
+  test('waits for the API to catch up to the chain tip', async () => {
+    // Set the API behind on purpose.
+    apiDb.currentBlockHeight = 3;
+    await db.sql`UPDATE chain_tip SET block_height = 5`;
+    importer = new BlockchainImporter({ db, apiDb, startingBlockHeight: 5 });
+    importer['apiBlockHeightRetryIntervalMs'] = 1000;
+
+    // Start import, this will trigger a 1s wait loop for the API block height to catch up.
+    const importPromise = new Promise<void>(resolve => {
+      void importer.import().then(() => resolve());
+    });
+
+    // Update the API block height after 500ms.
+    await sleep(500);
+    apiDb.currentBlockHeight = 5;
+
+    // The import finishes then.
+    await importPromise;
+  });
 });

tests/blockchain-importer.test.ts

@@ -142,8 +142,8 @@ describe('FT routes', () => {
               l10n_uri: null,
               l10n_default: true,
               description: 'test',
-              image: null,
-              cached_image: null,
+              image: 'http://test.com/image.png',
+              cached_image: 'http://test.com/image.png?processed=true',
             },
             attributes: [
               {
@@ -186,6 +186,8 @@ describe('FT routes', () => {
         sip: 16,
         description: 'test',
         name: 'hello-world',
+        image: 'http://test.com/image.png',
+        cached_image: 'http://test.com/image.png?processed=true',
         attributes: [
           {
             display_type: 'number',

tests/ft.test.ts

@@ -23,6 +23,10 @@ export async function startTestApiServer(db: PgStore): Promise<TestFastifyServer
   return await buildApiServer({ db });
 }
 
+export const sleep = (time: number) => {
+  return new Promise(resolve => setTimeout(resolve, time));
+};
+
 export class MockPgBlockchainApiStore extends PgBlockchainApiStore {
   constructor() {
     super(postgres());