feat: throttle requests to rate limited domains (#97)

* feat: first draft

* feat: draft 2

* feat: validate host status before fetch

* test: more headers

* chore: test progress

* fix: use timestamptz for time columns

* test: call resume

* docs: add readme docs

* fix: set default 1 hour for retry

* build: remove unsafe-perm from Dockerfile

* chore: upgrade undici

* fix: multiple header values

Author: rafaelcr

Dockerfile

@@ -4,8 +4,7 @@ WORKDIR /app
 COPY . .
 
 RUN apk add --no-cache --virtual .build-deps git
-RUN npm config set unsafe-perm true && \
-    npm ci && \
+RUN npm ci && \
     npm run build && \
     npm run generate:git-info && \
     npm prune --production

README.md

@@ -147,7 +147,7 @@ The
 component constantly listens for the following Stacks Blockchain API events:
 
 * **Smart contract log events**
-    
+
     If a contract `print` event conforms to SIP-019, it finds the affected tokens and marks them for
     metadata refresh.
 
@@ -192,3 +192,7 @@ metadata ingestion.
 This job fetches the metadata JSON object for a single token as well as other relevant properties
 depending on the token type (symbol, decimals, etc.). Once fetched, it parses and ingests this data
 to save it into the local database for API endpoints to return.
+
+If a `429` (Too Many Requests) status code is returned by a hostname used to fetch metadata, the
+service will cease all further requests to it until a reasonable amount of time has passed or until
+the time specified by the host in a `Retry-After` response header.

migrations/1670264425574_smart-contracts.ts

@@ -34,12 +34,12 @@ export function up(pgm: MigrationBuilder): void {
       type: 'numeric',
     },
     created_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
       default: pgm.func('(NOW())'),
       notNull: true,
     },
     updated_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
     },
   });
   pgm.createConstraint('smart_contracts', 'smart_contracts_principal_unique', 'UNIQUE(principal)');

migrations/1670265062169_tokens.ts

@@ -47,12 +47,12 @@ export function up(pgm: MigrationBuilder): void {
       type: 'numeric',
     },
     created_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
       default: pgm.func('(NOW())'),
       notNull: true,
     },
     updated_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
     },
   });
   pgm.createConstraint(

migrations/1670266371036_jobs.ts

@@ -25,12 +25,12 @@ export function up(pgm: MigrationBuilder): void {
       default: 0,
     },
     created_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
       default: pgm.func('(NOW())'),
       notNull: true,
     },
     updated_at: {
-      type: 'timestamp',
+      type: 'timestamptz',
     },
   });
   pgm.createConstraint('jobs', 'jobs_token_id_fk', 'FOREIGN KEY(token_id) REFERENCES tokens(id)');

migrations/1675368570737_rate-limited-hosts.ts

@@ -0,0 +1,32 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { MigrationBuilder, ColumnDefinitions } from 'node-pg-migrate';
+
+export const shorthands: ColumnDefinitions | undefined = undefined;
+
+export function up(pgm: MigrationBuilder): void {
+  pgm.createTable('rate_limited_hosts', {
+    id: {
+      type: 'serial',
+      primaryKey: true,
+    },
+    hostname: {
+      type: 'text',
+      notNull: true,
+    },
+    created_at: {
+      type: 'timestamptz',
+      default: pgm.func('(NOW())'),
+      notNull: true,
+    },
+    retry_after: {
+      type: 'timestamptz',
+      notNull: true,
+    },
+  });
+  pgm.createConstraint(
+    'rate_limited_hosts',
+    'rate_limited_hosts_hostname_unique',
+    'UNIQUE(hostname)'
+  );
+  pgm.createIndex('rate_limited_hosts', ['hostname']);
+}

package-lock.json

@@ -67,6 +67,12 @@ interface Env {
    * hours).
    */
   METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL: number;
+  /**
+   * Time that must elapse between a 429 'Too many requests' response returned by a hostname and the
+   * next request that is sent to it (seconds). This value will be overridden by the `Retry-After`
+   * header returned by the domain, if any.
+   */
+  METADATA_RATE_LIMITED_HOST_RETRY_AFTER: number;
 
   /** Whether or not the `JobQueue` will continue to try retryable failed jobs indefinitely. */
   JOB_QUEUE_STRICT_MODE: boolean;
@@ -225,6 +231,10 @@ export function getEnvVars(): Env {
         type: 'number',
         default: 86_400, // 24 hours
       },
+      METADATA_RATE_LIMITED_HOST_RETRY_AFTER: {
+        type: 'number',
+        default: 3600, // 1 hour
+      },
       JOB_QUEUE_CONCURRENCY_LIMIT: {
         type: 'number',
         default: 5,

src/env.ts

@@ -21,6 +21,9 @@ import {
   METADATA_COLUMNS,
   METADATA_ATTRIBUTES_COLUMNS,
   METADATA_PROPERTIES_COLUMNS,
+  DbRateLimitedHostInsert,
+  DbRateLimitedHost,
+  RATE_LIMITED_HOSTS_COLUMNS,
 } from './types';
 import { connectPostgres } from './postgres-tools';
 import { BasePgStore } from './postgres-tools/base-pg-store';
@@ -393,7 +396,38 @@ export class PgStore extends BasePgStore {
       INSERT INTO jobs (token_id) (SELECT id AS token_id FROM token_inserts)
       ON CONFLICT (token_id) WHERE smart_contract_id IS NULL DO
         UPDATE SET updated_at = NOW(), status = 'pending'
-      RETURNING *
+      RETURNING ${this.sql(JOBS_COLUMNS)}
+    `;
+  }
+
+  async insertRateLimitedHost(args: {
+    values: DbRateLimitedHostInsert;
+  }): Promise<DbRateLimitedHost> {
+    const retryAfter = args.values.retry_after.toString();
+    const results = await this.sql<DbRateLimitedHost[]>`
+      INSERT INTO rate_limited_hosts (hostname, created_at, retry_after)
+      VALUES (${args.values.hostname}, DEFAULT, NOW() + INTERVAL '${this.sql(retryAfter)} seconds')
+      ON CONFLICT ON CONSTRAINT rate_limited_hosts_hostname_unique DO
+        UPDATE SET retry_after = EXCLUDED.retry_after
+      RETURNING ${this.sql(RATE_LIMITED_HOSTS_COLUMNS)}
+    `;
+    return results[0];
+  }
+
+  async getRateLimitedHost(args: { hostname: string }): Promise<DbRateLimitedHost | undefined> {
+    const results = await this.sql<DbRateLimitedHost[]>`
+      SELECT ${this.sql(RATE_LIMITED_HOSTS_COLUMNS)}
+      FROM rate_limited_hosts
+      WHERE hostname = ${args.hostname}
+    `;
+    if (results.count > 0) {
+      return results[0];
+    }
+  }
+
+  async deleteRateLimitedHost(args: { hostname: string }): Promise<void> {
+    await this.sql`
+      DELETE FROM rate_limited_hosts WHERE hostname = ${args.hostname}
     `;
   }
 

src/pg/pg-store.ts

@@ -85,6 +85,19 @@ export type DbJob = {
   updated_at?: string;
 };
 
+export type DbRateLimitedHostInsert = {
+  hostname: string;
+  // Will be converted into a timestamp upon insert.
+  retry_after: number;
+};
+
+export type DbRateLimitedHost = {
+  id: number;
+  hostname: string;
+  created_at: string;
+  retry_after: string;
+};
+
 export type DbFtInsert = {
   name: string | null;
   symbol: string | null;
@@ -238,3 +251,5 @@ export const METADATA_ATTRIBUTES_COLUMNS = [
 ];
 
 export const METADATA_PROPERTIES_COLUMNS = ['id', 'metadata_id', 'name', 'value'];
+
+export const RATE_LIMITED_HOSTS_COLUMNS = ['id', 'hostname', 'created_at', 'retry_after'];