Merge pull request #2063 from hirosystems/chore/merge-dev-beta

Cut v8.0.0-beta.6

Author: zone117x

.github/workflows/ci.yml

@@ -147,6 +147,9 @@ jobs:
       - name: Setup integration environment
         run: |
           sudo ufw disable
+          mkdir -p src/tests-event-replay/.tmp/local/
+          sudo chown 999:999 src/tests-event-replay/.tmp/local/
+          sudo chmod -R 777 src/tests-event-replay/.tmp/local/
           docker compose -f docker/docker-compose.dev.postgres.yml up -d
           npm run devenv:logs -- --no-color &> docker-compose-logs.txt &
 

CHANGELOG.md

@@ -37,6 +37,15 @@
 
 * refactor from Express to Fastify ([aa0e51e](https://github.com/hirosystems/stacks-blockchain-api/commit/aa0e51e557491daff1a98dd36c4e952e05c58dd4)), closes [#2042](https://github.com/hirosystems/stacks-blockchain-api/issues/2042)
 
+* index on `principal_stx_txs` table for faster `/v1/address/{addr}/transactions` lookups ([#2059](https://github.com/hirosystems/stacks-blockchain-api/issues/2059)) ([ab64ab7](https://github.com/hirosystems/stacks-blockchain-api/commit/ab64ab7148a3656f81f0a3c5a176c40caca3345a))
+
+## [7.14.0](https://github.com/hirosystems/stacks-blockchain-api/compare/v7.13.2...v7.14.0) (2024-08-20)
+
+
+### Features
+
+* export events tsv directly to postgres instance ([#2048](https://github.com/hirosystems/stacks-blockchain-api/issues/2048)) ([#2058](https://github.com/hirosystems/stacks-blockchain-api/issues/2058)) ([a1f5b12](https://github.com/hirosystems/stacks-blockchain-api/commit/a1f5b12675118f6d7742c54e3420c38151aef4a7))
+
 ## [7.13.2](https://github.com/hirosystems/stacks-blockchain-api/compare/v7.13.1...v7.13.2) (2024-08-05)
 
 

docker/docker-compose.dev.bitcoind.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   bitcoind:
     image: "blockstack/bitcoind:v0.20.99.0"

docker/docker-compose.dev.postgres.yml

@@ -1,11 +1,12 @@
-version: '3.7'
 services:
   postgres:
-    image: "postgres:14"
+    image: "postgres:15"
     ports:
       - "5490:5432"
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: stacks_blockchain_api
       POSTGRES_PORT: 5432
+    volumes:
+      - ../src/tests-event-replay/.tmp/local/:/root/

docker/docker-compose.dev.stacks-blockchain.yml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   stacks-blockchain:
     image: 'hirosystems/stacks-api-e2e:stacks3.0-0a2c0e2'

migrations/1724229744369_principal_stx_txs_idx.js

@@ -0,0 +1,22 @@
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.up = pgm => {
+  pgm.createIndex(
+    'principal_stx_txs', 
+    [
+      'principal',
+      { name: 'block_height', order: 'DESC' },
+      { name: 'microblock_sequence', order: 'DESC' },
+      { name: 'tx_index', order: 'DESC' }],
+    {
+      name: 'idx_principal_stx_txs_optimized',
+      where: 'canonical = TRUE AND microblock_canonical = TRUE',
+    }
+  );
+};
+
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.down = pgm => {
+  pgm.dropIndex('principal_stx_txs', ['principal', 'block_height', 'microblock_sequence', 'tx_index'], {
+    name: 'idx_principal_stx_txs_optimized',
+  });
+};

package-lock.json

@@ -118,7 +118,7 @@
     "dotenv-flow": "3.2.0",
     "duckdb": "0.9.2",
     "ecpair": "2.1.0",
-    "elliptic": "6.5.4",
+    "elliptic": "6.5.7",
     "escape-goat": "3.0.0",
     "evt": "1.10.1",
     "express": "4.19.2",
@@ -150,7 +150,7 @@
     "tiny-secp256k1": "2.2.1",
     "ts-unused-exports": "7.0.3",
     "uuid": "8.3.2",
-    "ws": "7.5.9",
+    "ws": "7.5.10",
     "zone-file": "2.0.0-beta.3"
   },
   "devDependencies": {

package.json

@@ -6,11 +6,12 @@ import { parseDbTx } from '../../../api/controllers/db-controller';
 import { FastifyPluginAsync } from 'fastify';
 import { Type, TypeBoxTypeProvider } from '@fastify/type-provider-typebox';
 import { Server } from 'node:http';
-import { LimitParam, OffsetParam } from '../../schemas/params';
-import { ResourceType } from '../../pagination';
+import { CursorOffsetParam, LimitParam, OffsetParam } from '../../schemas/params';
+import { getPagingQueryLimit, pagingQueryLimits, ResourceType } from '../../pagination';
 import { PaginatedResponse } from '../../schemas/util';
 import { NakamotoBlock, NakamotoBlockSchema } from '../../schemas/entities/block';
 import { TransactionSchema } from '../../schemas/entities/transactions';
+import { BlockListV2ResponseSchema } from '../../schemas/responses/responses';
 
 export const BlockRoutesV2: FastifyPluginAsync<
   Record<never, never>,
@@ -28,21 +29,29 @@ export const BlockRoutesV2: FastifyPluginAsync<
         tags: ['Blocks'],
         querystring: Type.Object({
           limit: LimitParam(ResourceType.Block),
-          offset: OffsetParam(),
+          offset: CursorOffsetParam({ resource: ResourceType.Block }),
+          cursor: Type.Optional(Type.String({ description: 'Cursor for pagination' })),
         }),
         response: {
-          200: PaginatedResponse(NakamotoBlockSchema),
+          200: BlockListV2ResponseSchema,
         },
       },
     },
     async (req, reply) => {
       const query = req.query;
-      const { limit, offset, results, total } = await fastify.db.v2.getBlocks(query);
-      const blocks: NakamotoBlock[] = results.map(r => parseDbNakamotoBlock(r));
+      const limit = getPagingQueryLimit(ResourceType.Block, req.query.limit);
+      const blockQuery = await fastify.db.v2.getBlocks({ ...query, limit });
+      if (query.cursor && !blockQuery.current_cursor) {
+        throw new NotFoundError('Cursor not found');
+      }
+      const blocks: NakamotoBlock[] = blockQuery.results.map(r => parseDbNakamotoBlock(r));
       await reply.send({
-        limit,
-        offset,
-        total,
+        limit: blockQuery.limit,
+        offset: blockQuery.offset,
+        total: blockQuery.total,
+        next_cursor: blockQuery.next_cursor,
+        prev_cursor: blockQuery.prev_cursor,
+        cursor: blockQuery.current_cursor,
         results: blocks,
       });
     }

src/api/routes/v2/blocks.ts

@@ -28,6 +28,23 @@ export const LimitParam = (
     })
   );
 
+export const CursorOffsetParam = (args: {
+  resource: ResourceType;
+  title?: string;
+  description?: string;
+  limitOverride?: number;
+  maxPages?: number;
+}) =>
+  Type.Optional(
+    Type.Integer({
+      default: 0,
+      maximum: pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
+      minimum: -pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
+      title: args.title ?? 'Offset',
+      description: args.description ?? 'Result offset',
+    })
+  );
+
 export const UnanchoredParamSchema = Type.Optional(
   Type.Boolean({
     default: false,

src/api/schemas/params.ts

@@ -1,5 +1,5 @@
 import { Static, Type } from '@sinclair/typebox';
-import { OptionalNullable, PaginatedResponse } from '../util';
+import { Nullable, OptionalNullable, PaginatedCursorResponse, PaginatedResponse } from '../util';
 import { MempoolStatsSchema } from '../entities/mempool-transactions';
 import { MempoolTransactionSchema, TransactionSchema } from '../entities/transactions';
 import { MicroblockSchema } from '../entities/microblock';
@@ -12,6 +12,7 @@ import {
   BurnchainRewardSchema,
   BurnchainRewardSlotHolderSchema,
 } from '../entities/burnchain-rewards';
+import { NakamotoBlockSchema } from '../entities/block';
 
 export const ErrorResponseSchema = Type.Object(
   {
@@ -178,3 +179,6 @@ export const RunFaucetResponseSchema = Type.Object(
   }
 );
 export type RunFaucetResponse = Static<typeof RunFaucetResponseSchema>;
+
+export const BlockListV2ResponseSchema = PaginatedCursorResponse(NakamotoBlockSchema);
+export type BlockListV2Response = Static<typeof BlockListV2ResponseSchema>;

src/api/schemas/responses/responses.ts

@@ -12,3 +12,17 @@ export const PaginatedResponse = <T extends TSchema>(type: T, options?: ObjectOp
     },
     options
   );
+
+export const PaginatedCursorResponse = <T extends TSchema>(type: T, options?: ObjectOptions) =>
+  Type.Object(
+    {
+      limit: Type.Integer({ examples: [20] }),
+      offset: Type.Integer({ examples: [0] }),
+      total: Type.Integer({ examples: [1] }),
+      next_cursor: Nullable(Type.String({ description: 'Next page cursor' })),
+      prev_cursor: Nullable(Type.String({ description: 'Previous page cursor' })),
+      cursor: Nullable(Type.String({ description: 'Current page cursor' })),
+      results: Type.Array(type),
+    },
+    options
+  );

src/api/schemas/util.ts

@@ -1143,6 +1143,16 @@ export type DbPaginatedResult<T> = {
   results: T[];
 };
 
+export type DbCursorPaginatedResult<T> = {
+  limit: number;
+  offset: number;
+  next_cursor: string | null;
+  prev_cursor: string | null;
+  current_cursor: string | null;
+  total: number;
+  results: T[];
+};
+
 export interface BlocksWithMetadata {
   results: {
     block: DbBlock;