From 8c29a2ae4e553d6e4aa16e14081129f87981cdf2 Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:22:38 +0000 Subject: [PATCH 1/6] feat(utils): add MultiEndpointClient for resilient HTTP requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a reusable HTTP client that supports: - Multiple endpoints with automatic failover - Per-endpoint retry with exponential backoff (100ms, 200ms, 400ms...) - Priority-based endpoint selection - Remembers last successful endpoint - Configurable timeout and max retries This utility will be used to enhance RPC/REST calls across the codebase to handle endpoint failures gracefully. Key features: - Gracefully handles AbortSignal.timeout availability (Node 17.3+) - Sorts endpoints by priority (higher priority tried first) - Provides getCurrentEndpoint() to check which endpoint is active - Comprehensive error messages when all endpoints fail Includes comprehensive test coverage (11 tests): - Single/multiple endpoint support - Retry and fallback behavior - Priority sorting - Error handling - Endpoint memory (remembers last successful) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../__tests__/multi-endpoint-client.spec.ts | 194 ++++++++++++++++++ packages/utils/src/index.ts | 1 + packages/utils/src/multi-endpoint-client.ts | 171 +++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 packages/utils/src/__tests__/multi-endpoint-client.spec.ts create mode 100644 packages/utils/src/multi-endpoint-client.ts diff --git a/packages/utils/src/__tests__/multi-endpoint-client.spec.ts b/packages/utils/src/__tests__/multi-endpoint-client.spec.ts new file mode 100644 index 0000000000..64da602c1f --- /dev/null +++ b/packages/utils/src/__tests__/multi-endpoint-client.spec.ts @@ -0,0 +1,194 @@ +import { apiClient } from "../api-client"; +import { + createMultiEndpointClient, + MultiEndpointClient, +} from "../multi-endpoint-client"; + +jest.mock("../api-client", () => ({ + apiClient: jest.fn(), +})); + +describe("MultiEndpointClient", () => { + beforeEach(() => { + (apiClient as jest.Mock).mockClear(); + }); + + it("should create client with single endpoint", () => { + const client = new MultiEndpointClient([ + { address: "https://endpoint1.com" }, + ]); + expect(client.getEndpoints()).toHaveLength(1); + }); + + it("should create client with multiple endpoints", () => { + const client = new MultiEndpointClient([ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ]); + expect(client.getEndpoints()).toHaveLength(2); + }); + + it("should throw error when no endpoints provided", () => { + expect(() => new MultiEndpointClient([])).toThrow( + "At least one endpoint must be provided" + ); + }); + + it("should successfully fetch from first endpoint", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock).mockResolvedValue(mockResult); + + const client = createMultiEndpointClient([ + { address: "https://endpoint1.com" }, + ]); + + const result = await client.fetch("/status"); + + expect(apiClient).toHaveBeenCalledTimes(1); + expect(apiClient).toHaveBeenCalledWith( + "https://endpoint1.com/status", + expect.any(Object) + ); + expect(result).toEqual(mockResult); + }); + + it("should retry on failure and succeed on second attempt", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock) + .mockRejectedValueOnce(new Error("Network error")) + .mockResolvedValueOnce(mockResult); + + const client = createMultiEndpointClient( + [{ address: "https://endpoint1.com" }], + { maxRetries: 2 } + ); + + const result = await client.fetch("/status"); + + expect(apiClient).toHaveBeenCalledTimes(2); + expect(result).toEqual(mockResult); + }); + + it("should fallback to second endpoint when first fails", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock) + .mockRejectedValueOnce(new Error("Endpoint 1 fail")) + .mockRejectedValueOnce(new Error("Endpoint 1 fail again")) + .mockResolvedValueOnce(mockResult); + + const client = createMultiEndpointClient( + [ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ], + { maxRetries: 2 } + ); + + const result = await client.fetch("/status"); + + expect(apiClient).toHaveBeenCalledTimes(3); + // Verify second endpoint was called + expect(apiClient).toHaveBeenCalledWith( + "https://endpoint2.com/status", + expect.any(Object) + ); + expect(result).toEqual(mockResult); + }); + + it("should throw error when all endpoints fail", async () => { + (apiClient as jest.Mock).mockRejectedValue(new Error("All failed")); + + const client = createMultiEndpointClient( + [ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ], + { maxRetries: 2 } + ); + + await expect(client.fetch("/status")).rejects.toThrow( + /All .* endpoints failed/ + ); + + // endpoint1 (2 attempts) + endpoint2 (2 attempts) = 4 total + expect(apiClient).toHaveBeenCalledTimes(4); + }); + + it("should remember last successful endpoint", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock) + .mockRejectedValueOnce(new Error("First endpoint failed")) + .mockRejectedValueOnce(new Error("First endpoint failed again")) + .mockResolvedValue(mockResult); + + const client = createMultiEndpointClient( + [ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ], + { maxRetries: 2 } + ); + + // First call should fail on endpoint1, succeed on endpoint2 + await client.fetch("/status"); + + // Clear the mock to count fresh calls + (apiClient as jest.Mock).mockClear(); + (apiClient as jest.Mock).mockResolvedValue(mockResult); + + // Second call should start with endpoint2 (last successful) + await client.fetch("/status"); + + // Should only make 1 call since it starts with the working endpoint + expect(apiClient).toHaveBeenCalledTimes(1); + expect(apiClient).toHaveBeenCalledWith( + "https://endpoint2.com/status", + expect.any(Object) + ); + }); + + it("should sort endpoints by priority", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock).mockResolvedValue(mockResult); + + const client = createMultiEndpointClient([ + { address: "https://low-priority.com", priority: 0 }, + { address: "https://high-priority.com", priority: 10 }, + { address: "https://medium-priority.com", priority: 5 }, + ]); + + await client.fetch("/status"); + + // Should try high priority first + expect(apiClient).toHaveBeenCalledWith( + "https://high-priority.com/status", + expect.any(Object) + ); + }); + + it("should use custom timeout", async () => { + const mockResult = { data: "success" }; + (apiClient as jest.Mock).mockResolvedValue(mockResult); + + const client = createMultiEndpointClient( + [{ address: "https://endpoint1.com" }], + { timeout: 10000 } + ); + + await client.fetch("/status"); + + expect(apiClient).toHaveBeenCalledWith( + expect.any(String), + expect.any(Object) + ); + }); + + it("should get current endpoint address", () => { + const client = createMultiEndpointClient([ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ]); + + expect(client.getCurrentEndpoint()).toBe("https://endpoint1.com"); + }); +}); diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 6c77a4aa32..02446e793e 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -17,6 +17,7 @@ export * from "./gas-utils"; export * from "./ibc-utils"; export * from "./litecoin"; export * from "./math"; +export * from "./multi-endpoint-client"; export * from "./object"; export * from "./pengu"; export * from "./penumbra"; diff --git a/packages/utils/src/multi-endpoint-client.ts b/packages/utils/src/multi-endpoint-client.ts new file mode 100644 index 0000000000..6a70748dae --- /dev/null +++ b/packages/utils/src/multi-endpoint-client.ts @@ -0,0 +1,171 @@ +import { apiClient, ClientOptions } from "./api-client"; + +/** + * Configuration for a single endpoint. + * Endpoints with higher priority are tried first. + */ +export interface EndpointConfig { + address: string; + priority?: number; +} + +/** + * Options for configuring multi-endpoint client behavior. + */ +export interface MultiEndpointOptions { + /** Maximum number of retries per endpoint before trying the next one. + * Default: 3 */ + maxRetries?: number; + /** Request timeout in milliseconds. Default: 5000ms */ + timeout?: number; + /** Multiplier for exponential backoff delay. Default: 2 (100ms, 200ms, 400ms...) */ + backoffMultiplier?: number; +} + +/** + * HTTP client that supports multiple endpoints with automatic failover and retry logic. + * + * Features: + * - Tries endpoints in priority order (higher priority first) + * - Retries each endpoint with exponential backoff + * - Remembers the last successful endpoint for future requests + * - Provides comprehensive error messages when all endpoints fail + * + * Example usage: + * ```typescript + * const client = createMultiEndpointClient([ + * { address: "https://rpc-osmosis.blockapsis.com", priority: 1 }, + * { address: "https://osmosis-rpc.polkachu.com", priority: 0 }, + * ]); + * + * const status = await client.fetch("/status"); + * ``` + */ +export class MultiEndpointClient { + private endpoints: EndpointConfig[]; + private currentIndex: number = 0; + private maxRetries: number; + private timeout: number; + private backoffMultiplier: number; + + constructor( + endpoints: EndpointConfig[], + options: MultiEndpointOptions = {} + ) { + if (!endpoints || endpoints.length === 0) { + throw new Error("At least one endpoint must be provided"); + } + + // Sort by priority (higher priority first) + this.endpoints = [...endpoints].sort( + (a, b) => (b.priority ?? 0) - (a.priority ?? 0) + ); + + this.maxRetries = options.maxRetries ?? 3; + this.timeout = options.timeout ?? 5000; + this.backoffMultiplier = options.backoffMultiplier ?? 2; + } + + /** + * Fetch data from the given path using multi-endpoint retry logic. + * + * @param path - The path to append to the endpoint URL (e.g., "/status") + * @param options - Additional fetch options (headers, body, etc.) + * @returns Promise resolving to the typed response + * @throws Error if all endpoints fail after all retries + */ + async fetch(path: string, options?: ClientOptions): Promise { + let lastError: Error | null = null; + + // Try each endpoint starting from the last successful one + for (let i = 0; i < this.endpoints.length; i++) { + const endpointIndex = (this.currentIndex + i) % this.endpoints.length; + const endpoint = this.endpoints[endpointIndex]; + + // Retry current endpoint with exponential backoff + for (let retry = 0; retry < this.maxRetries; retry++) { + try { + const url = `${endpoint.address}${path}`; + + // AbortSignal.timeout is only available in Node 17.3+ and modern browsers + const timeoutSignal = + typeof AbortSignal.timeout === "function" + ? AbortSignal.timeout(this.timeout) + : undefined; + + const result = await apiClient(url, { + ...options, + ...(timeoutSignal && { signal: timeoutSignal }), + }); + + // Success! Remember this endpoint for future requests + this.currentIndex = endpointIndex; + return result; + } catch (error) { + lastError = error as Error; + + // Log the failure for debugging + if (retry === this.maxRetries - 1) { + console.warn( + `[MultiEndpointClient] Endpoint ${endpoint.address} failed after ${this.maxRetries} retries. ` + + (i < this.endpoints.length - 1 + ? "Trying next endpoint..." + : "No more endpoints available."), + lastError.message + ); + } + + // Wait before retry with exponential backoff + // Don't wait on the last retry if there are more endpoints to try + if (retry < this.maxRetries - 1) { + const backoffDelay = Math.pow(this.backoffMultiplier, retry) * 100; + await new Promise((resolve) => setTimeout(resolve, backoffDelay)); + } + } + } + } + + // All endpoints exhausted + throw new Error( + `All ${this.endpoints.length} endpoints failed after ${this.maxRetries} retries each. ` + + `Last error: ${lastError?.message || "Unknown error"}` + ); + } + + /** + * Get the current best endpoint address. + * Useful for direct access when needed. + */ + getCurrentEndpoint(): string { + return this.endpoints[this.currentIndex].address; + } + + /** + * Get all configured endpoints. + */ + getEndpoints(): EndpointConfig[] { + return [...this.endpoints]; + } +} + +/** + * Factory function to create a MultiEndpointClient instance. + * + * @param endpoints - Array of endpoint configurations + * @param options - Optional configuration for retry behavior + * @returns A new MultiEndpointClient instance + * + * @example + * ```typescript + * const client = createMultiEndpointClient( + * chain.apis.rpc, + * { maxRetries: 5, timeout: 10000 } + * ); + * ``` + */ +export function createMultiEndpointClient( + endpoints: EndpointConfig[] | { address: string }[], + options?: MultiEndpointOptions +): MultiEndpointClient { + return new MultiEndpointClient(endpoints, options); +} From 30e0737abf304ec7eb9fba16241f47be901e3b3d Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:26:38 +0000 Subject: [PATCH 2/6] feat(server): add multi-endpoint support to createNodeQuery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhances createNodeQuery to use all available REST endpoints from chain asset lists instead of only the first one. Changes: - Iterates through all chain.apis.rest endpoints - Retries each endpoint with exponential backoff before moving to next - Adds optional maxRetries (default: 3) and timeout (default: 5000ms) params - Maintains backward compatibility - existing code works unchanged - Gracefully handles AbortSignal.timeout availability Endpoint source: The REST endpoints come from the chain's asset list (osmosis-labs/assetlists). Each chain can have multiple REST endpoints for redundancy. This function will: 1. Try each endpoint in order from the chain.apis.rest array 2. Retry each endpoint up to maxRetries times with exponential backoff 3. Move to the next endpoint if all retries fail 4. Throw an error only if all endpoints have been exhausted Benefits ALL queries using createNodeQuery: - Balance queries (cosmos/bank/balances.ts) - Fee estimation (osmosis/txfees/*.ts) - Transaction simulation (cosmos/tx/simulate.ts) - Staking queries (cosmos/staking/validators.ts) - Governance queries (cosmos/governance/proposals.ts) Test coverage: - Updated 5 existing tests for backward compatibility - Added 5 new tests for retry/fallback behavior - All 10 tests passing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../__tests__/create-node-query.spec.ts | 139 +++++++++++++++++- .../server/src/queries/create-node-query.ts | 88 ++++++++++- 2 files changed, 215 insertions(+), 12 deletions(-) diff --git a/packages/server/src/queries/__tests__/create-node-query.spec.ts b/packages/server/src/queries/__tests__/create-node-query.spec.ts index 1fd8be2717..5ca473b207 100644 --- a/packages/server/src/queries/__tests__/create-node-query.spec.ts +++ b/packages/server/src/queries/__tests__/create-node-query.spec.ts @@ -25,7 +25,8 @@ describe("createNodeQuery", () => { const result = await query({ chainList: MockChains, ...params }); expect(apiClient).toHaveBeenCalledWith( - `https://lcd-osmosis.keplr.app${path}` + `https://lcd-osmosis.keplr.app${path}`, + expect.any(Object) ); expect(result).toEqual(mockResult); }); @@ -45,7 +46,8 @@ describe("createNodeQuery", () => { const result = await query({ chainList: MockChains, ...params }); expect(apiClient).toHaveBeenCalledWith( - `https://lcd-osmosis.keplr.app${path(params)}` + `https://lcd-osmosis.keplr.app${path(params)}`, + expect.any(Object) ); expect(result).toEqual(mockResult); }); @@ -83,7 +85,8 @@ describe("createNodeQuery", () => { const result = await query({ chainList: MockChains, ...params }); expect(apiClient).toHaveBeenCalledWith( - `https://lcd-cosmoshub.keplr.app${path(params)}` + `https://lcd-cosmoshub.keplr.app${path(params)}`, + expect.any(Object) ); expect(result).toEqual(mockResult); }); @@ -109,8 +112,136 @@ describe("createNodeQuery", () => { expect(apiClient).toHaveBeenCalledWith( `https://lcd-cosmoshub.keplr.app${path(params)}`, - { body: "stringBody" } + expect.objectContaining({ + body: "stringBody", + }) ); expect(result).toEqual(mockResult); }); + + describe("multi-endpoint retry logic", () => { + it("should retry on first endpoint failure and succeed on second attempt", async () => { + const mockResult = { data: "test" }; + (apiClient as jest.Mock) + .mockRejectedValueOnce(new Error("Network timeout")) + .mockResolvedValueOnce(mockResult); + + const query = createNodeQuery<{ data: string }>({ + path: "/test", + maxRetries: 2, + }); + + const result = await query({ chainList: MockChains }); + + // Should have been called twice (first failure, then success) + expect(apiClient).toHaveBeenCalledTimes(2); + expect(result).toEqual(mockResult); + }); + + it("should fallback to second endpoint when first endpoint fails all retries", async () => { + const mockResult = { data: "success" }; + const mockChains = [ + { + ...MockChains[0], + apis: { + rest: [ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ], + rpc: [], + }, + }, + ]; + + // First endpoint fails 2 times, second endpoint succeeds + (apiClient as jest.Mock) + .mockRejectedValueOnce(new Error("Endpoint 1 fail")) + .mockRejectedValueOnce(new Error("Endpoint 1 fail again")) + .mockResolvedValueOnce(mockResult); + + const query = createNodeQuery<{ data: string }>({ + path: "/test", + maxRetries: 2, + }); + + const result = await query({ chainList: mockChains }); + + expect(apiClient).toHaveBeenCalledTimes(3); + // Verify second endpoint was called + expect(apiClient).toHaveBeenCalledWith( + "https://endpoint2.com/test", + expect.any(Object) + ); + expect(result).toEqual(mockResult); + }); + + it("should throw error when all endpoints are exhausted", async () => { + const mockChains = [ + { + ...MockChains[0], + apis: { + rest: [ + { address: "https://endpoint1.com" }, + { address: "https://endpoint2.com" }, + ], + rpc: [], + }, + }, + ]; + + // All attempts fail + (apiClient as jest.Mock).mockRejectedValue(new Error("All failed")); + + const query = createNodeQuery<{ data: string }>({ + path: "/test", + maxRetries: 2, + }); + + await expect(query({ chainList: mockChains })).rejects.toThrow( + /All 2 REST endpoints failed/ + ); + + // Should have tried: endpoint1 (2 times) + endpoint2 (2 times) = 4 calls + expect(apiClient).toHaveBeenCalledTimes(4); + }); + + it("should respect custom timeout parameter", async () => { + const mockResult = { data: "test" }; + (apiClient as jest.Mock).mockResolvedValue(mockResult); + + const customTimeout = 10000; + const query = createNodeQuery<{ data: string }>({ + path: "/test", + timeout: customTimeout, + }); + + await query({ chainList: MockChains }); + + // Should be called with some options + expect(apiClient).toHaveBeenCalledWith( + expect.any(String), + expect.any(Object) + ); + }); + + it("should throw error when chain has no REST endpoints", async () => { + const mockChains = [ + { + ...MockChains[0], + apis: { + rest: [], + rpc: [], + }, + }, + ]; + + const query = createNodeQuery<{ data: string }>({ + path: "/test", + }); + + await expect(query({ chainList: mockChains })).rejects.toThrow( + /No REST endpoints available/ + ); + }); + }); }); diff --git a/packages/server/src/queries/create-node-query.ts b/packages/server/src/queries/create-node-query.ts index 27070d5b43..848ed8443d 100644 --- a/packages/server/src/queries/create-node-query.ts +++ b/packages/server/src/queries/create-node-query.ts @@ -14,6 +14,14 @@ import { runIfFn } from "@osmosis-labs/utils"; * the request will be sent to the RPC endpoint since it's * assumed the LCD rest endpoint only accepts GET requests. * + * ENDPOINT FALLBACK: + * The REST endpoints come from the chain's asset list (osmosis-labs/assetlists). + * Each chain can have multiple REST endpoints for redundancy. This function will: + * 1. Try each endpoint in order from the chain.apis.rest array + * 2. Retry each endpoint up to `maxRetries` times with exponential backoff + * 3. Move to the next endpoint if all retries fail + * 4. Throw an error only if all endpoints have been exhausted + * * - [ ]: add node query error handling (like deserializing code) * and extend `ApiClientError` */ @@ -21,6 +29,8 @@ export const createNodeQuery = | unknown = unknown>({ path, options, + maxRetries = 3, + timeout = 5000, }: { path: string | ((params: PathParameters) => string); /** Additional query options such as @@ -29,6 +39,11 @@ export const createNodeQuery = * due to the node-query invariant of rest services * only accepting GET requests. */ options?: (params: PathParameters) => ClientOptions; + /** Maximum number of retries per endpoint before trying the next one. + * Default: 3 */ + maxRetries?: number; + /** Request timeout in milliseconds. Default: 5000ms */ + timeout?: number; }) => async ( ...params: PathParameters extends Record @@ -47,15 +62,72 @@ export const createNodeQuery = if (!chain) throw new Error(`Chain ${chainId} not found`); + // Get all available REST endpoints from the chain's asset list + const restEndpoints = chain.apis.rest; + + if (!restEndpoints || restEndpoints.length === 0) { + throw new Error(`No REST endpoints available for chain ${chainId}`); + } + const opts = options?.(...(params as [PathParameters])); + const pathStr = runIfFn( + path, + ...((params as [PathParameters & { chainId?: string }]) ?? []) + ); + + let lastError: Error | null = null; + + // Try each endpoint with retries + for (let endpointIndex = 0; endpointIndex < restEndpoints.length; endpointIndex++) { + const endpoint = restEndpoints[endpointIndex]; + const baseUrl = endpoint.address; + + // Retry current endpoint with exponential backoff + for (let retry = 0; retry < maxRetries; retry++) { + try { + const url = new URL(pathStr, baseUrl); + + // AbortSignal.timeout is only available in Node 17.3+ and modern browsers + const timeoutSignal = + typeof AbortSignal.timeout === "function" + ? AbortSignal.timeout(timeout) + : undefined; + + const result = await apiClient(url.toString(), { + ...opts, + ...(timeoutSignal && { signal: timeoutSignal }), + }); + + // Success! Return immediately + return result; + } catch (error) { + lastError = error as Error; + + // Log the failure for debugging + if (retry === maxRetries - 1) { + // Last retry for this endpoint + console.warn( + `[createNodeQuery] Endpoint ${baseUrl} failed after ${maxRetries} retries. ` + + (endpointIndex < restEndpoints.length - 1 + ? "Trying next endpoint..." + : "No more endpoints available."), + lastError.message + ); + } + + // Wait before retry with exponential backoff (100ms, 200ms, 400ms, ...) + // Don't wait on the last retry if there are more endpoints to try + if (retry < maxRetries - 1) { + const backoffDelay = Math.pow(2, retry) * 100; + await new Promise((resolve) => setTimeout(resolve, backoffDelay)); + } + } + } + } - const url = new URL( - runIfFn( - path, - ...((params as [PathParameters & { chainId?: string }]) ?? []) - ), - chain.apis.rest[0].address + // All endpoints exhausted + throw new Error( + `All ${restEndpoints.length} REST endpoints failed for chain ${chainId} after ${maxRetries} retries each. ` + + `Last error: ${lastError?.message || "Unknown error"}` ); - if (opts) return apiClient(url.toString(), opts); - else return apiClient(url.toString()); }; From 92c6cbd7bb14188795e99166b7226ce6ff1014e3 Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:29:03 +0000 Subject: [PATCH 3/6] feat(server): add multi-endpoint support to queryRPCStatus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates queryRPCStatus to accept either single endpoint (legacy) or multiple endpoints with automatic retry/fallback. Changes: - New API: queryRPCStatus({ rpcUrls: string[] }) - Legacy API still works: queryRPCStatus({ restUrl: string }) - Uses MultiEndpointClient for automatic failover - Maintains backward compatibility This improves resilience for: - IBC transfer time estimation - Block height polling - Chain status checks Example usage: // Old (still works) await queryRPCStatus({ restUrl: "https://rpc.osmosis.zone" }) // New (automatic failover) await queryRPCStatus({ rpcUrls: [ "https://rpc.osmosis.zone", "https://osmosis-rpc.polkachu.com", "https://rpc-osmosis.blockapsis.com" ] }) Implementation details: - Detects which API is being used via "rpcUrls" in params - Creates MultiEndpointClient with 3 retries and 5s timeout - Handles both standard and non-standard RPC response formats 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../server/src/queries/cosmos/rpc-status.ts | 60 ++++++++++++++++--- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/server/src/queries/cosmos/rpc-status.ts b/packages/server/src/queries/cosmos/rpc-status.ts index 6a603ce46e..5f987236c3 100644 --- a/packages/server/src/queries/cosmos/rpc-status.ts +++ b/packages/server/src/queries/cosmos/rpc-status.ts @@ -1,4 +1,4 @@ -import { apiClient } from "@osmosis-labs/utils"; +import { apiClient, createMultiEndpointClient } from "@osmosis-labs/utils"; export type Status = { node_info: { @@ -45,14 +45,56 @@ export type QueryStatusResponse = { result: Status; }; -export async function queryRPCStatus({ - restUrl, -}: { - restUrl: string; -}): Promise { - const data = await apiClient( - restUrl + "/status" - ); +/** + * Query RPC status from a chain node. + * + * Supports both single endpoint (legacy) and multiple endpoints with automatic fallback. + * When multiple RPC URLs are provided, will try each endpoint with retry logic before failing. + * + * @param params - Either { restUrl: string } for single endpoint or { rpcUrls: string[] } for multi-endpoint + * @returns The RPC status response + * + * @example + * // Single endpoint (legacy, backward compatible) + * const status = await queryRPCStatus({ restUrl: "https://rpc.osmosis.zone" }); + * + * // Multiple endpoints with automatic fallback + * const status = await queryRPCStatus({ + * rpcUrls: [ + * "https://rpc.osmosis.zone", + * "https://osmosis-rpc.polkachu.com", + * "https://rpc-osmosis.blockapsis.com" + * ] + * }); + */ +export async function queryRPCStatus( + params: { restUrl: string } | { rpcUrls: string[] } +): Promise { + let data: QueryStatusResponse | Status; + + // Check if using new multi-endpoint API + if ("rpcUrls" in params) { + const { rpcUrls } = params; + + if (!rpcUrls || rpcUrls.length === 0) { + throw new Error("At least one RPC URL must be provided"); + } + + // Use multi-endpoint client for automatic retry and fallback + const client = createMultiEndpointClient( + rpcUrls.map((url) => ({ address: url })), + { + maxRetries: 3, + timeout: 5000, + } + ); + + data = await client.fetch("/status"); + } else { + // Legacy single endpoint - backward compatible + const { restUrl } = params; + data = await apiClient(restUrl + "/status"); + } // some chains return a nonstandard response that does not include the jsonrpc field // but rather just the status object From 209e9e89e5f8afacc0cc9d3262b5c1ddc1b9fd9c Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:29:47 +0000 Subject: [PATCH 4/6] feat(bridge): use all available RPC endpoints for IBC transfer estimation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates IBC bridge provider to pass all RPC endpoints from chain config to queryRPCStatus instead of only the first one. Changes: - Maps all chain.apis.rpc endpoints to array - Passes rpcUrls array to queryRPCStatus for automatic failover - Updates error messages to be more descriptive Before: const fromRpc = fromChain?.apis.rpc[0]?.address; const toRpc = toChain?.apis.rpc[0]?.address; await queryRPCStatus({ restUrl: fromRpc }) await queryRPCStatus({ restUrl: toRpc }) After: const fromRpcUrls = fromChain?.apis.rpc.map(rpc => rpc.address); const toRpcUrls = toChain?.apis.rpc.map(rpc => rpc.address); await queryRPCStatus({ rpcUrls: fromRpcUrls }) await queryRPCStatus({ rpcUrls: toRpcUrls }) Impact: - IBC transfer time estimates no longer fail if primary RPC is down - Automatically tries all available RPC endpoints with retry logic - Better user experience during network issues - More accurate transfer time estimates with increased reliability Location: estimateTransferTime() method at packages/bridge/src/ibc/index.ts:315 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- packages/bridge/src/ibc/index.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/bridge/src/ibc/index.ts b/packages/bridge/src/ibc/index.ts index 33bc033b61..d99fb3f9a3 100644 --- a/packages/bridge/src/ibc/index.ts +++ b/packages/bridge/src/ibc/index.ts @@ -321,20 +321,26 @@ export class IbcBridgeProvider implements BridgeProvider { ); const toChain = this.ctx.chainList.find((c) => c.chain_id === toChainId); - const fromRpc = fromChain?.apis.rpc[0]?.address; - const toRpc = toChain?.apis.rpc[0]?.address; + // Get all RPC endpoints for automatic fallback + const fromRpcUrls = fromChain?.apis.rpc.map((rpc) => rpc.address) ?? []; + const toRpcUrls = toChain?.apis.rpc.map((rpc) => rpc.address) ?? []; - if (!fromChain || !toChain || !fromRpc || !toRpc) { + if ( + !fromChain || + !toChain || + fromRpcUrls.length === 0 || + toRpcUrls.length === 0 + ) { throw new BridgeQuoteError({ bridgeId: IbcBridgeProvider.ID, errorType: "UnsupportedQuoteError", - message: "Chain not found", + message: "Chain not found or no RPC endpoints available", }); } const [fromBlockTimeMs, toBlockTimeMs] = await Promise.all([ - queryRPCStatus({ restUrl: fromRpc }).then(calcAverageBlockTimeMs), - queryRPCStatus({ restUrl: toRpc }).then(calcAverageBlockTimeMs), + queryRPCStatus({ rpcUrls: fromRpcUrls }).then(calcAverageBlockTimeMs), + queryRPCStatus({ rpcUrls: toRpcUrls }).then(calcAverageBlockTimeMs), ]); // convert to seconds From f999f29e1c3ac16a3c772364e7ccab8282e3a851 Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:30:41 +0000 Subject: [PATCH 5/6] feat(tx): add multi-endpoint support to PollingStatusSubscription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhances PollingStatusSubscription to accept single or multiple RPC URLs with automatic failover during block polling. Changes: - Constructor now accepts string | string[] for rpc parameter - Converts single string to array internally for consistent handling - Uses new queryRPCStatus multi-endpoint API when multiple URLs provided - Maintains backward compatibility with single URL - Added validation to ensure at least one URL is provided Benefits: - Block polling continues even if primary RPC fails - Automatic failover to alternative endpoints - More resilient IBC timeout tracking - Better user experience during network issues Example usage: // Old (still works) new PollingStatusSubscription("https://rpc.osmosis.zone") // New (automatic failover) new PollingStatusSubscription([ "https://rpc.osmosis.zone", "https://osmosis-rpc.polkachu.com" ]) Implementation details: - Stores URLs in protected readonly rpcUrls array - Detects single vs multiple URLs and calls appropriate queryRPCStatus API - Enhances error logging to show number of endpoints being used Location: packages/tx/src/poll-status.ts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- packages/tx/src/poll-status.ts | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/tx/src/poll-status.ts b/packages/tx/src/poll-status.ts index d110f0f9ca..6a42f3b0ea 100644 --- a/packages/tx/src/poll-status.ts +++ b/packages/tx/src/poll-status.ts @@ -11,10 +11,23 @@ export class PollingStatusSubscription { protected _handlers: StatusHandler[] = []; + protected readonly rpcUrls: string[]; + + /** + * @param rpc - Single RPC URL or array of RPC URLs for multi-endpoint support + * @param defaultBlockTimeMs - Default block time in milliseconds (default: 7500ms) + */ constructor( - protected readonly rpc: string, + rpc: string | string[], protected readonly defaultBlockTimeMs = 7500 - ) {} + ) { + // Support both single string (backward compatible) and array of URLs + this.rpcUrls = Array.isArray(rpc) ? rpc : [rpc]; + + if (this.rpcUrls.length === 0) { + throw new Error("At least one RPC URL must be provided"); + } + } get subscriptionCount(): number { return this._subscriptionCount; @@ -39,7 +52,12 @@ export class PollingStatusSubscription { let timeoutId: NodeJS.Timeout | undefined; while (this._subscriptionCount > 0) { try { - const status = await queryRPCStatus({ restUrl: this.rpc }); + // Use multi-endpoint support if multiple URLs provided + const status = + this.rpcUrls.length === 1 + ? await queryRPCStatus({ restUrl: this.rpcUrls[0] }) + : await queryRPCStatus({ rpcUrls: this.rpcUrls }); + const blockTime = calcAverageBlockTimeMs( status, this.defaultBlockTimeMs @@ -49,7 +67,9 @@ export class PollingStatusSubscription { timeoutId = setTimeout(resolve, blockTime); }); } catch (e: any) { - console.error(`Failed to fetch /status: ${e}`); + console.error( + `Failed to fetch /status from ${this.rpcUrls.length} endpoint(s): ${e}` + ); } } if (timeoutId) clearTimeout(timeoutId); From c47d26b61bf57dcd69ae0d8a1f777093c5d18f93 Mon Sep 17 00:00:00 2001 From: JohnnyWyles Date: Wed, 12 Nov 2025 14:31:35 +0000 Subject: [PATCH 6/6] feat(tx): add automatic WebSocket reconnection and endpoint failover to TxTracer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhances TxTracer with comprehensive WebSocket failover logic to maintain IBC transfer status tracking during network issues. Changes: - Constructor now accepts string | string[] for url parameter - Automatic reconnection with exponential backoff (1s, 2s, 4s, 8s max) - Tries each endpoint multiple times before moving to next - After exhausting all endpoints, waits 10s before cycling back - Preserves all subscriptions across reconnections - Prevents reconnection on manual close - Adds comprehensive logging for debugging Reconnection flow: 1. Try endpoint 0 (maxReconnectAttempts times with backoff) 2. If all fail, try endpoint 1 (maxReconnectAttempts times) 3. If all fail, try endpoint 2 (maxReconnectAttempts times) 4. After all endpoints exhausted, wait 10s and cycle back to endpoint 0 New state management: - urls: readonly string[] - Array of WebSocket URLs - currentUrlIndex: number - Tracks which endpoint is active - reconnectAttempts: number - Counts retry attempts for current endpoint - maxReconnectAttempts: number - Configurable (default: 3) - isManualClose: boolean - Prevents auto-reconnect on user close Event handlers: - onOpen: Resets reconnect counter, re-subscribes all handlers - onClose: Triggers reconnect logic unless manual close - onError: Logs error and lets onClose handle reconnection Benefits: - IBC transfer status tracking continues during RPC issues - Automatic recovery without user intervention - Prevents lost WebSocket subscriptions - Better visibility with console logging Example usage: // Old (still works) new TxTracer("https://rpc.osmosis.zone") // New (automatic failover) new TxTracer([ "https://rpc.osmosis.zone", "https://osmosis-rpc.polkachu.com" ], "/websocket", { maxReconnectAttempts: 5 }) Location: packages/tx/src/tracer.ts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- packages/tx/src/tracer.ts | 121 +++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/tx/src/tracer.ts b/packages/tx/src/tracer.ts index 2318924909..8276d8a778 100644 --- a/packages/tx/src/tracer.ts +++ b/packages/tx/src/tracer.ts @@ -24,6 +24,9 @@ type Listeners = { * Subscribes to new blocks and transactions by attaching a WebSocket connection to the * chain's RPC endpoint. This allows for real-time updates on the chain's state. * + * Supports multiple WebSocket endpoints with automatic failover. If the connection fails, + * will automatically try the next endpoint with exponential backoff. + * * Create one instance per block or tx subscription. */ export class TxTracer { @@ -55,16 +58,45 @@ export class TxTracer { protected listeners: Listeners = {}; + // Multi-endpoint support + protected readonly urls: string[]; + protected currentUrlIndex: number = 0; + protected reconnectAttempts: number = 0; + protected maxReconnectAttempts: number = 3; + protected reconnectTimeoutId?: NodeJS.Timeout; + protected isManualClose: boolean = false; + + /** + * @param url - Single RPC URL or array of RPC URLs for multi-endpoint failover + * @param wsEndpoint - WebSocket endpoint path (default: "/websocket") + * @param options - Additional options including custom WebSocket constructor + */ constructor( - protected readonly url: string, + url: string | string[], protected readonly wsEndpoint: string = "/websocket", protected readonly options: { wsObject?: new (url: string, protocols?: string | string[]) => WebSocket; + /** Maximum reconnect attempts per endpoint before trying next. Default: 3 */ + maxReconnectAttempts?: number; } = {} ) { + // Support both single string (backward compatible) and array of URLs + this.urls = Array.isArray(url) ? url : [url]; + + if (this.urls.length === 0) { + throw new Error("At least one RPC URL must be provided"); + } + + this.maxReconnectAttempts = options.maxReconnectAttempts ?? 3; + this.open(); } + /** Get the current RPC URL being used */ + protected get url(): string { + return this.urls[this.currentUrlIndex]; + } + protected getWsEndpoint(): string { let url = this.url; if (url.startsWith("http")) { @@ -82,18 +114,75 @@ export class TxTracer { } open() { + const wsUrl = this.getWsEndpoint(); + console.log(`[TxTracer] Connecting to WebSocket: ${wsUrl}`); + this.ws = this.options.wsObject - ? new this.options.wsObject(this.getWsEndpoint()) - : new WebSocket(this.getWsEndpoint()); + ? new this.options.wsObject(wsUrl) + : new WebSocket(wsUrl); this.ws.onopen = this.onOpen; this.ws.onmessage = this.onMessage; this.ws.onclose = this.onClose; + this.ws.onerror = this.onError; } close() { + this.isManualClose = true; + if (this.reconnectTimeoutId) { + clearTimeout(this.reconnectTimeoutId); + this.reconnectTimeoutId = undefined; + } this.ws.close(); } + /** + * Attempt to reconnect with exponential backoff and endpoint failover. + * Tries the current endpoint multiple times before moving to the next one. + */ + protected reconnect() { + if (this.isManualClose) { + console.log("[TxTracer] Manual close, not reconnecting"); + return; + } + + this.reconnectAttempts++; + + // Check if we should try the next endpoint + if (this.reconnectAttempts > this.maxReconnectAttempts) { + this.reconnectAttempts = 0; + const previousIndex = this.currentUrlIndex; + this.currentUrlIndex = (this.currentUrlIndex + 1) % this.urls.length; + + // If we've tried all endpoints, wait longer before cycling again + if (this.currentUrlIndex === 0 && this.urls.length > 1) { + console.warn( + `[TxTracer] All ${this.urls.length} endpoints failed. Cycling back to first endpoint after delay.` + ); + // Wait 10 seconds before trying all endpoints again + this.reconnectTimeoutId = setTimeout(() => { + this.open(); + }, 10000); + return; + } + + console.warn( + `[TxTracer] Switching from endpoint ${previousIndex} to ${this.currentUrlIndex}` + ); + } + + // Exponential backoff: 1s, 2s, 4s + const backoffDelay = Math.min(Math.pow(2, this.reconnectAttempts - 1) * 1000, 8000); + + console.log( + `[TxTracer] Reconnect attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts} ` + + `for endpoint ${this.currentUrlIndex} in ${backoffDelay}ms` + ); + + this.reconnectTimeoutId = setTimeout(() => { + this.open(); + }, backoffDelay); + } + get numberOfSubscriberOrPendingQuery(): number { return ( this.newBlockSubscribes.length + @@ -132,6 +221,12 @@ export class TxTracer { } protected readonly onOpen = (e: Event) => { + console.log(`[TxTracer] WebSocket connected successfully to ${this.url}`); + + // Reset reconnect attempts on successful connection + this.reconnectAttempts = 0; + this.isManualClose = false; + if (this.newBlockSubscribes.length > 0) { this.sendSubscribeBlockRpc(); } @@ -210,9 +305,29 @@ export class TxTracer { }; protected readonly onClose = (e: CloseEvent) => { + console.warn( + `[TxTracer] WebSocket closed. Code: ${e.code}, Reason: ${e.reason || "No reason provided"}` + ); + for (const listener of this.listeners.close ?? []) { listener(e); } + + // Attempt to reconnect unless this was a manual close + if (!this.isManualClose) { + this.reconnect(); + } + }; + + protected readonly onError = (e: ErrorEvent) => { + console.error(`[TxTracer] WebSocket error:`, e.message || e); + + for (const listener of this.listeners.error ?? []) { + listener(e); + } + + // Note: onError is typically followed by onClose, so reconnection + // will be handled there. We just log the error here. }; /**