feat: Add Multi-Endpoint Support with Automatic Retry and Failover

packages/bridge/src/ibc/index.ts

@@ -321,20 +321,26 @@ export class IbcBridgeProvider implements BridgeProvider {
     );
     const toChain = this.ctx.chainList.find((c) => c.chain_id === toChainId);
 
-    const fromRpc = fromChain?.apis.rpc[0]?.address;
-    const toRpc = toChain?.apis.rpc[0]?.address;
+    // Get all RPC endpoints for automatic fallback
+    const fromRpcUrls = fromChain?.apis.rpc.map((rpc) => rpc.address) ?? [];
+    const toRpcUrls = toChain?.apis.rpc.map((rpc) => rpc.address) ?? [];
 
-    if (!fromChain || !toChain || !fromRpc || !toRpc) {
+    if (
+      !fromChain ||
+      !toChain ||
+      fromRpcUrls.length === 0 ||
+      toRpcUrls.length === 0
+    ) {
       throw new BridgeQuoteError({
         bridgeId: IbcBridgeProvider.ID,
         errorType: "UnsupportedQuoteError",
-        message: "Chain not found",
+        message: "Chain not found or no RPC endpoints available",
       });
     }
 
     const [fromBlockTimeMs, toBlockTimeMs] = await Promise.all([
-      queryRPCStatus({ restUrl: fromRpc }).then(calcAverageBlockTimeMs),
-      queryRPCStatus({ restUrl: toRpc }).then(calcAverageBlockTimeMs),
+      queryRPCStatus({ rpcUrls: fromRpcUrls }).then(calcAverageBlockTimeMs),
+      queryRPCStatus({ rpcUrls: toRpcUrls }).then(calcAverageBlockTimeMs),
     ]);
 
     // convert to seconds

packages/server/src/queries/__tests__/create-node-query.spec.ts

@@ -25,7 +25,8 @@ describe("createNodeQuery", () => {
     const result = await query({ chainList: MockChains, ...params });
 
     expect(apiClient).toHaveBeenCalledWith(
-      `https://lcd-osmosis.keplr.app${path}`
+      `https://lcd-osmosis.keplr.app${path}`,
+      expect.any(Object)
     );
     expect(result).toEqual(mockResult);
   });
@@ -45,7 +46,8 @@ describe("createNodeQuery", () => {
     const result = await query({ chainList: MockChains, ...params });
 
     expect(apiClient).toHaveBeenCalledWith(
-      `https://lcd-osmosis.keplr.app${path(params)}`
+      `https://lcd-osmosis.keplr.app${path(params)}`,
+      expect.any(Object)
     );
     expect(result).toEqual(mockResult);
   });
@@ -83,7 +85,8 @@ describe("createNodeQuery", () => {
     const result = await query({ chainList: MockChains, ...params });
 
     expect(apiClient).toHaveBeenCalledWith(
-      `https://lcd-cosmoshub.keplr.app${path(params)}`
+      `https://lcd-cosmoshub.keplr.app${path(params)}`,
+      expect.any(Object)
     );
     expect(result).toEqual(mockResult);
   });
@@ -109,8 +112,136 @@ describe("createNodeQuery", () => {
 
     expect(apiClient).toHaveBeenCalledWith(
       `https://lcd-cosmoshub.keplr.app${path(params)}`,
-      { body: "stringBody" }
+      expect.objectContaining({
+        body: "stringBody",
+      })
     );
     expect(result).toEqual(mockResult);
   });
+
+  describe("multi-endpoint retry logic", () => {
+    it("should retry on first endpoint failure and succeed on second attempt", async () => {
+      const mockResult = { data: "test" };
+      (apiClient as jest.Mock)
+        .mockRejectedValueOnce(new Error("Network timeout"))
+        .mockResolvedValueOnce(mockResult);
+
+      const query = createNodeQuery<{ data: string }>({
+        path: "/test",
+        maxRetries: 2,
+      });
+
+      const result = await query({ chainList: MockChains });
+
+      // Should have been called twice (first failure, then success)
+      expect(apiClient).toHaveBeenCalledTimes(2);
+      expect(result).toEqual(mockResult);
+    });
+
+    it("should fallback to second endpoint when first endpoint fails all retries", async () => {
+      const mockResult = { data: "success" };
+      const mockChains = [
+        {
+          ...MockChains[0],
+          apis: {
+            rest: [
+              { address: "https://endpoint1.com" },
+              { address: "https://endpoint2.com" },
+            ],
+            rpc: [],
+          },
+        },
+      ];
+
+      // First endpoint fails 2 times, second endpoint succeeds
+      (apiClient as jest.Mock)
+        .mockRejectedValueOnce(new Error("Endpoint 1 fail"))
+        .mockRejectedValueOnce(new Error("Endpoint 1 fail again"))
+        .mockResolvedValueOnce(mockResult);
+
+      const query = createNodeQuery<{ data: string }>({
+        path: "/test",
+        maxRetries: 2,
+      });
+
+      const result = await query({ chainList: mockChains });
+
+      expect(apiClient).toHaveBeenCalledTimes(3);
+      // Verify second endpoint was called
+      expect(apiClient).toHaveBeenCalledWith(
+        "https://endpoint2.com/test",
+        expect.any(Object)
+      );
+      expect(result).toEqual(mockResult);
+    });
+
+    it("should throw error when all endpoints are exhausted", async () => {
+      const mockChains = [
+        {
+          ...MockChains[0],
+          apis: {
+            rest: [
+              { address: "https://endpoint1.com" },
+              { address: "https://endpoint2.com" },
+            ],
+            rpc: [],
+          },
+        },
+      ];
+
+      // All attempts fail
+      (apiClient as jest.Mock).mockRejectedValue(new Error("All failed"));
+
+      const query = createNodeQuery<{ data: string }>({
+        path: "/test",
+        maxRetries: 2,
+      });
+
+      await expect(query({ chainList: mockChains })).rejects.toThrow(
+        /All 2 REST endpoints failed/
+      );
+
+      // Should have tried: endpoint1 (2 times) + endpoint2 (2 times) = 4 calls
+      expect(apiClient).toHaveBeenCalledTimes(4);
+    });
+
+    it("should respect custom timeout parameter", async () => {
+      const mockResult = { data: "test" };
+      (apiClient as jest.Mock).mockResolvedValue(mockResult);
+
+      const customTimeout = 10000;
+      const query = createNodeQuery<{ data: string }>({
+        path: "/test",
+        timeout: customTimeout,
+      });
+
+      await query({ chainList: MockChains });
+
+      // Should be called with some options
+      expect(apiClient).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.any(Object)
+      );
+    });
+
+    it("should throw error when chain has no REST endpoints", async () => {
+      const mockChains = [
+        {
+          ...MockChains[0],
+          apis: {
+            rest: [],
+            rpc: [],
+          },
+        },
+      ];
+
+      const query = createNodeQuery<{ data: string }>({
+        path: "/test",
+      });
+
+      await expect(query({ chainList: mockChains })).rejects.toThrow(
+        /No REST endpoints available/
+      );
+    });
+  });
 });

packages/server/src/queries/cosmos/rpc-status.ts

@@ -1,4 +1,4 @@
-import { apiClient } from "@osmosis-labs/utils";
+import { apiClient, createMultiEndpointClient } from "@osmosis-labs/utils";
 
 export type Status = {
   node_info: {
@@ -45,14 +45,56 @@ export type QueryStatusResponse = {
   result: Status;
 };
 
-export async function queryRPCStatus({
-  restUrl,
-}: {
-  restUrl: string;
-}): Promise<QueryStatusResponse> {
-  const data = await apiClient<QueryStatusResponse | Status>(
-    restUrl + "/status"
-  );
+/**
+ * Query RPC status from a chain node.
+ *
+ * Supports both single endpoint (legacy) and multiple endpoints with automatic fallback.
+ * When multiple RPC URLs are provided, will try each endpoint with retry logic before failing.
+ *
+ * @param params - Either { restUrl: string } for single endpoint or { rpcUrls: string[] } for multi-endpoint
+ * @returns The RPC status response
+ *
+ * @example
+ * // Single endpoint (legacy, backward compatible)
+ * const status = await queryRPCStatus({ restUrl: "https://rpc.osmosis.zone" });
+ *
+ * // Multiple endpoints with automatic fallback
+ * const status = await queryRPCStatus({
+ *   rpcUrls: [
+ *     "https://rpc.osmosis.zone",
+ *     "https://osmosis-rpc.polkachu.com",
+ *     "https://rpc-osmosis.blockapsis.com"
+ *   ]
+ * });
+ */
+export async function queryRPCStatus(
+  params: { restUrl: string } | { rpcUrls: string[] }
+): Promise<QueryStatusResponse> {
+  let data: QueryStatusResponse | Status;
+
+  // Check if using new multi-endpoint API
+  if ("rpcUrls" in params) {
+    const { rpcUrls } = params;
+
+    if (!rpcUrls || rpcUrls.length === 0) {
+      throw new Error("At least one RPC URL must be provided");
+    }
+
+    // Use multi-endpoint client for automatic retry and fallback
+    const client = createMultiEndpointClient(
+      rpcUrls.map((url) => ({ address: url })),
+      {
+        maxRetries: 3,
+        timeout: 5000,
+      }
+    );
+
+    data = await client.fetch<QueryStatusResponse | Status>("/status");
+  } else {
+    // Legacy single endpoint - backward compatible
+    const { restUrl } = params;
+    data = await apiClient<QueryStatusResponse | Status>(restUrl + "/status");
+  }
 
   // some chains return a nonstandard response that does not include the jsonrpc field
   // but rather just the status object

packages/server/src/queries/create-node-query.ts

@@ -14,13 +14,23 @@ import { runIfFn } from "@osmosis-labs/utils";
  *  the request will be sent to the RPC endpoint since it's
  *  assumed the LCD rest endpoint only accepts GET requests.
  *
+ *  ENDPOINT FALLBACK:
+ *  The REST endpoints come from the chain's asset list (osmosis-labs/assetlists).
+ *  Each chain can have multiple REST endpoints for redundancy. This function will:
+ *  1. Try each endpoint in order from the chain.apis.rest array
+ *  2. Retry each endpoint up to `maxRetries` times with exponential backoff
+ *  3. Move to the next endpoint if all retries fail
+ *  4. Throw an error only if all endpoints have been exhausted
+ *
  *  - [ ]: add node query error handling (like deserializing code)
  *  and extend `ApiClientError`
  */
 export const createNodeQuery =
   <Result, PathParameters extends Record<any, any> | unknown = unknown>({
     path,
     options,
+    maxRetries = 3,
+    timeout = 5000,
   }: {
     path: string | ((params: PathParameters) => string);
     /** Additional query options such as
@@ -29,6 +39,11 @@ export const createNodeQuery =
      *  due to the node-query invariant of rest services
      *  only accepting GET requests. */
     options?: (params: PathParameters) => ClientOptions;
+    /** Maximum number of retries per endpoint before trying the next one.
+     *  Default: 3 */
+    maxRetries?: number;
+    /** Request timeout in milliseconds. Default: 5000ms */
+    timeout?: number;
   }) =>
   async (
     ...params: PathParameters extends Record<any, any>
@@ -47,15 +62,72 @@ export const createNodeQuery =
 
     if (!chain) throw new Error(`Chain ${chainId} not found`);
 
+    // Get all available REST endpoints from the chain's asset list
+    const restEndpoints = chain.apis.rest;
+
+    if (!restEndpoints || restEndpoints.length === 0) {
+      throw new Error(`No REST endpoints available for chain ${chainId}`);
+    }
+
     const opts = options?.(...(params as [PathParameters]));
+    const pathStr = runIfFn(
+      path,
+      ...((params as [PathParameters & { chainId?: string }]) ?? [])
+    );
+
+    let lastError: Error | null = null;
+
+    // Try each endpoint with retries
+    for (let endpointIndex = 0; endpointIndex < restEndpoints.length; endpointIndex++) {
+      const endpoint = restEndpoints[endpointIndex];
+      const baseUrl = endpoint.address;
+
+      // Retry current endpoint with exponential backoff
+      for (let retry = 0; retry < maxRetries; retry++) {
+        try {
+          const url = new URL(pathStr, baseUrl);
+
+          // AbortSignal.timeout is only available in Node 17.3+ and modern browsers
+          const timeoutSignal =
+            typeof AbortSignal.timeout === "function"
+              ? AbortSignal.timeout(timeout)
+              : undefined;
+
+          const result = await apiClient<Result>(url.toString(), {
+            ...opts,
+            ...(timeoutSignal && { signal: timeoutSignal }),
+          });
+
+          // Success! Return immediately
+          return result;
+        } catch (error) {
+          lastError = error as Error;
+
+          // Log the failure for debugging
+          if (retry === maxRetries - 1) {
+            // Last retry for this endpoint
+            console.warn(
+              `[createNodeQuery] Endpoint ${baseUrl} failed after ${maxRetries} retries. ` +
+                (endpointIndex < restEndpoints.length - 1
+                  ? "Trying next endpoint..."
+                  : "No more endpoints available."),
+              lastError.message
+            );
+          }
+
+          // Wait before retry with exponential backoff (100ms, 200ms, 400ms, ...)
+          // Don't wait on the last retry if there are more endpoints to try
+          if (retry < maxRetries - 1) {
+            const backoffDelay = Math.pow(2, retry) * 100;
+            await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+          }
+        }
+      }
+    }
 
-    const url = new URL(
-      runIfFn(
-        path,
-        ...((params as [PathParameters & { chainId?: string }]) ?? [])
-      ),
-      chain.apis.rest[0].address
+    // All endpoints exhausted
+    throw new Error(
+      `All ${restEndpoints.length} REST endpoints failed for chain ${chainId} after ${maxRetries} retries each. ` +
+        `Last error: ${lastError?.message || "Unknown error"}`
     );
-    if (opts) return apiClient<Result>(url.toString(), opts);
-    else return apiClient<Result>(url.toString());
   };