Merge pull request #148 from autonomys/fix-serialize-bigint

jfrank-summit · web-flow · commit e7977f0cf1ba · 2025-12-02T12:43:27.000-07:00
fix: properly serialize bigint in objects
diff --git a/.github/workflows/image-deployment.yml b/.github/workflows/image-deployment.yml
@@ -8,9 +8,7 @@ on:
         required: true
         type: choice
         options:
-          - files_gateway_taurus_staging
           - files_gateway_mainnet_staging
-          - files_gateway_taurus_production
           - files_gateway_mainnet_production
       gateway_image_tag:
         description: 'Docker image tag visit in the releases page at (https://github.com/autonomys/subspace/pkgs/container/gateway)'
@@ -53,7 +51,7 @@ jobs:
             echo "Using manual inputs"
           else
             # Push trigger - use defaults
-            TARGET_MACHINES="files_gateway_taurus_staging"
+            TARGET_MACHINES="files_gateway_mainnet_staging"
             GATEWAY_IMAGE_TAG=""  # or set a default like "latest"
             INDEXER_IMAGE_TAG=""  # or set a default like "latest"
             FILE_RETRIEVER_IMAGE_TAG=""  # or set a default like "latest"
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
@@ -7,7 +7,6 @@ fact_caching = memory
 retry_files_enabled = False
 deprecation_warnings = False
 callback_whitelist = profile_tasks
-stdout_callback = yaml
 
 [ssh_connection]
 ssh_args = -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=60s
diff --git a/ansible/auto-files-gateway-deployment.yaml b/ansible/auto-files-gateway-deployment.yaml
@@ -61,6 +61,10 @@
         echo "$env_file" > {{ env_file_path }}
       register: env_output
 
+    - name: Pull docker images
+      ansible.builtin.shell: |
+        /usr/bin/docker compose --env-file {{ env_file_path }} -f ~/deploy/auto-files-gateway/docker/file-retriever/docker-compose.yml -f ~/deploy/auto-files-gateway/docker/file-retriever/docker-compose.prod.yml -f ~/deploy/auto-files-gateway/docker/object-mapping-indexer/docker-compose.yml -f ~/deploy/auto-files-gateway/docker/object-mapping-indexer/docker-compose.prod.yml pull
+
     - name: Launch docker compose
       ansible.builtin.shell: |
         /usr/bin/docker compose --env-file {{ env_file_path }} -f ~/deploy/auto-files-gateway/docker/file-retriever/docker-compose.yml -f ~/deploy/auto-files-gateway/docker/file-retriever/docker-compose.prod.yml -f ~/deploy/auto-files-gateway/docker/object-mapping-indexer/docker-compose.yml -f ~/deploy/auto-files-gateway/docker/object-mapping-indexer/docker-compose.prod.yml up -d
diff --git a/services/file-retriever/__tests__/toSerializable.spec.ts b/services/file-retriever/__tests__/toSerializable.spec.ts
@@ -0,0 +1,178 @@
+import { toSerializable } from '../src/utils/express.js'
+
+describe('toSerializable', () => {
+  describe('null and undefined handling', () => {
+    it('should return null as-is', () => {
+      expect(toSerializable(null)).toBeNull()
+    })
+
+    it('should return undefined as-is', () => {
+      expect(toSerializable(undefined)).toBeUndefined()
+    })
+  })
+
+  describe('primitive handling', () => {
+    it('should return strings as-is', () => {
+      expect(toSerializable('hello')).toBe('hello')
+    })
+
+    it('should return numbers as-is', () => {
+      expect(toSerializable(42)).toBe(42)
+    })
+
+    it('should return booleans as-is', () => {
+      expect(toSerializable(true)).toBe(true)
+      expect(toSerializable(false)).toBe(false)
+    })
+  })
+
+  describe('bigint handling', () => {
+    it('should convert bigint to string', () => {
+      expect(toSerializable(BigInt(123456789))).toBe('123456789')
+    })
+
+    it('should convert large bigint to string', () => {
+      const largeBigInt = BigInt('9007199254740993')
+      expect(toSerializable(largeBigInt)).toBe('9007199254740993')
+    })
+  })
+
+  describe('Date handling', () => {
+    it('should convert Date to ISO string', () => {
+      const date = new Date('2025-12-02T10:30:00.000Z')
+      expect(toSerializable(date)).toBe('2025-12-02T10:30:00.000Z')
+    })
+
+    it('should handle Date with different timezone', () => {
+      const date = new Date(0) // Unix epoch
+      expect(toSerializable(date)).toBe('1970-01-01T00:00:00.000Z')
+    })
+  })
+
+  describe('array handling', () => {
+    it('should process simple arrays', () => {
+      expect(toSerializable([1, 2, 3])).toEqual([1, 2, 3])
+    })
+
+    it('should recursively process array elements', () => {
+      const date = new Date('2025-12-02T10:30:00.000Z')
+      const input = [BigInt(123), date, 'hello']
+      expect(toSerializable(input)).toEqual([
+        '123',
+        '2025-12-02T10:30:00.000Z',
+        'hello',
+      ])
+    })
+
+    it('should handle nested arrays', () => {
+      const input = [[BigInt(1)], [BigInt(2)]]
+      expect(toSerializable(input)).toEqual([['1'], ['2']])
+    })
+  })
+
+  describe('object handling', () => {
+    it('should process simple objects', () => {
+      expect(toSerializable({ a: 1, b: 2 })).toEqual({ a: 1, b: 2 })
+    })
+
+    it('should recursively process object values', () => {
+      const date = new Date('2025-12-02T10:30:00.000Z')
+      const input = {
+        size: BigInt(1024),
+        timestamp: date,
+        name: 'test',
+      }
+      expect(toSerializable(input)).toEqual({
+        size: '1024',
+        timestamp: '2025-12-02T10:30:00.000Z',
+        name: 'test',
+      })
+    })
+
+    it('should handle nested objects', () => {
+      const input = {
+        outer: {
+          inner: {
+            value: BigInt(42),
+          },
+        },
+      }
+      expect(toSerializable(input)).toEqual({
+        outer: {
+          inner: {
+            value: '42',
+          },
+        },
+      })
+    })
+  })
+
+  describe('complex nested structures', () => {
+    it('should handle ExtendedIPLDMetadata-like structure', () => {
+      const input = {
+        cid: 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi',
+        blockHeight: 12345,
+        blockHash: '0xabc123',
+        extrinsicId: 'ext-001',
+        extrinsicHash: '0xdef456',
+        indexInBlock: 0,
+        links: ['link1', 'link2'],
+        blake3Hash: 'hash123',
+        timestamp: new Date('2025-12-02T10:30:00.000Z'),
+        size: BigInt('1048576'),
+        type: 'file',
+      }
+
+      const result = toSerializable(input)
+
+      expect(result).toEqual({
+        cid: 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi',
+        blockHeight: 12345,
+        blockHash: '0xabc123',
+        extrinsicId: 'ext-001',
+        extrinsicHash: '0xdef456',
+        indexInBlock: 0,
+        links: ['link1', 'link2'],
+        blake3Hash: 'hash123',
+        timestamp: '2025-12-02T10:30:00.000Z',
+        size: '1048576',
+        type: 'file',
+      })
+    })
+
+    it('should handle array of objects with mixed types', () => {
+      const input = [
+        {
+          id: 1,
+          value: BigInt(100),
+          created: new Date('2025-01-01T00:00:00Z'),
+        },
+        {
+          id: 2,
+          value: BigInt(200),
+          created: new Date('2025-01-02T00:00:00Z'),
+        },
+      ]
+
+      expect(toSerializable(input)).toEqual([
+        { id: 1, value: '100', created: '2025-01-01T00:00:00.000Z' },
+        { id: 2, value: '200', created: '2025-01-02T00:00:00.000Z' },
+      ])
+    })
+
+    it('should handle objects with array properties containing special types', () => {
+      const input = {
+        timestamps: [
+          new Date('2025-01-01T00:00:00Z'),
+          new Date('2025-01-02T00:00:00Z'),
+        ],
+        sizes: [BigInt(100), BigInt(200)],
+      }
+
+      expect(toSerializable(input)).toEqual({
+        timestamps: ['2025-01-01T00:00:00.000Z', '2025-01-02T00:00:00.000Z'],
+        sizes: ['100', '200'],
+      })
+    })
+  })
+})
diff --git a/services/file-retriever/src/http/controllers/file.ts b/services/file-retriever/src/http/controllers/file.ts
@@ -3,7 +3,7 @@ import { authMiddleware } from '../middlewares/auth.js'
 import { fileComposer } from '../../services/fileComposer.js'
 import { pipeline } from 'stream'
 import { logger } from '../../drivers/logger.js'
-import { asyncSafeHandler } from '../../utils/express.js'
+import { asyncSafeHandler, toSerializable } from '../../utils/express.js'
 import { uniqueHeaderValue } from '../../utils/http.js'
 import { HttpError } from '../middlewares/error.js'
 import { dsnFetcher } from '../../services/dsnFetcher.js'
@@ -28,7 +28,7 @@ fileRouter.get(
     }
 
     const metadata = await dsnFetcher.fetchNodeMetadata(cid)
-    res.status(200).json(metadata)
+    res.status(200).json(toSerializable(metadata))
   }),
 )
 
diff --git a/services/file-retriever/src/http/controllers/node.ts b/services/file-retriever/src/http/controllers/node.ts
@@ -1,6 +1,6 @@
 import { Router } from 'express'
 import { dsnFetcher } from '../../services/dsnFetcher.js'
-import { asyncSafeHandler } from '../../utils/express.js'
+import { asyncSafeHandler, toSerializable } from '../../utils/express.js'
 import { safeIPLDDecode } from '../../utils/dagData.js'
 
 const nodeRouter = Router()
@@ -23,7 +23,7 @@ nodeRouter.get(
 
     const ipldNode = safeIPLDDecode(node)
 
-    res.json(ipldNode)
+    res.json(toSerializable(ipldNode))
   }),
 )
 
diff --git a/services/file-retriever/src/utils/express.ts b/services/file-retriever/src/utils/express.ts
@@ -11,3 +11,26 @@ export const asyncSafeHandler = (
     }
   }
 }
+
+export const toSerializable = <T>(obj: T): T => {
+  if (obj === null || obj === undefined) {
+    return obj
+  }
+  if (typeof obj === 'bigint') {
+    return obj.toString() as unknown as T
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(toSerializable) as unknown as T
+  }
+  if (obj instanceof Date) {
+    return obj.toISOString() as unknown as T
+  }
+  if (typeof obj === 'object') {
+    const result: Record<string, unknown> = {}
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = toSerializable(value)
+    }
+    return result as T
+  }
+  return obj
+}
