diff --git a/.github/workflows/snapshot-fork-benchmarks.yml b/.github/workflows/snapshot-fork-benchmarks.yml index 61ecd08c..8fd7f622 100644 --- a/.github/workflows/snapshot-fork-benchmarks.yml +++ b/.github/workflows/snapshot-fork-benchmarks.yml @@ -58,6 +58,7 @@ jobs: - cloudflare-r2 - tigris - azure-blob + - tensorlake # vercel-blob is disabled pending an adapter fix: its manifest-based # snapshot/fork emulation is unreliable on Vercel Blob's eventually- # consistent overwrites (stale reads + lost deletes). See providers.ts. @@ -80,7 +81,7 @@ jobs: run: rm -rf results/snapshot-fork/ - name: Run snapshot/fork test run: | - . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts \ --provider ${{ matrix.provider }} \ diff --git a/.github/workflows/storage-benchmarks.yml b/.github/workflows/storage-benchmarks.yml index a5bd4c12..52acb915 100644 --- a/.github/workflows/storage-benchmarks.yml +++ b/.github/workflows/storage-benchmarks.yml @@ -60,6 +60,7 @@ jobs: - vercel-blob - gcs - azure-blob + - tensorlake file_size: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.file_size != '' && fromJson(format('["{0}"]', github.event.inputs.file_size))) || (github.event_name == 'push' && fromJson('["1MB"]')) || fromJson('["1MB","4MB","10MB","16MB"]') }} steps: - uses: actions/checkout@v4 @@ -79,7 +80,7 @@ jobs: run: rm -rf results/storage/ - name: Run storage benchmark run: | - . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' FILE_SIZE="${{ matrix.file_size }}" npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts \ diff --git a/benchmarks/scripts/provider-vars.json b/benchmarks/scripts/provider-vars.json index c6523de1..5a71929c 100644 --- a/benchmarks/scripts/provider-vars.json +++ b/benchmarks/scripts/provider-vars.json @@ -168,6 +168,12 @@ "AZURE_ACCOUNT_NAME", "AZURE_ACCOUNT_KEY", "AZURE_CONTAINER" + ], + "tensorlake": [ + "TENSORLAKE_API_KEY", + "TENSORLAKE_ORGANIZATION_ID", + "TENSORLAKE_PROJECT_ID", + "TENSORLAKE_FILESYSTEM" ] }, "scale": { @@ -267,6 +273,12 @@ "AZURE_SNAPSHOT_ACCOUNT_NAME", "AZURE_SNAPSHOT_ACCOUNT_KEY", "AZURE_SNAPSHOT_CONTAINER" + ], + "tensorlake": [ + "TENSORLAKE_API_KEY", + "TENSORLAKE_ORGANIZATION_ID", + "TENSORLAKE_PROJECT_ID", + "TENSORLAKE_FILESYSTEM" ] } } diff --git a/benchmarks/storage/providers.ts b/benchmarks/storage/providers.ts index 253c2da2..42b489cf 100644 --- a/benchmarks/storage/providers.ts +++ b/benchmarks/storage/providers.ts @@ -5,6 +5,7 @@ import { tigris } from '@storagesdk/adapters/tigris'; import { vercel } from '@storagesdk/adapters/vercel'; import { gcs } from '@storagesdk/adapters/gcs'; import { azure } from '@storagesdk/adapters/azure'; +import { tensorlake } from '@storagesdk/adapters/tensorlake'; import type { StorageProviderConfig } from './types.js'; /** @@ -187,6 +188,28 @@ export const storageProviders: StorageProviderConfig[] = [ }), }, }, + { + name: 'tensorlake', + requiredEnvVars: [ + 'TENSORLAKE_API_KEY', + 'TENSORLAKE_ORGANIZATION_ID', + 'TENSORLAKE_PROJECT_ID', + 'TENSORLAKE_FILESYSTEM', + ], + bucket: process.env.TENSORLAKE_FILESYSTEM!, + createStorage: () => new Storage({ + adapter: tensorlake({ + apiKey: process.env.TENSORLAKE_API_KEY!, + organizationId: process.env.TENSORLAKE_ORGANIZATION_ID!, + projectId: process.env.TENSORLAKE_PROJECT_ID!, + filesystem: process.env.TENSORLAKE_FILESYSTEM!, + ...(process.env.TENSORLAKE_API_URL + ? { apiUrl: process.env.TENSORLAKE_API_URL } + : {}), + }), + }), + fileSizes: [1 * 1024 * 1024, 4 * 1024 * 1024, 10 * 1024 * 1024, 16 * 1024 * 1024], + }, // // add providers above ]; diff --git a/package.json b/package.json index 60927169..434883df 100644 --- a/package.json +++ b/package.json @@ -47,12 +47,14 @@ "bench:storage:s3": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --provider aws-s3", "bench:storage:r2": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --provider cloudflare-r2", "bench:storage:tigris": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --provider tigris", + "bench:storage:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --provider tensorlake", "bench:storage:1mb": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --file-size 1MB", "bench:storage:4mb": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --file-size 4MB", "bench:storage:10mb": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --file-size 10MB", "bench:storage:16mb": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/storage.bench.ts --file-size 16MB", "bench:snapshot-fork": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts", "bench:snapshot-fork:tigris": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --provider tigris", + "bench:snapshot-fork:tensorlake": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --provider tensorlake", "bench:snapshot-fork:s3": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --provider aws-s3", "bench:snapshot-fork:wide": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --dataset wide", "bench:snapshot-fork:deep": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --dataset deep", @@ -137,8 +139,8 @@ "@computesdk/vercel": "^1.7.31", "@electric-sql/pglite": "^0.2.14", "@google-cloud/storage": "^7.21.0", - "@storagesdk/adapters": "^0.7.1", - "@storagesdk/core": "^0.4.2", + "@storagesdk/adapters": "^0.10.0", + "@storagesdk/core": "^0.5.0", "@superserve/sdk": "^0.8.1", "@tigrisdata/storage": "^3.16.0", "@vercel/blob": "^2.5.0", @@ -148,7 +150,8 @@ "p-limit": "^7.3.0", "pg": "^8.22.0", "playwright-core": "^1.61.1", - "sql.js": "^1.13.0" + "sql.js": "^1.13.0", + "tensorlake": "0.5.89" }, "devDependencies": { "@changesets/cli": "^2.27.12", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 98934634..a39768d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -153,11 +153,11 @@ importers: specifier: ^7.21.0 version: 7.21.0 '@storagesdk/adapters': - specifier: ^0.7.1 - version: 0.7.1(@aws-sdk/client-s3@3.1092.0)(@aws-sdk/lib-storage@3.1092.0(@aws-sdk/client-s3@3.1092.0))(@aws-sdk/s3-presigned-post@3.1092.0)(@aws-sdk/s3-request-presigner@3.1092.0)(@azure/storage-blob@12.33.0)(@google-cloud/storage@7.21.0)(@tigrisdata/storage@3.17.2)(@vercel/blob@2.6.1) + specifier: ^0.10.0 + version: 0.10.0(@aws-sdk/client-s3@3.1092.0)(@aws-sdk/lib-storage@3.1092.0(@aws-sdk/client-s3@3.1092.0))(@aws-sdk/s3-presigned-post@3.1092.0)(@aws-sdk/s3-request-presigner@3.1092.0)(@azure/storage-blob@12.33.0)(@google-cloud/storage@7.21.0)(@tigrisdata/storage@3.17.2)(@vercel/blob@2.6.1)(tensorlake@0.5.89(bufferutil@4.1.0)(utf-8-validate@6.0.6)) '@storagesdk/core': - specifier: ^0.4.2 - version: 0.4.2 + specifier: ^0.5.0 + version: 0.5.0 '@superserve/sdk': specifier: ^0.8.1 version: 0.8.1 @@ -188,6 +188,9 @@ importers: sql.js: specifier: ^1.13.0 version: 1.14.1 + tensorlake: + specifier: 0.5.89 + version: 0.5.89(bufferutil@4.1.0)(utf-8-validate@6.0.6) devDependencies: '@changesets/cli': specifier: ^2.27.12 @@ -2099,8 +2102,8 @@ packages: resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} engines: {node: '>=14.0.0'} - '@storagesdk/adapters@0.7.1': - resolution: {integrity: sha512-0mkZVdg/AVfazyIvT602SFV4qOZB2IcEoDmzu/WqMS0qKQGmMLk2Qe7AT57OqUOQshM61iYA4e2bz6xUGll40A==} + '@storagesdk/adapters@0.10.0': + resolution: {integrity: sha512-/742qc+DxY2nVFFMB+RI1NHnIC5hO+clE1Fu3F9nqnwuMp+tDehz6nPfaendAZZ4apgIdZ8EhGY2vQgRbRTjZQ==} engines: {node: '>=20'} peerDependencies: '@aws-sdk/client-s3': ^3.0.0 @@ -2109,9 +2112,12 @@ packages: '@aws-sdk/s3-request-presigner': ^3.0.0 '@azure/storage-blob': ^12.0.0 '@google-cloud/storage': ^7.0.0 + '@mesadev/sdk': ^0.38.0 '@octokit/rest': ^21.0.0 - '@tigrisdata/storage': ^3.11.0 + '@pierre/storage': ^1.13.0 + '@tigrisdata/storage': ^3.17.1 '@vercel/blob': ^2.3.0 + tensorlake: '>=0.5.89 <0.6.0' vitest: ^4.0.0 webdav: ^5.10.0 peerDependenciesMeta: @@ -2127,19 +2133,25 @@ packages: optional: true '@google-cloud/storage': optional: true + '@mesadev/sdk': + optional: true '@octokit/rest': optional: true + '@pierre/storage': + optional: true '@tigrisdata/storage': optional: true '@vercel/blob': optional: true + tensorlake: + optional: true vitest: optional: true webdav: optional: true - '@storagesdk/core@0.4.2': - resolution: {integrity: sha512-TASJvFpf4tHM52Sgg3AWMF1VsW6tiZAf6cG1PWktyqKWm+P4Iaz8cILEpD1VncvC4DzFYKeXRkq/xgaY3phKtg==} + '@storagesdk/core@0.5.0': + resolution: {integrity: sha512-v4Ya5h4Ulc5yj+FO0iNCm05hvc1UvvFJxT/e44Ip/WcXMHpHAAQioOGG+S89iKJ+YXywRhCjwz0S2141gH/agA==} engines: {node: '>=20'} '@superserve/sdk@0.7.8': @@ -4905,6 +4917,11 @@ packages: engines: {node: '>=22.0.0'} hasBin: true + tensorlake@0.5.89: + resolution: {integrity: sha512-ibLb35jxG0B77Rpmu6trjYYJ+qEBU5YhtQvzCYrTtkA3d8tCJIOYkKTfwULTyz6sgNO7lj5JPFA+vHnSHZvszw==} + engines: {node: '>=22.0.0'} + hasBin: true + term-canvas@0.0.5: resolution: {integrity: sha512-eZ3rIWi5yLnKiUcsW8P79fKyooaLmyLWAGqBhFspqMxRNUiB4GmHHk5AzQ4LxvFbJILaXqQZLwbbATLOhCFwkw==} @@ -7743,9 +7760,9 @@ snapshots: '@smithy/util-buffer-from': 2.2.0 tslib: 2.8.1 - '@storagesdk/adapters@0.7.1(@aws-sdk/client-s3@3.1092.0)(@aws-sdk/lib-storage@3.1092.0(@aws-sdk/client-s3@3.1092.0))(@aws-sdk/s3-presigned-post@3.1092.0)(@aws-sdk/s3-request-presigner@3.1092.0)(@azure/storage-blob@12.33.0)(@google-cloud/storage@7.21.0)(@tigrisdata/storage@3.17.2)(@vercel/blob@2.6.1)': + '@storagesdk/adapters@0.10.0(@aws-sdk/client-s3@3.1092.0)(@aws-sdk/lib-storage@3.1092.0(@aws-sdk/client-s3@3.1092.0))(@aws-sdk/s3-presigned-post@3.1092.0)(@aws-sdk/s3-request-presigner@3.1092.0)(@azure/storage-blob@12.33.0)(@google-cloud/storage@7.21.0)(@tigrisdata/storage@3.17.2)(@vercel/blob@2.6.1)(tensorlake@0.5.89(bufferutil@4.1.0)(utf-8-validate@6.0.6))': dependencies: - '@storagesdk/core': 0.4.2 + '@storagesdk/core': 0.5.0 optionalDependencies: '@aws-sdk/client-s3': 3.1092.0 '@aws-sdk/lib-storage': 3.1092.0(@aws-sdk/client-s3@3.1092.0) @@ -7755,8 +7772,9 @@ snapshots: '@google-cloud/storage': 7.21.0 '@tigrisdata/storage': 3.17.2 '@vercel/blob': 2.6.1 + tensorlake: 0.5.89(bufferutil@4.1.0)(utf-8-validate@6.0.6) - '@storagesdk/core@0.4.2': {} + '@storagesdk/core@0.5.0': {} '@superserve/sdk@0.7.8': {} @@ -10828,6 +10846,14 @@ snapshots: - bufferutil - utf-8-validate + tensorlake@0.5.89(bufferutil@4.1.0)(utf-8-validate@6.0.6): + dependencies: + undici: 8.3.0 + ws: 8.21.1(bufferutil@4.1.0)(utf-8-validate@6.0.6) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + term-canvas@0.0.5: {} term-size@2.2.1: {}