From 68e177fce69e91c792f4bdf4826f883c16c81220 Mon Sep 17 00:00:00 2001 From: matical Date: Mon, 28 Apr 2025 01:20:14 -0700 Subject: [PATCH 1/9] refac(seal): deduplication fetching with preserved array --- packages/seal/src/client.ts | 2 +- packages/seal/src/key-server.ts | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/seal/src/client.ts b/packages/seal/src/client.ts index 4d3ebf6b9..8e4505562 100644 --- a/packages/seal/src/client.ts +++ b/packages/seal/src/client.ts @@ -218,7 +218,7 @@ export class SealClient { /** * Fetch keys from the key servers and update the cache. * - * It is recommended to call this function once for all ids of all encrypted obejcts if + * It is recommended to call this function once for all ids of all encrypted objects if * there are multiple, then call decrypt for each object. This avoids calling fetchKey * individually for each decrypt. * diff --git a/packages/seal/src/key-server.ts b/packages/seal/src/key-server.ts index 206921f72..dba52e39a 100644 --- a/packages/seal/src/key-server.ts +++ b/packages/seal/src/key-server.ts @@ -57,9 +57,12 @@ export async function retrieveKeyServers({ objectIds: string[]; client: SealCompatibleClient; }): Promise { - // todo: do not fetch the same object ID if this is fetched before. - return await Promise.all( - objectIds.map(async (objectId) => { + const uniqueIds = Array.from(new Set(objectIds)); + const fetchedServers: Record = {}; + + // Only fetch key server information for each unique objectId. + await Promise.all( + uniqueIds.map(async (objectId) => { let res; try { res = await client.core.getObject({ @@ -74,7 +77,7 @@ export async function retrieveKeyServers({ throw new UnsupportedFeatureError(`Unsupported key type ${ks.keyType}`); } - return { + fetchedServers[objectId] = { objectId, name: ks.name, url: ks.url, @@ -83,6 +86,15 @@ export async function retrieveKeyServers({ }; }), ); + + return objectIds.map((objectId) => { + if (!fetchedServers[objectId]) { + console.error(`KeyServer ${objectId} not found in fetched servers`); + throw new InvalidGetObjectError(`KeyServer ${objectId} not found`); + } + + return fetchedServers[objectId]; + }); } /** From c1ff5e2c745f11939b71edaea465eb0db236bc9f Mon Sep 17 00:00:00 2001 From: matical Date: Mon, 28 Apr 2025 19:57:09 -0700 Subject: [PATCH 2/9] chore(seal): remove debug logging --- packages/seal/src/key-server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/seal/src/key-server.ts b/packages/seal/src/key-server.ts index dba52e39a..9006df698 100644 --- a/packages/seal/src/key-server.ts +++ b/packages/seal/src/key-server.ts @@ -89,7 +89,6 @@ export async function retrieveKeyServers({ return objectIds.map((objectId) => { if (!fetchedServers[objectId]) { - console.error(`KeyServer ${objectId} not found in fetched servers`); throw new InvalidGetObjectError(`KeyServer ${objectId} not found`); } From dd5e78fb616f4a854d9b8fec6778d8ee512ee76a Mon Sep 17 00:00:00 2001 From: matical Date: Mon, 28 Apr 2025 21:13:11 -0700 Subject: [PATCH 3/9] fix(seal): preserve duplicate key servers in threshold calc --- packages/seal/src/client.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/seal/src/client.ts b/packages/seal/src/client.ts index 8e4505562..91f50673c 100644 --- a/packages/seal/src/client.ts +++ b/packages/seal/src/client.ts @@ -162,10 +162,12 @@ export class SealClient { for (const objectId of this.#serverObjectIds) { serverObjectIdsMap.set(objectId, (serverObjectIdsMap.get(objectId) ?? 0) + 1); } + const servicesMap = new Map(); for (const service of services) { servicesMap.set(service, (servicesMap.get(service) ?? 0) + 1); } + for (const [objectId, count] of serverObjectIdsMap) { if (servicesMap.get(objectId) !== count) { throw new InconsistentKeyServersError( @@ -173,6 +175,7 @@ export class SealClient { ); } } + // Check that the threshold can be met with the client's key servers. if (threshold > this.#serverObjectIds.length) { throw new InvalidThresholdError( @@ -246,7 +249,7 @@ export class SealClient { } let completedServerCount = 0; - const remainingKeyServers = new Set(); + const remainingKeyServers = new Map(); const fullIds = ids.map((id) => createFullId(DST, sessionKey.getPackageId(), id)); // Count a server as completed if it has keys for all fullIds. @@ -256,10 +259,11 @@ export class SealClient { for (const fullId of fullIds) { if (!this.#cachedKeys.has(`${fullId}:${server.objectId}`)) { hasAllKeys = false; - remainingKeyServers.add(server); + remainingKeyServers.set(server.objectId, server); break; } } + if (hasAllKeys) { completedServerCount++; } @@ -271,7 +275,7 @@ export class SealClient { } // Check server validities. - for (const server of remainingKeyServers) { + for (const server of remainingKeyServers.values()) { if (server.keyType !== KeyServerType.BonehFranklinBLS12381) { throw new InvalidKeyServerError( `Server ${server.objectId} has invalid key type: ${server.keyType}`, @@ -285,7 +289,7 @@ export class SealClient { const controller = new AbortController(); const errors: Error[] = []; - const keyFetches = [...remainingKeyServers].map(async (server) => { + const keyFetches = [...remainingKeyServers.values()].map(async (server) => { try { const allKeys = await fetchKeysForAllIds( server.url, @@ -296,6 +300,7 @@ export class SealClient { this.#timeout, controller.signal, ); + // Check validity of the keys and add them to the cache. const receivedIds = new Set(); for (const { fullId, key } of allKeys) { @@ -310,6 +315,7 @@ export class SealClient { console.warn('Received invalid key from key server ' + server.objectId); continue; } + this.#cachedKeys.set(`${fullId}:${server.objectId}`, keyElement); receivedIds.add(fullId); } @@ -321,9 +327,11 @@ export class SealClient { receivedIds.size === expectedIds.size && [...receivedIds].every((id) => expectedIds.has(id)); - // Return early if the completed servers is more than threshold. + // Count each occurrence of this servers objectId from the original keyServers array. if (hasAllKeys) { - completedServerCount++; + const occurrences = keyServers.filter((ks) => ks.objectId === server.objectId).length; + completedServerCount += occurrences; + if (completedServerCount >= threshold) { controller.abort(); } @@ -332,6 +340,7 @@ export class SealClient { if (!controller.signal.aborted) { errors.push(error as Error); } + // If there are too many errors that the threshold is not attainable, return early with error. if (remainingKeyServers.size - errors.length < threshold - completedServerCount) { controller.abort(error); From 9d13b18153ee87af9f5543b14adc49b136c9061a Mon Sep 17 00:00:00 2001 From: matical Date: Tue, 29 Apr 2025 10:10:17 -0700 Subject: [PATCH 4/9] chore: adds changeset --- .changeset/slimy-plants-add.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/slimy-plants-add.md diff --git a/.changeset/slimy-plants-add.md b/.changeset/slimy-plants-add.md new file mode 100644 index 000000000..42e177fa2 --- /dev/null +++ b/.changeset/slimy-plants-add.md @@ -0,0 +1,5 @@ +--- +'@mysten/seal': minor +--- + +introduced deduplication of key server object fetching From 40e34af3979fd4e67ba2a811a88df9e4146116e0 Mon Sep 17 00:00:00 2001 From: matical Date: Tue, 29 Apr 2025 11:38:52 -0700 Subject: [PATCH 5/9] refac(seal): batched server fetching --- packages/seal/src/client.ts | 13 ++++----- packages/seal/src/key-server.ts | 51 ++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/packages/seal/src/client.ts b/packages/seal/src/client.ts index 91f50673c..9c8e2f5e5 100644 --- a/packages/seal/src/client.ts +++ b/packages/seal/src/client.ts @@ -249,7 +249,7 @@ export class SealClient { } let completedServerCount = 0; - const remainingKeyServers = new Map(); + const remainingKeyServers = new Set(); const fullIds = ids.map((id) => createFullId(DST, sessionKey.getPackageId(), id)); // Count a server as completed if it has keys for all fullIds. @@ -259,7 +259,7 @@ export class SealClient { for (const fullId of fullIds) { if (!this.#cachedKeys.has(`${fullId}:${server.objectId}`)) { hasAllKeys = false; - remainingKeyServers.set(server.objectId, server); + remainingKeyServers.add(server); break; } } @@ -275,7 +275,7 @@ export class SealClient { } // Check server validities. - for (const server of remainingKeyServers.values()) { + for (const server of remainingKeyServers) { if (server.keyType !== KeyServerType.BonehFranklinBLS12381) { throw new InvalidKeyServerError( `Server ${server.objectId} has invalid key type: ${server.keyType}`, @@ -289,7 +289,7 @@ export class SealClient { const controller = new AbortController(); const errors: Error[] = []; - const keyFetches = [...remainingKeyServers.values()].map(async (server) => { + const keyFetches = [...remainingKeyServers].map(async (server) => { try { const allKeys = await fetchKeysForAllIds( server.url, @@ -327,10 +327,9 @@ export class SealClient { receivedIds.size === expectedIds.size && [...receivedIds].every((id) => expectedIds.has(id)); - // Count each occurrence of this servers objectId from the original keyServers array. + // Return early if the completed servers is more than threshold. if (hasAllKeys) { - const occurrences = keyServers.filter((ks) => ks.objectId === server.objectId).length; - completedServerCount += occurrences; + completedServerCount++; if (completedServerCount >= threshold) { controller.abort(); diff --git a/packages/seal/src/key-server.ts b/packages/seal/src/key-server.ts index 9006df698..0196500c7 100644 --- a/packages/seal/src/key-server.ts +++ b/packages/seal/src/key-server.ts @@ -58,41 +58,52 @@ export async function retrieveKeyServers({ client: SealCompatibleClient; }): Promise { const uniqueIds = Array.from(new Set(objectIds)); - const fetchedServers: Record = {}; + const { objects } = await client.core.getObjects({ + objectIds: uniqueIds, + }); - // Only fetch key server information for each unique objectId. - await Promise.all( - uniqueIds.map(async (objectId) => { - let res; - try { - res = await client.core.getObject({ - objectId, - }); - } catch (e) { - throw new InvalidGetObjectError(`KeyServer ${objectId} not found; ${(e as Error).message}`); + // Create a single pass lookup map from objectId to key server data. + const serverDataMap = Object.fromEntries( + objects.map((res, i) => { + const objectId = uniqueIds[i]; + + if (res instanceof Error) { + throw new InvalidGetObjectError( + `KeyServer ${objectId} not found; ${(res as Error).message}`, + ); } - const ks = KeyServerMove.parse(res.object.content); + const ks = KeyServerMove.parse(res.content); if (ks.keyType !== 0) { throw new UnsupportedFeatureError(`Unsupported key type ${ks.keyType}`); } - fetchedServers[objectId] = { + return [ objectId, - name: ks.name, - url: ks.url, - keyType: KeyServerType.BonehFranklinBLS12381, - pk: new Uint8Array(ks.pk), - }; + { + objectId, + name: ks.name, + url: ks.url, + pk: new Uint8Array(ks.pk), + }, + ]; }), ); + // Return the preserved order of the input objectIds, creating a new object for each occurence. return objectIds.map((objectId) => { - if (!fetchedServers[objectId]) { + const data = serverDataMap[objectId]; + if (!data) { throw new InvalidGetObjectError(`KeyServer ${objectId} not found`); } - return fetchedServers[objectId]; + return { + objectId, + name: data.name, + url: data.url, + keyType: KeyServerType.BonehFranklinBLS12381, + pk: data.pk, + }; }); } From e57271c5a7a0efa938bdcc39a4772819fbeae115 Mon Sep 17 00:00:00 2001 From: matical Date: Fri, 9 May 2025 10:47:16 -0700 Subject: [PATCH 6/9] refac(seal): update changeset --- .changeset/slimy-plants-add.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/slimy-plants-add.md b/.changeset/slimy-plants-add.md index 42e177fa2..9ef2b22f5 100644 --- a/.changeset/slimy-plants-add.md +++ b/.changeset/slimy-plants-add.md @@ -1,5 +1,5 @@ --- -'@mysten/seal': minor +'@mysten/seal': patch --- introduced deduplication of key server object fetching From b30442a3c7317f2de5a056d263bccf54df631db2 Mon Sep 17 00:00:00 2001 From: matical Date: Tue, 13 May 2025 18:00:10 -0700 Subject: [PATCH 7/9] chore: conflict reso --- packages/seal/src/client.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/seal/src/client.ts b/packages/seal/src/client.ts index 9c8e2f5e5..3d8adff3c 100644 --- a/packages/seal/src/client.ts +++ b/packages/seal/src/client.ts @@ -162,12 +162,10 @@ export class SealClient { for (const objectId of this.#serverObjectIds) { serverObjectIdsMap.set(objectId, (serverObjectIdsMap.get(objectId) ?? 0) + 1); } - const servicesMap = new Map(); for (const service of services) { servicesMap.set(service, (servicesMap.get(service) ?? 0) + 1); } - for (const [objectId, count] of serverObjectIdsMap) { if (servicesMap.get(objectId) !== count) { throw new InconsistentKeyServersError( @@ -175,7 +173,6 @@ export class SealClient { ); } } - // Check that the threshold can be met with the client's key servers. if (threshold > this.#serverObjectIds.length) { throw new InvalidThresholdError( @@ -263,7 +260,6 @@ export class SealClient { break; } } - if (hasAllKeys) { completedServerCount++; } @@ -300,7 +296,6 @@ export class SealClient { this.#timeout, controller.signal, ); - // Check validity of the keys and add them to the cache. const receivedIds = new Set(); for (const { fullId, key } of allKeys) { @@ -315,7 +310,6 @@ export class SealClient { console.warn('Received invalid key from key server ' + server.objectId); continue; } - this.#cachedKeys.set(`${fullId}:${server.objectId}`, keyElement); receivedIds.add(fullId); } @@ -339,7 +333,6 @@ export class SealClient { if (!controller.signal.aborted) { errors.push(error as Error); } - // If there are too many errors that the threshold is not attainable, return early with error. if (remainingKeyServers.size - errors.length < threshold - completedServerCount) { controller.abort(error); From 07d26d0b5cf3f48962d2e99a9684809e3faf7cd0 Mon Sep 17 00:00:00 2001 From: matical Date: Fri, 16 May 2025 10:01:40 -0700 Subject: [PATCH 8/9] refac(seal): batched requests without dedup --- packages/seal/src/key-server.ts | 52 +++++++++------------------------ 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/packages/seal/src/key-server.ts b/packages/seal/src/key-server.ts index 8e104fd32..8c388a913 100644 --- a/packages/seal/src/key-server.ts +++ b/packages/seal/src/key-server.ts @@ -62,52 +62,28 @@ export async function retrieveKeyServers({ objectIds: string[]; client: SealCompatibleClient; }): Promise { - const uniqueIds = Array.from(new Set(objectIds)); const { objects } = await client.core.getObjects({ - objectIds: uniqueIds, + objectIds, }); - // Create a single pass lookup map from objectId to key server data. - const serverDataMap = Object.fromEntries( - objects.map((res, i) => { - const objectId = uniqueIds[i]; - - if (res instanceof Error) { - throw new InvalidGetObjectError( - `KeyServer ${objectId} not found; ${(res as Error).message}`, - ); - } - - const ks = KeyServerMove.parse(res.content); - if (ks.keyType !== 0) { - throw new UnsupportedFeatureError(`Unsupported key type ${ks.keyType}`); - } - - return [ - objectId, - { - objectId, - name: ks.name, - url: ks.url, - pk: new Uint8Array(ks.pk), - }, - ]; - }), - ); - - // Return the preserved order of the input objectIds, creating a new object for each occurence. - return objectIds.map((objectId) => { - const data = serverDataMap[objectId]; - if (!data) { - throw new InvalidGetObjectError(`KeyServer ${objectId} not found`); + return objects.map((res, i) => { + const objectId = objectIds[i]; + + if (res instanceof Error) { + throw new InvalidGetObjectError(`KeyServer ${objectId} not found; ${res.message}`); + } + + const ks = KeyServerMove.parse(res.content); + if (ks.keyType !== 0) { + throw new UnsupportedFeatureError(`Unsupported key type ${ks.keyType}`); } return { objectId, - name: data.name, - url: data.url, + name: ks.name, + url: ks.url, keyType: KeyServerType.BonehFranklinBLS12381, - pk: data.pk, + pk: new Uint8Array(ks.pk), }; }); } From 0c65225df6485b2df1df5eb9a09f3bfc846ef412 Mon Sep 17 00:00:00 2001 From: matical Date: Fri, 16 May 2025 10:04:38 -0700 Subject: [PATCH 9/9] chore: update changeset --- .changeset/slimy-plants-add.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/slimy-plants-add.md b/.changeset/slimy-plants-add.md index 9ef2b22f5..bf40208d3 100644 --- a/.changeset/slimy-plants-add.md +++ b/.changeset/slimy-plants-add.md @@ -2,4 +2,4 @@ '@mysten/seal': patch --- -introduced deduplication of key server object fetching +introduced batched requests for key server objects