Skip to content

Commit ddda2e0

Browse files
authored
revert: PR #1090 ensure EAI_AGAIN middleware is added to new file systems TDE-1268 (#1093)
Reverts "fix: ensure EAI_AGAIN middleware is added to new file systems TDE-1268 (#1090)" This reverts commit 48ec902. #### Motivation Release v4.5.1 of Argo Tasks caused workflows (e.g. when running `tileindex-validate`) to fail with the following error: ```json {"level":60,"time":1727812404265,"pid":14,"hostname":"test-afage-tileindex-validate-sq9t2-main-1070049358","id":"01J94TA0KEYQ4MZCHHXCPCMQYY","err":{"type":"TypeError","message":"Cannot set properties of undefined (setting 'onFileSystemCreated')","stack":"TypeError: Cannot set properties of undefined (setting 'onFileSystemCreated')\n at setupS3FileSystem (file:///app/fs.register.js:68:46)\n at fsClient.credentials.onFileSystemCreated (file:///app/fs.register.js:72:13)\n at FsAwsS3ProviderV3.find (file:///app/node_modules/@chunkd/source-aws/build/provider.js:100:85)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async FsAwsS3V3.details (file:///app/node_modules/@chunkd/source-aws/build/s3.fs.js:90:31)\n at async asyncFilter (file:///app/utils/chunk.js:7:22)\n at async FileSystemAbstraction.toArray (file:///app/node_modules/@chunkd/core/build/index.js:49:22)\n at async getFiles (file:///app/utils/chunk.js:64:26)\n at async Object.load (file:///app/commands/tileindex-validate/tileindex.validate.js:27:23)\n at async Object.handler (file:///app/commands/tileindex-validate/tileindex.validate.js:198:23)"},"msg":"Command:Failed"} ``` #### Modification Revert PR #1090. #### Checklist - [x] Tests updated - [ ] Docs updated N/A - [x] Issue linked in Title
1 parent b7dc19b commit ddda2e0

File tree

2 files changed

+17
-93
lines changed

2 files changed

+17
-93
lines changed

src/__test__/fs.test.ts

Lines changed: 0 additions & 62 deletions
This file was deleted.

src/fs.register.ts

Lines changed: 17 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { S3Client } from '@aws-sdk/client-s3';
44
import { FileSystem } from '@chunkd/core';
55
import { fsa } from '@chunkd/fs';
66
import { AwsCredentialConfig, FsAwsS3 } from '@chunkd/source-aws';
7-
import { FsAwsS3V3 } from '@chunkd/source-aws-v3';
7+
import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3';
88
import { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types';
99

1010
import { logger } from './log.js';
@@ -63,48 +63,34 @@ export function eaiAgainBuilder(timeout: (attempt: number) => number): BuildMidd
6363
return eaiAgain;
6464
}
6565

66-
/**
67-
* When a new AWS file system is created copy across the middleware, but only if the middleware does not exist
68-
*
69-
* @param fsClient Filesystem to setup
70-
*/
71-
export function setupS3FileSystem(fsClient: FsAwsS3V3): void {
72-
fsClient.credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
73-
logger.debug({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
74-
75-
// TODO this cast can be removed once chunkd is upgraded
76-
if (fs.protocol === 's3') setupS3FileSystem(fs as FsAwsS3V3);
77-
78-
fsa.register(acc.prefix, fs);
79-
};
66+
const client = new S3Client();
67+
export const s3Fs = new FsAwsS3V3(client);
68+
client.middlewareStack.add(
69+
eaiAgainBuilder((attempt: number) => 100 + attempt * 1000),
70+
{ name: 'EAI_AGAIN', step: 'build' },
71+
);
72+
client.middlewareStack.add(fqdn, { name: 'FQDN', step: 'finalizeRequest' });
8073

81-
// There doesnt appear to be a has or find, so the only option is to list all middleware
82-
// which returns a list in a format: "FQDN - finalizeRequest"
83-
const middleware = fsClient.client.middlewareStack.identify();
74+
FsAwsS3.MaxListCount = 1000;
75+
s3Fs.credentials.onFileSystemCreated = (acc: AwsCredentialConfig, fs: FileSystem): void => {
76+
logger.debug({ prefix: acc.prefix, roleArn: acc.roleArn }, 'FileSystem:Register');
8477

85-
if (middleware.find((f) => f.startsWith('FQDN ')) == null) {
78+
if (fs.protocol === 's3') {
79+
// TODO this cast can be removed once chunkd is upgraded
80+
const fsS3 = fs as FsAwsS3V3;
81+
const fsClient = fsS3.s3 as S3LikeV3;
8682
fsClient.client.middlewareStack.add(fqdn, { name: 'FQDN', step: 'finalizeRequest' });
8783
}
8884

89-
if (middleware.find((f) => f.startsWith('EAI_AGAIN ')) == null) {
90-
fsClient.client.middlewareStack.add(
91-
eaiAgainBuilder((attempt: number) => 100 + attempt * 1000),
92-
{ name: 'EAI_AGAIN', step: 'build' },
93-
);
94-
}
95-
}
96-
97-
FsAwsS3.MaxListCount = 1000;
85+
fsa.register(acc.prefix, fs);
86+
};
9887

9988
function splitConfig(x: string): string[] {
10089
if (x.startsWith('[')) return JSON.parse(x) as string[];
10190
return x.split(',');
10291
}
10392

10493
export function registerFileSystem(opts: { config?: string }): void {
105-
const s3Fs = new FsAwsS3V3(new S3Client());
106-
setupS3FileSystem(s3Fs);
107-
10894
fsa.register('s3://', s3Fs);
10995

11096
const configPath = opts.config ?? process.env['AWS_ROLE_CONFIG_PATH'];

0 commit comments

Comments
 (0)