feat: EAI_AGAIN middleware TDE-1114 (#949)

#### Motivation

Retry workflows that fail due to EAI_AGAIN errors. 

#### Modification

Added an SDK middleware to try 3 times when receiving an `EAI_AGAIN`
error.

#### Checklist

_If not applicable, provide explanation of why._

- [x] Tests updated
- [ ] Docs updated -n/a
- [x] Issue linked in Title

---------

Co-authored-by: Blayne Chard <[email protected]>

Author: MDavidson17

src/__test__/eai_againRetry.test.ts

@@ -0,0 +1,54 @@
+import assert from 'node:assert';
+import { beforeEach, describe, it } from 'node:test';
+
+import { BuildHandler, MetadataBearer } from '@smithy/types';
+
+import { eaiAgainBuilder } from '../fs.register.js';
+
+let callCount = 0;
+
+function fakeNextBuilder(failCount: number): BuildHandler<object, MetadataBearer> {
+  const fakeNext: BuildHandler<object, MetadataBearer> = () => {
+    // fail a specified number of times and then succeed
+    callCount += 1;
+    if (callCount < 1 + failCount) {
+      return Promise.reject({ code: 'EAI_AGAIN', hostname: 'nz-imagery.s3.ap-southeast-2.amazonaws.com' });
+    } else {
+      return Promise.resolve({ output: { $metadata: {} }, response: {} });
+    }
+  };
+  return fakeNext;
+}
+
+describe('eai_againRetryMiddleware', () => {
+  beforeEach(() => {
+    callCount = 0;
+  });
+  it('should run next once if it succeeds', () => {
+    const fakeNext = fakeNextBuilder(0);
+    eaiAgainBuilder(() => 0)(fakeNext, {})({ input: {}, request: {} });
+    assert.equal(callCount, 1);
+  });
+
+  it('should try three times when getting EAI_AGAIN errors', async () => {
+    const fakeNext = fakeNextBuilder(2);
+    await eaiAgainBuilder(() => 0)(fakeNext, {})({ input: {}, request: {} });
+    assert.equal(callCount, 3);
+  });
+
+  it('should throw error if fails with unknown error type', () => {
+    const fakeNext: BuildHandler<object, MetadataBearer> = () => {
+      return Promise.reject({ message: 'ERROR MESSAGE' });
+    };
+    assert.rejects(eaiAgainBuilder(() => 0)(fakeNext, {})({ input: {}, request: {} }), {
+      message: 'ERROR MESSAGE',
+    });
+  });
+
+  it('should throw error if next fails with EAI_AGAIN three times', () => {
+    const fakeNext = fakeNextBuilder(3);
+    assert.rejects(eaiAgainBuilder(() => 0)(fakeNext, {})({ input: {}, request: {} }), {
+      message: 'EAI_AGAIN maximum tries (3) exceeded',
+    });
+  });
+});

src/fs.register.ts

@@ -1,10 +1,11 @@
+import { setTimeout } from 'node:timers/promises';
+
 import { S3Client } from '@aws-sdk/client-s3';
 import { FileSystem } from '@chunkd/core';
 import { fsa } from '@chunkd/fs';
-import { AwsCredentialConfig } from '@chunkd/source-aws';
-import { FsAwsS3 } from '@chunkd/source-aws';
+import { AwsCredentialConfig, FsAwsS3 } from '@chunkd/source-aws';
 import { FsAwsS3V3, S3LikeV3 } from '@chunkd/source-aws-v3';
-import { FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types';
+import { BuildMiddleware, FinalizeRequestMiddleware, MetadataBearer } from '@smithy/types';
 
 import { logger } from './log.js';
 
@@ -31,8 +32,43 @@ export const fqdn: FinalizeRequestMiddleware<object, MetadataBearer> = (next) =>
   };
 };
 
+/**
+ * AWS SDK middleware logic to try 3 times if receiving an EAI_AGAIN error
+ */
+export function eaiAgainBuilder(timeout: (attempt: number) => number): BuildMiddleware<object, MetadataBearer> {
+  const eaiAgain: BuildMiddleware<object, MetadataBearer> = (next) => {
+    const maxTries = 3;
+    let totalDelay = 0;
+    return async (args) => {
+      for (let attempt = 1; attempt <= maxTries; attempt++) {
+        try {
+          return await next(args);
+        } catch (error) {
+          if (error && typeof error === 'object' && 'code' in error && 'hostname' in error) {
+            if (error.code !== 'EAI_AGAIN') {
+              throw error;
+            }
+            const delay = timeout(attempt);
+            totalDelay += delay;
+            logger.warn({ host: error.hostname, attempt, delay, totalDelay }, `eai_again:retry`);
+            await setTimeout(timeout(attempt));
+          } else {
+            throw error;
+          }
+        }
+      }
+      throw new Error(`EAI_AGAIN maximum tries (${maxTries}) exceeded`);
+    };
+  };
+  return eaiAgain;
+}
+
 const client = new S3Client();
 export const s3Fs = new FsAwsS3V3(client);
+client.middlewareStack.add(
+  eaiAgainBuilder((attempt: number) => 100 + attempt * 1000),
+  { name: 'EAI_AGAIN', step: 'build' },
+);
 client.middlewareStack.add(fqdn, { name: 'FQDN', step: 'finalizeRequest' });
 
 FsAwsS3.MaxListCount = 1000;