Enable AWS S3 image storage : Issue #208 (#320)

Author: suratdas

.env

@@ -14,11 +14,11 @@ POSTGRES_DB=vrt_db_dev
 
 # static 
 STATIC_SERVICE=hdd # hdd | s3 - hdd as default if not provided
-
-# AWS_ACCESS_KEY_ID=
-# AWS_SECRET_ACCESS_KEY=
-# AWS_REGION=
-# AWS_S3_BUCKET_NAME=
+# Enter below values if STATIC_SERVICE=s3
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_S3_BUCKET_NAME=
 
 # optional
 #HTTPS_KEY_PATH='./secrets/ssl.key'

package-lock.json

@@ -26,6 +26,8 @@
     "node": ">=18.12.0"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.717.0",
+    "@aws-sdk/s3-request-presigner": "^3.717.0",
     "@nestjs/cache-manager": "^2.1.0",
     "@nestjs/common": "^10.2.5",
     "@nestjs/config": "^3.1.1",

package.json

@@ -26,7 +26,9 @@ export class OdiffService implements ImageComparator {
 
   constructor(private staticService: StaticService) {
     if (!isHddStaticServiceConfigured()) {
-      throw new Error('OdiffService can only be used with HddService');
+      return undefined;
+      // If we throw an exception, the application does not start.
+      // throw new Error('OdiffService can only be used with HddService');
     }
     this.hddService = this.staticService as unknown as HddService;
   }

src/compare/libs/odiff/odiff.service.ts

@@ -8,7 +8,6 @@ import { readFileSync, existsSync } from 'fs';
 import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import { HDD_IMAGE_PATH } from './static/hdd/constants';
-import { isHddStaticServiceConfigured } from './static/utils';
 
 function getHttpsOptions(): HttpsOptions | null {
   const keyPath = './secrets/ssl.key';
@@ -35,16 +34,13 @@ async function bootstrap() {
     app.use(bodyParser.json({ limit: process.env.BODY_PARSER_JSON_LIMIT }));
   }
 
-  // serve images only if hdd configuration
-  if (isHddStaticServiceConfigured()) {
-    app.useStaticAssets(join(process.cwd(), HDD_IMAGE_PATH), {
-      maxAge: 31536000,
-      // allow cors
-      setHeaders: (res) => {
-        res.set('Access-Control-Allow-Origin', '*');
-      },
-    });
-  }
+  app.useStaticAssets(join(process.cwd(), HDD_IMAGE_PATH), {
+    maxAge: 31536000,
+    // allow cors
+    setHeaders: (res) => {
+      res.set('Access-Control-Allow-Origin', '*');
+    },
+  });
 
   await app.listen(process.env.APP_PORT || 3000);
 }

src/main.ts

@@ -1,39 +1,75 @@
-import { PNGWithMetadata } from 'pngjs';
+import { PNG, PNGWithMetadata } from 'pngjs';
 import { Logger } from '@nestjs/common';
 import { Static } from '../static.interface';
-// import { S3Client } from '@aws-sdk/client-s3';
+import { DeleteObjectCommand, GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 
 export class AWSS3Service implements Static {
   private readonly logger: Logger = new Logger(AWSS3Service.name);
-  // private readonly AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
-  // private readonly AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
-  // private readonly AWS_REGION = process.env.AWS_REGION;
-  // private readonly AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME;
+  private readonly AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
+  private readonly AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
+  private readonly AWS_REGION = process.env.AWS_REGION;
+  private readonly AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME;
 
-  // private s3Client: S3Client;
+  private s3Client: S3Client;
 
   constructor() {
-    // this.s3Client = new S3Client({
-    //     credentials: {
-    //         accessKeyId: this.AWS_ACCESS_KEY_ID,
-    //         secretAccessKey: this.AWS_SECRET_ACCESS_KEY,
-    //     },
-    //     region: this.AWS_REGION,
-    // });
+    this.s3Client = new S3Client({
+      credentials: {
+        accessKeyId: this.AWS_ACCESS_KEY_ID,
+        secretAccessKey: this.AWS_SECRET_ACCESS_KEY,
+      },
+      region: this.AWS_REGION,
+    });
     this.logger.log('AWS S3 service is being used for file storage.');
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise<string> {
-    throw new Error('Method not implemented.');
+
+  async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise<string> {
+    const imageName = `${Date.now()}.${type}.png`;
+    try {
+      await this.s3Client.send(
+        new PutObjectCommand({
+          Bucket: this.AWS_S3_BUCKET_NAME,
+          Key: imageName,
+          ContentType: 'image/png',
+          Body: imageBuffer,
+        })
+      );
+      return imageName;
+    } catch (ex) {
+      throw new Error('Could not save file at AWS S3 : ' + ex);
+    }
+  }
+
+  async getImage(fileName: string): Promise<PNGWithMetadata> {
+    if (!fileName) return null;
+    try {
+      const command = new GetObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: fileName });
+      const s3Response = await this.s3Client.send(command);
+      const stream = s3Response.Body as Readable;
+      return PNG.sync.read(Buffer.concat(await stream.toArray()));
+    } catch (ex) {
+      this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`);
+    }
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getImage(fileName: string): Promise<PNGWithMetadata> {
-    throw new Error('Method not implemented.');
+  async getImageUrl(imageName: string): Promise<string> {
+    const command = new GetObjectCommand({
+      Bucket: `${this.AWS_S3_BUCKET_NAME}`,
+      Key: imageName,
+    });
+    return getSignedUrl(this.s3Client, command, { expiresIn: 3600 });
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  deleteImage(imageName: string): Promise<boolean> {
-    throw new Error('Method not implemented.');
+  async deleteImage(imageName: string): Promise<boolean> {
+    if (!imageName) return false;
+    try {
+      await this.s3Client.send(new DeleteObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: imageName }));
+      return true;
+    } catch (error) {
+      this.logger.log(`Failed to delete file at AWS S3 for image ${imageName}:`, error);
+      return false;
+    }
   }
 }

src/static/aws/s3.service.ts

@@ -21,6 +21,10 @@ export class HddService implements Static {
     return path.resolve(HDD_IMAGE_PATH, imageName);
   }
 
+  getImageUrl(imageName: string): Promise<string> {
+    return Promise.resolve('/' + imageName);
+  }
+
   async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise<string> {
     try {
       new PNG().parse(imageBuffer);

src/static/hdd/hdd.service.ts

@@ -0,0 +1,23 @@
+import { Controller, Get, Logger, Param, Res } from '@nestjs/common';
+import { Response } from 'express';
+import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
+import { StaticService } from './static.service';
+
+@ApiTags('images')
+@Controller('images')
+export class StaticController {
+  private readonly logger: Logger = new Logger(StaticController.name);
+  constructor(private staticService: StaticService) {}
+
+  @Get('/:fileName')
+  @ApiOkResponse()
+  async getUrlAndRedirect(@Param('fileName') fileName: string, @Res() res: Response) {
+    try {
+      const url = await this.staticService.getImageUrl(fileName);
+      res.redirect(url);
+    } catch (error) {
+      this.logger.error('Error fetching file from S3:' + fileName, error);
+      res.status(500).send('Error occurred while getting the file.');
+    }
+  }
+}

src/static/static.controller.ts

@@ -4,4 +4,5 @@ export interface Static {
   saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise<string>;
   getImage(fileName: string): Promise<PNGWithMetadata>;
   deleteImage(imageName: string): Promise<boolean>;
+  getImageUrl(imageName: string): Promise<string>;
 }

src/static/static.interface.ts

@@ -2,10 +2,12 @@ import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
 import { StaticService } from './static.service';
 import { StaticFactoryService } from './static.factory';
+import { StaticController } from './static.controller';
 
 @Module({
   imports: [ConfigModule],
   providers: [StaticService, StaticFactoryService],
   exports: [StaticService],
+  controllers: [StaticController],
 })
 export class StaticModule {}