Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 50 additions & 56 deletions src/commands/upload-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,11 @@ import { CLIUtils } from '../utils/cli.utils';
import { ConfigService } from '../services/config.service';
import path from 'node:path';
import { DriveFileService } from '../services/drive/drive-file.service';
import { ErrorUtils } from '../utils/errors.utils';
import { NotValidDirectoryError } from '../types/command.types';
import { ValidationService } from '../services/validation.service';
import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
import { ThumbnailService } from '../services/thumbnail.service';
import { BufferStream } from '../utils/stream.utils';
import { isFileThumbnailable } from '../utils/thumbnail.utils';
import { isFileThumbnailable, tryUploadThumbnail } from '../utils/thumbnail.utils';
import { Readable } from 'node:stream';

export default class UploadFile extends Command {
Expand Down Expand Up @@ -46,9 +44,6 @@ export default class UploadFile extends Command {
const filePath = await this.getFilePath(flags['file'], nonInteractive);

const stats = await stat(filePath);
if (!stats.size) {
throw new Error('The file is empty. Uploading empty files is not allowed.');
}

const fileInfo = path.parse(filePath);
const fileType = fileInfo.ext.replaceAll('.', '');
Expand All @@ -68,11 +63,9 @@ export default class UploadFile extends Command {
thumbnailUpload: 0,
};

// 1. Prepare the network
const networkFacade = await CLIUtils.prepareNetwork({ loginUserDetails: user, jsonFlag: flags['json'] });
// Prepare the network
const networkFacade = CLIUtils.prepareNetwork({ loginUserDetails: user, jsonFlag: flags['json'] });

// 2. Upload file to the Network
const readStream = createReadStream(filePath);
const networkUploadTimer = CLIUtils.timer();
const progressBar = CLIUtils.progress(
{
Expand All @@ -83,44 +76,52 @@ export default class UploadFile extends Command {
);
progressBar?.start(100, 0);

let fileId: string | undefined;
let bufferStream: BufferStream | undefined;
let fileStream: Readable = readStream;
const isThumbnailable = isFileThumbnailable(fileType);
if (isThumbnailable) {
bufferStream = new BufferStream();
fileStream = readStream.pipe(bufferStream);
}
const fileSize = stats.size ?? 0;

const progressCallback = (progress: number) => {
progressBar?.update(progress * 100 * 0.99);
};
if (fileSize > 0) {
// Upload file to the Network
const readStream = createReadStream(filePath);
let fileStream: Readable = readStream;

const fileId = await new Promise((resolve: (fileId: string) => void, reject) => {
const state = networkFacade.uploadFile(
fileStream,
stats.size,
user.bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
progressCallback,
);
process.on('SIGINT', () => {
state.stop();
process.exit(1);
if (isThumbnailable) {
bufferStream = new BufferStream();
fileStream = readStream.pipe(bufferStream);
}

const progressCallback = (progress: number) => {
progressBar?.update(progress * 100 * 0.99);
};

fileId = await new Promise((resolve: (fileId: string) => void, reject) => {
const state = networkFacade.uploadFile(
fileStream,
fileSize,
user.bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
progressCallback,
);
process.on('SIGINT', () => {
state.stop();
process.exit(1);
});
});
});
}
timings.networkUpload = networkUploadTimer.stop();

// 3. Create the file in Drive
// Create the file in Drive
const driveUploadTimer = CLIUtils.timer();
const createdDriveFile = await DriveFileService.instance.createFile({
plainName: fileInfo.name,
type: fileType,
size: stats.size,
size: fileSize,
folderUuid: destinationFolderUuid,
fileId: fileId,
bucket: user.bucket,
Expand All @@ -131,22 +132,14 @@ export default class UploadFile extends Command {
timings.driveUpload = driveUploadTimer.stop();

const thumbnailTimer = CLIUtils.timer();
try {
if (isThumbnailable && bufferStream) {
const thumbnailBuffer = bufferStream.getBuffer();

if (thumbnailBuffer) {
await ThumbnailService.instance.uploadThumbnail(
thumbnailBuffer,
fileType,
user.bucket,
createdDriveFile.uuid,
networkFacade,
);
}
}
} catch (error) {
ErrorUtils.report(error, { command: this.id });
if (fileSize > 0 && isThumbnailable && bufferStream) {
void tryUploadThumbnail({
bufferStream,
fileType,
userBucket: user.bucket,
fileUuid: createdDriveFile.uuid,
networkFacade,
});
}
timings.thumbnailUpload = thumbnailTimer.stop();

Expand All @@ -164,15 +157,16 @@ export default class UploadFile extends Command {
Thumbnail: ${CLIUtils.formatDuration(timings.thumbnailUpload)}\n`,
);
this.log('\n');
// eslint-disable-next-line max-len
const message = `File uploaded successfully in ${CLIUtils.formatDuration(totalTime)}, view it at ${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}`;
const message =
`File uploaded successfully in ${CLIUtils.formatDuration(totalTime)}, view it at ` +
`${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}`;
CLIUtils.success(this.log.bind(this), message);
return {
success: true,
message,
file: {
...createdDriveFile,
plainName: fileInfo.name,
plainName: createdDriveFile.name,
},
};
};
Expand Down
65 changes: 36 additions & 29 deletions src/services/network/upload/upload-file.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { isAlreadyExistsError } from '../../../utils/errors.utils';
import { stat } from 'node:fs/promises';
import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
import { createFileStreamWithBuffer, tryUploadThumbnail } from '../../../utils/thumbnail.utils';
import { BufferStream } from '../../../utils/stream.utils';
import { DriveFileItem } from '../../../types/drive.types';
import { CLIUtils } from '../../../utils/cli.utils';

export class UploadFileService {
Expand Down Expand Up @@ -64,49 +66,54 @@ export class UploadFileService {
network,
bucket,
parentFolderUuid,
}: UploadFileWithRetryParams): Promise<string | null> {
}: UploadFileWithRetryParams): Promise<DriveFileItem | null> {
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
const stats = await stat(file.absolutePath);
if (!stats.size) {
logger.warn(`Skipping empty file: ${file.relativePath}`);
return null;
}
const fileSize = stats.size ?? 0;

const fileType = extname(file.absolutePath).replaceAll('.', '');
const { fileStream, bufferStream } = createFileStreamWithBuffer({
path: file.absolutePath,
fileType,
});

let fileId: string | undefined;
let thumbnailStream: BufferStream | undefined;

const timings = {
networkUpload: 0,
driveUpload: 0,
thumbnailUpload: 0,
};

const uploadTimer = CLIUtils.timer();
const fileId = await new Promise<string>((resolve, reject) => {
network.uploadFile(
fileStream,
stats.size,
bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
() => {},
);
});
timings.networkUpload = uploadTimer.stop();
if (fileSize > 0) {
const { fileStream, bufferStream } = createFileStreamWithBuffer({
path: file.absolutePath,
fileType,
});

const uploadTimer = CLIUtils.timer();
thumbnailStream = bufferStream;

fileId = await new Promise<string>((resolve, reject) => {
network.uploadFile(
fileStream,
fileSize,
bucket,
(err: Error | null, res: string | null) => {
if (err) {
return reject(err);
}
resolve(res as string);
},
() => {},
);
});
timings.networkUpload = uploadTimer.stop();
}

const driveTimer = CLIUtils.timer();
const createdDriveFile = await DriveFileService.instance.createFile({
plainName: file.name,
type: fileType,
size: stats.size,
size: fileSize,
folderUuid: parentFolderUuid,
fileId,
bucket,
Expand All @@ -117,9 +124,9 @@ export class UploadFileService {
timings.driveUpload = driveTimer.stop();

const thumbnailTimer = CLIUtils.timer();
if (bufferStream) {
if (thumbnailStream && fileSize > 0) {
void tryUploadThumbnail({
bufferStream,
bufferStream: thumbnailStream,
fileType,
userBucket: bucket,
fileUuid: createdDriveFile.uuid,
Expand All @@ -139,7 +146,7 @@ export class UploadFileService {
Total: ${CLIUtils.formatDuration(totalTime)}\n`,
);

return createdDriveFile.fileId;
return createdDriveFile;
} catch (error: unknown) {
if (isAlreadyExistsError(error)) {
const msg = `File ${file.name} already exists, skipping...`;
Expand Down
4 changes: 2 additions & 2 deletions src/utils/thumbnail.utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,14 @@ export const tryUploadThumbnail = async ({
fileUuid,
networkFacade,
}: {
bufferStream: BufferStream;
bufferStream?: BufferStream;
fileType: string;
userBucket: string;
fileUuid: string;
networkFacade: NetworkFacade;
}) => {
try {
const thumbnailBuffer = bufferStream.getBuffer();
const thumbnailBuffer = bufferStream?.getBuffer();
if (thumbnailBuffer) {
await ThumbnailService.instance.uploadThumbnail(thumbnailBuffer, fileType, userBucket, fileUuid, networkFacade);
}
Expand Down
2 changes: 0 additions & 2 deletions src/webdav/handlers/GET.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ export class GETRequestHandler implements WebDavMethodHandler {
const { driveFileService, authService, networkFacade } = this.dependencies;
const resource = await WebDavUtils.getRequestedResource(req.url);

if (resource.name.startsWith('._')) throw new NotFoundError('File not found');

webdavLogger.info(`[GET] Request received item at ${resource.url}`);
const driveFile = await WebDavUtils.getDriveFileFromResource({
url: resource.url,
Expand Down
Loading