Merge pull request #1829 from TriliumNext/monorepo/express-partial-content

Integrate express-partial-content into monorepo

Author: eliandoran

apps/edit-docs/tsconfig.app.json

@@ -18,6 +18,9 @@
     "eslint.config.mjs"
   ],
   "references": [
+    {
+      "path": "../server/tsconfig.app.json"
+    },
     {
       "path": "../desktop/tsconfig.app.json"
     },

apps/edit-docs/tsconfig.json

@@ -3,6 +3,9 @@
   "files": [],
   "include": [],
   "references": [
+    {
+      "path": "../server"
+    },
     {
       "path": "../desktop"
     },

apps/server/package.json

@@ -53,7 +53,7 @@
     "@anthropic-ai/sdk": "0.40.1",
     "@braintree/sanitize-url": "7.1.1",
     "@triliumnext/commons": "workspace:*",
-    "@triliumnext/express-partial-content": "1.0.1",
+    "@triliumnext/express-partial-content": "workspace:*",
     "@triliumnext/turndown-plugin-gfm": "workspace:*",
     "archiver": "7.0.1",
     "async-mutex": "0.5.0",

apps/server/src/services/import/mime.spec.ts

@@ -45,6 +45,11 @@ describe("#getMime", () => {
             ["test.zip"], "application/zip"
         ],
 
+        [
+            "MP4 videos are supported",
+            ["video.mp4"], "video/mp4"
+        ],
+
         [
             "unknown MIME type not recognized by mimeTypes.lookup",
             ["test.fake"], false

apps/server/src/services/import/mime.ts

@@ -71,7 +71,8 @@ const EXTENSION_TO_MIME = new Map<string, string>([
     [".ts", "text/x-typescript"],
     [".excalidraw", "application/json"],
     [".mermaid", "text/vnd.mermaid"],
-    [".mmd", "text/vnd.mermaid"]
+    [".mmd", "text/vnd.mermaid"],
+    [".mp4", "video/mp4"]   // https://github.com/jshttp/mime-types/issues/138
 ]);
 
 /** @returns false if MIME is not detected */

apps/server/tsconfig.app.json

@@ -36,6 +36,9 @@
     {
       "path": "../../packages/turndown-plugin-gfm/tsconfig.lib.json"
     },
+    {
+      "path": "../../packages/express-partial-content/tsconfig.lib.json"
+    },
     {
       "path": "../../packages/commons/tsconfig.lib.json"
     }

apps/server/tsconfig.json

@@ -6,6 +6,9 @@
     {
       "path": "../../packages/turndown-plugin-gfm"
     },
+    {
+      "path": "../../packages/express-partial-content"
+    },
     {
       "path": "../../packages/commons"
     },

packages/express-partial-content/.swcrc

@@ -0,0 +1,22 @@
+{
+  "jsc": {
+    "target": "es2017",
+    "parser": {
+      "syntax": "typescript",
+      "decorators": true,
+      "dynamicImport": true
+    },
+    "transform": {
+      "decoratorMetadata": true,
+      "legacyDecorator": true
+    },
+    "keepClassNames": true,
+    "externalHelpers": true,
+    "loose": true
+  },
+  "module": {
+    "type": "commonjs"
+  },
+  "sourceMaps": true,
+  "exclude": ["jest.config.ts",".*\\.spec.tsx?$",".*\\.test.tsx?$","./src/jest-setup.ts$","./**/jest-setup.ts$",".*.js$"]
+}

packages/express-partial-content/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Sukant Gujar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/express-partial-content/README.md

@@ -0,0 +1,149 @@
+![Travis CI Status](https://travis-ci.com/SukantGujar/express-partial-content.svg?branch=master)
+
+# About
+
+A HTTP 206 Partial Content handler to serve any readable stream partially in Express.
+
+Based on this blog post: https://www.codeproject.com/Articles/813480/HTTP-Partial-Content-In-Node-js.
+
+# Installation
+
+`yarn add express-partial-content`
+
+OR
+
+`npm install express-partial-content`
+
+> Note: `Express` package is a peer dependency for `express-partial-content` and must be present in dependencies of the host package.
+
+# Usage
+
+From the `express-file-server` example:
+
+1.  Implement a `ContentProvider` function which prepares and returns a `Content` object:
+
+        import { promisify } from "util";
+        import fs from "fs";
+        import { Range, ContentDoesNotExistError, ContentProvider } from "express-partial-content";
+        import {logger} from "./logger";
+
+        const statAsync = promisify(fs.stat);
+        const existsAsync = promisify(fs.exists);
+
+        export const fileContentProvider: ContentProvider = async (req: Request) => {
+          // Read file name from route params.
+          const fileName = req.params.name;
+          const file = `${__dirname}/files/${fileName}`;
+          if (!(await existsAsync(file))) {
+            throw new ContentDoesNotExistError(`File doesn't exist: ${file}`);
+          }
+          const stats = await statAsync(file);
+          const totalSize = stats.size;
+          const mimeType = "application/octet-stream";
+          const getStream = (range?: Range) => {
+            if (!range) {
+              // Request if for complete content.
+              return fs.createReadStream(file);
+            }
+            // Partial content request.
+            const { start, end } = range;
+            logger.debug(`start: ${start}, end: ${end}`);
+            return fs.createReadStream(file, { start, end });
+          };
+          return {
+            fileName,
+            totalSize,
+            mimeType,
+            getStream
+          };
+        };
+
+2.  In your express code, use `createPartialContentHandler` factory method to generate an express handler for serving partial content for the route of your choice:
+
+        import {createPartialContentHandler} from "express-partial-content";
+        import {logger} from "./logger";
+
+        const handler = createPartialContentHandler(fileContentProvider, logger);
+
+        const app = express();
+        const port = 8080;
+
+        // File name is a route param.
+        app.get("/files/:name", handler);
+
+        app.listen(port, () => {
+          logger.debug("Server started!");
+        });
+
+3.  Run your server and use a multi-part/multi-connection download utility like [aria2c](https://aria2.github.io/) to test it:
+
+        aria -x5 -k1M http://localhost:8080/files/readme.txt
+
+# Examples
+
+There one examples in the `src/examples` folder:
+
+1. `express-file-server`: Implements a file based `ContentProvider`.
+
+## Running the examples:
+
+1.  `express-file-server`: Run the following commands, the server will listen on http://localhost:8080/.
+
+        yarn build:dev
+        yarn copy-assets
+        yarn run:examples:file
+
+## Connecting to the running server:
+
+Browse to `https://localhost:8080/files/readme.txt`
+
+# Reference
+
+## createPartialContentHandler function:
+
+This is a factory method which generates a partial content handler for express routes.
+
+### Arguments:
+
+- `contentProvider`: An `async` function which returns a Promise resolved to a `Content` object (see below).
+- `logger`: Any logging implementation which has a `debug(message:string, extra: any)` method. Either `winston` or `bunyan` loggers should work.
+
+### Returns:
+
+- Express Route Handler: `createPartialContentHandler` returns an express handler which can be mapped to an Express route to serve partial content.
+
+## ContentProvider function:
+
+This function _needs to be implemented by you_. It's purpose is to fetch and return `Content` object containing necessary metadata and methods to stream the content partially. This method is invoked by the express handler (returned by `createPartialContentHandler`) on each request.
+
+### Arguments:
+
+- `Request`: It receives the `Request` object as it's only input. Use the information available in `Request` to find the requested content, e.g. through `Request.params` or query string, headers etc.
+
+### Returns:
+
+- `Promise<Content>`: See below.
+
+### Throws:
+
+- `ContentDoesNotExistError`: Throw this to indicate that the content doesn't exist. The generated express handler will return a 404 in this case.
+  > Note: Any message provided to the `ContentDoesNotExistError` object is returned to the client.
+
+## Content object:
+
+This object contains metadata and methods which describe the content. The `ContentProvider` method builds and returns it.
+
+### Properties:
+
+All the properties of this object are used to return content metadata to the client as various `Response` headers.
+
+- `fileName`: Used as the `Content-Disposition` header's `filename` value.
+- `mimeType`: Used as the `Content-Type` header value.
+- `totalSize`: Used as the `Content-Length` header value.
+
+### Methods:
+
+- `getStream(range?: Range)`: This method should return a readable stream initialized to the provided `range` (optional). You need to handle two cases:
+
+  - range is `null`: When `range` is not-specified, the client is requesting the full content. In this case, return the stream as it is.
+  - range is `{start, end}`: When client requests partial content, the `start` and `end` values will point to the corresponding byte positions (0 based and inclusive) of the content. You need to return stream limited to these positions.

packages/express-partial-content/eslint.config.mjs

@@ -0,0 +1,23 @@
+import baseConfig from "../../eslint.config.mjs";
+
+export default [
+    ...baseConfig,
+    {
+      "files": [
+        "**/*.json"
+      ],
+      "rules": {
+        "@nx/dependency-checks": [
+          "error",
+          {
+            "ignoredFiles": [
+              "{projectRoot}/eslint.config.{js,cjs,mjs}"
+            ]
+          }
+        ]
+      },
+      "languageOptions": {
+        "parser": (await import('jsonc-eslint-parser'))
+      }
+    }
+];

packages/express-partial-content/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@triliumnext/express-partial-content",
+  "description": "A partial content handler implementation for any readable stream with Express. Based on this blog post: https://www.codeproject.com/Articles/813480/HTTP-Partial-Content-In-Node-js.",
+  "license": "MIT",
+  "version": "1.1.0",
+  "type": "module",
+  "private": true,
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "keywords": [
+    "partial-content",
+    "206",
+    "stream",
+    "typescript"
+  ],
+  "nx": {
+    "name": "express-partial-content",
+    "targets": {
+      "build": {
+        "executor": "@nx/js:swc",
+        "outputs": [
+          "{options.outputPath}"
+        ],
+        "options": {
+          "outputPath": "packages/express-partial-content/dist",
+          "tsConfig": "packages/express-partial-content/tsconfig.lib.json",
+          "packageJson": "packages/express-partial-content/package.json",
+          "main": "packages/express-partial-content/src/index.ts",
+          "stripLeadingPaths": true
+        }
+      }
+    }
+  },
+  "dependencies": {
+    "tslib": "^2.3.0"
+  }
+}

packages/express-partial-content/src/Content.ts

@@ -0,0 +1,22 @@
+import type { Range } from "./Range.js";
+import { Stream } from "stream";
+export interface Content {
+  /**
+   * Returns a readable stream based on the provided range (optional).
+   * @param {Range} range The start-end range of stream data.
+   * @returns {Stream} A readable stream
+   */
+  getStream(range?: Range): Stream;
+  /**
+   * Total size of the content
+   */
+  readonly totalSize: number;
+  /**
+   * Mime type to be sent in Content-Type header
+   */
+  readonly mimeType: string;
+  /**
+   * File name to be sent in Content-Disposition header
+   */
+  readonly fileName: string;
+};

packages/express-partial-content/src/ContentDoesNotExistError.ts

@@ -0,0 +1,2 @@
+export class ContentDoesNotExistError extends Error {
+}

packages/express-partial-content/src/ContentProvider.ts

@@ -0,0 +1,6 @@
+import type { Request } from "express";
+import type { Content } from "./Content.js";
+/**
+ * @type {function (Request): Promise<Content>}
+ */
+export type ContentProvider = (req: Request) => Promise<Content>;

packages/express-partial-content/src/Logger.ts

@@ -0,0 +1,3 @@
+export interface Logger {
+  debug(message: string, extra?: any): void;
+}

packages/express-partial-content/src/Range.ts

@@ -0,0 +1,4 @@
+export type Range = {
+  start: number;
+  end: number;
+};

packages/express-partial-content/src/RangeParserError.ts

@@ -0,0 +1,5 @@
+export class RangeParserError extends Error {
+  constructor(start: any, end: any) {
+    super(`Invalid start and end values: ${start}-${end}.`);
+  }
+}