common: Add audio denoiser package

.changeset/common-jeans-cry.md

@@ -0,0 +1,5 @@
+---
+"@whereby.com/audio-denoiser": major
+---
+
+Release audio denoiser

.changeset/cute-peas-speak.md

@@ -0,0 +1,5 @@
+---
+"@whereby.com/core": minor
+---
+
+Add audio denoiser support

.github/workflows/build-test-release.yml

@@ -71,6 +71,15 @@ jobs:
                   package: "camera-effects"
                   dest_dir: "camera-effects"
 
+            - name: Deploy CDN assets - audio-denoiser
+              if: steps.changesets.outputs.published == 'true' && contains(fromJSON(steps.changesets.outputs.publishedPackages).*.name, '@whereby.com/audio-denoiser')
+              uses: ./.github/actions/deploy-cdn
+              with:
+                  aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  package: "audio-denoiser"
+                  dest_dir: "audio-denoiser"
+
             - name: Send Slack message
               if: steps.changesets.outputs.published == 'true'
               uses: ./.github/actions/notify-slack

.github/workflows/canary-release.yml

@@ -109,6 +109,14 @@ jobs:
                   package: "camera-effects"
                   dest_dir: "camera-effects"
 
+            - name: Deploy CDN assets - audio-denoiser
+              uses: ./.github/actions/deploy-cdn
+              with:
+                  aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  package: "audio-denoiser"
+                  dest_dir: "audio-denoiser"
+
             - name: Add final reaction
               uses: peter-evans/create-or-update-comment@v4
               with:

packages/audio-denoiser/.gitignore

@@ -0,0 +1,15 @@
+*.env
+*.log
+
+dist/
+node_modules/
+storybook-static/
+yalc.lock
+.yalc/
+/test-results/
+/playwright-report/
+/playwright/.cache/
+
+yarn-error.log
+
+*.tgz

packages/audio-denoiser/README.md

@@ -0,0 +1,52 @@
+# `@whereby.com/audio-denoiser`
+
+Audio denoiser (noise suppression) for microphone streams.
+
+Wraps the input `MediaStream` in an `AudioWorklet` that runs an RNNoise-based
+WebAssembly model, returning a new `MediaStream` with the cleaned audio.
+Static assets (the WASM model and worklet script) are hosted on a CDN and
+loaded at runtime.
+
+## Installation
+
+```bash
+npm install @whereby.com/audio-denoiser
+```
+
+or
+
+```bash
+yarn add @whereby.com/audio-denoiser
+```
+
+or
+
+```bash
+pnpm add @whereby.com/audio-denoiser
+```
+
+## Usage
+
+```typescript
+import { applyAudioDenoiser, canUse } from "@whereby.com/audio-denoiser";
+
+if (canUse()) {
+    const { outputStream, stop } = await applyAudioDenoiser({
+        inputStream: micStream,
+        doCaptureException: (err, ctx) => reportError(err, ctx),
+    });
+
+    // hand `outputStream` to your RTC pipeline; call `stop()` when done
+}
+```
+
+`applyAudioDenoiser` also returns `audioContext` and `denoiserNode` for
+consumers that need to share the underlying `AudioContext` (e.g. wiring an
+audio analyzer onto the same node without creating a second source).
+
+## Development
+
+The static assets (WASM model + worklet script) are hosted on a CDN in
+production builds. To exercise the local copies during development, set the
+environment variable `REACT_APP_IS_DEV=true` before building. When unset (or
+`false`), the build references the CDN as in production.

packages/audio-denoiser/assets/denoiser/processor.ext.js

@@ -0,0 +1,70 @@
+let instance;
+let heapFloat32;
+
+class DenoiserProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super({
+            ...options,
+            numberOfInputs: 1,
+            numberOfOutputs: 1,
+            outputChannelCount: [1],
+        });
+
+        this.alive = true;
+
+        (async () => {
+            try {
+                if (!instance) {
+                    const wasmModule = (await WebAssembly.instantiate(options.processorOptions.wasmBuffer)).module;
+                    instance = new WebAssembly.Instance(wasmModule).exports;
+                    heapFloat32 = new Float32Array(instance.memory.buffer);
+                }
+                this.state = instance.newState();
+                this.active = true;
+                this.port.onmessage = ({ data: keepalive }) => {
+                    if (this.alive) {
+                        if (!keepalive) {
+                            this.active = false;
+                            this.alive = false;
+                            instance.deleteState(this.state);
+                            this.state = undefined;
+                        }
+                    }
+                };
+            } catch (ex) {
+                this.port.postMessage({ error: ex.toString() });
+            }
+        })();
+    }
+
+    process([input], [output]) {
+        if (this.active) {
+            if (!input.length) {
+                return true;
+            }
+            // ensure the state is truthy before proceeding, otherwise just passthrough audio
+            if (!this.state) {
+                try {
+                    output[0].set(input[0]);
+                } catch (_) {}
+                return true;
+            }
+
+            heapFloat32.set(input[0], instance.getInput(this.state) / 4);
+            const o = output[0];
+            const ptr4 = instance.pipe(this.state, o.length) / 4;
+            if (ptr4) o.set(heapFloat32.subarray(ptr4, ptr4 + o.length));
+            return true;
+        }
+        if (this.alive) {
+            // not yet loaded, or error initalizing wasm, so try to passthrough audio
+            try {
+                output[0].set(input[0]);
+            } catch (_) {}
+            return true;
+        }
+        return false; // we signal it is ok to destroy the processor
+    }
+}
+
+registerProcessor("denoiser", DenoiserProcessor);

packages/audio-denoiser/eslint.config.mjs

@@ -0,0 +1,9 @@
+import baseConfig from "@whereby.com/eslint-config/base";
+
+/** @type {import('typescript-eslint').Config} */
+export default [
+    {
+        ignores: ["dist/**", "assets/**"],
+    },
+    ...baseConfig,
+];

packages/audio-denoiser/package.json

@@ -0,0 +1,46 @@
+{
+    "name": "@whereby.com/audio-denoiser",
+    "version": "0.1.0",
+    "description": "Audio denoiser (noise suppression) for microphone streams",
+    "main": "./dist/index.cjs",
+    "module": "./dist/index.mjs",
+    "types": "./dist/index.d.ts",
+    "publishConfig": {
+        "access": "public"
+    },
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.mjs",
+            "require": "./dist/index.cjs",
+            "default": "./dist/index.mjs"
+        }
+    },
+    "files": [
+        "dist/**",
+        "!dist/cdn",
+        "!dist/assets"
+    ],
+    "scripts": {
+        "build": "rollup -c",
+        "build:dev": "REACT_APP_IS_DEV=true rollup -c",
+        "build:prod": "REACT_APP_IS_DEV=false rollup -c",
+        "clean": "rimraf dist",
+        "lint": "eslint .",
+        "lint:fix": "eslint --fix ."
+    },
+    "devDependencies": {
+        "@rollup/plugin-url": "^8.0.2",
+        "@whereby.com/eslint-config": "workspace:*",
+        "@whereby.com/jest-config": "workspace:*",
+        "@whereby.com/prettier-config": "workspace:*",
+        "@whereby.com/rollup-config": "workspace:*",
+        "@whereby.com/tsconfig": "workspace:*",
+        "rimraf": "^5.0.5",
+        "rollup": "^4.22.4"
+    },
+    "dependencies": {
+        "@whereby.com/media": "workspace:*"
+    },
+    "prettier": "@whereby.com/prettier-config"
+}