diff --git a/.changeset/common-jeans-cry.md b/.changeset/common-jeans-cry.md new file mode 100644 index 000000000..3513541f4 --- /dev/null +++ b/.changeset/common-jeans-cry.md @@ -0,0 +1,5 @@ +--- +"@whereby.com/audio-denoiser": major +--- + +Release audio denoiser diff --git a/.changeset/cute-peas-speak.md b/.changeset/cute-peas-speak.md new file mode 100644 index 000000000..4c5192bb3 --- /dev/null +++ b/.changeset/cute-peas-speak.md @@ -0,0 +1,5 @@ +--- +"@whereby.com/core": minor +--- + +Add audio denoiser support diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml index 6031ae2fb..b1ce79b4d 100644 --- a/.github/workflows/build-test-release.yml +++ b/.github/workflows/build-test-release.yml @@ -71,6 +71,15 @@ jobs: package: "camera-effects" dest_dir: "camera-effects" + - name: Deploy CDN assets - audio-denoiser + if: steps.changesets.outputs.published == 'true' && contains(fromJSON(steps.changesets.outputs.publishedPackages).*.name, '@whereby.com/audio-denoiser') + uses: ./.github/actions/deploy-cdn + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + package: "audio-denoiser" + dest_dir: "audio-denoiser" + - name: Send Slack message if: steps.changesets.outputs.published == 'true' uses: ./.github/actions/notify-slack diff --git a/.github/workflows/canary-release.yml b/.github/workflows/canary-release.yml index ab571bb18..9908a2f2f 100644 --- a/.github/workflows/canary-release.yml +++ b/.github/workflows/canary-release.yml @@ -109,6 +109,14 @@ jobs: package: "camera-effects" dest_dir: "camera-effects" + - name: Deploy CDN assets - audio-denoiser + uses: ./.github/actions/deploy-cdn + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + package: "audio-denoiser" + dest_dir: "audio-denoiser" + - name: Add final reaction uses: peter-evans/create-or-update-comment@v4 with: diff --git a/packages/audio-denoiser/.gitignore b/packages/audio-denoiser/.gitignore new file mode 100644 index 000000000..944759baa --- /dev/null +++ b/packages/audio-denoiser/.gitignore @@ -0,0 +1,15 @@ +*.env +*.log + +dist/ +node_modules/ +storybook-static/ +yalc.lock +.yalc/ +/test-results/ +/playwright-report/ +/playwright/.cache/ + +yarn-error.log + +*.tgz diff --git a/packages/audio-denoiser/README.md b/packages/audio-denoiser/README.md new file mode 100644 index 000000000..d5fee3e4f --- /dev/null +++ b/packages/audio-denoiser/README.md @@ -0,0 +1,52 @@ +# `@whereby.com/audio-denoiser` + +Audio denoiser (noise suppression) for microphone streams. + +Wraps the input `MediaStream` in an `AudioWorklet` that runs an RNNoise-based +WebAssembly model, returning a new `MediaStream` with the cleaned audio. +Static assets (the WASM model and worklet script) are hosted on a CDN and +loaded at runtime. + +## Installation + +```bash +npm install @whereby.com/audio-denoiser +``` + +or + +```bash +yarn add @whereby.com/audio-denoiser +``` + +or + +```bash +pnpm add @whereby.com/audio-denoiser +``` + +## Usage + +```typescript +import { applyAudioDenoiser, canUse } from "@whereby.com/audio-denoiser"; + +if (canUse()) { + const { outputStream, stop } = await applyAudioDenoiser({ + inputStream: micStream, + doCaptureException: (err, ctx) => reportError(err, ctx), + }); + + // hand `outputStream` to your RTC pipeline; call `stop()` when done +} +``` + +`applyAudioDenoiser` also returns `audioContext` and `denoiserNode` for +consumers that need to share the underlying `AudioContext` (e.g. wiring an +audio analyzer onto the same node without creating a second source). + +## Development + +The static assets (WASM model + worklet script) are hosted on a CDN in +production builds. To exercise the local copies during development, set the +environment variable `REACT_APP_IS_DEV=true` before building. When unset (or +`false`), the build references the CDN as in production. diff --git a/packages/audio-denoiser/assets/denoiser/model.ext.wasm b/packages/audio-denoiser/assets/denoiser/model.ext.wasm new file mode 100755 index 000000000..b3a0298aa Binary files /dev/null and b/packages/audio-denoiser/assets/denoiser/model.ext.wasm differ diff --git a/packages/audio-denoiser/assets/denoiser/processor.ext.js b/packages/audio-denoiser/assets/denoiser/processor.ext.js new file mode 100644 index 000000000..52864757b --- /dev/null +++ b/packages/audio-denoiser/assets/denoiser/processor.ext.js @@ -0,0 +1,70 @@ +let instance; +let heapFloat32; + +class DenoiserProcessor extends AudioWorkletProcessor { + constructor(options) { + super({ + ...options, + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + }); + + this.alive = true; + + (async () => { + try { + if (!instance) { + const wasmModule = (await WebAssembly.instantiate(options.processorOptions.wasmBuffer)).module; + instance = new WebAssembly.Instance(wasmModule).exports; + heapFloat32 = new Float32Array(instance.memory.buffer); + } + this.state = instance.newState(); + this.active = true; + this.port.onmessage = ({ data: keepalive }) => { + if (this.alive) { + if (!keepalive) { + this.active = false; + this.alive = false; + instance.deleteState(this.state); + this.state = undefined; + } + } + }; + } catch (ex) { + this.port.postMessage({ error: ex.toString() }); + } + })(); + } + + process([input], [output]) { + if (this.active) { + if (!input.length) { + return true; + } + // ensure the state is truthy before proceeding, otherwise just passthrough audio + if (!this.state) { + try { + output[0].set(input[0]); + } catch (_) {} + return true; + } + + heapFloat32.set(input[0], instance.getInput(this.state) / 4); + const o = output[0]; + const ptr4 = instance.pipe(this.state, o.length) / 4; + if (ptr4) o.set(heapFloat32.subarray(ptr4, ptr4 + o.length)); + return true; + } + if (this.alive) { + // not yet loaded, or error initalizing wasm, so try to passthrough audio + try { + output[0].set(input[0]); + } catch (_) {} + return true; + } + return false; // we signal it is ok to destroy the processor + } +} + +registerProcessor("denoiser", DenoiserProcessor); diff --git a/packages/audio-denoiser/eslint.config.mjs b/packages/audio-denoiser/eslint.config.mjs new file mode 100644 index 000000000..56dafefdf --- /dev/null +++ b/packages/audio-denoiser/eslint.config.mjs @@ -0,0 +1,9 @@ +import baseConfig from "@whereby.com/eslint-config/base"; + +/** @type {import('typescript-eslint').Config} */ +export default [ + { + ignores: ["dist/**", "assets/**"], + }, + ...baseConfig, +]; diff --git a/packages/audio-denoiser/package.json b/packages/audio-denoiser/package.json new file mode 100644 index 000000000..35aa008f7 --- /dev/null +++ b/packages/audio-denoiser/package.json @@ -0,0 +1,46 @@ +{ + "name": "@whereby.com/audio-denoiser", + "version": "0.1.0", + "description": "Audio denoiser (noise suppression) for microphone streams", + "main": "./dist/index.cjs", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.cjs", + "default": "./dist/index.mjs" + } + }, + "files": [ + "dist/**", + "!dist/cdn", + "!dist/assets" + ], + "scripts": { + "build": "rollup -c", + "build:dev": "REACT_APP_IS_DEV=true rollup -c", + "build:prod": "REACT_APP_IS_DEV=false rollup -c", + "clean": "rimraf dist", + "lint": "eslint .", + "lint:fix": "eslint --fix ." + }, + "devDependencies": { + "@rollup/plugin-url": "^8.0.2", + "@whereby.com/eslint-config": "workspace:*", + "@whereby.com/jest-config": "workspace:*", + "@whereby.com/prettier-config": "workspace:*", + "@whereby.com/rollup-config": "workspace:*", + "@whereby.com/tsconfig": "workspace:*", + "rimraf": "^5.0.5", + "rollup": "^4.22.4" + }, + "dependencies": { + "@whereby.com/media": "workspace:*" + }, + "prettier": "@whereby.com/prettier-config" +} diff --git a/packages/audio-denoiser/rollup.config.js b/packages/audio-denoiser/rollup.config.js new file mode 100644 index 000000000..325a258b0 --- /dev/null +++ b/packages/audio-denoiser/rollup.config.js @@ -0,0 +1,243 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +const typescript = require("rollup-plugin-typescript2"); +const commonjs = require("@rollup/plugin-commonjs"); +const replace = require("@rollup/plugin-replace"); +const nodeResolve = require("@rollup/plugin-node-resolve"); +const url = require("@rollup/plugin-url"); +const { dts } = require("rollup-plugin-dts"); +const dotenv = require("dotenv"); +const fs = require("fs"); +const path = require("path"); +const pkg = require("./package.json"); + +dotenv.config({ + path: `../../.env`, +}); + +const baseConfig = require("@whereby.com/rollup-config/base"); +const replaceValues = baseConfig(__dirname, {}).replaceValues; + +const peerDependencies = [...Object.keys(pkg.peerDependencies || {})]; +const dependencies = [...Object.keys(pkg.dependencies || {})]; +const external = [...dependencies, ...peerDependencies]; + +function makeCdnPath() { + const major = pkg.version.split(".")[0]; + const minor = pkg.version.split(".")[1]; + const patch = pkg.version.split(".")[2]; + + let tag = ""; + const preRelease = pkg.version.split("-")[1]; + if (preRelease) { + tag = `-${preRelease.replace(/\./g, "-")}`; + } + + return `v${major}-${minor}-${patch}${tag}`; +} + +const CDN_BASE_URL = process.env.CDN_BASE_URL || "https://cdn.srv.whereby.com/audio-denoiser"; + +const IS_DEV = process.env.REACT_APP_IS_DEV === "true"; + +const createReplaceValues = () => ({ + preventAssignment: true, + delimiters: ["", ""], + values: { + ...replaceValues.values, + __ASSET_CDN_BASE_URL__: IS_DEV ? "" : `${CDN_BASE_URL}/${makeCdnPath()}`, + __USE_CDN_ASSETS__: IS_DEV ? "false" : "true", + }, +}); + +const copyAssetsToCdn = () => ({ + name: "copy-assets-to-cdn", + async generateBundle() { + const assetsDir = path.join(__dirname, "assets"); + const versionPath = makeCdnPath(); + const cdnAssetsDir = path.join(__dirname, "dist/cdn", versionPath, "assets"); + + if (!fs.existsSync(cdnAssetsDir)) { + fs.mkdirSync(cdnAssetsDir, { recursive: true }); + } + + const copyDir = (src, dest) => { + if (!fs.existsSync(dest)) { + fs.mkdirSync(dest, { recursive: true }); + } + const entries = fs.readdirSync(src, { withFileTypes: true }); + for (const entry of entries) { + const srcPath = path.join(src, entry.name); + const destPath = path.join(dest, entry.name); + if (entry.isDirectory()) { + copyDir(srcPath, destPath); + } else if (!entry.name.endsWith(".ts")) { + fs.copyFileSync(srcPath, destPath); + } + } + }; + + copyDir(assetsDir, cdnAssetsDir); + }, + async writeBundle() { + const virtualFile = path.join(__dirname, "dist/cdn/virtual-cdn-assets.js"); + if (fs.existsSync(virtualFile)) { + fs.unlinkSync(virtualFile); + } + }, +}); + +const tsOptions = { + tsconfig: "tsconfig.build.json", +}; + +const wasmPlugin = url({ + include: ["**/*.wasm"], + limit: 0, + fileName: "assets/denoiser/[name][extname]", + publicPath: "./", + destDir: path.join(__dirname, "dist"), + emitFiles: true, +}); + +// AudioWorklet processor script — must be served as a standalone URL (the +// browser fetches and evaluates it in the worklet global scope), not bundled. +const workletProcessorPlugin = url({ + include: ["**/processor.ext.js"], + limit: 0, + fileName: "assets/denoiser/[name][extname]", + publicPath: "./", + destDir: path.join(__dirname, "dist"), + emitFiles: true, +}); + +const handleUrlImports = () => ({ + name: "handle-url-imports", + resolveId(source, importer) { + const baseDir = importer ? path.dirname(importer) : __dirname; + if (source.includes("?url")) { + const cleanSource = source.replace("?url", ""); + return path.resolve(baseDir, cleanSource); + } + return null; + }, +}); + +const externalizeAssets = () => ({ + name: "externalize-assets", + resolveId(source) { + if (source.includes("assets/") && !source.endsWith(".js") && !source.endsWith(".ts")) { + return { id: source, external: true }; + } + return null; + }, +}); + +const plugins = [ + ...(IS_DEV ? [handleUrlImports(), wasmPlugin, workletProcessorPlugin] : [externalizeAssets()]), + nodeResolve({ + preferBuiltins: false, + browser: true, + extensions: [".mjs", ".js", ".ts", ".json", ".node"], + }), + commonjs(), + typescript(tsOptions), + replace(createReplaceValues()), +]; + +module.exports = [ + // Main entry point - ESM build + { + input: "src/index.ts", + output: [ + { + format: "esm", + file: "dist/index.mjs", + exports: "named", + inlineDynamicImports: true, + }, + ], + plugins, + external, + }, + + // Legacy ESM build + { + input: "src/index.ts", + output: [ + { + format: "es", + file: "dist/legacy-esm.js", + exports: "named", + inlineDynamicImports: true, + }, + ], + plugins, + external, + }, + + // CommonJS build + { + input: "src/index.ts", + output: [ + { + format: "cjs", + file: "dist/index.cjs", + exports: "named", + inlineDynamicImports: true, + }, + ], + plugins, + external, + }, + + // CDN Assets - copy assets/ to dist/cdn/v{ver}/assets/ + { + input: "virtual-cdn-assets", + output: { + dir: "dist/cdn", + format: "esm", + }, + plugins: [ + { + name: "virtual-cdn-assets", + resolveId(id) { + if (id === "virtual-cdn-assets") { + return id; + } + return null; + }, + load(id) { + if (id === "virtual-cdn-assets") { + return "export default null;"; + } + return null; + }, + }, + copyAssetsToCdn(), + ], + }, + + // Type definitions - ESM + { + input: "src/index.ts", + output: [{ file: "dist/index.d.mts", format: "esm" }], + external, + plugins: [dts(tsOptions)], + }, + + // Type definitions - ES + { + input: "src/index.ts", + output: [{ file: "dist/index.d.ts", format: "es" }], + external, + plugins: [dts(tsOptions)], + }, + + // Type definitions - CJS + { + input: "src/index.ts", + output: [{ file: "dist/index.d.cts", format: "cjs" }], + external, + plugins: [dts(tsOptions)], + }, +]; diff --git a/packages/audio-denoiser/src/assetUrls.ts b/packages/audio-denoiser/src/assetUrls.ts new file mode 100644 index 000000000..c37ed4696 --- /dev/null +++ b/packages/audio-denoiser/src/assetUrls.ts @@ -0,0 +1,20 @@ +const CDN_BASE_URL = "__ASSET_CDN_BASE_URL__"; + +declare const __USE_CDN_ASSETS__: boolean; +const USE_CDN = __USE_CDN_ASSETS__; + +const getAssetUrl = (path: string): string => { + if (USE_CDN) { + return `${CDN_BASE_URL}/${path}`; + } + throw new Error(`Local asset import required for: ${path}`); +}; + +export const assetUrls = { + denoiser: { + wasm: USE_CDN ? getAssetUrl("assets/denoiser/model.ext.wasm") : null, + processor: USE_CDN ? getAssetUrl("assets/denoiser/processor.ext.js") : null, + }, +}; + +export const USE_CDN_ASSETS = USE_CDN; diff --git a/packages/audio-denoiser/src/audioContext.ts b/packages/audio-denoiser/src/audioContext.ts new file mode 100644 index 000000000..26cbd4864 --- /dev/null +++ b/packages/audio-denoiser/src/audioContext.ts @@ -0,0 +1,21 @@ +type CacheKey = `${string}:${number | "default"}`; + +const audioContexts = new Map(); + +const AudioContextCtor: typeof AudioContext | undefined = + typeof AudioContext !== "undefined" + ? AudioContext + : (globalThis as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; + +export function getAudioContext(name: string, options?: AudioContextOptions): AudioContext { + if (!AudioContextCtor) { + throw new Error("AudioContext is not supported in this environment"); + } + const key: CacheKey = `${name}:${options?.sampleRate ?? "default"}`; + let ctx = audioContexts.get(key); + if (!ctx) { + ctx = new AudioContextCtor(options); + audioContexts.set(key, ctx); + } + return ctx; +} diff --git a/packages/audio-denoiser/src/index.ts b/packages/audio-denoiser/src/index.ts new file mode 100644 index 000000000..837185167 --- /dev/null +++ b/packages/audio-denoiser/src/index.ts @@ -0,0 +1,169 @@ +import { trackAnnotations } from "@whereby.com/media"; + +import { assetUrls, USE_CDN_ASSETS } from "./assetUrls"; +import { getAudioContext } from "./audioContext"; + +export type CaptureExceptionContext = { + tags?: Record; + extra?: Record; +}; + +export type CaptureExceptionFn = (err: Error, ctx?: CaptureExceptionContext) => void; + +// Dev-mode imports emit relative URLs like "./assets/denoiser/model.ext.wasm". +// Resolve them against this module's URL so consumers don't need to mount the +// dist/assets folder at a known origin path. Already-absolute (http/blob/data) +// URLs pass through unchanged — that covers the CDN production path. +const resolveAssetUrl = (url: string): string => { + if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("blob:") || url.startsWith("data:")) { + return url; + } + return new URL(url, import.meta.url).href; +}; + +const getWasmUrl = async (): Promise => { + if (USE_CDN_ASSETS) { + return assetUrls.denoiser.wasm!; + } + const mod = (await import("../assets/denoiser/model.ext.wasm")) as { default: string }; + return resolveAssetUrl(mod.default); +}; + +const getProcessorUrl = async (): Promise => { + if (USE_CDN_ASSETS) { + return assetUrls.denoiser.processor!; + } + const mod = (await import("../assets/denoiser/processor.ext.js?url")) as { default: string }; + return resolveAssetUrl(mod.default); +}; + +let wasmBufferPromise: Promise | null = null; +const loadWasmBuffer = async (doCaptureException?: CaptureExceptionFn): Promise => { + if (!wasmBufferPromise) { + wasmBufferPromise = (async () => { + const url = await getWasmUrl(); + const response = await fetch(url); + if (!response.ok) { + const error = new Error(`Failed to fetch denoiser model: ${response.status} ${response.statusText}`); + doCaptureException?.(error, { tags: { from: "audioDenoiser.loadWasmBuffer" } }); + wasmBufferPromise = null; + throw error; + } + return response.arrayBuffer(); + })(); + } + return wasmBufferPromise; +}; + +let workletRegistered: WeakSet | null = null; +const ensureWorkletRegistered = async (context: BaseAudioContext): Promise => { + if (!workletRegistered) workletRegistered = new WeakSet(); + if (workletRegistered.has(context)) return; + const processorUrl = await getProcessorUrl(); + await context.audioWorklet.addModule(processorUrl); + workletRegistered.add(context); +}; + +class Denoiser extends AudioWorkletNode { + constructor(context: AudioContext, wasmBuffer: ArrayBuffer) { + super(context, "denoiser", { + channelCountMode: "explicit", + channelCount: 1, + channelInterpretation: "speakers", + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + wasmBuffer, + }, + }); + } +} + +export type ApplyAudioDenoiserParams = { + inputStream: MediaStream; + doCaptureException?: CaptureExceptionFn; +}; + +export type AudioDenoiserHandle = { + outputStream: MediaStream; + audioContext: AudioContext; + denoiserNode: AudioWorkletNode; + stop: () => void; +}; + +export const canUse = (): boolean => typeof AudioWorkletNode !== "undefined" && typeof AudioContext !== "undefined"; + +export const warmup = async (): Promise => { + await loadWasmBuffer(); +}; + +export const applyAudioDenoiser = async ({ + inputStream, + doCaptureException, +}: ApplyAudioDenoiserParams): Promise => { + const inputTrack = inputStream.getAudioTracks()[0]; + const sampleRate = inputTrack?.getSettings().sampleRate; + + const audioContext = getAudioContext("audiodenoiser", sampleRate ? { sampleRate } : undefined); + const destination = audioContext.createMediaStreamDestination(); + + const [wasmBuffer] = await Promise.all([loadWasmBuffer(doCaptureException), ensureWorkletRegistered(audioContext)]); + + const source = audioContext.createMediaStreamSource(inputStream); + const denoiserNode = new Denoiser(audioContext, wasmBuffer); + + denoiserNode.port.onmessage = (event: MessageEvent<{ error?: string }>) => { + if (event.data?.error) { + doCaptureException?.(new Error(event.data.error), { + tags: { from: "audioDenoiser.onmessage" }, + extra: { sampleRate }, + }); + } + }; + denoiserNode.onprocessorerror = (errorEvent) => { + const event = errorEvent as ErrorEvent; + const error = new Error(`Denoiser processor error: ${event.error} - ${event.message}`); + doCaptureException?.(error, { + tags: { from: "audioDenoiser.onprocessorerror" }, + extra: { sampleRate }, + }); + }; + + denoiserNode.connect(destination); + source.connect(denoiserNode); + + const outputTrack = destination.stream.getAudioTracks()[0]; + + // Toggling `enabled` on the output track has no effect on the input track + // (and therefore on what gets denoised + sent). Forward it so muting the + // output track mutes the underlying mic. + if (outputTrack && inputTrack) { + Object.defineProperty(outputTrack, "enabled", { + get() { + return inputTrack.enabled; + }, + set(value: boolean) { + inputTrack.enabled = value; + }, + }); + } + if (outputTrack) { + trackAnnotations(outputTrack).isEffectTrack = true; + } + + let stopped = false; + const stop = () => { + if (stopped) return; + stopped = true; + try { + denoiserNode.port.postMessage(false); + source.disconnect(); + denoiserNode.disconnect(); + } catch (error) { + console.error("Error stopping audio denoiser", error); + } + }; + + return { outputStream: destination.stream, audioContext, denoiserNode, stop }; +}; diff --git a/packages/audio-denoiser/src/types/assets.d.ts b/packages/audio-denoiser/src/types/assets.d.ts new file mode 100644 index 000000000..089ba1a7a --- /dev/null +++ b/packages/audio-denoiser/src/types/assets.d.ts @@ -0,0 +1,9 @@ +declare module "*.wasm" { + const content: string; + export default content; +} + +declare module "*?url" { + const content: string; + export default content; +} diff --git a/packages/audio-denoiser/tsconfig.build.json b/packages/audio-denoiser/tsconfig.build.json new file mode 100644 index 000000000..f4b6c94c6 --- /dev/null +++ b/packages/audio-denoiser/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "@whereby.com/tsconfig/build.json", + "exclude": ["dist", "node_modules"] +} diff --git a/packages/audio-denoiser/tsconfig.json b/packages/audio-denoiser/tsconfig.json new file mode 100644 index 000000000..f738511ae --- /dev/null +++ b/packages/audio-denoiser/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@whereby.com/tsconfig/base.json", + "exclude": ["dist", "node_modules"] +} diff --git a/packages/browser-sdk/package.json b/packages/browser-sdk/package.json index 85329e1c7..64ec4af05 100644 --- a/packages/browser-sdk/package.json +++ b/packages/browser-sdk/package.json @@ -55,6 +55,7 @@ "build-storybook": "storybook build" }, "devDependencies": { + "@whereby.com/audio-denoiser": "workspace:*", "@whereby.com/camera-effects": "workspace:*", "@whereby.com/eslint-config": "workspace:*", "@whereby.com/jest-config": "workspace:*", diff --git a/packages/browser-sdk/src/lib/react/useRoomConnection/index.ts b/packages/browser-sdk/src/lib/react/useRoomConnection/index.ts index 181efc892..a978238bd 100644 --- a/packages/browser-sdk/src/lib/react/useRoomConnection/index.ts +++ b/packages/browser-sdk/src/lib/react/useRoomConnection/index.ts @@ -135,6 +135,12 @@ export function useRoomConnection( const clearCameraEffect = React.useCallback(async () => { await client.clearCameraEffect(); }, [client]); + const enableAudioDenoiser = React.useCallback(async () => { + await client.enableAudioDenoiser(); + }, [client]); + const disableAudioDenoiser = React.useCallback(async () => { + await client.disableAudioDenoiser(); + }, [client]); const { events, ...state } = roomConnectionState; @@ -174,6 +180,8 @@ export function useRoomConnection( switchCameraEffect, switchCameraEffectCustom, clearCameraEffect, + enableAudioDenoiser, + disableAudioDenoiser, }, }; } diff --git a/packages/browser-sdk/src/lib/react/useRoomConnection/types.ts b/packages/browser-sdk/src/lib/react/useRoomConnection/types.ts index 5208d7c0d..ea8372d5e 100644 --- a/packages/browser-sdk/src/lib/react/useRoomConnection/types.ts +++ b/packages/browser-sdk/src/lib/react/useRoomConnection/types.ts @@ -47,4 +47,6 @@ export interface RoomConnectionActions { switchCameraEffect: (effectId: string) => Promise; switchCameraEffectCustom: (imageUrl: string) => Promise; clearCameraEffect: () => Promise; + enableAudioDenoiser: () => Promise; + disableAudioDenoiser: () => Promise; } diff --git a/packages/browser-sdk/src/stories/components/VideoExperience.tsx b/packages/browser-sdk/src/stories/components/VideoExperience.tsx index c99f5a87b..4b8aeabc7 100644 --- a/packages/browser-sdk/src/stories/components/VideoExperience.tsx +++ b/packages/browser-sdk/src/stories/components/VideoExperience.tsx @@ -24,6 +24,7 @@ export default function VideoExperience({ joinRoomOnLoad, showBreakoutGroups, showCameraEffects, + showAudioDenoiser, }: { displayName?: string; roomName: string; @@ -35,10 +36,13 @@ export default function VideoExperience({ joinRoomOnLoad?: boolean; showBreakoutGroups?: boolean; showCameraEffects?: boolean; + showAudioDenoiser?: boolean; }) { const [chatMessage, setChatMessage] = useState(""); const [isLocalScreenshareActive, setIsLocalScreenshareActive] = useState(false); const [effectPresets, setEffectPresets] = useState>([]); + const [audioDenoiserSupported, setAudioDenoiserSupported] = useState(null); + const [audioDenoiserOn, setAudioDenoiserOn] = useState(false); const { state, actions, events } = useRoomConnection(roomName, { localMediaOptions: { @@ -95,6 +99,8 @@ export default function VideoExperience({ switchCameraEffect, switchCameraEffectCustom, clearCameraEffect, + enableAudioDenoiser, + disableAudioDenoiser, } = actions; async function loadBackgroundEffects() { @@ -105,6 +111,22 @@ export default function VideoExperience({ setEffectPresets(usablePresets); } + async function loadAudioDenoiserSupport() { + if (!showAudioDenoiser) return; + const { canUse } = await import("@whereby.com/audio-denoiser"); + setAudioDenoiserSupported(canUse()); + } + + async function handleEnableAudioDenoiser() { + await enableAudioDenoiser(); + setAudioDenoiserOn(true); + } + + async function handleDisableAudioDenoiser() { + await disableAudioDenoiser(); + setAudioDenoiserOn(false); + } + useEffect(() => { if (!joinRoomOnLoad) return; @@ -116,6 +138,7 @@ export default function VideoExperience({ if (!localParticipant?.stream) return; loadBackgroundEffects(); + loadAudioDenoiserSupport(); }, [localParticipant?.stream]); function showIncomingChatMessageNotification({ message }: ChatMessageEvent) { @@ -455,6 +478,30 @@ export default function VideoExperience({ ) : null} + {showAudioDenoiser ? ( +
+ Audio denoiser:{" "} + {audioDenoiserSupported === null + ? "Checking support…" + : audioDenoiserSupported + ? audioDenoiserOn + ? "On" + : "Off" + : "Not supported in this browser"} + {audioDenoiserSupported ? ( + <> + {" "} + + + + ) : null} +
+ ) : null} +
{[localParticipant, ...remoteParticipants].map((participant, i) => { const isSpotlighted = !!spotlightedParticipants.find((p) => p.id === participant?.id); diff --git a/packages/browser-sdk/src/stories/custom-ui.stories.tsx b/packages/browser-sdk/src/stories/custom-ui.stories.tsx index 570fe6932..61ea97c83 100644 --- a/packages/browser-sdk/src/stories/custom-ui.stories.tsx +++ b/packages/browser-sdk/src/stories/custom-ui.stories.tsx @@ -342,3 +342,17 @@ export const RoomConnectionWithCameraEffects = ({ return ; }; + +export const RoomConnectionWithAudioDenoiser = ({ + roomUrl, + displayName, +}: { + roomUrl: string; + displayName?: string; +}) => { + if (!roomUrl || !roomUrl.match(roomRegEx)) { + return

Set room url on the Controls panel

; + } + + return ; +}; diff --git a/packages/core/package.json b/packages/core/package.json index 58a360957..c5b506c88 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -81,9 +81,13 @@ "events": "^3.3.0" }, "peerDependencies": { + "@whereby.com/audio-denoiser": "*", "@whereby.com/camera-effects": "*" }, "peerDependenciesMeta": { + "@whereby.com/audio-denoiser": { + "optional": true + }, "@whereby.com/camera-effects": { "optional": true } diff --git a/packages/core/src/client/RoomConnection/index.ts b/packages/core/src/client/RoomConnection/index.ts index 59c302ba4..9e0347c4e 100644 --- a/packages/core/src/client/RoomConnection/index.ts +++ b/packages/core/src/client/RoomConnection/index.ts @@ -77,6 +77,7 @@ import { import { selectRoomConnectionState } from "./selector"; import { BaseClient } from "../BaseClient"; import { doCameraEffectsSwitchPreset } from "../../redux/slices/cameraEffects"; +import { doAudioDenoiserDisable, doAudioDenoiserEnable } from "../../redux/slices/audioDenoiser"; export class RoomConnectionClient extends BaseClient { protected options: Partial; @@ -677,6 +678,33 @@ export class RoomConnectionClient extends BaseClient { + try { + await this.store.dispatch(doAudioDenoiserEnable()); + } catch (error) { + return Promise.reject(error); + } + return Promise.resolve(); + } + + /** + * Disable audio noise suppression and revert to the raw microphone stream. + */ + public async disableAudioDenoiser(): Promise { + try { + await this.store.dispatch(doAudioDenoiserDisable()); + } catch (error) { + return Promise.reject(error); + } + return Promise.resolve(); + } + /** * Destroy the client. * This method will stop the app and reset the client state. diff --git a/packages/core/src/redux/slices/audioDenoiser.ts b/packages/core/src/redux/slices/audioDenoiser.ts new file mode 100644 index 000000000..44aec8438 --- /dev/null +++ b/packages/core/src/redux/slices/audioDenoiser.ts @@ -0,0 +1,205 @@ +import { createSlice, PayloadAction } from "@reduxjs/toolkit"; +import { createAppAsyncThunk } from "../thunk"; +import { RootState } from "../store"; +import { startAppListening } from "../listenerMiddleware"; +import { doAppStop } from "./app"; +import { + doLocalStreamEffect, + doSwitchLocalStream, + selectLocalMediaIsSwitchingStream, + selectLocalMediaStream, +} from "./localMedia"; + +type StopFn = () => void; + +export interface AudioDenoiserState { + /** + * User intent — true if the consumer asked for denoising on. Persists + * across mid-flight stops triggered by device/stream switches so we can + * auto-reapply after the switch settles. + */ + wanted: boolean; + isSwitching: boolean; + error?: unknown; + raw: { + stop?: StopFn; + outputStream?: MediaStream; + audioContext?: AudioContext; + denoiserNode?: AudioWorkletNode; + }; +} + +const initialState: AudioDenoiserState = { + wanted: false, + isSwitching: false, + raw: {}, +}; + +export const audioDenoiserSlice = createSlice({ + name: "audioDenoiser", + initialState, + reducers: { + audioDenoiserSwitching(state, action: PayloadAction<{ isSwitching: boolean }>) { + state.isSwitching = action.payload.isSwitching; + }, + audioDenoiserWantedSet(state, action: PayloadAction<{ wanted: boolean }>) { + state.wanted = action.payload.wanted; + }, + audioDenoiserApplied( + state, + action: PayloadAction<{ + stop: StopFn; + outputStream: MediaStream; + audioContext: AudioContext; + denoiserNode: AudioWorkletNode; + }>, + ) { + // Cast bypasses Immer's WritableDraft expansion for AudioContext / + // AudioWorkletNode (which expose ReadonlyMap-ish fields that Immer's + // type mapper can't reconcile). Immer leaves class instances alone + // at runtime, so this is type-level only. + state.raw = action.payload as typeof state.raw; + state.error = undefined; + state.isSwitching = false; + }, + audioDenoiserCleared(state) { + state.raw = {}; + state.error = undefined; + state.isSwitching = false; + }, + audioDenoiserError(state, action: PayloadAction<{ error: unknown }>) { + state.error = action.payload.error; + state.isSwitching = false; + }, + }, +}); + +export const { + audioDenoiserSwitching, + audioDenoiserWantedSet, + audioDenoiserApplied, + audioDenoiserCleared, + audioDenoiserError, +} = audioDenoiserSlice.actions; + +export const selectAudioDenoiserRaw = (state: RootState) => state.audioDenoiser.raw; +export const selectAudioDenoiserWanted = (state: RootState) => state.audioDenoiser.wanted; +export const selectIsAudioDenoiserSwitching = (state: RootState) => state.audioDenoiser.isSwitching; +export const selectAudioDenoiserEnabled = (state: RootState) => !!state.audioDenoiser.raw.stop; +export const selectAudioDenoiserError = (state: RootState) => state.audioDenoiser.error; + +const doAudioDenoiserApply = createAppAsyncThunk( + "audioDenoiser/apply", + async (_, { getState, dispatch, rejectWithValue }) => { + const state = getState(); + if (selectLocalMediaIsSwitchingStream(state)) return; + + const localStream = selectLocalMediaStream(state); + if (!localStream?.getAudioTracks()?.[0]) { + // No audio input to wrap; nothing to do but stay in "wanted" so we + // re-apply once a stream becomes available. + return; + } + if (selectAudioDenoiserEnabled(state)) return; + + dispatch(audioDenoiserSwitching({ isSwitching: true })); + try { + let mod: typeof import("@whereby.com/audio-denoiser"); + try { + mod = await import("@whereby.com/audio-denoiser"); + } catch { + throw new Error( + "@whereby.com/audio-denoiser is not installed. Add it as a dependency to enable audio denoising.", + ); + } + + const { applyAudioDenoiser, canUse } = mod; + if (!canUse()) { + throw new Error("Audio denoiser is not supported in this browser"); + } + + const handle = await applyAudioDenoiser({ + inputStream: localStream, + doCaptureException: (err, ctx) => { + console.error("[audioDenoiser]", err, ctx); + }, + }); + + await dispatch(doLocalStreamEffect({ effectStream: handle.outputStream, only: "audio" })); + + dispatch( + audioDenoiserApplied({ + stop: handle.stop, + outputStream: handle.outputStream, + audioContext: handle.audioContext, + denoiserNode: handle.denoiserNode, + }), + ); + } catch (error) { + dispatch(audioDenoiserError({ error })); + return rejectWithValue(error); + } + }, +); + +const doAudioDenoiserTeardown = createAppAsyncThunk("audioDenoiser/teardown", async (_, { getState, dispatch }) => { + const raw = selectAudioDenoiserRaw(getState()); + if (!raw.stop) return; + + dispatch(audioDenoiserSwitching({ isSwitching: true })); + try { + if (raw.outputStream) { + await dispatch(doLocalStreamEffect({ effectStream: undefined, only: "audio" })); + } + raw.stop(); + } finally { + dispatch(audioDenoiserCleared()); + } +}); + +export const doAudioDenoiserEnable = createAppAsyncThunk("audioDenoiser/enable", async (_, { dispatch }) => { + dispatch(audioDenoiserWantedSet({ wanted: true })); + await dispatch(doAudioDenoiserApply()); +}); + +export const doAudioDenoiserDisable = createAppAsyncThunk( + "audioDenoiser/disable", + async (arg, { dispatch }) => { + if (!arg?.keepWanted) { + dispatch(audioDenoiserWantedSet({ wanted: false })); + } + await dispatch(doAudioDenoiserTeardown()); + }, +); + +// Pause the denoiser while the local stream is switching (mic/cam device +// change). Keep `wanted` so we reapply once the switch completes. +startAppListening({ + actionCreator: doSwitchLocalStream.pending, + effect: async (_, { getState, dispatch }) => { + if (selectAudioDenoiserEnabled(getState())) { + await dispatch(doAudioDenoiserDisable({ keepWanted: true })); + } + }, +}); + +// After a stream switch, reapply the denoiser onto the new stream if the +// user still wants it on. +startAppListening({ + actionCreator: doSwitchLocalStream.fulfilled, + effect: async (_, { getState, dispatch }) => { + const state = getState(); + if (selectAudioDenoiserWanted(state) && !selectAudioDenoiserEnabled(state)) { + await dispatch(doAudioDenoiserApply()); + } + }, +}); + +startAppListening({ + actionCreator: doAppStop, + effect: (_, { dispatch, getState }) => { + if (selectAudioDenoiserRaw(getState()).stop) { + dispatch(doAudioDenoiserDisable()); + } + }, +}); diff --git a/packages/core/src/redux/slices/localMedia.ts b/packages/core/src/redux/slices/localMedia.ts index e88f61087..9af6a3896 100644 --- a/packages/core/src/redux/slices/localMedia.ts +++ b/packages/core/src/redux/slices/localMedia.ts @@ -807,9 +807,10 @@ startAppListening({ if (!stream) return; + const beforeEffectTracks = selectLocalMediaBeforeEffectTracks(state); const deviceData = getDeviceData({ - audioTrack: stream.getAudioTracks()[0], - videoTrack: stream.getVideoTracks()[0], + audioTrack: beforeEffectTracks?.audio || stream.getAudioTracks()[0], + videoTrack: beforeEffectTracks?.video || stream.getVideoTracks()[0], devices, }); diff --git a/packages/core/src/redux/store.ts b/packages/core/src/redux/store.ts index 335509ea3..dc24cec93 100644 --- a/packages/core/src/redux/store.ts +++ b/packages/core/src/redux/store.ts @@ -25,11 +25,13 @@ import { spotlightsSlice } from "./slices/spotlights"; import { streamingSlice } from "./slices/streaming"; import { waitingParticipantsSlice } from "./slices/waitingParticipants"; import { cameraEffectsSlice } from "./slices/cameraEffects"; +import { audioDenoiserSlice } from "./slices/audioDenoiser"; const IS_DEV = process.env.REACT_APP_IS_DEV === "true"; const appReducer = combineReducers({ app: appSlice.reducer, + audioDenoiser: audioDenoiserSlice.reducer, authorization: authorizationSlice.reducer, breakout: breakoutSlice.reducer, cameraEffects: cameraEffectsSlice.reducer, @@ -70,6 +72,10 @@ export const rootReducer: AppReducer = (state, action) => { ...cameraEffectsSlice.getInitialState(), ...state?.cameraEffects, }, + audioDenoiser: { + ...audioDenoiserSlice.getInitialState(), + ...state?.audioDenoiser, + }, }; return appReducer(resetState, action); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44db2d836..34cc4c672 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -481,6 +481,37 @@ importers: specifier: 'catalog:' version: 5.8.3 + packages/audio-denoiser: + dependencies: + '@whereby.com/media': + specifier: workspace:* + version: link:../media + devDependencies: + '@rollup/plugin-url': + specifier: ^8.0.2 + version: 8.0.2(rollup@4.44.1) + '@whereby.com/eslint-config': + specifier: workspace:* + version: link:../../tooling/eslint + '@whereby.com/jest-config': + specifier: workspace:* + version: link:../../tooling/jest + '@whereby.com/prettier-config': + specifier: workspace:* + version: link:../../tooling/prettier + '@whereby.com/rollup-config': + specifier: workspace:* + version: link:../../tooling/rollup + '@whereby.com/tsconfig': + specifier: workspace:* + version: link:../../tooling/tsconfig + rimraf: + specifier: ^5.0.5 + version: 5.0.10 + rollup: + specifier: ^4.22.4 + version: 4.44.1 + packages/browser-sdk: dependencies: '@radix-ui/react-popover': @@ -523,6 +554,9 @@ importers: '@vitejs/plugin-react': specifier: ^4.2.1 version: 4.6.0(vite@5.4.19(@types/node@22.15.34)(lightningcss@1.30.2)(terser@5.43.1)) + '@whereby.com/audio-denoiser': + specifier: workspace:* + version: link:../audio-denoiser '@whereby.com/camera-effects': specifier: workspace:* version: link:../camera-effects @@ -638,6 +672,9 @@ importers: '@reduxjs/toolkit': specifier: ^2.2.3 version: 2.8.2(react-redux@9.2.0(@types/react@19.1.8)(react@19.1.0)(redux@5.0.1))(react@19.1.0) + '@whereby.com/audio-denoiser': + specifier: '*' + version: link:../audio-denoiser '@whereby.com/camera-effects': specifier: '*' version: link:../camera-effects