[ts-sdk] Add Blob as MP4 input

ts/examples/vite-example/src/App.tsx

@@ -11,6 +11,7 @@ import WhipExample from './examples/WhipExample';
 import DemoExample from './examples/Demo';
 import MultipleOutputs from './examples/MultipleOutputs';
 import MediaStreamInput from './examples/MediaStreamExample';
+import UploadMp4Example from './examples/UploadMp4Example';
 
 setWasmBundleUrl('/assets/smelter.wasm');
 
@@ -25,6 +26,7 @@ function App() {
     camera: <Camera />,
     screenCapture: <ScreenCapture />,
     mediaStream: <MediaStreamInput />,
+    uploadMp4: <UploadMp4Example />,
     home: <Home />,
     demo: <DemoExample />,
   };
@@ -49,6 +51,7 @@ function App() {
         <button onClick={() => setCurrentExample('camera')}>Camera</button>
         <button onClick={() => setCurrentExample('screenCapture')}>Screen Capture</button>
         <button onClick={() => setCurrentExample('mediaStream')}>MediaStream</button>
+        <button onClick={() => setCurrentExample('uploadMp4')}>Upload MP4</button>
 
         <h3>Smelter rendering engine examples</h3>
         <button onClick={() => setCurrentExample('counter')}>Counter</button>

ts/examples/vite-example/src/components/SmelterVideo.tsx

@@ -1,36 +1,32 @@
-import React, { useCallback, useEffect, useState } from 'react';
-import Smelter from '@swmansion/smelter-web-wasm';
+import React, { useCallback } from 'react';
+import type Smelter from '@swmansion/smelter-web-wasm';
 
 type VideoProps = React.DetailedHTMLProps<
   React.VideoHTMLAttributes<HTMLVideoElement>,
   HTMLVideoElement
 >;
 
 type CompositorVideoProps = {
-  onVideoCreate?: (smelter: Smelter) => Promise<void>;
-  onVideoStarted?: (smelter: Smelter) => Promise<void>;
+  outputId: string;
+  onVideoCreated?: (smelter: Smelter) => Promise<void>;
+  smelter: Smelter;
   children: React.ReactElement;
 } & VideoProps;
 
 export default function CompositorVideo(props: CompositorVideoProps) {
-  const { onVideoCreate, onVideoStarted, children, ...videoProps } = props;
-  const [smelter, setSmelter] = useState<Smelter | undefined>(undefined);
+  const { outputId, onVideoCreated, children, smelter, ...videoProps } = props;
 
   const videoRef = useCallback(
     async (video: HTMLVideoElement | null) => {
       if (!video) {
         return;
       }
 
-      const smelter = new Smelter({});
-
-      await smelter.init();
-
-      if (onVideoCreate) {
-        await onVideoCreate(smelter);
+      if (onVideoCreated) {
+        await onVideoCreated(smelter);
       }
 
-      const { stream } = await smelter.registerOutput('output', children, {
+      const { stream } = await smelter.registerOutput(outputId, children, {
         type: 'stream',
         video: {
           resolution: {
@@ -41,27 +37,13 @@ export default function CompositorVideo(props: CompositorVideoProps) {
         audio: true,
       });
 
-      await smelter.start();
-      setSmelter(smelter);
-
-      if (onVideoStarted) {
-        await onVideoStarted(smelter);
-      }
       if (stream) {
         video.srcObject = stream;
         await video.play();
       }
     },
-    [onVideoCreate, onVideoStarted, videoProps.width, videoProps.height, children]
+    [onVideoCreated, videoProps.width, videoProps.height, smelter, outputId]
   );
 
-  useEffect(() => {
-    return () => {
-      if (smelter) {
-        void smelter.terminate();
-      }
-    };
-  }, [smelter]);
-
   return <video ref={videoRef} {...videoProps} />;
 }

ts/examples/vite-example/src/examples/MultipleOutputs.tsx

@@ -1,7 +1,8 @@
-import React, { useCallback, useEffect, useState } from 'react';
+import { useEffect, useState } from 'react';
 import Smelter from '@swmansion/smelter-web-wasm';
 import { InputStream, Rescaler, Text, Tiles, useInputStreams, View } from '@swmansion/smelter';
 import NotoSansFont from '../../assets/NotoSans.ttf';
+import CompositorVideo from '../components/SmelterVideo';
 
 const FIRST_MP4_URL =
   'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4';
@@ -138,52 +139,4 @@ function useSmelter(): Smelter | undefined {
   return smelter;
 }
 
-type VideoProps = React.DetailedHTMLProps<
-  React.VideoHTMLAttributes<HTMLVideoElement>,
-  HTMLVideoElement
->;
-
-type CompositorVideoProps = {
-  outputId: string;
-  onVideoCreated?: (smelter: Smelter) => Promise<void>;
-  smelter: Smelter;
-  children: React.ReactElement;
-} & VideoProps;
-
-function CompositorVideo(props: CompositorVideoProps) {
-  const { outputId, onVideoCreated, children, smelter: initialSmelter, ...videoProps } = props;
-  const [smelter, _setSmelter] = useState<Smelter>(initialSmelter);
-
-  const videoRef = useCallback(
-    async (video: HTMLVideoElement | null) => {
-      if (!video) {
-        return;
-      }
-
-      if (onVideoCreated) {
-        await onVideoCreated(smelter);
-      }
-
-      const { stream } = await smelter.registerOutput(outputId, children, {
-        type: 'stream',
-        video: {
-          resolution: {
-            width: Number(videoProps.width ?? video.width),
-            height: Number(videoProps.height ?? video.height),
-          },
-        },
-        audio: true,
-      });
-
-      if (stream) {
-        video.srcObject = stream;
-        await video.play();
-      }
-    },
-    [onVideoCreated, videoProps.width, videoProps.height, smelter, outputId]
-  );
-
-  return <video ref={videoRef} {...videoProps} />;
-}
-
 export default MultipleOutputs;

ts/examples/vite-example/src/examples/UploadMp4Example.tsx

@@ -0,0 +1,99 @@
+import React, { useCallback, useEffect, useState } from 'react';
+import Smelter from '@swmansion/smelter-web-wasm';
+import { InputStream, Rescaler, useInputStreams, View, Text } from '@swmansion/smelter';
+import CompositorVideo from '../components/SmelterVideo';
+import NotoSansFont from '../../assets/NotoSans.ttf';
+
+function UploadMp4Example() {
+  const smelter = useSmelter();
+
+  const onCreate = useCallback(async (smelter: Smelter) => {
+    await smelter.registerFont(NotoSansFont);
+  }, []);
+  const onUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    if (!e.target.files) {
+      console.error('No files were uploaded');
+      return;
+    }
+
+    let file = e.target.files[0];
+
+    if (!smelter) {
+      console.error('Smelter has not been initialized yet');
+      return;
+    }
+
+    await smelter.unregisterInput('file');
+    await smelter.registerInput('file', { type: 'mp4', blob: file });
+  };
+
+  if (!smelter) {
+    return <div className="card" />;
+  }
+
+  return (
+    <div className="card">
+      <div style={{ margin: 20 }}>
+        <label htmlFor="upload-input" style={{ padding: 10 }}>
+          Upload MP4
+        </label>
+        <input id="upload-input" type="file" onChange={onUpload} accept="video/mp4" />
+      </div>
+      <CompositorVideo
+        outputId="output"
+        width={1280}
+        height={720}
+        smelter={smelter}
+        onVideoCreated={onCreate}>
+        <Scene />
+      </CompositorVideo>
+    </div>
+  );
+}
+
+function Scene() {
+  const inputs = useInputStreams();
+  if (!inputs['file']) {
+    return (
+      <View style={{ backgroundColor: 'black' }}>
+        <View style={{ top: 340, left: 560 }}>
+          <Text style={{ fontSize: 24, color: 'white' }}>Upload an MP4 file</Text>
+        </View>
+      </View>
+    );
+  }
+  return (
+    <View style={{ backgroundColor: 'black' }}>
+      <Rescaler>
+        <InputStream inputId="file" />
+      </Rescaler>
+    </View>
+  );
+}
+
+function useSmelter(): Smelter | undefined {
+  const [smelter, setSmelter] = useState<Smelter>();
+  useEffect(() => {
+    const smelter = new Smelter();
+
+    let cancel = false;
+    const promise = (async () => {
+      await smelter.init();
+      await smelter.start();
+      if (!cancel) {
+        setSmelter(smelter);
+      }
+    })();
+
+    return () => {
+      cancel = true;
+      void (async () => {
+        await promise.catch(() => {});
+        await smelter.terminate();
+      })();
+    };
+  }, []);
+  return smelter;
+}
+
+export default UploadMp4Example;

ts/smelter-core/src/api/input.ts

@@ -13,6 +13,7 @@ import { _smelterInternals } from '@swmansion/smelter';
  */
 export type RegisterInputRequest =
   | Api.RegisterInput
+  | { type: 'mp4_blob'; blob: any }
   | { type: 'camera' }
   | { type: 'screen_capture' }
   | { type: 'stream'; stream: any };
@@ -52,6 +53,13 @@ export function intoRegisterInput(input: RegisterInput): RegisterInputRequest {
 }
 
 function intoMp4RegisterInput(input: Inputs.RegisterMp4Input): RegisterInputRequest {
+  if (input.blob) {
+    return {
+      type: 'mp4_blob',
+      blob: input.blob,
+    };
+  }
+
   return {
     type: 'mp4',
     url: input.url,

ts/smelter-web-wasm/src/compositor/api.ts

@@ -52,7 +52,9 @@ export function intoRegisterOutputRequest(request: RegisterOutput): Output.Regis
 }
 
 export type RegisterInput =
-  | { type: 'mp4'; url: string }
+  | ({ type: 'mp4' } & RegisterMP4Input)
   | { type: 'camera' }
   | { type: 'screen_capture' }
   | { type: 'stream'; stream: MediaStream };
+
+type RegisterMP4Input = { url: string } | { blob: Blob };

ts/smelter-web-wasm/src/mainContext/input.ts

@@ -1,6 +1,6 @@
 import type { Input as CoreInput } from '@swmansion/smelter-core';
 import type { WorkerMessage } from '../workerApi';
-import { assert } from '../utils';
+import { assert, downloadToArrayBuffer } from '../utils';
 import { handleRegisterCameraInput } from './input/camera';
 import { handleRegisterScreenCaptureInput } from './input/screenCapture';
 import { handleRegisterStreamInput } from './input/stream';
@@ -23,7 +23,11 @@ export async function handleRegisterInputRequest(
 ): Promise<RegisterInputResult> {
   if (body.type === 'mp4') {
     assert(body.url, 'mp4 URL is required');
-    return handleRegisterMp4Input(ctx, inputId, body.url);
+    const arrayBuffer = await downloadToArrayBuffer(body.url);
+    return await handleRegisterMp4Input(ctx, inputId, arrayBuffer);
+  } else if (body.type === 'mp4_blob') {
+    const arrayBuffer = await (body.blob as Blob).arrayBuffer();
+    return await handleRegisterMp4Input(ctx, inputId, arrayBuffer);
   } else if (body.type === 'camera') {
     return await handleRegisterCameraInput(ctx, inputId);
   } else if (body.type === 'screen_capture') {

ts/smelter-web-wasm/src/mainContext/input/mp4.ts

@@ -21,11 +21,8 @@ export class Mp4Input implements Input {
 export async function handleRegisterMp4Input(
   ctx: InstanceContext,
   inputId: string,
-  url: string
+  arrayBuffer: ArrayBuffer
 ): Promise<RegisterInputResult> {
-  const response = await fetch(url);
-  const arrayBuffer = await response.arrayBuffer();
-
   const metadata = await parseMp4(arrayBuffer);
 
   let messagePort;

ts/smelter-web-wasm/src/utils.ts

@@ -24,3 +24,8 @@ export async function sleep(timeoutMs: number): Promise<void> {
 export function framerateToDurationMs(framerate: Framerate): number {
   return (1000 * framerate.den) / framerate.num;
 }
+
+export async function downloadToArrayBuffer(url: string): Promise<ArrayBuffer> {
+  const response = await fetch(url);
+  return await response.arrayBuffer();
+}

ts/smelter/src/types/registerInput.ts

@@ -39,6 +39,10 @@ export type RegisterMp4Input = {
    * Path to the MP4 file (location on the server where Smelter server is deployed).
    */
   serverPath?: string | null;
+  /**
+   * Blob of the MP4 file (available only in smelter-web-wasm).
+   */
+  blob?: any | null;
   /**
    * (**default=`false`**) If input should be played in the loop. <span class="badge badge--primary">Added in v0.4.0</span>
    */