HACK DO NOT SUBMIT. Integration example for replay videos.

packages/replay-next/src/suspense/ScreenshotCache.ts

@@ -1,4 +1,4 @@
-import { ExecutionPoint, ScreenShot } from "@replayio/protocol";
+import { ExecutionPoint, ScreenShot, Video } from "@replayio/protocol";
 import { createCache } from "suspense";
 
 import { paintHashCache } from "replay-next/src/suspense/PaintHashCache";
@@ -19,3 +19,16 @@ export const screenshotCache = createCache<
     return screenShot;
   },
 });
+
+export const videoCache = createCache<
+  [replayClient: ReplayClientInterface],
+  Video[]
+>({
+  config: { immutable: true },
+  debugLabel: "ScreenshotCache",
+  getKey: ([client]) => client.getRecordingId() || "",
+  load: async ([client]) => {
+    const videos = await client.getVideos();
+    return videos;
+  },
+});

packages/shared/client/ReplayClient.ts

@@ -45,6 +45,7 @@ import {
   TimeStampedPoint,
   TimeStampedPointRange,
   VariableMapping,
+  Video,
   annotations,
   createPauseResult,
   findPointsResults,
@@ -810,6 +811,15 @@ export class ReplayClient implements ReplayClientInterface {
     return screen;
   }
 
+  async getVideos(): Promise<Video[]> {
+    const sessionId = await this.waitForSession();
+    const { videos } = await client.Graphics.getVideos(
+      { mimeType: "video/webm" },
+      sessionId
+    );
+    return videos;
+  }
+
   async mapExpressionToGeneratedScope(expression: string, location: Location): Promise<string> {
     const sessionId = await this.waitForSession();
     const result = await client.Debugger.mapExpressionToGeneratedScope(

pages/_document.tsx

@@ -48,6 +48,8 @@ const csp = (props: any) => {
     // Required to inline images from the database and from external avaters
     `img-src 'self' data: https:`,
 
+    `media-src 'self' data: https:`,
+
     // Required for our logpoint analysis cache (which uses a Web worker)
     `worker-src 'self' blob:`,
   ]

src/ui/components/Video/Video.tsx

@@ -31,22 +31,38 @@ export default function Video() {
   useLayoutEffect(() => {
     const containerElement = document.getElementById("video") as HTMLDivElement;
     const graphicsElement = document.getElementById("graphics") as HTMLImageElement;
+    const videoElement = document.getElementById("webmvideo") as HTMLVideoElement;
+    const videoSourceElement = document.getElementById("videoSrc") as HTMLSourceElement;
     const graphicsOverlayElement = document.getElementById("overlay-graphics") as HTMLDivElement;
 
     let prevState: Partial<State> = {};
     let stalledTimeout: NodeJS.Timeout | null = null;
 
     // Keep graphics in sync with the imperatively managed screenshot state
     state.listen(nextState => {
-      if (nextState.screenShot != prevState.screenShot) {
-        const { screenShot } = nextState;
-        if (screenShot) {
-          graphicsElement.src = `data:${screenShot.mimeType};base64,${screenShot.data}`;
-        } else {
-          graphicsElement.src = "";
+      if (nextState.videos.length > 0 && !videoElement.src) {
+        graphicsElement.hidden = true;
+        videoElement.hidden = false;
+        videoElement.src = `data:video/webm;base64,${nextState.videos[0].data}`;
+      } else {
+        if (nextState.screenShot != prevState.screenShot) {
+          const { screenShot } = nextState;
+          if (screenShot) {
+            graphicsElement.src = `data:${screenShot.mimeType};base64,${screenShot.data}`;
+            graphicsElement.hidden = false;
+            videoElement.hidden = true;
+          } else {
+            graphicsElement.src = "";
+            videoSourceElement.src = "";
+          }
         }
       }
 
+      const { videos, paintIndex } = nextState;
+      if (videos.length > 0 && paintIndex !== null && paintIndex > 0) {
+        videoElement.currentTime = (paintIndex - 1) * (1.0 / videos[0].fps);
+      }
+
       // Show loading progress bar if graphics stall for longer than 5s
       const isLoading = nextState.status === "loading";
       const wasLoading = prevState.status === "loading";
@@ -116,6 +132,9 @@ export default function Video() {
       </div>
 
       <img className={styles.Image} id="graphics" onClick={onClick} />
+      <video className={styles.Image} id="webmvideo" onClick={onClick}>
+        <source type="video/webm" id="videoSrc"/>
+      </video>
 
       {/* Graphics that are relative to the rendered screenshot go here; this container is automatically positioned to align with the screenshot */}
       <div className={styles.Graphics} id="overlay-graphics">

src/ui/components/Video/imperative/MutableGraphicsState.tsx

@@ -11,7 +11,7 @@
 //
 // This approach is unusual, but it's arguably cleaner than sharing these values via the DOM.
 
-import { ExecutionPoint, ScreenShot } from "@replayio/protocol";
+import { ExecutionPoint, ScreenShot, Video } from "@replayio/protocol";
 
 import { fitImageToContainer, getDimensions } from "replay-next/src/utils/image";
 import { shallowEqual } from "shared/utils/compare";
@@ -34,8 +34,10 @@ export interface State {
   localScale: number;
   recordingScale: number;
   screenShot: ScreenShot | undefined;
+  videos: Video[];
   screenShotType: ScreenShotType | undefined;
   status: Status;
+  paintIndex: number | null;
 }
 
 export const state = createState<State>({
@@ -52,6 +54,8 @@ export const state = createState<State>({
   screenShot: undefined,
   screenShotType: undefined,
   status: "loading",
+  paintIndex: null,
+  videos: [],
 });
 
 let lock: Object | null = null;
@@ -62,9 +66,11 @@ export async function updateState(
     didResize?: boolean;
     executionPoint: ExecutionPoint | null;
     screenShot: ScreenShot | null;
+    videos: Video[];
     screenShotType: ScreenShotType | null;
     status: Status;
     time: number;
+    paintIndex: number | null;
   }> = {}
 ) {
   const prevState = state.read();
@@ -76,6 +82,8 @@ export async function updateState(
     screenShotType = prevState.screenShotType,
     status = prevState.status,
     time = prevState.currentTime,
+    paintIndex = prevState.paintIndex,
+    videos = prevState.videos,
   } = options;
 
   if (shallowEqual(options, { didResize })) {
@@ -92,7 +100,41 @@ export async function updateState(
   let graphicsRect = prevState.graphicsRect;
   let localScale = prevState.localScale;
   let recordingScale = prevState.recordingScale;
-  if (screenShot && (screenShot != prevState.screenShot || didResize)) {
+  if (videos.length > 0) {
+    const naturalDimensions = {
+      aspectRatio: videos[0].width / videos[0].height,
+      height: videos[0].height,
+      width: videos[0].width,
+    }
+
+    if (lock !== localLock) {
+      return;
+    }
+
+    const naturalHeight = naturalDimensions.height;
+    const naturalWidth = naturalDimensions.width;
+
+    const containerRect = graphicsElement.getBoundingClientRect();
+    const scaledDimensions = fitImageToContainer({
+      containerHeight: containerRect.height,
+      containerWidth: containerRect.width,
+      imageHeight: naturalHeight,
+      imageWidth: naturalWidth,
+    });
+
+    const clientHeight = scaledDimensions.height;
+    const clientWidth = scaledDimensions.width;
+
+    localScale = clientWidth / naturalWidth;
+    recordingScale = 1.0;
+
+    graphicsRect = {
+      height: clientHeight,
+      left: containerRect.left + (containerRect.width - clientWidth) / 2,
+      top: containerRect.top + (containerRect.height - clientHeight) / 2,
+      width: clientWidth,
+    };
+  } else if (screenShot && (screenShot != prevState.screenShot || didResize)) {
     const naturalDimensions = await getDimensions(screenShot.data, screenShot.mimeType);
     if (lock !== localLock) {
       return;
@@ -132,6 +174,8 @@ export async function updateState(
     screenShot: screenShot || undefined,
     screenShotType: screenShotType || undefined,
     status,
+    paintIndex,
+    videos
   };
 
   if (!shallowEqual(prevState, nextState)) {

src/ui/components/Video/imperative/updateGraphics.ts

@@ -1,9 +1,9 @@
 import { ExecutionPoint, ScreenShot } from "@replayio/protocol";
 
-import { PaintsCache, findMostRecentPaint } from "protocol/PaintsCache";
+import { PaintsCache, findMostRecentPaint, findMostRecentPaintIndex } from "protocol/PaintsCache";
 import { RepaintGraphicsCache } from "protocol/RepaintGraphicsCache";
 import { paintHashCache } from "replay-next/src/suspense/PaintHashCache";
-import { screenshotCache } from "replay-next/src/suspense/ScreenshotCache";
+import { screenshotCache, videoCache } from "replay-next/src/suspense/ScreenshotCache";
 import { ReplayClientInterface } from "shared/client/types";
 import { updateState } from "ui/components/Video/imperative/MutableGraphicsState";
 
@@ -32,9 +32,11 @@ export async function updateGraphics({
   }
 
   const promises: Promise<ScreenShot | undefined>[] = [];
+  const videos = await videoCache.readAsync(replayClient);
 
   // If the current time is before the first paint, we should show nothing
   const paintPoint = findMostRecentPaint(time);
+  const paintIndex = findMostRecentPaintIndex(time);
   const isBeforeFirstCachedPaint = !paintPoint || !paintPoint.paintHash;
   if (isBeforeFirstCachedPaint) {
     updateState(graphicsElement, {
@@ -44,7 +46,7 @@ export async function updateGraphics({
       status: executionPoint ? "loading" : "loaded",
       time,
     });
-  } else {
+  } else if (videos.length == 0) {
     const cachedScreenShot = paintHashCache.getValueIfCached(paintPoint.paintHash);
     if (cachedScreenShot) {
       // If this screenshot has already been cached, skip fetching it again
@@ -63,20 +65,34 @@ export async function updateGraphics({
   }
 
   let repaintGraphicsScreenShot: ScreenShot | undefined = undefined;
-  if (executionPoint) {
-    const promise = fetchRepaintGraphics({
+  // if (executionPoint) {
+  //   const promise = fetchRepaintGraphics({
+  //     executionPoint,
+  //     replayClient,
+  //     time,
+  //   }).then(screenShot => {
+  //     repaintGraphicsScreenShot = screenShot;
+
+  //     return screenShot;
+  //   });
+
+  //   promises.push(promise);
+  // }
+
+  if (videos.length > 0) {
+    updateState(graphicsElement, {
       executionPoint,
-      replayClient,
+      screenShotType: repaintGraphicsScreenShot != null ? "repaint" : "cached-paint",
+      status: "loaded",
       time,
-    }).then(screenShot => {
-      repaintGraphicsScreenShot = screenShot;
-
-      return screenShot;
+      paintIndex,
+      videos
     });
 
-    promises.push(promise);
+    return true;
   }
 
+
   if (promises.length === 0) {
     // If we are before the first paint and have no execution point to request a repaint,
     // then we should clear out the currently visible graphics and bail out
@@ -97,6 +113,7 @@ export async function updateGraphics({
       screenShotType: repaintGraphicsScreenShot != null ? "repaint" : "cached-paint",
       status: "loaded",
       time,
+      paintIndex,
     });
 
     if (repaintGraphicsScreenShot != null) {