ui: VideoFrames plugin

ui/src/plugins/dev.perfetto.VideoFrames/index.ts

@@ -0,0 +1,79 @@
+// Copyright (C) 2026 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import './styles.scss';
+import type {PerfettoPlugin} from '../../public/plugin';
+import type {Trace} from '../../public/trace';
+import {TrackNode} from '../../public/workspace';
+import {NUM, STR_NULL} from '../../trace_processor/query_result';
+import {VideoFramePlayer} from './video_frame_player';
+import {createVideoFramesTrack} from './video_frames_track';
+import {VideoFramesSelectionTab} from './video_frames_selection_tab';
+
+interface StreamInfo {
+  displayId: number;
+  displayName: string;
+}
+
+export default class implements PerfettoPlugin {
+  static readonly id = 'dev.perfetto.VideoFrames';
+
+  async onTraceLoad(ctx: Trace): Promise<void> {
+    const res = await ctx.engine.query(`
+      SELECT display_id AS displayId, MAX(display_name) AS displayName
+      FROM __intrinsic_video_frames
+      GROUP BY display_id
+      ORDER BY display_id
+    `);
+
+    const streams: StreamInfo[] = [];
+    const it = res.iter({displayId: NUM, displayName: STR_NULL});
+    for (; it.valid(); it.next()) {
+      streams.push({
+        displayId: it.displayId,
+        displayName: it.displayName ?? `Display ${it.displayId}`,
+      });
+    }
+    if (streams.length === 0) return;
+
+    const group = new TrackNode({
+      name: 'Video Frames',
+      isSummary: true,
+      sortOrder: -55,
+    });
+
+    for (const stream of streams) {
+      const uri = `/video_frames/${stream.displayId}`;
+      const player = new VideoFramePlayer(ctx, uri, stream.displayId);
+
+      ctx.tracks.registerTrack({
+        uri,
+        renderer: createVideoFramesTrack(ctx, uri, stream.displayId, player),
+      });
+      group.addChildInOrder(new TrackNode({uri, name: stream.displayName}));
+
+      ctx.selection.registerAreaSelectionTab(
+        new VideoFramesSelectionTab(
+          ctx,
+          uri,
+          stream.displayId,
+          stream.displayName,
+          player,
+        ),
+      );
+    }
+
+    ctx.defaultWorkspace.addChildInOrder(group);
+  }
+}

ui/src/plugins/dev.perfetto.VideoFrames/styles.scss

@@ -0,0 +1,86 @@
+// Copyright (C) 2026 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Bare default: safe in any container (e.g. the area-selection tab whose
+// shell we don't control). max-width keeps it from overflowing
+// horizontally; height stays intrinsic so it can never collapse to zero.
+.pf-video-frame-preview {
+  display: block;
+  max-width: 100%;
+  border: 1px solid var(--md-sys-color-outline-variant);
+  border-radius: 4px;
+}
+
+// Hover-preview shown in the track tooltip, same UX as the Screenshots
+// track's thumbnail: just the decoded frame sized to itself and capped,
+// with no wrapper box -- so there is no letterbox / white padding around
+// the image. A smaller cap keeps the preview compact so it minimises how
+// far it extends over the ruler above the (short) slice track row.
+.pf-video-frame-tooltip__img {
+  display: block;
+  max-width: 180px;
+  max-height: 180px;
+}
+
+// Layout scoping for the VideoFrame details panel. DetailsShell
+// (fillHeight) already gives a fixed-height, non-scrolling shell with a
+// flex-column body; these rules make the height chain down to the preview
+// canvas *definite* so the canvas is bounded to the available height
+// (never scrolls) and rescales to fit -- preserving aspect ratio via
+// object-fit -- as the panel is resized.
+.pf-video-frame-shell {
+  // The panel body never scrolls; the preview rescales to fit instead.
+  .pf-content {
+    overflow: hidden;
+  }
+
+  // Fill the body and shrink below intrinsic content height (the standard
+  // flex/grid min-height:0 fix). box-sizing + the default 8px grid margin
+  // would otherwise push total content 16px past the box and re-introduce
+  // a scrollbar, so the grid fills the content box exactly: height 100%
+  // minus its own vertical margins. stretch gives each column the full row
+  // height, so the preview column has a definite height to scale into.
+  .pf-grid-layout {
+    box-sizing: border-box;
+    height: calc(100% - 16px);
+    min-height: 0;
+    align-items: stretch;
+  }
+
+  // Only the preview section fills + centers; the details (table) section
+  // keeps its natural height.
+  .pf-video-frame-preview-section {
+    display: flex;
+    flex-direction: column;
+    min-height: 0;
+
+    article {
+      flex: 1;
+      min-height: 0;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+    }
+
+    // Within the bounded section, fill the box and letterbox the decoded
+    // frame to preserve aspect ratio. The canvas *buffer* stays native
+    // resolution (set in drawFrame); this only controls displayed size.
+    .pf-video-frame-preview {
+      width: 100%;
+      height: 100%;
+      max-width: 100%;
+      object-fit: contain;
+    }
+  }
+}

ui/src/plugins/dev.perfetto.VideoFrames/video_frame_details_panel.ts

@@ -0,0 +1,137 @@
+// Copyright (C) 2026 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import m from 'mithril';
+import {Time} from '../../base/time';
+import {Timestamp} from '../../components/widgets/timestamp';
+import type {TrackEventDetailsPanel} from '../../public/details_panel';
+import type {TrackEventSelection} from '../../public/selection';
+import {Button, ButtonBar} from '../../widgets/button';
+import {Intent} from '../../widgets/common';
+import {DetailsShell} from '../../widgets/details_shell';
+import {GridLayout} from '../../widgets/grid_layout';
+import {Section} from '../../widgets/section';
+import {Select} from '../../widgets/select';
+import {Tree, TreeNode} from '../../widgets/tree';
+import type {VideoFramePlayer} from './video_frame_player';
+
+// Playback speed options, in real-time multiples.
+const PLAYBACK_RATES = [0.1, 0.2, 0.5, 1, 1.5, 2];
+
+export class VideoFrameDetailsPanel implements TrackEventDetailsPanel {
+  private readonly player: VideoFramePlayer;
+
+  constructor(player: VideoFramePlayer) {
+    this.player = player;
+  }
+
+  async load(sel: TrackEventSelection) {
+    // Skip re-decode when the selection update was triggered by us
+    // (playback loop, next/prev, or the player's own selectAndSeek path).
+    // The player already has currentIdx pointing at this event.
+    if (this.player.playing) return;
+    await this.player.ensureFramesLoaded();
+    if (this.player.frames[this.player.currentIdx]?.id === sel.eventId) return;
+    await this.player.seek(sel.eventId);
+  }
+
+  render() {
+    const p = this.player;
+    const frame = p.currentFrame;
+    if (frame === undefined) {
+      return m(DetailsShell, {title: 'Video Frame'}, m('span', 'Loading...'));
+    }
+    const detailRows = [
+      m(TreeNode, {left: 'Frame number', right: `${frame.frameNumber}`}),
+      m(TreeNode, {
+        left: 'Timestamp',
+        right: m(Timestamp, {trace: p.trace, ts: Time.fromRaw(frame.ts)}),
+      }),
+    ];
+    for (const err of p.errors) {
+      detailRows.push(m(TreeNode, {left: 'Stream error', right: err}));
+    }
+    return m(
+      DetailsShell,
+      {
+        title: 'Video Frame',
+        description: `Frame ${frame.frameNumber}`,
+        buttons: this.renderControls(),
+        className: 'pf-video-frame-shell',
+      },
+      m(
+        GridLayout,
+        m(Section, {title: 'Details'}, m(Tree, detailRows)),
+        m(
+          Section,
+          {title: 'Preview', className: 'pf-video-frame-preview-section'},
+          p.webCodecsAvailable
+            ? m('canvas.pf-video-frame-preview', {
+                oncreate: ({dom}) => p.attachCanvas(dom as HTMLCanvasElement),
+                onremove: () => p.detachCanvas(),
+              })
+            : m(
+                'span',
+                'Frame preview requires WebCodecs, which is unavailable in ' +
+                  'this browser or context (a secure context / https is ' +
+                  'needed).',
+              ),
+        ),
+      ),
+    );
+  }
+
+  private renderControls(): m.Children {
+    const p = this.player;
+    const idx = p.currentIdx;
+    const total = p.frames.length;
+    return m(
+      ButtonBar,
+      m(Button, {
+        icon: 'skip_previous',
+        compact: true,
+        disabled: idx <= 0 || p.playing,
+        onclick: () => p.prev(),
+      }),
+      m(Button, {
+        icon: p.playing ? 'pause' : 'play_arrow',
+        intent: p.playing ? Intent.Warning : Intent.Primary,
+        compact: true,
+        onclick: () => p.togglePlay(),
+      }),
+      m(Button, {
+        icon: 'skip_next',
+        compact: true,
+        disabled: idx >= total - 1 || p.playing,
+        onclick: () => p.next(),
+      }),
+      m(
+        Select,
+        {
+          title: 'Playback speed',
+          onchange: (e: Event) => {
+            p.setPlaybackRate(Number((e.target as HTMLSelectElement).value));
+          },
+        },
+        PLAYBACK_RATES.map((rate) =>
+          m(
+            'option',
+            {value: rate, selected: rate === p.playbackRate},
+            `${rate}x`,
+          ),
+        ),
+      ),
+    );
+  }
+}