diff --git a/torchci/clickhouse_queries/vllm/docker_build_runtime/params.json b/torchci/clickhouse_queries/vllm/docker_build_runtime/params.json new file mode 100644 index 0000000000..7f2c8af8ca --- /dev/null +++ b/torchci/clickhouse_queries/vllm/docker_build_runtime/params.json @@ -0,0 +1,14 @@ +{ + "params": { + "repo": "String", + "startTime": "DateTime64(3)", + "stopTime": "DateTime64(3)" + }, + "tests": [ + { + "repo": "https://github.com/vllm-project/vllm.git", + "startTime": "2025-10-01T00:00:00.000", + "stopTime": "2025-11-01T00:00:00.000" + } + ] +} diff --git a/torchci/clickhouse_queries/vllm/docker_build_runtime/query.sql b/torchci/clickhouse_queries/vllm/docker_build_runtime/query.sql new file mode 100644 index 0000000000..8041cdb538 --- /dev/null +++ b/torchci/clickhouse_queries/vllm/docker_build_runtime/query.sql @@ -0,0 +1,32 @@ +-- vLLM Docker Build Image Runtime Trends (main branch only) +-- Tracks runtime for the ":docker: build image" job specifically +-- This is a critical job for build speed monitoring + +WITH jobs AS ( + SELECT + tupleElement(job, 'name') AS job_name, + tupleElement(job, 'started_at') AS job_started_at, + tupleElement(job, 'finished_at') AS job_finished_at, + tupleElement(job, 'state') AS job_state, + tupleElement(build, 'number') AS build_number + FROM vllm.vllm_buildkite_jobs + WHERE + tupleElement(pipeline, 'repository') = {repo: String } + AND tupleElement(build, 'branch') = 'main' + AND tupleElement(job, 'name') = ':docker: build image' + AND tupleElement(job, 'started_at') IS NOT NULL + AND tupleElement(job, 'finished_at') IS NOT NULL + AND tupleElement(job, 'started_at') >= {startTime: DateTime64(3) } + AND tupleElement(job, 'started_at') < {stopTime: DateTime64(3) } + AND lowerUTF8(tupleElement(job, 'state')) IN ( + 'passed', 'finished', 'success', 'failed' + ) +) + +SELECT + job_started_at AS timestamp, + build_number, + round(dateDiff('second', job_started_at, job_finished_at) / 60.0, 2) + AS runtime_minutes +FROM jobs +ORDER BY job_started_at ASC diff --git a/torchci/clickhouse_queries/vllm/job_runtime_trends/params.json b/torchci/clickhouse_queries/vllm/job_runtime_trends/params.json new file mode 100644 index 0000000000..5f8714d676 --- /dev/null +++ b/torchci/clickhouse_queries/vllm/job_runtime_trends/params.json @@ -0,0 +1,16 @@ +{ + "params": { + "repo": "String", + "startTime": "DateTime64(3)", + "stopTime": "DateTime64(3)", + "jobGroups": "Array(String)" + }, + "tests": [ + { + "repo": "https://github.com/vllm-project/vllm.git", + "startTime": "2025-10-01T00:00:00.000", + "stopTime": "2025-10-08T00:00:00.000", + "jobGroups": ["main", "amd", "torch_nightly"] + } + ] +} diff --git a/torchci/clickhouse_queries/vllm/job_runtime_trends/query.sql b/torchci/clickhouse_queries/vllm/job_runtime_trends/query.sql new file mode 100644 index 0000000000..48c347009a --- /dev/null +++ b/torchci/clickhouse_queries/vllm/job_runtime_trends/query.sql @@ -0,0 +1,66 @@ +-- vLLM Job Runtime Trends (main branch only) +-- Aggregates per-job runtime statistics by day +-- Shows count, mean, p90, and max runtime for each job per day +-- Supports filtering by job groups: AMD, Torch Nightly, or Main + +WITH jobs AS ( + SELECT + tupleElement(job, 'name') AS job_name, + tupleElement(job, 'started_at') AS job_started_at, + tupleElement(job, 'finished_at') AS job_finished_at, + tupleElement(job, 'state') AS job_state, + tupleElement(build, 'branch') AS branch + FROM vllm.vllm_buildkite_jobs + WHERE + tupleElement(pipeline, 'repository') = {repo: String } + AND tupleElement(build, 'branch') = 'main' + AND tupleElement(job, 'started_at') IS NOT NULL + AND tupleElement(job, 'finished_at') IS NOT NULL + AND tupleElement(job, 'started_at') >= {startTime: DateTime64(3) } + AND tupleElement(job, 'started_at') < {stopTime: DateTime64(3) } + AND lowerUTF8(tupleElement(job, 'state')) IN ( + 'passed', 'finished', 'success', 'failed' + ) + -- Job group filtering: AMD, Torch Nightly, or Main + AND ( + ( + has({jobGroups: Array(String)}, 'amd') + AND positionCaseInsensitive(tupleElement(job, 'name'), 'AMD') + > 0 + ) + OR ( + has({jobGroups: Array(String)}, 'torch_nightly') + AND positionCaseInsensitive( + tupleElement(job, 'name'), 'Torch Nightly' + ) + > 0 + ) + OR ( + has({jobGroups: Array(String)}, 'main') + AND positionCaseInsensitive(tupleElement(job, 'name'), 'AMD') + = 0 + AND positionCaseInsensitive( + tupleElement(job, 'name'), 'Torch Nightly' + ) + = 0 + ) + ) +) + +SELECT + job_name, + toDate(job_started_at) AS date, + count() AS count, + round(avg(dateDiff('second', job_started_at, job_finished_at) / 60.0), 2) + AS mean_runtime_minutes, + round( + quantile(0.9) ( + dateDiff('second', job_started_at, job_finished_at) / 60.0 + ), + 2 + ) AS p90_runtime_minutes, + round(max(dateDiff('second', job_started_at, job_finished_at) / 60.0), 2) + AS max_runtime_minutes +FROM jobs +GROUP BY job_name, date +ORDER BY job_name ASC, date ASC diff --git a/torchci/components/metrics/vllm/CiDurationsPanel.tsx b/torchci/components/metrics/vllm/CiDurationsPanel.tsx index 63b8c49425..30e5db0462 100644 --- a/torchci/components/metrics/vllm/CiDurationsPanel.tsx +++ b/torchci/components/metrics/vllm/CiDurationsPanel.tsx @@ -230,28 +230,11 @@ export default function CiDurationsPanel({ ...getLineSeries(dailyMeanSuccess, dailyMeanNonCanceled), ...getScatterSeriesByState(source), ], - dataZoom: [ - { - type: "slider", - show: true, - xAxisIndex: 0, - bottom: 0, - start: 0, - end: 100, - height: 25, - }, - { - type: "inside", - xAxisIndex: 0, - start: 0, - end: 100, - }, - ], }; return ( Build #${buildNumber}
` + : `Daily Average
`; + result += `Time: ${formattedTime}
`; + result += `Runtime: ${runtime.toFixed(1)} min`; + + return result; +} + +// Helper function to handle click events +function handleBuildClick(params: any) { + if (params?.componentType === "series") { + const data = Array.isArray(params.data) ? params.data : [params.data]; + const buildNumber = data[2]; + if (buildNumber !== undefined && buildNumber !== null) { + const url = `https://buildkite.com/vllm/ci/builds/${buildNumber}/`; + if (typeof window !== "undefined") { + window.open(url, "_blank"); + } + } + } +} + +export default function DockerBuildRuntimePanel({ + data, +}: { + data: DockerBuildData[] | undefined; +}) { + const { darkMode } = useDarkMode(); + + // Process data for chart + const chartData = (data || []).map((d) => [ + dayjs(d.timestamp).toISOString(), + d.runtime_minutes, + d.build_number, + ]); + + // Calculate daily average for trend line + const groupedByDay = _.groupBy(data || [], (d) => + dayjs(d.timestamp).format("YYYY-MM-DD") + ); + + const dailyAvg = Object.entries(groupedByDay) + .map(([day, records]) => { + const avgRuntime = _.meanBy(records, "runtime_minutes"); + return { + day, + value: Number(avgRuntime.toFixed(1)), + }; + }) + .sort((a, b) => (a.day < b.day ? -1 : 1)); + + // Calculate statistics + const runtimes = (data || []).map((d) => d.runtime_minutes); + const avgRuntime = runtimes.length ? _.mean(runtimes).toFixed(1) : "N/A"; + const p90Runtime = runtimes.length + ? runtimes + .sort((a, b) => a - b) + [Math.floor(runtimes.length * 0.9)].toFixed(1) + : "N/A"; + + const options: EChartsOption = { + title: { + text: "Docker Build Image Runtime", + subtext: `Avg: ${avgRuntime}m | P90: ${p90Runtime}m | Total builds: ${runtimes.length}`, + textStyle: { + fontSize: 14, + }, + }, + legend: { + top: 24, + data: ["Individual Builds", "Daily Average"], + }, + grid: { top: 60, right: 20, bottom: 80, left: 60 }, + dataset: [{ source: chartData }, { source: dailyAvg }], + xAxis: { + type: "time", + axisLabel: { + hideOverlap: true, + formatter: (value: number) => dayjs(value).format("M/D"), + }, + }, + yAxis: { + type: "value", + name: "Runtime (minutes)", + nameLocation: "middle", + nameGap: 45, + nameRotate: 90, + axisLabel: { + formatter: (value: number) => `${value}m`, + }, + }, + series: [ + { + name: "Individual Builds", + type: "scatter", + datasetIndex: 0, + symbolSize: 6, + itemStyle: { color: COLOR_SUCCESS, opacity: 0.6 }, + }, + { + name: "Daily Average", + type: "line", + datasetIndex: 1, + smooth: true, + encode: { x: "day", y: "value" }, + lineStyle: { color: COLOR_WARNING, width: 2 }, + itemStyle: { color: COLOR_WARNING }, + showSymbol: true, + symbolSize: 4, + }, + ], + tooltip: { + trigger: "item", + formatter: formatTooltip, + }, + }; + + return ( + + ); +} diff --git a/torchci/components/metrics/vllm/DurationDistributionPanel.tsx b/torchci/components/metrics/vllm/DurationDistributionPanel.tsx index b4f9e4ba65..bf261e365d 100644 --- a/torchci/components/metrics/vllm/DurationDistributionPanel.tsx +++ b/torchci/components/metrics/vllm/DurationDistributionPanel.tsx @@ -164,28 +164,11 @@ export default function DurationDistributionPanel({ axisPointer: { type: "shadow" }, formatter: formatDistributionTooltip, }, - dataZoom: [ - { - type: "slider", - show: true, - xAxisIndex: 0, - bottom: 0, - start: 0, - end: 100, - height: 25, - }, - { - type: "inside", - xAxisIndex: 0, - start: 0, - end: 100, - }, - ], }; return ( diff --git a/torchci/components/metrics/vllm/JobRuntimePanel.tsx b/torchci/components/metrics/vllm/JobRuntimePanel.tsx new file mode 100644 index 0000000000..80ed0c5dc1 --- /dev/null +++ b/torchci/components/metrics/vllm/JobRuntimePanel.tsx @@ -0,0 +1,397 @@ +import { + Box, + Paper, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TableSortLabel, + TextField, +} from "@mui/material"; +import dayjs from "dayjs"; +import { EChartsOption } from "echarts"; +import ReactECharts from "echarts-for-react"; +import { useDarkMode } from "lib/DarkModeContext"; +import React, { useState } from "react"; +import { getCrosshairTooltipConfig, getReactEChartsProps } from "./chartUtils"; +import { COLOR_SUCCESS, COLOR_WARNING } from "./constants"; + +interface JobRuntimeData { + job_name: string; + date: string; + count: number; + mean_runtime_minutes: number; + p90_runtime_minutes: number; + max_runtime_minutes: number; +} + +interface JobAggregatedStats { + job_name: string; + count: number; + mean: number; + p90: number; + max: number; +} + +type SortField = "job_name" | "count" | "mean" | "p90" | "max"; +type SortOrder = "asc" | "desc"; + +// Helper function to aggregate job statistics across all dates +function aggregateJobStats(data: JobRuntimeData[]): JobAggregatedStats[] { + const jobMap = new Map(); + + // Group by job name + data.forEach((row) => { + if (!jobMap.has(row.job_name)) { + jobMap.set(row.job_name, []); + } + jobMap.get(row.job_name)!.push(row); + }); + + // Aggregate statistics + const result: JobAggregatedStats[] = []; + jobMap.forEach((rows, jobName) => { + const totalCount = rows.reduce((sum, r) => sum + r.count, 0); + const avgMean = + rows.reduce((sum, r) => sum + r.mean_runtime_minutes * r.count, 0) / + totalCount; + const avgP90 = + rows.reduce((sum, r) => sum + r.p90_runtime_minutes * r.count, 0) / + totalCount; + const overallMax = Math.max(...rows.map((r) => r.max_runtime_minutes)); + + result.push({ + job_name: jobName, + count: totalCount, + mean: avgMean, + p90: avgP90, + max: overallMax, + }); + }); + + return result; +} + +// Helper function to format runtime with unit +function formatRuntime(minutes: number | null | undefined): string { + if (minutes === null || minutes === undefined) return "-"; + return minutes.toFixed(1) + "m"; +} + +// Helper function to format tooltip +function formatChartTooltip(params: any): string { + if (!Array.isArray(params) || params.length === 0) return ""; + + const date = params[0].axisValue; + let result = `${date}
`; + + params.forEach((p: any) => { + if (p.value !== undefined && p.value !== null) { + result += `${p.marker} ${p.seriesName}: ${p.value.toFixed( + 1 + )}m
`; + } + }); + + return result; +} + +// Helper function to get line chart series +function getLineSeries( + dates: string[], + meanData: number[], + p90Data: number[] +): any[] { + return [ + { + name: "Mean Runtime", + type: "line", + data: meanData, + smooth: true, + symbol: "circle", + symbolSize: 6, + itemStyle: { color: COLOR_SUCCESS }, + lineStyle: { width: 2 }, + emphasis: { focus: "series" }, + }, + { + name: "P90 Runtime", + type: "line", + data: p90Data, + smooth: true, + symbol: "diamond", + symbolSize: 7, + itemStyle: { color: COLOR_WARNING }, + lineStyle: { width: 2, type: "dashed" }, + emphasis: { focus: "series" }, + }, + ]; +} + +export default function JobRuntimePanel({ + data, +}: { + data: JobRuntimeData[] | undefined; +}) { + const { darkMode } = useDarkMode(); + const [sortField, setSortField] = useState("mean"); + const [sortOrder, setSortOrder] = useState("desc"); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedJob, setSelectedJob] = useState(null); + + // Aggregate statistics for the table + const aggregatedStats = aggregateJobStats(data || []); + + // Filter by search query + const filteredStats = aggregatedStats.filter((job) => + job.job_name.toLowerCase().includes(searchQuery.toLowerCase()) + ); + + // Sort the filtered data + const sortedStats = [...filteredStats].sort((a, b) => { + let aValue: number | string = a[sortField]; + let bValue: number | string = b[sortField]; + + if (sortField === "job_name") { + aValue = (aValue as string).toLowerCase(); + bValue = (bValue as string).toLowerCase(); + return sortOrder === "asc" + ? aValue < bValue + ? -1 + : 1 + : aValue > bValue + ? -1 + : 1; + } + + return sortOrder === "asc" + ? (aValue as number) - (bValue as number) + : (bValue as number) - (aValue as number); + }); + + // Auto-select first job if nothing is selected or if selected job is no longer in the list + React.useEffect(() => { + if (sortedStats.length > 0) { + if ( + !selectedJob || + !sortedStats.some((s) => s.job_name === selectedJob) + ) { + setSelectedJob(sortedStats[0].job_name); + } + } + }, [sortedStats, selectedJob]); + + // Handle sort request + function handleSort(field: SortField) { + if (sortField === field) { + setSortOrder(sortOrder === "asc" ? "desc" : "asc"); + } else { + setSortField(field); + setSortOrder("desc"); + } + } + + // Handle row click + function handleRowClick(jobName: string) { + setSelectedJob(jobName); + } + + // Prepare chart data for selected job + const selectedJobData = + selectedJob && data + ? data + .filter((d) => d.job_name === selectedJob) + .sort((a, b) => a.date.localeCompare(b.date)) + : []; + + const chartDates = selectedJobData.map((d) => dayjs(d.date).format("MMM D")); + const chartMeanData = selectedJobData.map((d) => d.mean_runtime_minutes); + const chartP90Data = selectedJobData.map((d) => d.p90_runtime_minutes); + + const chartOptions: EChartsOption = { + title: { + text: selectedJob ? "Runtime Trend" : "Select a job to view", + subtext: selectedJob || "Click a row in the table", + textStyle: { + fontSize: 14, + }, + subtextStyle: { + fontSize: 16, + fontWeight: "bold", + color: darkMode ? "#fff" : "#333", + }, + }, + legend: { + top: 40, + data: ["Mean Runtime", "P90 Runtime"], + }, + grid: { top: 80, right: 20, bottom: 60, left: 60 }, + xAxis: { + type: "category", + data: chartDates, + name: "Date", + nameLocation: "middle", + nameGap: 35, + axisLabel: { + rotate: 45, + fontSize: 10, + }, + }, + yAxis: { + type: "value", + name: "Runtime (minutes)", + nameLocation: "middle", + nameGap: 45, + axisLabel: { + formatter: (value: number) => `${value}m`, + }, + }, + series: + selectedJobData.length > 0 + ? getLineSeries(chartDates, chartMeanData, chartP90Data) + : [], + tooltip: getCrosshairTooltipConfig(darkMode, formatChartTooltip), + }; + + return ( + + + {/* Table on the left */} + + setSearchQuery(e.target.value)} + sx={{ mb: 1 }} + fullWidth + /> + + + + + + handleSort("job_name")} + > + Job Name + + + + handleSort("count")} + > + Count + + + + handleSort("mean")} + > + Mean + + + + handleSort("p90")} + > + P90 + + + + handleSort("max")} + > + Max + + + + + + {sortedStats.map((job) => ( + handleRowClick(job.job_name)} + selected={selectedJob === job.job_name} + sx={{ + cursor: "pointer", + "&.Mui-selected": { + backgroundColor: darkMode + ? "rgba(144, 202, 249, 0.16)" + : "rgba(25, 118, 210, 0.12)", + }, + "&.Mui-selected:hover": { + backgroundColor: darkMode + ? "rgba(144, 202, 249, 0.24)" + : "rgba(25, 118, 210, 0.18)", + }, + }} + > + + {job.job_name} + + {job.count} + + {formatRuntime(job.mean)} + + + {formatRuntime(job.p90)} + + + {formatRuntime(job.max)} + + + ))} + +
+ + + + {/* Chart on the right */} + + + + + + ); +} diff --git a/torchci/components/metrics/vllm/constants.ts b/torchci/components/metrics/vllm/constants.ts index c1e85785e5..390d8c8622 100644 --- a/torchci/components/metrics/vllm/constants.ts +++ b/torchci/components/metrics/vllm/constants.ts @@ -1,6 +1,59 @@ -// Shared color constants for vLLM metrics charts +// Shared constants for vLLM metrics dashboard and charts -// Data visualization colors +// ============================================================================ +// Layout Constants +// ============================================================================ +export const ROW_HEIGHT = 375; +export const METRIC_CARD_HEIGHT = 200; +export const JOB_RUNTIME_PANEL_HEIGHT = ROW_HEIGHT + 150; + +// ============================================================================ +// Repository Constants +// ============================================================================ +export const VLLM_REPO_URL = "https://github.com/vllm-project/vllm.git"; +export const VLLM_REPO_SHORT = "vllm-project/vllm"; +export const PIPELINE_NAME = "CI"; + +// ============================================================================ +// Query Parameter Defaults +// ============================================================================ +export const DEFAULT_MIN_RUNS_RETRY_STATS = 5; +export const DEFAULT_MIN_RUNS_JOB_RELIABILITY = 3; + +// ============================================================================ +// Tab Styling Configuration +// ============================================================================ +export const TAB_CONFIG = { + containerSx: (darkMode: boolean) => ({ + borderBottom: 2, + borderColor: "divider", + mb: 3, + mt: 2, + bgcolor: darkMode ? "rgba(255, 255, 255, 0.05)" : "rgba(0, 0, 0, 0.02)", + borderRadius: "8px 8px 0 0", + px: 2, + }), + tabsSx: { + "& .MuiTab-root": { + fontSize: "1rem", + fontWeight: 600, + minHeight: 56, + textTransform: "none", + px: 3, + }, + "& .Mui-selected": { + fontWeight: 700, + }, + }, + indicatorSx: { + height: 3, + borderRadius: "3px 3px 0 0", + }, +}; + +// ============================================================================ +// Data Visualization Colors +// ============================================================================ export const COLOR_SUCCESS = "#3ba272"; // Green - for successful/passing states export const COLOR_ERROR = "#ee6666"; // Red - for failures/errors export const COLOR_WARNING = "#fc9403"; // Orange - for warnings/manual actions diff --git a/torchci/pages/metrics/vllm.tsx b/torchci/pages/metrics/vllm.tsx index a5e140ae75..29dbab059e 100644 --- a/torchci/pages/metrics/vllm.tsx +++ b/torchci/pages/metrics/vllm.tsx @@ -7,15 +7,19 @@ import { Link, Skeleton, Stack, + Tab, + Tabs, Typography, } from "@mui/material"; import CiDurationsPanel from "components/metrics/vllm/CiDurationsPanel"; import CommitsOnRedTrendPanel from "components/metrics/vllm/CommitsOnRedTrendPanel"; +import DockerBuildRuntimePanel from "components/metrics/vllm/DockerBuildRuntimePanel"; import DurationDistributionPanel from "components/metrics/vllm/DurationDistributionPanel"; import JobGroupFilter, { JobGroup, } from "components/metrics/vllm/JobGroupFilter"; import JobReliabilityPanel from "components/metrics/vllm/JobReliabilityPanel"; +import JobRuntimePanel from "components/metrics/vllm/JobRuntimePanel"; import MergesPanel from "components/metrics/vllm/MergesPanel"; import MostRetriedJobsTable from "components/metrics/vllm/MostRetriedJobsTable"; import ReliabilityPanel from "components/metrics/vllm/ReliabilityPanel"; @@ -30,6 +34,17 @@ import { VllmDualScalarPanel, VllmScalarPanel, } from "components/metrics/vllm/VllmScalarPanel"; +import { + DEFAULT_MIN_RUNS_JOB_RELIABILITY, + DEFAULT_MIN_RUNS_RETRY_STATS, + JOB_RUNTIME_PANEL_HEIGHT, + METRIC_CARD_HEIGHT, + PIPELINE_NAME, + ROW_HEIGHT, + TAB_CONFIG, + VLLM_REPO_SHORT, + VLLM_REPO_URL, +} from "components/metrics/vllm/constants"; import dayjs from "dayjs"; import { useDarkMode } from "lib/DarkModeContext"; import { useClickHouseAPIImmutable } from "lib/GeneralUtils"; @@ -37,11 +52,6 @@ import _ from "lodash"; import React, { useState } from "react"; import { TimeRangePicker } from "../metrics"; -const ROW_HEIGHT = 375; -const METRIC_CARD_HEIGHT = 200; // Height for key metric cards (reduced by ~20% from default) - -// moved MergesPanel and CiDurationsPanel to components - // Helper function to safely extract PR cycle data values function getPrCycleValue( data: any[] | undefined, @@ -216,6 +226,11 @@ export default function Page() { "torch_nightly", "main", ]); + const [selectedTab, setSelectedTab] = useState(0); + + const handleTabChange = (_: React.SyntheticEvent, newValue: number) => { + setSelectedTab(newValue); + }; const timeParams = { startTime: startTime.utc().format("YYYY-MM-DDTHH:mm:ss.SSS"), @@ -237,7 +252,7 @@ export default function Page() { { ...timeParams, granularity: "day", - repo: "vllm-project/vllm", + repo: VLLM_REPO_SHORT, } ); @@ -245,9 +260,8 @@ export default function Page() { "vllm/ci_run_duration", { ...timeParams, - // Buildkite uses full repo URL with .git in vLLM dataset - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, } ); @@ -255,8 +269,8 @@ export default function Page() { "vllm/ci_run_duration", { ...prevTimeParams, - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, } ); @@ -311,7 +325,7 @@ export default function Page() { "vllm/pr_cycle_time_breakdown", { ...timeParams, - repo: "vllm-project/vllm", + repo: VLLM_REPO_SHORT, } ); @@ -319,7 +333,7 @@ export default function Page() { "vllm/pr_cycle_time_breakdown", { ...prevTimeParams, - repo: "vllm-project/vllm", + repo: VLLM_REPO_SHORT, } ); @@ -328,8 +342,8 @@ export default function Page() { { ...timeParams, granularity: "day", - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, } ); @@ -337,8 +351,8 @@ export default function Page() { const { data: retryData } = useClickHouseAPIImmutable("vllm/rebuild_rate", { ...timeParams, granularity: "day", - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, }); @@ -346,9 +360,9 @@ export default function Page() { "vllm/job_retry_stats", { ...timeParams, - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", - minRuns: 5, + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, + minRuns: DEFAULT_MIN_RUNS_RETRY_STATS, jobGroups: selectedJobGroups, } ); @@ -357,20 +371,37 @@ export default function Page() { "vllm/job_reliability", { ...timeParams, - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", - minRuns: 3, + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, + minRuns: DEFAULT_MIN_RUNS_JOB_RELIABILITY, jobGroups: selectedJobGroups, } ); + const { data: jobRuntimeTrendsData } = useClickHouseAPIImmutable( + "vllm/job_runtime_trends", + { + ...timeParams, + repo: VLLM_REPO_URL, + jobGroups: selectedJobGroups, + } + ); + + const { data: dockerBuildRuntimeData } = useClickHouseAPIImmutable( + "vllm/docker_build_runtime", + { + ...timeParams, + repo: VLLM_REPO_URL, + } + ); + const { data: trunkHealthData } = useClickHouseAPIImmutable( "vllm/trunk_health", { ...timeParams, granularity: "day", - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, } ); @@ -379,8 +410,8 @@ export default function Page() { "vllm/trunk_recovery_time", { ...timeParams, - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, } ); @@ -391,8 +422,8 @@ export default function Page() { { ...prevTimeParams, granularity: "day", - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, } ); @@ -402,8 +433,8 @@ export default function Page() { { ...prevTimeParams, granularity: "day", - repo: "https://github.com/vllm-project/vllm.git", - pipelineName: "CI", + repo: VLLM_REPO_URL, + pipelineName: PIPELINE_NAME, jobGroups: selectedJobGroups, } ); @@ -413,7 +444,7 @@ export default function Page() { { ...prevTimeParams, granularity: "day", - repo: "vllm-project/vllm", + repo: VLLM_REPO_SHORT, } ); @@ -662,7 +693,7 @@ export default function Page() { : _.meanBy(recoveryTimes, "recovery_hours"); return ( -
+ - {/* Section 1: Key Metrics Summary Cards */} + {/* Overview - Always Visible */} Key Metrics Overview @@ -822,214 +853,234 @@ export default function Page() { /> - {/* Section 2: CI Reliability */} - - - CI Reliability - - - - (v ?? 1) < 0.85, - tooltip: - "Percentage of main branch builds with zero hard test failures. Builds with only soft failures (flaky tests) count as passed. Canceled builds excluded from calculation.", - delta: overallSuccessRateDelta, - }, - ]} - /> - (v ?? 0) > 10, - tooltip: - "Count of main branch CI runs with hard test failures (soft failures excluded) in selected time period.", - delta: totalFailedDelta, - }, - ]} - /> - (v ?? 0) > 0.01, - tooltip: - "Percentage of jobs that were manually or automatically retried. Low values (<1%) indicate stable infrastructure. High values may indicate flaky tests or infrastructure issues.", - delta: null, // TODO: Add delta when we have previous retry data - }, - ]} - /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - Trunk Health - - - - (v ?? 0) > 12, - tooltip: - "Average time trunk stays broken before being fixed. Measured from when trunk first breaks (success→failure) to when it's fixed (failure→success). Includes nights, weekends, and investigation time. Lower is better.", - delta: null, // TODO: Calculate when we have previous recovery data - }, - ]} - /> - - - - - - - - - - - - - - - - - + {/* Tabs for detailed sections */} + + + + + + + - {/* Section 3: CI Duration Analysis */} - - - CI Duration Analysis - - - - - - - - - - - - - - - + {/* Tab 0: Reliability */} + {selectedTab === 0 && ( + <> + + (v ?? 1) < 0.85, + tooltip: + "Percentage of main branch builds with zero hard test failures. Builds with only soft failures (flaky tests) count as passed. Canceled builds excluded from calculation.", + delta: overallSuccessRateDelta, + }, + ]} + /> + (v ?? 0) > 10, + tooltip: + "Count of main branch CI runs with hard test failures (soft failures excluded) in selected time period.", + delta: totalFailedDelta, + }, + ]} + /> + (v ?? 0) > 0.01, + tooltip: + "Percentage of jobs that were manually or automatically retried. Low values (<1%) indicate stable infrastructure. High values may indicate flaky tests or infrastructure issues.", + delta: null, // TODO: Add delta when we have previous retry data + }, + ]} + /> + (v ?? 0) > 12, + tooltip: + "Average time trunk stays broken before being fixed. Measured from when trunk first breaks (success→failure) to when it's fixed (failure→success). Includes nights, weekends, and investigation time. Lower is better.", + delta: null, // TODO: Calculate when we have previous recovery data + }, + ]} + /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + )} - {/* Section 4: PR Cycle Metrics */} - - - PR Cycle Metrics - - - - (v ?? 0) > 0.5, - tooltip: - "Percentage of merged PRs where a human clicked 'Merge' button instead of using GitHub auto-merge. Includes both clean manual merges AND force merges. High values may indicate slow merge queues or low CI trust.", - delta: manualMergedPctDelta, - }, - ]} - /> - (v ?? 0) > 24, - badThreshold2: (v) => (v ?? 0) > 72, - tooltip: - "Time from PR ready (labeled 'ready' or created) to first human review comment. P50 = median, P90 = 90th percentile. Excludes bot reviews.", - delta: timeToReviewP50Delta, - delta2: timeToReviewP90Delta, - }, - ]} - /> - (v ?? 0) > 48, - badThreshold2: (v) => (v ?? 0) > 120, - tooltip: - "Time from first human review to first approval from a maintainer (MEMBER/OWNER/COLLABORATOR). P50 = median, P90 = 90th percentile.", - delta: timeToApprovalP50Delta, - delta2: timeToApprovalP90Delta, - }, - ]} - /> - - - - - - -
+ {/* Tab 1: Duration Analysis */} + {selectedTab === 1 && ( + <> + + + + + + + + + + + + + + + + + + + + + + + )} + + {/* Tab 2: Source Control */} + {selectedTab === 2 && ( + <> + + (v ?? 0) > 0.5, + tooltip: + "Percentage of merged PRs where a human clicked 'Merge' button instead of using GitHub auto-merge. Includes both clean manual merges AND force merges. High values may indicate slow merge queues or low CI trust.", + delta: manualMergedPctDelta, + }, + ]} + /> + (v ?? 0) > 24, + badThreshold2: (v) => (v ?? 0) > 72, + tooltip: + "Time from PR ready (labeled 'ready' or created) to first human review comment. P50 = median, P90 = 90th percentile. Excludes bot reviews.", + delta: timeToReviewP50Delta, + delta2: timeToReviewP90Delta, + }, + ]} + /> + (v ?? 0) > 48, + badThreshold2: (v) => (v ?? 0) > 120, + tooltip: + "Time from first human review to first approval from a maintainer (MEMBER/OWNER/COLLABORATOR). P50 = median, P90 = 90th percentile.", + delta: timeToApprovalP50Delta, + delta2: timeToApprovalP90Delta, + }, + ]} + /> + + + + + + + + )} + ); }