From 8ebd59820e92467f927ba9c3dabd4385196b9870 Mon Sep 17 00:00:00 2001 From: Yasmine Frigui Date: Wed, 19 Aug 2026 11:40:44 -0400 Subject: [PATCH 1/6] feat: add project evaluator template gallery --- js/app/src/Routes.tsx | 12 + .../CreateProjectEvaluatorSlideover.tsx | 72 ++- .../ProjectEvaluatorSlideoverRoutes.tsx | 37 ++ .../ProjectEvaluatorsEmptyState.tsx | 532 ++++++++++++++---- .../projectEvaluatorTemplatesQuery.graphql.ts | 216 +++++++ .../projectEvaluatorTemplates.test.ts | 60 ++ .../evaluators/projectEvaluatorPaths.ts | 4 + .../evaluators/projectEvaluatorTemplates.ts | 155 +++++ 8 files changed, 938 insertions(+), 150 deletions(-) create mode 100644 js/app/src/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql.ts create mode 100644 js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts create mode 100644 js/app/src/pages/project/evaluators/projectEvaluatorTemplates.ts diff --git a/js/app/src/Routes.tsx b/js/app/src/Routes.tsx index 7c8fb68c623..59cbd47da16 100644 --- a/js/app/src/Routes.tsx +++ b/js/app/src/Routes.tsx @@ -85,6 +85,7 @@ import { LoggedOutPage, LoginPage, NewCodeProjectEvaluatorPage, + NewLlmFromTemplateProjectEvaluatorPage, NewLlmProjectEvaluatorPage, OAuth2ConsentPage, PlaygroundPage, @@ -496,6 +497,17 @@ export const appRouteObjects = createRoutesFromElements( }, }} /> + } + handle={{ + agentRoute: { + label: "New Project Evaluator From Template", + description: + "Create a project LLM evaluator seeded from a classification evaluator template. The templateName route param is the template's name.", + }, + }} + /> } diff --git a/js/app/src/pages/project/evaluators/CreateProjectEvaluatorSlideover.tsx b/js/app/src/pages/project/evaluators/CreateProjectEvaluatorSlideover.tsx index 25883d941a8..39d9ad32ddb 100644 --- a/js/app/src/pages/project/evaluators/CreateProjectEvaluatorSlideover.tsx +++ b/js/app/src/pages/project/evaluators/CreateProjectEvaluatorSlideover.tsx @@ -49,19 +49,25 @@ import { type EvaluatorStoreProps, } from "@phoenix/store/evaluatorStore"; +type SeededLlmEvaluatorInitialState = { + name: string; + description: string; + outputConfigs: AnnotationConfig[]; + defaultMessages: PlaygroundChatTemplate["messages"]; + templateFormat: TemplateFormat; + includeExplanation: boolean; +}; + export type ProjectEvaluatorCreationMode = | { kind: "scratch" } | { kind: "newCode" } | { kind: "copy"; - initialState: { - name: string; - description: string; - outputConfigs: AnnotationConfig[]; - defaultMessages: PlaygroundChatTemplate["messages"]; - templateFormat: TemplateFormat; - includeExplanation: boolean; - }; + initialState: SeededLlmEvaluatorInitialState; + } + | { + kind: "template"; + initialState: SeededLlmEvaluatorInitialState; } | { kind: "code"; @@ -86,6 +92,9 @@ function getProjectEvaluatorCreationTitle( if (creationMode.kind === "copy") { return `Copy LLM evaluator “${creationMode.initialState.name}”`; } + if (creationMode.kind === "template") { + return `Create “${creationMode.initialState.name}” evaluator`; + } return `Attach code evaluator “${creationMode.name}”`; } @@ -116,15 +125,19 @@ function CreateProjectEvaluatorDialogForMode( props: Parameters[0] ) { const { creationMode } = props; - if (creationMode.kind === "scratch" || creationMode.kind === "copy") { + if ( + creationMode.kind === "scratch" || + creationMode.kind === "copy" || + creationMode.kind === "template" + ) { const defaultMessages = - creationMode.kind === "copy" - ? creationMode.initialState.defaultMessages - : getSpanEvaluatorDefaultMessages(); + creationMode.kind === "scratch" + ? getSpanEvaluatorDefaultMessages() + : creationMode.initialState.defaultMessages; const templateFormat = - creationMode.kind === "copy" - ? creationMode.initialState.templateFormat - : undefined; + creationMode.kind === "scratch" + ? undefined + : creationMode.initialState.templateFormat; return ( ( + projectEvaluatorTemplatesQuery, + {}, + { fetchPolicy: "store-and-network" } + ); + const template = data.classificationEvaluatorConfigs.find( + (config) => config.name === templateName + ); + if (!template) { + return ( + + ); + } + return ( + + ); +} + export function CopyLlmProjectEvaluatorPage() { const projectId = useRouteProjectId(); const onOpenChange = useCloseSlideover(); diff --git a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx index f3e9dcad768..e027fc61970 100644 --- a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx +++ b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx @@ -1,174 +1,458 @@ import { css } from "@emotion/react"; -import type { ReactNode } from "react"; -import { Suspense } from "react"; +import { Suspense, useState } from "react"; import { useLazyLoadQuery } from "react-relay"; import { useNavigate } from "react-router"; -import { Flex, Icon, Icons, Skeleton, Text } from "@phoenix/components"; import { - EmptyState, - EmptyStateArea, - EmptyStateGraphic, -} from "@phoenix/components/core/empty"; + Badge, + Button, + Counter, + Flex, + Heading, + Skeleton, + Text, +} from "@phoenix/components"; import { LineClamp } from "@phoenix/components/core/utility/LineClamp"; import { ErrorBoundary } from "@phoenix/components/exception"; -import type { projectEvaluatorOptionsQuery } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorOptionsQuery.graphql"; -import { projectEvaluatorOptionsQuery as projectEvaluatorOptionsQueryNode } from "@phoenix/pages/project/evaluators/projectEvaluatorOptions"; +import type { projectEvaluatorTemplatesQuery as ProjectEvaluatorTemplatesQueryType } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql"; import { useProjectEvaluatorPaths } from "@phoenix/pages/project/evaluators/projectEvaluatorPaths"; +import { + getProjectEvaluatorTemplateChoices, + getProjectEvaluatorTemplateMetadata, + type ProjectEvaluatorTemplate, + projectEvaluatorTemplatesQuery, +} from "@phoenix/pages/project/evaluators/projectEvaluatorTemplates"; + +const RECOMMENDED_CATEGORY = "Recommended"; +const GALLERY_SKELETON_HEIGHT = 440; -const MAX_COPY_CARDS = 4; -const MAX_ATTACH_CARDS = 3; -const EVALUATOR_CARD_HEIGHT = 90; +type TemplateWithMetadata = { + config: ProjectEvaluatorTemplate; + metadata: ReturnType; +}; export function ProjectEvaluatorsEmptyState() { return ( - - - } - title="No evaluators for this project" - description="Add an evaluator to score spans, traces, or sessions automatically as they arrive." - /> - - }> - - - - - +
+ + }> + + + +
); } function EvaluatorGallery() { const navigate = useNavigate(); const paths = useProjectEvaluatorPaths(); - const data = useLazyLoadQuery( - projectEvaluatorOptionsQueryNode, + const [selectedCategory, setSelectedCategory] = + useState(RECOMMENDED_CATEGORY); + const [selectedTemplateName, setSelectedTemplateName] = useState< + string | null + >(null); + const data = useLazyLoadQuery( + projectEvaluatorTemplatesQuery, {}, { fetchPolicy: "store-and-network" } ); - const evaluators = data.evaluators.edges.map(({ evaluator }) => evaluator); - const copyEvaluators = evaluators - .filter((evaluator) => evaluator.__typename === "LLMEvaluator") - .slice(0, MAX_COPY_CARDS); - const attachEvaluators = evaluators - .filter((evaluator) => evaluator.__typename === "CodeEvaluator") - .slice(0, MAX_ATTACH_CARDS); + const templates: TemplateWithMetadata[] = + data.classificationEvaluatorConfigs.map((config) => ({ + config, + metadata: getProjectEvaluatorTemplateMetadata(config.name), + })); + const useCases = Array.from( + new Set(templates.map(({ metadata }) => metadata.useCase)) + ); + const categoryItems = [ + { + name: RECOMMENDED_CATEGORY, + count: templates.filter(({ metadata }) => metadata.recommended).length, + }, + ...useCases.map((useCase) => ({ + name: useCase, + count: templates.filter(({ metadata }) => metadata.useCase === useCase) + .length, + })), + ]; + const visibleTemplates = templates.filter(({ metadata }) => + selectedCategory === RECOMMENDED_CATEGORY + ? metadata.recommended + : metadata.useCase === selectedCategory + ); + const selectedTemplate = + visibleTemplates.find( + ({ config }) => config.name === selectedTemplateName + ) ?? visibleTemplates[0]; + return ( - -
- } />} - title="Create new LLM evaluator" - description="Author an LLM-as-a-judge evaluator from scratch." - onPress={() => navigate(paths.newLlm)} - /> - {copyEvaluators.map((evaluator) => ( - navigate(paths.copyLlm(evaluator.id))} - /> - ))} -
-
- } />} - title="Create new code evaluator" - description="Author a Python or TypeScript evaluator from scratch." - onPress={() => navigate(paths.newCode)} - /> - {attachEvaluators.map((evaluator) => ( - navigate(paths.attachCode(evaluator.id))} +
+ + +
+ + {selectedCategory} + +
    + {visibleTemplates.map(({ config, metadata }) => { + const isSelected = config.name === selectedTemplate?.config.name; + return ( +
  • + +
    • + ); + })} +
+
+ +
- + ) : ( + + No templates are available in this category. + + )} + +
); } -function EvaluatorCard({ - icon, - title, - description, - onPress, +function EvaluatorTemplateDetails({ + template, + onUseTemplate, }: { - icon?: ReactNode; - title: string; - description: ReactNode; - onPress: () => void; + template: TemplateWithMetadata; + onUseTemplate: () => void; }) { + const { config, metadata } = template; + const choices = getProjectEvaluatorTemplateChoices(config); return ( - +
+
+
Scope
+
{capitalize(metadata.scope)}
+
+
+
Optimization
+
{capitalize(config.optimizationDirection.toLowerCase())}
+
+
+ + + Output choices + +
    + {choices.map(({ label, score }) => ( +
  • + {label} + + {score} + +
    • + ))} +
+ + + ); } +function capitalize(value: string): string { + return `${value.charAt(0).toUpperCase()}${value.slice(1)}`; +} + function EvaluatorGallerySkeleton() { - const column = ( -
- - -
- ); return ( - + ); } function EvaluatorGalleryError() { return ( - Existing evaluators could not be loaded. + Evaluator templates could not be loaded. ); } -const evaluatorItemButtonCSS = css` - display: flex; - flex-direction: column; - gap: var(--global-dimension-size-50); - height: ${EVALUATOR_CARD_HEIGHT}px; - padding: var(--global-dimension-size-200); - border-radius: var(--global-rounding-small); - border: 1px solid var(--global-border-color-default); - background-color: transparent; - cursor: pointer; - text-align: left; - transition: background-color 0.2s ease; - &:hover { - background-color: var(--global-color-gray-200); +const galleryCSS = css` + box-sizing: border-box; + display: grid; + grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr) minmax( + 260px, + 1fr + ); + height: 100%; + min-height: ${GALLERY_SKELETON_HEIGHT}px; + overflow: hidden; + background-color: var(--global-background-color-default); + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates, + .project-evaluator-gallery__details { + padding: var(--global-dimension-size-200); + } + + .project-evaluator-gallery__categories { + display: flex; + flex-direction: column; + } + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates { + border-right: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__category-list, + .project-evaluator-gallery__template-list, + .project-evaluator-gallery__choice-list { + list-style: none; + margin: 0; + padding: 0; + } + + .project-evaluator-gallery__category-list { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-50); + margin-top: var(--global-dimension-size-100); + } + + .project-evaluator-gallery__scratch-actions { + margin-top: auto; + padding-top: var(--global-dimension-size-200); + border-top: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__category-button, + .project-evaluator-gallery__template-card { + width: 100%; + border: var(--global-border-size-thin) solid transparent; + border-radius: var(--global-rounding-small); + background-color: transparent; + color: inherit; + cursor: pointer; + text-align: left; + + &:hover { + background-color: var(--global-list-item-hover-background-color); + } + + &:focus-visible { + outline: var(--focus-ring-thickness) solid var(--focus-ring-color); + outline-offset: var(--focus-ring-offset); + } + + &[aria-pressed="true"] { + border-color: var(--global-border-color-default); + background-color: var(--global-list-item-selected-background-color); + } + } + + .project-evaluator-gallery__category-button { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--global-dimension-size-100); + padding: var(--global-dimension-size-75) var(--global-dimension-size-100); + } + + .project-evaluator-gallery__template-list { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); + gap: var(--global-dimension-size-100); + margin-top: var(--global-dimension-size-100); + } + + .project-evaluator-gallery__template-card { + display: flex; + min-height: var(--global-dimension-size-1400); + flex-direction: column; + gap: var(--global-dimension-size-100); + padding: var(--global-dimension-size-150); + border-color: var(--global-border-color-default); + } + + .project-evaluator-gallery__definition-list { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--global-dimension-size-100); + margin: 0; + + div { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-25); + } + + dt { + color: var(--global-text-color-500); + font-size: var(--global-font-size-xs); + } + + dd { + margin: 0; + font-size: var(--global-font-size-s); + } + } + + .project-evaluator-gallery__choice-list { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-50); + + li { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--global-dimension-size-100); + padding-bottom: var(--global-dimension-size-50); + border-bottom: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + } + + @media (max-width: 900px) { + grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr); + + .project-evaluator-gallery__templates { + border-right: 0; + } + + .project-evaluator-gallery__details { + grid-column: 1 / -1; + border-top: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + } + + @media (max-width: 600px) { + grid-template-columns: 1fr; + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates { + border-right: 0; + border-bottom: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__details { + grid-column: auto; + border-top: 0; + } } `; -const evaluatorColumnCSS = css` - display: flex; - flex-direction: column; - gap: var(--global-dimension-size-125); - flex: 1; +const galleryOuterCSS = css` + box-sizing: border-box; + width: 100%; + height: 100%; + min-height: ${GALLERY_SKELETON_HEIGHT}px; +`; + +const sectionHeadingCSS = css` + font-size: var(--global-font-size-s); + line-height: var(--global-line-height-s); + font-weight: 600; `; diff --git a/js/app/src/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql.ts b/js/app/src/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql.ts new file mode 100644 index 00000000000..f6a20f09bc6 --- /dev/null +++ b/js/app/src/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql.ts @@ -0,0 +1,216 @@ +/** + * @generated SignedSource<> + * @lightSyntaxTransform + */ + +/* tslint:disable */ +/* eslint-disable */ +// @ts-nocheck + +import { ConcreteRequest } from 'relay-runtime'; +import { FragmentRefs } from "relay-runtime"; +export type OptimizationDirection = "MAXIMIZE" | "MINIMIZE" | "NONE"; +export type projectEvaluatorTemplatesQuery$variables = Record; +export type projectEvaluatorTemplatesQuery$data = { + readonly classificationEvaluatorConfigs: ReadonlyArray<{ + readonly choices: any; + readonly description: string | null; + readonly messages: ReadonlyArray<{ + readonly " $fragmentSpreads": FragmentRefs<"promptUtils_promptMessages">; + }>; + readonly name: string; + readonly optimizationDirection: OptimizationDirection; + }>; +}; +export type projectEvaluatorTemplatesQuery = { + response: projectEvaluatorTemplatesQuery$data; + variables: projectEvaluatorTemplatesQuery$variables; +}; + +const node: ConcreteRequest = (function(){ +var v0 = { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "name", + "storageKey": null +}, +v1 = { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "description", + "storageKey": null +}, +v2 = { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "choices", + "storageKey": null +}, +v3 = { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "optimizationDirection", + "storageKey": null +}, +v4 = { + "kind": "InlineFragment", + "selections": [ + { + "alias": null, + "args": null, + "concreteType": "TextContentValue", + "kind": "LinkedField", + "name": "text", + "plural": false, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "text", + "storageKey": null + } + ], + "storageKey": null + } + ], + "type": "TextContentPart", + "abstractKey": null +}, +v5 = { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "role", + "storageKey": null +}; +return { + "fragment": { + "argumentDefinitions": [], + "kind": "Fragment", + "metadata": null, + "name": "projectEvaluatorTemplatesQuery", + "selections": [ + { + "alias": null, + "args": null, + "concreteType": "ClassificationEvaluatorConfig", + "kind": "LinkedField", + "name": "classificationEvaluatorConfigs", + "plural": true, + "selections": [ + (v0/*:: as any*/), + (v1/*:: as any*/), + (v2/*:: as any*/), + (v3/*:: as any*/), + { + "alias": null, + "args": null, + "concreteType": "PromptMessage", + "kind": "LinkedField", + "name": "messages", + "plural": true, + "selections": [ + { + "kind": "InlineDataFragmentSpread", + "name": "promptUtils_promptMessages", + "selections": [ + { + "alias": null, + "args": null, + "concreteType": null, + "kind": "LinkedField", + "name": "content", + "plural": true, + "selections": [ + (v4/*:: as any*/) + ], + "storageKey": null + }, + (v5/*:: as any*/) + ], + "args": null, + "argumentDefinitions": [] + } + ], + "storageKey": null + } + ], + "storageKey": null + } + ], + "type": "Query", + "abstractKey": null + }, + "kind": "Request", + "operation": { + "argumentDefinitions": [], + "kind": "Operation", + "name": "projectEvaluatorTemplatesQuery", + "selections": [ + { + "alias": null, + "args": null, + "concreteType": "ClassificationEvaluatorConfig", + "kind": "LinkedField", + "name": "classificationEvaluatorConfigs", + "plural": true, + "selections": [ + (v0/*:: as any*/), + (v1/*:: as any*/), + (v2/*:: as any*/), + (v3/*:: as any*/), + { + "alias": null, + "args": null, + "concreteType": "PromptMessage", + "kind": "LinkedField", + "name": "messages", + "plural": true, + "selections": [ + { + "alias": null, + "args": null, + "concreteType": null, + "kind": "LinkedField", + "name": "content", + "plural": true, + "selections": [ + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "__typename", + "storageKey": null + }, + (v4/*:: as any*/) + ], + "storageKey": null + }, + (v5/*:: as any*/) + ], + "storageKey": null + } + ], + "storageKey": null + } + ] + }, + "params": { + "cacheID": "c6c80e4cdc5351aba88d5ba1f49b456f", + "id": null, + "metadata": {}, + "name": "projectEvaluatorTemplatesQuery", + "operationKind": "query", + "text": "query projectEvaluatorTemplatesQuery {\n classificationEvaluatorConfigs {\n name\n description\n choices\n optimizationDirection\n messages {\n ...promptUtils_promptMessages\n }\n }\n}\n\nfragment promptUtils_promptMessages on PromptMessage {\n content {\n __typename\n ... on TextContentPart {\n text {\n text\n }\n }\n }\n role\n}\n" + } +}; +})(); + +(node as any).hash = "b938efd265919dc8c2d52b8aa9d45a15"; + +export default node; diff --git a/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts b/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts new file mode 100644 index 00000000000..cc6e91ecf8b --- /dev/null +++ b/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts @@ -0,0 +1,60 @@ +import type { ProjectEvaluatorTemplate } from "../projectEvaluatorTemplates"; +import { + buildTemplateCreationMode, + getProjectEvaluatorTemplateChoices, + getProjectEvaluatorTemplateMetadata, +} from "../projectEvaluatorTemplates"; + +const template = { + name: "hallucination", + description: "Detect unsupported claims.", + choices: { grounded: 0, hallucinated: 1 }, + optimizationDirection: "MINIMIZE", + messages: [], +} satisfies ProjectEvaluatorTemplate; + +describe("project evaluator templates", () => { + it("maps known templates to their gallery metadata", () => { + expect(getProjectEvaluatorTemplateMetadata("hallucination")).toEqual({ + useCase: "Answer quality", + scope: "span", + recommended: true, + kind: "LLM", + }); + }); + + it("uses safe fallback metadata for new backend templates", () => { + expect(getProjectEvaluatorTemplateMetadata("new_template")).toEqual({ + useCase: "Other", + scope: "span", + recommended: false, + kind: "LLM", + }); + }); + + it("validates choices before building the seeded creation mode", () => { + expect(buildTemplateCreationMode(template)).toEqual({ + kind: "template", + initialState: { + name: "hallucination", + description: "Detect unsupported claims.", + outputConfigs: [ + { + name: "hallucination", + optimizationDirection: "MINIMIZE", + values: [ + { label: "grounded", score: 0 }, + { label: "hallucinated", score: 1 }, + ], + }, + ], + defaultMessages: [], + templateFormat: "MUSTACHE", + includeExplanation: false, + }, + }); + expect( + getProjectEvaluatorTemplateChoices({ choices: { invalid: "score" } }) + ).toEqual([]); + }); +}); diff --git a/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts b/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts index 592c8ccea33..e81386788d3 100644 --- a/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts +++ b/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts @@ -38,6 +38,10 @@ export function useProjectEvaluatorPaths() { return { list: withCurrentSearch(list), newLlm: withCurrentSearch(newLlmProjectEvaluatorPath(rootPath)), + newLlmFromTemplate: (templateName: string) => + withCurrentSearch( + `${list}/new/template/${encodeURIComponent(templateName)}` + ), newCode: withCurrentSearch(newCodeProjectEvaluatorPath(rootPath)), copyLlm: (evaluatorId: string) => withCurrentSearch( diff --git a/js/app/src/pages/project/evaluators/projectEvaluatorTemplates.ts b/js/app/src/pages/project/evaluators/projectEvaluatorTemplates.ts new file mode 100644 index 00000000000..e5a6f2f0716 --- /dev/null +++ b/js/app/src/pages/project/evaluators/projectEvaluatorTemplates.ts @@ -0,0 +1,155 @@ +import { graphql } from "react-relay"; +import z from "zod"; + +import { inferIncludeExplanationFromPrompt } from "@phoenix/components/evaluators/utils"; +import type { projectEvaluatorTemplatesQuery$data } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql"; +import type { ProjectEvaluatorCreationMode } from "@phoenix/pages/project/evaluators/CreateProjectEvaluatorSlideover"; +import { convertPromptVersionMessagesToPlaygroundInstanceMessages } from "@phoenix/utils/promptUtils"; + +export const projectEvaluatorTemplatesQuery = graphql` + query projectEvaluatorTemplatesQuery { + classificationEvaluatorConfigs { + name + description + choices + optimizationDirection + messages { + ...promptUtils_promptMessages + } + } + } +`; + +export type ProjectEvaluatorTemplate = + projectEvaluatorTemplatesQuery$data["classificationEvaluatorConfigs"][number]; + +export type EvaluatorTemplateMetadata = { + useCase: string; + scope: "span" | "trace" | "session"; + recommended: boolean; + kind: "LLM" | "CODE"; + longDescription?: string; +}; + +const DEFAULT_TEMPLATE_METADATA = { + useCase: "Other", + scope: "span", + recommended: false, + kind: "LLM", +} as const satisfies EvaluatorTemplateMetadata; + +const PROJECT_EVALUATOR_TEMPLATE_METADATA: Partial< + Record +> = { + hallucination: { + useCase: "Answer quality", + scope: "span", + recommended: true, + kind: "LLM", + }, + correctness: { + useCase: "Answer quality", + scope: "span", + recommended: true, + kind: "LLM", + }, + conciseness: { + useCase: "Answer quality", + scope: "span", + recommended: false, + kind: "LLM", + }, + document_relevance: { + useCase: "Retrieval & context", + scope: "span", + recommended: false, + kind: "LLM", + }, + faithfulness: { + useCase: "Retrieval & context", + scope: "span", + recommended: false, + kind: "LLM", + }, + tool_invocation: { + useCase: "Agents & tool use", + scope: "span", + recommended: false, + kind: "LLM", + }, + tool_selection: { + useCase: "Agents & tool use", + scope: "span", + recommended: true, + kind: "LLM", + }, + tool_response_handling: { + useCase: "Agents & tool use", + scope: "span", + recommended: false, + kind: "LLM", + }, + refusal: { + useCase: "Safety", + scope: "span", + recommended: false, + kind: "LLM", + }, + toxicity: { + useCase: "Safety", + scope: "span", + recommended: true, + kind: "LLM", + }, + user_friction: { + useCase: "User experience", + scope: "span", + recommended: false, + kind: "LLM", + }, +}; + +export function getProjectEvaluatorTemplateMetadata( + templateName: string +): EvaluatorTemplateMetadata { + return ( + PROJECT_EVALUATOR_TEMPLATE_METADATA[templateName] ?? + DEFAULT_TEMPLATE_METADATA + ); +} + +export function getProjectEvaluatorTemplateChoices(config: { + choices: unknown; +}): { label: string; score: number }[] { + const parsedChoices = z + .record(z.string(), z.number()) + .safeParse(config.choices); + const choices = parsedChoices.success ? parsedChoices.data : {}; + return Object.entries(choices).map(([label, score]) => ({ label, score })); +} + +export function buildTemplateCreationMode( + config: ProjectEvaluatorTemplate +): ProjectEvaluatorCreationMode { + return { + kind: "template", + initialState: { + name: config.name, + description: config.description ?? "", + outputConfigs: [ + { + name: config.name, + optimizationDirection: config.optimizationDirection, + values: getProjectEvaluatorTemplateChoices(config), + }, + ], + defaultMessages: convertPromptVersionMessagesToPlaygroundInstanceMessages( + { + promptMessagesRefs: config.messages, + } + ), + templateFormat: "MUSTACHE", + includeExplanation: inferIncludeExplanationFromPrompt(undefined), + }, + }; +} From 7daf9bcda270a9bd1af82b40c88c8f1e1122655f Mon Sep 17 00:00:00 2001 From: Yasmine Frigui Date: Wed, 19 Aug 2026 15:08:30 -0400 Subject: [PATCH 2/6] feat: move project evaluator gallery to full-page route --- js/app/src/Routes.tsx | 50 ++ .../evaluators/AddProjectEvaluatorMenu.tsx | 12 +- .../ProjectEvaluatorGalleryPage.tsx | 467 +++++++++++++++ .../ProjectEvaluatorSlideoverRoutes.tsx | 43 +- .../ProjectEvaluatorsEmptyState.tsx | 551 +++++------------- .../evaluators/projectEvaluatorPaths.ts | 19 +- js/app/src/pages/project/index.tsx | 1 + 7 files changed, 726 insertions(+), 417 deletions(-) create mode 100644 js/app/src/pages/project/evaluators/ProjectEvaluatorGalleryPage.tsx diff --git a/js/app/src/Routes.tsx b/js/app/src/Routes.tsx index 59cbd47da16..8b43975435c 100644 --- a/js/app/src/Routes.tsx +++ b/js/app/src/Routes.tsx @@ -85,6 +85,9 @@ import { LoggedOutPage, LoginPage, NewCodeProjectEvaluatorPage, + NewGalleryCodeProjectEvaluatorPage, + NewGalleryLlmFromTemplateProjectEvaluatorPage, + NewGalleryLlmProjectEvaluatorPage, NewLlmFromTemplateProjectEvaluatorPage, NewLlmProjectEvaluatorPage, OAuth2ConsentPage, @@ -97,6 +100,7 @@ import { ProfilePage, ProfilePreferencesPage, ProjectEvaluatorsPage, + ProjectEvaluatorGalleryPage, projectEvaluatorsLoader, ProjectIndexPage, projectLoader, @@ -532,6 +536,52 @@ export const appRouteObjects = createRoutesFromElements( /> + } + handle={{ + crumb: () => "Evaluator gallery", + agentRoute: { + label: "Project Evaluator Gallery", + description: + "Browse evaluator templates and start a project evaluator from a template or from scratch.", + }, + }} + > + } + handle={{ + agentRoute: { + label: "New Project LLM Evaluator From Gallery", + description: + "Author a new LLM-as-a-judge evaluator from scratch while browsing the evaluator gallery.", + }, + }} + /> + } + handle={{ + agentRoute: { + label: "New Project Code Evaluator From Gallery", + description: + "Author a new Python or TypeScript code evaluator from scratch while browsing the evaluator gallery.", + }, + }} + /> + } + handle={{ + agentRoute: { + label: "New Project Evaluator From Gallery Template", + description: + "Create a project LLM evaluator seeded from the selected evaluator gallery template.", + }, + }} + /> + {/* The evaluator details page is a full page rather than a tab, mirroring the dataset evaluator details route. The edit slideover nests beneath it so it opens over the details view. */} diff --git a/js/app/src/pages/project/evaluators/AddProjectEvaluatorMenu.tsx b/js/app/src/pages/project/evaluators/AddProjectEvaluatorMenu.tsx index 52d48e376e3..f7a99389d1c 100644 --- a/js/app/src/pages/project/evaluators/AddProjectEvaluatorMenu.tsx +++ b/js/app/src/pages/project/evaluators/AddProjectEvaluatorMenu.tsx @@ -72,13 +72,23 @@ function AddProjectEvaluatorMenuItems() { { - if (action === "createEvaluator") { + if (action === "browseGallery") { + navigate(paths.gallery); + } else if (action === "createEvaluator") { navigate(paths.newLlm); } else if (action === "createCodeEvaluator") { navigate(paths.newCode); } }} > + + } />} + id="browseGallery" + > + Browse evaluator gallery + + ; +}; + +export function ProjectEvaluatorGalleryPage() { + return ( +
+ + }> + + + + + + +
+ ); +} + +function EvaluatorGallery() { + const navigate = useNavigate(); + const paths = useProjectEvaluatorPaths(); + const [selectedCategory, setSelectedCategory] = + useState(RECOMMENDED_CATEGORY); + const [selectedTemplateName, setSelectedTemplateName] = useState< + string | null + >(null); + const data = useLazyLoadQuery( + projectEvaluatorTemplatesQuery, + {}, + { fetchPolicy: "store-and-network" } + ); + const templates: TemplateWithMetadata[] = + data.classificationEvaluatorConfigs.map((config) => ({ + config, + metadata: getProjectEvaluatorTemplateMetadata(config.name), + })); + const useCases = Array.from( + new Set(templates.map(({ metadata }) => metadata.useCase)) + ); + const categoryItems = [ + { + name: RECOMMENDED_CATEGORY, + count: templates.filter(({ metadata }) => metadata.recommended).length, + }, + ...useCases.map((useCase) => ({ + name: useCase, + count: templates.filter(({ metadata }) => metadata.useCase === useCase) + .length, + })), + ]; + const visibleTemplates = templates.filter(({ metadata }) => + selectedCategory === RECOMMENDED_CATEGORY + ? metadata.recommended + : metadata.useCase === selectedCategory + ); + const selectedTemplate = + visibleTemplates.find( + ({ config }) => config.name === selectedTemplateName + ) ?? visibleTemplates[0]; + + return ( +
+ + +
+ + {selectedCategory} + +
    + {visibleTemplates.map(({ config, metadata }) => { + const isSelected = config.name === selectedTemplate?.config.name; + return ( +
  • + +
    • + ); + })} +
+
+ + +
+ ); +} + +function EvaluatorTemplateDetails({ + template, + onUseTemplate, +}: { + template: TemplateWithMetadata; + onUseTemplate: () => void; +}) { + const { config, metadata } = template; + const choices = getProjectEvaluatorTemplateChoices(config); + return ( + + + {config.name} + + {metadata.kind} + {metadata.useCase} + + + {config.description} + + {metadata.longDescription ? ( + + {metadata.longDescription} + + ) : null} + +
+
+
Scope
+
{capitalize(metadata.scope)}
+
+
+
Optimization
+
{capitalize(config.optimizationDirection.toLowerCase())}
+
+
+ + + Output choices + +
    + {choices.map(({ label, score }) => ( +
  • + {label} + + {score} + +
    • + ))} +
+ + + + ); +} + +function capitalize(value: string): string { + return `${value.charAt(0).toUpperCase()}${value.slice(1)}`; +} + +function EvaluatorGallerySkeleton() { + return ( + + ); +} + +function EvaluatorGalleryError() { + return ( + + Evaluator templates could not be loaded. + + ); +} + +const galleryCSS = css` + box-sizing: border-box; + display: grid; + grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr) minmax( + 260px, + 1fr + ); + height: 100%; + min-height: ${GALLERY_SKELETON_HEIGHT}px; + overflow: hidden; + background-color: var(--global-background-color-default); + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates, + .project-evaluator-gallery__details { + padding: var(--global-dimension-size-200); + } + + .project-evaluator-gallery__categories { + display: flex; + flex-direction: column; + } + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates { + border-right: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__category-list, + .project-evaluator-gallery__template-list, + .project-evaluator-gallery__choice-list { + list-style: none; + margin: 0; + padding: 0; + } + + .project-evaluator-gallery__category-list { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-50); + margin-top: var(--global-dimension-size-100); + } + + .project-evaluator-gallery__scratch-actions { + margin-top: auto; + padding-top: var(--global-dimension-size-200); + border-top: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__category-button, + .project-evaluator-gallery__template-card { + width: 100%; + border: var(--global-border-size-thin) solid transparent; + border-radius: var(--global-rounding-small); + background-color: transparent; + color: inherit; + cursor: pointer; + text-align: left; + + &:hover { + background-color: var(--global-list-item-hover-background-color); + } + + &:focus-visible { + outline: var(--focus-ring-thickness) solid var(--focus-ring-color); + outline-offset: var(--focus-ring-offset); + } + + &[aria-pressed="true"] { + border-color: var(--global-border-color-default); + background-color: var(--global-list-item-selected-background-color); + } + } + + .project-evaluator-gallery__category-button { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--global-dimension-size-100); + padding: var(--global-dimension-size-75) var(--global-dimension-size-100); + } + + .project-evaluator-gallery__template-list { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); + gap: var(--global-dimension-size-100); + margin-top: var(--global-dimension-size-100); + } + + .project-evaluator-gallery__template-card { + display: flex; + min-height: var(--global-dimension-size-1400); + flex-direction: column; + gap: var(--global-dimension-size-100); + padding: var(--global-dimension-size-150); + border-color: var(--global-border-color-default); + } + + .project-evaluator-gallery__definition-list { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--global-dimension-size-100); + margin: 0; + + div { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-25); + } + + dt { + color: var(--global-text-color-500); + font-size: var(--global-font-size-xs); + } + + dd { + margin: 0; + font-size: var(--global-font-size-s); + } + } + + .project-evaluator-gallery__choice-list { + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-50); + + li { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--global-dimension-size-100); + padding-bottom: var(--global-dimension-size-50); + border-bottom: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + } + + @media (max-width: 900px) { + overflow-x: hidden; + overflow-y: auto; + grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr); + + .project-evaluator-gallery__templates { + border-right: 0; + } + + .project-evaluator-gallery__details { + grid-column: 1 / -1; + border-top: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + } + + @media (max-width: 600px) { + grid-template-columns: 1fr; + + .project-evaluator-gallery__categories, + .project-evaluator-gallery__templates { + border-right: 0; + border-bottom: var(--global-border-size-thin) solid + var(--global-border-color-default); + } + + .project-evaluator-gallery__details { + grid-column: auto; + border-top: 0; + } + } +`; + +const galleryOuterCSS = css` + box-sizing: border-box; + flex: 1 1 auto; + width: 100%; + height: 100%; + min-height: ${GALLERY_SKELETON_HEIGHT}px; + overflow: hidden; +`; + +const sectionHeadingCSS = css` + font-size: var(--global-font-size-s); + line-height: var(--global-line-height-s); + font-weight: 600; +`; diff --git a/js/app/src/pages/project/evaluators/ProjectEvaluatorSlideoverRoutes.tsx b/js/app/src/pages/project/evaluators/ProjectEvaluatorSlideoverRoutes.tsx index 9273ecd7286..f1b03f61d7e 100644 --- a/js/app/src/pages/project/evaluators/ProjectEvaluatorSlideoverRoutes.tsx +++ b/js/app/src/pages/project/evaluators/ProjectEvaluatorSlideoverRoutes.tsx @@ -77,6 +77,20 @@ export function NewLlmProjectEvaluatorPage() { ); } +export function NewGalleryLlmProjectEvaluatorPage() { + const projectId = useRouteProjectId(); + const { gallery } = useProjectEvaluatorPaths(); + const onOpenChange = useCloseSlideover(gallery); + return ( + + ); +} + export function NewCodeProjectEvaluatorPage() { const projectId = useRouteProjectId(); const onOpenChange = useCloseSlideover(); @@ -90,11 +104,38 @@ export function NewCodeProjectEvaluatorPage() { ); } +export function NewGalleryCodeProjectEvaluatorPage() { + const projectId = useRouteProjectId(); + const { gallery } = useProjectEvaluatorPaths(); + const onOpenChange = useCloseSlideover(gallery); + return ( + + ); +} + export function NewLlmFromTemplateProjectEvaluatorPage() { + return ; +} + +export function NewGalleryLlmFromTemplateProjectEvaluatorPage() { + const { gallery } = useProjectEvaluatorPaths(); + return ; +} + +function NewLlmFromTemplateProjectEvaluatorSlideover({ + closeTo, +}: { + closeTo?: string; +}) { const projectId = useRouteProjectId(); const { templateName } = useParams(); invariant(templateName, "templateName is required"); - const onOpenChange = useCloseSlideover(); + const onOpenChange = useCloseSlideover(closeTo); const data = useLazyLoadQuery( projectEvaluatorTemplatesQuery, {}, diff --git a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx index e027fc61970..ecacc4238fa 100644 --- a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx +++ b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx @@ -1,458 +1,187 @@ import { css } from "@emotion/react"; -import { Suspense, useState } from "react"; +import type { ReactNode } from "react"; +import { Suspense } from "react"; import { useLazyLoadQuery } from "react-relay"; import { useNavigate } from "react-router"; +import { Flex, Icon, Icons, Skeleton, Text } from "@phoenix/components"; import { - Badge, - Button, - Counter, - Flex, - Heading, - Skeleton, - Text, -} from "@phoenix/components"; + EmptyState, + EmptyStateArea, + EmptyStateGraphic, +} from "@phoenix/components/core/empty"; import { LineClamp } from "@phoenix/components/core/utility/LineClamp"; import { ErrorBoundary } from "@phoenix/components/exception"; -import type { projectEvaluatorTemplatesQuery as ProjectEvaluatorTemplatesQueryType } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql"; +import type { projectEvaluatorOptionsQuery } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorOptionsQuery.graphql"; +import { projectEvaluatorOptionsQuery as projectEvaluatorOptionsQueryNode } from "@phoenix/pages/project/evaluators/projectEvaluatorOptions"; import { useProjectEvaluatorPaths } from "@phoenix/pages/project/evaluators/projectEvaluatorPaths"; -import { - getProjectEvaluatorTemplateChoices, - getProjectEvaluatorTemplateMetadata, - type ProjectEvaluatorTemplate, - projectEvaluatorTemplatesQuery, -} from "@phoenix/pages/project/evaluators/projectEvaluatorTemplates"; - -const RECOMMENDED_CATEGORY = "Recommended"; -const GALLERY_SKELETON_HEIGHT = 440; -type TemplateWithMetadata = { - config: ProjectEvaluatorTemplate; - metadata: ReturnType; -}; +const MAX_COPY_CARDS = 4; +const MAX_ATTACH_CARDS = 3; +const EVALUATOR_CARD_HEIGHT = 90; export function ProjectEvaluatorsEmptyState() { + const navigate = useNavigate(); + const paths = useProjectEvaluatorPaths(); return ( -
- - }> - - - -
+ + + } + title="No evaluators for this project" + description="Add an evaluator to score spans, traces, or sessions automatically as they arrive." + action={{ + type: "strip", + items: [ + { + kind: "button", + variant: "primary", + children: "Browse evaluator gallery", + onPress: () => navigate(paths.gallery), + }, + ], + }} + /> + + }> + + + + + ); } -function EvaluatorGallery() { +function EvaluatorOptions() { const navigate = useNavigate(); const paths = useProjectEvaluatorPaths(); - const [selectedCategory, setSelectedCategory] = - useState(RECOMMENDED_CATEGORY); - const [selectedTemplateName, setSelectedTemplateName] = useState< - string | null - >(null); - const data = useLazyLoadQuery( - projectEvaluatorTemplatesQuery, + const data = useLazyLoadQuery( + projectEvaluatorOptionsQueryNode, {}, { fetchPolicy: "store-and-network" } ); - const templates: TemplateWithMetadata[] = - data.classificationEvaluatorConfigs.map((config) => ({ - config, - metadata: getProjectEvaluatorTemplateMetadata(config.name), - })); - const useCases = Array.from( - new Set(templates.map(({ metadata }) => metadata.useCase)) - ); - const categoryItems = [ - { - name: RECOMMENDED_CATEGORY, - count: templates.filter(({ metadata }) => metadata.recommended).length, - }, - ...useCases.map((useCase) => ({ - name: useCase, - count: templates.filter(({ metadata }) => metadata.useCase === useCase) - .length, - })), - ]; - const visibleTemplates = templates.filter(({ metadata }) => - selectedCategory === RECOMMENDED_CATEGORY - ? metadata.recommended - : metadata.useCase === selectedCategory - ); - const selectedTemplate = - visibleTemplates.find( - ({ config }) => config.name === selectedTemplateName - ) ?? visibleTemplates[0]; - + const evaluators = data.evaluators.edges.map(({ evaluator }) => evaluator); + const copyEvaluators = evaluators + .filter((evaluator) => evaluator.__typename === "LLMEvaluator") + .slice(0, MAX_COPY_CARDS); + const attachEvaluators = evaluators + .filter((evaluator) => evaluator.__typename === "CodeEvaluator") + .slice(0, MAX_ATTACH_CARDS); return ( -
- - -
- - {selectedCategory} - -
    - {visibleTemplates.map(({ config, metadata }) => { - const isSelected = config.name === selectedTemplate?.config.name; - return ( -
  • - -
    • - ); - })} -
-
- - -
+ ))} + +
+ } />} + title="Create new code evaluator" + description="Author a Python or TypeScript evaluator from scratch." + onPress={() => navigate(paths.newCode)} + /> + {attachEvaluators.map((evaluator) => ( + navigate(paths.attachCode(evaluator.id))} + /> + ))} +
+ ); } -function EvaluatorTemplateDetails({ - template, - onUseTemplate, +function EvaluatorCard({ + icon, + title, + description, + onPress, }: { - template: TemplateWithMetadata; - onUseTemplate: () => void; + icon?: ReactNode; + title: string; + description: ReactNode; + onPress: () => void; }) { - const { config, metadata } = template; - const choices = getProjectEvaluatorTemplateChoices(config); return ( - - - {config.name} - - {metadata.kind} - {metadata.useCase} - - - {config.description} + - + + + {description} + + + ); } -function capitalize(value: string): string { - return `${value.charAt(0).toUpperCase()}${value.slice(1)}`; -} - -function EvaluatorGallerySkeleton() { +function EvaluatorOptionsSkeleton() { + const column = ( +
+ + +
+ ); return ( - + ); } -function EvaluatorGalleryError() { +function EvaluatorOptionsError() { return ( - Evaluator templates could not be loaded. + Existing evaluators could not be loaded. ); } -const galleryCSS = css` - box-sizing: border-box; - display: grid; - grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr) minmax( - 260px, - 1fr - ); - height: 100%; - min-height: ${GALLERY_SKELETON_HEIGHT}px; - overflow: hidden; - background-color: var(--global-background-color-default); - - .project-evaluator-gallery__categories, - .project-evaluator-gallery__templates, - .project-evaluator-gallery__details { - padding: var(--global-dimension-size-200); - } - - .project-evaluator-gallery__categories { - display: flex; - flex-direction: column; - } - - .project-evaluator-gallery__categories, - .project-evaluator-gallery__templates { - border-right: var(--global-border-size-thin) solid - var(--global-border-color-default); - } - - .project-evaluator-gallery__category-list, - .project-evaluator-gallery__template-list, - .project-evaluator-gallery__choice-list { - list-style: none; - margin: 0; - padding: 0; +const evaluatorItemButtonCSS = css` + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-50); + height: ${EVALUATOR_CARD_HEIGHT}px; + padding: var(--global-dimension-size-200); + border-radius: var(--global-rounding-small); + border: 1px solid var(--global-border-color-default); + background-color: transparent; + cursor: pointer; + text-align: left; + transition: background-color 0.2s ease; + &:hover { + background-color: var(--global-color-gray-200); } - - .project-evaluator-gallery__category-list { - display: flex; - flex-direction: column; - gap: var(--global-dimension-size-50); - margin-top: var(--global-dimension-size-100); - } - - .project-evaluator-gallery__scratch-actions { - margin-top: auto; - padding-top: var(--global-dimension-size-200); - border-top: var(--global-border-size-thin) solid - var(--global-border-color-default); - } - - .project-evaluator-gallery__category-button, - .project-evaluator-gallery__template-card { - width: 100%; - border: var(--global-border-size-thin) solid transparent; - border-radius: var(--global-rounding-small); - background-color: transparent; - color: inherit; - cursor: pointer; - text-align: left; - - &:hover { - background-color: var(--global-list-item-hover-background-color); - } - - &:focus-visible { - outline: var(--focus-ring-thickness) solid var(--focus-ring-color); - outline-offset: var(--focus-ring-offset); - } - - &[aria-pressed="true"] { - border-color: var(--global-border-color-default); - background-color: var(--global-list-item-selected-background-color); - } - } - - .project-evaluator-gallery__category-button { - display: flex; - align-items: center; - justify-content: space-between; - gap: var(--global-dimension-size-100); - padding: var(--global-dimension-size-75) var(--global-dimension-size-100); - } - - .project-evaluator-gallery__template-list { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(190px, 1fr)); - gap: var(--global-dimension-size-100); - margin-top: var(--global-dimension-size-100); - } - - .project-evaluator-gallery__template-card { - display: flex; - min-height: var(--global-dimension-size-1400); - flex-direction: column; - gap: var(--global-dimension-size-100); - padding: var(--global-dimension-size-150); - border-color: var(--global-border-color-default); - } - - .project-evaluator-gallery__definition-list { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: var(--global-dimension-size-100); - margin: 0; - - div { - display: flex; - flex-direction: column; - gap: var(--global-dimension-size-25); - } - - dt { - color: var(--global-text-color-500); - font-size: var(--global-font-size-xs); - } - - dd { - margin: 0; - font-size: var(--global-font-size-s); - } - } - - .project-evaluator-gallery__choice-list { - display: flex; - flex-direction: column; - gap: var(--global-dimension-size-50); - - li { - display: flex; - align-items: center; - justify-content: space-between; - gap: var(--global-dimension-size-100); - padding-bottom: var(--global-dimension-size-50); - border-bottom: var(--global-border-size-thin) solid - var(--global-border-color-default); - } - } - - @media (max-width: 900px) { - grid-template-columns: minmax(160px, 0.65fr) minmax(320px, 1.5fr); - - .project-evaluator-gallery__templates { - border-right: 0; - } - - .project-evaluator-gallery__details { - grid-column: 1 / -1; - border-top: var(--global-border-size-thin) solid - var(--global-border-color-default); - } - } - - @media (max-width: 600px) { - grid-template-columns: 1fr; - - .project-evaluator-gallery__categories, - .project-evaluator-gallery__templates { - border-right: 0; - border-bottom: var(--global-border-size-thin) solid - var(--global-border-color-default); - } - - .project-evaluator-gallery__details { - grid-column: auto; - border-top: 0; - } - } -`; - -const galleryOuterCSS = css` - box-sizing: border-box; - width: 100%; - height: 100%; - min-height: ${GALLERY_SKELETON_HEIGHT}px; `; -const sectionHeadingCSS = css` - font-size: var(--global-font-size-s); - line-height: var(--global-line-height-s); - font-weight: 600; +const evaluatorColumnCSS = css` + display: flex; + flex-direction: column; + gap: var(--global-dimension-size-125); + flex: 1; `; diff --git a/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts b/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts index e81386788d3..1420a953b70 100644 --- a/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts +++ b/js/app/src/pages/project/evaluators/projectEvaluatorPaths.ts @@ -6,6 +6,9 @@ import { useProjectRootPath } from "@phoenix/hooks/useProjectRootPath"; const projectEvaluatorsPath = (projectRootPath: string) => `${projectRootPath}/evaluators`; +const projectEvaluatorGalleryPath = (projectRootPath: string) => + `${projectRootPath}/evaluator-gallery`; + /** * Exported for the loader that forwards the legacy `?createLlmEvaluator` and * `?createCodeEvaluator` links, which has a path rather than a project root. @@ -27,22 +30,30 @@ export const newCodeProjectEvaluatorPath = (projectRootPath: string) => */ export function useProjectEvaluatorPaths() { const { rootPath } = useProjectRootPath(); - // A slideover is a sub-view of the list, not a new destination, so opening - // and closing one carries the page's URL state -- above all a custom time - // range, which would otherwise be dropped on the way in and again on the way - // out. + // A slideover is a sub-view of its parent page, not a new destination, so + // opening and closing one carries the page's URL state -- above all a custom + // time range, which would otherwise be dropped on the way in and again on + // the way out. const { search } = useLocation(); return useMemo(() => { const list = projectEvaluatorsPath(rootPath); + const gallery = projectEvaluatorGalleryPath(rootPath); const withCurrentSearch = (path: string) => `${path}${search}`; return { list: withCurrentSearch(list), + gallery: withCurrentSearch(gallery), newLlm: withCurrentSearch(newLlmProjectEvaluatorPath(rootPath)), newLlmFromTemplate: (templateName: string) => withCurrentSearch( `${list}/new/template/${encodeURIComponent(templateName)}` ), newCode: withCurrentSearch(newCodeProjectEvaluatorPath(rootPath)), + galleryNewLlm: withCurrentSearch(`${gallery}/new/llm`), + galleryNewLlmFromTemplate: (templateName: string) => + withCurrentSearch( + `${gallery}/new/template/${encodeURIComponent(templateName)}` + ), + galleryNewCode: withCurrentSearch(`${gallery}/new/code`), copyLlm: (evaluatorId: string) => withCurrentSearch( `${list}/new/copy/${encodeURIComponent(evaluatorId)}` diff --git a/js/app/src/pages/project/index.tsx b/js/app/src/pages/project/index.tsx index 9e7da0860de..5e1a7f477bd 100644 --- a/js/app/src/pages/project/index.tsx +++ b/js/app/src/pages/project/index.tsx @@ -5,6 +5,7 @@ export * from "./ProjectSpansPage"; export * from "./ProjectTracesPage"; export * from "./projectLoader"; export * from "./evaluators/ProjectEvaluatorsPage"; +export * from "./evaluators/ProjectEvaluatorGalleryPage"; export * from "./evaluators/projectEvaluatorsLoader"; export * from "./evaluators/ProjectEvaluatorSlideoverRoutes"; export * from "./metrics/ProjectMetricsPage"; From 0ff5e796a84b549b77c603b33e81f1908ed70683 Mon Sep 17 00:00:00 2001 From: Yasmine Frigui Date: Wed, 19 Aug 2026 15:57:25 -0400 Subject: [PATCH 3/6] chore: refine project evaluator gallery --- .../ProjectEvaluatorsEmptyState.tsx | 2 +- .../projectEvaluatorTemplates.test.ts | 60 ------------------- 2 files changed, 1 insertion(+), 61 deletions(-) delete mode 100644 js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts diff --git a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx index ecacc4238fa..b4c61398fd7 100644 --- a/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx +++ b/js/app/src/pages/project/evaluators/ProjectEvaluatorsEmptyState.tsx @@ -42,7 +42,7 @@ export function ProjectEvaluatorsEmptyState() { { kind: "button", variant: "primary", - children: "Browse evaluator gallery", + children: "Browse the whole library", onPress: () => navigate(paths.gallery), }, ], diff --git a/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts b/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts deleted file mode 100644 index cc6e91ecf8b..00000000000 --- a/js/app/src/pages/project/evaluators/__tests__/projectEvaluatorTemplates.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { ProjectEvaluatorTemplate } from "../projectEvaluatorTemplates"; -import { - buildTemplateCreationMode, - getProjectEvaluatorTemplateChoices, - getProjectEvaluatorTemplateMetadata, -} from "../projectEvaluatorTemplates"; - -const template = { - name: "hallucination", - description: "Detect unsupported claims.", - choices: { grounded: 0, hallucinated: 1 }, - optimizationDirection: "MINIMIZE", - messages: [], -} satisfies ProjectEvaluatorTemplate; - -describe("project evaluator templates", () => { - it("maps known templates to their gallery metadata", () => { - expect(getProjectEvaluatorTemplateMetadata("hallucination")).toEqual({ - useCase: "Answer quality", - scope: "span", - recommended: true, - kind: "LLM", - }); - }); - - it("uses safe fallback metadata for new backend templates", () => { - expect(getProjectEvaluatorTemplateMetadata("new_template")).toEqual({ - useCase: "Other", - scope: "span", - recommended: false, - kind: "LLM", - }); - }); - - it("validates choices before building the seeded creation mode", () => { - expect(buildTemplateCreationMode(template)).toEqual({ - kind: "template", - initialState: { - name: "hallucination", - description: "Detect unsupported claims.", - outputConfigs: [ - { - name: "hallucination", - optimizationDirection: "MINIMIZE", - values: [ - { label: "grounded", score: 0 }, - { label: "hallucinated", score: 1 }, - ], - }, - ], - defaultMessages: [], - templateFormat: "MUSTACHE", - includeExplanation: false, - }, - }); - expect( - getProjectEvaluatorTemplateChoices({ choices: { invalid: "score" } }) - ).toEqual([]); - }); -}); From f7e3228ddf4384612abf685739f87e5a757ca8a4 Mon Sep 17 00:00:00 2001 From: Yasmine Frigui Date: Wed, 19 Aug 2026 16:15:35 -0400 Subject: [PATCH 4/6] feat(evaluators): align gallery template metadata --- .../ProjectEvaluatorGalleryPage.tsx | 51 ++- .../evaluators/projectEvaluatorTemplates.ts | 295 +++++++++++++++--- 2 files changed, 293 insertions(+), 53 deletions(-) diff --git a/js/app/src/pages/project/evaluators/ProjectEvaluatorGalleryPage.tsx b/js/app/src/pages/project/evaluators/ProjectEvaluatorGalleryPage.tsx index ed540cb80c8..ac1a5299e2b 100644 --- a/js/app/src/pages/project/evaluators/ProjectEvaluatorGalleryPage.tsx +++ b/js/app/src/pages/project/evaluators/ProjectEvaluatorGalleryPage.tsx @@ -17,6 +17,7 @@ import { ErrorBoundary } from "@phoenix/components/exception"; import type { projectEvaluatorTemplatesQuery as ProjectEvaluatorTemplatesQueryType } from "@phoenix/pages/project/evaluators/__generated__/projectEvaluatorTemplatesQuery.graphql"; import { useProjectEvaluatorPaths } from "@phoenix/pages/project/evaluators/projectEvaluatorPaths"; import { + getProjectEvaluatorTemplateCategoryLabel, getProjectEvaluatorTemplateChoices, getProjectEvaluatorTemplateMetadata, type ProjectEvaluatorTemplate, @@ -64,24 +65,40 @@ function EvaluatorGallery() { config, metadata: getProjectEvaluatorTemplateMetadata(config.name), })); - const useCases = Array.from( - new Set(templates.map(({ metadata }) => metadata.useCase)) + const categories = Array.from( + new Set( + templates.map(({ metadata }) => + getProjectEvaluatorTemplateCategoryLabel(metadata.category) + ) + ) ); + const recommendedTemplateCount = templates.filter( + ({ metadata }) => metadata.recommended + ).length; const categoryItems = [ { name: RECOMMENDED_CATEGORY, - count: templates.filter(({ metadata }) => metadata.recommended).length, + count: recommendedTemplateCount, }, - ...useCases.map((useCase) => ({ - name: useCase, - count: templates.filter(({ metadata }) => metadata.useCase === useCase) - .length, + ...categories.map((category) => ({ + name: category, + count: templates.filter( + ({ metadata }) => + getProjectEvaluatorTemplateCategoryLabel(metadata.category) === + category + ).length, })), ]; + const activeCategory = categoryItems.some( + ({ name }) => name === selectedCategory + ) + ? selectedCategory + : (categoryItems[0]?.name ?? RECOMMENDED_CATEGORY); const visibleTemplates = templates.filter(({ metadata }) => - selectedCategory === RECOMMENDED_CATEGORY + activeCategory === RECOMMENDED_CATEGORY ? metadata.recommended - : metadata.useCase === selectedCategory + : getProjectEvaluatorTemplateCategoryLabel(metadata.category) === + activeCategory ); const selectedTemplate = visibleTemplates.find( @@ -99,7 +116,7 @@ function EvaluatorGallery() {
    {categoryItems.map(({ name, count }) => { - const isSelected = name === selectedCategory; + const isSelected = name === activeCategory; return (