feat: add project evaluator template gallery

js/app/src/Routes.tsx

@@ -9,6 +9,7 @@ import { RouterProvider } from "react-router/dom";
 
 import { buildRouteInfoCatalog } from "@phoenix/agent/tools/getRouteInfo/catalog";
 import { registerRouteInfoCatalog } from "@phoenix/agent/tools/getRouteInfo/routeCatalogRegistry";
+import type { ParentCrumbToFn } from "@phoenix/hooks/useMatchesWithCrumb";
 import type { DatasetEvaluatorDetailsLoaderData } from "@phoenix/pages/dataset/evaluators/datasetEvaluatorDetailsLoader";
 import { datasetEvaluatorDetailsLoader } from "@phoenix/pages/dataset/evaluators/datasetEvaluatorDetailsLoader";
 import { DatasetEvaluatorDetailsPage } from "@phoenix/pages/dataset/evaluators/DatasetEvaluatorDetailsPage";
@@ -85,6 +86,10 @@ import {
   LoggedOutPage,
   LoginPage,
   NewCodeProjectEvaluatorPage,
+  NewGalleryCodeProjectEvaluatorPage,
+  NewGalleryLlmFromTemplateProjectEvaluatorPage,
+  NewGalleryLlmProjectEvaluatorPage,
+  NewLlmFromTemplateProjectEvaluatorPage,
   NewLlmProjectEvaluatorPage,
   OAuth2ConsentPage,
   PlaygroundPage,
@@ -96,6 +101,7 @@ import {
   ProfilePage,
   ProfilePreferencesPage,
   ProjectEvaluatorsPage,
+  ProjectEvaluatorGalleryPage,
   projectEvaluatorsLoader,
   ProjectIndexPage,
   projectLoader,
@@ -496,6 +502,17 @@ export const appRouteObjects = createRoutesFromElements(
                     },
                   }}
                 />
+                <Route
+                  path="new/template/:templateName"
+                  element={<NewLlmFromTemplateProjectEvaluatorPage />}
+                  handle={{
+                    agentRoute: {
+                      label: "New Project Evaluator From Template",
+                      description:
+                        "Create a project LLM evaluator seeded from a classification evaluator template. The templateName route param is the template's name.",
+                    },
+                  }}
+                />
                 <Route
                   path="new/copy/:evaluatorId"
                   element={<CopyLlmProjectEvaluatorPage />}
@@ -520,6 +537,56 @@ export const appRouteObjects = createRoutesFromElements(
                 />
               </Route>
             </Route>
+            <Route
+              path="evaluator-gallery"
+              element={<ProjectEvaluatorGalleryPage />}
+              handle={{
+                crumb: () => "Evaluator gallery",
+                parentCrumbTo: (({ parentPathname, search }) => ({
+                  pathname: `${parentPathname}/evaluators`,
+                  search,
+                })) satisfies ParentCrumbToFn,
+                agentRoute: {
+                  label: "Project Evaluator Gallery",
+                  description:
+                    "Browse evaluator templates and start a project evaluator from a template or from scratch.",
+                },
+              }}
+            >
+              <Route
+                path="new/llm"
+                element={<NewGalleryLlmProjectEvaluatorPage />}
+                handle={{
+                  agentRoute: {
+                    label: "New Project LLM Evaluator From Gallery",
+                    description:
+                      "Author a new LLM-as-a-judge evaluator from scratch while browsing the evaluator gallery.",
+                  },
+                }}
+              />
+              <Route
+                path="new/code"
+                element={<NewGalleryCodeProjectEvaluatorPage />}
+                handle={{
+                  agentRoute: {
+                    label: "New Project Code Evaluator From Gallery",
+                    description:
+                      "Author a new Python or TypeScript code evaluator from scratch while browsing the evaluator gallery.",
+                  },
+                }}
+              />
+              <Route
+                path="new/template/:templateName"
+                element={<NewGalleryLlmFromTemplateProjectEvaluatorPage />}
+                handle={{
+                  agentRoute: {
+                    label: "New Project Evaluator From Gallery Template",
+                    description:
+                      "Create a project LLM evaluator seeded from the selected evaluator gallery template.",
+                  },
+                }}
+              />
+            </Route>
             {/* The evaluator details page is a full page rather than a tab,
                 mirroring the dataset evaluator details route. The edit
                 slideover nests beneath it so it opens over the details view. */}

js/app/src/components/nav/NavBreadcrumb.tsx

@@ -1,4 +1,4 @@
-import { Link } from "react-router";
+import { Link, useLocation } from "react-router";
 
 import {
   Breadcrumb,
@@ -10,6 +10,7 @@ import { useMatchesWithCrumb } from "@phoenix/hooks/useMatchesWithCrumb";
 
 export function NavBreadcrumb() {
   const matchesWithCrumb = useMatchesWithCrumb();
+  const { search } = useLocation();
   const numMatches = matchesWithCrumb.length;
   return (
     <Breadcrumbs size="L">
@@ -18,12 +19,18 @@ export function NavBreadcrumb() {
         const copyableItems = match.handle.copy
           ? match.handle?.copy(match.loaderData)
           : [];
+        const nextMatch = matchesWithCrumb[index + 1];
+        const breadcrumbDestination =
+          nextMatch?.handle.parentCrumbTo?.({
+            parentPathname: match.pathname,
+            search,
+          }) ?? match.pathname;
         const isLastCrumb = index === numMatches - 1;
         const showCopyableItems = isLastCrumb && copyableItems.length;
         return (
           <Breadcrumb key={index}>
             <Flex direction="row" gap="size-100">
-              <Link to={match.pathname} title={crumb}>
+              <Link to={breadcrumbDestination} title={crumb}>
                 {crumb}
               </Link>
               {showCopyableItems ? (

js/app/src/hooks/useMatchesWithCrumb.ts

@@ -1,4 +1,4 @@
-import { useMatches } from "react-router";
+import { type To, useMatches } from "react-router";
 
 import type { CopyActionMenuItem } from "@phoenix/components";
 
@@ -8,12 +8,23 @@ export type CrumbFn = (data: unknown) => string;
  */
 export type CopyItem = CopyActionMenuItem;
 export type CopyFn = (data: unknown) => CopyActionMenuItem[];
+/**
+ * Resolves a custom destination for the preceding breadcrumb.
+ * @param params - Breadcrumb navigation context.
+ * @param params.parentPathname - The matched pathname of the preceding crumb.
+ * @param params.search - The current URL search string.
+ */
+export type ParentCrumbToFn = (params: {
+  parentPathname: string;
+  search: string;
+}) => To;
 type Matches = ReturnType<typeof useMatches>;
 type Match = Matches[number];
 type RouteMatchWithCrumb = Match & {
   handle: {
     crumb: CrumbFn;
     copy?: CopyFn;
+    parentCrumbTo?: ParentCrumbToFn;
   };
 };
 

js/app/src/pages/project/evaluators/AddProjectEvaluatorMenu.tsx

@@ -72,13 +72,23 @@ function AddProjectEvaluatorMenuItems() {
       <Menu
         aria-label="Add evaluator"
         onAction={(action) => {
-          if (action === "createEvaluator") {
+          if (action === "browseGallery") {
+            navigate(paths.gallery);
+          } else if (action === "createEvaluator") {
             navigate(paths.newLlm);
           } else if (action === "createCodeEvaluator") {
             navigate(paths.newCode);
           }
         }}
       >
+        <MenuSection>
+          <MenuItem
+            leadingContent={<Icon svg={<Icons.Grid />} />}
+            id="browseGallery"
+          >
+            Browse the whole library
+          </MenuItem>
+        </MenuSection>
         <MenuSection>
           <MenuSectionTitle title="LLM evaluator" />
           <MenuItem

js/app/src/pages/project/evaluators/CreateProjectEvaluatorSlideover.tsx

@@ -49,19 +49,25 @@ import {
   type EvaluatorStoreProps,
 } from "@phoenix/store/evaluatorStore";
 
+type SeededLlmEvaluatorInitialState = {
+  name: string;
+  description: string;
+  outputConfigs: AnnotationConfig[];
+  defaultMessages: PlaygroundChatTemplate["messages"];
+  templateFormat: TemplateFormat;
+  includeExplanation: boolean;
+};
+
 export type ProjectEvaluatorCreationMode =
   | { kind: "scratch" }
   | { kind: "newCode" }
   | {
       kind: "copy";
-      initialState: {
-        name: string;
-        description: string;
-        outputConfigs: AnnotationConfig[];
-        defaultMessages: PlaygroundChatTemplate["messages"];
-        templateFormat: TemplateFormat;
-        includeExplanation: boolean;
-      };
+      initialState: SeededLlmEvaluatorInitialState;
+    }
+  | {
+      kind: "template";
+      initialState: SeededLlmEvaluatorInitialState;
     }
   | {
       kind: "code";
@@ -86,6 +92,9 @@ function getProjectEvaluatorCreationTitle(
   if (creationMode.kind === "copy") {
     return `Copy LLM evaluator “${creationMode.initialState.name}”`;
   }
+  if (creationMode.kind === "template") {
+    return `Create “${creationMode.initialState.name}” evaluator`;
+  }
   return `Attach code evaluator “${creationMode.name}”`;
 }
 
@@ -116,15 +125,19 @@ function CreateProjectEvaluatorDialogForMode(
   props: Parameters<typeof CreateProjectEvaluatorDialog>[0]
 ) {
   const { creationMode } = props;
-  if (creationMode.kind === "scratch" || creationMode.kind === "copy") {
+  if (
+    creationMode.kind === "scratch" ||
+    creationMode.kind === "copy" ||
+    creationMode.kind === "template"
+  ) {
     const defaultMessages =
-      creationMode.kind === "copy"
-        ? creationMode.initialState.defaultMessages
-        : getSpanEvaluatorDefaultMessages();
+      creationMode.kind === "scratch"
+        ? getSpanEvaluatorDefaultMessages()
+        : creationMode.initialState.defaultMessages;
     const templateFormat =
-      creationMode.kind === "copy"
-        ? creationMode.initialState.templateFormat
-        : undefined;
+      creationMode.kind === "scratch"
+        ? undefined
+        : creationMode.initialState.templateFormat;
     return (
       <EvaluatorPlaygroundProvider
         defaultMessages={defaultMessages}
@@ -173,17 +186,24 @@ const CreateProjectEvaluatorDialog = ({
         ),
       } satisfies EvaluatorStoreProps;
     }
-    const copiedState =
-      creationMode.kind === "copy" ? creationMode.initialState : undefined;
-    const defaultEvaluatorName = copiedState?.name
-      ? `${copiedState.name} copy`
-      : creationMode.kind === "code"
-        ? creationMode.name
-        : DEFAULT_LLM_EVALUATOR_STORE_VALUES.evaluator.globalName;
+    const seededState =
+      creationMode.kind === "copy" || creationMode.kind === "template"
+        ? creationMode.initialState
+        : undefined;
+    const defaultEvaluatorName =
+      creationMode.kind === "copy"
+        ? creationMode.initialState.name
+          ? `${creationMode.initialState.name} copy`
+          : DEFAULT_LLM_EVALUATOR_STORE_VALUES.evaluator.globalName
+        : creationMode.kind === "template"
+          ? creationMode.initialState.name
+          : creationMode.kind === "code"
+            ? creationMode.name
+            : DEFAULT_LLM_EVALUATOR_STORE_VALUES.evaluator.globalName;
     const outputConfigs =
       creationMode.kind === "code"
         ? creationMode.outputConfigs
-        : (copiedState?.outputConfigs ??
+        : (seededState?.outputConfigs ??
           DEFAULT_LLM_EVALUATOR_STORE_VALUES.outputConfigs);
     return {
       ...DEFAULT_LLM_EVALUATOR_STORE_VALUES,
@@ -193,15 +213,15 @@ const CreateProjectEvaluatorDialog = ({
         description:
           creationMode.kind === "code"
             ? creationMode.description
-            : (copiedState?.description ?? ""),
+            : (seededState?.description ?? ""),
         inputMapping: { pathMapping: {}, literalMapping: {} },
         kind: creationMode.kind === "code" ? "CODE" : "LLM",
         includeExplanation:
-          copiedState?.includeExplanation ??
+          seededState?.includeExplanation ??
           DEFAULT_LLM_EVALUATOR_STORE_VALUES.evaluator.includeExplanation,
       },
       outputConfigs:
-        copiedState || creationMode.kind === "code"
+        seededState || creationMode.kind === "code"
           ? outputConfigs
           : outputConfigs[0]
             ? [{ ...outputConfigs[0], name: defaultEvaluatorName }]