Fix display of metrics from results

Add new pipeline fields to store result docs (for metrics calc)

Author: yamalight

app/components/pipeline/nodes/StepNode.tsx

@@ -44,9 +44,8 @@ export function StepNode({ data }: { data: ProcessingStep }) {
   const { averageTiming, averagePrompt, averageCompletion, averageCost } =
     useMemo(() => {
       // const timings = data.
-      const results = litlytics.docs;
-      const resultDocs = litlytics.pipeline.results?.map((r) => r.doc) ?? [];
-      const res = results.concat(resultDocs).filter((doc) => doc);
+      const results = litlytics.pipeline.resultDocs ?? [];
+      const res = results.filter((doc) => doc);
       if (!res.length) {
         return {};
       }

app/components/pipeline/nodes/output/OutputNode.tsx

@@ -50,9 +50,8 @@ export function OutputNode() {
 
   const { timing, prompt, completion, cost } = useMemo(() => {
     // const timings = data.
-    const results = litlytics.docs;
-    const resultDocs = litlytics.pipeline.results?.map((r) => r.doc) ?? [];
-    const res = results.concat(resultDocs).filter((doc) => doc);
+    const results = litlytics.pipeline.resultDocs ?? [];
+    const res = results.filter((doc) => doc);
     if (!res.length) {
       return {};
     }
@@ -88,6 +87,9 @@ export function OutputNode() {
         litlytics.setPipelineStatus(status);
       },
     });
+
+    // write final status to update on end
+    litlytics.setPipelineStatus({ status: 'done' });
   };
 
   const running =

examples/compare.json

@@ -75,13 +75,13 @@
   "results": [
     {
       "stepId": "step_1",
-      "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to put a smile on my face!\n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
+      "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to put a smile on my face! \n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
       "usage": {
-        "completionTokens": 90,
+        "completionTokens": 91,
         "promptTokens": 139,
-        "totalTokens": 229
+        "totalTokens": 230
       },
-      "timingMs": 1649,
+      "timingMs": 4028,
       "doc": {
         "id": "textdoc",
         "name": "Default document",
@@ -90,16 +90,46 @@
         "processingResults": [
           {
             "stepId": "step_1",
-            "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to put a smile on my face!\n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
+            "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to put a smile on my face! \n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
             "usage": {
-              "completionTokens": 90,
+              "completionTokens": 91,
               "promptTokens": 139,
-              "totalTokens": 229
+              "totalTokens": 230
             },
-            "timingMs": 1649
+            "timingMs": 4028
           }
         ]
       }
     }
+  ],
+  "resultDocs": [
+    {
+      "id": "textdoc",
+      "name": "Default document",
+      "content": "Post is a photo of my cute black cat with white mustache.",
+      "test": true,
+      "processingResults": [
+        {
+          "stepId": "step_0",
+          "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to make me smile! \n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
+          "usage": {
+            "completionTokens": 88,
+            "promptTokens": 89,
+            "totalTokens": 177
+          },
+          "timingMs": 3736
+        },
+        {
+          "stepId": "step_1",
+          "result": "🐾 Meet my adorable little buddy! 🐱✨ This is my cute black cat with a charming white mustache! Isn’t he just the cutest? 😻 He always knows how to put a smile on my face! \n\nWhat do you think of his mustache? Let me know in the comments! 💬👇\n\n#CatLovers #CuteCats #BlackCat #CatMustache #FurryFriend #PetLove #Caturday",
+          "usage": {
+            "completionTokens": 91,
+            "promptTokens": 139,
+            "totalTokens": 230
+          },
+          "timingMs": 4028
+        }
+      ]
+    }
   ]
 }

examples/complaints.json

@@ -8,8 +8,10 @@ export async function runPipeline(
   litlytics: LitLytics,
   onStatus: (status: PipelineStatus) => void
 ) {
+  const pipeline = structuredClone(litlytics.pipeline);
+
   // get source
-  const source = litlytics.pipeline.source;
+  const source = pipeline.source;
   // validate source
   if (!source) {
     throw new Error('Source is required!');
@@ -19,7 +21,7 @@ export async function runPipeline(
   onStatus({ status: 'sourcing' });
 
   // get all documents from the source
-  const docs: Doc[] = litlytics.pipeline.source.docs;
+  const docs: Doc[] = pipeline.source.docs;
   // validate docs
   if (!docs?.length) {
     throw new Error('At least one document is required!');
@@ -31,7 +33,7 @@ export async function runPipeline(
   // get all connections to source
   let stepIds = source.connectsTo;
   // find all connected steps
-  let nextStep = litlytics.pipeline.steps.find((s) => stepIds.includes(s.id));
+  let nextStep = pipeline.steps.find((s) => stepIds.includes(s.id));
   // while there are follow-up steps - continue;
   while (nextStep !== undefined) {
     onStatus({ status: 'step', currentStep: nextStep });
@@ -43,7 +45,7 @@ export async function runPipeline(
           litlytics.runStep({
             step: nextStep!,
             source,
-            allSteps: litlytics.pipeline.steps,
+            allSteps: pipeline.steps,
             doc,
             allDocs: docs,
           })
@@ -64,7 +66,7 @@ export async function runPipeline(
       aggregateResult = (await litlytics.runStep({
         step: nextStep!,
         source,
-        allSteps: litlytics.pipeline.steps,
+        allSteps: pipeline.steps,
         doc: aggregateResult,
         allDocs: docs,
       })) as Doc;
@@ -74,16 +76,18 @@ export async function runPipeline(
     stepIds = nextStep.connectsTo;
     // if we're at the output - handle results and break
     if (stepIds.includes(OUTPUT_ID)) {
+      // store processed docs
+      pipeline.resultDocs = newDocs.filter((doc) => doc);
       // process and store final results
-      litlytics.pipeline.results = getResults(litlytics, newDocs);
+      pipeline.results = getResults(litlytics, newDocs);
       break;
     }
-    nextStep = litlytics.pipeline.steps.find((s) => stepIds.includes(s.id));
+    nextStep = pipeline.steps.find((s) => stepIds.includes(s.id));
   }
 
   onStatus({ status: 'done' });
 
   // save result to pipeline
   // pipeline.results = finalResult;
-  return litlytics.pipeline;
+  return pipeline;
 }

examples/etsy.json

@@ -1,3 +1,4 @@
+import type { Doc } from '../doc/Document';
 import type { Result } from '../output/Output';
 import type { ProcessingStep, SourceStep } from '../step/Step';
 
@@ -6,6 +7,7 @@ export interface Pipeline {
   pipelineDescription?: string;
   pipelinePlan?: string;
   source: SourceStep;
+  resultDocs?: Doc[];
   results?: Result[];
   steps: ProcessingStep[];
   provider?: string;