chore(core): update type definition for insight (#1323)

* feat(core): update type definition for insight dump

* feat(core): update type definition for insight dump

* fix(core): error state for waitFor

* fix(core): error state for waitFor

* fix(core): test cases

* fix(core): cases in report

Author: yuyutaotao

apps/report/src/components/sidebar/index.tsx

@@ -49,12 +49,11 @@ const SideItem = (props: {
         }}
         bordered={false}
       >
-        cache
+        Cache
       </Tag>
     ) : null;
 
-  const deepThinkEl = (task as ExecutionTaskInsightLocate)?.log?.dump
-    ?.deepThink ? (
+  const deepThinkEl = (task as ExecutionTaskInsightLocate)?.param?.deepThink ? (
     <Tag
       bordered={false}
       style={{
@@ -66,7 +65,7 @@ const SideItem = (props: {
         lineHeight: '16px',
       }}
     >
-      deepthink
+      DeepThink
     </Tag>
   ) : null;
 
@@ -83,10 +82,7 @@ const SideItem = (props: {
     }
 
     const isAssertFinishedWithWarning =
-      isFinished &&
-      task.subType === 'Assert' &&
-      task.output === false &&
-      task.log.isWaitForAssert;
+      isFinished && task.subType === 'WaitFor' && task.output === false;
 
     if (isAssertFinishedWithWarning) {
       return iconForStatus('finishedWithWarning');

apps/report/src/components/store/index.tsx

@@ -204,7 +204,7 @@ export const useExecutionDump = create<DumpStoreType>((set, get) => {
       });
       console.log('will set task', task);
       if (task.type === 'Insight') {
-        const dump = (task as ExecutionTaskInsightLocate).log?.dump!;
+        const dump = (task as ExecutionTaskInsightLocate).log;
         set({
           insightDump: dump,
         });

packages/core/src/agent/agent.ts

@@ -36,6 +36,7 @@ export type TestStatus =
 import yaml from 'js-yaml';
 
 import {
+  getVersion,
   groupedActionDumpFileExt,
   processCacheConfig,
   reportHTMLContent,
@@ -336,6 +337,7 @@ export class Agent<
 
   resetDump() {
     this.dump = {
+      sdkVersion: getVersion(),
       groupName: this.opts.groupName!,
       groupDescription: this.opts.groupDescription,
       executions: [],
@@ -959,15 +961,19 @@ export class Agent<
       screenshotIncluded:
         opt?.screenshotIncluded ??
         defaultInsightExtractOption.screenshotIncluded,
-      isWaitForAssert: opt?.isWaitForAssert,
       doNotThrowError: opt?.doNotThrowError,
     };
 
-    const { output, executor, thought } = await this.taskExecutor.assert(
-      assertion,
-      modelConfig,
-      insightOpt,
-    );
+    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+
+    const { output, executor, thought } =
+      await this.taskExecutor.createTypeQueryExecution<boolean>(
+        'Assert',
+        textPrompt,
+        modelConfig,
+        insightOpt,
+        multimodalPrompt,
+      );
     await this.afterTaskRunning(executor, true);
 
     const message = output
@@ -1114,7 +1120,6 @@ export class Agent<
     };
     // 4. build ExecutionDump
     const executionDump: ExecutionDump = {
-      sdkVersion: '',
       logTime: now,
       name: `Log - ${title || 'untitled'}`,
       description: opt?.content || '',

packages/core/src/agent/tasks.ts

@@ -864,7 +864,7 @@ export class TaskExecutor {
   }
 
   private createTypeQueryTask(
-    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',
+    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',
     demand: InsightExtractParam,
     modelConfig: IModelConfig,
     opt?: InsightExtractOption,
@@ -906,10 +906,14 @@ export class TaskExecutor {
         const ifTypeRestricted = type !== 'Query';
         let demandInput = demand;
         let keyOfResult = 'result';
-        if (ifTypeRestricted && type === 'Assert') {
+        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {
           keyOfResult = 'StatementIsTruthy';
+          const booleanPrompt =
+            type === 'Assert'
+              ? `Boolean, whether the following statement is true: ${demand}`
+              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;
           demandInput = {
-            [keyOfResult]: `Boolean, whether the following statement is true: ${demand}`,
+            [keyOfResult]: booleanPrompt,
           };
         } else if (ifTypeRestricted) {
           demandInput = {
@@ -931,7 +935,7 @@ export class TaskExecutor {
             outputResult = data;
           } else {
             assert(
-              data?.[keyOfResult] !== undefined,
+              type !== 'WaitFor' ? data?.[keyOfResult] !== undefined : true,
               'No result in query data',
             );
             outputResult = (data as any)[keyOfResult];
@@ -940,7 +944,7 @@ export class TaskExecutor {
 
         return {
           output: outputResult,
-          log: { dump: insightDump, isWaitForAssert: opt?.isWaitForAssert },
+          log: insightDump,
           usage,
           thought,
         };
@@ -992,21 +996,6 @@ export class TaskExecutor {
     };
   }
 
-  async assert(
-    assertion: TUserPrompt,
-    modelConfig: IModelConfig,
-    opt?: InsightExtractOption,
-  ): Promise<ExecutionResult<boolean>> {
-    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
-    return await this.createTypeQueryExecution<boolean>(
-      'Assert',
-      textPrompt,
-      modelConfig,
-      opt,
-      multimodalPrompt,
-    );
-  }
-
   private async appendErrorPlan(
     taskExecutor: Executor,
     errorMsg: string,
@@ -1074,15 +1063,13 @@ export class TaskExecutor {
     const overallStartTime = Date.now();
     let startTime = Date.now();
     let errorThought = '';
-    let hitError = false;
     while (Date.now() - overallStartTime < timeoutMs) {
       startTime = Date.now();
       const queryTask = await this.createTypeQueryTask(
-        'Assert',
+        'WaitFor',
         textPrompt,
         modelConfig,
         {
-          isWaitForAssert: true,
           doNotThrowError: true,
         },
         multimodalPrompt,
@@ -1096,20 +1083,6 @@ export class TaskExecutor {
           }
         | undefined;
 
-      // If executor enters error state, stop polling immediately
-      if (taskExecutor.isInErrorState()) {
-        errorThought =
-          taskExecutor.latestErrorTask()?.errorMessage ||
-          `Error occurred during waitFor: ${textPrompt}`;
-        hitError = true;
-        break;
-      }
-
-      if (!result) {
-        errorThought = `No result from assertion: ${textPrompt}`;
-        break;
-      }
-
       if (result?.output) {
         return {
           output: undefined,
@@ -1119,6 +1092,7 @@ export class TaskExecutor {
 
       errorThought =
         result?.thought ||
+        (!result && `No result from assertion: ${textPrompt}`) ||
         `unknown error when waiting for assertion: ${textPrompt}`;
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
@@ -1128,15 +1102,6 @@ export class TaskExecutor {
       }
     }
 
-    // If executor is already in error state, don't try to append error plan
-    // Just return the executor with existing error information
-    if (hitError) {
-      return {
-        output: undefined,
-        executor: taskExecutor,
-      };
-    }
-
     return this.appendErrorPlan(
       taskExecutor,
       `waitFor timeout: ${errorThought}`,

packages/core/src/ai-model/action-executor.ts

@@ -7,7 +7,6 @@ import type {
   ExecutionTaskReturn,
   ExecutorContext,
 } from '@/types';
-import { getVersion } from '@/utils';
 import { assert } from '@midscene/shared/utils';
 
 export class Executor {
@@ -119,6 +118,7 @@ export class Executor {
             task.subType === 'Locate' ||
               task.subType === 'Query' ||
               task.subType === 'Assert' ||
+              task.subType === 'WaitFor' ||
               task.subType === 'Boolean' ||
               task.subType === 'Number' ||
               task.subType === 'String',
@@ -199,7 +199,6 @@ export class Executor {
 
   dump(): ExecutionDump {
     const dumpData: ExecutionDump = {
-      sdkVersion: getVersion(),
       logTime: Date.now(),
       name: this.name,
       tasks: this.tasks,

packages/core/src/insight/utils.ts

@@ -4,15 +4,13 @@ import type {
   InsightDump,
   PartialInsightDumpFromSDK,
 } from '@/types';
-import { getVersion } from '@/utils';
 import { uuid } from '@midscene/shared/utils';
 
 export function emitInsightDump(
   data: PartialInsightDumpFromSDK,
   dumpSubscriber?: DumpSubscriber,
 ) {
   const baseData: DumpMeta = {
-    sdkVersion: getVersion(),
     logTime: Date.now(),
   };
   const finalData: InsightDump = {

packages/core/src/types.ts

@@ -170,7 +170,6 @@ export interface InsightTaskInfo {
 }
 
 export interface DumpMeta {
-  sdkVersion: string;
   logTime: number;
 }
 
@@ -200,7 +199,7 @@ export interface InsightDump extends DumpMeta {
 
 export type PartialInsightDumpFromSDK = Omit<
   InsightDump,
-  'sdkVersion' | 'logTime' | 'logId' | 'model_name'
+  'logTime' | 'logId' | 'model_name'
 >;
 
 export type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;
@@ -413,15 +412,13 @@ export interface ExecutionTaskInsightLocateOutput {
   element: LocateResultElement | null;
 }
 
-export interface ExecutionTaskInsightDumpLog {
-  dump?: InsightDump;
-}
+export type ExecutionTaskInsightDump = InsightDump;
 
 export type ExecutionTaskInsightLocateApply = ExecutionTaskApply<
   'Insight',
   ExecutionTaskInsightLocateParam,
   ExecutionTaskInsightLocateOutput,
-  ExecutionTaskInsightDumpLog
+  ExecutionTaskInsightDump
 >;
 
 export type ExecutionTaskInsightLocate =
@@ -442,7 +439,7 @@ export type ExecutionTaskInsightQueryApply = ExecutionTaskApply<
   'Insight',
   ExecutionTaskInsightQueryParam,
   any,
-  ExecutionTaskInsightDumpLog
+  ExecutionTaskInsightDump
 >;
 
 export type ExecutionTaskInsightQuery =
@@ -459,7 +456,7 @@ export type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<
   'Insight',
   ExecutionTaskInsightAssertionParam,
   InsightAssertionResponse,
-  ExecutionTaskInsightDumpLog
+  ExecutionTaskInsightDump
 >;
 
 export type ExecutionTaskInsightAssertion =
@@ -507,6 +504,7 @@ export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
 Grouped dump
 */
 export interface GroupedActionDump {
+  sdkVersion: string;
   groupName: string;
   groupDescription?: string;
   modelBriefs: string[];

packages/core/src/yaml.ts

@@ -13,8 +13,6 @@ export interface LocateOption {
 export interface InsightExtractOption {
   domIncluded?: boolean | 'visible-only';
   screenshotIncluded?: boolean;
-  // To make the assert in the "waitfor" section display the warning icon in report
-  isWaitForAssert?: boolean;
   doNotThrowError?: boolean;
 }
 

packages/core/tests/unit-test/executor/index.test.ts

@@ -6,7 +6,6 @@ import type {
   ExecutionTaskInsightLocateApply,
   InsightDump,
 } from '@/index';
-import { globalModelConfigManager } from '@midscene/shared/env';
 import { fakeInsight } from 'tests/utils';
 import { describe, expect, it, vi } from 'vitest';
 
@@ -105,7 +104,7 @@ describe(
       expect(tasks.length).toBe(inputTasks.length);
       expect(tasks[0].status).toBe('finished');
       expect(tasks[0].output).toMatchSnapshot();
-      expect(tasks[0].log?.dump).toBeTruthy();
+      expect(tasks[0].log).toBeTruthy();
       expect(tasks[0].timing?.end).toBeTruthy();
       expect(tasks[0].hitBy?.from).not.toBe('Cache');
 

packages/visualizer/src/utils/replay-scripts.ts

@@ -140,15 +140,11 @@ export const allScriptsFromDump = (
   const dimensionsSet = new Set<string>();
   let firstWidth: number | undefined = undefined;
   let firstHeight: number | undefined = undefined;
-  let sdkVersion: string | undefined = undefined;
+  const sdkVersion: string | undefined = dump.sdkVersion;
 
   const modelBriefsSet = new Set<string>();
 
   dump.executions.forEach((execution) => {
-    if (execution.sdkVersion) {
-      sdkVersion = execution.sdkVersion;
-    }
-
     execution.tasks.forEach((task) => {
       if (task.uiContext?.size?.width) {
         const w = task.uiContext.size.width;
@@ -339,7 +335,7 @@ export const generateAnimationScripts = (
       }
       const context = insightTask.uiContext;
       if (context?.screenshotBase64) {
-        const insightDump = insightTask.log?.dump;
+        const insightDump = insightTask.log;
         const insightContentLength = context.tree
           ? treeToList(context.tree).length
           : 0;