feat(web-integration): enhance element caching by adding XPath support and improving cache hit logic

quanru · quanru · commit 8c69013c7bc2 · 2025-05-09T18:54:48.000+08:00
diff --git a/packages/core/src/insight/index.ts b/packages/core/src/insight/index.ts
@@ -173,7 +173,7 @@ export default class Insight<
     const elements: BaseElement[] = [];
     (parseResult.elements || []).forEach((item) => {
       if ('id' in item) {
-        const element = elementById(item.id);
+        const element = elementById(item?.id);
 
         if (!element) {
           console.warn(
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
@@ -383,7 +383,7 @@ export interface ExecutionTaskApply<
     param: TaskParam,
     context: ExecutorContext,
   ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>
-    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>
+  | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>
     | undefined
     | void;
 }
diff --git a/packages/shared/src/extractor/index.ts b/packages/shared/src/extractor/index.ts
@@ -30,4 +30,8 @@ export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-ext
 
 export { setNodeHashCacheListOnWindow } from './util';
 
-export { getXpathsById, getElementInfoByNode } from './locator';
+export {
+  getXpathsById,
+  getElementInfoByNode,
+  getElementInfoByXpath,
+} from './locator';
diff --git a/packages/shared/src/extractor/locator.ts b/packages/shared/src/extractor/locator.ts
@@ -32,6 +32,17 @@ const findFirstAncestorWithId = (element: Element): Element | null => {
 };
 
 const getElementXPath = (element: Node): string => {
+  // 处理文本节点
+  if (element.nodeType === Node.TEXT_NODE) {
+    // 获取父元素的 XPath
+    const parentNode = element.parentNode;
+    if (parentNode && parentNode.nodeType === Node.ELEMENT_NODE) {
+      const parentXPath = getElementXPath(parentNode);
+      return `${parentXPath}/text()`;
+    }
+    return '';
+  }
+
   if (element.nodeType !== Node.ELEMENT_NODE) return '';
 
   const el = element as Element;
@@ -99,19 +110,37 @@ function generateXPaths(node: Node | null): string[] {
 }
 
 export function getXpathsById(id: string): string[] | null {
-  let node = getNodeFromCacheList(id);
+  const node = getNodeFromCacheList(id);
 
   if (!node) {
     return null;
   }
 
-  if (isTextElement(node)) {
-    node = node.parentElement;
-  }
-
   return generateXPaths(node);
 }
 
 export function getElementInfoByNode(node: Node): ElementInfo | null {
   return collectElementInfo(node, window, document);
 }
+
+export function getElementInfoByXpath(xpath: string): ElementInfo | null {
+  const xpathResult = document.evaluate(
+    xpath,
+    document,
+    null,
+    XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
+    null,
+  );
+
+  if (xpathResult.snapshotLength !== 1) {
+    return null;
+  }
+
+  const node = xpathResult.snapshotItem(0);
+
+  if (!node) {
+    return null;
+  }
+
+  return collectElementInfo(node, window, document);
+}
diff --git a/packages/web-integration/src/common/task-cache.ts b/packages/web-integration/src/common/task-cache.ts
@@ -1,10 +1,6 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import type {
-  AIElementLocatorResponse,
-  LocateResultElement,
-  PlanningAIResponse,
-} from '@midscene/core';
+import type { LocateResultElement, PlanningAIResponse } from '@midscene/core';
 import type { vlmPlanning } from '@midscene/core/ai-model';
 import { stringifyDumpData, writeLogFile } from '@midscene/core/utils';
 import { getMidsceneRunSubDir } from '@midscene/shared/common';
@@ -13,10 +9,12 @@ import { getRunningPkgInfo } from '@midscene/shared/fs';
 import { getDebug } from '@midscene/shared/logger';
 import { ifInBrowser } from '@midscene/shared/utils';
 import semver from 'semver';
-import type { PlaywrightWebPage } from '../playwright';
-import type { PuppeteerWebPage } from '../puppeteer';
 import type { WebPage } from './page';
-import { type WebUIContext, replaceIllegalPathCharsAndSpace } from './utils';
+import {
+  type WebUIContext,
+  checkElementExistsByXPath,
+  replaceIllegalPathCharsAndSpace,
+} from './utils';
 const debug = getDebug('cache');
 
 export type PlanTask = {
@@ -55,7 +53,9 @@ export type LocateTask = {
       height: number;
     };
   };
-  response: AIElementLocatorResponse;
+  response: {
+    xpaths: string[];
+  };
   element: LocateResultElement;
 };
 
@@ -190,24 +190,6 @@ export class TaskCache {
    * @param userPrompt String type, representing user prompt information
    * @return Returns a Promise object that resolves to a boolean or object
    */
-  async matchCache(
-    pageContext: WebUIContext,
-    type: 'plan',
-    userPrompt: string,
-    cacheGroup: AiTasks,
-  ): Promise<PlanTask['response']>;
-  async matchCache(
-    pageContext: WebUIContext,
-    type: 'ui-tars-plan',
-    userPrompt: string,
-    cacheGroup: AiTasks,
-  ): Promise<UITarsPlanTask['response']>;
-  async matchCache(
-    pageContext: WebUIContext,
-    type: 'locate',
-    userPrompt: string,
-    cacheGroup: AiTasks,
-  ): Promise<LocateTask['response']>;
   async matchCache(
     pageContext: WebUIContext,
     type: 'plan' | 'locate' | 'ui-tars-plan',
@@ -243,69 +225,24 @@ export class TaskCache {
 
       // The corresponding element cannot be found in the new context
       if (taskRes?.type === 'locate') {
-        const xpaths = taskRes.response?.elements?.[0]?.xpaths;
+        const xpaths = taskRes.response?.xpaths;
 
         if (!xpaths || !xpaths.length) {
           debug('no xpaths in cached response');
           return false;
         }
 
-        for (const xpath of xpaths) {
-          if (this.page.pageType === 'playwright') {
-            try {
-              const playwrightPage = (this.page as PlaywrightWebPage)
-                .underlyingPage;
-              const xpathLocator = playwrightPage.locator(`xpath=${xpath}`);
-              const xpathCount = await xpathLocator.count();
-              if (xpathCount === 1) {
-                debug(
-                  'cache hit, type: %s, prompt: %s, xpath: %s',
-                  type,
-                  userPrompt,
-                  xpath,
-                );
-                const xpathElement = await xpathLocator.first();
-                await xpathElement.evaluate((element) => {
-                  element.setAttribute('data-midscene', 'cache-hit');
-                  element.scrollIntoView();
-                });
-                return taskRes.response;
-              }
-            } catch (error) {
-              debug('playwright xpath locator error', error);
-            }
-          } else if (this.page.pageType === 'puppeteer') {
-            try {
-              const puppeteerPage = (this.page as PuppeteerWebPage)
-                .underlyingPage;
-              const xpathElements = await puppeteerPage.$$(`xpath=${xpath}`);
-              if (xpathElements && xpathElements.length === 1) {
-                debug(
-                  'cache hit, type: %s, prompt: %s, xpath: %s',
-                  type,
-                  userPrompt,
-                  xpath,
-                );
-                await xpathElements[0].evaluate((element) => {
-                  element.setAttribute('data-midscene', 'cache-hit');
-                  element.scrollIntoView();
-                });
-                return taskRes.response;
-              }
-            } catch (error) {
-              debug('puppeteer xpath locator error', error);
-            }
-          } else {
-            debug('unknown page type, will not match cache', {
-              pageType: this.page.pageType,
-            });
-          }
-
-          debug('cannot match element with same id in current page', {
-            element: taskRes.element,
-          });
-          return false;
+        const elementExists = await checkElementExistsByXPath(
+          this.page,
+          xpaths,
+          { type, userPrompt, debug },
+        );
+
+        if (elementExists) {
+          return taskRes.response;
         }
+
+        return false;
       }
 
       if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt) {
diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts
@@ -196,21 +196,21 @@ export class PageTaskExecutor {
               cachePrompt,
             );
             let elementInfo: ElementInfo | null = null;
-            const oldId = locateCache?.elements?.[0]?.id;
+            const xpaths = locateCache?.xpaths;
             let newId = null;
             try {
-              if (oldId) {
+              if (xpaths?.length) {
                 // hit cache, use new id
                 const elementInfosScriptContent =
                   getElementInfosScriptContent();
                 elementInfo = await this.page.evaluateJavaScript?.(
-                  `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByNode(document.querySelector('[data-midscene="cache-hit"]'))`,
+                  `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpaths[0]}')`,
                 );
                 newId = elementInfo?.id;
-                debug('get new id by node', newId);
+                debug('get new id by xpath', elementInfo?.id);
               }
             } catch (error) {
-              debug('get element info by node error: ', error);
+              debug('get element info by xpath error: ', error);
             }
             let cacheHitFlag = false;
 
@@ -253,12 +253,7 @@ export class PageTaskExecutor {
                 },
                 prompt: cachePrompt,
                 response: {
-                  elements: [
-                    {
-                      id: newId || element.id,
-                      xpaths,
-                    },
-                  ],
+                  xpaths,
                 },
                 element,
               });
diff --git a/packages/web-integration/src/common/utils.ts b/packages/web-integration/src/common/utils.ts
@@ -14,6 +14,8 @@ import { assert, logMsg, uuid } from '@midscene/shared/utils';
 import dayjs from 'dayjs';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { Page as PuppeteerPage } from 'puppeteer';
+import type { PlaywrightWebPage } from '../playwright';
+import type { PuppeteerWebPage } from '../puppeteer';
 import { WebElementInfo } from '../web-element';
 import type { WebPage } from './page';
 
@@ -183,3 +185,76 @@ export function forceClosePopup(
     }
   });
 }
+
+function replaceTextInXPath(xpath: string) {
+  return xpath.replace('/text()', ''); // /text() can't get locator
+}
+
+export async function checkElementExistsByXPath(
+  page: WebPage,
+  xpaths: string[],
+  logData: {
+    type: string;
+    userPrompt: string;
+    debug: (...args: any[]) => void;
+  },
+): Promise<boolean> {
+  const { type, userPrompt, debug } = logData;
+
+  for (const xpath of xpaths) {
+    if (page.pageType === 'playwright') {
+      try {
+        const playwrightPage = (page as PlaywrightWebPage).underlyingPage;
+        const xpathLocator = playwrightPage.locator(
+          `xpath=${replaceTextInXPath(xpath)}`,
+        );
+        const xpathCount = await xpathLocator.count();
+        if (xpathCount === 1) {
+          debug(
+            'cache hit, type: %s, prompt: %s, xpath: %s',
+            type,
+            userPrompt,
+            xpath,
+          );
+          const xpathElement = await xpathLocator.first();
+          await xpathElement.evaluate((element: Element) => {
+            element.scrollIntoView();
+            element.setAttribute('data-midscene', 'cache-hit');
+          });
+          return true;
+        }
+      } catch (error) {
+        debug('playwright xpath locator error', error);
+      }
+    } else if (page.pageType === 'puppeteer') {
+      try {
+        const puppeteerPage = (page as PuppeteerWebPage).underlyingPage;
+        const xpathElements = await puppeteerPage.$$(
+          `xpath=${replaceTextInXPath(xpath)}`,
+        );
+        if (xpathElements && xpathElements.length === 1) {
+          debug(
+            'cache hit, type: %s, prompt: %s, xpath: %s',
+            type,
+            userPrompt,
+            xpath,
+          );
+          await xpathElements[0].evaluate((element: Element) => {
+            element.scrollIntoView();
+            element.setAttribute('data-midscene', 'cache-hit');
+          });
+          return true;
+        }
+      } catch (error) {
+        debug('puppeteer xpath locator error', error);
+      }
+    } else {
+      debug('unknown page type, will not match cache', {
+        pageType: page.pageType,
+      });
+    }
+  }
+
+  debug('cannot match element with same id in current page');
+  return false;
+}
diff --git a/packages/web-integration/tests/ai/web/puppeteer/open-new-tab.test.ts b/packages/web-integration/tests/ai/web/puppeteer/open-new-tab.test.ts
@@ -1,6 +1,6 @@
 import { PuppeteerAgent } from '@/puppeteer';
 import { sleep } from '@midscene/core/utils';
-import { afterEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, describe, it, vi } from 'vitest';
 import { launchPage } from './utils';
 
 vi.setConfig({
@@ -16,7 +16,9 @@ describe('agent with forceSameTabNavigation', () => {
   });
 
   it('open new tab', async () => {
-    const { originPage, reset } = await launchPage('https://www.bing.com/');
+    const { originPage, reset } = await launchPage('https://www.bing.com/', {
+      headless: false,
+    });
     resetFn = reset;
     const agent = new PuppeteerAgent(originPage, {
       cacheId: 'puppeteer-open-new-tab',
diff --git a/packages/web-integration/tests/unit-test/task-cache.test.ts b/packages/web-integration/tests/unit-test/task-cache.test.ts

Original file line number	Diff line number	Diff line change
`@@ -383,7 +383,7 @@ export interface ExecutionTaskApply<`
`383`	`383`	`param: TaskParam,`
`384`	`384`	`context: ExecutorContext,`
`385`	`385`	`) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>`
`386`		`- \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>`
	`386`	`+ \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>`
`387`	`387`	`\| undefined`
`388`	`388`	`\| void;`
`389`	`389`	`}`