fix(playwright,puppeteer): only intercept requests for non-GET methods or payloads

packages/playwright-crawler/src/internals/utils/playwright-utils.ts

@@ -205,7 +205,7 @@ export async function gotoExtended(
     const { url, method, headers, payload } = request;
     const isEmpty = (o?: object) => !o || Object.keys(o).length === 0;
 
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
         log.deprecated(
             'Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
@@ -235,6 +235,8 @@ export async function gotoExtended(
         };
 
         await page.route('**/*', interceptRequestHandler);
+    } else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers!);
     }
 
     return page.goto(url, gotoOptions);

packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts

@@ -490,7 +490,7 @@ export async function gotoExtended(
     const { url, method, headers, payload } = request;
     const isEmpty = (o?: object) => !o || Object.keys(o).length === 0;
 
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
         log.deprecated(
             'Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
@@ -517,6 +517,8 @@ export async function gotoExtended(
         };
 
         await addInterceptRequestHandler(page, interceptRequestHandler);
+    } else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers!);
     }
 
     return page.goto(url, gotoOptions as Dictionary);

test/core/playwright_utils.test.ts

@@ -271,6 +271,33 @@ describe('playwrightUtils', () => {
         }
     }, 60_000);
 
+    test('gotoExtended() applies headers natively for GET requests without a payload', async () => {
+        const browser = await chromium.launch({ headless: true });
+
+        try {
+            const page = await browser.newPage();
+            const routeSpy = vitest.spyOn(page, 'route');
+            const deprecatedSpy = vitest.spyOn(log, 'deprecated');
+
+            const request = new Request({
+                url: `${serverAddress}/special/getDebug`,
+                headers: {
+                    'User-Agent': 'Demo UA',
+                },
+            });
+
+            const response = await playwrightUtils.gotoExtended(page, request);
+
+            const { method, headers } = JSON.parse(await response!.text());
+            expect(method).toBe('GET');
+            expect(headers['user-agent']).toBe('Demo UA');
+            expect(routeSpy).not.toHaveBeenCalled();
+            expect(deprecatedSpy).not.toHaveBeenCalled();
+        } finally {
+            await browser.close();
+        }
+    }, 60_000);
+
     describe('shadow root expansion', () => {
         let browser: Browser;
         beforeAll(async () => {

test/core/puppeteer_utils.test.ts

@@ -395,6 +395,33 @@ describe('puppeteerUtils', () => {
             }
         });
 
+        test('gotoExtended() applies headers natively for GET requests without a payload', async () => {
+            const browser = await launchPuppeteer(launchContext);
+
+            try {
+                const page = await browser.newPage();
+                const interceptionSpy = vitest.spyOn(page, 'setRequestInterception');
+                const deprecatedSpy = vitest.spyOn(log, 'deprecated');
+
+                const request = new Request({
+                    url: `${serverAddress}/special/getDebug`,
+                    headers: {
+                        'User-Agent': 'Demo UA',
+                    },
+                });
+
+                const response = await puppeteerUtils.gotoExtended(page, request, { waitUntil: 'networkidle' });
+
+                const { method, headers } = JSON.parse(await response!.text());
+                expect(method).toBe('GET');
+                expect(headers['user-agent']).toBe('Demo UA');
+                expect(interceptionSpy).not.toHaveBeenCalled();
+                expect(deprecatedSpy).not.toHaveBeenCalled();
+            } finally {
+                await browser.close();
+            }
+        });
+
         describe('infiniteScroll()', () => {
             function isAtBottom() {
                 return window.innerHeight + window.pageYOffset >= document.body.offsetHeight;