diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts index 6c6d34bf0b81..c7295524b6b2 100644 --- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts +++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts @@ -205,7 +205,7 @@ export async function gotoExtended( const { url, method, headers, payload } = request; const isEmpty = (o?: object) => !o || Object.keys(o).length === 0; - if (method !== 'GET' || payload || !isEmpty(headers)) { + if (method !== 'GET' || payload) { // This is not deprecated, we use it to log only once. log.deprecated( 'Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' + @@ -235,6 +235,8 @@ export async function gotoExtended( }; await page.route('**/*', interceptRequestHandler); + } else if (!isEmpty(headers)) { + await page.setExtraHTTPHeaders(headers!); } return page.goto(url, gotoOptions); diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts index 8c061421f19e..9d01f6db0ee5 100644 --- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts +++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts @@ -490,7 +490,7 @@ export async function gotoExtended( const { url, method, headers, payload } = request; const isEmpty = (o?: object) => !o || Object.keys(o).length === 0; - if (method !== 'GET' || payload || !isEmpty(headers)) { + if (method !== 'GET' || payload) { // This is not deprecated, we use it to log only once. log.deprecated( 'Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' + @@ -517,6 +517,8 @@ export async function gotoExtended( }; await addInterceptRequestHandler(page, interceptRequestHandler); + } else if (!isEmpty(headers)) { + await page.setExtraHTTPHeaders(headers!); } return page.goto(url, gotoOptions as Dictionary); diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts index d17eeebe9c40..b765e5462739 100644 --- a/test/core/playwright_utils.test.ts +++ b/test/core/playwright_utils.test.ts @@ -271,6 +271,33 @@ describe('playwrightUtils', () => { } }, 60_000); + test('gotoExtended() applies headers natively for GET requests without a payload', async () => { + const browser = await chromium.launch({ headless: true }); + + try { + const page = await browser.newPage(); + const routeSpy = vitest.spyOn(page, 'route'); + const deprecatedSpy = vitest.spyOn(log, 'deprecated'); + + const request = new Request({ + url: `${serverAddress}/special/getDebug`, + headers: { + 'User-Agent': 'Demo UA', + }, + }); + + const response = await playwrightUtils.gotoExtended(page, request); + + const { method, headers } = JSON.parse(await response!.text()); + expect(method).toBe('GET'); + expect(headers['user-agent']).toBe('Demo UA'); + expect(routeSpy).not.toHaveBeenCalled(); + expect(deprecatedSpy).not.toHaveBeenCalled(); + } finally { + await browser.close(); + } + }, 60_000); + describe('shadow root expansion', () => { let browser: Browser; beforeAll(async () => { diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts index 9f5bdde8ddb1..af5570e3c295 100644 --- a/test/core/puppeteer_utils.test.ts +++ b/test/core/puppeteer_utils.test.ts @@ -395,6 +395,33 @@ describe('puppeteerUtils', () => { } }); + test('gotoExtended() applies headers natively for GET requests without a payload', async () => { + const browser = await launchPuppeteer(launchContext); + + try { + const page = await browser.newPage(); + const interceptionSpy = vitest.spyOn(page, 'setRequestInterception'); + const deprecatedSpy = vitest.spyOn(log, 'deprecated'); + + const request = new Request({ + url: `${serverAddress}/special/getDebug`, + headers: { + 'User-Agent': 'Demo UA', + }, + }); + + const response = await puppeteerUtils.gotoExtended(page, request, { waitUntil: 'networkidle' }); + + const { method, headers } = JSON.parse(await response!.text()); + expect(method).toBe('GET'); + expect(headers['user-agent']).toBe('Demo UA'); + expect(interceptionSpy).not.toHaveBeenCalled(); + expect(deprecatedSpy).not.toHaveBeenCalled(); + } finally { + await browser.close(); + } + }); + describe('infiniteScroll()', () => { function isAtBottom() { return window.innerHeight + window.pageYOffset >= document.body.offsetHeight;