From d806111b5c7f1230c237882814a910b18f4edc8e Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 27 Feb 2025 17:54:35 +0100 Subject: [PATCH 01/45] Re-org timing of stub injected file --- packages/plugins/injection/src/esbuild.ts | 9 ++++++--- packages/plugins/injection/src/xpack.ts | 13 +++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/plugins/injection/src/esbuild.ts b/packages/plugins/injection/src/esbuild.ts index 2097cf388..814f9dea0 100644 --- a/packages/plugins/injection/src/esbuild.ts +++ b/packages/plugins/injection/src/esbuild.ts @@ -57,9 +57,6 @@ export const getEsbuildPlugin = ( async () => { const content = getContentToInject(contentsToInject[InjectPosition.MIDDLE]); - // Safe to delete the temp file now, the hook will take over. - await rm(absoluteFilePath); - return { // We can't use an empty string otherwise esbuild will crash. contents: content || ' ', @@ -72,6 +69,12 @@ export const getEsbuildPlugin = ( // InjectPosition.START and InjectPosition.END onEnd(async (result) => { + // Safe to delete the temp file now. + // We wait the end of the build to avoid any sub-builds + // that would re-use the parent build's options (with our modified 'inject') + // to fail to resolve it. + await rm(absoluteFilePath); + if (!result.metafile) { log.warn('Missing metafile from build result.'); return; diff --git a/packages/plugins/injection/src/xpack.ts b/packages/plugins/injection/src/xpack.ts index e4b1c6071..6c2cc6d8e 100644 --- a/packages/plugins/injection/src/xpack.ts +++ b/packages/plugins/injection/src/xpack.ts @@ -43,9 +43,13 @@ export const getXpackPlugin = ); // NOTE: RSpack MAY try to resolve the entry points before the loader is ready. - // There must be some race condition around this, because it's not always the case. + // There must be some race condition around this, because it's not always failing. if (context.bundler.name === 'rspack') { outputFileSync(filePath, ''); + compiler.hooks.shutdown.tapPromise(PLUGIN_NAME, async () => { + // Delete the fake file we created. + await rm(filePath); + }); } // Handle the InjectPosition.MIDDLE. @@ -108,13 +112,6 @@ export const getXpackPlugin = await addInjections(log, toInject, contentsToInject, context.cwd); }); - if (context.bundler.name === 'rspack') { - compiler.hooks.done.tapPromise(PLUGIN_NAME, async () => { - // Delete the fake file we created. - await rm(filePath); - }); - } - // Handle the InjectPosition.START and InjectPosition.END. // This is a re-implementation of the BannerPlugin, // that is compatible with all versions of webpack and rspack, From a4a2613eae7ad90d28bc3768e76803332cfdee70 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 27 Feb 2025 17:59:37 +0100 Subject: [PATCH 02/45] Fix the prepare-link revert --- packages/tools/src/commands/prepare-link/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/tools/src/commands/prepare-link/index.ts b/packages/tools/src/commands/prepare-link/index.ts index 1d56bb2cb..ac9d56cbf 100644 --- a/packages/tools/src/commands/prepare-link/index.ts +++ b/packages/tools/src/commands/prepare-link/index.ts @@ -43,7 +43,10 @@ class PrepareLink extends Command { const pkgJsonPath = path.resolve(ROOT, pkg.location, 'package.json'); const pkgJson = require(pkgJsonPath); if (this.revert) { - pkgJson.exports = { '.': './src/index.ts' }; + pkgJson.exports = { + './dist/src': './dist/src/index.js', + '.': './src/index.ts', + }; } else { pkgJson.exports = pkgJson.publishConfig.exports; } From f3efb8b3796dee4c29ed5675f5b1edd8537b8268 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 14:56:00 +0100 Subject: [PATCH 03/45] Fix random rspack crashes --- packages/core/src/helpers.ts | 3 +++ packages/plugins/injection/src/xpack.ts | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 66fe10c75..68d2fa912 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -217,6 +217,9 @@ export const isInjectionFile = (filename: string) => filename.includes(INJECTED_ export const rm = async (dir: string) => { return fsp.rm(dir, { force: true, maxRetries: 3, recursive: true }); }; +export const rmSync = async (dir: string) => { + return fs.rmSync(dir, { force: true, maxRetries: 3, recursive: true }); +}; // Mkdir recursively. export const mkdir = async (dir: string) => { diff --git a/packages/plugins/injection/src/xpack.ts b/packages/plugins/injection/src/xpack.ts index 6c2cc6d8e..412ec7e52 100644 --- a/packages/plugins/injection/src/xpack.ts +++ b/packages/plugins/injection/src/xpack.ts @@ -3,7 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { INJECTED_FILE } from '@dd/core/constants'; -import { getUniqueId, outputFileSync, rm } from '@dd/core/helpers'; +import { getUniqueId, outputFileSync, rmSync } from '@dd/core/helpers'; import type { GlobalContext, Logger, PluginOptions, ToInjectItem } from '@dd/core/types'; import { InjectPosition } from '@dd/core/types'; import { createRequire } from 'module'; @@ -46,9 +46,12 @@ export const getXpackPlugin = // There must be some race condition around this, because it's not always failing. if (context.bundler.name === 'rspack') { outputFileSync(filePath, ''); - compiler.hooks.shutdown.tapPromise(PLUGIN_NAME, async () => { - // Delete the fake file we created. - await rm(filePath); + // WARNING: Can't use shutdown.tapPromise as it would randomly crash the process. + // An alternative can be to use both `done` and `failed` hooks. + // Seems to be fixed in rspack@1.2.* + compiler.hooks.shutdown.tap(PLUGIN_NAME, () => { + // Delete the file we created. + rmSync(filePath); }); } From e2f6d267677491dde3c0b71aa736796ce89eacbd Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 15:00:19 +0100 Subject: [PATCH 04/45] Slightly change esbuild stub file strategy - use a tmpdir() to store the stub file. - do not remove it at the end of the build. - change how we update the initialOptions. --- packages/plugins/injection/src/esbuild.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/plugins/injection/src/esbuild.ts b/packages/plugins/injection/src/esbuild.ts index 814f9dea0..629b09840 100644 --- a/packages/plugins/injection/src/esbuild.ts +++ b/packages/plugins/injection/src/esbuild.ts @@ -3,17 +3,20 @@ // Copyright 2019-Present Datadog, Inc. import { INJECTED_FILE } from '@dd/core/constants'; -import { getEsbuildEntries, getUniqueId, outputFile, rm } from '@dd/core/helpers'; +import { getEsbuildEntries, getUniqueId, outputFile } from '@dd/core/helpers'; import type { Logger, PluginOptions, GlobalContext, ResolvedEntry } from '@dd/core/types'; import { InjectPosition } from '@dd/core/types'; import { getAbsolutePath } from '@dd/internal-build-report-plugin/helpers'; -import fsp from 'fs/promises'; +import fs from 'fs'; +import os from 'os'; import path from 'path'; import { PLUGIN_NAME } from './constants'; import { getContentToInject } from './helpers'; import type { ContentsToInject } from './types'; +const fsp = fs.promises; + export const getEsbuildPlugin = ( log: Logger, context: GlobalContext, @@ -23,13 +26,15 @@ export const getEsbuildPlugin = ( const { onStart, onLoad, onEnd, esbuild, initialOptions } = build; const entries: ResolvedEntry[] = []; const filePath = `${getUniqueId()}.${InjectPosition.MIDDLE}.${INJECTED_FILE}.js`; - const absoluteFilePath = path.resolve(context.bundler.outDir, filePath); + const tmpDir = fs.realpathSync(os.tmpdir()); + const absoluteFilePath = path.resolve(tmpDir, filePath); const injectionRx = new RegExp(`${filePath}$`); // InjectPosition.MIDDLE // Inject the file in the build using the "inject" option. // NOTE: This is made "safer" for sub-builds by actually creating the file. - initialOptions.inject = initialOptions.inject || []; + const initialInject = initialOptions.inject; + initialOptions.inject = initialInject ? [...initialInject] : []; initialOptions.inject.push(absoluteFilePath); onStart(async () => { @@ -37,8 +42,7 @@ export const getEsbuildPlugin = ( entries.push(...(await getEsbuildEntries(build, context, log))); // Remove our injected file from the config, so we reduce our chances to leak our changes. - initialOptions.inject = - initialOptions.inject?.filter((file) => file !== absoluteFilePath) || []; + build.initialOptions.inject = initialInject; try { // Create the MIDDLE file because esbuild will crash if it doesn't exist. @@ -69,12 +73,6 @@ export const getEsbuildPlugin = ( // InjectPosition.START and InjectPosition.END onEnd(async (result) => { - // Safe to delete the temp file now. - // We wait the end of the build to avoid any sub-builds - // that would re-use the parent build's options (with our modified 'inject') - // to fail to resolve it. - await rm(absoluteFilePath); - if (!result.metafile) { log.warn('Missing metafile from build result.'); return; From 6b45388d1603143b717ce296f301f9807715a266 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 15:01:16 +0100 Subject: [PATCH 05/45] Remove unecessary complexity from runBundlers' isolated runs --- packages/tests/src/_jest/helpers/runBundlers.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/tests/src/_jest/helpers/runBundlers.ts b/packages/tests/src/_jest/helpers/runBundlers.ts index adf8ee003..299f6749d 100644 --- a/packages/tests/src/_jest/helpers/runBundlers.ts +++ b/packages/tests/src/_jest/helpers/runBundlers.ts @@ -223,7 +223,7 @@ export const runBundlers = async ( const errors: string[] = []; // Generate a seed to avoid collision of builds. - const seed: string = `${jest.getSeed()}.${getUniqueId()}`; + const seed: string = `${Math.abs(jest.getSeed())}.${getUniqueId()}`; const bundlersToRun = BUNDLERS.filter( (bundler) => !bundlers || bundlers.includes(bundler.name), @@ -237,14 +237,11 @@ export const runBundlers = async ( : bundlerOverrides || {}; const runBundlerFunction = async (bundler: Bundler) => { - const bundlerOverride = bundlerOverridesResolved[bundler.name] || {}; - - let result: Awaited>; - // Isolate each runs to avoid conflicts between tests. - await jest.isolateModulesAsync(async () => { - result = await bundler.run(workingDir, pluginOverrides, bundlerOverride); - }); - return result!; + return bundler.run( + workingDir, + pluginOverrides, + bundlerOverridesResolved[bundler.name] || {}, + ); }; // Run the bundlers sequentially to ease the resources usage. From d5001bf168de4311c9397a838fb6155fad570ec6 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 16:40:01 +0100 Subject: [PATCH 06/45] Add isXpack helper to detect webpack/rspack bundlers --- packages/core/src/helpers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 68d2fa912..14ca8c95a 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -13,6 +13,7 @@ import type { RequestInit } from 'undici-types'; import type { BuildReport, + BundlerFullName, Entry, File, GlobalContext, @@ -212,6 +213,10 @@ export const truncateString = ( // Is the file coming from the injection plugin? export const isInjectionFile = (filename: string) => filename.includes(INJECTED_FILE); +// From a bundler's name, is it part of the "xpack" family? +export const isXpack = (bundlerName: BundlerFullName) => + ['rspack', 'webpack4', 'webpack5', 'webpack'].includes(bundlerName); + // Replacing fs-extra with local helpers. // Delete folders recursively. export const rm = async (dir: string) => { From a02a3c409b8ba0d489a2fd096f790d4a39b55e31 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 16:43:44 +0100 Subject: [PATCH 07/45] Better and stricter separation of needs for the injection plugin --- packages/plugins/injection/src/esbuild.ts | 12 ++- packages/plugins/injection/src/index.ts | 92 +++++++++++------------ 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/packages/plugins/injection/src/esbuild.ts b/packages/plugins/injection/src/esbuild.ts index 629b09840..532ddcbe5 100644 --- a/packages/plugins/injection/src/esbuild.ts +++ b/packages/plugins/injection/src/esbuild.ts @@ -23,7 +23,7 @@ export const getEsbuildPlugin = ( contentsToInject: ContentsToInject, ): PluginOptions['esbuild'] => ({ setup(build) { - const { onStart, onLoad, onEnd, esbuild, initialOptions } = build; + const { onStart, onResolve, onLoad, onEnd, esbuild, initialOptions } = build; const entries: ResolvedEntry[] = []; const filePath = `${getUniqueId()}.${InjectPosition.MIDDLE}.${INJECTED_FILE}.js`; const tmpDir = fs.realpathSync(os.tmpdir()); @@ -53,6 +53,16 @@ export const getEsbuildPlugin = ( } }); + onResolve( + { + filter: injectionRx, + }, + async (args) => { + // Mark the file as being injected by us. + return { path: args.path, namespace: PLUGIN_NAME }; + }, + ); + onLoad( { filter: injectionRx, diff --git a/packages/plugins/injection/src/index.ts b/packages/plugins/injection/src/index.ts index 00e0f51e1..f5fd4357d 100644 --- a/packages/plugins/injection/src/index.ts +++ b/packages/plugins/injection/src/index.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getUniqueId, isInjectionFile } from '@dd/core/helpers'; +import { getUniqueId, isInjectionFile, isXpack } from '@dd/core/helpers'; import { InjectPosition, type GlobalContext, @@ -35,56 +35,50 @@ export const getInjectionPlugins = (bundler: any, context: GlobalContext): Plugi injections.set(getUniqueId(), item); }; - const plugins: PluginOptions[] = [ - { - name: PLUGIN_NAME, - enforce: 'post', - // Bundler specific part of the plugin. - // We use it to: - // - Inject the content in the right places, each bundler offers this differently. - esbuild: getEsbuildPlugin(log, context, contentsToInject), - webpack: getXpackPlugin(bundler, log, context, injections, contentsToInject), - rspack: getXpackPlugin(bundler, log, context, injections, contentsToInject), - rollup: getRollupPlugin(contentsToInject), - vite: { ...getRollupPlugin(contentsToInject), enforce: 'pre' }, - // Universal part of the plugin. - // We use it to: - // - Prepare the injections. - // - Handle the resolution of the injection file. - async buildStart() { - // In xpack, we need to prepare the injections before the build starts. - // So we do it in their specific plugin. - if (['webpack', 'rspack'].includes(context.bundler.name)) { - return; - } + const plugin: PluginOptions = { + name: PLUGIN_NAME, + enforce: 'post', + // Bundler specific part of the plugin. + // We use it to: + // - Inject the content in the right places, each bundler offers this differently. + esbuild: getEsbuildPlugin(log, context, contentsToInject), + webpack: getXpackPlugin(bundler, log, context, injections, contentsToInject), + rspack: getXpackPlugin(bundler, log, context, injections, contentsToInject), + rollup: getRollupPlugin(contentsToInject), + vite: { ...getRollupPlugin(contentsToInject), enforce: 'pre' }, + }; - // Prepare the injections. - await addInjections(log, injections, contentsToInject, context.cwd); - }, - async resolveId(source) { - if (isInjectionFile(source)) { - return { id: source }; - } + // We need to handle the resolution in xpack, + // and it's easier to use unplugin's hooks for it. + if (isXpack(context.bundler.fullName)) { + plugin.loadInclude = (id) => { + if (isInjectionFile(id)) { + // console.log('loadInclude', id); + return true; + } - return null; - }, - loadInclude(id) { - if (isInjectionFile(id)) { - return true; - } + return null; + }; - return null; - }, - load(id) { - if (isInjectionFile(id)) { - return { - code: getContentToInject(contentsToInject[InjectPosition.MIDDLE]), - }; - } - return null; - }, - }, - ]; + plugin.load = (id) => { + if (isInjectionFile(id)) { + // console.log('load', id); + return { + code: getContentToInject(contentsToInject[InjectPosition.MIDDLE]), + }; + } + return null; + }; + } else { + // In xpack, we need to prepare the injections BEFORE the build starts. + // Otherwise, the bundler doesn't have the content when it needs it. + // So we do it in their specific plugin. + // Here for all the other non-xpack bundlers. + plugin.buildStart = async () => { + // Prepare the injections. + await addInjections(log, injections, contentsToInject, context.cwd); + }; + } - return plugins; + return [plugin]; }; From 6aa1fe835c0d5320da8c2af54deb897ceb239a74 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 16:45:41 +0100 Subject: [PATCH 08/45] Widen the file stub solution for xpack to webpack too. --- packages/plugins/injection/src/xpack.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/plugins/injection/src/xpack.ts b/packages/plugins/injection/src/xpack.ts index 412ec7e52..353b7f97f 100644 --- a/packages/plugins/injection/src/xpack.ts +++ b/packages/plugins/injection/src/xpack.ts @@ -44,15 +44,20 @@ export const getXpackPlugin = // NOTE: RSpack MAY try to resolve the entry points before the loader is ready. // There must be some race condition around this, because it's not always failing. - if (context.bundler.name === 'rspack') { - outputFileSync(filePath, ''); - // WARNING: Can't use shutdown.tapPromise as it would randomly crash the process. - // An alternative can be to use both `done` and `failed` hooks. - // Seems to be fixed in rspack@1.2.* - compiler.hooks.shutdown.tap(PLUGIN_NAME, () => { - // Delete the file we created. - rmSync(filePath); - }); + outputFileSync(filePath, ''); + // WARNING: Can't use shutdown.tapPromise as rspack would randomly crash the process. + // Seems to be fixed in rspack@1.2.* + // We also do it for webpack, as it fixes some resolution edge cases. + const hookFn = () => { + // Delete the file we created. + rmSync(filePath); + }; + // Webpack4 doesn't have the "shutdown" hook. + if (compiler.hooks.shutdown) { + compiler.hooks.shutdown.tap(PLUGIN_NAME, hookFn); + } else { + compiler.hooks.done.tap(PLUGIN_NAME, hookFn); + compiler.hooks.failed.tap(PLUGIN_NAME, hookFn); } // Handle the InjectPosition.MIDDLE. From 6a7fb2a44d866058fa28f7417a0ccbade7df7367 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 16:46:24 +0100 Subject: [PATCH 09/45] Small inconsequential diff --- packages/published/webpack-plugin/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index f4a189514..a1a73e26a 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -22,7 +22,7 @@ "module": "./dist/src/index.mjs", "types": "./dist/src/index.d.ts", "exports": { - "./dist/src/*": "./dist/src/*", + "./dist/src": "./dist/src/index.js", ".": "./src/index.ts" }, "publishConfig": { From fc0df3b9670e98514299e2069a833efd3826db81 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 16:51:29 +0100 Subject: [PATCH 10/45] Fix analytics --- packages/factory/src/helpers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/factory/src/helpers.ts b/packages/factory/src/helpers.ts index 2acd8e11c..ec532e5a3 100644 --- a/packages/factory/src/helpers.ts +++ b/packages/factory/src/helpers.ts @@ -118,7 +118,9 @@ export const getContext = ({ }, }; - const passedEnv: Env = (process.env.BUILD_PLUGINS_ENV as Env) || 'development'; + // Use "production" if there is no env passed. + const passedEnv: Env = (process.env.BUILD_PLUGINS_ENV as Env) || 'production'; + // Fallback to "development" if the passed env is wrong. const env: Env = ALL_ENVS.includes(passedEnv) ? passedEnv : 'development'; const context: GlobalContext = { auth: options.auth, From 5b58a980adb4859ec42dd6fee6b058524cea365c Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 18:15:11 +0100 Subject: [PATCH 11/45] Upload artifacts only when failed. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 67bde95ff..fdc77c175 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: key: cache-playwright-binaries-${{ hashFiles('yarn.lock') }} - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() && failure() }} + if: ${{ failure() }} with: name: playwright path: | From c89d1f3f6e63f1412f35432905209d30f649110d Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 28 Feb 2025 18:15:25 +0100 Subject: [PATCH 12/45] Fix some exports values --- packages/plugins/error-tracking/package.json | 1 + packages/plugins/rum/package.json | 1 + packages/plugins/telemetry/package.json | 1 + packages/published/esbuild-plugin/package.json | 6 ++++-- packages/published/rollup-plugin/package.json | 6 ++++-- packages/published/rspack-plugin/package.json | 6 ++++-- packages/published/vite-plugin/package.json | 6 ++++-- packages/published/webpack-plugin/package.json | 6 ++++-- packages/tools/src/commands/prepare-link/index.ts | 1 + 9 files changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/plugins/error-tracking/package.json b/packages/plugins/error-tracking/package.json index 2c19e7e22..1b89e2788 100644 --- a/packages/plugins/error-tracking/package.json +++ b/packages/plugins/error-tracking/package.json @@ -11,6 +11,7 @@ "url": "https://github.com/DataDog/build-plugins", "directory": "packages/plugins/error-tracking" }, + "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./sourcemaps/*": "./src/sourcemaps/*.ts", diff --git a/packages/plugins/rum/package.json b/packages/plugins/rum/package.json index 31d58bbb4..1249b100f 100644 --- a/packages/plugins/rum/package.json +++ b/packages/plugins/rum/package.json @@ -16,6 +16,7 @@ "entry": "./src/built/rum-browser-sdk.ts" } }, + "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./*": "./src/*.ts" diff --git a/packages/plugins/telemetry/package.json b/packages/plugins/telemetry/package.json index 5b63009f1..60084bbbe 100644 --- a/packages/plugins/telemetry/package.json +++ b/packages/plugins/telemetry/package.json @@ -11,6 +11,7 @@ "url": "https://github.com/DataDog/build-plugins", "directory": "packages/plugins/telemetry" }, + "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./esbuild-plugin/*": "./src/esbuild-plugin/*.ts", diff --git a/packages/published/esbuild-plugin/package.json b/packages/published/esbuild-plugin/package.json index fd291184d..67ecf6b4d 100644 --- a/packages/published/esbuild-plugin/package.json +++ b/packages/published/esbuild-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/esbuild-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.0", + "version": "2.5.1-dev-3", "license": "MIT", "author": "Datadog", "description": "Datadog ESBuild Plugin", @@ -23,6 +23,7 @@ "types": "./dist/src/index.d.ts", "exports": { "./dist/src": "./dist/src/index.js", + "./dist/src/*": "./dist/src/*", ".": "./src/index.ts" }, "publishConfig": { @@ -79,5 +80,6 @@ }, "peerDependencies": { "esbuild": ">=0.x" - } + }, + "stableVersion": "2.5.0" } diff --git a/packages/published/rollup-plugin/package.json b/packages/published/rollup-plugin/package.json index 9a88e5594..db6b0dc21 100644 --- a/packages/published/rollup-plugin/package.json +++ b/packages/published/rollup-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/rollup-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.0", + "version": "2.5.1-dev-3", "license": "MIT", "author": "Datadog", "description": "Datadog Rollup Plugin", @@ -23,6 +23,7 @@ "types": "./dist/src/index.d.ts", "exports": { "./dist/src": "./dist/src/index.js", + "./dist/src/*": "./dist/src/*", ".": "./src/index.ts" }, "publishConfig": { @@ -79,5 +80,6 @@ }, "peerDependencies": { "rollup": ">= 3.x < 5.x" - } + }, + "stableVersion": "2.5.0" } diff --git a/packages/published/rspack-plugin/package.json b/packages/published/rspack-plugin/package.json index ad3fcf395..b429a3f09 100644 --- a/packages/published/rspack-plugin/package.json +++ b/packages/published/rspack-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/rspack-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.0", + "version": "2.5.1-dev-3", "license": "MIT", "author": "Datadog", "description": "Datadog Rspack Plugin", @@ -23,6 +23,7 @@ "types": "./dist/src/index.d.ts", "exports": { "./dist/src": "./dist/src/index.js", + "./dist/src/*": "./dist/src/*", ".": "./src/index.ts" }, "publishConfig": { @@ -79,5 +80,6 @@ }, "peerDependencies": { "@rspack/core": "1.x" - } + }, + "stableVersion": "2.5.0" } diff --git a/packages/published/vite-plugin/package.json b/packages/published/vite-plugin/package.json index 621fbca5a..c48da8149 100644 --- a/packages/published/vite-plugin/package.json +++ b/packages/published/vite-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/vite-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.0", + "version": "2.5.1-dev-3", "license": "MIT", "author": "Datadog", "description": "Datadog Vite Plugin", @@ -23,6 +23,7 @@ "types": "./dist/src/index.d.ts", "exports": { "./dist/src": "./dist/src/index.js", + "./dist/src/*": "./dist/src/*", ".": "./src/index.ts" }, "publishConfig": { @@ -79,5 +80,6 @@ }, "peerDependencies": { "vite": "5.x" - } + }, + "stableVersion": "2.5.0" } diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index a1a73e26a..60909ca92 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/webpack-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.0", + "version": "2.5.1-dev-3", "license": "MIT", "author": "Datadog", "description": "Datadog Webpack Plugin", @@ -23,6 +23,7 @@ "types": "./dist/src/index.d.ts", "exports": { "./dist/src": "./dist/src/index.js", + "./dist/src/*": "./dist/src/*", ".": "./src/index.ts" }, "publishConfig": { @@ -79,5 +80,6 @@ }, "peerDependencies": { "webpack": ">= 4.x < 6.x" - } + }, + "stableVersion": "2.5.0" } diff --git a/packages/tools/src/commands/prepare-link/index.ts b/packages/tools/src/commands/prepare-link/index.ts index ac9d56cbf..4c7c65347 100644 --- a/packages/tools/src/commands/prepare-link/index.ts +++ b/packages/tools/src/commands/prepare-link/index.ts @@ -45,6 +45,7 @@ class PrepareLink extends Command { if (this.revert) { pkgJson.exports = { './dist/src': './dist/src/index.js', + './dist/src/*': './dist/src/*', '.': './src/index.ts', }; } else { From 3563c7e793a88bd8aae2154ecaab0dd33a8c3344 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 3 Mar 2025 09:52:58 +0100 Subject: [PATCH 13/45] Increase e2e timeout --- packages/tests/playwright.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tests/playwright.config.ts b/packages/tests/playwright.config.ts index 4251ae213..e2291d356 100644 --- a/packages/tests/playwright.config.ts +++ b/packages/tests/playwright.config.ts @@ -28,7 +28,7 @@ export default defineConfig({ bundlers: getRequestedBundlers(), trace: 'retain-on-failure', }, - timeout: 5_000, + timeout: 60_000, /* Configure projects for each bundler */ // TODO Also build and test for ESM. projects: FULL_NAME_BUNDLERS.map((bundler) => [ From 5796ea2f50ad05ffc1b899f6bca21851d72eaf5a Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 3 Mar 2025 10:07:32 +0100 Subject: [PATCH 14/45] Add global timeout for CI --- packages/tests/playwright.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/tests/playwright.config.ts b/packages/tests/playwright.config.ts index e2291d356..41dcb08fc 100644 --- a/packages/tests/playwright.config.ts +++ b/packages/tests/playwright.config.ts @@ -28,6 +28,7 @@ export default defineConfig({ bundlers: getRequestedBundlers(), trace: 'retain-on-failure', }, + globalTimeout: process.env.CI ? 20 * 60 * 1000 : undefined, timeout: 60_000, /* Configure projects for each bundler */ // TODO Also build and test for ESM. From c9119bc11cf64aaa11822153e57d68b8c552ee1e Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 3 Mar 2025 10:12:47 +0100 Subject: [PATCH 15/45] Add playwright alias --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index af67c1ae1..4bad748ad 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "loop-published": "yarn workspaces foreach -A --include \"@datadog/*\" --exclude \"@datadog/build-plugins\"", "loop": "yarn loop-published -pti", "oss": "yarn cli oss -d packages -l mit", + "playwright": "yarn workspace @dd/tests playwright", "publish:all": "yarn loop --no-private npm publish", "typecheck:all": "yarn workspaces foreach -Apti run typecheck", "version:all": "yarn loop-published version ${0} --immediate", From e2b7f25137ba8fa2805c204ca7c77b48da9726a9 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 3 Mar 2025 15:00:00 +0100 Subject: [PATCH 16/45] Fix package.json breaking the bundled types --- packages/plugins/error-tracking/package.json | 1 - packages/plugins/rum/package.json | 1 - packages/plugins/telemetry/package.json | 1 - 3 files changed, 3 deletions(-) diff --git a/packages/plugins/error-tracking/package.json b/packages/plugins/error-tracking/package.json index 1b89e2788..2c19e7e22 100644 --- a/packages/plugins/error-tracking/package.json +++ b/packages/plugins/error-tracking/package.json @@ -11,7 +11,6 @@ "url": "https://github.com/DataDog/build-plugins", "directory": "packages/plugins/error-tracking" }, - "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./sourcemaps/*": "./src/sourcemaps/*.ts", diff --git a/packages/plugins/rum/package.json b/packages/plugins/rum/package.json index 1249b100f..31d58bbb4 100644 --- a/packages/plugins/rum/package.json +++ b/packages/plugins/rum/package.json @@ -16,7 +16,6 @@ "entry": "./src/built/rum-browser-sdk.ts" } }, - "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./*": "./src/*.ts" diff --git a/packages/plugins/telemetry/package.json b/packages/plugins/telemetry/package.json index 60084bbbe..5b63009f1 100644 --- a/packages/plugins/telemetry/package.json +++ b/packages/plugins/telemetry/package.json @@ -11,7 +11,6 @@ "url": "https://github.com/DataDog/build-plugins", "directory": "packages/plugins/telemetry" }, - "types": "./src/index.ts", "exports": { ".": "./src/index.ts", "./esbuild-plugin/*": "./src/esbuild-plugin/*.ts", From 6c717e7c55a724b2eebd11e36c6837fd27335ad0 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 4 Mar 2025 11:15:26 +0100 Subject: [PATCH 17/45] Handle 0ms in formatDuration --- packages/core/src/helpers.ts | 9 ++++++--- packages/tests/src/unit/core/helpers.test.ts | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 14ca8c95a..c430527bc 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -37,9 +37,12 @@ export const formatDuration = (duration: number) => { const minutes = d.getUTCMinutes(); const seconds = d.getUTCSeconds(); const milliseconds = d.getUTCMilliseconds(); - return `${days ? `${days}d ` : ''}${hours ? `${hours}h ` : ''}${minutes ? `${minutes}m ` : ''}${ - seconds ? `${seconds}s ` : '' - }${milliseconds ? `${milliseconds}ms` : ''}`.trim(); + const timeString = + `${days ? `${days}d ` : ''}${hours ? `${hours}h ` : ''}${minutes ? `${minutes}m ` : ''}${ + seconds ? `${seconds}s` : '' + }`.trim(); + // Split here so we can show 0ms in case we have a duration of 0. + return `${timeString}${!timeString || milliseconds ? ` ${milliseconds}ms` : ''}`.trim(); }; // https://esbuild.github.io/api/#glob-style-entry-points diff --git a/packages/tests/src/unit/core/helpers.test.ts b/packages/tests/src/unit/core/helpers.test.ts index 645c46633..ad929b0e9 100644 --- a/packages/tests/src/unit/core/helpers.test.ts +++ b/packages/tests/src/unit/core/helpers.test.ts @@ -25,10 +25,15 @@ jest.mock('fs', () => require('memfs').fs); describe('Core Helpers', () => { describe('formatDuration', () => { test.each([ + [0, '0ms'], [10, '10ms'], + [10000, '10s'], [10010, '10s 10ms'], + [1000000, '16m 40s'], [1000010, '16m 40s 10ms'], + [10000000, '2h 46m 40s'], [10000010, '2h 46m 40s 10ms'], + [1000000000, '11d 13h 46m 40s'], [1000000010, '11d 13h 46m 40s 10ms'], ])('Should format duration %s => %s', async (ms, expected) => { const { formatDuration } = await import('@dd/core/helpers'); From 9f5603d709651c89fa1b1ed52a4c0c6482997133 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 4 Mar 2025 11:16:52 +0100 Subject: [PATCH 18/45] Add time logger --- packages/core/src/types.ts | 2 ++ packages/factory/src/helpers.ts | 18 ++++++++++++ packages/tests/src/_jest/helpers/mocks.ts | 2 ++ .../tests/src/unit/factory/helpers.test.ts | 29 +++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index b8600142d..22008bf83 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -99,6 +99,8 @@ export type ToInjectItem = { export type GetLogger = (name: string) => Logger; export type Logger = { getLogger: GetLogger; + time: (label: string, level?: LogLevel) => () => void; + timeEnd: (label: string, level?: LogLevel) => void; error: (text: any) => void; warn: (text: any) => void; info: (text: any) => void; diff --git a/packages/factory/src/helpers.ts b/packages/factory/src/helpers.ts index ec532e5a3..b0ca28d92 100644 --- a/packages/factory/src/helpers.ts +++ b/packages/factory/src/helpers.ts @@ -3,6 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { ALL_ENVS } from '@dd/core/constants'; +import { formatDuration } from '@dd/core/helpers'; import type { BuildReport, BundlerFullName, @@ -12,6 +13,7 @@ import type { GetLogger, GlobalContext, LogLevel, + Logger, Options, OptionsWithDefaults, } from '@dd/core/types'; @@ -81,11 +83,27 @@ export const getLoggerFactory = } }; + const timerMap = new Map(); + const timeEnd: Logger['timeEnd'] = (label: string, level: LogLevel = 'debug') => { + const start = timerMap.get(label); + if (start) { + timerMap.delete(label); + const duration = Date.now() - start.timestamp; + log(`[${c.cyan(label)}] : ${c.cyan(formatDuration(duration))}`, level); + } + }; + const time: Logger['time'] = (label: string, level: LogLevel = 'debug') => { + timerMap.set(label, { timestamp: Date.now(), logLevel: level }); + return () => timeEnd(label, level); + }; + return { getLogger: (subName: string) => { const logger = getLoggerFactory(build, logLevel); return logger(`${cleanedName}${NAME_SEP}${subName}`); }, + time, + timeEnd, error: (text: any) => log(text, 'error'), warn: (text: any) => log(text, 'warn'), info: (text: any) => log(text, 'info'), diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index 955d2d038..cbc15097a 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -40,6 +40,8 @@ export const defaultPluginOptions: GetPluginsOptions = { export const mockLogFn = jest.fn((text: any, level: LogLevel) => {}); const logFn: Logger = { getLogger: jest.fn(), + time: jest.fn(), + timeEnd: jest.fn(), error: (text: any) => { mockLogFn(text, 'error'); }, diff --git a/packages/tests/src/unit/factory/helpers.test.ts b/packages/tests/src/unit/factory/helpers.test.ts index e3c8990d5..c5cd44c75 100644 --- a/packages/tests/src/unit/factory/helpers.test.ts +++ b/packages/tests/src/unit/factory/helpers.test.ts @@ -155,6 +155,35 @@ describe('Factory Helpers', () => { }); }); + describe('Time logger', () => { + test('Should log a duration.', () => { + const [logger] = setupLogger('testLogger'); + // Basic usage. + logger.time('test time 1'); + logger.timeEnd('test time 1'); + + // Using the return timeEnd function. + const timeEnd = logger.time('test time 2'); + timeEnd(); + + // Use a specific log level. + const timeEnd2 = logger.time('test time 3', 'error'); + timeEnd2(); + + expect(logMock).toHaveBeenCalledTimes(2); + expect(errorMock).toHaveBeenCalledTimes(1); + expect(getOutput(logMock, 0)).toBe( + `[debug|esbuild|testLogger] [test time 1] : 0ms`, + ); + expect(getOutput(logMock, 1)).toBe( + `[debug|esbuild|testLogger] [test time 2] : 0ms`, + ); + expect(getOutput(errorMock, 0)).toBe( + `[error|esbuild|testLogger] [test time 3] : 0ms`, + ); + }); + }); + describe('Sub logger', () => { test('Should return a logger factory.', () => { const [logger] = setupLogger('testLogger'); From 50deda4e820b4d036773ac4f4c5dab7f20c69a13 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 4 Mar 2025 11:19:30 +0100 Subject: [PATCH 19/45] Add time debugs in some places --- packages/plugins/build-report/src/esbuild.ts | 16 ++++++++++++++++ .../telemetry/src/esbuild-plugin/index.ts | 4 ++++ packages/plugins/telemetry/src/index.ts | 13 ++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index 041038ad4..441617c54 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -31,6 +31,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const resolvedEntries: ResolvedEntry[] = []; build.onStart(async () => { + const timeEnd = log.time('process entries'); // Store entry names based on the configuration. resolvedEntries.push(...(await getEsbuildEntries(build, context, log))); for (const entry of resolvedEntries) { @@ -41,9 +42,11 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt entryNames.set(cleanedName, cleanedName); } } + timeEnd(); }); build.onEnd((result) => { + const collectTimeEnd = log.time('collecting errors and warnings'); const cwd = context.cwd; for (const error of result.errors) { context.build.errors.push(error.text); @@ -51,6 +54,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt for (const warning of result.warnings) { context.build.warnings.push(warning.text); } + collectTimeEnd(); if (!result.metafile) { const warning = 'Missing metafile from build report.'; @@ -68,8 +72,10 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const reportInputsIndexed: Record = {}; const reportOutputsIndexed: Record = {}; + const indexTimeEnd = log.time('indexing metafile data'); const metaInputsIndexed = reIndexMeta(result.metafile.inputs, cwd); const metaOutputsIndexed = reIndexMeta(result.metafile.outputs, cwd); + indexTimeEnd(); // From a proxy entry point, created by our injection plugin, get the real path. const getRealPathFromInjectionProxy = (entryPoint: string): string => { @@ -94,6 +100,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt }; // Loop through inputs. + const inputsTimeEnd = log.time('looping through inputs'); for (const [filename, input] of Object.entries(result.metafile.inputs)) { if (isInjectionFile(filename)) { continue; @@ -113,8 +120,10 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt reportInputsIndexed[filepath] = file; inputs.push(file); } + inputsTimeEnd(); // Loop through outputs. + const outputTimeEnd = log.time('looping through outputs'); for (const [filename, output] of Object.entries(result.metafile.outputs)) { const fullPath = getAbsolutePath(cwd, filename); const cleanedName = cleanName(context, fullPath); @@ -193,8 +202,10 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt tempEntryFiles.push(entry); } } + outputTimeEnd(); // Loop through sourcemaps. + const sourcemapsTimeEnd = log.time('looping through sourcemaps'); for (const sourcemap of tempSourcemaps) { const outputFilepath = sourcemap.filepath.replace(/\.map$/, ''); const foundOutput = reportOutputsIndexed[outputFilepath]; @@ -206,6 +217,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt sourcemap.inputs.push(foundOutput); } + sourcemapsTimeEnd(); // Build our references for the entries. const references = { @@ -272,6 +284,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt }; // Loop through entries. + const entriesTimeEnd = log.time('looping through entries'); // TODO This is slightly underperformant due to getAllImports' recursivity. for (const entryFile of tempEntryFiles) { const entryInputs: Record = {}; @@ -300,8 +313,10 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt entries.push(entryFile); } + entriesTimeEnd(); // Loop through all inputs to aggregate dependencies and dependents. + const depsTimeEnd = log.time('aggregate dependencies and dependents'); for (const input of inputs) { const metaFile = references.inputs.meta[input.filepath]; if (!metaFile) { @@ -326,6 +341,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt dependencyFile.dependents.add(input); } } + depsTimeEnd(); context.build.outputs = outputs; context.build.inputs = inputs; diff --git a/packages/plugins/telemetry/src/esbuild-plugin/index.ts b/packages/plugins/telemetry/src/esbuild-plugin/index.ts index b617b80ba..1d873a0a2 100644 --- a/packages/plugins/telemetry/src/esbuild-plugin/index.ts +++ b/packages/plugins/telemetry/src/esbuild-plugin/index.ts @@ -19,14 +19,18 @@ export const getEsbuildPlugin = ( // We force esbuild to produce its metafile. build.initialOptions.metafile = true; + const wrapTimeEnd = logger.time('wrapping plugins'); wrapPlugins(build, globalContext.cwd); + wrapTimeEnd(); build.onEnd(async (result: BuildResult) => { if (!result.metafile) { logger.warn("Missing metafile, can't proceed with modules data."); return; } + const resultTimeEnd = logger.time('getting plugins results'); const { plugins, modules } = getPluginsResults(); + resultTimeEnd(); bundlerContext.report = { timings: { diff --git a/packages/plugins/telemetry/src/index.ts b/packages/plugins/telemetry/src/index.ts index 7df11edde..56578c81c 100644 --- a/packages/plugins/telemetry/src/index.ts +++ b/packages/plugins/telemetry/src/index.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import type { GlobalContext, GetPlugins, PluginOptions } from '@dd/core/types'; +import type { GlobalContext, GetPlugins, PluginOptions, Logger } from '@dd/core/types'; import { addMetrics } from './common/aggregator'; import { defaultFilters } from './common/filters'; @@ -56,14 +56,17 @@ export const getPlugins: GetPlugins = ( webpack: getWebpackPlugin(bundlerContext, context), rspack: getWebpackPlugin(bundlerContext, context), }; + let buildTimeEnd: ReturnType; // Universal plugin. const universalPlugin: PluginOptions = { name: 'datadog-universal-telemetry-plugin', enforce: 'post', buildStart() { + buildTimeEnd = log.time('build'); context.build.start = context.build.start || Date.now(); }, buildEnd() { + buildTimeEnd(); realBuildEnd = Date.now(); }, @@ -76,22 +79,30 @@ export const getPlugins: GetPlugins = ( const metrics: Set = new Set(); const optionsDD = getOptionsDD(telemetryOptions); + const metricsTimeEnd = log.time(`adding metrics`); addMetrics(context, optionsDD, metrics, bundlerContext.report); + metricsTimeEnd(); // TODO Extract the files output in an internal plugin. + const writeTimeEnd = log.time(`writing to files`); await outputFiles( { report: bundlerContext.report, metrics }, telemetryOptions.output, log, context.bundler.outDir, ); + writeTimeEnd(); + const reportTimeEnd = log.time('outputing report'); outputTexts(context, log, bundlerContext.report); + reportTimeEnd(); + const sendTimeEnd = log.time('sending metrics to Datadog.'); await sendMetrics( metrics, { apiKey: context.auth?.apiKey, endPoint: telemetryOptions.endPoint }, log, ); + sendTimeEnd(); }, }; From 370674d0b8901618f431e3a4dd553a1d24c4f207 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 4 Mar 2025 14:56:15 +0100 Subject: [PATCH 20/45] Revert version bump --- packages/published/esbuild-plugin/package.json | 5 ++--- packages/published/rollup-plugin/package.json | 5 ++--- packages/published/rspack-plugin/package.json | 5 ++--- packages/published/vite-plugin/package.json | 5 ++--- packages/published/webpack-plugin/package.json | 5 ++--- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/packages/published/esbuild-plugin/package.json b/packages/published/esbuild-plugin/package.json index 67ecf6b4d..334ee18f9 100644 --- a/packages/published/esbuild-plugin/package.json +++ b/packages/published/esbuild-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/esbuild-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.1-dev-3", + "version": "2.5.0", "license": "MIT", "author": "Datadog", "description": "Datadog ESBuild Plugin", @@ -80,6 +80,5 @@ }, "peerDependencies": { "esbuild": ">=0.x" - }, - "stableVersion": "2.5.0" + } } diff --git a/packages/published/rollup-plugin/package.json b/packages/published/rollup-plugin/package.json index db6b0dc21..32229b937 100644 --- a/packages/published/rollup-plugin/package.json +++ b/packages/published/rollup-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/rollup-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.1-dev-3", + "version": "2.5.0", "license": "MIT", "author": "Datadog", "description": "Datadog Rollup Plugin", @@ -80,6 +80,5 @@ }, "peerDependencies": { "rollup": ">= 3.x < 5.x" - }, - "stableVersion": "2.5.0" + } } diff --git a/packages/published/rspack-plugin/package.json b/packages/published/rspack-plugin/package.json index b429a3f09..82514d44d 100644 --- a/packages/published/rspack-plugin/package.json +++ b/packages/published/rspack-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/rspack-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.1-dev-3", + "version": "2.5.0", "license": "MIT", "author": "Datadog", "description": "Datadog Rspack Plugin", @@ -80,6 +80,5 @@ }, "peerDependencies": { "@rspack/core": "1.x" - }, - "stableVersion": "2.5.0" + } } diff --git a/packages/published/vite-plugin/package.json b/packages/published/vite-plugin/package.json index c48da8149..4e5e3cf33 100644 --- a/packages/published/vite-plugin/package.json +++ b/packages/published/vite-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/vite-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.1-dev-3", + "version": "2.5.0", "license": "MIT", "author": "Datadog", "description": "Datadog Vite Plugin", @@ -80,6 +80,5 @@ }, "peerDependencies": { "vite": "5.x" - }, - "stableVersion": "2.5.0" + } } diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index 60909ca92..0f7d1b404 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -1,7 +1,7 @@ { "name": "@datadog/webpack-plugin", "packageManager": "yarn@4.0.2", - "version": "2.5.1-dev-3", + "version": "2.5.0", "license": "MIT", "author": "Datadog", "description": "Datadog Webpack Plugin", @@ -80,6 +80,5 @@ }, "peerDependencies": { "webpack": ">= 4.x < 6.x" - }, - "stableVersion": "2.5.0" + } } From 08fc2f241bbcfa33712fac6cd4ba6976a9b1295c Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 5 Mar 2025 11:55:03 +0100 Subject: [PATCH 21/45] Slight update on metrics sending logs --- packages/plugins/telemetry/src/common/sender.ts | 15 +++++---------- packages/plugins/telemetry/src/index.ts | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/plugins/telemetry/src/common/sender.ts b/packages/plugins/telemetry/src/common/sender.ts index bef00e4c4..8b0b58685 100644 --- a/packages/plugins/telemetry/src/common/sender.ts +++ b/packages/plugins/telemetry/src/common/sender.ts @@ -2,16 +2,15 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { doRequest, formatDuration } from '@dd/core/helpers'; +import { doRequest } from '@dd/core/helpers'; import type { Logger } from '@dd/core/types'; import type { MetricToSend } from '@dd/telemetry-plugin/types'; -export const sendMetrics = ( +export const sendMetrics = async ( metrics: Set, auth: { apiKey?: string; endPoint: string }, log: Logger, ) => { - const startSending = Date.now(); if (!auth.apiKey) { log.info(`Won't send metrics to Datadog: missing API Key.`); return; @@ -46,11 +45,7 @@ Metrics: series: MetricToSend[]; }), }), - }) - .then(() => { - log.debug(`Sent metrics in ${formatDuration(Date.now() - startSending)}.`); - }) - .catch((e) => { - log.error(`Error sending metrics ${e}`); - }); + }).catch((e) => { + log.error(`Error sending metrics ${e}`); + }); }; diff --git a/packages/plugins/telemetry/src/index.ts b/packages/plugins/telemetry/src/index.ts index 56578c81c..f9686f8aa 100644 --- a/packages/plugins/telemetry/src/index.ts +++ b/packages/plugins/telemetry/src/index.ts @@ -96,7 +96,7 @@ export const getPlugins: GetPlugins = ( outputTexts(context, log, bundlerContext.report); reportTimeEnd(); - const sendTimeEnd = log.time('sending metrics to Datadog.'); + const sendTimeEnd = log.time('sending metrics to Datadog'); await sendMetrics( metrics, { apiKey: context.auth?.apiKey, endPoint: telemetryOptions.endPoint }, From 68075c8dcb2bc6c06645156648896b17037ca995 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 6 Mar 2025 15:31:30 +0100 Subject: [PATCH 22/45] Fix esbuild report on external dependencies --- packages/plugins/build-report/src/esbuild.ts | 61 ++++++++++++++++++-- packages/plugins/build-report/src/helpers.ts | 2 +- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index 441617c54..a075e8201 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -12,6 +12,7 @@ import type { PluginOptions, ResolvedEntry, } from '@dd/core/types'; +import path from 'path'; import { cleanName, getAbsolutePath, getType } from './helpers'; @@ -147,7 +148,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt // It has no inputs, but still relates to its entryPoint. if (output.entryPoint && !inputFiles.length) { const inputFound = - reportInputsIndexed[getAbsolutePath(cwd, output.entryPoint!)]; + reportInputsIndexed[getAbsolutePath(cwd, output.entryPoint)]; if (!inputFound) { log.debug( `Input ${output.entryPoint} not found for output ${cleanedName}`, @@ -275,9 +276,36 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt } for (const imported of metaFile.imports) { - const importPath = getAbsolutePath(cwd, imported.path); + const isRelative = imported.path.match(/^\.\.?\//); + const root = isRelative ? path.dirname(filePath) : cwd; + const absoluteImportPath = getAbsolutePath(root, imported.path); + + if (imported.external) { + if ( + isFileSupported(imported.path) && + !references.inputs.report[imported.path] + ) { + // If it's an absolute external import, we can't trust our own getAbsolutePath(). + // We can't know what its "root" could be. + const filepath = isRelative ? absoluteImportPath : imported.path; + // But we can still add it to the report. + const inputFile: Input = { + filepath, + name: cleanName(context, imported.path), + size: 0, + type: 'external', + dependencies: new Set(), + dependents: new Set(), + }; + references.inputs.report[filepath] = inputFile; + inputs.push(inputFile); + } + // We can't follow external imports. + continue; + } + // Look for the other inputs. - getAllImports(importPath, ref, allImports); + getAllImports(absoluteImportPath, ref, allImports); } return allImports; @@ -318,6 +346,12 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt // Loop through all inputs to aggregate dependencies and dependents. const depsTimeEnd = log.time('aggregate dependencies and dependents'); for (const input of inputs) { + // The metafile does not contain external dependencies. + // So we can only fill in their dependents. + if (input.type === 'external') { + continue; + } + const metaFile = references.inputs.meta[input.filepath]; if (!metaFile) { log.debug(`Could not find metafile's ${input.name}`); @@ -328,11 +362,26 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt if (!isFileSupported(dependency.path)) { continue; } - const dependencyPath = getAbsolutePath(cwd, dependency.path); - const dependencyFile = references.inputs.report[dependencyPath]; + + const isRelative = dependency.path.match(/^\.?\.\//); + const root = isRelative ? path.dirname(input.filepath) : cwd; + const absoluteDependencyPath = getAbsolutePath(root, dependency.path); + + let dependencyFile: Input | undefined; + if (dependency.external) { + // If it's an absolute external import, we can't trust our own getAbsolutePath(). + // We can't know what its "root" could be. + const filepath = isRelative ? absoluteDependencyPath : dependency.path; + // In case of externals, we use their path directly. + dependencyFile = references.inputs.report[filepath]; + } else { + dependencyFile = references.inputs.report[absoluteDependencyPath]; + } if (!dependencyFile) { - log.debug(`Could not find input file of ${dependency.path}`); + log.debug( + `Could not find input file of ${dependency.path} imported from ${input.name}`, + ); continue; } diff --git a/packages/plugins/build-report/src/helpers.ts b/packages/plugins/build-report/src/helpers.ts index aeaee5dcb..19ff8925f 100644 --- a/packages/plugins/build-report/src/helpers.ts +++ b/packages/plugins/build-report/src/helpers.ts @@ -88,7 +88,7 @@ export const getAbsolutePath = (cwd: string, filepath: string) => { return INJECTED_FILE; } - if (filepath.startsWith(cwd)) { + if (filepath.startsWith(cwd) || path.isAbsolute(filepath)) { return filepath; } return path.resolve(cwd, filepath); From 95770a64d1c2995791b9622e217dc350c4dd35b0 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 6 Mar 2025 15:36:35 +0100 Subject: [PATCH 23/45] Add more time debug --- packages/plugins/build-report/src/esbuild.ts | 2 ++ packages/plugins/build-report/src/rollup.ts | 12 ++++++++++++ packages/plugins/build-report/src/xpack.ts | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index a075e8201..def7555e9 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -47,6 +47,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt }); build.onEnd((result) => { + const buildReportEnd = log.time('build report'); const collectTimeEnd = log.time('collecting errors and warnings'); const cwd = context.cwd; for (const error of result.errors) { @@ -395,6 +396,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt context.build.outputs = outputs; context.build.inputs = inputs; context.build.entries = entries; + buildReportEnd(); }); }, }; diff --git a/packages/plugins/build-report/src/rollup.ts b/packages/plugins/build-report/src/rollup.ts index c99be1b27..2e77f3b2b 100644 --- a/packages/plugins/build-report/src/rollup.ts +++ b/packages/plugins/build-report/src/rollup.ts @@ -55,6 +55,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti importsReport[cleanId] = report; }, writeBundle(options, bundle) { + const buildReportEnd = log.time('build report'); const inputs: Input[] = []; const outputs: Output[] = []; const tempEntryFiles: Entry[] = []; @@ -65,6 +66,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti const reportOutputsIndexed: Record = {}; // Complete the importsReport with missing dependents and dependencies. + const depsCompleteEnd = log.time('completing dependencies and dependents'); for (const [filepath, { dependencies, dependents }] of Object.entries(importsReport)) { for (const dependency of dependencies) { const cleanedDependency = cleanPath(dependency); @@ -98,8 +100,10 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti importsReport[cleanedDependent].dependencies.add(filepath); } } + depsCompleteEnd(); // Fill in inputs and outputs. + const inputsOutputsEnd = log.time('filling inputs and outputs'); for (const [filename, asset] of Object.entries(bundle)) { const filepath = getAbsolutePath(context.bundler.outDir, filename); const size = @@ -153,8 +157,10 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti reportOutputsIndexed[file.filepath] = file; outputs.push(file); } + inputsOutputsEnd(); // Fill in inputs' dependencies and dependents. + const depsEnd = log.time('filling dependencies and dependents'); for (const input of inputs) { const importReport = importsReport[input.filepath]; if (!importReport) { @@ -184,9 +190,11 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti input.dependents.add(foundInput); } } + depsEnd(); // Fill in sourcemaps' inputs if necessary if (tempSourcemaps.length) { + const sourcemapsEnd = log.time('filling sourcemaps inputs'); for (const sourcemap of tempSourcemaps) { const outputPath = sourcemap.filepath.replace(/\.map$/, ''); const foundOutput = reportOutputsIndexed[outputPath]; @@ -198,6 +206,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti sourcemap.inputs.push(foundOutput); } + sourcemapsEnd(); } // Gather all outputs from a filepath, following imports. @@ -239,6 +248,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti }; // Fill in entries + const entriesEnd = log.time('filling entries'); for (const entryFile of tempEntryFiles) { const entryOutputs = getAllOutputs(entryFile.filepath); entryFile.outputs = Object.values(entryOutputs); @@ -251,10 +261,12 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti entryFile.size = entryFile.outputs.reduce((acc, output) => acc + output.size, 0); entries.push(entryFile); } + entriesEnd(); context.build.inputs = inputs; context.build.outputs = outputs; context.build.entries = entries; + buildReportEnd(); }, }; }; diff --git a/packages/plugins/build-report/src/xpack.ts b/packages/plugins/build-report/src/xpack.ts index 404ba2716..aa145e209 100644 --- a/packages/plugins/build-report/src/xpack.ts +++ b/packages/plugins/build-report/src/xpack.ts @@ -140,15 +140,19 @@ export const getXpackPlugin = compilation.hooks.finishModules.tap( PLUGIN_NAME, (finishedModules: Iterable) => { + const graphEnd = log.time('dependency graph'); // First loop to create indexes. + const indexEnd = log.time('indexing modules'); for (const module of finishedModules) { const keysToIndex = getKeysToIndex(module); for (const key of keysToIndex) { moduleIndex.set(key, module); } } + indexEnd(); // Second loop to create the dependency graph. + const inputEnd = log.time('building inputs'); for (const module of finishedModules) { const moduleIdentifier = module.identifier(); const dependencies: Set = new Set( @@ -216,8 +220,10 @@ export const getXpackPlugin = inputs.push(file); reportInputsIndexed.set(moduleIdentifier, file); } + inputEnd(); // Assign dependencies and dependents. + const assignEnd = log.time('assigning dependencies and dependents'); for (const input of inputs) { const depsReport = tempDeps.get(input.filepath); @@ -244,11 +250,14 @@ export const getXpackPlugin = input.dependents.add(depInput); } } + assignEnd(); + graphEnd(); }, ); }); compiler.hooks.afterEmit.tap(PLUGIN_NAME, (result: Compilation) => { + const reportEnd = log.time('build report'); const chunks = result.chunks; const assets = result.getAssets(); @@ -258,6 +267,7 @@ export const getXpackPlugin = ); }; + const chunkEnd = log.time('indexing chunks'); const chunkGraph = result.chunkGraph; for (const chunk of chunks) { const files = getChunkFiles(chunk); @@ -287,8 +297,10 @@ export const getXpackPlugin = modulesPerFile.set(file, [...fileModules, ...chunkModules]); } } + chunkEnd(); // Build outputs + const outputEnd = log.time('building outputs'); for (const asset of assets) { const file: Output = { size: asset.source.size() || 0, @@ -324,8 +336,10 @@ export const getXpackPlugin = file.inputs.push(inputFound); } } + outputEnd(); // Fill in inputs for sourcemaps. + const sourcemapsEnd = log.time('filling sourcemaps inputs'); for (const sourcemap of tempSourcemaps) { const outputFound = reportOutputsIndexed.get( sourcemap.filepath.replace(/\.map$/, ''), @@ -338,8 +352,10 @@ export const getXpackPlugin = sourcemap.inputs.push(outputFound); } + sourcemapsEnd(); // Build entries + const entriesEnd = log.time('building entries'); for (const [name, entrypoint] of result.entrypoints) { const entryOutputs: Output[] = []; const entryInputs: Input[] = []; @@ -396,6 +412,7 @@ export const getXpackPlugin = entries.push(file); } + entriesEnd(); // Save everything in the context. for (const error of result.errors) { @@ -407,5 +424,6 @@ export const getXpackPlugin = context.build.inputs = inputs; context.build.outputs = outputs; context.build.entries = entries; + reportEnd(); }); }; From 627935044cb37d3828b91b29c716d9e22ee3bb6a Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 6 Mar 2025 17:40:56 +0100 Subject: [PATCH 24/45] Add a dimmed starting point for time logs --- packages/factory/src/helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/factory/src/helpers.ts b/packages/factory/src/helpers.ts index b0ca28d92..828dc02aa 100644 --- a/packages/factory/src/helpers.ts +++ b/packages/factory/src/helpers.ts @@ -93,6 +93,7 @@ export const getLoggerFactory = } }; const time: Logger['time'] = (label: string, level: LogLevel = 'debug') => { + log(c.dim(`[${c.cyan(label)}] : start`), level); timerMap.set(label, { timestamp: Date.now(), logLevel: level }); return () => timeEnd(label, level); }; From bf3f59eb89fcc0b16d66058e479a66406126350d Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 7 Mar 2025 16:29:07 +0100 Subject: [PATCH 25/45] Move helpers `getAbsolutePath`, `getNearestCommonDirectory` and `debugFilesPlugins` to `@dd/core` --- packages/core/src/helpers.ts | 123 ++++++++++++++++++ packages/plugins/build-report/src/esbuild.ts | 4 +- packages/plugins/build-report/src/helpers.ts | 13 -- packages/plugins/build-report/src/rollup.ts | 3 +- packages/plugins/build-report/src/xpack.ts | 4 +- packages/plugins/bundler-report/src/index.ts | 25 +--- packages/plugins/injection/src/esbuild.ts | 3 +- packages/plugins/injection/src/helpers.ts | 3 +- packages/tests/src/_jest/helpers/mocks.ts | 2 +- .../unit/plugins/build-report/index.test.ts | 3 +- .../src/unit/plugins/injection/index.test.ts | 4 +- .../src/unit/plugins/telemetry/index.test.ts | 2 +- packages/tools/src/helpers.ts | 88 +------------ 13 files changed, 139 insertions(+), 138 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index c430527bc..5453e259b 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -16,8 +16,10 @@ import type { BundlerFullName, Entry, File, + GetCustomPlugins, GlobalContext, Input, + IterableElement, Logger, Output, RequestOpts, @@ -422,3 +424,124 @@ export const unserializeBuildReport = (report: SerializedBuildReport): BuildRepo outputs, }; }; + +// Will only prepend the cwd if not already there. +export const getAbsolutePath = (cwd: string, filepath: string) => { + if (isInjectionFile(filepath)) { + return INJECTED_FILE; + } + + if (filepath.startsWith(cwd) || path.isAbsolute(filepath)) { + return filepath; + } + return path.resolve(cwd, filepath); +}; + +// From a list of path, return the nearest common directory. +export const getNearestCommonDirectory = (dirs: string[], cwd?: string) => { + const dirsToCompare = [...dirs]; + + // We include the CWD because it's part of the paths we want to compare. + if (cwd) { + dirsToCompare.push(cwd); + } + + const splitPaths = dirsToCompare.map((dir) => { + const absolutePath = cwd ? getAbsolutePath(cwd, dir) : dir; + return absolutePath.split(path.sep); + }); + + // Use the shortest length for faster results. + const minLength = Math.min(...splitPaths.map((parts) => parts.length)); + const commonParts = []; + + for (let i = 0; i < minLength; i++) { + // We use the first path as our basis. + const component = splitPaths[0][i]; + if (splitPaths.every((parts) => parts[i] === component)) { + commonParts.push(component); + } else { + break; + } + } + + return commonParts.length > 0 ? commonParts.join(path.sep) : path.sep; +}; + +// Returns a customPlugin to output some debug files. +type CustomPlugins = ReturnType; +export const debugFilesPlugins = (context: GlobalContext): CustomPlugins => { + const rollupPlugin: IterableElement['rollup'] = { + writeBundle(options, bundle) { + outputJsonSync( + path.resolve(context.bundler.outDir, `output.${context.bundler.fullName}.json`), + bundle, + ); + }, + }; + + const xpackPlugin: IterableElement['webpack'] & + IterableElement['rspack'] = (compiler) => { + type Stats = Parameters[1]>[0]; + + compiler.hooks.done.tap('bundler-outputs', (stats: Stats) => { + const statsJson = stats.toJson({ + all: false, + assets: true, + children: true, + chunks: true, + chunkGroupAuxiliary: true, + chunkGroupChildren: true, + chunkGroups: true, + chunkModules: true, + chunkRelations: true, + entrypoints: true, + errors: true, + ids: true, + modules: true, + nestedModules: true, + reasons: true, + relatedAssets: true, + warnings: true, + }); + outputJsonSync( + path.resolve(context.bundler.outDir, `output.${context.bundler.fullName}.json`), + statsJson, + ); + }); + }; + + return [ + { + name: 'build-report', + enforce: 'post', + writeBundle() { + outputJsonSync( + path.resolve(context.bundler.outDir, `report.${context.bundler.fullName}.json`), + serializeBuildReport(context.build), + ); + }, + }, + { + name: 'bundler-outputs', + enforce: 'post', + esbuild: { + setup(build) { + build.onEnd((result) => { + outputJsonSync( + path.resolve( + context.bundler.outDir, + `output.${context.bundler.fullName}.json`, + ), + result.metafile, + ); + }); + }, + }, + rspack: xpackPlugin, + rollup: rollupPlugin, + vite: rollupPlugin, + webpack: xpackPlugin, + }, + ]; +}; diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index def7555e9..29c4aff72 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getEsbuildEntries, isInjectionFile } from '@dd/core/helpers'; +import { getAbsolutePath, getEsbuildEntries, isInjectionFile } from '@dd/core/helpers'; import type { Logger, Entry, @@ -14,7 +14,7 @@ import type { } from '@dd/core/types'; import path from 'path'; -import { cleanName, getAbsolutePath, getType } from './helpers'; +import { cleanName, getType } from './helpers'; // Re-index metafile data for easier access. const reIndexMeta = (obj: Record, cwd: string) => diff --git a/packages/plugins/build-report/src/helpers.ts b/packages/plugins/build-report/src/helpers.ts index 19ff8925f..8fd00234f 100644 --- a/packages/plugins/build-report/src/helpers.ts +++ b/packages/plugins/build-report/src/helpers.ts @@ -5,7 +5,6 @@ import { INJECTED_FILE } from '@dd/core/constants'; import { isInjectionFile } from '@dd/core/helpers'; import type { GlobalContext } from '@dd/core/types'; -import path from 'path'; // Will match any last part of a path after a dot or slash and is a word character. const EXTENSION_RX = /\.(?!.*(?:\.|\/|\\))(\w{1,})/g; @@ -82,18 +81,6 @@ export const cleanPath = (filepath: string) => { ); }; -// Will only prepend the cwd if not already there. -export const getAbsolutePath = (cwd: string, filepath: string) => { - if (isInjectionFile(filepath)) { - return INJECTED_FILE; - } - - if (filepath.startsWith(cwd) || path.isAbsolute(filepath)) { - return filepath; - } - return path.resolve(cwd, filepath); -}; - // Extract a name from a path based on the context (out dir and cwd). export const cleanName = (context: GlobalContext, filepath: string) => { if (isInjectionFile(filepath)) { diff --git a/packages/plugins/build-report/src/rollup.ts b/packages/plugins/build-report/src/rollup.ts index 2e77f3b2b..6ab479040 100644 --- a/packages/plugins/build-report/src/rollup.ts +++ b/packages/plugins/build-report/src/rollup.ts @@ -2,9 +2,10 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { getAbsolutePath } from '@dd/core/helpers'; import type { Logger, Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types'; -import { cleanName, cleanPath, cleanReport, getAbsolutePath, getType } from './helpers'; +import { cleanName, cleanPath, cleanReport, getType } from './helpers'; export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOptions['rollup'] => { const importsReport: Record< diff --git a/packages/plugins/build-report/src/xpack.ts b/packages/plugins/build-report/src/xpack.ts index aa145e209..6f4a06c9e 100644 --- a/packages/plugins/build-report/src/xpack.ts +++ b/packages/plugins/build-report/src/xpack.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjectionFile } from '@dd/core/helpers'; +import { getAbsolutePath, isInjectionFile } from '@dd/core/helpers'; import type { Logger, Entry, @@ -13,7 +13,7 @@ import type { PluginOptions, } from '@dd/core/types'; -import { cleanName, getAbsolutePath, getType } from './helpers'; +import { cleanName, getType } from './helpers'; export const getXpackPlugin = ( diff --git a/packages/plugins/bundler-report/src/index.ts b/packages/plugins/bundler-report/src/index.ts index 04f34f7d4..d3b3a3aca 100644 --- a/packages/plugins/bundler-report/src/index.ts +++ b/packages/plugins/bundler-report/src/index.ts @@ -2,35 +2,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { getNearestCommonDirectory } from '@dd/core/helpers'; import type { GlobalContext, PluginOptions } from '@dd/core/types'; import path from 'path'; export const PLUGIN_NAME = 'datadog-bundler-report-plugin'; -// From a list of path, return the nearest common directory. -const getNearestCommonDirectory = (dirs: string[], cwd: string) => { - const splitPaths = dirs.map((dir) => { - const absolutePath = path.isAbsolute(dir) ? dir : path.resolve(cwd, dir); - return absolutePath.split(path.sep); - }); - - // Use the shortest length for faster results. - const minLength = Math.min(...splitPaths.map((parts) => parts.length)); - const commonParts = []; - - for (let i = 0; i < minLength; i++) { - // We use the first path as our basis. - const component = splitPaths[0][i]; - if (splitPaths.every((parts) => parts[i] === component)) { - commonParts.push(component); - } else { - break; - } - } - - return commonParts.length > 0 ? commonParts.join(path.sep) : path.sep; -}; - const handleCwd = (dirs: string[], context: GlobalContext) => { const nearestDir = getNearestCommonDirectory(dirs, context.cwd); if (nearestDir !== path.sep) { diff --git a/packages/plugins/injection/src/esbuild.ts b/packages/plugins/injection/src/esbuild.ts index 532ddcbe5..77ae39b87 100644 --- a/packages/plugins/injection/src/esbuild.ts +++ b/packages/plugins/injection/src/esbuild.ts @@ -3,10 +3,9 @@ // Copyright 2019-Present Datadog, Inc. import { INJECTED_FILE } from '@dd/core/constants'; -import { getEsbuildEntries, getUniqueId, outputFile } from '@dd/core/helpers'; +import { getAbsolutePath, getEsbuildEntries, getUniqueId, outputFile } from '@dd/core/helpers'; import type { Logger, PluginOptions, GlobalContext, ResolvedEntry } from '@dd/core/types'; import { InjectPosition } from '@dd/core/types'; -import { getAbsolutePath } from '@dd/internal-build-report-plugin/helpers'; import fs from 'fs'; import os from 'os'; import path from 'path'; diff --git a/packages/plugins/injection/src/helpers.ts b/packages/plugins/injection/src/helpers.ts index 9bf643dd6..c6dd2042c 100644 --- a/packages/plugins/injection/src/helpers.ts +++ b/packages/plugins/injection/src/helpers.ts @@ -2,10 +2,9 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { doRequest, truncateString } from '@dd/core/helpers'; +import { doRequest, getAbsolutePath, truncateString } from '@dd/core/helpers'; import type { Logger, ToInjectItem } from '@dd/core/types'; import { InjectPosition } from '@dd/core/types'; -import { getAbsolutePath } from '@dd/internal-build-report-plugin/helpers'; import { readFile } from 'fs/promises'; import { AFTER_INJECTION, BEFORE_INJECTION, DISTANT_FILE_RX } from './constants'; diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index cbc15097a..96acc1975 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { getAbsolutePath } from '@dd/core/helpers'; import type { BuildReport, File, @@ -11,7 +12,6 @@ import type { LogLevel, Options, } from '@dd/core/types'; -import { getAbsolutePath } from '@dd/internal-build-report-plugin/helpers'; import { getSourcemapsConfiguration } from '@dd/tests/unit/plugins/error-tracking/testHelpers'; import { getTelemetryConfiguration } from '@dd/tests/unit/plugins/telemetry/testHelpers'; import { configXpack } from '@dd/tools/bundlers'; diff --git a/packages/tests/src/unit/plugins/build-report/index.test.ts b/packages/tests/src/unit/plugins/build-report/index.test.ts index 6f8d7ca8d..bf2e25262 100644 --- a/packages/tests/src/unit/plugins/build-report/index.test.ts +++ b/packages/tests/src/unit/plugins/build-report/index.test.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { serializeBuildReport, unserializeBuildReport } from '@dd/core/helpers'; +import { serializeBuildReport, unserializeBuildReport, debugFilesPlugins } from '@dd/core/helpers'; import type { Input, Entry, @@ -21,7 +21,6 @@ import { } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import type { BundlerOptionsOverrides } from '@dd/tests/_jest/helpers/types'; -import { debugFilesPlugins } from '@dd/tools/helpers'; import path from 'path'; const sortFiles = (a: File | Output | Entry, b: File | Output | Entry) => { diff --git a/packages/tests/src/unit/plugins/injection/index.test.ts b/packages/tests/src/unit/plugins/injection/index.test.ts index 6d50d28dd..ab1e25782 100644 --- a/packages/tests/src/unit/plugins/injection/index.test.ts +++ b/packages/tests/src/unit/plugins/injection/index.test.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { outputFileSync } from '@dd/core/helpers'; +import { debugFilesPlugins, outputFileSync } from '@dd/core/helpers'; import type { Assign, BundlerFullName, Options, ToInjectItem } from '@dd/core/types'; import { InjectPosition } from '@dd/core/types'; import { AFTER_INJECTION, BEFORE_INJECTION } from '@dd/internal-injection-plugin/constants'; @@ -14,7 +14,7 @@ import { } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import { header, licenses } from '@dd/tools/commands/oss/templates'; -import { debugFilesPlugins, escapeStringForRegExp, execute } from '@dd/tools/helpers'; +import { escapeStringForRegExp, execute } from '@dd/tools/helpers'; import chalk from 'chalk'; import { readFileSync, writeFileSync } from 'fs'; import { glob } from 'glob'; diff --git a/packages/tests/src/unit/plugins/telemetry/index.test.ts b/packages/tests/src/unit/plugins/telemetry/index.test.ts index 65daf8033..ec85a721b 100644 --- a/packages/tests/src/unit/plugins/telemetry/index.test.ts +++ b/packages/tests/src/unit/plugins/telemetry/index.test.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { debugFilesPlugins } from '@dd/core/helpers'; import type { GlobalContext, Options } from '@dd/core/types'; import { addMetrics } from '@dd/telemetry-plugin/common/aggregator'; import type { MetricToSend } from '@dd/telemetry-plugin/types'; @@ -12,7 +13,6 @@ import { } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; import type { Bundler } from '@dd/tests/_jest/helpers/types'; -import { debugFilesPlugins } from '@dd/tools/helpers'; import nock from 'nock'; // Used to intercept metrics. diff --git a/packages/tools/src/helpers.ts b/packages/tools/src/helpers.ts index 4a6170fee..df086c4b8 100644 --- a/packages/tools/src/helpers.ts +++ b/packages/tools/src/helpers.ts @@ -3,16 +3,8 @@ // Copyright 2019-Present Datadog, Inc. import { ALL_BUNDLERS, SUPPORTED_BUNDLERS } from '@dd/core/constants'; -import { outputJsonSync, readJsonSync, serializeBuildReport } from '@dd/core/helpers'; -import type { - BundlerFullName, - BundlerName, - GetCustomPlugins, - GetPlugins, - GlobalContext, - IterableElement, - OptionsWithDefaults, -} from '@dd/core/types'; +import { readJsonSync } from '@dd/core/helpers'; +import type { BundlerFullName, BundlerName, GetPlugins, OptionsWithDefaults } from '@dd/core/types'; import { getContext } from '@dd/factory/helpers'; import chalk from 'chalk'; import { execFile, execFileSync } from 'child_process'; @@ -260,79 +252,3 @@ export const getBundlerPicture = (bundler: string) => { export const isInternalPluginWorkspace = (workspace: Workspace) => workspace.name.startsWith('@dd/internal-'); - -// Returns a customPlugin to output some debug files. -type CustomPlugins = ReturnType; -export const debugFilesPlugins = (context: GlobalContext): CustomPlugins => { - const rollupPlugin: IterableElement['rollup'] = { - writeBundle(options, bundle) { - outputJsonSync( - path.resolve(context.bundler.outDir, `output.${context.bundler.fullName}.json`), - bundle, - ); - }, - }; - - const xpackPlugin: IterableElement['webpack'] & - IterableElement['rspack'] = (compiler) => { - type Stats = Parameters[1]>[0]; - - compiler.hooks.done.tap('bundler-outputs', (stats: Stats) => { - const statsJson = stats.toJson({ - all: false, - assets: true, - children: true, - chunks: true, - chunkGroupAuxiliary: true, - chunkGroupChildren: true, - chunkGroups: true, - chunkModules: true, - chunkRelations: true, - entrypoints: true, - errors: true, - ids: true, - modules: true, - nestedModules: true, - reasons: true, - relatedAssets: true, - warnings: true, - }); - outputJsonSync( - path.resolve(context.bundler.outDir, `output.${context.bundler.fullName}.json`), - statsJson, - ); - }); - }; - - return [ - { - name: 'build-report', - writeBundle() { - outputJsonSync( - path.resolve(context.bundler.outDir, `report.${context.bundler.fullName}.json`), - serializeBuildReport(context.build), - ); - }, - }, - { - name: 'bundler-outputs', - esbuild: { - setup(build) { - build.onEnd((result) => { - outputJsonSync( - path.resolve( - context.bundler.outDir, - `output.${context.bundler.fullName}.json`, - ), - result.metafile, - ); - }); - }, - }, - rspack: xpackPlugin, - rollup: rollupPlugin, - vite: rollupPlugin, - webpack: xpackPlugin, - }, - ]; -}; From 08bba97c72a6216a1599207edf9d7433b7dba135 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 7 Mar 2025 16:34:10 +0100 Subject: [PATCH 26/45] Factorise build commands --- packages/published/esbuild-plugin/package.json | 5 +++-- packages/published/rollup-plugin/package.json | 5 +++-- packages/published/rspack-plugin/package.json | 5 +++-- packages/published/vite-plugin/package.json | 5 +++-- packages/published/webpack-plugin/package.json | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/published/esbuild-plugin/package.json b/packages/published/esbuild-plugin/package.json index 334ee18f9..86fb83ea0 100644 --- a/packages/published/esbuild-plugin/package.json +++ b/packages/published/esbuild-plugin/package.json @@ -42,11 +42,12 @@ "dist" ], "scripts": { - "build": "yarn clean && rollup --config rollup.config.mjs", + "buildCmd": "rollup --config rollup.config.mjs", + "build": "yarn clean && yarn buildCmd", "clean": "rm -rf dist", "prepack": "yarn build", "typecheck": "tsc --noEmit", - "watch": "yarn clean && rollup --config rollup.config.mjs --watch" + "watch": "yarn build --watch" }, "dependencies": { "async-retry": "1.3.3", diff --git a/packages/published/rollup-plugin/package.json b/packages/published/rollup-plugin/package.json index 32229b937..35b59bfb1 100644 --- a/packages/published/rollup-plugin/package.json +++ b/packages/published/rollup-plugin/package.json @@ -42,11 +42,12 @@ "dist" ], "scripts": { - "build": "yarn clean && rollup --config rollup.config.mjs", + "buildCmd": "rollup --config rollup.config.mjs", + "build": "yarn clean && yarn buildCmd", "clean": "rm -rf dist", "prepack": "yarn build", "typecheck": "tsc --noEmit", - "watch": "yarn clean && rollup --config rollup.config.mjs --watch" + "watch": "yarn build --watch" }, "dependencies": { "async-retry": "1.3.3", diff --git a/packages/published/rspack-plugin/package.json b/packages/published/rspack-plugin/package.json index 82514d44d..555c16c23 100644 --- a/packages/published/rspack-plugin/package.json +++ b/packages/published/rspack-plugin/package.json @@ -42,11 +42,12 @@ "dist" ], "scripts": { - "build": "yarn clean && rollup --config rollup.config.mjs", + "buildCmd": "rollup --config rollup.config.mjs", + "build": "yarn clean && yarn buildCmd", "clean": "rm -rf dist", "prepack": "yarn build", "typecheck": "tsc --noEmit", - "watch": "yarn clean && rollup --config rollup.config.mjs --watch" + "watch": "yarn build --watch" }, "dependencies": { "async-retry": "1.3.3", diff --git a/packages/published/vite-plugin/package.json b/packages/published/vite-plugin/package.json index 4e5e3cf33..025222bdd 100644 --- a/packages/published/vite-plugin/package.json +++ b/packages/published/vite-plugin/package.json @@ -42,11 +42,12 @@ "dist" ], "scripts": { - "build": "yarn clean && rollup --config rollup.config.mjs", + "buildCmd": "rollup --config rollup.config.mjs", + "build": "yarn clean && yarn buildCmd", "clean": "rm -rf dist", "prepack": "yarn build", "typecheck": "tsc --noEmit", - "watch": "yarn clean && rollup --config rollup.config.mjs --watch" + "watch": "yarn build --watch" }, "dependencies": { "async-retry": "1.3.3", diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index 0f7d1b404..4c4e1b88a 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -42,11 +42,12 @@ "dist" ], "scripts": { - "build": "yarn clean && rollup --config rollup.config.mjs", + "buildCmd": "rollup --config rollup.config.mjs", + "build": "yarn clean && yarn buildCmd", "clean": "rm -rf dist", "prepack": "yarn build", "typecheck": "tsc --noEmit", - "watch": "yarn clean && rollup --config rollup.config.mjs --watch" + "watch": "yarn build --watch" }, "dependencies": { "async-retry": "1.3.3", From a67c63c3b386f4c0b7d6f3c58eeebe7a92f8cf3b Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 10 Mar 2025 14:42:44 +0100 Subject: [PATCH 27/45] Fix rollup's build-report with external dependencies support --- packages/core/src/helpers.ts | 25 +++++++++- packages/plugins/build-report/src/helpers.ts | 8 ++-- packages/plugins/build-report/src/rollup.ts | 47 ++++++++++++++++++- packages/plugins/bundler-report/src/index.ts | 39 ++++++++++----- .../tests/src/unit/factory/helpers.test.ts | 33 +++++++++++-- 5 files changed, 128 insertions(+), 24 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 5453e259b..6c2ef54e2 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -437,6 +437,24 @@ export const getAbsolutePath = (cwd: string, filepath: string) => { return path.resolve(cwd, filepath); }; +// Find the highest package.json from the current directory. +export const getHighestPackageJsonDir = (currentDir: string): string | undefined => { + let highestPackage; + let current = getAbsolutePath(process.cwd(), currentDir); + let currentDepth = current.split('/').length; + while (currentDepth > 0) { + const packagePath = path.resolve(current, `package.json`); + // Check if package.json exists in the current directory. + if (fs.existsSync(packagePath)) { + highestPackage = current; + } + // Remove the last part of the path. + current = current.split('/').slice(0, -1).join('/'); + currentDepth--; + } + return highestPackage; +}; + // From a list of path, return the nearest common directory. export const getNearestCommonDirectory = (dirs: string[], cwd?: string) => { const dirsToCompare = [...dirs]; @@ -447,7 +465,7 @@ export const getNearestCommonDirectory = (dirs: string[], cwd?: string) => { } const splitPaths = dirsToCompare.map((dir) => { - const absolutePath = cwd ? getAbsolutePath(cwd, dir) : dir; + const absolutePath = getAbsolutePath(cwd || process.cwd(), dir); return absolutePath.split(path.sep); }); @@ -465,7 +483,10 @@ export const getNearestCommonDirectory = (dirs: string[], cwd?: string) => { } } - return commonParts.length > 0 ? commonParts.join(path.sep) : path.sep; + return commonParts.length > 0 + ? // Use "|| path.sep" to cover for the [''] case. + commonParts.join(path.sep) || path.sep + : path.sep; }; // Returns a customPlugin to output some debug files. diff --git a/packages/plugins/build-report/src/helpers.ts b/packages/plugins/build-report/src/helpers.ts index 8fd00234f..31d42fa8c 100644 --- a/packages/plugins/build-report/src/helpers.ts +++ b/packages/plugins/build-report/src/helpers.ts @@ -3,7 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { INJECTED_FILE } from '@dd/core/constants'; -import { isInjectionFile } from '@dd/core/helpers'; +import { getAbsolutePath, isInjectionFile } from '@dd/core/helpers'; import type { GlobalContext } from '@dd/core/types'; // Will match any last part of a path after a dot or slash and is a word character. @@ -101,7 +101,7 @@ export const cleanName = (context: GlobalContext, filepath: string) => { .split('!') .pop()! // Remove outDir's path. - .replace(context.bundler.outDir, '') + .replace(getAbsolutePath(context.cwd, context.bundler.outDir), '') // Remove the cwd's path. .replace(context.cwd, '') // Remove node_modules path. @@ -110,7 +110,7 @@ export const cleanName = (context: GlobalContext, filepath: string) => { // Remove query parameters. .split(QUERY_RX) .shift()! - // Remove leading slashes. - .replace(/^\/+/, '') + // Remove leading dots and slashes. + .replace(/^((\.\.?)?\/)+/, '') ); }; diff --git a/packages/plugins/build-report/src/rollup.ts b/packages/plugins/build-report/src/rollup.ts index 6ab479040..0a994f7a5 100644 --- a/packages/plugins/build-report/src/rollup.ts +++ b/packages/plugins/build-report/src/rollup.ts @@ -131,6 +131,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti // We don't want to include commonjs wrappers that have a path like: // \u0000{{path}}?commonjs-proxy if (cleanPath(modulepath) !== modulepath) { + log.debug(`Not including ${modulepath} in the report.`); continue; } const moduleFile: Input = { @@ -149,6 +150,42 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti } } + // Add imports as inputs. + // These are external imports since they are declared in the output file. + if ('imports' in asset) { + for (const importName of asset.imports) { + const cleanedImport = cleanPath(importName); + const importReport = importsReport[cleanedImport]; + if (!importReport) { + log.debug( + `Could not find the import report for ${cleanedImport} from ${file.name}.`, + ); + continue; + } + + if (reportInputsIndexed[cleanedImport]) { + log.debug( + `Input report already there for ${cleanedImport} from ${file.name}.`, + ); + continue; + } + + const importFile: Input = { + name: cleanName(context, importName), + dependencies: new Set(), + dependents: new Set(), + filepath: cleanedImport, + // Since it's external, we don't have the size. + size: 0, + type: 'external', + }; + file.inputs.push(importFile); + + reportInputsIndexed[importFile.filepath] = importFile; + inputs.push(importFile); + } + } + // Store entries for later filling. // As we may not have reported its outputs and inputs yet. if ('isEntry' in asset && asset.isEntry) { @@ -221,12 +258,18 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti // Get its output. const foundOutput = reportOutputsIndexed[filepath]; if (!foundOutput) { - log.debug(`Could not find output for ${filename}`); + // If it's been reported in the indexes, it means it's an external here. + const isExternal = !!reportInputsIndexed[filename]; + // Do not log about externals, we don't expect to find them. + if (!isExternal) { + log.debug(`Could not find output for ${filename}`); + } return allOutputs; } allOutputs[filepath] = foundOutput; - const asset = bundle[filename]; + // Rollup indexes on the filepath relative to the outDir. + const asset = bundle[cleanName(context, filepath)]; if (!asset) { log.debug(`Could not find asset for ${filename}`); return allOutputs; diff --git a/packages/plugins/bundler-report/src/index.ts b/packages/plugins/bundler-report/src/index.ts index d3b3a3aca..46fe5c842 100644 --- a/packages/plugins/bundler-report/src/index.ts +++ b/packages/plugins/bundler-report/src/index.ts @@ -2,16 +2,27 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getNearestCommonDirectory } from '@dd/core/helpers'; +import { + getAbsolutePath, + getNearestCommonDirectory, + getHighestPackageJsonDir, +} from '@dd/core/helpers'; import type { GlobalContext, PluginOptions } from '@dd/core/types'; import path from 'path'; export const PLUGIN_NAME = 'datadog-bundler-report-plugin'; -const handleCwd = (dirs: string[], context: GlobalContext) => { - const nearestDir = getNearestCommonDirectory(dirs, context.cwd); +// Compute the CWD based on a list of directories and the outDir. +const getCwd = (dirs: Set, outDir: string) => { + const highestPackage = getHighestPackageJsonDir(outDir); + if (highestPackage) { + return highestPackage; + } + + // Fall back to the nearest common directory. + const nearestDir = getNearestCommonDirectory(Array.from(dirs)); if (nearestDir !== path.sep) { - context.cwd = nearestDir; + return nearestDir; } }; @@ -41,15 +52,19 @@ export const getBundlerReportPlugins = (context: GlobalContext): PluginOptions[] directories.add(outputOptions.dir); } else if (outputOptions.file) { context.bundler.outDir = path.dirname(outputOptions.file); - directories.add(outputOptions.dir); + directories.add(context.bundler.outDir); } + // We need an absolute path for rollup because of the way we have to compute its CWD. + // It's relative to process.cwd(), because there is no cwd options for rollup. + context.bundler.outDir = getAbsolutePath(process.cwd(), context.bundler.outDir); + // Vite has the "root" option we're using. if (context.bundler.name === 'vite') { return; } - handleCwd(Array.from(directories), context); + context.cwd = getCwd(directories, context.bundler.outDir) || context.cwd; }; const rollupPlugin: () => PluginOptions['rollup'] & PluginOptions['vite'] = () => { @@ -73,12 +88,14 @@ export const getBundlerReportPlugins = (context: GlobalContext): PluginOptions[] } if ('output' in options) { - handleOutputOptions(options.output); + const outputOptions = Array.isArray(options.output) + ? options.output + : [options.output]; + for (const output of outputOptions) { + handleOutputOptions(output); + } } }, - outputOptions(options) { - handleOutputOptions(options); - }, }; }; @@ -117,7 +134,7 @@ export const getBundlerReportPlugins = (context: GlobalContext): PluginOptions[] if (config.root) { context.cwd = config.root; } else { - handleCwd(Array.from(directories), context); + context.cwd = getCwd(directories, context.bundler.outDir) || context.cwd; } }, }, diff --git a/packages/tests/src/unit/factory/helpers.test.ts b/packages/tests/src/unit/factory/helpers.test.ts index c5cd44c75..183293f69 100644 --- a/packages/tests/src/unit/factory/helpers.test.ts +++ b/packages/tests/src/unit/factory/helpers.test.ts @@ -28,6 +28,8 @@ const getOutput = (mock: jest.Mock, index: number) => stripAnsi(mock.mock.calls[ describe('Factory Helpers', () => { // Intercept contexts to verify it at the moment they're used. const initialContexts: Record = {}; + const cwds: Record = {}; + let workingDir: string; beforeAll(async () => { const pluginConfig: Options = { @@ -40,11 +42,19 @@ describe('Factory Helpers', () => { // These are functions, so they can't be serialized with parse/stringify. initialContexts[bundlerName].inject = context.inject; - return []; + return [ + { + name: 'custom-plugin', + buildStart() { + cwds[bundlerName] = context.cwd; + }, + }, + ]; }, }; - await runBundlers(pluginConfig); + const result = await runBundlers(pluginConfig); + workingDir = result.workingDir; }); describe('getContext', () => { @@ -62,6 +72,10 @@ describe('Factory Helpers', () => { expect(context.version).toBe(version); expect(context.inject).toEqual(expect.any(Function)); }); + + test('Should update to the right CWD.', () => { + expect(cwds[name]).toBe(workingDir); + }); }); }); @@ -170,15 +184,24 @@ describe('Factory Helpers', () => { const timeEnd2 = logger.time('test time 3', 'error'); timeEnd2(); - expect(logMock).toHaveBeenCalledTimes(2); - expect(errorMock).toHaveBeenCalledTimes(1); + expect(logMock).toHaveBeenCalledTimes(4); + expect(errorMock).toHaveBeenCalledTimes(2); expect(getOutput(logMock, 0)).toBe( - `[debug|esbuild|testLogger] [test time 1] : 0ms`, + `[debug|esbuild|testLogger] [test time 1] : start`, ); expect(getOutput(logMock, 1)).toBe( + `[debug|esbuild|testLogger] [test time 1] : 0ms`, + ); + expect(getOutput(logMock, 2)).toBe( + `[debug|esbuild|testLogger] [test time 2] : start`, + ); + expect(getOutput(logMock, 3)).toBe( `[debug|esbuild|testLogger] [test time 2] : 0ms`, ); expect(getOutput(errorMock, 0)).toBe( + `[error|esbuild|testLogger] [test time 3] : start`, + ); + expect(getOutput(errorMock, 1)).toBe( `[error|esbuild|testLogger] [test time 3] : 0ms`, ); }); From 52250a59af09d322064a3dd4a538fe3c2f81eeb0 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 10 Mar 2025 17:12:34 +0100 Subject: [PATCH 28/45] Remove resetModule from jest as it's not needed anymore and prevents some mocks to work as expected (memfs for isntance) --- packages/tests/jest.config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/tests/jest.config.js b/packages/tests/jest.config.js index 796986f99..aa8e266e1 100644 --- a/packages/tests/jest.config.js +++ b/packages/tests/jest.config.js @@ -7,8 +7,6 @@ module.exports = { clearMocks: true, globalSetup: '/src/_jest/globalSetup.ts', preset: 'ts-jest/presets/js-with-ts', - // Without it, vite import is silently crashing the process with code SIGHUP 129 - resetModules: true, roots: ['./src/unit/'], setupFilesAfterEnv: ['/src/_jest/setupAfterEnv.ts'], testEnvironment: 'node', From 097f92c6daa8e1f95f62c2adcabd1694fa2232be Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Mon, 10 Mar 2025 17:12:56 +0100 Subject: [PATCH 29/45] Clean readme from unecessary information --- packages/tests/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/tests/README.md b/packages/tests/README.md index faf1b0912..1b347b2ea 100644 --- a/packages/tests/README.md +++ b/packages/tests/README.md @@ -58,7 +58,6 @@ Here's a bootstrap to get you going: ```typescript import type { Options } from '@dd/core/types'; -import type { CleanupFn } from '@dd/tests/_jest/helpers/runBundlers'; import { runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; describe('My very awesome plugin', () => { @@ -174,7 +173,6 @@ The best way would be to freeze the content you need to test, at the moment you ```typescript import type { GlobalContext, Options } from '@dd/core/types'; import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; -import type { CleanupFn } from '@dd/tests/_jest/helpers/runBundlers'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; describe('Global Context Plugin', () => { @@ -221,7 +219,6 @@ Giving the following, more involved example: import { serializeBuildReport, unserializeBuildReport } from '@dd/core/helpers'; import type { BuildReport, Options } from '@dd/core/types'; import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; -import type { CleanupFn } from '@dd/tests/_jest/helpers/runBundlers'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; describe('Build Reports', () => { From c76c6b07daa3ade13b49f483f710a9c7e8cd7923 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 09:27:28 +0100 Subject: [PATCH 30/45] Add some tests --- packages/tests/src/unit/core/helpers.test.ts | 78 +++++++++ .../unit/plugins/build-report/helpers.test.ts | 164 ++++++++++++++++-- 2 files changed, 231 insertions(+), 11 deletions(-) diff --git a/packages/tests/src/unit/core/helpers.test.ts b/packages/tests/src/unit/core/helpers.test.ts index ad929b0e9..a71f30df5 100644 --- a/packages/tests/src/unit/core/helpers.test.ts +++ b/packages/tests/src/unit/core/helpers.test.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { INJECTED_FILE } from '@dd/core/constants'; import { getEsbuildEntries } from '@dd/core/helpers'; import type { RequestOpts, ResolvedEntry } from '@dd/core/types'; import { @@ -368,4 +369,81 @@ describe('Core Helpers', () => { }, ); }); + + describe('getAbsolutePath', () => { + test.each([ + // With the injection file. + ['/path/to', `./to/1293.${INJECTED_FILE}.js`, INJECTED_FILE], + // With a path with no prefix. + ['/path/to', 'file.js', '/path/to/file.js'], + // With a path with a dot prefix. + ['/path/to', './file.js', '/path/to/file.js'], + ['/path/to', '../file.js', '/path/file.js'], + ['/path/to', '../../file.js', '/file.js'], + ['/path/to', '../../../file.js', '/file.js'], + // With an absolute path. + ['/path/to', '/file.js', '/file.js'], + ])('Should resolve "%s" with "%s" to "%s"', async (base, relative, expected) => { + const { getAbsolutePath } = await import('@dd/core/helpers'); + expect(getAbsolutePath(base, relative)).toBe(expected); + }); + }); + + describe('getNearestCommonDirectory', () => { + test.each([ + { + // With a single path. + directories: ['/path/to'], + expected: '/path/to', + }, + { + // Basic usage. + directories: ['/path/to', '/path/to/other'], + expected: '/path/to', + }, + { + // With a different root directory. + directories: ['/path/to', '/path2/to/other'], + expected: '/', + }, + { + // With an absolute file. + directories: ['/path/to', '/'], + expected: '/', + }, + { + // With a given cwd. + cwd: '/path', + directories: ['/path/to', './', '/path/to/other'], + expected: '/path', + }, + ])('Should find the nearest common directory', async ({ directories, cwd, expected }) => { + const { getNearestCommonDirectory } = await import('@dd/core/helpers'); + expect(getNearestCommonDirectory(directories, cwd)).toBe(expected); + }); + }); + + describe('getHighestPackageJsonDir', () => { + beforeEach(() => { + vol.fromJSON({ + '/path1/to/package.json': '', + '/path2/to/other/package.json': '', + '/path3/to/other/deeper/package.json': '', + }); + }); + + afterEach(() => { + vol.reset(); + }); + + test.each([ + ['/path1/to', '/path1/to'], + ['/path2/to/other/project/directory', '/path2/to/other'], + ['/path3/to/other/deeper/who/knows', '/path3/to/other/deeper'], + ['/', undefined], + ])('Should find the highest package.json', async (dirpath, expected) => { + const { getHighestPackageJsonDir } = await import('@dd/core/helpers'); + expect(getHighestPackageJsonDir(dirpath)).toBe(expected); + }); + }); }); diff --git a/packages/tests/src/unit/plugins/build-report/helpers.test.ts b/packages/tests/src/unit/plugins/build-report/helpers.test.ts index 31c1215b4..cd2a8427a 100644 --- a/packages/tests/src/unit/plugins/build-report/helpers.test.ts +++ b/packages/tests/src/unit/plugins/build-report/helpers.test.ts @@ -2,21 +2,163 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getType } from '@dd/internal-build-report-plugin/helpers'; +import { INJECTED_FILE } from '@dd/core/constants'; +import { cleanPath, cleanName, getType } from '@dd/internal-build-report-plugin/helpers'; +import { getContextMock, getMockBuild } from '@dd/tests/_jest/helpers/mocks'; describe('Build report plugin helpers', () => { describe('getType', () => { const expectations = [ - ['unknown', 'unknown'], - ['webpack/runtime', 'runtime'], - ['path/to/file.js', 'js'], - [ - '/loaders/load.js??ref--4-0!/tests/_virtual_.%2Fsrc%2Ffixtures%2Fproject%2Fmain1.js%3Fadd-custom-injection', - 'js', - ], + { + name: 'unknown', + filepath: 'unknown', + expected: 'unknown', + }, + { + name: 'webpack runtime', + filepath: 'webpack/runtime', + expected: 'runtime', + }, + { + name: 'file with extension', + filepath: 'path/to/file.js', + expected: 'js', + }, + { + name: 'complex loader path', + filepath: + '/loaders/load.js??ref--4-0!/tests/_virtual_.%2Fsrc%2Ffixtures%2Fproject%2Fmain1.js%3Fadd-custom-injection', + expected: 'js', + }, ]; - test.each(expectations)('Should return the right type for "%s".', (filepath, type) => { - expect(getType(filepath)).toBe(type); - }); + test.each(expectations)( + 'Should return the right type for "$name".', + ({ filepath, expected }) => { + expect(getType(filepath)).toBe(expected); + }, + ); + }); + + describe('cleanName', () => { + const expectations = [ + { + name: 'injected file', + filepath: `./${INJECTED_FILE}`, + expected: INJECTED_FILE, + }, + { + name: 'unknown file', + filepath: 'unknown', + expected: 'unknown', + }, + { + name: 'webpack runtime', + filepath: 'webpack/runtime/make namespace object', + expected: 'make-namespace-object', + }, + { + name: 'loader path', + filepath: + 'webpack/loaders/load.js??ruleSet[1].rules[0].use[0]!/current/working/directory/path.js', + expected: 'path.js', + }, + { + name: 'cwd', + filepath: '/current/working/directory/src/path.js', + expected: 'src/path.js', + }, + { + name: 'outDir', + filepath: '/current/working/directory/dist/path.js', + expected: 'path.js', + }, + { + name: 'node_modules dependency', + filepath: '/current/working/directory/node_modules/module/path.js', + expected: 'module/path.js', + }, + { + name: 'query parameters', + filepath: '/current/working/directory/path.js?query=param', + expected: 'path.js', + }, + { + name: 'encoded query parameters', + filepath: '/current/working/directory/path.js%3Fquery=param', + expected: 'path.js', + }, + { + name: 'pipe query parameters', + filepath: '/current/working/directory/path.js|query=param', + expected: 'path.js', + }, + { + name: 'leading dots and slashes', + filepath: '../../path.js', + expected: 'path.js', + }, + { + name: 'some composition', + filepath: + 'webpack/loaders/load.js??ruleSet[1].rules[0].use[0]!/current/working/directory/node_modules/module/path.js?query=param', + expected: 'module/path.js', + }, + ]; + test.each(expectations)( + 'Should return a cleaned name for "$name".', + ({ filepath, expected }) => { + const context = getContextMock({ + cwd: '/current/working/directory', + bundler: { + ...getMockBuild().bundler, + outDir: '/current/working/directory/dist', + }, + }); + expect(cleanName(context, filepath)).toBe(expected); + }, + ); + }); + + describe('cleanPath', () => { + const expectations = [ + { + name: 'loader path', + filepath: + 'webpack/loaders/load.js??ruleSet[1].rules[0].use[0]!/current/working/directory/path.js', + expected: '/current/working/directory/path.js', + }, + { + name: 'query parameters', + filepath: '/current/working/directory/path.js?query=param', + expected: '/current/working/directory/path.js', + }, + { + name: 'encoded query parameters', + filepath: '/current/working/directory/path.js%3Fquery=param', + expected: '/current/working/directory/path.js', + }, + { + name: 'pipe query parameters', + filepath: '/current/working/directory/path.js|query=param', + expected: '/current/working/directory/path.js', + }, + { + name: 'leading invisible characters', + filepath: '\u0000/current/working/directory/path.js', + expected: '/current/working/directory/path.js', + }, + { + name: 'some composition', + filepath: + '\u0000/webpack/loaders/load.js??ruleSet[1].rules[0].use[0]!/current/working/directory/node_modules/module/path.js?query=param', + expected: '/current/working/directory/node_modules/module/path.js', + }, + ]; + test.each(expectations)( + 'Should return a cleaned path for "$name".', + ({ filepath, expected }) => { + expect(cleanPath(filepath)).toBe(expected); + }, + ); }); }); From baa70bdf552b6cee03a24539b92b9a508de1006f Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 09:29:04 +0100 Subject: [PATCH 31/45] Better types and comments --- global.d.ts | 46 ++++++++++++-- packages/tools/src/rollupConfig.mjs | 93 ++++++++++++++++++++++++----- 2 files changed, 119 insertions(+), 20 deletions(-) diff --git a/global.d.ts b/global.d.ts index b38f6a88b..74d36eedc 100644 --- a/global.d.ts +++ b/global.d.ts @@ -2,13 +2,51 @@ import type { Env } from '@dd/core/types'; declare global { namespace NodeJS { - interface ProcessEnv { - [key: string]: string | undefined; + interface ProcessEnv extends NodeJS.ProcessEnv { + /** + * To use when building the plugins with `yarn build:all`. + * + * If passed, the build will include our rollup's plugin to the build. + * + * Be sure to have build rollup's plugin with `yarn workspace @datadog/rollup-plugin build`. + */ + ADD_BUILD_PLUGINS?: '1'; + /** + * The environment in which the plugins will execute. + * + * For instance, we only submit logs to Datadog when the environment is `production`. + */ BUILD_PLUGINS_ENV?: Env; - NO_CLEANUP?: '1'; + /** + * Defined in github actions when running in CI. + */ + CI?: '1'; + /** + * Defined in github actions when running in CI. + * + * The commit SHA that triggered the workflow. + */ + GITHUB_SHA?: string; + /** + * Run jest in silent mode. + */ + JEST_SILENT?: '1'; + /** + * To also build the plugins before running the tests when using `yarn test:unit`. + */ NEED_BUILD?: '1'; + /** + * To skip the cleanup of the temporary working dirs where we build `runBundlers()`. + */ + NO_CLEANUP?: '1'; + /** + * The list of bundlers to use in our tests. + */ REQUESTED_BUNDLERS?: string; - JEST_SILENT?: '1'; + /** + * Defined by yarn and targets the root of the project. + */ + PROJECT_CWD?: string; } } } diff --git a/packages/tools/src/rollupConfig.mjs b/packages/tools/src/rollupConfig.mjs index 0f92613dd..17ed3c14b 100644 --- a/packages/tools/src/rollupConfig.mjs +++ b/packages/tools/src/rollupConfig.mjs @@ -2,6 +2,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +// @ts-check + import babel from '@rollup/plugin-babel'; import commonjs from '@rollup/plugin-commonjs'; import esmShim from '@rollup/plugin-esm-shim'; @@ -15,12 +17,32 @@ import path from 'path'; import dts from 'rollup-plugin-dts'; import esbuild from 'rollup-plugin-esbuild'; -const CWD = process.env.PROJECT_CWD; +const CWD = process.env.PROJECT_CWD || process.cwd(); + +/** + * @typedef {{ + * module: string; + * main: string; + * name: string; + * peerDependencies: Record; + * dependencies: Record + * }} PackageJson + * @typedef {import('rollup').InputPluginOption} InputPluginOption + * @typedef {import('rollup').Plugin} Plugin + * @typedef {import('@dd/core/types').Assign< + * import('rollup').RollupOptions, + * { + * external?: string[]; + * plugins?: InputPluginOption[]; + * } + * >} RollupOptions + * @typedef {import('rollup').OutputOptions} OutputOptions + */ /** - * @param {{module: string; main: string;}} packageJson - * @param {import('rollup').RollupOptions} config - * @returns {import('rollup').RollupOptions} + * @param {PackageJson} packageJson + * @param {RollupOptions} config + * @returns {RollupOptions} */ export const bundle = (packageJson, config) => ({ input: 'src/index.ts', @@ -31,7 +53,6 @@ export const bundle = (packageJson, config) => ({ // All dependencies are external dependencies. ...Object.keys(packageJson.dependencies), // These should be internal only and never be anywhere published. - '@dd/core', '@dd/tools', '@dd/tests', // We never want to include Node.js built-in modules in the bundle. @@ -53,14 +74,14 @@ export const bundle = (packageJson, config) => ({ json(), commonjs(), nodeResolve({ preferBuiltins: true }), - ...config.plugins, + ...(config.plugins || []), ], }); /** - * @param {{module: string; main: string;}} packageJson - * @param {Partial} overrides - * @returns {import('rollup').OutputOptions} + * @param {PackageJson} packageJson + * @param {Partial} overrides + * @returns {OutputOptions} */ const getOutput = (packageJson, overrides = {}) => { const filename = overrides.format === 'esm' ? packageJson.module : packageJson.main; @@ -88,13 +109,53 @@ const getOutput = (packageJson, overrides = {}) => { }; /** - * @param {{module: string; main: string;}} packageJson - * @returns {import('rollup').RollupOptions[]} + * @param {PackageJson} packageJson + * @returns {Promise} + */ +const getPlugins = async (packageJson) => { + /** + * @type {InputPluginOption[]} + */ + const plugins = [esbuild()]; + if (!process.env.ADD_BUILD_PLUGINS) { + return plugins; + } + try { + // This may not exist before at least one build. + // eslint-disable-next-line import/no-unresolved + const { datadogRollupPlugin } = await import('@datadog/rollup-plugin/dist/src'); + plugins.push( + datadogRollupPlugin({ + auth: {}, + logLevel: 'debug', + telemetry: { + prefix: 'build.rollup.build-plugins', + tags: [ + `package:${packageJson.name}`, + 'service:build-plugins', + `env:${process.env.BUILD_PLUGINS_ENV || 'development'}`, + `sha:${process.env.GITHUB_SHA || 'local'}`, + `ci:${process.env.CI ? 1 : 0}`, + ], + timestamp: Number(process.env.CI_PIPELINE_TIMESTAMP || Date.now()), + }, + }), + ); + } catch (e) { + console.log('Could not load @datadog/rollup-plugin, skipping.'); + } + return plugins; +}; + +/** + * @param {PackageJson} packageJson + * @returns {Promise} */ export const getDefaultBuildConfigs = async (packageJson) => { + const plugins = await getPlugins(packageJson); // Verify if we have anything else to build from plugins. const pkgs = glob.sync('packages/plugins/**/package.json', { cwd: CWD }); - const pluginBuilds = []; + const subBuilds = []; for (const pkg of pkgs) { const { default: content } = await import(path.resolve(CWD, pkg), { assert: { type: 'json' }, @@ -108,10 +169,10 @@ export const getDefaultBuildConfigs = async (packageJson) => { `Will also build ${chalk.green.bold(content.name)} additional files: ${chalk.green.bold(Object.keys(content.toBuild).join(', '))}`, ); - pluginBuilds.push( + subBuilds.push( ...Object.entries(content.toBuild).map(([name, config]) => { return bundle(packageJson, { - plugins: [esbuild()], + plugins, external: config.external, input: { [name]: path.join(CWD, path.dirname(pkg), config.entry), @@ -131,7 +192,7 @@ export const getDefaultBuildConfigs = async (packageJson) => { const configs = [ // Main bundle. bundle(packageJson, { - plugins: [esbuild()], + plugins, input: { index: 'src/index.ts', }, @@ -140,7 +201,7 @@ export const getDefaultBuildConfigs = async (packageJson) => { getOutput(packageJson, { format: 'cjs' }), ], }), - ...pluginBuilds, + ...subBuilds, // Bundle type definitions. // FIXME: This build is sloooow. // Check https://github.com/timocov/dts-bundle-generator From cdcdf832a7d57e9d789d41e85623b09707a2bd99 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 10:21:48 +0100 Subject: [PATCH 32/45] Output the working dir in case of no cleanup --- packages/tests/src/_jest/helpers/runBundlers.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/tests/src/_jest/helpers/runBundlers.ts b/packages/tests/src/_jest/helpers/runBundlers.ts index 299f6749d..2d7395874 100644 --- a/packages/tests/src/_jest/helpers/runBundlers.ts +++ b/packages/tests/src/_jest/helpers/runBundlers.ts @@ -231,6 +231,10 @@ export const runBundlers = async ( const workingDir = await prepareWorkingDir(seed); + if (NO_CLEANUP) { + console.log(`[NO_CLEANUP] Working directory: ${workingDir}`); + } + const bundlerOverridesResolved = typeof bundlerOverrides === 'function' ? bundlerOverrides(workingDir) From dbc407eb1a05b59e8cff0e5c3abdb5c7b988a168 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 17:29:06 +0100 Subject: [PATCH 33/45] Fix esbuild external reporting --- packages/plugins/build-report/src/esbuild.ts | 31 +++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index 29c4aff72..b8576e384 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -281,16 +281,16 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const root = isRelative ? path.dirname(filePath) : cwd; const absoluteImportPath = getAbsolutePath(root, imported.path); + // We need to register external imports, as this is the first time we see them. if (imported.external) { - if ( - isFileSupported(imported.path) && - !references.inputs.report[imported.path] - ) { - // If it's an absolute external import, we can't trust our own getAbsolutePath(). + if (isFileSupported(imported.path)) { + // If it's an absolute external import, + // we can't trust our own getAbsolutePath(). // We can't know what its "root" could be. const filepath = isRelative ? absoluteImportPath : imported.path; + // But we can still add it to the report. - const inputFile: Input = { + const inputFile: Input = references.inputs.report[filepath] || { filepath, name: cleanName(context, imported.path), size: 0, @@ -298,8 +298,25 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt dependencies: new Set(), dependents: new Set(), }; + + if ('dependencies' in file) { + // file is an Input, so we add the external to its dependencies, + // and we add file to the external's dependents. + inputFile.dependents.add(file); + file.dependencies.add(inputFile); + } + + if ('inputs' in file && !file.inputs.includes(inputFile)) { + // file is an Output, so we add the external to its inputs. + file.inputs.push(inputFile); + } + + if (!inputs.includes(inputFile)) { + inputs.push(inputFile); + } + references.inputs.report[filepath] = inputFile; - inputs.push(inputFile); + allImports[inputFile.filepath] = inputFile as T; } // We can't follow external imports. continue; From 4978e0481da398f3f17d9e2d1e2f5833ddc3f885 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 17:29:47 +0100 Subject: [PATCH 34/45] Add external reporting to xpack --- packages/plugins/build-report/src/xpack.ts | 62 ++++++++++++++++++---- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/packages/plugins/build-report/src/xpack.ts b/packages/plugins/build-report/src/xpack.ts index 6f4a06c9e..57c27b528 100644 --- a/packages/plugins/build-report/src/xpack.ts +++ b/packages/plugins/build-report/src/xpack.ts @@ -63,6 +63,12 @@ export const getXpackPlugin = * 2. Once the build is finished and emitted, we can compute the outputs and the entries. */ + const cleanExternalName = (name: string) => { + // Removes "external " prefix and surrounding quotes from external dependency names + // Example: 'external var "lodash"' -> 'lodash' + return name.replace(/(^external[^"]+"|"$)/g, ''); + }; + // Index the module by its identifier, resource, request, rawRequest, and userRequest. const getKeysToIndex = (mod: Module): Set => { const values: Record = { @@ -96,6 +102,11 @@ export const getXpackPlugin = } } else { keysToIndex.add(value); + // RSpack only use "external ..." for external dependencies. + // So we need to clean and add the actual name to the index too. + if (value.startsWith('external ')) { + keysToIndex.add(cleanExternalName(value)); + } } } @@ -134,6 +145,19 @@ export const getXpackPlugin = } }; + const isExternal = (mod: Module) => { + if ('externalType' in mod && mod.externalType) { + return true; + } + if ('external' in mod && mod.external) { + return true; + } + if (mod.identifier?.().startsWith('external ')) { + return true; + } + return false; + }; + // Intercept the compilation to then get the modules. compiler.hooks.thisCompilation.tap(PLUGIN_NAME, (compilation) => { // Intercept the modules to build the dependency graph. @@ -155,13 +179,14 @@ export const getXpackPlugin = const inputEnd = log.time('building inputs'); for (const module of finishedModules) { const moduleIdentifier = module.identifier(); + const moduleName = cleanName(context, moduleIdentifier); const dependencies: Set = new Set( getAllDependencies(module) .map((dep) => { const mod = getModuleFromDep(module, dep); // Ignore those we can't identify. - if (!mod || !mod.identifier()) { + if (!mod?.identifier()) { return false; } @@ -177,7 +202,9 @@ export const getXpackPlugin = return false; } - return identifier; + return isExternal(mod) + ? cleanExternalName(identifier) + : identifier; }) .filter(Boolean) as string[], ); @@ -209,16 +236,31 @@ export const getXpackPlugin = tempDeps.set(moduleIdentifier, moduleDeps); // Store the inputs. - const file: Input = { - size: module.size() || 0, - name: cleanName(context, moduleIdentifier), - dependencies: new Set(), - dependents: new Set(), - filepath: moduleIdentifier, - type: getType(moduleIdentifier), - }; + const file: Input = isExternal(module) + ? { + size: 0, + name: cleanExternalName(moduleName), + dependencies: new Set(), + dependents: new Set(), + filepath: moduleIdentifier, + type: 'external', + } + : { + size: module.size() || 0, + name: moduleName, + dependencies: new Set(), + dependents: new Set(), + filepath: moduleIdentifier, + type: getType(moduleIdentifier), + }; + inputs.push(file); reportInputsIndexed.set(moduleIdentifier, file); + + // If it's an external dependency, we also need to index it by its cleaned name. + if (isExternal(module)) { + reportInputsIndexed.set(cleanExternalName(moduleIdentifier), file); + } } inputEnd(); From e5b6667a891f870bf7d0058731b9cd559ec114a3 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 17:31:39 +0100 Subject: [PATCH 35/45] Cover externals in tests --- .../unit/plugins/build-report/index.test.ts | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/packages/tests/src/unit/plugins/build-report/index.test.ts b/packages/tests/src/unit/plugins/build-report/index.test.ts index bf2e25262..d9f216379 100644 --- a/packages/tests/src/unit/plugins/build-report/index.test.ts +++ b/packages/tests/src/unit/plugins/build-report/index.test.ts @@ -54,7 +54,12 @@ const getPluginConfig: ( }; }; +const isFileThirdParty = (file: Input | Output) => { + return file.filepath.includes('node_modules') || file.type === 'external'; +}; + describe('Build Report Plugin', () => { + // TODO: Account for external dependencies, and test namings. describe('Basic build', () => { const bundlerOutdir: Record = {}; const buildReports: Record = {}; @@ -173,9 +178,27 @@ describe('Build Report Plugin', () => { let workingDir: string; beforeAll(async () => { + // Mark some dependencies as external to ensure it's correctly reported too. + const rollupExternals = { + external: ['supports-color'], + }; + const xpackExternals = { + externals: { + 'supports-color': 'supports-color', + }, + }; const result = await runBundlers( getPluginConfig(bundlerOutdir, buildReports), - getComplexBuildOverrides(), + getComplexBuildOverrides({ + rollup: rollupExternals, + vite: rollupExternals, + webpack4: xpackExternals, + webpack5: xpackExternals, + rspack: xpackExternals, + esbuild: { + external: ['supports-color'], + }, + }), ); workingDir = result.workingDir; }); @@ -221,7 +244,7 @@ describe('Build Report Plugin', () => { 'hard_project/src/srcFile1.js', 'hard_project/workspaces/app/workspaceFile0.js', 'hard_project/workspaces/app/workspaceFile1.js', - 'supports-color/browser.js', + 'supports-color', ]); }); @@ -232,9 +255,7 @@ describe('Build Report Plugin', () => { .sort(sortFiles); // Only list the common dependencies and remove any particularities from bundlers. - const thirdParties = inputs!.filter((input) => - input.filepath.includes('node_modules'), - ); + const thirdParties = inputs!.filter((input) => isFileThirdParty(input)); expect(thirdParties.map((d) => d.name).sort()).toEqual([ 'ansi-styles/index.js', @@ -245,7 +266,7 @@ describe('Build Report Plugin', () => { 'color-convert/route.js', 'color-name/index.js', 'escape-string-regexp/index.js', - 'supports-color/browser.js', + 'supports-color', ]); }); @@ -293,7 +314,7 @@ describe('Build Report Plugin', () => { 'ansi-styles/index.js', 'chalk/templates.js', 'escape-string-regexp/index.js', - 'supports-color/browser.js', + 'supports-color', ], // It should also have a single dependent which is main1. dependents: ['hard_project/main1.js'], @@ -450,10 +471,10 @@ describe('Build Report Plugin', () => { )!; const entryInputs = entry.inputs.filter(filterOutParticularities); const dependencies = entryInputs.filter((input) => - input.filepath.includes('node_modules'), + isFileThirdParty(input), ); const mainFiles = entryInputs.filter( - (input) => !input.filepath.includes('node_modules'), + (input) => !isFileThirdParty(input), ); expect(dependencies).toHaveLength(dependenciesLength); From 0634b2aad02f98fcaf945308083247d6adfe8792 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 17:48:20 +0100 Subject: [PATCH 36/45] Fix rollup's build-report on inputs/outputs --- packages/plugins/build-report/src/rollup.ts | 24 ++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/plugins/build-report/src/rollup.ts b/packages/plugins/build-report/src/rollup.ts index 0a994f7a5..8285f0eb5 100644 --- a/packages/plugins/build-report/src/rollup.ts +++ b/packages/plugins/build-report/src/rollup.ts @@ -61,6 +61,7 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti const outputs: Output[] = []; const tempEntryFiles: Entry[] = []; const tempSourcemaps: Output[] = []; + const tempOutputsImports: Record = {}; const entries: Entry[] = []; const reportInputsIndexed: Record = {}; @@ -128,10 +129,9 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti if ('modules' in asset) { for (const [modulepath, module] of Object.entries(asset.modules)) { - // We don't want to include commonjs wrappers that have a path like: + // We don't want to include commonjs wrappers and proxies that are like: // \u0000{{path}}?commonjs-proxy if (cleanPath(modulepath) !== modulepath) { - log.debug(`Not including ${modulepath} in the report.`); continue; } const moduleFile: Input = { @@ -157,9 +157,11 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti const cleanedImport = cleanPath(importName); const importReport = importsReport[cleanedImport]; if (!importReport) { - log.debug( - `Could not find the import report for ${cleanedImport} from ${file.name}.`, - ); + // We may not have this yet as it could be one of the chunks + // produced by the current build. + tempOutputsImports[ + getAbsolutePath(context.bundler.outDir, cleanedImport) + ] = file; continue; } @@ -197,6 +199,18 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti } inputsOutputsEnd(); + for (const [filepath, output] of Object.entries(tempOutputsImports)) { + const outputReport = reportOutputsIndexed[filepath]; + if (!outputReport) { + log.debug(`Could not find the output report for ${filepath}.`); + continue; + } + + if (!output.inputs.includes(outputReport)) { + output.inputs.push(outputReport); + } + } + // Fill in inputs' dependencies and dependents. const depsEnd = log.time('filling dependencies and dependents'); for (const input of inputs) { From 4c64d6f14c87bfaa80ce352ce60b8d344eb7aed1 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 17:49:32 +0100 Subject: [PATCH 37/45] Remove unused dependencies --- packages/tests/src/_jest/fixtures/hard_project/package.json | 5 ----- packages/tests/src/_jest/fixtures/yarn.lock | 3 --- 2 files changed, 8 deletions(-) diff --git a/packages/tests/src/_jest/fixtures/hard_project/package.json b/packages/tests/src/_jest/fixtures/hard_project/package.json index ac25238f5..5574de16a 100644 --- a/packages/tests/src/_jest/fixtures/hard_project/package.json +++ b/packages/tests/src/_jest/fixtures/hard_project/package.json @@ -6,10 +6,5 @@ "packageManager": "yarn@4.2.1", "dependencies": { "chalk": "2.3.1" - }, - "devDependencies": { - "react": "19.0.0", - "react-dom": "19.0.0", - "react-router-dom": "6.28.0" } } diff --git a/packages/tests/src/_jest/fixtures/yarn.lock b/packages/tests/src/_jest/fixtures/yarn.lock index e8d79974f..ee71a905c 100644 --- a/packages/tests/src/_jest/fixtures/yarn.lock +++ b/packages/tests/src/_jest/fixtures/yarn.lock @@ -34,9 +34,6 @@ __metadata: resolution: "@tests/hard_project@workspace:hard_project" dependencies: chalk: "npm:2.3.1" - react: "npm:19.0.0" - react-dom: "npm:19.0.0" - react-router-dom: "npm:6.28.0" languageName: unknown linkType: soft From df9bdae684e39885c204b0a96aea03d83256a685 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 11 Mar 2025 18:02:21 +0100 Subject: [PATCH 38/45] Remove values from tests as bundlers can act differently --- packages/tests/src/unit/plugins/telemetry/index.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tests/src/unit/plugins/telemetry/index.test.ts b/packages/tests/src/unit/plugins/telemetry/index.test.ts index ec85a721b..4372fe5b9 100644 --- a/packages/tests/src/unit/plugins/telemetry/index.test.ts +++ b/packages/tests/src/unit/plugins/telemetry/index.test.ts @@ -212,17 +212,17 @@ describe('Telemetry Universal Plugin', () => { describe('Entry metrics', () => { test.each([ { metric: 'entries.size', tags: ['entryName:app1'] }, - { metric: 'entries.modules.count', tags: ['entryName:app1'], value: 13 }, + { metric: 'entries.modules.count', tags: ['entryName:app1'] }, { metric: 'entries.assets.count', tags: ['entryName:app1'] }, { metric: 'entries.size', tags: ['entryName:app2'] }, - { metric: 'entries.modules.count', tags: ['entryName:app2'], value: 5 }, + { metric: 'entries.modules.count', tags: ['entryName:app2'] }, { metric: 'entries.assets.count', tags: ['entryName:app2'] }, - ])('Should have $metric with $tags', ({ metric, tags, value }) => { + ])('Should have $metric with $tags', ({ metric, tags }) => { const entryMetrics = metrics[name].filter((m) => m.metric.startsWith('entries'), ); - const metricToTest = getMetric(metric, tags, value); + const metricToTest = getMetric(metric, tags); const foundMetrics = entryMetrics.filter( (m) => m.metric === metric && tags.every((t) => m.tags.includes(t)), ); From e7905c6d6d32cca7677b776af828aa35d1b302b9 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 09:58:09 +0100 Subject: [PATCH 39/45] Revert logger.time helper will be added in a separate PR --- packages/core/src/types.ts | 2 - packages/factory/src/helpers.ts | 19 ---------- packages/plugins/build-report/src/esbuild.ts | 18 --------- packages/plugins/build-report/src/rollup.ts | 12 ------ packages/plugins/build-report/src/xpack.ts | 18 --------- .../telemetry/src/esbuild-plugin/index.ts | 4 -- packages/plugins/telemetry/src/index.ts | 15 ++------ packages/tests/src/_jest/helpers/mocks.ts | 2 - .../tests/src/unit/factory/helpers.test.ts | 38 ------------------- 9 files changed, 3 insertions(+), 125 deletions(-) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 22008bf83..b8600142d 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -99,8 +99,6 @@ export type ToInjectItem = { export type GetLogger = (name: string) => Logger; export type Logger = { getLogger: GetLogger; - time: (label: string, level?: LogLevel) => () => void; - timeEnd: (label: string, level?: LogLevel) => void; error: (text: any) => void; warn: (text: any) => void; info: (text: any) => void; diff --git a/packages/factory/src/helpers.ts b/packages/factory/src/helpers.ts index 828dc02aa..ec532e5a3 100644 --- a/packages/factory/src/helpers.ts +++ b/packages/factory/src/helpers.ts @@ -3,7 +3,6 @@ // Copyright 2019-Present Datadog, Inc. import { ALL_ENVS } from '@dd/core/constants'; -import { formatDuration } from '@dd/core/helpers'; import type { BuildReport, BundlerFullName, @@ -13,7 +12,6 @@ import type { GetLogger, GlobalContext, LogLevel, - Logger, Options, OptionsWithDefaults, } from '@dd/core/types'; @@ -83,28 +81,11 @@ export const getLoggerFactory = } }; - const timerMap = new Map(); - const timeEnd: Logger['timeEnd'] = (label: string, level: LogLevel = 'debug') => { - const start = timerMap.get(label); - if (start) { - timerMap.delete(label); - const duration = Date.now() - start.timestamp; - log(`[${c.cyan(label)}] : ${c.cyan(formatDuration(duration))}`, level); - } - }; - const time: Logger['time'] = (label: string, level: LogLevel = 'debug') => { - log(c.dim(`[${c.cyan(label)}] : start`), level); - timerMap.set(label, { timestamp: Date.now(), logLevel: level }); - return () => timeEnd(label, level); - }; - return { getLogger: (subName: string) => { const logger = getLoggerFactory(build, logLevel); return logger(`${cleanedName}${NAME_SEP}${subName}`); }, - time, - timeEnd, error: (text: any) => log(text, 'error'), warn: (text: any) => log(text, 'warn'), info: (text: any) => log(text, 'info'), diff --git a/packages/plugins/build-report/src/esbuild.ts b/packages/plugins/build-report/src/esbuild.ts index b8576e384..df40510c4 100644 --- a/packages/plugins/build-report/src/esbuild.ts +++ b/packages/plugins/build-report/src/esbuild.ts @@ -32,7 +32,6 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const resolvedEntries: ResolvedEntry[] = []; build.onStart(async () => { - const timeEnd = log.time('process entries'); // Store entry names based on the configuration. resolvedEntries.push(...(await getEsbuildEntries(build, context, log))); for (const entry of resolvedEntries) { @@ -43,12 +42,9 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt entryNames.set(cleanedName, cleanedName); } } - timeEnd(); }); build.onEnd((result) => { - const buildReportEnd = log.time('build report'); - const collectTimeEnd = log.time('collecting errors and warnings'); const cwd = context.cwd; for (const error of result.errors) { context.build.errors.push(error.text); @@ -56,7 +52,6 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt for (const warning of result.warnings) { context.build.warnings.push(warning.text); } - collectTimeEnd(); if (!result.metafile) { const warning = 'Missing metafile from build report.'; @@ -74,10 +69,8 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const reportInputsIndexed: Record = {}; const reportOutputsIndexed: Record = {}; - const indexTimeEnd = log.time('indexing metafile data'); const metaInputsIndexed = reIndexMeta(result.metafile.inputs, cwd); const metaOutputsIndexed = reIndexMeta(result.metafile.outputs, cwd); - indexTimeEnd(); // From a proxy entry point, created by our injection plugin, get the real path. const getRealPathFromInjectionProxy = (entryPoint: string): string => { @@ -102,7 +95,6 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt }; // Loop through inputs. - const inputsTimeEnd = log.time('looping through inputs'); for (const [filename, input] of Object.entries(result.metafile.inputs)) { if (isInjectionFile(filename)) { continue; @@ -122,10 +114,8 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt reportInputsIndexed[filepath] = file; inputs.push(file); } - inputsTimeEnd(); // Loop through outputs. - const outputTimeEnd = log.time('looping through outputs'); for (const [filename, output] of Object.entries(result.metafile.outputs)) { const fullPath = getAbsolutePath(cwd, filename); const cleanedName = cleanName(context, fullPath); @@ -204,10 +194,8 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt tempEntryFiles.push(entry); } } - outputTimeEnd(); // Loop through sourcemaps. - const sourcemapsTimeEnd = log.time('looping through sourcemaps'); for (const sourcemap of tempSourcemaps) { const outputFilepath = sourcemap.filepath.replace(/\.map$/, ''); const foundOutput = reportOutputsIndexed[outputFilepath]; @@ -219,7 +207,6 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt sourcemap.inputs.push(foundOutput); } - sourcemapsTimeEnd(); // Build our references for the entries. const references = { @@ -330,7 +317,6 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt }; // Loop through entries. - const entriesTimeEnd = log.time('looping through entries'); // TODO This is slightly underperformant due to getAllImports' recursivity. for (const entryFile of tempEntryFiles) { const entryInputs: Record = {}; @@ -359,10 +345,8 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt entries.push(entryFile); } - entriesTimeEnd(); // Loop through all inputs to aggregate dependencies and dependents. - const depsTimeEnd = log.time('aggregate dependencies and dependents'); for (const input of inputs) { // The metafile does not contain external dependencies. // So we can only fill in their dependents. @@ -408,12 +392,10 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt dependencyFile.dependents.add(input); } } - depsTimeEnd(); context.build.outputs = outputs; context.build.inputs = inputs; context.build.entries = entries; - buildReportEnd(); }); }, }; diff --git a/packages/plugins/build-report/src/rollup.ts b/packages/plugins/build-report/src/rollup.ts index 8285f0eb5..adce15942 100644 --- a/packages/plugins/build-report/src/rollup.ts +++ b/packages/plugins/build-report/src/rollup.ts @@ -56,7 +56,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti importsReport[cleanId] = report; }, writeBundle(options, bundle) { - const buildReportEnd = log.time('build report'); const inputs: Input[] = []; const outputs: Output[] = []; const tempEntryFiles: Entry[] = []; @@ -68,7 +67,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti const reportOutputsIndexed: Record = {}; // Complete the importsReport with missing dependents and dependencies. - const depsCompleteEnd = log.time('completing dependencies and dependents'); for (const [filepath, { dependencies, dependents }] of Object.entries(importsReport)) { for (const dependency of dependencies) { const cleanedDependency = cleanPath(dependency); @@ -102,10 +100,8 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti importsReport[cleanedDependent].dependencies.add(filepath); } } - depsCompleteEnd(); // Fill in inputs and outputs. - const inputsOutputsEnd = log.time('filling inputs and outputs'); for (const [filename, asset] of Object.entries(bundle)) { const filepath = getAbsolutePath(context.bundler.outDir, filename); const size = @@ -197,7 +193,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti reportOutputsIndexed[file.filepath] = file; outputs.push(file); } - inputsOutputsEnd(); for (const [filepath, output] of Object.entries(tempOutputsImports)) { const outputReport = reportOutputsIndexed[filepath]; @@ -212,7 +207,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti } // Fill in inputs' dependencies and dependents. - const depsEnd = log.time('filling dependencies and dependents'); for (const input of inputs) { const importReport = importsReport[input.filepath]; if (!importReport) { @@ -242,11 +236,9 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti input.dependents.add(foundInput); } } - depsEnd(); // Fill in sourcemaps' inputs if necessary if (tempSourcemaps.length) { - const sourcemapsEnd = log.time('filling sourcemaps inputs'); for (const sourcemap of tempSourcemaps) { const outputPath = sourcemap.filepath.replace(/\.map$/, ''); const foundOutput = reportOutputsIndexed[outputPath]; @@ -258,7 +250,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti sourcemap.inputs.push(foundOutput); } - sourcemapsEnd(); } // Gather all outputs from a filepath, following imports. @@ -306,7 +297,6 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti }; // Fill in entries - const entriesEnd = log.time('filling entries'); for (const entryFile of tempEntryFiles) { const entryOutputs = getAllOutputs(entryFile.filepath); entryFile.outputs = Object.values(entryOutputs); @@ -319,12 +309,10 @@ export const getRollupPlugin = (context: GlobalContext, log: Logger): PluginOpti entryFile.size = entryFile.outputs.reduce((acc, output) => acc + output.size, 0); entries.push(entryFile); } - entriesEnd(); context.build.inputs = inputs; context.build.outputs = outputs; context.build.entries = entries; - buildReportEnd(); }, }; }; diff --git a/packages/plugins/build-report/src/xpack.ts b/packages/plugins/build-report/src/xpack.ts index 57c27b528..a3481110e 100644 --- a/packages/plugins/build-report/src/xpack.ts +++ b/packages/plugins/build-report/src/xpack.ts @@ -164,19 +164,15 @@ export const getXpackPlugin = compilation.hooks.finishModules.tap( PLUGIN_NAME, (finishedModules: Iterable) => { - const graphEnd = log.time('dependency graph'); // First loop to create indexes. - const indexEnd = log.time('indexing modules'); for (const module of finishedModules) { const keysToIndex = getKeysToIndex(module); for (const key of keysToIndex) { moduleIndex.set(key, module); } } - indexEnd(); // Second loop to create the dependency graph. - const inputEnd = log.time('building inputs'); for (const module of finishedModules) { const moduleIdentifier = module.identifier(); const moduleName = cleanName(context, moduleIdentifier); @@ -262,10 +258,8 @@ export const getXpackPlugin = reportInputsIndexed.set(cleanExternalName(moduleIdentifier), file); } } - inputEnd(); // Assign dependencies and dependents. - const assignEnd = log.time('assigning dependencies and dependents'); for (const input of inputs) { const depsReport = tempDeps.get(input.filepath); @@ -292,14 +286,11 @@ export const getXpackPlugin = input.dependents.add(depInput); } } - assignEnd(); - graphEnd(); }, ); }); compiler.hooks.afterEmit.tap(PLUGIN_NAME, (result: Compilation) => { - const reportEnd = log.time('build report'); const chunks = result.chunks; const assets = result.getAssets(); @@ -309,7 +300,6 @@ export const getXpackPlugin = ); }; - const chunkEnd = log.time('indexing chunks'); const chunkGraph = result.chunkGraph; for (const chunk of chunks) { const files = getChunkFiles(chunk); @@ -339,10 +329,8 @@ export const getXpackPlugin = modulesPerFile.set(file, [...fileModules, ...chunkModules]); } } - chunkEnd(); // Build outputs - const outputEnd = log.time('building outputs'); for (const asset of assets) { const file: Output = { size: asset.source.size() || 0, @@ -378,10 +366,8 @@ export const getXpackPlugin = file.inputs.push(inputFound); } } - outputEnd(); // Fill in inputs for sourcemaps. - const sourcemapsEnd = log.time('filling sourcemaps inputs'); for (const sourcemap of tempSourcemaps) { const outputFound = reportOutputsIndexed.get( sourcemap.filepath.replace(/\.map$/, ''), @@ -394,10 +380,8 @@ export const getXpackPlugin = sourcemap.inputs.push(outputFound); } - sourcemapsEnd(); // Build entries - const entriesEnd = log.time('building entries'); for (const [name, entrypoint] of result.entrypoints) { const entryOutputs: Output[] = []; const entryInputs: Input[] = []; @@ -454,7 +438,6 @@ export const getXpackPlugin = entries.push(file); } - entriesEnd(); // Save everything in the context. for (const error of result.errors) { @@ -466,6 +449,5 @@ export const getXpackPlugin = context.build.inputs = inputs; context.build.outputs = outputs; context.build.entries = entries; - reportEnd(); }); }; diff --git a/packages/plugins/telemetry/src/esbuild-plugin/index.ts b/packages/plugins/telemetry/src/esbuild-plugin/index.ts index 1d873a0a2..b617b80ba 100644 --- a/packages/plugins/telemetry/src/esbuild-plugin/index.ts +++ b/packages/plugins/telemetry/src/esbuild-plugin/index.ts @@ -19,18 +19,14 @@ export const getEsbuildPlugin = ( // We force esbuild to produce its metafile. build.initialOptions.metafile = true; - const wrapTimeEnd = logger.time('wrapping plugins'); wrapPlugins(build, globalContext.cwd); - wrapTimeEnd(); build.onEnd(async (result: BuildResult) => { if (!result.metafile) { logger.warn("Missing metafile, can't proceed with modules data."); return; } - const resultTimeEnd = logger.time('getting plugins results'); const { plugins, modules } = getPluginsResults(); - resultTimeEnd(); bundlerContext.report = { timings: { diff --git a/packages/plugins/telemetry/src/index.ts b/packages/plugins/telemetry/src/index.ts index f9686f8aa..e8096d7b5 100644 --- a/packages/plugins/telemetry/src/index.ts +++ b/packages/plugins/telemetry/src/index.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import type { GlobalContext, GetPlugins, PluginOptions, Logger } from '@dd/core/types'; +import type { GlobalContext, GetPlugins, PluginOptions } from '@dd/core/types'; import { addMetrics } from './common/aggregator'; import { defaultFilters } from './common/filters'; @@ -56,17 +56,15 @@ export const getPlugins: GetPlugins = ( webpack: getWebpackPlugin(bundlerContext, context), rspack: getWebpackPlugin(bundlerContext, context), }; - let buildTimeEnd: ReturnType; + // Universal plugin. const universalPlugin: PluginOptions = { name: 'datadog-universal-telemetry-plugin', enforce: 'post', buildStart() { - buildTimeEnd = log.time('build'); context.build.start = context.build.start || Date.now(); }, buildEnd() { - buildTimeEnd(); realBuildEnd = Date.now(); }, @@ -79,30 +77,23 @@ export const getPlugins: GetPlugins = ( const metrics: Set = new Set(); const optionsDD = getOptionsDD(telemetryOptions); - const metricsTimeEnd = log.time(`adding metrics`); addMetrics(context, optionsDD, metrics, bundlerContext.report); - metricsTimeEnd(); // TODO Extract the files output in an internal plugin. - const writeTimeEnd = log.time(`writing to files`); await outputFiles( { report: bundlerContext.report, metrics }, telemetryOptions.output, log, context.bundler.outDir, ); - writeTimeEnd(); - const reportTimeEnd = log.time('outputing report'); + outputTexts(context, log, bundlerContext.report); - reportTimeEnd(); - const sendTimeEnd = log.time('sending metrics to Datadog'); await sendMetrics( metrics, { apiKey: context.auth?.apiKey, endPoint: telemetryOptions.endPoint }, log, ); - sendTimeEnd(); }, }; diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index 96acc1975..b1864f4cb 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -40,8 +40,6 @@ export const defaultPluginOptions: GetPluginsOptions = { export const mockLogFn = jest.fn((text: any, level: LogLevel) => {}); const logFn: Logger = { getLogger: jest.fn(), - time: jest.fn(), - timeEnd: jest.fn(), error: (text: any) => { mockLogFn(text, 'error'); }, diff --git a/packages/tests/src/unit/factory/helpers.test.ts b/packages/tests/src/unit/factory/helpers.test.ts index 183293f69..2d74c2552 100644 --- a/packages/tests/src/unit/factory/helpers.test.ts +++ b/packages/tests/src/unit/factory/helpers.test.ts @@ -169,44 +169,6 @@ describe('Factory Helpers', () => { }); }); - describe('Time logger', () => { - test('Should log a duration.', () => { - const [logger] = setupLogger('testLogger'); - // Basic usage. - logger.time('test time 1'); - logger.timeEnd('test time 1'); - - // Using the return timeEnd function. - const timeEnd = logger.time('test time 2'); - timeEnd(); - - // Use a specific log level. - const timeEnd2 = logger.time('test time 3', 'error'); - timeEnd2(); - - expect(logMock).toHaveBeenCalledTimes(4); - expect(errorMock).toHaveBeenCalledTimes(2); - expect(getOutput(logMock, 0)).toBe( - `[debug|esbuild|testLogger] [test time 1] : start`, - ); - expect(getOutput(logMock, 1)).toBe( - `[debug|esbuild|testLogger] [test time 1] : 0ms`, - ); - expect(getOutput(logMock, 2)).toBe( - `[debug|esbuild|testLogger] [test time 2] : start`, - ); - expect(getOutput(logMock, 3)).toBe( - `[debug|esbuild|testLogger] [test time 2] : 0ms`, - ); - expect(getOutput(errorMock, 0)).toBe( - `[error|esbuild|testLogger] [test time 3] : start`, - ); - expect(getOutput(errorMock, 1)).toBe( - `[error|esbuild|testLogger] [test time 3] : 0ms`, - ); - }); - }); - describe('Sub logger', () => { test('Should return a logger factory.', () => { const [logger] = setupLogger('testLogger'); From a91d4d869c2997347700d564c0a467e31143d597 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 10:09:53 +0100 Subject: [PATCH 40/45] Cleaning commented lines --- packages/plugins/injection/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/plugins/injection/src/index.ts b/packages/plugins/injection/src/index.ts index f5fd4357d..36ddd72a0 100644 --- a/packages/plugins/injection/src/index.ts +++ b/packages/plugins/injection/src/index.ts @@ -53,7 +53,6 @@ export const getInjectionPlugins = (bundler: any, context: GlobalContext): Plugi if (isXpack(context.bundler.fullName)) { plugin.loadInclude = (id) => { if (isInjectionFile(id)) { - // console.log('loadInclude', id); return true; } @@ -62,7 +61,6 @@ export const getInjectionPlugins = (bundler: any, context: GlobalContext): Plugi plugin.load = (id) => { if (isInjectionFile(id)) { - // console.log('load', id); return { code: getContentToInject(contentsToInject[InjectPosition.MIDDLE]), }; From 794daf9acd631069a9322a618bf9cadef7e0cbdb Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 10:20:15 +0100 Subject: [PATCH 41/45] Some more cleaning --- packages/tests/src/unit/plugins/build-report/index.test.ts | 1 - .../src/unit/tools/src/commands/create-plugin/index.test.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/tests/src/unit/plugins/build-report/index.test.ts b/packages/tests/src/unit/plugins/build-report/index.test.ts index d9f216379..9e72368ae 100644 --- a/packages/tests/src/unit/plugins/build-report/index.test.ts +++ b/packages/tests/src/unit/plugins/build-report/index.test.ts @@ -59,7 +59,6 @@ const isFileThirdParty = (file: Input | Output) => { }; describe('Build Report Plugin', () => { - // TODO: Account for external dependencies, and test namings. describe('Basic build', () => { const bundlerOutdir: Record = {}; const buildReports: Record = {}; diff --git a/packages/tests/src/unit/tools/src/commands/create-plugin/index.test.ts b/packages/tests/src/unit/tools/src/commands/create-plugin/index.test.ts index b050f2943..28d95554a 100644 --- a/packages/tests/src/unit/tools/src/commands/create-plugin/index.test.ts +++ b/packages/tests/src/unit/tools/src/commands/create-plugin/index.test.ts @@ -6,6 +6,7 @@ import { getMirroredFixtures } from '@dd/tests/_jest/helpers/mocks'; import commands from '@dd/tools/commands/create-plugin/index'; import { ROOT } from '@dd/tools/constants'; import { Cli } from 'clipanion'; +import { vol } from 'memfs'; jest.mock('fs', () => require('memfs').fs); @@ -19,12 +20,11 @@ describe('Command create-plugin', () => { beforeEach(() => { // Mock the files that are touched by yarn cli create-plugin. - // FIXME: Using require here because clipanion + memfs somehow breaks memfs' singleton. - require('memfs').vol.fromJSON(fixtures, ROOT); + vol.fromJSON(fixtures, ROOT); }); afterEach(() => { - require('memfs').vol.reset(); + vol.reset(); }); const cases = [ From 59c968aac43b59bfff93111a5718171a3a235fb3 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 10:21:58 +0100 Subject: [PATCH 42/45] Revert meta plugin --- packages/tools/src/rollupConfig.mjs | 44 ++--------------------------- 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/packages/tools/src/rollupConfig.mjs b/packages/tools/src/rollupConfig.mjs index 17ed3c14b..116f22ada 100644 --- a/packages/tools/src/rollupConfig.mjs +++ b/packages/tools/src/rollupConfig.mjs @@ -108,51 +108,11 @@ const getOutput = (packageJson, overrides = {}) => { }; }; -/** - * @param {PackageJson} packageJson - * @returns {Promise} - */ -const getPlugins = async (packageJson) => { - /** - * @type {InputPluginOption[]} - */ - const plugins = [esbuild()]; - if (!process.env.ADD_BUILD_PLUGINS) { - return plugins; - } - try { - // This may not exist before at least one build. - // eslint-disable-next-line import/no-unresolved - const { datadogRollupPlugin } = await import('@datadog/rollup-plugin/dist/src'); - plugins.push( - datadogRollupPlugin({ - auth: {}, - logLevel: 'debug', - telemetry: { - prefix: 'build.rollup.build-plugins', - tags: [ - `package:${packageJson.name}`, - 'service:build-plugins', - `env:${process.env.BUILD_PLUGINS_ENV || 'development'}`, - `sha:${process.env.GITHUB_SHA || 'local'}`, - `ci:${process.env.CI ? 1 : 0}`, - ], - timestamp: Number(process.env.CI_PIPELINE_TIMESTAMP || Date.now()), - }, - }), - ); - } catch (e) { - console.log('Could not load @datadog/rollup-plugin, skipping.'); - } - return plugins; -}; - /** * @param {PackageJson} packageJson * @returns {Promise} */ export const getDefaultBuildConfigs = async (packageJson) => { - const plugins = await getPlugins(packageJson); // Verify if we have anything else to build from plugins. const pkgs = glob.sync('packages/plugins/**/package.json', { cwd: CWD }); const subBuilds = []; @@ -172,7 +132,7 @@ export const getDefaultBuildConfigs = async (packageJson) => { subBuilds.push( ...Object.entries(content.toBuild).map(([name, config]) => { return bundle(packageJson, { - plugins, + plugins: [esbuild()], external: config.external, input: { [name]: path.join(CWD, path.dirname(pkg), config.entry), @@ -192,7 +152,7 @@ export const getDefaultBuildConfigs = async (packageJson) => { const configs = [ // Main bundle. bundle(packageJson, { - plugins, + plugins: [esbuild()], input: { index: 'src/index.ts', }, From 5446e31d8d78303d3217c80ef6ce35d15bdbd851 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 10:22:57 +0100 Subject: [PATCH 43/45] Remove comment about solution that isn't working --- global.d.ts | 8 -------- packages/tools/src/rollupConfig.mjs | 1 - 2 files changed, 9 deletions(-) diff --git a/global.d.ts b/global.d.ts index 74d36eedc..c3ac3c956 100644 --- a/global.d.ts +++ b/global.d.ts @@ -3,14 +3,6 @@ import type { Env } from '@dd/core/types'; declare global { namespace NodeJS { interface ProcessEnv extends NodeJS.ProcessEnv { - /** - * To use when building the plugins with `yarn build:all`. - * - * If passed, the build will include our rollup's plugin to the build. - * - * Be sure to have build rollup's plugin with `yarn workspace @datadog/rollup-plugin build`. - */ - ADD_BUILD_PLUGINS?: '1'; /** * The environment in which the plugins will execute. * diff --git a/packages/tools/src/rollupConfig.mjs b/packages/tools/src/rollupConfig.mjs index 116f22ada..edee58cda 100644 --- a/packages/tools/src/rollupConfig.mjs +++ b/packages/tools/src/rollupConfig.mjs @@ -164,7 +164,6 @@ export const getDefaultBuildConfigs = async (packageJson) => { ...subBuilds, // Bundle type definitions. // FIXME: This build is sloooow. - // Check https://github.com/timocov/dts-bundle-generator bundle(packageJson, { plugins: [dts()], output: { From 731a560857f9e204f111c2f455c78804e0e4aff7 Mon Sep 17 00:00:00 2001 From: Yoann Moinet <597828+yoannmoinet@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:55:50 +0100 Subject: [PATCH 44/45] Remove missed `async`. Co-authored-by: jakub-g --- packages/core/src/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 6c2ef54e2..7b78d3cb3 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -227,7 +227,7 @@ export const isXpack = (bundlerName: BundlerFullName) => export const rm = async (dir: string) => { return fsp.rm(dir, { force: true, maxRetries: 3, recursive: true }); }; -export const rmSync = async (dir: string) => { +export const rmSync = (dir: string) => { return fs.rmSync(dir, { force: true, maxRetries: 3, recursive: true }); }; From c2576cabf151a56a1738bd5e0c9330d6aa3bddaf Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 12 Mar 2025 11:49:54 +0100 Subject: [PATCH 45/45] Revert log timing --- packages/plugins/telemetry/src/common/sender.ts | 15 ++++++++++----- packages/plugins/telemetry/src/index.ts | 2 -- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/plugins/telemetry/src/common/sender.ts b/packages/plugins/telemetry/src/common/sender.ts index 8b0b58685..bef00e4c4 100644 --- a/packages/plugins/telemetry/src/common/sender.ts +++ b/packages/plugins/telemetry/src/common/sender.ts @@ -2,15 +2,16 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { doRequest } from '@dd/core/helpers'; +import { doRequest, formatDuration } from '@dd/core/helpers'; import type { Logger } from '@dd/core/types'; import type { MetricToSend } from '@dd/telemetry-plugin/types'; -export const sendMetrics = async ( +export const sendMetrics = ( metrics: Set, auth: { apiKey?: string; endPoint: string }, log: Logger, ) => { + const startSending = Date.now(); if (!auth.apiKey) { log.info(`Won't send metrics to Datadog: missing API Key.`); return; @@ -45,7 +46,11 @@ Metrics: series: MetricToSend[]; }), }), - }).catch((e) => { - log.error(`Error sending metrics ${e}`); - }); + }) + .then(() => { + log.debug(`Sent metrics in ${formatDuration(Date.now() - startSending)}.`); + }) + .catch((e) => { + log.error(`Error sending metrics ${e}`); + }); }; diff --git a/packages/plugins/telemetry/src/index.ts b/packages/plugins/telemetry/src/index.ts index e8096d7b5..7df11edde 100644 --- a/packages/plugins/telemetry/src/index.ts +++ b/packages/plugins/telemetry/src/index.ts @@ -56,7 +56,6 @@ export const getPlugins: GetPlugins = ( webpack: getWebpackPlugin(bundlerContext, context), rspack: getWebpackPlugin(bundlerContext, context), }; - // Universal plugin. const universalPlugin: PluginOptions = { name: 'datadog-universal-telemetry-plugin', @@ -86,7 +85,6 @@ export const getPlugins: GetPlugins = ( log, context.bundler.outDir, ); - outputTexts(context, log, bundlerContext.report); await sendMetrics(