Skip to content

Commit c2e89f8

Browse files
authored
feat(core): add multi hash fn (#29935)
Adds function to compute multiple glob hashes in native code at the same time, greatly speeding up certain plugin performance.
1 parent 2ebdd2e commit c2e89f8

File tree

23 files changed

+547
-240
lines changed

23 files changed

+547
-240
lines changed

docs/generated/devkit/createNodesFromFiles.md

+6-6
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@
1010

1111
#### Parameters
1212

13-
| Name | Type |
14-
| :------------ | :------------------------------------------------------------------------- |
15-
| `createNodes` | [`CreateNodesFunction`](../../devkit/documents/CreateNodesFunction)\<`T`\> |
16-
| `configFiles` | readonly `string`[] |
17-
| `options` | `T` |
18-
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
13+
| Name | Type |
14+
| :------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
15+
| `createNodes` | (`projectConfigurationFile`: `string`, `options`: `T`, `context`: [`CreateNodesContext`](../../devkit/documents/CreateNodesContext), `idx`: `number`) => [`CreateNodesResult`](../../devkit/documents/CreateNodesResult) \| `Promise`\<[`CreateNodesResult`](../../devkit/documents/CreateNodesResult)\> |
16+
| `configFiles` | readonly `string`[] |
17+
| `options` | `T` |
18+
| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) |
1919

2020
#### Returns
2121

docs/generated/devkit/isDaemonEnabled.md

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
# Function: isDaemonEnabled
22

3-
**isDaemonEnabled**(): `boolean`
3+
**isDaemonEnabled**(`nxJson?`): `boolean`
4+
5+
#### Parameters
6+
7+
| Name | Type |
8+
| :------- | :----------------------------------------------------------------------------------------- |
9+
| `nxJson` | [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\> |
410

511
#### Returns
612

packages/devkit/src/utils/calculate-hash-for-create-nodes.ts

+30-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ import {
55
hashArray,
66
} from 'nx/src/devkit-exports';
77

8-
import { hashObject, hashWithWorkspaceContext } from 'nx/src/devkit-internals';
8+
import {
9+
hashMultiGlobWithWorkspaceContext,
10+
hashObject,
11+
hashWithWorkspaceContext,
12+
} from 'nx/src/devkit-internals';
913

1014
export async function calculateHashForCreateNodes(
1115
projectRoot: string,
@@ -21,3 +25,28 @@ export async function calculateHashForCreateNodes(
2125
hashObject(options),
2226
]);
2327
}
28+
29+
export async function calculateHashesForCreateNodes(
30+
projectRoots: string[],
31+
options: object,
32+
context: CreateNodesContext | CreateNodesContextV2,
33+
additionalGlobs: string[][] = []
34+
): Promise<string[]> {
35+
if (
36+
additionalGlobs.length &&
37+
additionalGlobs.length !== projectRoots.length
38+
) {
39+
throw new Error(
40+
'If additionalGlobs is provided, it must be the same length as projectRoots'
41+
);
42+
}
43+
return hashMultiGlobWithWorkspaceContext(
44+
context.workspaceRoot,
45+
projectRoots.map((projectRoot, idx) => [
46+
join(projectRoot, '**/*'),
47+
...(additionalGlobs.length ? additionalGlobs[idx] : []),
48+
])
49+
).then((hashes) => {
50+
return hashes.map((hash) => hashArray([hash, hashObject(options)]));
51+
});
52+
}

packages/eslint/src/plugins/plugin.ts

+23-18
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@ import {
1111
TargetConfiguration,
1212
writeJsonFile,
1313
} from '@nx/devkit';
14-
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
14+
import {
15+
calculateHashesForCreateNodes,
16+
calculateHashForCreateNodes,
17+
} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
1518
import { existsSync } from 'node:fs';
1619
import { basename, dirname, join, normalize, sep } from 'node:path/posix';
1720
import { hashObject } from 'nx/src/hasher/file-hasher';
@@ -186,10 +189,10 @@ const internalCreateNodesV2 = async (
186189
configFilePath: string,
187190
options: EslintPluginOptions,
188191
context: CreateNodesContextV2,
189-
eslintConfigFiles: string[],
190192
projectRootsByEslintRoots: Map<string, string[]>,
191193
lintableFilesPerProjectRoot: Map<string, string[]>,
192-
projectsCache: Record<string, CreateNodesResult['projects']>
194+
projectsCache: Record<string, CreateNodesResult['projects']>,
195+
hashByRoot: Map<string, string>
193196
): Promise<CreateNodesResult> => {
194197
const configDir = dirname(configFilePath);
195198

@@ -201,19 +204,7 @@ const internalCreateNodesV2 = async (
201204
const projects: CreateNodesResult['projects'] = {};
202205
await Promise.all(
203206
projectRootsByEslintRoots.get(configDir).map(async (projectRoot) => {
204-
const parentConfigs = eslintConfigFiles.filter((eslintConfig) =>
205-
isSubDir(projectRoot, dirname(eslintConfig))
206-
);
207-
const hash = await calculateHashForCreateNodes(
208-
projectRoot,
209-
options,
210-
{
211-
configFiles: eslintConfigFiles,
212-
nxJsonConfiguration: context.nxJsonConfiguration,
213-
workspaceRoot: context.workspaceRoot,
214-
},
215-
[...parentConfigs, join(projectRoot, '.eslintignore')]
216-
);
207+
const hash = hashByRoot.get(projectRoot);
217208

218209
if (projectsCache[hash]) {
219210
// We can reuse the projects in the cache.
@@ -280,17 +271,31 @@ export const createNodesV2: CreateNodesV2<EslintPluginOptions> = [
280271
options,
281272
context
282273
);
274+
const hashes = await calculateHashesForCreateNodes(
275+
projectRoots,
276+
options,
277+
context,
278+
projectRoots.map((root) => {
279+
const parentConfigs = eslintConfigFiles.filter((eslintConfig) =>
280+
isSubDir(root, dirname(eslintConfig))
281+
);
282+
return [...parentConfigs, join(root, '.eslintignore')];
283+
})
284+
);
285+
const hashByRoot = new Map<string, string>(
286+
projectRoots.map((r, i) => [r, hashes[i]])
287+
);
283288
try {
284289
return await createNodesFromFiles(
285290
(configFile, options, context) =>
286291
internalCreateNodesV2(
287292
configFile,
288293
options,
289294
context,
290-
eslintConfigFiles,
291295
projectRootsByEslintRoots,
292296
lintableFilesPerProjectRoot,
293-
targetsCache
297+
targetsCache,
298+
hashByRoot
294299
),
295300
eslintConfigFiles,
296301
options,

packages/jest/src/plugins/plugin.ts

+110-47
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {
22
CreateNodes,
33
CreateNodesContext,
4+
CreateNodesContextV2,
45
createNodesFromFiles,
56
CreateNodesV2,
67
getPackageManagerCommand,
@@ -13,7 +14,10 @@ import {
1314
TargetConfiguration,
1415
writeJsonFile,
1516
} from '@nx/devkit';
16-
import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
17+
import {
18+
calculateHashesForCreateNodes,
19+
calculateHashForCreateNodes,
20+
} from '@nx/devkit/src/utils/calculate-hash-for-create-nodes';
1721
import {
1822
clearRequireCache,
1923
loadConfigFile,
@@ -72,17 +76,70 @@ export const createNodesV2: CreateNodesV2<JestPluginOptions> = [
7276
// Cache jest preset(s) to avoid penalties of module load times. Most of jest configs will use the same preset.
7377
const presetCache: Record<string, unknown> = {};
7478

79+
const packageManagerWorkspacesGlob = combineGlobPatterns(
80+
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
81+
);
82+
options = normalizeOptions(options);
83+
84+
const { roots: projectRoots, configFiles: validConfigFiles } =
85+
configFiles.reduce(
86+
(acc, configFile) => {
87+
const potentialRoot = dirname(configFile);
88+
if (
89+
checkIfConfigFileShouldBeProject(
90+
configFile,
91+
potentialRoot,
92+
packageManagerWorkspacesGlob,
93+
context
94+
)
95+
) {
96+
acc.roots.push(potentialRoot);
97+
acc.configFiles.push(configFile);
98+
}
99+
return acc;
100+
},
101+
{
102+
roots: [],
103+
configFiles: [],
104+
} as {
105+
roots: string[];
106+
configFiles: string[];
107+
}
108+
);
109+
110+
const hashes = await calculateHashesForCreateNodes(
111+
projectRoots,
112+
options,
113+
context
114+
);
115+
75116
try {
76117
return await createNodesFromFiles(
77-
(configFile, options, context) =>
78-
createNodesInternal(
79-
configFile,
118+
async (configFilePath, options, context, idx) => {
119+
const projectRoot = projectRoots[idx];
120+
const hash = hashes[idx];
121+
122+
targetsCache[hash] ??= await buildJestTargets(
123+
configFilePath,
124+
projectRoot,
80125
options,
81126
context,
82-
targetsCache,
83127
presetCache
84-
),
85-
configFiles,
128+
);
129+
130+
const { targets, metadata } = targetsCache[hash];
131+
132+
return {
133+
projects: {
134+
[projectRoot]: {
135+
root: projectRoot,
136+
targets,
137+
metadata,
138+
},
139+
},
140+
};
141+
},
142+
validConfigFiles,
86143
options,
87144
context
88145
);
@@ -98,35 +155,63 @@ export const createNodesV2: CreateNodesV2<JestPluginOptions> = [
98155
*/
99156
export const createNodes: CreateNodes<JestPluginOptions> = [
100157
jestConfigGlob,
101-
(...args) => {
158+
async (configFilePath, options, context) => {
102159
logger.warn(
103160
'`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will change to the createNodesV2 API.'
104161
);
105162

106-
return createNodesInternal(...args, {}, {});
163+
const projectRoot = dirname(configFilePath);
164+
165+
const packageManagerWorkspacesGlob = combineGlobPatterns(
166+
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
167+
);
168+
169+
if (
170+
!checkIfConfigFileShouldBeProject(
171+
configFilePath,
172+
projectRoot,
173+
packageManagerWorkspacesGlob,
174+
context
175+
)
176+
) {
177+
return {};
178+
}
179+
180+
options = normalizeOptions(options);
181+
182+
const { targets, metadata } = await buildJestTargets(
183+
configFilePath,
184+
projectRoot,
185+
options,
186+
context,
187+
{}
188+
);
189+
190+
return {
191+
projects: {
192+
[projectRoot]: {
193+
root: projectRoot,
194+
targets,
195+
metadata,
196+
},
197+
},
198+
};
107199
},
108200
];
109201

110-
async function createNodesInternal(
202+
function checkIfConfigFileShouldBeProject(
111203
configFilePath: string,
112-
options: JestPluginOptions,
113-
context: CreateNodesContext,
114-
targetsCache: Record<string, JestTargets>,
115-
presetCache: Record<string, unknown>
116-
) {
117-
const projectRoot = dirname(configFilePath);
118-
119-
const packageManagerWorkspacesGlob = combineGlobPatterns(
120-
getGlobPatternsFromPackageManagerWorkspaces(context.workspaceRoot)
121-
);
122-
204+
projectRoot: string,
205+
packageManagerWorkspacesGlob: string,
206+
context: CreateNodesContext | CreateNodesContextV2
207+
): boolean {
123208
// Do not create a project if package.json and project.json isn't there.
124209
const siblingFiles = readdirSync(join(context.workspaceRoot, projectRoot));
125210
if (
126211
!siblingFiles.includes('package.json') &&
127212
!siblingFiles.includes('project.json')
128213
) {
129-
return {};
214+
return false;
130215
} else if (
131216
!siblingFiles.includes('project.json') &&
132217
siblingFiles.includes('package.json')
@@ -136,7 +221,7 @@ async function createNodesInternal(
136221
const isPackageJsonProject = minimatch(path, packageManagerWorkspacesGlob);
137222

138223
if (!isPackageJsonProject) {
139-
return {};
224+
return false;
140225
}
141226
}
142227

@@ -148,31 +233,9 @@ async function createNodesInternal(
148233
// The `getJestProjectsAsync` function uses the project graph, which leads to a
149234
// circular dependency. We can skip this since it's no intended to be used for
150235
// an Nx project.
151-
return {};
236+
return false;
152237
}
153-
154-
options = normalizeOptions(options);
155-
156-
const hash = await calculateHashForCreateNodes(projectRoot, options, context);
157-
targetsCache[hash] ??= await buildJestTargets(
158-
configFilePath,
159-
projectRoot,
160-
options,
161-
context,
162-
presetCache
163-
);
164-
165-
const { targets, metadata } = targetsCache[hash];
166-
167-
return {
168-
projects: {
169-
[projectRoot]: {
170-
root: projectRoot,
171-
targets,
172-
metadata,
173-
},
174-
},
175-
};
238+
return true;
176239
}
177240

178241
async function buildJestTargets(

packages/nx/src/config/nx-json.ts

+5-4
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,16 @@ import { existsSync } from 'fs';
22
import { dirname, join } from 'path';
33

44
import type { ChangelogRenderOptions } from '../../release/changelog-renderer';
5-
import { readJsonFile } from '../utils/fileutils';
6-
import { PackageManager } from '../utils/package-manager';
7-
import { workspaceRoot } from '../utils/workspace-root';
8-
import {
5+
import type { PackageManager } from '../utils/package-manager';
6+
import type {
97
InputDefinition,
108
TargetConfiguration,
119
TargetDependencyConfig,
1210
} from './workspace-json-project-json';
1311

12+
import { readJsonFile } from '../utils/fileutils';
13+
import { workspaceRoot } from '../utils/workspace-root';
14+
1415
export type ImplicitDependencyEntry<T = '*' | string[]> = {
1516
[key: string]: T | ImplicitJsonSubsetDependency<T>;
1617
};

0 commit comments

Comments
 (0)