From 2d1b57ba1a285bdde178712b2e04dae6bb392d5f Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 2 Aug 2024 10:05:01 +0100 Subject: [PATCH 1/4] ci: create script which generates path filters from internal dependencies map --- scripts/generate-filters.js | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 scripts/generate-filters.js diff --git a/scripts/generate-filters.js b/scripts/generate-filters.js new file mode 100644 index 0000000000..8218059b29 --- /dev/null +++ b/scripts/generate-filters.js @@ -0,0 +1,76 @@ +// @ts-check +/** + * This script generates filters for https://github.com/dorny/paths-filter + * + * It is used in CI to find all the packages which have been modified or + * have a dependency (including transitive) that has been modified. + * + * This enables running checks only for the packages affected by the changes. + */ +const fs = require("fs"); +const path = require("path"); + +function main() { + const packageIgnore = JSON.parse(process.env.PACKAGE_IGNORE || "[]"); + const commonFilters = JSON.parse(process.env.COMMON_FILTERS || "[]"); + + const pnpmLockfilePath = path.join(__dirname, "..", "pnpm-lock.json"); + if (!fs.existsSync(pnpmLockfilePath)) { + console.warn( + `${pnpmLockfilePath} doesn't exist, please run: yq -p yaml -o json pnpm-lock.yaml | tee pnpm-lock.json` + ); + process.exit(1); + } + + const pnpmLockfile = JSON.parse(fs.readFileSync(pnpmLockfilePath, "utf8")); + + // Find all direct internal dependencies for all packages + const internalDependenciesMap = {}; + for (const [package, allDependencies] of Object.entries( + pnpmLockfile.importers + )) { + const internalDependencies = Object.values(allDependencies) + .flatMap((dependencies) => Object.values(dependencies)) + .map((dependency) => dependency.version) + .filter((version) => version.startsWith("link:")) + .map((version) => version.replace("link:", "")) + .map((version) => path.join(package, version)); + internalDependenciesMap[package] = internalDependencies; + } + + // Add transitive internal dependencies + for (const dependencies of Object.values(internalDependenciesMap)) { + const dependencyQueue = [...dependencies]; + while (dependencyQueue.length !== 0) { + const dependency = dependencyQueue.pop(); + for (const transitiveDependency of internalDependenciesMap[dependency]) { + if (!dependencies.includes(transitiveDependency)) { + dependencies.push(transitiveDependency); + dependencyQueue.push(transitiveDependency); + } + } + } + } + + // Generate filters + const filters = {}; + for (const [package, dependencies] of Object.entries( + internalDependenciesMap + )) { + // Ignore packages that start with one of the prefixes in PACKAGE_IGNORE + if (packageIgnore.some((prefix) => package.startsWith(prefix))) { + continue; + } + // Calculate glob patterns for the package and its dependencies + const packageFilters = [package, ...dependencies].map((dependency) => + path.join(dependency, "**") + ); + // Set filters for the package + filters[package] = [...commonFilters, ...packageFilters]; + } + + // Pretty print the filters + console.log(JSON.stringify(filters, null, 2)); +} + +main(); From 5d6c0cddc4e16c98bd065d4ea2684203d46ce337 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 2 Aug 2024 10:05:31 +0100 Subject: [PATCH 2/4] ci: add json pnpm lockfile to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 797af0ccc7..90d8e1e986 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,6 @@ Brewfile.lock.json # Performance profiling application .clinic + +# Input to scripts/generate-filters.js; generated in CI +pnpm-lock.json From 0cc2836dc85d8bac6a79a03252a34a9c18eee8ac Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 2 Aug 2024 10:05:51 +0100 Subject: [PATCH 3/4] ci: lint and test v-next packages lazily --- .github/workflows/v-next-ci.yml | 35 ++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/v-next-ci.yml b/.github/workflows/v-next-ci.yml index 7af8775c67..79778fb069 100644 --- a/.github/workflows/v-next-ci.yml +++ b/.github/workflows/v-next-ci.yml @@ -7,6 +7,7 @@ on: branches: - "**" paths: + - ".github/workflows/v-next-ci.yml" - "v-next/**" - "config-v-next/**" workflow_dispatch: @@ -42,16 +43,42 @@ jobs: name: List packages runs-on: ubuntu-latest outputs: - packages: ${{ steps.list.outputs.packages }} + packages: ${{ steps.filter.outputs.changes }} steps: - uses: actions/checkout@v4 - - id: list + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: yq -p yaml -o json pnpm-lock.yaml | tee pnpm-lock.json + - id: generate + env: + PACKAGE_IGNORE: | + [ + ".", + "packages/", + "v-next/example-project" + ] + COMMON_FILTERS: | + [ + ".github/workflows/v-next-ci.yml", + "config-v-next/**" + ] run: | - echo "packages=$(ls -d v-next/* | xargs -n 1 basename | grep -v example-project | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + echo "filters<> $GITHUB_OUTPUT + node scripts/generate-filters.js | + yq -Po yaml 'with_entries(.key |= sub("^v-next/", ""))' | + tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - id: filter + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + with: + filters: ${{ steps.generate.outputs.filters }} lint: needs: list-packages + if: needs.list-packages.outputs.packages != '[]' + strategy: fail-fast: false matrix: @@ -83,6 +110,8 @@ jobs: ci: needs: list-packages + if: needs.list-packages.outputs.packages != '[]' + strategy: fail-fast: false matrix: From f8dc86ce9b62c2867bd98fcce120544980cf5f4c Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 2 Aug 2024 17:08:58 +0100 Subject: [PATCH 4/4] ci: trigger v-next ci on changes to the pnpm lockfile --- .github/workflows/v-next-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/v-next-ci.yml b/.github/workflows/v-next-ci.yml index f9bedbd48c..ea3f56acbd 100644 --- a/.github/workflows/v-next-ci.yml +++ b/.github/workflows/v-next-ci.yml @@ -8,11 +8,13 @@ on: - ".github/workflows/v-next-ci.yml" - "v-next/**" - "config-v-next/**" + - "pnpm-lock.yaml" pull_request: paths: - ".github/workflows/v-next-ci.yml" - "v-next/**" - "config-v-next/**" + - "pnpm-lock.yaml" workflow_dispatch: concurrency: @@ -64,7 +66,8 @@ jobs: COMMON_FILTERS: | [ ".github/workflows/v-next-ci.yml", - "config-v-next/**" + "config-v-next/**", + "pnpm-lock.yaml" ] run: | echo "filters<> $GITHUB_OUTPUT