[CI] Add GH workflow to run dev build using submodules at head (#5561)

Author: chalin

.github/workflows/build-dev.yml

@@ -0,0 +1,88 @@
+name: Dev build
+
+on:
+  workflow_dispatch:
+    inputs:
+      submodule_path_regex:
+        description:
+          Regex of submodule paths to updated to HEAD before building.
+        default: content-modules
+        type: string
+
+jobs:
+  build-and-test:
+    name: BUILD and CHECK LINKS
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create NPM cache-hash input file
+        run: |
+          mkdir -p tmp
+          jq '{devDependencies, dependencies, engines, gitHubActionCacheKey}' package.json > tmp/package-ci.json
+
+      - name: Create and use reduced-dependencies package.json
+        run: |
+          jq '.devDependencies |= with_entries(
+                  select(.key | test("prefix|hugo|css"))
+                )
+                | del(.dependencies, .optionalDependencies)' \
+            package.json > tmp/package-min.json
+          cp tmp/package-min.json package.json
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+          cache-dependency-path: tmp/package-ci.json
+
+      - run: npm install --omit=optional
+
+      - name: Fetch all submodules at HEAD
+        run: npm run update:submodule
+
+      - name:
+          Revert to pinned versions of submodules except for
+          submodule_path_regex
+        run: npm run _sync -- ${{ inputs.submodule_path_regex }}
+
+      - run: npm run log:test-and-fix
+        continue-on-error: true
+        env:
+          GET: no
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-log-etc
+          path: |
+            tmp/build-log.txt
+            tmp/package*.json
+            static/refcache.json
+
+  check-refcache:
+    name: REFCACHE updates?
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with: { name: build-log-etc }
+      - name: Fail when refcache contains entries with HTTP status 4XX
+        run: |
+          if grep -B 1 -e '"StatusCode": 4' static/refcache.json; then
+            echo "Run 'npx gulp prune' to remove 4xx entries from the refcache"
+            exit 1
+          fi
+      - name: Does the refcache need updating?
+        run: npm run diff:fail
+
+  check-build-log-for-issues:
+    name: WARNINGS in build log?
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with: { name: build-log-etc }
+      - run: cat tmp/build-log.txt
+      - run: scripts/check-build-log.sh

package.json

@@ -1,6 +1,7 @@
 {
   "Notes": [
-    "The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case."
+    "The 'all' runs _all_ named scripts in sequence, even if one fails; and exits with failure in that case.",
+    "check:links actually has the side-effect of 'fixing' the refcache; this is why test-and-fix uses _fix:most"
   ],
   "scripts": {
     "__check:links": "make --keep-going check-links",
@@ -20,6 +21,7 @@
     "_diff:fail": "git diff --name-only --exit-code || exit 1",
     "_fail": "exit 1",
     "_filename-error": "echo 'ERROR: the following files violate naming conventions; fix using: `npm run fix:filenames`'; echo; npm run -s _ls-bad-filenames; exit 1",
+    "_fix:most": "npm run all -- $(npm -s run _list:fix:* | grep -Ev 'refcache|submodule')",
     "_get:no": "echo SKIPPING get operation",
     "_get:submodule:non-lang": "npm run _get:submodule -- content-modules/opentelemetry-specification themes/docsy",
     "_get:submodule": "set -x && git submodule update --init ${DEPTH:- --depth 999}",
@@ -30,13 +32,13 @@
     "_list:dict": "grep '^\\s*\"@cspell' package.json | awk -F: '{print $1}'  | tr -d '\"'",
     "_list:fix:*": "npm run --loglevel=warn | grep -Ee '^\\s*fix:[^:]+$' | grep -v 'fix:all'",
     "_ls-bad-filenames": "find assets content static -name '*_*' ! -name '[_.]*'",
+    "_pin": "scripts/pin-submodules.pl",
     "_prebuild": "npm run seq -- get:submodule cp:spec",
     "_prepare:docsy": "cd themes/docsy && npm install",
     "_prettier:any": "npx prettier --ignore-path ''",
     "_rename-to-kebab-case": "find assets content static -name '*_*' ! -name '[_.]*' -exec sh -c 'mv \"$1\" \"${1//_/-}\"' _ {} \\;",
     "_serve:hugo": "hugo server --buildDrafts --minify",
     "_serve:netlify": "netlify dev -c \"npm run _serve:hugo -- --renderToMemory\"",
-    "_sync": "scripts/sync-submodules.pl",
     "all": "bash -c 'x=0; for c in \"$@\"; do npm run $c || x=$((x+1)); done; ((!x)) || (echo \"ERROR: some scripts failed!\" && exit 1)' -",
     "build:preview": "set -x && npm run _build -- --minify",
     "build:production": "npm run _hugo -- --minify",
@@ -59,7 +61,7 @@
     "cp:spec": "scripts/content-modules/cp-pages.sh",
     "diff:check": "npm run _diff:check || (echo; echo 'WARNING: the files above have not been committed'; echo)",
     "diff:fail": "npm run _diff:check || (echo; echo 'ERROR: the files above have changed. Locally rerun `npm run test-and-fix` and commit changes'; echo; exit 1)",
-    "fix:all": "npm run seq -- $(npm -s run _list:fix:*)",
+    "fix:all": "npm run all -- $(npm -s run _list:fix:*)",
     "fix:dict": "find content/en layouts -name \"*.md\" -print0 | xargs -0 scripts/normalize-cspell-front-matter.pl",
     "fix:filenames": "npm run _rename-to-kebab-case",
     "fix:format": "npm run format",
@@ -69,12 +71,13 @@
     "fix:i18n": "npm run fix:i18n:new",
     "fix:markdown": "npm run check:markdown -- --fix",
     "fix:refcache": "npm run check:links",
-    "fix:submodules": "npm run _sync",
+    "fix:submodules": "npm run update:submodule && npm run _pin",
     "fix:text": "npm run check:text -- --fix",
     "fix": "npm run fix:all",
     "format": "npm run _check:format -- --write && npm run _check:format:ja+zh -- --write",
     "get:submodule": "npm run _get:${GET:-submodule} --",
     "log:check:links": "npm run check:links | tee tmp/build-log.txt",
+    "log:test-and-fix": "npm run test-and-fix | tee tmp/build-log.txt",
     "make:public": "make public ls-public",
     "netlify-build:preview": "npm run seq -- build:preview diff:check",
     "netlify-build:production": "npm run seq -- build:production diff:check",
@@ -93,17 +96,11 @@
     "serve:hugo": "npm run _serve:hugo -- --renderToMemory",
     "serve:netlify": "npm run _serve:netlify",
     "serve": "npm run serve:hugo --",
-    "sync": "scripts/sync-submodules.pl",
-    "test-and-fix": "npm run seq -- check fix:dict fix:filenames",
+    "test-and-fix": "npm run seq -- fix:submodules _fix:most test",
     "test": "npm run check",
-    "update:docsy-dep": "npm install --save-dev autoprefixer@latest postcss-cli@latest",
     "update:htmltest-config": "scripts/htmltest-config.sh",
-    "update:hugo": "npm install --save-dev --save-exact hugo-extended@latest",
-    "update:hugo+": "npm run update:hugo && npm run update:docsy-dep",
-    "update:netlify": "npm install --save-optional netlify-cli@latest",
-    "update:other-pkg": "npm install --save-dev gulp@latest",
+    "update:pkg:hugo": "npm install --save-dev --save-exact hugo-extended@latest",
     "update:pkgs": "npx npm-check-updates -u",
-    "update:submodule:lang": "npm run seq -- update:submodule _get:submodule:non-lang",
     "update:submodule": "set -x && git submodule update --remote ${DEPTH:- --depth 999}"
   },
   "devDependencies": {
@@ -157,5 +154,5 @@
     "proseWrap": "always",
     "singleQuote": true
   },
-  "spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap postnetlify prebuild precheck preinstall postbuild postget refcache textlint -"
+  "spelling": "cSpell:ignore docsy elemetry htmltest hugo loglevel netlify nowrap pkgs postnetlify prebuild precheck preinstall postbuild postget refcache textlint -"
 }

scripts/sync-submodules.pl scripts/pin-submodules.pl

@@ -1,9 +1,17 @@
 #!/usr/bin/perl -w
+#
+# Switch submodules to their pinned versions, all submodules by default. Pass
+# submodule-path regex to specify submodules to skip.
 
 use strict;
 use warnings;
 use FileHandle;
 
+my $submodule_skip_regex = shift @ARGV;
+
+print "Using submodule-path skip regex: $submodule_skip_regex\n\n"
+  if $submodule_skip_regex;
+
 my $file = '.gitmodules';
 my $fh = FileHandle->new($file, 'r') or die "Error opening $file: $!";
 my $content = do { local $/; <$fh> };
@@ -17,6 +25,11 @@
     my $submodule_path = $submodules[$i];
     my $commit = $submodules[$i + 1];
 
+    if ($submodule_skip_regex && $submodule_path =~ /$submodule_skip_regex/) {
+      printf "Skipping $submodule_path\n";
+      next;
+    }
+
     my $command = "cd $submodule_path && git switch --detach $commit";
     print "> $command\n";
     system($command);