Release/publish process improvements: version-bump safeguard, CodeQL release detection, and a published-version inventory

.github/scripts/check-codeql-release.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -euo pipefail
+
+# Checks whether a newer CodeQL CLI has been released upstream than the one
+# pinned in .codeqlversion, and if so, opens a tracking issue with the manual
+# update checklist from CONTRIBUTING.md ("Keeping CodeQL versions current").
+#
+# Nothing else in the repo watches for new upstream CodeQL CLI releases today
+# - this script/workflow only detects and files the issue. Actually bumping
+# .codeqlversion, running `codeql pack upgrade`, fixing any resulting
+# compile/test breakage, and bumping affected pack versions all remain
+# manual steps for a maintainer/contributor to pick up.
+
+CURRENT_VERSION=$(cat .codeqlversion)
+LATEST_TAG=$(gh api repos/github/codeql-cli-binaries/releases/latest --jq '.tag_name')
+LATEST_VERSION=${LATEST_TAG#v}
+
+echo "[+] Currently pinned CodeQL CLI version: $CURRENT_VERSION"
+echo "[+] Latest CodeQL CLI release upstream:  $LATEST_VERSION"
+
+if [[ "$LATEST_VERSION" == "$CURRENT_VERSION" ]]; then
+    echo "[+] Already up to date, nothing to do."
+    exit 0
+fi
+
+echo "[+] New CodeQL CLI release detected: $CURRENT_VERSION -> $LATEST_VERSION"
+
+TITLE="CodeQL CLI v$LATEST_VERSION is available (currently pinned: v$CURRENT_VERSION)"
+
+# Idempotency: don't open a duplicate issue if one is already open for this version.
+EXISTING=$(gh issue list --state open --search "\"$LATEST_VERSION\" in:title" --json number --jq 'length')
+if [[ "$EXISTING" -gt 0 ]]; then
+    echo "[+] An open issue already mentions v$LATEST_VERSION in its title, skipping."
+    exit 0
+fi
+
+# Create the tracking label if it doesn't already exist (never fails the run).
+if ! gh label list --search "codeql-cli-update" --json name --jq '.[].name' | grep -qx "codeql-cli-update"; then
+    gh label create "codeql-cli-update" --color "0E8A16" \
+        --description "Tracks updating the pinned upstream CodeQL CLI/library version" || true
+fi
+
+BINARY_URL="https://github.com/github/codeql-cli-binaries/releases/tag/$LATEST_TAG"
+BUNDLE_URL="https://github.com/github/codeql-action/releases/tag/codeql-bundle-v$LATEST_VERSION"
+REPO="${GITHUB_REPOSITORY:-GitHubSecurityLab/CodeQL-Community-Packs}"
+REPO_BLOB_URL="https://github.com/$REPO/blob/main"
+
+BODY=$(cat <<EOF
+A new CodeQL CLI release is available upstream: **v$LATEST_VERSION** ([binary]($BINARY_URL), [bundle]($BUNDLE_URL)).
+
+This repo is currently pinned to **v$CURRENT_VERSION** in [\`.codeqlversion\`]($REPO_BLOB_URL/.codeqlversion).
+
+Updating is a manual process today (see [CONTRIBUTING.md § Keeping CodeQL versions current]($REPO_BLOB_URL/CONTRIBUTING.md#keeping-codeql-versions-current)):
+
+- [ ] Update \`.codeqlversion\` to \`$LATEST_VERSION\`.
+- [ ] Run \`codeql pack upgrade <dir>\` for each pack directory to refresh its \`codeql-pack.lock.yml\`.
+- [ ] Fix any compilation/test errors caused by upstream API changes (usually the hardest part - see [#124](https://github.com/GitHubSecurityLab/CodeQL-Community-Packs/pull/124) for an example of what this can involve).
+- [ ] Bump the \`version:\` field of every pack that changed, so \`publish.yml\` actually publishes the update once merged.
+- [ ] Update the "Supported CodeQL versions" table in CONTRIBUTING.md.
+
+This issue was opened automatically by [\`detect-codeql-release.yml\`]($REPO_BLOB_URL/.github/workflows/detect-codeql-release.yml). If you're not picking this up right away, feel free to leave it open as a tracker - a duplicate won't be filed while this stays open.
+EOF
+)
+
+echo "[+] Opening tracking issue"
+gh issue create --title "$TITLE" --body "$BODY" --label "codeql-cli-update"

.github/scripts/generate-version-inventory.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+# Generates a Markdown table of every pack's published (GHCR) version next
+# to its local qlpack.yml version, so there's a single place to answer
+# "what's live?" without paging through the GHCR Packages UI or grepping
+# every qlpack.yml by hand (see CONTRIBUTING.md § How packages get
+# published).
+#
+# Usage: generate-version-inventory.sh > PACKAGE_VERSIONS.md
+
+LANGUAGES=(cpp csharp go java javascript python ruby)
+
+# subdir -> (GHCR package suffix, human label)
+declare -A PACKAGE_SUFFIX=(
+    [src]="queries"
+    [lib]="libs"
+    [ext]="extensions"
+    [ext-library-sources]="library-sources"
+)
+declare -A SUBDIR_LABEL=(
+    [src]="Queries (src)"
+    [lib]="Library (lib)"
+    [ext]="Extensions (ext)"
+    [ext-library-sources]="Library sources (ext-library-sources)"
+)
+
+echo "<!-- Generated by .github/scripts/generate-version-inventory.sh - do not edit by hand. -->"
+echo "# Published package versions"
+echo
+echo "This table is regenerated on a schedule by [\`publish-version-inventory.yml\`](./.github/workflows/publish-version-inventory.yml)."
+echo "\"Published\" is what's currently live on [GHCR](https://github.com/orgs/GitHubSecurityLab/packages?repo_name=CodeQL-Community-Packs)."
+echo "\"On main\" is the version declared in that pack's \`qlpack.yml\` on the default branch - if it differs from"
+echo "\"Published\", that version will be published the next time \`main\` is pushed to (see [publish.yml](./.github/workflows/publish.yml))."
+echo
+echo "| Language | Pack | Published | On main | Package |"
+echo "| --- | --- | --- | --- | --- |"
+
+for language in "${LANGUAGES[@]}"; do
+    for subdir in src lib ext ext-library-sources; do
+        qlpack_path="$language/$subdir/qlpack.yml"
+        if [[ ! -f "$qlpack_path" ]]; then
+            continue
+        fi
+
+        package="codeql-$language-${PACKAGE_SUFFIX[$subdir]}"
+        current_version=$(grep '^version:' "$qlpack_path" | awk '{print $2}' | tr -d '"'\''')
+        published_version=$(gh api "/orgs/githubsecuritylab/packages/container/$package/versions" --jq '.[0].metadata.container.tags[0]' 2>/dev/null || echo "_none_")
+
+        echo "| $language | ${SUBDIR_LABEL[$subdir]} | \`$published_version\` | \`$current_version\` | [\`$package\`](https://github.com/GitHubSecurityLab/CodeQL-Community-Packs/pkgs/container/$package) |"
+    done
+done

.github/scripts/pr-version-bump-check.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -euo pipefail
+
+# Warns (non-blocking) when a PR changes files inside a pack directory but
+# does not bump that pack's own `version:` in qlpack.yml.
+#
+# publish.yml only republishes a pack when its qlpack.yml `version:` differs
+# from what is currently on GHCR (see pr-suites-packs.sh for the companion
+# check that fires when a bump *was* made). If content changes merge without
+# a version bump, the change silently never ships - this script surfaces
+# that gap as an advisory PR comment so it isn't forgotten.
+#
+# Usage: pr-version-bump-check.sh <pr_number> <language>
+
+PR_NUMBER=${1}
+LANGUAGE=${2}
+
+mapfile -t CHANGED_FILES < <(gh pr view "$PR_NUMBER" --json files --jq '.files.[].path')
+
+# subdir -> GHCR package name suffix (matches publish.yml)
+declare -A PACKAGE_SUFFIX=(
+    [src]="queries"
+    [lib]="libs"
+    [ext]="extensions"
+    [ext-library-sources]="library-sources"
+)
+
+# publish.yml's `extensions`/`library_sources_extensions` jobs only matrix over
+# csharp/java today (see #144 for the proposal to extend this to python/go).
+# go/ext and python/ext directories exist but aren't wired into publish.yml,
+# so bumping their version wouldn't actually make them publish - don't check
+# those pack types for other languages, to avoid telling contributors a
+# version bump will ship something that publish.yml doesn't yet publish.
+SUBDIRS=(src lib)
+if [[ "$LANGUAGE" == "csharp" || "$LANGUAGE" == "java" ]]; then
+    SUBDIRS+=(ext ext-library-sources)
+fi
+
+for subdir in "${SUBDIRS[@]}"; do
+    qlpack_path="$LANGUAGE/$subdir/qlpack.yml"
+    if [[ ! -f "$qlpack_path" ]]; then
+        # Not every language has an ext / ext-library-sources pack.
+        continue
+    fi
+
+    changed=false
+    for file in "${CHANGED_FILES[@]}"; do
+        if [[ "$file" == "$LANGUAGE/$subdir/"* ]]; then
+            changed=true
+            break
+        fi
+    done
+
+    if [[ "$changed" != true ]]; then
+        continue
+    fi
+
+    package="codeql-$LANGUAGE-${PACKAGE_SUFFIX[$subdir]}"
+    echo "[+] Files changed under $LANGUAGE/$subdir - checking whether $package's version was bumped"
+
+    PUBLISHED_VERSION=$(gh api "/orgs/githubsecuritylab/packages/container/$package/versions" --jq '.[0].metadata.container.tags[0]' 2>/dev/null || echo "unknown")
+    CURRENT_VERSION=$(grep '^version:' "$qlpack_path" | awk '{print $2}' | tr -d '"'\''')
+
+    if [[ "$PUBLISHED_VERSION" == "unknown" ]]; then
+        echo "[!] Could not resolve published version for $package - skipping (package may not exist yet)"
+        continue
+    fi
+
+    if [[ "$PUBLISHED_VERSION" == "$CURRENT_VERSION" ]]; then
+        comment="Files changed in \`$LANGUAGE/$subdir\` but \`$qlpack_path\` version is still \`$CURRENT_VERSION\`, matching what is already published for \`$package\`. This change will NOT be published when this PR merges. If it should ship, bump \`version:\` in \`$qlpack_path\` (in this PR or a fast-follow)."
+        if [[ ! $(gh pr view "$PR_NUMBER" --json comments --jq '.comments.[].body' | grep "$comment") ]]; then
+            echo "[+] Commenting on PR: version bump appears to be missing for $package"
+            gh pr comment "$PR_NUMBER" \
+                --body "$comment"
+        fi
+    else
+        echo "[+] $package version already bumped ($PUBLISHED_VERSION -> $CURRENT_VERSION), nothing to flag"
+    fi
+done
+
+echo "[+] Complete"

.github/workflows/ci.yml

@@ -251,3 +251,40 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
           ./.github/scripts/pr-configs.sh "${{ github.event.number }}"
+
+  version-bump-check:
+    name: Warn if pack version bump is missing
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write # required by pr-version-bump-check.sh (gh pr comment)
+      issues: write # required by pr-version-bump-check.sh (gh pr comment)
+      packages: read # required by pr-version-bump-check.sh (gh api packages)
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
+        id: changes
+        with:
+          filters: |
+            src:
+              - '${{ matrix.language }}/**'
+
+      # Advisory only (never fails the build): publish.yml only republishes a
+      # pack when its qlpack.yml `version:` differs from what's on GHCR, so a
+      # content change that merges without a version bump silently never
+      # ships. This flags that gap on the PR instead of failing it, since the
+      # repo's actual practice sometimes intentionally splits a content
+      # change and its version bump across sequential PRs.
+      - name: "Check for forgotten version bump"
+        if: steps.changes.outputs.src == 'true'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          ./.github/scripts/pr-version-bump-check.sh "${{ github.event.number }}" "${{ matrix.language }}"

.github/workflows/detect-codeql-release.yml

@@ -0,0 +1,22 @@
+name: Detect new CodeQL CLI release
+
+on:
+  schedule:
+    - cron: '0 13 * * 1' # Every Monday at 13:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Check for a new CodeQL CLI release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          ./.github/scripts/check-codeql-release.sh

.github/workflows/publish-version-inventory.yml

@@ -0,0 +1,57 @@
+name: Update published package version inventory
+
+on:
+  schedule:
+    - cron: '0 6 * * 1' # Every Monday at 06:00 UTC (offset from the other scheduled jobs)
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+  packages: read
+
+jobs:
+  update-inventory:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Generate PACKAGE_VERSIONS.md
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          ./.github/scripts/generate-version-inventory.sh > PACKAGE_VERSIONS.md
+
+      - name: Check for changes
+        id: diff
+        run: |
+          if [[ -z "$(git status --porcelain -- PACKAGE_VERSIONS.md)" ]]; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Reuses one stable branch/PR (force-pushed each run) instead of opening
+      # a new PR every time the inventory drifts, since this file changes
+      # often (any time any pack is published) and is fully machine-generated.
+      - name: Open or update the inventory PR
+        if: steps.diff.outputs.changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          BRANCH: chore/package-inventory-update
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          git checkout -b "$BRANCH"
+          git add PACKAGE_VERSIONS.md
+          git commit -m "chore: update published package version inventory"
+          git push --force origin "HEAD:$BRANCH"
+
+          if [[ "$(gh pr list --head "$BRANCH" --state open --json number --jq 'length')" == "0" ]]; then
+            gh pr create --base main --head "$BRANCH" \
+              --title "chore: update published package version inventory" \
+              --body "Automated update of \`PACKAGE_VERSIONS.md\` (see [\`publish-version-inventory.yml\`](./.github/workflows/publish-version-inventory.yml))."
+          else
+            echo "[+] Existing PR for $BRANCH updated via force-push."
+          fi