diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 00000000..ebd21ffb --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,110 @@ +name: build-images +run-name: build-images ${{ github.event.release.tag_name }} +on: + release: + types: [published] + +permissions: + contents: read + packages: write + +jobs: + prepare: + if: ${{ github.event_name == 'release' }} + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.release_tag.outputs.tag }} + version: ${{ steps.release_tag.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Resolve release tag & version + id: release_tag + run: | + git fetch --tags --force + TAG="${{ github.event.release.tag_name }}" + if [ -z "$TAG" ]; then + echo "No Git tag found to check out" >&2 + exit 1 + fi + VER_NO_V="${TAG#v}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VER_NO_V" >> $GITHUB_OUTPUT + + build-image: + needs: prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: rag-backend + dockerfile: services/rag-backend/Dockerfile + - name: admin-backend + dockerfile: services/admin-backend/Dockerfile + - name: document-extractor + dockerfile: services/document-extractor/Dockerfile + - name: mcp-server + dockerfile: services/mcp-server/Dockerfile + - name: frontend + dockerfile: services/frontend/apps/chat-app/Dockerfile + - name: admin-frontend + dockerfile: services/frontend/apps/admin-app/Dockerfile + env: + REGISTRY: ghcr.io + IMAGE_NS: ${{ github.repository }} + VERSION: ${{ needs.prepare.outputs.version }} + TAG: ${{ needs.prepare.outputs.tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Checkout release tag + run: git checkout "$TAG" + - name: Normalize IMAGE_NS to lowercase + run: echo "IMAGE_NS=$(echo '${{ env.IMAGE_NS }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_PAT }} + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + - name: Build & push ${{ matrix.name }} + run: | + docker buildx build --push \ + -t "$REGISTRY/$IMAGE_NS/${{ matrix.name }}:${VERSION}" \ + -t "$REGISTRY/$IMAGE_NS/${{ matrix.name }}:latest" \ + -f "${{ matrix.dockerfile }}" . + - name: Capture digest + run: | + sudo apt-get update && sudo apt-get install -y jq + ref="$REGISTRY/$IMAGE_NS/${{ matrix.name }}:${VERSION}" + digest=$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest.Digest}}' | jq -r . || true) + jq -n --arg name "${{ matrix.name }}" --arg tag "$VERSION" --arg digest "$digest" '{($name): {tag: $tag, digest: $digest}}' > digest.json + - name: Upload digest artifact + uses: actions/upload-artifact@v4 + with: + name: image-digest-${{ matrix.name }} + path: digest.json + + collect-digests: + needs: [build-image] + runs-on: ubuntu-latest + steps: + - name: Download digest artifacts + uses: actions/download-artifact@v4 + with: + pattern: image-digest-* + merge-multiple: false + - name: Merge digests + run: | + sudo apt-get update && sudo apt-get install -y jq + jq -s 'reduce .[] as $item ({}; . * $item)' image-digest-*/digest.json > image-digests.json + - name: Upload merged digests + uses: actions/upload-artifact@v4 + with: + name: image-digests + path: image-digests.json diff --git a/.github/workflows/bump-chart-version.yml b/.github/workflows/bump-chart-version.yml new file mode 100644 index 00000000..8a238309 --- /dev/null +++ b/.github/workflows/bump-chart-version.yml @@ -0,0 +1,59 @@ +name: bump-chart-version +on: + workflow_dispatch: + inputs: + chart_version: + description: "Chart version to set (does not touch appVersion)" + required: true + type: string + ref: + description: "Git ref to bump (default: main)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + +jobs: + bump: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || 'main' }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install deps + run: | + python -m pip install --upgrade pip + python -m pip install "pyyaml==6.0.2" "packaging==25.0" + + - name: Bump chart version only + env: + CHART_VERSION: ${{ inputs.chart_version }} + run: | + if [ -z "${CHART_VERSION}" ]; then + echo "chart_version input is required" >&2 + exit 1 + fi + python tools/bump_chart_versions.py --mode chart-only --chart-version "$CHART_VERSION" + + - name: Open PR for chart version bump + uses: peter-evans/create-pull-request@v6 + with: + base: main + branch: chore/chart-bump-${{ inputs.chart_version }} + title: "chore(release): bump chart version to ${{ inputs.chart_version }}" + body: | + Bump Chart.yaml version to ${{ inputs.chart_version }} (appVersion unchanged). + commit-message: "chore(release): bump chart version to ${{ inputs.chart_version }}" + add-paths: | + infrastructure/**/Chart.yaml + token: ${{ secrets.PR_AUTOMATION_TOKEN }} + labels: chart-bump diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000..caf971d5 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,66 @@ +name: create-release +on: + pull_request_target: + types: [closed] + branches: [main] + +permissions: + contents: write + +jobs: + release: + if: >- + ${{ + github.event.pull_request.merged == true && + contains(github.event.pull_request.labels.*.name, 'refresh-locks') + }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.PR_AUTOMATION_TOKEN }} + + - name: Derive version from PR title + id: ver + run: | + TITLE="${{ github.event.pull_request.title }}" + VERSION=$(echo "$TITLE" | sed -nE 's/.*([0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?).*/\1/p' || true) + if [ -z "$VERSION" ]; then + echo "Could not extract version from PR title: $TITLE" >&2 + exit 1 + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Verify appVersion matches release version (clean semver) + env: + RELEASE_VERSION: ${{ steps.ver.outputs.version }} + run: | + if echo "$RELEASE_VERSION" | grep -q '\.post'; then + echo "Release version must be clean semver (no .post): $RELEASE_VERSION" >&2 + exit 1 + fi + APP_VERSION=$(awk '/^appVersion:/ {print $2}' infrastructure/rag/Chart.yaml | tr -d "\"'") + if [ -z "$APP_VERSION" ]; then + echo "Could not read appVersion from infrastructure/rag/Chart.yaml" >&2 + exit 1 + fi + if [ "$APP_VERSION" != "$RELEASE_VERSION" ]; then + echo "Chart appVersion ($APP_VERSION) does not match release version ($RELEASE_VERSION)" >&2 + exit 1 + fi + + - name: Create Git tag + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git tag -a "v${{ steps.ver.outputs.version }}" -m "Release v${{ steps.ver.outputs.version }}" + git push origin "v${{ steps.ver.outputs.version }}" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ steps.ver.outputs.version }} + name: v${{ steps.ver.outputs.version }} + generate_release_notes: true + token: ${{ secrets.PR_AUTOMATION_TOKEN }} diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index ad63eca7..75662f58 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -12,6 +12,12 @@ env: jobs: changes: + if: >- + ${{ + !contains(github.event.pull_request.labels.*.name, 'prepare-release') && + !contains(github.event.pull_request.labels.*.name, 'refresh-locks') && + !contains(github.event.pull_request.labels.*.name, 'chart-bump') + }} name: Detect Changes runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 00000000..ea157328 --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,96 @@ +name: prepare-release +on: + pull_request: + types: [closed] + branches: + - main + +permissions: + contents: write + pull-requests: write + +jobs: + changes: + if: >- + ${{ + github.event.pull_request.merged && + !contains(github.event.pull_request.labels.*.name, 'prepare-release') && + !contains(github.event.pull_request.labels.*.name, 'refresh-locks') && + !contains(github.event.pull_request.labels.*.name, 'chart-bump') + }} + name: Detect release-relevant changes + runs-on: ubuntu-latest + outputs: + releasable: ${{ steps.filter.outputs.releasable }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Filter paths + id: filter + uses: dorny/paths-filter@v2 + with: + filters: | + releasable: + - 'services/**' + - 'libs/**' + + prepare: + if: ${{ needs.changes.outputs.releasable == 'true' }} + runs-on: ubuntu-latest + needs: [changes] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '25' + + - name: Install semantic-release deps + run: npm ci + + - name: verify-dependencies-integrity + run: npm audit signatures + + - name: Compute next version (dry-run) + id: semrel + env: + GITHUB_TOKEN: ${{ secrets.PR_AUTOMATION_TOKEN }} + run: | + npx semantic-release --dry-run --no-ci | tee semrel.log + BASE_VERSION=$(grep -Eo "next release version is [0-9]+\.[0-9]+\.[0-9]+" semrel.log | awk '{print $5}') + if [ -z "$BASE_VERSION" ]; then echo "No new release required"; exit 1; fi + VERSION="${BASE_VERSION}.post$(date +%Y%m%d%H%M%S)" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install bump script deps + run: | + python -m pip install --upgrade pip + python -m pip install "tomlkit==0.13.3" "pyyaml==6.0.2" "packaging==25.0" + + - name: Bump internal libs only (no service pins) + run: | + python tools/bump_pyproject_deps.py --version "${{ steps.semrel.outputs.version }}" --bump-libs + + - name: Commit and open PR + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/release-${{ steps.semrel.outputs.version }} + title: "chore(release): prepare ${{ steps.semrel.outputs.version }}" + body: | + Prepare release ${{ steps.semrel.outputs.version }} + - bump internal libs versions only + commit-message: "chore(release): prepare ${{ steps.semrel.outputs.version }}" + add-paths: | + libs/**/pyproject.toml + labels: prepare-release + token: ${{ secrets.PR_AUTOMATION_TOKEN }} diff --git a/.github/workflows/promote-clean-semver.yml b/.github/workflows/promote-clean-semver.yml new file mode 100644 index 00000000..d32049a5 --- /dev/null +++ b/.github/workflows/promote-clean-semver.yml @@ -0,0 +1,174 @@ +name: promote-clean-semver +on: + workflow_dispatch: + inputs: + clean_version: + description: "Clean semver to promote (no .post)" + required: true + type: string + ref: + description: "Git ref to use (defaults to main)" + required: false + type: string + pre_version: + description: "Pre-release version (optional, for bookkeeping)" + required: false + type: string + dry_run: + description: "If true, skip publishing to PyPI (only TestPyPI)" + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + packages: write + +jobs: + promote: + name: Publish clean semver, refresh locks, bump appVersion + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || 'main' }} + + - name: Load version metadata (workflow_run) + if: ${{ github.event_name == 'workflow_run' }} + uses: actions/download-artifact@v4 + with: + name: pre-release-meta + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + path: meta + + - name: Determine versions + id: versions + run: | + set -euo pipefail + CLEAN_VERSION_INPUT="${{ inputs.clean_version || '' }}" + if [ -f meta/version.env ]; then + source meta/version.env + fi + CLEAN_VERSION="${CLEAN_VERSION_INPUT:-${CLEAN_VERSION:-}}" + if [ -z "${CLEAN_VERSION:-}" ]; then + echo "CLEAN_VERSION is required (input or artifact)" >&2 + exit 1 + fi + if echo "$CLEAN_VERSION" | grep -q '\.post'; then + echo "CLEAN_VERSION must be clean semver (no .post): $CLEAN_VERSION" >&2 + exit 1 + fi + echo "clean_version=$CLEAN_VERSION" >> $GITHUB_OUTPUT + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Poetry and deps + run: | + pip install poetry==2.1.3 + python -m pip install --upgrade pip + python -m pip install "tomlkit==0.13.3" "pyyaml==6.0.2" "packaging==25.0" + + - name: Configure TestPyPI repository + run: | + poetry config repositories.testpypi https://test.pypi.org/legacy/ + + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Update internal libs and service pins to clean version + env: + VERSION: ${{ steps.versions.outputs.clean_version }} + run: | + python tools/bump_pyproject_deps.py --version "$VERSION" --bump-libs --bump-service-pins + + - name: Publish clean semver to TestPyPI and PyPI (core-first) + env: + CLEAN_VERSION: ${{ steps.versions.outputs.clean_version }} + POETRY_HTTP_BASIC_TESTPYPI_USERNAME: __token__ + POETRY_HTTP_BASIC_TESTPYPI_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run || 'false' }} + run: | + set -euo pipefail + if [ -z "${POETRY_HTTP_BASIC_TESTPYPI_PASSWORD:-}" ]; then + echo "Missing TEST_PYPI_TOKEN secret" >&2 + exit 1 + fi + if [ "${DRY_RUN:-false}" != "true" ] && [ -z "${POETRY_PYPI_TOKEN_PYPI:-}" ]; then + echo "Missing PYPI_TOKEN secret" >&2 + exit 1 + fi + + source tools/publish_libs.sh + + echo "Publishing clean version $CLEAN_VERSION (core-first) to TestPyPI..." + publish_lib "rag-core-lib" "-r testpypi" "$CLEAN_VERSION" + wait_for_index "rag-core-lib" "$CLEAN_VERSION" "https://test.pypi.org" "TestPyPI" + for lib in rag-core-api admin-api-lib extractor-api-lib; do + publish_lib "$lib" "-r testpypi" "$CLEAN_VERSION" + done + + if [ "${DRY_RUN:-false}" = "true" ]; then + echo "Dry run enabled: skipping PyPI publish." + else + echo "Publishing clean version $CLEAN_VERSION (core-first) to PyPI..." + publish_lib "rag-core-lib" "" "$CLEAN_VERSION" + wait_for_index "rag-core-lib" "$CLEAN_VERSION" "https://pypi.org" "PyPI" + for lib in rag-core-api admin-api-lib extractor-api-lib; do + publish_lib "$lib" "" "$CLEAN_VERSION" + done + fi + + - name: Clear poetry caches + run: | + poetry cache clear --all pypi -n || true + poetry cache clear --all testpypi -n || true + + - name: Refresh service lockfiles + env: + VERSION: ${{ steps.versions.outputs.clean_version }} + run: | + for svc in services/rag-backend services/admin-backend services/document-extractor services/mcp-server; do + if [ -f "$svc/pyproject.toml" ]; then + echo "Locking $svc" + ( + cd "$svc" + poetry lock -v || ( + echo "Lock failed, clearing caches and retrying..."; + poetry cache clear --all pypi -n || true; + poetry cache clear --all testpypi -n || true; + sleep 10; + poetry lock -v + ) + ) + fi + done + + - name: Bump Chart appVersion to clean version (leave chart version for manual chart publish) + env: + APP_VERSION: ${{ steps.versions.outputs.clean_version }} + run: | + python tools/bump_chart_versions.py --app-version "$APP_VERSION" --mode app-only + + - name: Open PR with updated locks, pins, and Chart appVersion + id: cpr + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/refresh-locks-${{ steps.versions.outputs.clean_version }}-${{ github.run_number }} + title: "chore(release): refresh service lockfiles for ${{ steps.versions.outputs.clean_version }}" + body: | + Refresh service poetry.lock files, dependency pins, and Chart appVersion for version ${{ steps.versions.outputs.clean_version }}. + commit-message: "chore(release): refresh service lockfiles, pins, and Chart appVersion" + add-paths: | + services/**/pyproject.toml + services/**/poetry.lock + infrastructure/**/Chart.yaml + libs/**/pyproject.toml + labels: refresh-locks + token: ${{ secrets.PR_AUTOMATION_TOKEN }} diff --git a/.github/workflows/publish-chart-packages.yml b/.github/workflows/publish-chart-packages.yml new file mode 100644 index 00000000..3c17e553 --- /dev/null +++ b/.github/workflows/publish-chart-packages.yml @@ -0,0 +1,113 @@ +name: publish-chart-packages +on: + pull_request: + branches: [main] + types: [closed] + workflow_dispatch: + inputs: + chart_version: + description: "Chart version to publish (default: read from Chart.yaml)" + required: false + type: string + ref: + description: "Git ref to package from (default: main)" + required: false + type: string + +permissions: + contents: write + packages: write + pages: write + id-token: write + +env: + OCI_REGISTRY: ghcr.io + +jobs: + publish: + runs-on: ubuntu-latest + if: | + (github.event_name == 'pull_request' && github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'chart-bump')) + || github.event_name == 'workflow_dispatch' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ inputs.ref || 'main' }} + + - name: Setup Helm + uses: azure/setup-helm@v4 + + - name: Login to GHCR for Helm OCI + run: echo ${{ secrets.GHCR_PAT }} | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Determine chart version + id: meta + run: | + set -euo pipefail + INPUT_VER="${{ inputs.chart_version }}" + FILE_VER=$(awk '/^version:/ {print $2}' infrastructure/rag/Chart.yaml | tr -d "\"'") + CHART_VERSION="${INPUT_VER:-$FILE_VER}" + if [ -z "$CHART_VERSION" ]; then + echo "Could not determine chart version" >&2 + exit 1 + fi + echo "chart_version=$CHART_VERSION" >> $GITHUB_OUTPUT + + - name: Verify chart version matches input (if provided) + env: + INPUT_VER: ${{ inputs.chart_version }} + FILE_VER: ${{ steps.meta.outputs.chart_version }} + run: | + if [ -n "$INPUT_VER" ] && [ "$INPUT_VER" != "$FILE_VER" ]; then + echo "Chart.yaml version ($FILE_VER) does not match input $INPUT_VER" >&2 + exit 1 + fi + + - name: Package chart + run: | + set -euo pipefail + CHART_DIR="infrastructure/rag" + mkdir -p dist + helm dependency update "$CHART_DIR" || true + helm package "$CHART_DIR" --destination dist + ls -la dist + + - name: Push chart to GHCR (OCI) + env: + CHART_VERSION: ${{ steps.meta.outputs.chart_version }} + run: | + set -euo pipefail + PKG=$(ls dist/*.tgz) + helm show chart "$PKG" | grep -E "^version: " + helm push "$PKG" oci://$OCI_REGISTRY/${{ github.repository_owner }}/charts + + - name: Build Helm repo index for Pages + env: + CHART_VERSION: ${{ steps.meta.outputs.chart_version }} + run: | + set -euo pipefail + PKG=$(ls dist/*.tgz) + REPO="${GITHUB_REPOSITORY#*/}" + BASE_URL="https://${GITHUB_REPOSITORY_OWNER}.github.io/${REPO}" + helm repo index dist --url "$BASE_URL" + echo "Index generated for $BASE_URL" + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: dist + + deploy-pages: + needs: publish + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/publish-pre-and-qa.yml b/.github/workflows/publish-pre-and-qa.yml new file mode 100644 index 00000000..ce33f4a1 --- /dev/null +++ b/.github/workflows/publish-pre-and-qa.yml @@ -0,0 +1,231 @@ +name: publish-pre-and-qa +on: + pull_request: + branches: [main] + types: [closed] + workflow_dispatch: + inputs: + clean_version: + description: "Clean semver to promote (no .post)" + required: true + type: string + ref: + description: "Git ref to use (defaults to main)" + required: false + type: string + pre_version: + description: "Pre-release version (optional, for bookkeeping)" + required: false + type: string + dry_run: + description: "If true, skip publishing to PyPI (only TestPyPI)" + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + packages: write + issues: write + +jobs: + publish-pre: + name: Publish pre-release to TestPyPI (core-first) + if: ${{ github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'prepare-release') }} + runs-on: ubuntu-latest + outputs: + version: ${{ steps.ver.outputs.version }} + clean_version: ${{ steps.ver.outputs.clean_version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Derive version from PR title + id: ver + run: | + set -euo pipefail + TITLE="${{ github.event.pull_request.title }}" + VERSION=$(echo "$TITLE" | sed -nE 's/.*prepare ([0-9]+\.[0-9]+\.[0-9]+(\.post[0-9]+)?).*/\1/p' || true) + if [ -z "$VERSION" ]; then + echo "Could not derive version from PR title: $TITLE" >&2 + exit 1 + fi + CLEAN_VERSION="${VERSION%%.post*}" + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "clean_version=$CLEAN_VERSION" >> $GITHUB_OUTPUT + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install Poetry + run: | + pip install poetry==2.1.3 + + - name: Configure TestPyPI repository + run: | + poetry config repositories.testpypi https://test.pypi.org/legacy/ + + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Publish rag-core-lib first (TestPyPI) and wait for index + env: + VERSION: ${{ steps.ver.outputs.version }} + POETRY_HTTP_BASIC_TESTPYPI_USERNAME: __token__ + POETRY_HTTP_BASIC_TESTPYPI_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: | + set -euo pipefail + if [ -z "${POETRY_HTTP_BASIC_TESTPYPI_PASSWORD:-}" ]; then + echo "Missing TEST_PYPI_TOKEN secret" >&2 + exit 1 + fi + source tools/publish_libs.sh + publish_lib "rag-core-lib" "-r testpypi" "$VERSION" + wait_for_index "rag-core-lib" "$VERSION" "https://test.pypi.org" "TestPyPI" + + - name: Publish remaining libs to TestPyPI + env: + VERSION: ${{ steps.ver.outputs.version }} + POETRY_HTTP_BASIC_TESTPYPI_USERNAME: __token__ + POETRY_HTTP_BASIC_TESTPYPI_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: | + set -euo pipefail + source tools/publish_libs.sh + for lib in rag-core-api admin-api-lib extractor-api-lib; do + path="libs/$lib" + [ -d "$path" ] || { echo "Missing $path" >&2; exit 1; } + publish_lib "$lib" "-r testpypi" "$VERSION" "$path" + done + + - name: Persist version metadata for downstream workflows + env: + VERSION: ${{ steps.ver.outputs.version }} + CLEAN_VERSION: ${{ steps.ver.outputs.clean_version }} + run: | + set -euo pipefail + mkdir -p meta + { + echo "PRE_VERSION=${VERSION}" + echo "CLEAN_VERSION=${CLEAN_VERSION}" + } > meta/version.env + # Upload artifact for workflow_run trigger consumption + - name: Upload version artifact + uses: actions/upload-artifact@v4 + with: + name: pre-release-meta + path: meta/version.env + retention-days: 5 + + qa-verify: + name: Build QA images and update Argo deployment repo + runs-on: ubuntu-latest + needs: publish-pre + if: ${{ needs.publish-pre.result == 'success' }} + env: + PRE_RELEASE_VERSION: ${{ needs.publish-pre.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Validate QA secrets are present + run: | + missing=() + [ -z "${{ secrets.STACKIT_REGISTRY_USERNAME }}" ] && missing+=("STACKIT_REGISTRY_USERNAME") + [ -z "${{ secrets.STACKIT_REGISTRY_PASSWORD }}" ] && missing+=("STACKIT_REGISTRY_PASSWORD") + [ -z "${{ secrets.DEPLOY_REPO_URL }}" ] && missing+=("DEPLOY_REPO_URL") + [ -z "${{ secrets.DEPLOY_REPO_TOKEN }}" ] && missing+=("DEPLOY_REPO_TOKEN") + [ -z "${{ secrets.QA_IMAGE_REGISTRY }}" ] && echo "QA_IMAGE_REGISTRY not set, defaulting to registry.onstackit.cloud/qa-rag-template" + if [ ${#missing[@]} -gt 0 ]; then + echo "Missing required secrets: ${missing[*]}" >&2 + exit 1 + fi + + - name: Set image registry and version + env: + QA_IMAGE_REGISTRY: ${{ secrets.QA_IMAGE_REGISTRY }} + run: | + IMAGE_REGISTRY="${QA_IMAGE_REGISTRY:-registry.onstackit.cloud/qa-rag-template}" + echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}" >> "$GITHUB_ENV" + echo "IMAGE_TAG=${PRE_RELEASE_VERSION}" >> "$GITHUB_ENV" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to STACKIT registry + env: + STACKIT_REGISTRY_USERNAME: ${{ secrets.STACKIT_REGISTRY_USERNAME }} + STACKIT_REGISTRY_PASSWORD: ${{ secrets.STACKIT_REGISTRY_PASSWORD }} + run: | + echo "$STACKIT_REGISTRY_PASSWORD" | docker login registry.onstackit.cloud -u "$STACKIT_REGISTRY_USERNAME" --password-stdin + + - name: Build and push QA images + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.IMAGE_TAG }} + run: | + set -euo pipefail + images=( + "rag-backend services/rag-backend/Dockerfile ." + "admin-backend services/admin-backend/Dockerfile ." + "document-extractor services/document-extractor/Dockerfile ." + "mcp-server services/mcp-server/Dockerfile ." + "frontend services/frontend/apps/chat-app/Dockerfile services/frontend" + "admin-frontend services/frontend/apps/admin-app/Dockerfile services/frontend" + ) + for entry in "${images[@]}"; do + IFS=" " read -r name dockerfile context <<<"$entry" + ref="$IMAGE_REGISTRY/$name:$IMAGE_TAG" + echo "Building and pushing $ref (Dockerfile=$dockerfile context=$context)" + docker buildx build --platform linux/amd64 -t "$ref" -f "$dockerfile" "$context" --push + done + + - name: Checkout deployment repo + env: + DEPLOY_REPO_URL: ${{ secrets.DEPLOY_REPO_URL }} + DEPLOY_REPO_BRANCH: ${{ secrets.DEPLOY_REPO_BRANCH || 'main' }} + DEPLOY_REPO_TOKEN: ${{ secrets.DEPLOY_REPO_TOKEN }} + run: | + set -euo pipefail + mkdir -p /tmp/deploy-repo + AUTH_URL="${DEPLOY_REPO_URL/https:\/\//https://${DEPLOY_REPO_TOKEN}@}" + git clone --depth 1 --branch "${DEPLOY_REPO_BRANCH:-main}" "$AUTH_URL" /tmp/deploy-repo + + - name: Update values file in deployment repo + env: + IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }} + IMAGE_TAG: ${{ env.IMAGE_TAG }} + DEPLOY_VALUES_FILE: ${{ secrets.DEPLOY_VALUES_FILE || 'values-qa.yaml' }} + run: | + set -euo pipefail + cd /tmp/deploy-repo + python -m pip install --quiet pyyaml + python "$GITHUB_WORKSPACE/tools/update_deploy_values.py" \ + --values-file "$DEPLOY_VALUES_FILE" \ + --image-registry "$IMAGE_REGISTRY" \ + --image-tag "$IMAGE_TAG" + + - name: Commit and push deployment repo changes + env: + DEPLOY_REPO_BRANCH: ${{ secrets.DEPLOY_REPO_BRANCH || 'main' }} + DEPLOY_REPO_TOKEN: ${{ secrets.DEPLOY_REPO_TOKEN }} + DEPLOY_REPO_URL: ${{ secrets.DEPLOY_REPO_URL }} + DEPLOY_GIT_USER_NAME: ${{ secrets.DEPLOY_GIT_USER_NAME || 'github-actions' }} + DEPLOY_GIT_USER_EMAIL: ${{ secrets.DEPLOY_GIT_USER_EMAIL || 'github-actions@users.noreply.github.com' }} + run: | + set -euo pipefail + cd /tmp/deploy-repo + git config user.name "${DEPLOY_GIT_USER_NAME}" + git config user.email "${DEPLOY_GIT_USER_EMAIL}" + git add . + if git diff --cached --quiet; then + echo "No changes to commit" + exit 0 + fi + git commit -m "chore: update QA images to ${IMAGE_TAG}" + AUTH_URL="${DEPLOY_REPO_URL/https:\/\//https://${DEPLOY_REPO_TOKEN}@}" + git push "$AUTH_URL" "HEAD:${DEPLOY_REPO_BRANCH:-main}" diff --git a/.github/workflows/semantic-release.yml b/.github/workflows/semantic-release.yml deleted file mode 100644 index 4a8c44e3..00000000 --- a/.github/workflows/semantic-release.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: semantic-release -on: - workflow_dispatch: - push: - # Only trigger on merged PRs, not on every PR push - pull_request: - types: [closed] - branches: - - main - -permissions: - contents: write - packages: write - -jobs: - semantic-release: - name: semantic-release - runs-on: ubuntu-latest - # Only run on push to main, manual dispatch, or when PR is merged - if: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - outputs: - new-release-published: ${{ steps.semantic-release.outputs.new-release-published }} - new-release-version: ${{ steps.semantic-release.outputs.new-release-version }} - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: setup-node - uses: actions/setup-node@v4 - with: - node-version: "22.20.0" - - - name: create-archives - run: | - TEMP_DIR=$(mktemp -d) - tar --warning=no-file-changed \ - --exclude=".git" \ - --exclude="/.git" \ - --exclude="node_modules" \ - -czf "$TEMP_DIR/action-main-release-trials.tar.gz" . - zip -r "$TEMP_DIR/action-main-release-trials.zip" . \ - -x ".git" "node_modules" - mv "$TEMP_DIR"/*.{tar.gz,zip} . - rm -rf "$TEMP_DIR" - - - name: install-dependencies - run: npm ci - - - name: verify-dependencies-integrity - run: npm audit signatures - - - name: create-semantic-release - id: semantic-release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Run semantic-release and capture the output - npx semantic-release > semantic-release-output.txt 2>&1 || true - - # Check if a new release was published by looking for the success message - if grep -q "Published release" semantic-release-output.txt; then - echo "new-release-published=true" >> $GITHUB_OUTPUT - - # Extract the version from the output - VERSION=$(grep "Published release" semantic-release-output.txt | sed -n 's/.*Published release \([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/p') - - if [[ -n "$VERSION" ]]; then - echo "new-release-version=$VERSION" >> $GITHUB_OUTPUT - echo "āœ… New release published: $VERSION" - else - echo "āŒ Could not extract version from semantic-release output" - exit 1 - fi - else - echo "new-release-published=false" >> $GITHUB_OUTPUT - echo "ā„¹ļø No new release published" - fi - - build-and-push-images: - name: build-and-push-images - runs-on: ubuntu-latest - needs: semantic-release - if: needs.semantic-release.outputs.new-release-published == 'true' - strategy: - matrix: - service: - - name: rag-backend - dockerfile: services/rag-backend/Dockerfile - image: rag-backend - - name: admin-backend - dockerfile: services/admin-backend/Dockerfile - image: admin-backend - - name: document-extractor - dockerfile: services/document-extractor/Dockerfile - image: document-extractor - - name: mcp-server - dockerfile: services/mcp-server/Dockerfile - image: mcp-server - - name: frontend - dockerfile: services/frontend/apps/chat-app/Dockerfile - image: frontend - - name: admin-frontend - dockerfile: services/frontend/apps/admin-app/Dockerfile - image: admin-frontend - steps: - - name: debug-job-inputs - run: | - echo "šŸ” Debug: needs.semantic-release.outputs.new-release-published = ${{ needs.semantic-release.outputs.new-release-published }}" - echo "šŸ” Debug: needs.semantic-release.outputs.new-release-version = ${{ needs.semantic-release.outputs.new-release-version }}" - - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: login-to-github-container-registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: build-and-push-${{ matrix.service.name }} - run: | - docker build \ - --tag ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:v${{ needs.semantic-release.outputs.new-release-version }} \ - --tag ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:latest \ - --file ${{ matrix.service.dockerfile }} \ - . - - docker push ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:v${{ needs.semantic-release.outputs.new-release-version }} - docker push ghcr.io/${{ github.repository_owner }}/rag-template/${{ matrix.service.image }}:latest - - - name: deployment-summary - if: strategy.job-index == 0 # Only run on first job in matrix - run: | - echo "## šŸš€ Deployment Summary" >> $GITHUB_STEP_SUMMARY - echo "**New version:** v${{ needs.semantic-release.outputs.new-release-version }}" >> $GITHUB_STEP_SUMMARY - echo "**Services built and deployed:**" >> $GITHUB_STEP_SUMMARY - echo "- rag-backend" >> $GITHUB_STEP_SUMMARY - echo "- admin-backend" >> $GITHUB_STEP_SUMMARY - echo "- document-extractor" >> $GITHUB_STEP_SUMMARY - echo "- mcp-server" >> $GITHUB_STEP_SUMMARY - echo "- frontend" >> $GITHUB_STEP_SUMMARY - echo "- admin-frontend" >> $GITHUB_STEP_SUMMARY - echo "**Registry:** ghcr.io/${{ github.repository_owner }}/rag-template" >> $GITHUB_STEP_SUMMARY diff --git a/infrastructure/rag/templates/backend/service.yaml b/infrastructure/rag/templates/backend/service.yaml index 847b5c7c..4c98e05e 100644 --- a/infrastructure/rag/templates/backend/service.yaml +++ b/infrastructure/rag/templates/backend/service.yaml @@ -2,6 +2,10 @@ apiVersion: v1 kind: Service metadata: name: {{ .Values.backend.name }} + {{- with .Values.backend.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.backend.service.type }} ports: diff --git a/infrastructure/rag/values.yaml b/infrastructure/rag/values.yaml index 8c759336..c8d95a29 100644 --- a/infrastructure/rag/values.yaml +++ b/infrastructure/rag/values.yaml @@ -131,6 +131,7 @@ backend: service: type: ClusterIP port: 8080 + annotations: {} pythonPathEnv: PYTHONPATH: src diff --git a/package-lock.json b/package-lock.json index dec6b938..f62eadbb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -112,6 +112,7 @@ "integrity": "sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.1", @@ -2468,6 +2469,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -2983,6 +2985,34 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/npm/node_modules/@npmcli/arborist/node_modules/abbrev": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", + "integrity": "sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==", + "dev": true, + "inBundle": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/@npmcli/arborist/node_modules/nopt": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz", + "integrity": "sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==", + "dev": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "abbrev": "^3.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, "node_modules/npm/node_modules/@npmcli/config": { "version": "10.4.3", "dev": true, @@ -3002,6 +3032,34 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/npm/node_modules/@npmcli/config/node_modules/abbrev": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-3.0.1.tgz", + "integrity": "sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==", + "dev": true, + "inBundle": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/@npmcli/config/node_modules/nopt": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-8.1.0.tgz", + "integrity": "sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==", + "dev": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "abbrev": "^3.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, "node_modules/npm/node_modules/@npmcli/fs": { "version": "4.0.0", "dev": true, @@ -3920,6 +3978,49 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/npm/node_modules/libnpmdiff/node_modules/@npmcli/installed-package-contents": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/installed-package-contents/-/installed-package-contents-3.0.0.tgz", + "integrity": "sha512-fkxoPuFGvxyrH+OQzyTkX2LUEamrF4jZSmxjAtPPHHGO0dqsQ8tTKjnIS8SAnPHdk2I03BDtSMR5K/4loKg79Q==", + "dev": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "npm-bundled": "^4.0.0", + "npm-normalize-package-bin": "^4.0.0" + }, + "bin": { + "installed-package-contents": "bin/index.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/libnpmdiff/node_modules/npm-bundled": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-4.0.0.tgz", + "integrity": "sha512-IxaQZDMsqfQ2Lz37VvyyEtKLe8FsRZuysmedy/N06TU1RyVppYKXrO4xIhR0F+7ubIBox6Q7nir6fQI3ej39iA==", + "dev": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "npm-normalize-package-bin": "^4.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm/node_modules/libnpmdiff/node_modules/npm-normalize-package-bin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-4.0.0.tgz", + "integrity": "sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==", + "dev": true, + "inBundle": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, "node_modules/npm/node_modules/libnpmexec": { "version": "10.1.9", "dev": true, @@ -4902,6 +5003,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -5578,6 +5680,7 @@ "integrity": "sha512-6qGjWccl5yoyugHt3jTgztJ9Y0JVzyH8/Voc/D8PlLat9pwxQYXz7W1Dpnq5h0/G5GCYGUaDSlYcyk3AMh5A6g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -6433,6 +6536,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, diff --git a/tools/bump_chart_versions.py b/tools/bump_chart_versions.py new file mode 100644 index 00000000..d349da86 --- /dev/null +++ b/tools/bump_chart_versions.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import argparse +import pathlib +import sys +import re + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def _to_chart_version(app_version: str) -> str: + """Convert app_version to a SemVer 2.0.0 compliant Helm chart version. + + Examples: + - "2.0.0.post20250904105936" -> "2.0.0-post.20250904105936" + - "2.0.1" -> "2.0.1" + - "2.0.1-rc.1" -> "2.0.1-rc.1" + - Fallback: if an unexpected format is provided, try to keep a valid semver + by extracting the leading MAJOR.MINOR.PATCH. + """ + # Case 1: our prepare-release format "X.Y.Z.post" + m = re.fullmatch(r"(?P\d+\.\d+\.\d+)\.post(?P\d+)", app_version) + if m: + return f"{m.group('base')}-post.{m.group('ts')}" + + # Case 2: already valid semver (optionally with pre-release or build metadata) + if re.fullmatch(r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", app_version): + return app_version + + # Fallback: keep only the base version if present + base = re.match(r"(\d+\.\d+\.\d+)", app_version) + return base.group(1) if base else app_version + + +def bump_chart(chart_path: pathlib.Path, app_version: str, mode: str): + data = yaml.safe_load(chart_path.read_text()) + if mode in ("app-and-chart", "app-only"): + if not app_version: + raise ValueError("app_version is required for mode app-and-chart or app-only") + data['appVersion'] = str(app_version) + if mode in ("app-and-chart", "chart-only"): + if mode == "chart-only": + if not app_version: + raise ValueError("chart-only mode requires chart_version provided via app_version argument") + data['version'] = str(app_version) + else: + data['version'] = _to_chart_version(str(app_version)) + chart_path.write_text(yaml.safe_dump(data, sort_keys=False)) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--app-version', help='App version to set (required for app-and-chart/app-only)') + p.add_argument('--chart-version', help='Chart version to set (required for chart-only)') + p.add_argument( + '--mode', + choices=['app-and-chart', 'app-only', 'chart-only'], + default='app-and-chart', + help='app-and-chart: bump appVersion and chart version; app-only: bump only appVersion; chart-only: bump only chart version' + ) + args = p.parse_args() + + app_version = args.app_version + if args.mode == 'chart-only': + app_version = args.chart_version + + charts = list((ROOT / 'infrastructure').glob('*/Chart.yaml')) + for ch in charts: + bump_chart(ch, app_version, args.mode) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/bump_pyproject_deps.py b/tools/bump_pyproject_deps.py new file mode 100644 index 00000000..8e3688b8 --- /dev/null +++ b/tools/bump_pyproject_deps.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +import argparse +import pathlib +import re +import sys +from typing import Any, List, Optional + +import tomlkit + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +# Only bump versions for internal libs here +LIBS_VERSION_FILES = [ + ROOT / 'libs' / 'rag-core-lib' / 'pyproject.toml', + ROOT / 'libs' / 'rag-core-api' / 'pyproject.toml', + ROOT / 'libs' / 'admin-api-lib' / 'pyproject.toml', + ROOT / 'libs' / 'extractor-api-lib' / 'pyproject.toml', +] + +# Service pins to update after libs are published +SERVICE_PINS = { + ROOT / 'services' / 'rag-backend' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.rag-core-api': '=={v}', + 'tool.poetry.group.prod.dependencies.rag-core-lib': '=={v}', + }, + ROOT / 'services' / 'admin-backend' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.admin-api-lib': '=={v}', + 'tool.poetry.group.prod.dependencies.rag-core-lib': '=={v}', + }, + ROOT / 'services' / 'document-extractor' / 'pyproject.toml': { + 'tool.poetry.group.prod.dependencies.extractor-api-lib': '=={v}', + }, +} + + +def replace_version_line(text: str, new_version: str) -> str: + lines = text.splitlines(keepends=True) + in_tool_poetry = False + for i, line in enumerate(lines): + stripped = line.strip() + if stripped.startswith('[tool.poetry]'): + in_tool_poetry = True + continue + if in_tool_poetry and stripped.startswith('[') and not stripped.startswith('[tool.poetry]'): + # left the section without finding version; stop scanning section + break + if in_tool_poetry and stripped.startswith('version'): + # Replace only the version value, keep indentation and spacing + lines[i] = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{new_version}"', line) + return ''.join(lines) + # If no version line found, append it to the [tool.poetry] section + out = ''.join(lines) + return out + f"\n[tool.poetry]\nversion = \"{new_version}\"\n" + + +def _get_table(doc: tomlkit.TOMLDocument, path: List[str]) -> Optional[Any]: + ref: Any = doc + for key in path: + try: + if key not in ref: # mapping-like check + return None + ref = ref[key] + except Exception: + return None + return ref + + +def bump(version: str, bump_libs: bool = True, bump_service_pins: bool = True): + # 1) bump libs versions (textual, non-destructive) + if bump_libs: + for file in LIBS_VERSION_FILES: + txt = file.read_text() + new_txt = replace_version_line(txt, version) + file.write_text(new_txt) + print(f"Updated {file} -> tool.poetry.version = {version}") + + # 2) bump service pins only inside [tool.poetry.group.prod.dependencies] + if bump_service_pins: + for file, mapping in SERVICE_PINS.items(): + txt = file.read_text() + doc = tomlkit.parse(txt) + deps = _get_table(doc, [ + 'tool', 'poetry', 'group', 'prod', 'dependencies' + ]) + if deps is None or not hasattr(deps, '__contains__'): + print(f"Skip {file}: prod dependencies table not found") + file.write_text(tomlkit.dumps(doc)) + continue + for dotted, template in mapping.items(): + pkg = dotted.split('.')[-1] + if pkg in deps: + val = template.format(v=version) + deps[pkg] = val + print(f"Pinned {file} -> {pkg} = {val}") + else: + print(f"Skip {file}: {pkg} not present in prod dependencies") + file.write_text(tomlkit.dumps(doc)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--version', required=True) + ap.add_argument('--bump-libs', action='store_true', help='Bump versions in internal libs only') + ap.add_argument('--bump-service-pins', action='store_true', help='Bump service dependency pins only') + args = ap.parse_args() + + # Backward compatibility: if neither flag is provided, do both + bump_libs = args.bump_libs or (not args.bump_libs and not args.bump_service_pins) + bump_service_pins = args.bump_service_pins or (not args.bump_libs and not args.bump_service_pins) + + bump(args.version, bump_libs=bump_libs, bump_service_pins=bump_service_pins) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/publish_libs.sh b/tools/publish_libs.sh new file mode 100755 index 00000000..cfd6ecc7 --- /dev/null +++ b/tools/publish_libs.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Shared helpers for publishing internal libs and waiting for index visibility. +set -euo pipefail + +publish_lib() { + local lib="$1" + local repo_flag="$2" # "" or "-r testpypi" + local version="$3" + local path="${4:-libs/$lib}" + if [ ! -d "$path" ]; then + echo "Missing $path" >&2 + exit 1 + fi + echo "Publishing $lib ($version) to ${repo_flag:-pypi default}" + (cd "$path" && poetry version "$version" && poetry build && poetry publish $repo_flag) +} + +wait_for_index() { + local name="$1" + local version="$2" + local base_url="$3" + local label="$4" + echo "Waiting for $name==$version on $label" + for _ in $(seq 1 60); do + json_ok=false + simple_ok=false + if curl -fsSL "$base_url/pypi/$name/json" | jq -e --arg v "$version" '.releases[$v] | length > 0' >/dev/null; then + json_ok=true + fi + if curl -fsSL "$base_url/simple/$name/" | grep -q "$version"; then + simple_ok=true + fi + if [ "$json_ok" = true ] && [ "$simple_ok" = true ]; then + echo "$name==$version visible on $label" + return 0 + fi + sleep 5 + done + echo "$name==$version not visible on $label after waiting" >&2 + return 1 +} diff --git a/tools/update-helm-values.py b/tools/update-helm-values.py deleted file mode 100644 index 283ad051..00000000 --- a/tools/update-helm-values.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -""" -Update Helm values.yaml with new version tags for specific services only, -preserving all existing formatting and disabling line wrapping via ruamel.yaml. -""" - -import sys -from ruamel.yaml import YAML -from ruamel.yaml.scalarstring import DoubleQuotedScalarString - -def update_helm_values(file_path, new_version): - """Update specific service image tags in values.yaml with ruamel.yaml.""" - yaml = YAML() - yaml.preserve_quotes = True - yaml.width = float('inf') # ← no wrapping, even for long lines - yaml.indent(mapping=2, sequence=4, offset=2) - - with open(file_path, 'r') as f: - data = yaml.load(f) - - services_to_update = [ - ('backend', 'mcp', 'image', 'tag'), - ('backend', 'image', 'tag'), - ('frontend', 'image', 'tag'), - ('adminBackend', 'image', 'tag'), - ('extractor', 'image', 'tag'), - ('adminFrontend', 'image', 'tag'), - ] - - new_tag = f"v{new_version}" - updated_count = 0 - - for path in services_to_update: - node = data - try: - for key in path[:-1]: - node = node[key] - tag_key = path[-1] - if tag_key in node: - old = node[tag_key] - if str(old) != new_tag: - if hasattr(old, 'style') and old.style in ('"', "'"): - node[tag_key] = DoubleQuotedScalarString(new_tag) - else: - node[tag_key] = new_tag - print(f"āœ… Updated {'.'.join(path)}: {old!r} → {new_tag!r}") - updated_count += 1 - else: - print(f"āš ļø {'.'.join(path)} already at {new_tag!r}") - else: - print(f"āŒ Could not find {'.'.join(path)}") - except (KeyError, TypeError) as e: - print(f"āŒ Could not access {'.'.join(path)}: {e}") - - if updated_count: - with open(file_path, 'w') as f: - yaml.dump(data, f) - print(f"\nāœ… Updated {updated_count} tag(s) in {file_path}") - return True - else: - print(f"\nāš ļø No changes needed in {file_path}") - return False - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: update-helm-values.py ") - sys.exit(1) - - version = sys.argv[1] - file_path = 'infrastructure/rag/values.yaml' - success = update_helm_values(file_path, version) - sys.exit(0 if success else 1) diff --git a/tools/update_deploy_values.py b/tools/update_deploy_values.py new file mode 100644 index 00000000..4f460830 --- /dev/null +++ b/tools/update_deploy_values.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Update deployment repo values file with image registry/tag overrides.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any, Dict + +import yaml + + +def ensure(mapping: Dict[str, Any], key: str) -> Dict[str, Any]: + """Ensure key exists and is a dict.""" + if key not in mapping or mapping[key] is None: + mapping[key] = {} + if not isinstance(mapping[key], dict): + raise TypeError(f"Expected dict at {key}, got {type(mapping[key])}") + return mapping[key] + + +def update_values(values_path: Path, image_registry: str, image_tag: str) -> None: + if values_path.exists(): + data = yaml.safe_load(values_path.read_text(encoding="utf-8")) or {} + else: + data = {} + + components = { + "backend": "rag-backend", + "adminBackend": "admin-backend", + "extractor": "document-extractor", + "frontend": "frontend", + "adminFrontend": "admin-frontend", + } + + for key, image_name in components.items(): + comp = ensure(data, key) + image_block = ensure(comp, "image") + image_block["repository"] = f"{image_registry}/{image_name}" + image_block["tag"] = image_tag + + backend = ensure(data, "backend") + mcp = ensure(backend, "mcp") + mcp_image = ensure(mcp, "image") + mcp_image["repository"] = f"{image_registry}/mcp-server" + mcp_image["tag"] = image_tag + + values_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Update image overrides in a values file.") + parser.add_argument("--values-file", required=True, help="Path to values-qa.yaml in deployment repo") + parser.add_argument("--image-registry", required=True, help="Image registry base (e.g. registry.onstackit.cloud/qa-rag-template)") + parser.add_argument("--image-tag", required=True, help="Image tag/version to set") + args = parser.parse_args() + + update_values(Path(args.values_file), args.image_registry, args.image_tag) + + +if __name__ == "__main__": + main()