.deb package caching

Artem Ermoshkin · Artem Ermoshkin · commit 1fd56c1bc8b4 · 2026-07-02T17:50:03.000+03:00
diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml
@@ -39,68 +39,33 @@ jobs:
           submodules: true
           ref: ${{ steps.tag.outputs.ref }}
 
+      - name: Prepare Debian package cache key
+        id: deb-package-cache-key
+        shell: bash
+        run: |
+          echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Debian package build cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            .deb-ccache
+            build_googleapis_deb
+            build-deb
+          key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            ${{ steps.deb-package-cache-key.outputs.prefix }}-
+            ubuntu-24.04-deb-packages-
+
       - name: Build .deb packages in Ubuntu 24.04 container
         shell: bash
         run: |
-          mkdir -p artifacts
+          mkdir -p artifacts .deb-ccache build_googleapis_deb build-deb
           docker run --rm --network host \
+            -e CCACHE_DIR=/source/.deb-ccache \
             -v "$PWD:/source" \
             ubuntu:24.04 \
-            bash -c '
-              set -e
-              export DEBIAN_FRONTEND=noninteractive
-              apt-get update
-              apt-get install -y \
-                build-essential \
-                cmake \
-                pkg-config \
-                git \
-                libidn11-dev \
-                libssl-dev \
-                zlib1g-dev \
-                libprotobuf-dev \
-                protobuf-compiler \
-                libgrpc++-dev \
-                protobuf-compiler-grpc \
-                libbrotli-dev \
-                liblz4-dev \
-                libzstd-dev \
-                libbz2-dev \
-                libxxhash-dev \
-                libsnappy-dev \
-                libdouble-conversion-dev \
-                libgtest-dev \
-                libre2-dev \
-                libc-ares-dev \
-                rapidjson-dev \
-                python3 \
-                python3-six \
-                ragel \
-                yasm
-
-              cd /source
-              cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
-              cmake --build build_googleapis_deb -j$(nproc)
-              cmake --build build_googleapis_deb --target package
-              dpkg -i build_googleapis_deb/*.deb
-
-              ./scripts/generate-debian-directory.sh
-              cmake -S . -B build-deb \
-                -DCMAKE_BUILD_TYPE=Release \
-                -DYDB_SDK_INSTALL=ON \
-                -DYDB_SDK_EXAMPLES=OFF \
-                -DYDB_SDK_TESTS=OFF \
-                -DYDB_SDK_ENABLE_OTEL_METRICS=ON \
-                -DYDB_SDK_ENABLE_OTEL_TRACE=ON \
-                -DBUILD_SHARED_LIBS=OFF \
-                -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
-                -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
-                -DCMAKE_PREFIX_PATH="/usr/share/yandex"
-              cmake --build build-deb --target package -j$(nproc)
-
-              cp build_googleapis_deb/*.deb /source/artifacts/
-              cp build-deb/*.deb            /source/artifacts/
-            '
+            bash /source/scripts/build_cpack_deb_packages.sh /source/artifacts
 
       - name: Smoke-test generated .deb packages
         shell: bash
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -262,72 +262,51 @@ jobs:
         with:
           submodules: true
 
+      - name: Prepare Debian package cache key
+        id: deb-package-cache-key
+        shell: bash
+        run: |
+          echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT"
+
       - name: Validate dpkg-buildpackage
         shell: bash
         run: |
           ./scripts/test_dpkg_buildpackage.sh
 
+      - name: Restore Debian package build cache
+        id: deb-package-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            .deb-ccache
+            build_googleapis_deb
+            build-deb
+          key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: |
+            ${{ steps.deb-package-cache-key.outputs.prefix }}-
+            ubuntu-24.04-deb-packages-
+
       - name: Build CPack .deb packages
         shell: bash
         run: |
+          mkdir -p .deb-ccache build_googleapis_deb build-deb
           docker run --rm --network host \
+            -e CCACHE_DIR=/source/.deb-ccache \
             -v "$PWD:/source" \
             ubuntu:24.04 \
-            bash -c '
-              set -e
-              export DEBIAN_FRONTEND=noninteractive
-              apt-get update
-              apt-get install -y \
-                build-essential \
-                cmake \
-                pkg-config \
-                git \
-                libidn11-dev \
-                libssl-dev \
-                zlib1g-dev \
-                libprotobuf-dev \
-                protobuf-compiler \
-                libgrpc++-dev \
-                protobuf-compiler-grpc \
-                libbrotli-dev \
-                liblz4-dev \
-                libzstd-dev \
-                libbz2-dev \
-                libxxhash-dev \
-                libsnappy-dev \
-                libdouble-conversion-dev \
-                libgtest-dev \
-                libre2-dev \
-                libc-ares-dev \
-                rapidjson-dev \
-                python3 \
-                python3-six \
-                ragel \
-                yasm
-
-              cd /source
-              cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
-              cmake --build build_googleapis_deb -j$(nproc)
-              cmake --build build_googleapis_deb --target package
-              dpkg -i build_googleapis_deb/*.deb
-
-              ./scripts/generate-debian-directory.sh
-              cmake -S . -B build-deb \
-                -DCMAKE_BUILD_TYPE=Release \
-                -DYDB_SDK_INSTALL=ON \
-                -DYDB_SDK_EXAMPLES=OFF \
-                -DYDB_SDK_TESTS=OFF \
-                -DYDB_SDK_ENABLE_OTEL_METRICS=ON \
-                -DYDB_SDK_ENABLE_OTEL_TRACE=ON \
-                -DBUILD_SHARED_LIBS=OFF \
-                -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
-                -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
-                -DCMAKE_PREFIX_PATH="/usr/share/yandex"
-              cmake --build build-deb --target package -j$(nproc)
-              cp build_googleapis_deb/*.deb build-deb/
-            '
+            bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb
 
       - name: Smoke-test generated .deb packages
         shell: bash
         run: |
           ./scripts/test_deb_packages.sh build-deb
+
+      - name: Save Debian package build cache
+        if: github.event_name != 'pull_request' && success() && steps.deb-package-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            .deb-ccache
+            build_googleapis_deb
+            build-deb
+          key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.github/workflows/warmup_cache.yaml b/.github/workflows/warmup_cache.yaml
@@ -81,3 +81,47 @@ jobs:
       with:
         path: build
         key: ${{ steps.coverage-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+
+  deb-packages:
+    name: Build Debian packages and cache artifacts
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Prepare Debian package cache key
+      id: deb-package-cache-key
+      shell: bash
+      run: |
+        echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT"
+    - name: Restore Debian package build cache
+      id: deb-package-cache
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          .deb-ccache
+          build_googleapis_deb
+          build-deb
+        key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          ${{ steps.deb-package-cache-key.outputs.prefix }}-
+          ubuntu-24.04-deb-packages-
+    - name: Build CPack .deb packages
+      shell: bash
+      run: |
+        mkdir -p .deb-ccache build_googleapis_deb build-deb
+        docker run --rm --network host \
+          -e CCACHE_DIR=/source/.deb-ccache \
+          -v "$PWD:/source" \
+          ubuntu:24.04 \
+          bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb
+    - name: Save Debian package build cache
+      if: steps.deb-package-cache.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          .deb-ccache
+          build_googleapis_deb
+          build-deb
+        key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }}
diff --git a/.gitignore b/.gitignore
@@ -50,6 +50,8 @@ __pycache__/
 /build
 /build_*
 /build-*
+/.deb-ccache/
+/artifacts/
 
 .idea/
 .vscode/
diff --git a/scripts/build_cpack_deb_packages.sh b/scripts/build_cpack_deb_packages.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+set -euo pipefail
+
+SOURCE_DIR="${SOURCE_DIR:-/source}"
+if [ ! -d "$SOURCE_DIR" ]; then
+    SOURCE_DIR=$(dirname "$(realpath "$0")")/..
+    SOURCE_DIR=$(realpath "$SOURCE_DIR")
+fi
+
+OUTPUT_DIR="${1:-build-deb}"
+
+cd "$SOURCE_DIR"
+mkdir -p "$OUTPUT_DIR"
+OUTPUT_DIR=$(realpath "$OUTPUT_DIR")
+
+export DEBIAN_FRONTEND=noninteractive
+export CCACHE_DIR="${CCACHE_DIR:-/root/.ccache}"
+mkdir -p "$CCACHE_DIR"
+
+if [ "${YDB_DEB_INSTALL_DEPS:-1}" = "1" ]; then
+    apt-get update
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        ccache \
+        cmake \
+        pkg-config \
+        git \
+        libidn11-dev \
+        libssl-dev \
+        zlib1g-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        libgrpc++-dev \
+        protobuf-compiler-grpc \
+        libbrotli-dev \
+        liblz4-dev \
+        libzstd-dev \
+        libbz2-dev \
+        libxxhash-dev \
+        libsnappy-dev \
+        libdouble-conversion-dev \
+        libgtest-dev \
+        libre2-dev \
+        libc-ares-dev \
+        rapidjson-dev \
+        python3 \
+        python3-six \
+        ragel \
+        yasm
+fi
+
+touch_existing_sources() {
+    for path in "$@"; do
+        if [ -e "$path" ]; then
+            find "$path" -type f -exec touch {} +
+        fi
+    done
+}
+
+rm -f build_googleapis_deb/*.deb build-deb/*.deb "$OUTPUT_DIR"/*.deb 2>/dev/null || true
+
+if command -v ccache >/dev/null 2>&1; then
+    ccache --zero-stats >/dev/null || true
+    CMAKE_COMPILER_LAUNCHER_ARGS=(
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+    )
+else
+    CMAKE_COMPILER_LAUNCHER_ARGS=()
+fi
+
+# Restored build-cache archives can have outputs newer than the checkout.
+# Touch package inputs so the incremental build cannot hide source changes.
+touch_existing_sources scripts/googleapis_deb third_party/api-common-protos
+cmake -S scripts/googleapis_deb -B build_googleapis_deb \
+    -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
+    "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}"
+cmake --build build_googleapis_deb -j"$(nproc)"
+cmake --build build_googleapis_deb --target package
+dpkg -i build_googleapis_deb/*.deb
+
+./scripts/generate-debian-directory.sh
+
+touch_existing_sources \
+    CMakeLists.txt \
+    cmake \
+    contrib \
+    include \
+    library \
+    plugins \
+    scripts/build_cpack_deb_packages.sh \
+    scripts/generate-debian-directory.sh \
+    src \
+    third_party \
+    tools \
+    util
+
+cmake -S . -B build-deb \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DYDB_SDK_INSTALL=ON \
+    -DYDB_SDK_EXAMPLES=OFF \
+    -DYDB_SDK_TESTS=OFF \
+    -DYDB_SDK_ENABLE_OTEL_METRICS=ON \
+    -DYDB_SDK_ENABLE_OTEL_TRACE=ON \
+    -DBUILD_SHARED_LIBS=OFF \
+    -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
+    -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
+    -DCMAKE_PREFIX_PATH="/usr/share/yandex" \
+    "${CMAKE_COMPILER_LAUNCHER_ARGS[@]}"
+cmake --build build-deb --target package -j"$(nproc)"
+
+cp -f build_googleapis_deb/*.deb "$OUTPUT_DIR"/
+if [ "$(realpath build-deb)" != "$OUTPUT_DIR" ]; then
+    cp -f build-deb/*.deb "$OUTPUT_DIR"/
+fi
+
+if command -v ccache >/dev/null 2>&1; then
+    ccache --show-stats || true
+fi
+
+chmod -R a+rwX "$CCACHE_DIR" build_googleapis_deb build-deb "$OUTPUT_DIR" debian 2>/dev/null || true
+ls -la "$OUTPUT_DIR"
