update caching and ci behaviour #735
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| unit: | |
| concurrency: | |
| group: unit-${{ github.ref }}-ubuntu-22.04-${{ matrix.compiler }} | |
| cancel-in-progress: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compiler: [clang, gcc] | |
| env: | |
| OS: ubuntu-22.04 | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| shell: bash | |
| run: | | |
| echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" | |
| - name: Restore cache files | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.ccache | |
| key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} | |
| restore-keys: | | |
| ubuntu-22.04-ccache-${{ matrix.compiler }}- | |
| - name: Install dependencies | |
| uses: ./.github/actions/prepare_vm | |
| - name: Build | |
| uses: ./.github/actions/build | |
| with: | |
| compiler: ${{ matrix.compiler }} | |
| - name: Test | |
| shell: bash | |
| run: | | |
| ctest -j$(nproc) --preset unit | |
| integration-build: | |
| concurrency: | |
| group: integration-build-${{ github.ref }}-${{ matrix.compiler }} | |
| cancel-in-progress: true | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compiler: [clang, gcc] | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| shell: bash | |
| run: | | |
| echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" | |
| - name: Restore cache files | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.ccache | |
| key: ubuntu-22.04-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} | |
| restore-keys: | | |
| ubuntu-22.04-ccache-${{ matrix.compiler }}- | |
| - name: Install dependencies | |
| uses: ./.github/actions/prepare_vm | |
| - name: Build | |
| uses: ./.github/actions/build | |
| with: | |
| compiler: ${{ matrix.compiler }} | |
| - name: Package integration build | |
| shell: bash | |
| run: | | |
| tar -C build -czf "integration-build-${{ matrix.compiler }}.tar.gz" . | |
| tar -C "$HOME" -czf "integration-deps-${{ matrix.compiler }}.tar.gz" ydb_deps | |
| - name: Upload integration build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-${{ matrix.compiler }} | |
| path: | | |
| integration-build-${{ matrix.compiler }}.tar.gz | |
| integration-deps-${{ matrix.compiler }}.tar.gz | |
| retention-days: 3 | |
| integration: | |
| needs: integration-build | |
| concurrency: | |
| group: integration-${{ github.ref }}-${{ matrix.ydb-version }}-${{ matrix.compiler }} | |
| cancel-in-progress: true | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ydb-version: [24.1, 24.2, 24.3, 24.4, 25.1, trunk] | |
| compiler: [clang, gcc] | |
| services: | |
| ydb: | |
| image: ydbplatform/local-ydb:${{ matrix.ydb-version }} | |
| ports: | |
| - 2135:2135 | |
| - 2136:2136 | |
| - 8765:8765 | |
| volumes: | |
| - /tmp/ydb_certs:/ydb_certs | |
| env: | |
| YDB_LOCAL_SURVIVE_RESTART: true | |
| YDB_USE_IN_MEMORY_PDISKS: true | |
| YDB_TABLE_ENABLE_PREPARED_DDL: true | |
| options: '-h localhost' | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Install CMake | |
| uses: lukka/get-cmake@v3.27.7 | |
| - name: Install runtime dependencies | |
| shell: bash | |
| run: | | |
| sudo add-apt-repository ppa:ubuntu-toolchain-r/test | |
| sudo apt-get -y update | |
| sudo apt-get -y install git gdb ninja-build libidn11-dev ragel yasm libc-ares-dev libre2-dev \ | |
| rapidjson-dev zlib1g-dev libxxhash-dev libzstd-dev libsnappy-dev libgtest-dev libgmock-dev \ | |
| libbz2-dev liblz4-dev libdouble-conversion-dev libssl-dev libstdc++-13-dev gcc-13 g++-13 | |
| - name: Download integration build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: integration-${{ matrix.compiler }} | |
| path: . | |
| - name: Unpack integration build | |
| shell: bash | |
| run: | | |
| mkdir -p build | |
| tar -C build -xzf "integration-build-${{ matrix.compiler }}.tar.gz" | |
| tar -C "$HOME" -xzf "integration-deps-${{ matrix.compiler }}.tar.gz" | |
| - name: Test | |
| shell: bash | |
| run: | | |
| IAM_REGEX='^(DriverAuth|TMetadataFixture|TJwtIamFixture|TOAuthIamFixture|OAuth_WithFacility)\.' | |
| YDB_VERSION=${{ matrix.ydb-version }} ctest -j2 --preset integration \ | |
| -E "${IAM_REGEX}" --output-on-failure | |
| case '${{ matrix.ydb-version }}' in | |
| 25.1|trunk) | |
| ./.github/scripts/run_iam_integration_tests.sh | |
| ;; | |
| esac | |
| test-install: | |
| name: "Test CMake Install" | |
| concurrency: | |
| group: test-install-${{ github.ref }}-${{ matrix.compiler }} | |
| cancel-in-progress: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compiler: [clang, gcc] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Install dependencies | |
| uses: ./.github/actions/prepare_vm | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| shell: bash | |
| run: | | |
| echo "timestamp=$(date -u +%Y-%m-%d-%H-%M-%S)" >> "$GITHUB_OUTPUT" | |
| - name: Restore cache files | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ~/.ccache | |
| key: ${{ runner.os }}-ccache-${{ matrix.compiler }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }} | |
| restore-keys: | | |
| ${{ runner.os }}-ccache-${{ matrix.compiler }}- | |
| - name: Build and Install SDK | |
| shell: bash | |
| run: | | |
| mkdir build | |
| cmake --preset release-${{ matrix.compiler }} -D YDB_SDK_INSTALL=On | |
| cmake --build --preset default | |
| cmake --install build --prefix ./_install | |
| - name: Verify Installation | |
| shell: bash | |
| run: | | |
| echo "Installation directory content for ${{ matrix.compiler }}:" | |
| ls -R ./_install | |
| if [ -z "$(ls -A ./_install)" ]; then | |
| echo "Error: Installation directory is empty!" | |
| exit 1 | |
| fi | |
| deb-packages: | |
| name: "Test Debian Packages" | |
| concurrency: | |
| group: deb-packages-${{ github.ref }} | |
| cancel-in-progress: true | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout PR | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha != '' | |
| with: | |
| submodules: true | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| if: github.event.pull_request.head.sha == '' | |
| with: | |
| submodules: true | |
| - name: Prepare Debian package cache key | |
| id: deb-package-cache-key | |
| shell: bash | |
| run: | | |
| echo "prefix=ubuntu-24.04-deb-packages-${{ hashFiles('CMakeLists.txt', 'cmake/**', 'contrib/**', 'include/**', 'library/**', 'plugins/**', 'scripts/build_cpack_deb_packages.sh', 'scripts/generate-debian-directory.sh', 'scripts/googleapis_deb/**', 'src/**', 'third_party/api-common-protos/**', 'tools/**', 'util/**') }}" >> "$GITHUB_OUTPUT" | |
| - name: Validate dpkg-buildpackage | |
| shell: bash | |
| run: | | |
| ./scripts/test_dpkg_buildpackage.sh | |
| - name: Restore Debian package build cache | |
| id: deb-package-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| .deb-ccache | |
| build_googleapis_deb | |
| build-deb | |
| key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| restore-keys: | | |
| ${{ steps.deb-package-cache-key.outputs.prefix }}- | |
| ubuntu-24.04-deb-packages- | |
| - name: Build CPack .deb packages | |
| shell: bash | |
| run: | | |
| mkdir -p .deb-ccache build_googleapis_deb build-deb | |
| docker run --rm --network host \ | |
| -e CCACHE_DIR=/source/.deb-ccache \ | |
| -v "$PWD:/source" \ | |
| ubuntu:24.04 \ | |
| bash /source/scripts/build_cpack_deb_packages.sh /source/build-deb | |
| - name: Smoke-test generated .deb packages | |
| shell: bash | |
| run: | | |
| ./scripts/test_deb_packages.sh build-deb | |
| - name: Save Debian package build cache | |
| if: github.event_name != 'pull_request' && success() && steps.deb-package-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| .deb-ccache | |
| build_googleapis_deb | |
| build-deb | |
| key: ${{ steps.deb-package-cache-key.outputs.prefix }}-${{ github.run_id }}-${{ github.run_attempt }} |