Add caching to Dockerfile #1506
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: ci | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*" | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| REGISTRY: ghcr.io | |
| CLICOLOR: 1 | |
| jobs: | |
| test: | |
| needs: [check, fmt, clippy, build-storage, build-storage-lake] | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| pg: | |
| - postgres:16 | |
| - postgres:17 | |
| - postgres:18 | |
| env: | |
| ADVERTISED_LISTENER_URL: tcp://localhost:9092 | |
| AWS_ACCESS_KEY_ID: minioadmin | |
| AWS_ALLOW_HTTP: true | |
| AWS_DEFAULT_REGION: eu-west-2 | |
| AWS_ENDPOINT: http://localhost:9000 | |
| AWS_SECRET_ACCESS_KEY: minioadmin | |
| CARGO_BUILD_BUILD_DIR: /mnt/build | |
| CLUSTER_ID: tansu-ci | |
| DATA_LAKE: s3://lake | |
| ICEBERG_CATALOG: http://localhost:8181/catalog | |
| ICEBERG_WAREHOUSE: tansu | |
| LAKEKEEPER_IMAGE: quay.io/lakekeeper/catalog:v0.8.5 | |
| POSTGRES_IMAGE: ${{matrix.pg}} | |
| PROMETHEUS_LISTENER_URL: tcp://[::]:9100 | |
| RUST_BACKTRACE: 1 | |
| RUST_LOG: debug,tansu_sans_io=warn,tansu_model=warn | |
| SCHEMA_REGISTRY: file://./etc/schema | |
| STORAGE_ENGINE: s3://tansu/ | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu | |
| steps: | |
| - run: sudo mkdir /mnt/build | |
| - run: sudo chmod a+rwxt /mnt/build | |
| - run: echo "PG_IMAGE_DASH=${POSTGRES_IMAGE/:/-}" >> $GITHUB_ENV | |
| - run: echo ${PG_IMAGE_DASH} | |
| - run: df -h | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - run: just ci | |
| - run: sudo apt-get install -y libclang-dev libkrb5-dev | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: cargo install cargo-nextest --locked | |
| - run: df -h | |
| - run: just test | |
| - run: df -h | |
| if: always() | |
| - run: ls -l | |
| if: always() | |
| - run: tar --create --verbose --gzip --file=logs-${{env.PG_IMAGE_DASH}}.tar.gz logs | |
| if: always() | |
| - uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: logs-${{env.PG_IMAGE_DASH}}.tar.gz | |
| path: logs-${{env.PG_IMAGE_DASH}}.tar.gz | |
| if-no-files-found: error | |
| - run: du -sh target | |
| - run: docker system df | |
| - run: df -h | |
| check: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: just check | |
| fmt: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: just fmt | |
| clippy: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: just clippy | |
| build-storage: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| storage: | |
| - dynostore | |
| - libsql | |
| - postgres | |
| - slatedb | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: sudo apt-get install -y libclang-dev libkrb5-dev | |
| - run: cargo build --no-default-features --features ${{matrix.storage}} | |
| build-storage-lake: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| storage: | |
| - dynostore | |
| - libsql | |
| - postgres | |
| - slatedb | |
| lake: | |
| - delta | |
| - iceberg | |
| - parquet | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: clippy, rustfmt | |
| cache: false | |
| - run: sudo apt-get install -y libclang-dev libkrb5-dev | |
| - run: cargo build --features ${{matrix.storage}} --features ${{matrix.lake}} | |
| compat-librdkafka: | |
| runs-on: ${{matrix.os}} | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| storage: | |
| - name: s3 | |
| engine: s3://tansu/ | |
| features: dynostore | |
| - name: postgres | |
| engine: postgres://postgres:postgres@localhost | |
| features: postgres | |
| - name: sqlite | |
| engine: sqlite://tansu.db | |
| features: libsql | |
| env: | |
| ADVERTISED_LISTENER_URL: tcp://localhost:9092 | |
| AWS_ACCESS_KEY_ID: minioadmin | |
| AWS_ALLOW_HTTP: true | |
| AWS_DEFAULT_REGION: eu-west-2 | |
| AWS_ENDPOINT: http://localhost:9000 | |
| AWS_SECRET_ACCESS_KEY: minioadmin | |
| CLUSTER_ID: librdkafka-integration | |
| LAKEKEEPER_IMAGE: unused | |
| RESULTS_FILE: results-${{matrix.storage.name}}.csv | |
| RUST_BACKTRACE: 1 | |
| RUST_LOG: warn | |
| STORAGE_ENGINE: ${{matrix.storage.engine}} | |
| TANSU_IMAGE: hello-world | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - run: sudo apt-get install -y libclang-dev libkrb5-dev | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache: false | |
| - run: just compat-infra-up | |
| - run: just compat-librdkafka '${{matrix.storage.engine}}' '${{matrix.storage.features}}' | |
| - if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: compat-librdkafka-results-${{matrix.storage.name}} | |
| path: ${{env.RESULTS_FILE}} | |
| if-no-files-found: warn | |
| compat-librdkafka-report: | |
| if: always() | |
| needs: [compat-librdkafka] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| pattern: compat-librdkafka-results-* | |
| path: results | |
| merge-multiple: true | |
| - run: | | |
| { | |
| echo "## librdkafka compatibility report card" | |
| echo | |
| echo "| storage engine | test | result |" | |
| echo "|---|---|---|" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| for f in results/results-*.csv; do | |
| [[ -e "${f}" ]] || continue | |
| storage="$(basename "${f}" .csv)" | |
| storage="${storage#results-}" | |
| awk -F, -v s="${storage}" \ | |
| '{printf "| %s | %s | %s |\n", s, $1, ($2 == "PASS" ? "✅ PASS" : "❌ FAIL")}' \ | |
| "${f}" | |
| done | sort >> "${GITHUB_STEP_SUMMARY}" | |
| compat-franz-go: | |
| if: false | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| storage: | |
| - engine: memory:// | |
| features: dynostore | |
| - engine: sqlite://tansu.db | |
| features: libsql | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUST_LOG: warn | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: extractions/setup-just@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache: false | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: stable | |
| - run: just compat-franz-go '${{matrix.storage.engine}}' '${{matrix.storage.features}}' | |
| cargo-publish-dry-run: | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: sudo apt-get install -y libclang-dev libkrb5-dev | |
| - run: cargo publish --dry-run --workspace | |
| typos: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: crate-ci/typos@v1.44.0 | |
| with: | |
| config: typos.toml | |
| src: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - run: | | |
| git archive --format=tar.gz HEAD >${{runner.temp}}/src.tar.gz | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: src | |
| path: ${{runner.temp}}/src.tar.gz | |
| if-no-files-found: error | |
| package: | |
| needs: [test, cargo-publish-dry-run, typos] | |
| runs-on: ${{matrix.os}} | |
| permissions: | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: docker/metadata-action@v6 | |
| id: meta | |
| with: | |
| images: ${{env.REGISTRY}}/${{github.repository}} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - uses: docker/setup-buildx-action@v4 | |
| with: | |
| buildkitd-flags: --debug | |
| - uses: docker/login-action@v4 | |
| with: | |
| registry: ${{env.REGISTRY}} | |
| username: ${{github.actor}} | |
| password: ${{secrets.GITHUB_TOKEN}} | |
| - uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: ${{github.actor == 'shortishly'}} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| smoke: | |
| if: ${{github.actor == 'shortishly'}} | |
| needs: [package] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| storage: | |
| - s3://tansu/ | |
| - postgres://postgres:postgres@db | |
| - sqlite://tansu.db | |
| kafka: | |
| - apache/kafka:3.9.0 | |
| - apache/kafka:3.8.1 | |
| - apache/kafka:3.7.0 | |
| runs-on: ${{matrix.os}} | |
| env: | |
| ADVERTISED_LISTENER_URL: tcp://tansu:9092 | |
| AWS_ACCESS_KEY_ID: minioadmin | |
| AWS_ALLOW_HTTP: true | |
| AWS_DEFAULT_REGION: auto | |
| AWS_ENDPOINT: http://minio:9000 | |
| AWS_SECRET_ACCESS_KEY: minioadmin | |
| CLUSTER_ID: kafka-python-integration | |
| KAFKA_IMAGE: ${{matrix.kafka}} | |
| PROMETHEUS_LISTENER_URL: tcp://localhost:9100 | |
| RUST_BACKTRACE: 1 | |
| RUST_LOG: warn,tansu_broker=debug,tansu_storage=debug,tansu_schema=debug | |
| STORAGE_ENGINE: ${{matrix.storage}} | |
| steps: | |
| - uses: docker/metadata-action@v6 | |
| id: meta | |
| with: | |
| images: ${{env.REGISTRY}}/${{github.repository}} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| - uses: actions/checkout@v6 | |
| with: | |
| repository: tansu-io/example-java | |
| path: integration | |
| - uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| etc/initdb.d/010-schema.sql | |
| sparse-checkout-cone-mode: false | |
| path: initdb | |
| - run: ls -lR | |
| - run: cp -v initdb/etc/initdb.d/010-schema.sql integration/etc/initdb.d | |
| - run: docker compose --ansi never --progress plain up --no-color --quiet-pull --detach | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: docker compose --ansi never exec minio /usr/bin/mc ready local | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: docker compose --ansi never exec minio /usr/bin/mc alias set local http://localhost:9000 ${{ env.AWS_ACCESS_KEY_ID }} ${{ env.AWS_SECRET_ACCESS_KEY }} | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: docker compose --ansi never exec minio /usr/bin/mc mb local/tansu | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: sudo apt-get update | |
| - run: sudo apt-get install -y bats | |
| - run: docker compose exec db /usr/bin/pg_isready --timeout=60 | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: docker compose logs | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| - run: bats --trace --verbose-run tests | |
| working-directory: integration | |
| env: | |
| TANSU_IMAGE: ghcr.io/tansu-io/tansu:${{steps.meta.outputs.version}} | |
| release: | |
| if: ${{github.actor == 'shortishly'}} | |
| needs: [test, src, typos] | |
| permissions: | |
| contents: write | |
| runs-on: ${{matrix.os}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - target: aarch64-unknown-linux-gnu | |
| os: ubuntu-24.04-arm | |
| flags: "" | |
| enabled: true | |
| - target: x86_64-unknown-linux-gnu | |
| os: ubuntu-latest | |
| flags: "" | |
| enabled: true | |
| - target: aarch64-apple-darwin | |
| os: macos-latest | |
| flags: "" | |
| enabled: true | |
| - target: x86_64-apple-darwin | |
| os: macos-latest | |
| flags: "" | |
| enabled: true | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache: false | |
| - run: rustc -vV | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| path: artifacts | |
| merge-multiple: true | |
| - if: ${{matrix.os == 'macos-latest'}} | |
| run: | | |
| rustup target add x86_64-apple-darwin | |
| rustup target add aarch64-apple-darwin | |
| - if: ${{matrix.target == 'aarch64-unknown-linux-gnu'}} | |
| run: | | |
| sudo apt update | |
| sudo apt install -y musl-tools zlib1g-dev libzstd-dev | |
| - if: ${{matrix.target == 'x86_64-unknown-linux-gnu'}} | |
| run: | | |
| sudo apt update | |
| sudo apt install -y musl-tools zlib1g-dev libzstd-dev | |
| - if: ${{matrix.enabled}} | |
| env: | |
| RUSTFLAGS: ${{matrix.flags}} | |
| TARGET: ${{matrix.target}} | |
| run: | | |
| cargo build --release --target ${{matrix.target}} --features delta,dynostore,iceberg,libsql,parquet,postgres | |
| dist=$(mktemp -d -p ${{runner.temp}}) | |
| mkdir "$dist"/bin | |
| cp target/"${TARGET}"/release/tansu "$dist"/bin | |
| cp compose.yaml example.env LICENSE README.md "$dist" | |
| cp -r etc "$dist" | |
| tar --create --verbose --gzip --file=tansu-"${TARGET}".tar.gz -C "$dist" . | |
| - if: ${{matrix.enabled && github.ref_type != 'tag'}} | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: tansu-${{matrix.target}} | |
| path: tansu-${{matrix.target}}.tar.gz | |
| if-no-files-found: error | |
| - if: ${{matrix.enabled && github.ref_type == 'tag'}} | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: tansu-${{matrix.target}}.tar.gz |