refactor!: remove experimental features from release builds

.github/workflows/README.md

@@ -228,6 +228,17 @@ docker run --rm -e TEST_LWD_INTEGRATION=1 zebra-tests
 - Network sync (`TEST_ZEBRA_EMPTY_SYNC`, `TEST_UPDATE_SYNC`)
 - State management (`TEST_DISK_REBUILD`)
 
+### Feature Testing Strategy
+
+Our CI/CD pipeline includes comprehensive testing of all Zebra features:
+
+#### Standard Features
+
+- Tested in regular unit and integration tests
+- Included in official releases
+- Part of the default test suite
+- Can be enabled via `FEATURES` build argument
+
 ### Pull Request Testing
 
 #### Continuous Validation

.github/workflows/ci-unit-tests-os.yml

@@ -73,7 +73,7 @@ jobs:
   ### Build and test Zebra on all OSes ###
   ########################################
   test:
-    name: Test ${{ matrix.rust }} on ${{ matrix.os }}${{ matrix.features }}
+    name: Test ${{ matrix.rust }} on ${{ matrix.os }}
     # The large timeout is to accommodate:
     # - macOS and Windows builds (typically 50-90 minutes), and
     timeout-minutes: 120
@@ -83,11 +83,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         rust: [stable, beta]
-        # TODO: When vars.EXPERIMENTAL_FEATURES has features in it, add it here.
-        #       Or work out a way to trim the space from the variable: GitHub doesn't allow empty variables.
-        #       Or use `default` for the empty feature set and EXPERIMENTAL_FEATURES, and update the branch protection rules.
-        #features: ${{ fromJSON(format('["", "{0}"]', vars.EXPERIMENTAL_FEATURES)) }}
-        features: [""]
+        # We only test with default features in this workflow
+        # Other feature combinations are tested in specific workflows
+        features: ["default-release-binaries"]
         exclude:
         # We're excluding macOS beta for the following reasons:
         # - the concurrent macOS runner limit is much lower than the Linux limit
@@ -121,8 +119,7 @@ jobs:
         #with:
         #  workspaces: ". -> C:\\zebra-target"
         with:
-          # Split the experimental features cache from the regular cache, to avoid linker errors.
-          # (These might be "disk full" errors, or they might be dependency resolution issues.)
+          # Split the cache by feature set to avoid linker errors
           key: ${{ matrix.features }}
 
       - name: Change target output directory on Windows

.github/workflows/release-binaries.yml

@@ -15,30 +15,9 @@ on:
       - released
 
 jobs:
-  # Each time this workflow is executed, a build will be triggered to create a new image
-  # with the corresponding tags using information from git
-
-  # The image will be named `zebra:<semver>-experimental`
-  build-experimental:
-    name: Build Experimental Features Release Docker
-    uses: ./.github/workflows/sub-build-docker-image.yml
-    with:
-      dockerfile_path: ./docker/Dockerfile
-      dockerfile_target: runtime
-      image_name: zebra
-      tag_suffix: -experimental
-      features: ${{ format('{0} {1}', vars.RUST_PROD_FEATURES, vars.RUST_EXPERIMENTAL_FEATURES) }}
-      rust_log: ${{ vars.RUST_LOG }}
-    # This step needs access to Docker Hub secrets to run successfully
-    secrets: inherit
-
   # The image will be named `zebra:<semver>`
-  # It should be built last, so tags with the same name point to the production build, not the experimental build.
   build:
     name: Build Release Docker
-    # Run this build last, regardless of whether experimental worked
-    needs: build-experimental
-    if: always()
     uses: ./.github/workflows/sub-build-docker-image.yml
     with:
       dockerfile_path: ./docker/Dockerfile
@@ -53,7 +32,7 @@ jobs:
   failure-issue:
     name: Open or update issues for release binaries failures
     # When a new job is added to this workflow, add it to this list.
-    needs: [ build, build-experimental ]
+    needs: [ build ]
     # Open tickets for any failed build in this workflow.
     if: failure() || cancelled()
     runs-on: ubuntu-latest

.github/workflows/sub-ci-unit-tests-docker.yml

@@ -68,9 +68,10 @@ jobs:
       - name: Run zebrad tests with experimental features
         env:
           NETWORK: ${{ inputs.network || vars.ZCASH_NETWORK }}
+          EXPERIMENTAL_FEATURES: ${{ vars.RUST_EXPERIMENTAL_FEATURES }}
         run: |
           docker pull ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
-          docker run --tty -e NETWORK -e RUN_ALL_EXPERIMENTAL_TESTS=1 ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
+          docker run --tty -e NETWORK -e RUN_ALL_EXPERIMENTAL_TESTS=1 -e EXPERIMENTAL_FEATURES ${{ vars.GAR_BASE }}/${{ vars.CI_IMAGE_NAME }}@${{ inputs.image_digest }}
 
   # Run state tests with fake activation heights.
   #

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to Zebra are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org).
 
+## [Unreleased]
+
+### Breaking Changes
+
+- Remove experimental features from release builds and workflows
+
+
 ## [Zebra 2.2.0](https://github.com/ZcashFoundation/zebra/releases/tag/v2.2.0) - 2025-02-03
 
 In this release, Zebra introduced an additional consensus check on the branch ID of Nu6 transactions

book/src/user/docker.md

@@ -50,6 +50,19 @@ See [Building Zebra](https://github.com/ZcashFoundation/zebra#building-zebra) fo
 
 You're able to specify various parameters when building or launching the Docker image, which are meant to be used by developers and CI pipelines. For example, specifying the Network where Zebra will run (Mainnet, Testnet, etc), or enabling features like metrics with Prometheus.
 
+### Building with Custom Features
+
+Zebra supports various features that can be enabled during build time using the `FEATURES` build argument:
+
+```shell
+# Build with specific features
+docker build -f ./docker/Dockerfile --target runtime \
+    --build-arg FEATURES="default-release-binaries prometheus" \
+    --tag zebra:custom .
+```
+
+### Building with Metrics Support
+
 For example, if we'd like to enable metrics on the image, we'd build it using the following `build-arg`:
 
 > [!IMPORTANT]

docker/Dockerfile

@@ -18,7 +18,6 @@
 # https://github.com/ZcashFoundation/zebra/settings/variables/actions
 ARG FEATURES="default-release-binaries"
 ARG TEST_FEATURES="lightwalletd-grpc-tests zebra-checkpoints"
-ARG EXPERIMENTAL_FEATURES=""
 
 ARG APP_HOME="/opt/zebrad"
 ARG RUST_VERSION=1.82.0
@@ -84,9 +83,7 @@ ENV ZEBRA_SKIP_IPV6_TESTS=${ZEBRA_SKIP_IPV6_TESTS:-1}
 # separately from the test and production image builds.
 ARG FEATURES
 ARG TEST_FEATURES
-ARG EXPERIMENTAL_FEATURES
-# TODO: add empty $EXPERIMENTAL_FEATURES when we can avoid adding an extra space to the end of the string
-ARG ENTRYPOINT_FEATURES="${FEATURES} ${TEST_FEATURES}"
+ENV ENTRYPOINT_FEATURES="${FEATURES} ${TEST_FEATURES}"
 
 # Build Zebra test binaries, but don't run them
 
@@ -126,9 +123,6 @@ COPY ./ ./
 
 # Entrypoint environment variables
 ENV ENTRYPOINT_FEATURES=${ENTRYPOINT_FEATURES}
-# We repeat the ARGs here, so they are available in the entrypoint.sh script for $RUN_ALL_EXPERIMENTAL_TESTS
-ARG EXPERIMENTAL_FEATURES="journald prometheus filter-reload"
-ENV ENTRYPOINT_FEATURES_EXPERIMENTAL="${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}"
 
 # By default, runs the entrypoint tests specified by the environmental variables (if any are set)
 ENTRYPOINT [ "/etc/zebrad/entrypoint.sh" ]

docker/entrypoint.sh

@@ -229,7 +229,7 @@ case "$1" in
       elif [[ "${RUN_ALL_EXPERIMENTAL_TESTS}" -eq "1" ]]; then
         # Run unit, basic acceptance tests, and ignored tests with experimental features.
         # If the lightwalletd environmental variables are set, we will also run those tests.
-        exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES_EXPERIMENTAL}" --workspace -- --nocapture --include-ignored --skip check_no_git_refs_in_cargo_lock
+        exec cargo test --locked --release --features "${ENTRYPOINT_FEATURES} ${EXPERIMENTAL_FEATURES}" --workspace -- --nocapture --include-ignored --skip check_no_git_refs_in_cargo_lock
 
       elif [[ "${RUN_CHECK_NO_GIT_REFS}" -eq "1" ]]; then
         # Run the check_no_git_refs_in_cargo_lock test.