Skip to content

Commit 3b14b6c

Browse files
ndyakovCopilot
ndyakov
and
Copilot
authored
feat(e2e): mock maintnotif e2e tests (#3639)
* lazy cluster topology reload * fix discrepancies between options structs * Update osscluster_lazy_reload_test.go Co-authored-by: Copilot <[email protected]> * Update osscluster.go Co-authored-by: Copilot <[email protected]> * wip fault with mock proxy * make lint happy * fix linter issues * faster tests with mocks * linter once again * add complex node test * add ci e2e * use correct redis container * e2e fix --------- Co-authored-by: Copilot <[email protected]>
1 parent 8596fc7 commit 3b14b6c

24 files changed

+4489
-166
lines changed

‎.github/workflows/test-e2e.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: E2E Tests
2+
3+
on:
4+
push:
5+
branches: [master, v9, 'v9.*']
6+
pull_request:
7+
branches: [master, v9, v9.7, v9.8, 'ndyakov/**', 'ofekshenawa/**', 'ce/**']
8+
9+
permissions:
10+
contents: read
11+
12+
jobs:
13+
test-e2e-mock:
14+
name: E2E Tests (Mock Proxy)
15+
runs-on: ubuntu-latest
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
go-version:
20+
- "1.23.x"
21+
- stable
22+
23+
steps:
24+
- name: Checkout code
25+
uses: actions/checkout@v6
26+
27+
- name: Set up Go ${{ matrix.go-version }}
28+
uses: actions/setup-go@v6
29+
with:
30+
go-version: ${{ matrix.go-version }}
31+
32+
- name: Start Docker services for E2E tests
33+
run: make docker.e2e.start
34+
35+
- name: Wait for services to be ready
36+
run: |
37+
echo "Waiting for Redis to be ready..."
38+
timeout 30 bash -c 'until docker exec redis-standalone redis-cli ping 2>/dev/null; do sleep 1; done'
39+
echo "Waiting for cae-resp-proxy to be ready..."
40+
timeout 30 bash -c 'until curl -s http://localhost:18100/stats > /dev/null; do sleep 1; done'
41+
echo "All services are ready!"
42+
43+
- name: Run E2E tests with mock proxy
44+
env:
45+
E2E_SCENARIO_TESTS: "true"
46+
run: |
47+
go test -v ./maintnotifications/e2e/ -timeout 30m -race
48+
continue-on-error: false
49+
50+
- name: Stop Docker services
51+
if: always()
52+
run: make docker.e2e.stop
53+
54+
- name: Show Docker logs on failure
55+
if: failure()
56+
run: |
57+
echo "=== Redis logs ==="
58+
docker logs redis-standalone 2>&1 | tail -100
59+
echo "=== cae-resp-proxy logs ==="
60+
docker logs cae-resp-proxy 2>&1 | tail -100
61+
echo "=== proxy-fault-injector logs ==="
62+
docker logs proxy-fault-injector 2>&1 | tail -100
63+

‎Makefile‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,17 @@ docker.start:
1414
docker.stop:
1515
docker compose --profile all down
1616

17+
docker.e2e.start:
18+
@echo "Starting Redis and cae-resp-proxy for E2E tests..."
19+
docker compose --profile e2e up -d --quiet-pull
20+
@echo "Waiting for services to be ready..."
21+
@sleep 3
22+
@echo "Services ready!"
23+
24+
docker.e2e.stop:
25+
@echo "Stopping E2E services..."
26+
docker compose --profile e2e down
27+
1728
test:
1829
$(MAKE) docker.start
1930
@if [ -z "$(REDIS_VERSION)" ]; then \
@@ -66,7 +77,31 @@ bench:
6677
export REDIS_VERSION=$(REDIS_VERSION) && \
6778
go test ./... -test.run=NONE -test.bench=. -test.benchmem -skip Example
6879

69-
.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt
80+
test.e2e:
81+
@echo "Running E2E tests with auto-start proxy..."
82+
$(MAKE) docker.e2e.start
83+
@echo "Running tests..."
84+
@E2E_SCENARIO_TESTS=true go test -v ./maintnotifications/e2e/ -timeout 30m || ($(MAKE) docker.e2e.stop && exit 1)
85+
$(MAKE) docker.e2e.stop
86+
@echo "E2E tests completed!"
87+
88+
test.e2e.docker:
89+
@echo "Running Docker-compatible E2E tests..."
90+
$(MAKE) docker.e2e.start
91+
@echo "Running unified injector tests..."
92+
@E2E_SCENARIO_TESTS=true go test -v -run "TestUnifiedInjector|TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/ -timeout 10m || ($(MAKE) docker.e2e.stop && exit 1)
93+
$(MAKE) docker.e2e.stop
94+
@echo "Docker E2E tests completed!"
95+
96+
test.e2e.logic:
97+
@echo "Running E2E logic tests (no proxy required)..."
98+
@E2E_SCENARIO_TESTS=true \
99+
REDIS_ENDPOINTS_CONFIG_PATH=/tmp/test_endpoints_verify.json \
100+
FAULT_INJECTION_API_URL=http://localhost:8080 \
101+
go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/
102+
@echo "Logic tests completed!"
103+
104+
.PHONY: all test test.ci test.ci.skip-vectorsets bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop
70105

71106
build:
72107
export RE_CLUSTER=$(RE_CLUSTER) && \

‎docker-compose.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ services:
2121
- sentinel
2222
- all-stack
2323
- all
24+
- e2e
2425

2526
osscluster:
2627
image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.4.0}
@@ -39,6 +40,43 @@ services:
3940
- all-stack
4041
- all
4142

43+
cae-resp-proxy:
44+
image: redislabs/client-resp-proxy:latest
45+
container_name: cae-resp-proxy
46+
environment:
47+
- TARGET_HOST=redis
48+
- TARGET_PORT=6379
49+
- LISTEN_PORT=17000,17001,17002,17003 # 4 proxy nodes: initially show 3, swap in 4th during SMIGRATED
50+
- LISTEN_HOST=0.0.0.0
51+
- API_PORT=3000
52+
- DEFAULT_INTERCEPTORS=cluster,hitless
53+
ports:
54+
- "17000:17000" # Proxy node 1 (host:container)
55+
- "17001:17001" # Proxy node 2 (host:container)
56+
- "17002:17002" # Proxy node 3 (host:container)
57+
- "17003:17003" # Proxy node 4 (host:container) - hidden initially, swapped in during SMIGRATED
58+
- "18100:3000" # HTTP API port (host:container)
59+
depends_on:
60+
- redis
61+
profiles:
62+
- e2e
63+
- all
64+
65+
proxy-fault-injector:
66+
build:
67+
context: .
68+
dockerfile: maintnotifications/e2e/cmd/proxy-fi-server/Dockerfile
69+
container_name: proxy-fault-injector
70+
ports:
71+
- "15000:5000" # Fault injector API port (host:container)
72+
depends_on:
73+
- cae-resp-proxy
74+
environment:
75+
- PROXY_API_URL=http://cae-resp-proxy:3000
76+
profiles:
77+
- e2e
78+
- all
79+
4280
sentinel-cluster:
4381
image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.4.0}
4482
platform: linux/amd64

‎maintnotifications/e2e/README_SCENARIOS.md‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,36 @@ This directory contains comprehensive end-to-end test scenarios for Redis push n
77

88
## Introduction
99

10-
To run those tests you would need a fault injector service, please review the client and feel free to implement your
11-
fault injector of choice. Those tests are tailored for Redis Enterprise, but can be adapted to other Redis distributions where
12-
a fault injector is available.
10+
These tests support two modes:
1311

14-
Once you have fault injector service up and running, you can execute the tests by running the `run-e2e-tests.sh` script.
15-
there are three environment variables that need to be set before running the tests:
12+
### 1. Mock Proxy Mode (Default)
13+
Uses a local Docker-based proxy ([cae-resp-proxy](https://github.com/redis-developer/cae-resp-proxy)) to simulate Redis Enterprise behavior. This mode:
14+
- Runs entirely locally without external dependencies
15+
- Provides fast feedback for development
16+
- Simulates cluster topology changes
17+
- Supports SMIGRATING and SMIGRATED notifications
1618

19+
To run in mock proxy mode:
20+
```bash
21+
make test.e2e
22+
```
23+
24+
### 2. Real Fault Injector Mode
25+
Uses a real Redis Enterprise fault injector service for comprehensive testing. This mode:
26+
- Tests against actual Redis Enterprise clusters
27+
- Validates real-world scenarios
28+
- Requires external fault injector setup
29+
30+
To run with a real fault injector, set these environment variables:
1731
- `REDIS_ENDPOINTS_CONFIG_PATH`: Path to Redis endpoints configuration
1832
- `FAULT_INJECTION_API_URL`: URL of the fault injector server
1933
- `E2E_SCENARIO_TESTS`: Set to `true` to enable scenario tests
2034

35+
Then run:
36+
```bash
37+
./scripts/run-e2e-tests.sh
38+
```
39+
2140
## Test Scenarios Overview
2241

2342
### 1. Basic Push Notifications (`scenario_push_notifications_test.go`)
@@ -44,7 +63,28 @@ there are three environment variables that need to be set before running the tes
4463
- Notification delivery consistency
4564
- Handoff behavior per endpoint type
4665

47-
### 3. Database Management Scenario (`scenario_database_management_test.go`)
66+
### 3. Unified Injector Scenarios (`scenario_unified_injector_test.go`)
67+
**Mock proxy-based notification testing**
68+
- **Purpose**: Test SMIGRATING and SMIGRATED notifications with simulated cluster topology changes
69+
- **Features Tested**:
70+
- SMIGRATING notifications (slot migration in progress)
71+
- SMIGRATED notifications (slot migration completed)
72+
- Cluster topology changes (node swap simulation)
73+
- Complex multi-step migration scenarios
74+
- **Configuration**: Uses local Docker proxy (cae-resp-proxy) with 4 nodes
75+
- **Duration**: ~10 seconds
76+
- **Key Validations**:
77+
- Notification delivery and parsing
78+
- Cluster state reload callbacks
79+
- Client resilience during migrations
80+
- Topology change handling
81+
- **Topology Simulation**:
82+
- Starts with 4 proxy nodes (17000-17003)
83+
- Initially exposes 3 nodes in CLUSTER SLOTS (17000, 17001, 17002)
84+
- On SMIGRATED, swaps node 2 for node 3 (simulates node replacement)
85+
- Verifies client continues to function after topology change
86+
87+
### 4. Database Management Scenario (`scenario_database_management_test.go`)
4888
**Dynamic database creation and deletion**
4989
- **Purpose**: Test database lifecycle management via fault injector
5090
- **Features Tested**: CREATE_DATABASE, DELETE_DATABASE endpoints

‎maintnotifications/e2e/cmd/proxy-fi-server/Dockerfile‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Build stage
2+
FROM golang:1.21-alpine AS builder
3+
4+
WORKDIR /build
5+
6+
# Copy go mod files
7+
COPY go.mod go.sum ./
8+
RUN go mod download
9+
10+
# Copy source code
11+
COPY . .
12+
13+
# Build the proxy-fi-server binary
14+
RUN cd maintnotifications/e2e/cmd/proxy-fi-server && \
15+
CGO_ENABLED=0 GOOS=linux go build -o /proxy-fi-server .
16+
17+
# Runtime stage
18+
FROM alpine:latest
19+
20+
RUN apk --no-cache add ca-certificates
21+
22+
# Create a non-root user
23+
RUN addgroup -g 1000 appuser && \
24+
adduser -D -u 1000 -G appuser appuser
25+
26+
WORKDIR /app
27+
28+
# Copy the binary from builder
29+
COPY --from=builder /proxy-fi-server .
30+
31+
# Change ownership of the app directory
32+
RUN chown -R appuser:appuser /app
33+
34+
# Switch to non-root user
35+
USER appuser
36+
37+
# Expose the fault injector API port
38+
EXPOSE 5000
39+
40+
# Run the server
41+
ENTRYPOINT ["/app/proxy-fi-server"]
42+
CMD ["--listen", "0.0.0.0:5000", "--proxy-api-url", "http://cae-resp-proxy:3000"]
43+

‎maintnotifications/e2e/cmd/proxy-fi-server/main.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"os"
7+
"os/signal"
8+
"syscall"
9+
10+
e2e "github.com/redis/go-redis/v9/maintnotifications/e2e"
11+
)
12+
13+
func main() {
14+
listenAddr := flag.String("listen", "0.0.0.0:5000", "Address to listen on for fault injector API")
15+
proxyAPIURL := flag.String("proxy-api-url", "http://localhost:18100", "URL of the cae-resp-proxy API (updated to avoid macOS Control Center conflict)")
16+
flag.Parse()
17+
18+
fmt.Printf("Starting Proxy Fault Injector Server...\n")
19+
fmt.Printf(" Listen address: %s\n", *listenAddr)
20+
fmt.Printf(" Proxy API URL: %s\n", *proxyAPIURL)
21+
22+
server := e2e.NewProxyFaultInjectorServerWithURL(*listenAddr, *proxyAPIURL)
23+
if err := server.Start(); err != nil {
24+
fmt.Fprintf(os.Stderr, "Failed to start server: %v\n", err)
25+
os.Exit(1)
26+
}
27+
28+
fmt.Printf("Proxy Fault Injector Server started successfully\n")
29+
fmt.Printf("Fault Injector API available at http://%s\n", *listenAddr)
30+
31+
// Wait for interrupt signal
32+
sigChan := make(chan os.Signal, 1)
33+
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
34+
<-sigChan
35+
36+
fmt.Println("\nShutting down...")
37+
if err := server.Stop(); err != nil {
38+
fmt.Fprintf(os.Stderr, "Error during shutdown: %v\n", err)
39+
os.Exit(1)
40+
}
41+
fmt.Println("Server stopped")
42+
}
43+

0 commit comments

Comments
 (0)