Skip to content

Commit 18c0f4f

Browse files
snormoresnormore
committed
client/daemon: route liveness / bfd / e2e testing
1 parent 6fa7295 commit 18c0f4f

File tree

2 files changed

+207
-45
lines changed

2 files changed

+207
-45
lines changed

‎e2e/docker/client/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ FROM ${BASE_IMAGE} AS base
44
FROM ubuntu:24.04
55

66
RUN apt-get update && \
7-
apt-get install -y curl jq iproute2 iputils-ping iproute2 net-tools tcpdump vim iperf fping iptables ethtool
7+
apt-get install -y curl jq iproute2 iputils-ping iproute2 net-tools tcpdump tshark vim iperf fping iptables ethtool
88

99
COPY --from=base /doublezero/bin/doublezero /usr/local/bin/
1010
COPY --from=base /doublezero/bin/doublezerod /usr/local/bin/

‎e2e/multi_client_test.go‎

Lines changed: 206 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
package e2e_test
44

55
import (
6+
"errors"
7+
"fmt"
68
"log/slog"
79
"os"
810
"path/filepath"
@@ -340,52 +342,148 @@ func runMultiClientIBRLWorkflowTest(t *testing.T, log *slog.Logger, dn *devnet.D
340342
require.NoError(t, err)
341343

342344
log.Info("--> Clients can reach each other via their DZ IPs")
345+
// --- Route liveness block matrix ---
346+
log.Info("==> Route liveness: block each client independently and require expected route behavior")
347+
const wait = 120 * time.Second
348+
const tick = 5 * time.Second
349+
350+
doRouteLivenessBaseline := func() {
351+
t.Helper()
352+
// Baseline should already be:
353+
// - c1 has routes to c2,c3
354+
// - c2 has route to c1, NOT to c3
355+
// - c3 has route to c1, NOT to c2
356+
requireEventuallyRoute(t, client1, client2DZIP, true, wait, tick, "baseline c1->c2")
357+
requireEventuallyRoute(t, client1, client3DZIP, true, wait, tick, "baseline c1->c3")
358+
requireEventuallyRoute(t, client2, client1DZIP, true, wait, tick, "baseline c2->c1")
359+
requireEventuallyRoute(t, client2, client3DZIP, false, wait, tick, "baseline c2->c3")
360+
requireEventuallyRoute(t, client3, client1DZIP, true, wait, tick, "baseline c3->c1")
361+
requireEventuallyRoute(t, client3, client2DZIP, false, wait, tick, "baseline c3->c2")
362+
363+
// Baseline liveness packets (dz0 present where peers exist; never on eth0/eth1)
364+
requireUDPLivenessOnDZ0(t, client1, client2DZIP, true, "baseline c1 liveness packets -> c2 on dz0")
365+
requireUDPLivenessOnDZ0(t, client1, client3DZIP, true, "baseline c1 liveness packets -> c3 on dz0")
366+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "baseline c2 liveness packets -> c1 on dz0 (disabled = routing-agnostic)")
367+
requireUDPLivenessOnDZ0(t, client2, client3DZIP, false, "baseline c2 liveness packets -> c3 none")
368+
requireUDPLivenessOnDZ0(t, client3, client1DZIP, true, "baseline c3 liveness packets -> c1 on dz0")
369+
requireUDPLivenessOnDZ0(t, client3, client2DZIP, false, "baseline c3 liveness packets -> c2 none")
370+
requireNoUDPLivenessOnEth01(t, client1, client2DZIP, "baseline no c1 liveness packets on eth0/1 -> c2")
371+
requireNoUDPLivenessOnEth01(t, client1, client3DZIP, "baseline no c1 liveness packets on eth0/1 -> c3")
372+
requireNoUDPLivenessOnEth01(t, client2, client1DZIP, "baseline no c2 liveness packets on eth0/1 -> c1")
373+
requireNoUDPLivenessOnEth01(t, client3, client1DZIP, "baseline no c3 liveness packets on eth0/1 -> c1")
374+
}
343375

344-
// Block UDP traffic to routeLivenessPort to client2 and client3 at the same time and check that client1 routes are dropped, and remain the same otherwise.
345-
log.Info("==> Blocking UDP traffic to route liveness port to client2 and client3 and checking that client1 routes are dropped")
346-
_, err = client2.Exec(t.Context(), []string{"iptables", "-A", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"})
347-
require.NoError(t, err)
348-
_, err = client3.Exec(t.Context(), []string{"iptables", "-A", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"})
349-
require.NoError(t, err)
350-
require.Eventually(t, func() bool {
351-
output, err := client1.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
352-
require.NoError(t, err)
353-
return !strings.Contains(string(output), client2DZIP) && !strings.Contains(string(output), client3DZIP)
354-
}, 120*time.Second, 5*time.Second, "client1 should not have route to client2 and client3")
355-
require.Never(t, func() bool {
356-
output, err := client2.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
357-
require.NoError(t, err)
358-
return strings.Contains(string(output), client3DZIP)
359-
}, 1*time.Second, 500*time.Millisecond, "client2 should not have route to client3")
360-
require.Never(t, func() bool {
361-
output, err := client3.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
362-
require.NoError(t, err)
363-
return strings.Contains(string(output), client2DZIP)
364-
}, 1*time.Second, 500*time.Millisecond, "client3 should not have route to client2")
365-
log.Info("--> client1 routes are dropped when client2 and client3 probes start failing")
376+
doRouteLivenessCaseA := func(pass int) {
377+
t.Helper()
378+
log.Info("==> Route liveness Case A (block client1)", "pass", pass)
379+
blockUDPLiveness(t, client1)
380+
381+
// Routes
382+
requireEventuallyRoute(t, client1, client2DZIP, false, wait, tick, "pass %d: block c1: c1->c2 removed")
383+
requireEventuallyRoute(t, client1, client3DZIP, false, wait, tick, "pass %d: block c1: c1->c3 removed")
384+
requireEventuallyRoute(t, client3, client1DZIP, false, wait, tick, "pass %d: block c1: c3->c1 removed")
385+
requireEventuallyRoute(t, client2, client1DZIP, true, wait, tick, "pass %d: block c1: c2->c1 remains")
386+
requireEventuallyRoute(t, client2, client3DZIP, false, wait, tick, "pass %d: block c1: c2->c3 remains absent")
387+
requireEventuallyRoute(t, client3, client2DZIP, false, wait, tick, "pass %d: block c1: c3->c2 remains absent")
388+
389+
// Liveness packets on doublezero0, none on eth0/1
390+
requireUDPLivenessOnDZ0(t, client1, client2DZIP, true, "pass %d: block c1: no c1 liveness packets -> c2 on dz0")
391+
requireUDPLivenessOnDZ0(t, client1, client3DZIP, true, "pass %d: block c1: no c1 liveness packets -> c3 on dz0")
392+
requireUDPLivenessOnDZ0(t, client3, client1DZIP, true, "pass %d: block c1: no c3 liveness packets -> c1 on dz0")
393+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: block c1: c2 still shows liveness packets -> c1 on dz0")
394+
requireNoUDPLivenessOnEth01(t, client1, client2DZIP, "pass %d: block c1: no c1 liveness packets on eth0/1 -> c2")
395+
requireNoUDPLivenessOnEth01(t, client1, client3DZIP, "pass %d: block c1: no c1 liveness packets on eth0/1 -> c3")
396+
requireNoUDPLivenessOnEth01(t, client3, client1DZIP, "pass %d: block c1: no c3 liveness packets on eth0/1 -> c1")
397+
requireNoUDPLivenessOnEth01(t, client2, client1DZIP, "pass %d: block c1: no c2 liveness packets on eth0/1 -> c1")
398+
399+
unblockUDPLiveness(t, client1)
400+
401+
// Routes restored
402+
requireEventuallyRoute(t, client1, client2DZIP, true, wait, tick, "pass %d: unblock c1: c1->c2 restored")
403+
requireEventuallyRoute(t, client1, client3DZIP, true, wait, tick, "pass %d: unblock c1: c1->c3 restored")
404+
requireEventuallyRoute(t, client3, client1DZIP, true, wait, tick, "pass %d: unblock c1: c3->c1 restored")
405+
406+
// Liveness packets on dz0; none on eth0/1
407+
requireUDPLivenessOnDZ0(t, client1, client2DZIP, true, "pass %d: unblock c1: c1 liveness packets -> c2 on dz0")
408+
requireUDPLivenessOnDZ0(t, client1, client3DZIP, true, "pass %d: unblock c1: c1 liveness packets -> c3 on dz0")
409+
requireUDPLivenessOnDZ0(t, client3, client1DZIP, true, "pass %d: unblock c1: c3 liveness packets -> c1 on dz0")
410+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: unblock c1: c2 liveness packets -> c1 on dz0")
411+
requireNoUDPLivenessOnEth01(t, client1, client2DZIP, "pass %d: unblock c1: none on eth0/1 -> c2")
412+
requireNoUDPLivenessOnEth01(t, client1, client3DZIP, "pass %d: unblock c1: none on eth0/1 -> c3")
413+
}
366414

367-
// Unblock UDP traffic to routeLivenessPort to client2 and client3 at the same time and check that client1 routes are restored, and remain the same otherwise.
368-
log.Info("==> Unblocking UDP traffic to route liveness port to client2 and client3 and checking that client1 routes are restored")
369-
_, err = client2.Exec(t.Context(), []string{"iptables", "-D", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"})
370-
require.NoError(t, err)
371-
_, err = client3.Exec(t.Context(), []string{"iptables", "-D", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"})
372-
require.NoError(t, err)
373-
require.Eventually(t, func() bool {
374-
output, err := client1.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
375-
require.NoError(t, err)
376-
return strings.Contains(string(output), client2DZIP) && strings.Contains(string(output), client3DZIP)
377-
}, 120*time.Second, 5*time.Second, "client1 should have route to client2 and client3")
378-
require.Never(t, func() bool {
379-
output, err := client2.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
380-
require.NoError(t, err)
381-
return strings.Contains(string(output), client3DZIP)
382-
}, 1*time.Second, 100*time.Millisecond, "client2 should not have route to client3")
383-
require.Never(t, func() bool {
384-
output, err := client3.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
385-
require.NoError(t, err)
386-
return strings.Contains(string(output), client2DZIP)
387-
}, 1*time.Second, 100*time.Millisecond, "client3 should not have route to client2")
388-
log.Info("--> client1 routes are restored when client2 and client3 probes start succeeding")
415+
doRouteLivenessCaseB := func(pass int) {
416+
t.Helper()
417+
log.Info("==> Route liveness Case B (block client2)", "pass", pass)
418+
blockUDPLiveness(t, client2)
419+
420+
// Routes
421+
requireEventuallyRoute(t, client1, client2DZIP, false, wait, tick, "pass %d: block c2: c1->c2 removed")
422+
requireEventuallyRoute(t, client2, client1DZIP, true, wait, tick, "pass %d: block c2: c2->c1 remains")
423+
requireEventuallyRoute(t, client2, client3DZIP, false, wait, tick, "pass %d: block c2: c2->c3 remains absent")
424+
requireEventuallyRoute(t, client3, client2DZIP, false, wait, tick, "pass %d: block c2: c3->c2 remains absent")
425+
requireEventuallyRoute(t, client1, client3DZIP, true, wait, tick, "pass %d: block c2: c1->c3 remains")
426+
requireEventuallyRoute(t, client3, client1DZIP, true, wait, tick, "pass %d: block c2: c3->c1 remains")
427+
428+
// Liveness packets
429+
requireUDPLivenessOnDZ0(t, client1, client2DZIP, true, "pass %d: block c2: c1 liveness packets -> c2 on dz0 (route withdrawn)")
430+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: block c2: c2 still shows liveness packets -> c1 on dz0")
431+
requireNoUDPLivenessOnEth01(t, client1, client2DZIP, "pass %d: block c2: no c1 liveness packets on eth0/1 -> c2")
432+
requireNoUDPLivenessOnEth01(t, client2, client1DZIP, "pass %d: block c2: no c2 liveness packets on eth0/1 -> c1")
433+
434+
unblockUDPLiveness(t, client2)
435+
436+
// Routes restored
437+
requireEventuallyRoute(t, client1, client2DZIP, true, wait, tick, "pass %d: unblock c2: c1->c2 restored")
438+
439+
// Liveness packets on dz0; none on eth0/1
440+
requireUDPLivenessOnDZ0(t, client1, client2DZIP, true, "pass %d: unblock c2: c1 liveness packets -> c2 on dz0")
441+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: unblock c2: c2 liveness packets -> c1 on dz0")
442+
requireNoUDPLivenessOnEth01(t, client1, client2DZIP, "pass %d: unblock c2: none on eth0/1 -> c2")
443+
}
444+
445+
doRouteLivenessCaseC := func(pass int) {
446+
t.Helper()
447+
log.Info("==> Route liveness Case C (block client3)", "pass", pass)
448+
blockUDPLiveness(t, client3)
449+
450+
// Routes
451+
requireEventuallyRoute(t, client1, client3DZIP, false, wait, tick, "pass %d: block c3: c1->c3 removed")
452+
requireEventuallyRoute(t, client3, client1DZIP, false, wait, tick, "pass %d: block c3: c3->c1 removed")
453+
requireEventuallyRoute(t, client1, client2DZIP, true, wait, tick, "pass %d: block c3: c1->c2 remains")
454+
requireEventuallyRoute(t, client2, client1DZIP, true, wait, tick, "pass %d: block c3: c2->c1 remains")
455+
requireEventuallyRoute(t, client2, client3DZIP, false, wait, tick, "pass %d: block c3: c2->c3 remains absent")
456+
requireEventuallyRoute(t, client3, client2DZIP, false, wait, tick, "pass %d: block c3: c3->c2 remains absent")
457+
458+
// Liveness packets
459+
requireUDPLivenessOnDZ0(t, client1, client3DZIP, true, "pass %d: block c3: c1 liveness packets -> c3 on dz0")
460+
requireUDPLivenessOnDZ0(t, client3, client1DZIP, true, "pass %d: block c3: c3 liveness packets -> c1 on dz0")
461+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: block c3: c2 still shows liveness packets -> c1 on dz0")
462+
requireNoUDPLivenessOnEth01(t, client1, client3DZIP, "pass %d: block c3: no c1 liveness packets on eth0/1 -> c3")
463+
requireNoUDPLivenessOnEth01(t, client3, client1DZIP, "pass %d: block c3: no c3 liveness packets on eth0/1 -> c1")
464+
requireNoUDPLivenessOnEth01(t, client2, client1DZIP, "pass %d: block c3: no c2 liveness packets on eth0/1 -> c1")
465+
466+
unblockUDPLiveness(t, client3)
467+
468+
// Routes restored
469+
requireEventuallyRoute(t, client1, client3DZIP, true, wait, tick, "pass %d: unblock c3: c1->c3 restored")
470+
requireEventuallyRoute(t, client3, client1DZIP, true, wait, tick, "pass %d: unblock c3: c3->c1 restored")
471+
472+
// Liveness packets on dz0; none on eth0/1
473+
requireUDPLivenessOnDZ0(t, client1, client3DZIP, true, "pass %d: unblock c3: c1 liveness packets -> c3 on dz0")
474+
requireUDPLivenessOnDZ0(t, client3, client1DZIP, true, "pass %d: unblock c3: c3 liveness packets -> c1 on dz0")
475+
requireUDPLivenessOnDZ0(t, client2, client1DZIP, true, "pass %d: unblock c3: c2 liveness packets -> c1 on dz0")
476+
requireNoUDPLivenessOnEth01(t, client1, client3DZIP, "pass %d: unblock c3: none on eth0/1 -> c3")
477+
}
478+
479+
// Run the matrix multiple times to check multiple iterations of the workflow.
480+
doRouteLivenessBaseline()
481+
doRouteLivenessCaseA(1)
482+
doRouteLivenessCaseB(1)
483+
doRouteLivenessCaseC(1)
484+
doRouteLivenessCaseA(2)
485+
486+
log.Info("--> Route liveness block matrix (repeat) complete")
389487

390488
// Disconnect client1.
391489
log.Info("==> Disconnecting client1 from IBRL")
@@ -544,3 +642,67 @@ func runMultiClientIBRLWithAllocatedIPWorkflowTest(t *testing.T, log *slog.Logge
544642
require.Equal(t, devnet.ClientSessionStatusDisconnected, status[0].DoubleZeroStatus.SessionStatus)
545643
log.Info("--> Confirmed clients are disconnected and do not have a DZ IP allocated")
546644
}
645+
646+
func blockUDPLiveness(t *testing.T, c *devnet.Client) {
647+
t.Helper()
648+
cmd := []string{"iptables", "-A", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"}
649+
_, err := c.Exec(t.Context(), cmd)
650+
require.NoError(t, err)
651+
}
652+
653+
func unblockUDPLiveness(t *testing.T, c *devnet.Client) {
654+
t.Helper()
655+
cmd := []string{"iptables", "-D", "INPUT", "-p", "udp", "--dport", strconv.Itoa(routeLivenessPort), "-j", "DROP"}
656+
_, err := c.Exec(t.Context(), cmd)
657+
require.NoError(t, err)
658+
}
659+
660+
func hasRoute(t *testing.T, from *devnet.Client, ip string) bool {
661+
t.Helper()
662+
out, err := from.Exec(t.Context(), []string{"ip", "r", "list", "dev", "doublezero0"})
663+
require.NoError(t, err)
664+
return strings.Contains(string(out), ip)
665+
}
666+
667+
func requireEventuallyRoute(t *testing.T, from *devnet.Client, ip string, want bool, wait, tick time.Duration, msg string) {
668+
t.Helper()
669+
require.Eventually(t, func() bool { return hasRoute(t, from, ip) == want }, wait, tick, msg)
670+
}
671+
672+
func requireUDPLivenessOnDZ0(t *testing.T, c *devnet.Client, host string, want bool, msg string) {
673+
t.Helper()
674+
n, err := udpLivenessCaptureCount(t, c, []string{"doublezero0"}, host)
675+
require.NoError(t, err)
676+
require.Equal(t, want, n > 0, msg)
677+
}
678+
679+
func requireNoUDPLivenessOnEth01(t *testing.T, c *devnet.Client, host string, msg string) {
680+
t.Helper()
681+
n, err := udpLivenessCaptureCount(t, c, []string{"eth0", "eth1"}, host)
682+
require.NoError(t, err)
683+
require.Equal(t, 0, n, msg)
684+
}
685+
686+
func udpLivenessCaptureCount(t *testing.T, c *devnet.Client, ifaces []string, host string) (int, error) {
687+
t.Helper()
688+
var iargs []string
689+
for _, i := range ifaces {
690+
iargs = append(iargs, "-i", i)
691+
}
692+
cmd := fmt.Sprintf(`tshark %s -a duration:1 -Y "not gre && ip.addr==%s && udp.port==%d"`, strings.Join(iargs, " "), host, routeLivenessPort)
693+
args := append([]string{"bash", "-lc"}, cmd)
694+
out, err := c.Exec(t.Context(), args)
695+
require.NoError(t, err)
696+
// Expect a line like: "9 packets captured"
697+
s := string(out)
698+
for line := range strings.SplitSeq(s, "\n") {
699+
if strings.Contains(line, " packets captured") {
700+
idx := strings.LastIndex(line, " packets captured")
701+
numStr := strings.TrimSpace(line[:idx])
702+
n, err := strconv.Atoi(numStr)
703+
require.NoError(t, err)
704+
return n, nil
705+
}
706+
}
707+
return 0, errors.New("no capture count found in output")
708+
}

0 commit comments

Comments
 (0)