Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions .github/workflows/ci-restore-isolation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# TEMPORARY: isolates the restore tests to validate the Badger bump without the full integration2
# suite's resource contention (OOM / zero-quorum loss). Runs only the restore packages, repeated,
# so a stable pass/fail signal isn't drowned out by unrelated flaky tests.
# Remove this file before merging PR #9771.
name: ci-restore-isolation

on:
push:
branches:
- chore/update-badger-ristretto
workflow_dispatch:

permissions:
contents: read

jobs:
restore-isolation:
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 90
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod

- name: Make Linux Build and Docker Image
run: make docker-image

- name: Prepare
run: |
export GOPATH=~/go
cp dgraph/dgraph ~/go/bin/dgraph
go clean -testcache

- name: TestIncrementalRestore x8
run: |
export GOPATH=~/go
go test -v -timeout=80m -tags=integration2 -count=8 \
-run '^TestIncrementalRestore$' ./systest/integration2/

- name: TestNamespaceAwareRestoreOnMultipleGroups x4
if: ${{ always() }}
run: |
export GOPATH=~/go
go test -v -timeout=80m -tags=integration2 -count=4 \
-run '^TestNamespaceAwareRestoreOnMultipleGroups$' ./systest/online-restore/namespace-aware/
37 changes: 33 additions & 4 deletions dgraphapi/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -519,14 +519,41 @@ func (hc *HTTPClient) RestoreTenant(c Cluster, backupPath string, backupId strin
return nil
}

// WaitForRestore waits for restore to complete on all alphas
// transientClusterErr reports whether err looks like a temporary cluster condition that a poll
// loop should retry through rather than fail on. Under load a restore can briefly put an alpha into
// draining mode, time out its health endpoint, or drop quorum; none of these mean the restore failed.
func transientClusterErr(err error) bool {
if err == nil {
return false
}
msg := err.Error()
for _, s := range []string{
"the server is in draining mode",
"i/o timeout",
"connection refused",
"connection reset",
"EOF",
"context deadline exceeded",
"no such host",
} {
if strings.Contains(msg, s) {
return true
}
}
return false
}

// WaitForRestore waits for restore to complete on all alphas. A restore under load can transiently
// fail health checks or lose quorum, so poll for a bounded time and retry through transient errors,
// failing only on an unexpected error or if the restore never completes.
func WaitForRestore(c Cluster) error {
const maxWaitAttempts = 300 // waitDurBeforeRetry (1s) apart => ~5m
loop:
for {
for range maxWaitAttempts {
time.Sleep(waitDurBeforeRetry)

resp, err := c.AlphasHealth()
if err != nil && strings.Contains(err.Error(), "the server is in draining mode") {
if transientClusterErr(err) {
continue loop
} else if err != nil {
return err
Expand All @@ -548,7 +575,9 @@ loop2:
time.Sleep(waitDurBeforeRetry)

alphasLogs, err := c.AlphasLogs()
if err != nil {
if transientClusterErr(err) {
continue loop2
} else if err != nil {
return err
}

Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ require (
github.com/Masterminds/semver/v3 v3.4.0
github.com/blevesearch/bleve/v2 v2.5.7
github.com/containerd/errdefs v1.0.0
github.com/dgraph-io/badger/v4 v4.9.2
github.com/dgraph-io/badger/v4 v4.9.4-0.20260708171235-bcef113934c1
github.com/dgraph-io/dgo/v250 v250.0.0
github.com/dgraph-io/gqlgen v0.13.2
github.com/dgraph-io/gqlparser/v2 v2.2.2
github.com/dgraph-io/graphql-transport-ws v0.0.0-20210511143556-2cef522f1f15
github.com/dgraph-io/ristretto/v2 v2.4.0
github.com/dgraph-io/ristretto/v2 v2.4.2
github.com/dgraph-io/simdjson-go v0.3.0
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da
github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.9.2 h1:Wb5qw8gElqwV1a8msHTeQKova9b1V10heFKMIiPd80E=
github.com/dgraph-io/badger/v4 v4.9.2/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
github.com/dgraph-io/badger/v4 v4.9.4-0.20260708171235-bcef113934c1 h1:xAhWII4M4H1ue7iM9xaxe9nEjNhrHnaUHbg4UyVVfuU=
github.com/dgraph-io/badger/v4 v4.9.4-0.20260708171235-bcef113934c1/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI=
github.com/dgraph-io/dgo/v250 v250.0.0 h1:zkVj8EOgNOK3s5XFEK7CJKRdftWqg5K6qGs4HEH5TcY=
github.com/dgraph-io/dgo/v250 v250.0.0/go.mod h1:OVSaapUnuqaY4beLe98CajukINwbVm0JRNp0SRBCz/w=
github.com/dgraph-io/gqlgen v0.13.2 h1:TNhndk+eHKj5qE7BenKKSYdSIdOGhLqxR1rCiMso9KM=
Expand All @@ -135,8 +135,8 @@ github.com/dgraph-io/gqlparser/v2 v2.2.2 h1:CnxXOKL4EPguKqcGV/z4u4VoW5izUkOTIsNM
github.com/dgraph-io/gqlparser/v2 v2.2.2/go.mod h1:MYS4jppjyx8b9tuUtjV7jU1UFZK6P9fvO8TsIsQtRKU=
github.com/dgraph-io/graphql-transport-ws v0.0.0-20210511143556-2cef522f1f15 h1:X2NRsgAtVUAp2nmTPCq+x+wTcRRrj74CEpy7E0Unsl4=
github.com/dgraph-io/graphql-transport-ws v0.0.0-20210511143556-2cef522f1f15/go.mod h1:7z3c/5w0sMYYZF5bHsrh8IH4fKwG5O5Y70cPH1ZLLRQ=
github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU=
github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgraph-io/ristretto/v2 v2.4.2 h1:x0cvjmUKxt764Yxdk2nr94we1AvPPAMh1rh5TQ+Jo80=
github.com/dgraph-io/ristretto/v2 v2.4.2/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E=
github.com/dgraph-io/simdjson-go v0.3.0 h1:h71LO7vR4LHMPUhuoGN8bqGm1VNfGOlAG8BI6iDUKw0=
github.com/dgraph-io/simdjson-go v0.3.0/go.mod h1:Otpysdjaxj9OGaJusn4pgQV7OFh2bELuHANq0I78uvY=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
Expand Down
28 changes: 22 additions & 6 deletions systest/online-restore/namespace-aware/restore_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,26 @@ import (
// return err
// }

// requireEventualJSON polls query until its response matches wantJSON or the deadline passes, then
// asserts. Right after a restore there is a brief window where the queried replica can be one Raft
// apply behind, so a single-shot assert flakes under load. Retrying tolerates that lag without
// masking a real bug: if the data never converges (e.g. a genuine restore data loss), the final
// CompareJSON still fails.
func requireEventualJSON(t *testing.T, gc *dgraphapi.GrpcClient, query, wantJSON string) {
t.Helper()
var last error
for range 60 {
resp, err := gc.Query(query)
if err != nil {
last = err
} else if last = dgraphapi.CompareJSON(wantJSON, string(resp.Json)); last == nil {
return
}
time.Sleep(time.Second)
}
require.NoError(t, last)
}

func commonTest(t *testing.T, existingCluster, freshCluster *dgraphtest.LocalCluster) {
hc, err := existingCluster.HTTPClient()
require.NoError(t, err)
Expand Down Expand Up @@ -79,9 +99,7 @@ func commonTest(t *testing.T, existingCluster, freshCluster *dgraphtest.LocalClu
count(uid)
}
}`
resp, err := gc.Query(query)
require.NoError(t, err)
require.NoError(t, dgraphapi.CompareJSON(fmt.Sprintf(`{"all":[{"count":%v}]}`, 100+ns), string(resp.Json)))
requireEventualJSON(t, gc, query, fmt.Sprintf(`{"all":[{"count":%v}]}`, 100+ns))

// other namespaces should have no data
for _, ns2 := range namespaces[1:] {
Expand Down Expand Up @@ -155,9 +173,7 @@ func commonIncRestoreTest(t *testing.T, existingCluster, freshCluster *dgraphtes
count(uid)
}
}`
resp, err := gc.Query(query)
require.NoError(t, err)
require.NoError(t, dgraphapi.CompareJSON(fmt.Sprintf(`{"all":[{"count":%v}]}`, 20*(j+1)), string(resp.Json)))
requireEventualJSON(t, gc, query, fmt.Sprintf(`{"all":[{"count":%v}]}`, 20*(j+1)))
}
}
}
Expand Down
Loading