Merge branch 'main' into utils-unit-test2

Author: Effi-S

cmd/config/cobra_test_exports.go

@@ -49,7 +49,7 @@ func StartDefaultSystem(t *testing.T) SystemConfig {
 	_, coordinator := mock.StartMockCoordinatorService(t)
 	conn := dbtest.PrepareTestEnv(t)
 	server := connection.NewLocalHostServerWithTLS(test.InsecureTLSConfig)
-	listen, err := server.Listener()
+	listen, err := server.Listener(t.Context())
 	require.NoError(t, err)
 	connection.CloseConnectionsLog(listen)
 	return SystemConfig{

go.mod

@@ -9,7 +9,7 @@ go 1.24.3
 require (
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cockroachdb/errors v1.11.3
-	github.com/consensys/gnark-crypto v0.14.0
+	github.com/consensys/gnark-crypto v0.18.1
 	github.com/docker/docker v28.0.0+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/fsouza/go-dockerclient v1.12.0
@@ -46,19 +46,18 @@ require (
 	github.com/IBM/idemix/bccsp/schemes/aries v0.0.0-20240913182345-72941a5f41cd // indirect
 	github.com/IBM/idemix/bccsp/schemes/weak-bb v0.0.0-20240913182345-72941a5f41cd // indirect
 	github.com/IBM/idemix/bccsp/types v0.0.0-20240913182345-72941a5f41cd // indirect
-	github.com/IBM/mathlib v0.0.3-0.20231011094432-44ee0eb539da // indirect
+	github.com/IBM/mathlib v0.0.3-0.20250709075152-a138079496c3 // indirect
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.14.3 // indirect
+	github.com/bits-and-blooms/bitset v1.20.0 // indirect
 	github.com/bytedance/sonic v1.12.3 // indirect
 	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/consensys/bavard v0.1.22 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
@@ -99,7 +98,6 @@ require (
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
-	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.6.0 // indirect
@@ -149,5 +147,4 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/v3 v3.5.1 // indirect
-	rsc.io/tmplfunc v0.0.3 // indirect
 )

go.sum

@@ -609,8 +609,8 @@ github.com/IBM/idemix/bccsp/schemes/weak-bb v0.0.0-20240913182345-72941a5f41cd h
 github.com/IBM/idemix/bccsp/schemes/weak-bb v0.0.0-20240913182345-72941a5f41cd/go.mod h1:OlxSkCgdOahDJzm8uyw45a9R7rehCC+0YGyh3gf0b40=
 github.com/IBM/idemix/bccsp/types v0.0.0-20240913182345-72941a5f41cd h1:KQzFMUQtrwJHKwwijxnQ8Lj2D7A+RnSll4Ugv20H3vQ=
 github.com/IBM/idemix/bccsp/types v0.0.0-20240913182345-72941a5f41cd/go.mod h1:4bYvi+a50aXxmHf1vwuvR+Wd8YXZ6AhT+0p5oK4xdOA=
-github.com/IBM/mathlib v0.0.3-0.20231011094432-44ee0eb539da h1:qqGozq4tF6EOVnWoTgBoJGudRKKZXSAYnEtDggzTnsw=
-github.com/IBM/mathlib v0.0.3-0.20231011094432-44ee0eb539da/go.mod h1:Tco9QzE3fQzjMS7nPbHDeFfydAzctStf1Pa8hsh6Hjs=
+github.com/IBM/mathlib v0.0.3-0.20250709075152-a138079496c3 h1:TelnQIceKrhWVmuFnMXyKyq0WUG5zMT6u+7wnRMkcFY=
+github.com/IBM/mathlib v0.0.3-0.20250709075152-a138079496c3/go.mod h1:O230ebw6/22B7T4C03b99ZcPtc5XAfBTOp+ZT+xmMCk=
 github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
@@ -629,8 +629,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA=
-github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU=
+github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ=
@@ -681,10 +681,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
-github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A=
-github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs=
-github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E=
-github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0=
+github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI=
+github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -877,7 +875,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
 github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -1044,9 +1041,6 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
-github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
-github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
@@ -2037,5 +2031,3 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
-rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=

integration/runner/ports.go

@@ -35,7 +35,7 @@ func (p *portAllocator) allocatePorts(t *testing.T, count int) []config.ServiceE
 func (p *portAllocator) allocate(t *testing.T) *connection.Endpoint {
 	t.Helper()
 	s := connection.NewLocalHostServerWithTLS(test.InsecureTLSConfig)
-	listener, err := s.Listener()
+	listener, err := s.Listener(t.Context())
 	require.NoError(t, err)
 	p.listeners = append(p.listeners, listener)
 	return &s.Endpoint

utils/connection/client_util_test.go

@@ -139,7 +139,7 @@ func TestGRPCRetryMultiEndpoints(t *testing.T) {
 
 	t.Log("Creating fake service address")
 	fakeServerConfig := connection.NewLocalHostServerWithTLS(test.InsecureTLSConfig)
-	l, err := fakeServerConfig.Listener()
+	l, err := fakeServerConfig.Listener(t.Context())
 	require.NoError(t, err)
 	t.Cleanup(func() {
 		connection.CloseConnectionsLog(l)

utils/connection/server_util.go

@@ -34,6 +34,12 @@ type (
 	}
 )
 
+var listenRetry = RetryProfile{
+	InitialInterval: 50 * time.Millisecond,
+	MaxInterval:     500 * time.Millisecond,
+	MaxElapsedTime:  2 * time.Minute,
+}
+
 // NewLocalHostServerWithTLS returns a default server config with endpoint "localhost:0" given server credentials.
 func NewLocalHostServerWithTLS(creds TLSConfig) *ServerConfig {
 	return &ServerConfig{
@@ -70,11 +76,23 @@ func (c *ServerConfig) GrpcServer() (*grpc.Server, error) {
 }
 
 // Listener instantiate a [net.Listener] and updates the config port with the effective port.
-func (c *ServerConfig) Listener() (net.Listener, error) {
+// If the port is predefined, it will retry to bind to the port until successful or until the context ends.
+func (c *ServerConfig) Listener(ctx context.Context) (net.Listener, error) {
 	if c.preAllocatedListener != nil {
 		return c.preAllocatedListener, nil
 	}
-	listener, err := net.Listen(grpcProtocol, c.Endpoint.Address())
+
+	var err error
+	var listener net.Listener
+	if c.Endpoint.Port == 0 {
+		listener, err = net.Listen(grpcProtocol, c.Endpoint.Address())
+	} else {
+		err = listenRetry.Execute(ctx, func() error {
+			var listenErr error
+			listener, listenErr = net.Listen(grpcProtocol, c.Endpoint.Address())
+			return listenErr
+		})
+	}
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to listen")
 	}
@@ -93,7 +111,7 @@ func (c *ServerConfig) Listener() (net.Listener, error) {
 // PreAllocateListener is used to allocate a port and bind to ahead of the server initialization.
 // It stores the listener object internally to be reused on subsequent calls to Listener().
 func (c *ServerConfig) PreAllocateListener() (net.Listener, error) {
-	listener, err := c.Listener()
+	listener, err := c.Listener(context.Background())
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +125,7 @@ func RunGrpcServer(
 	serverConfig *ServerConfig,
 	register func(server *grpc.Server),
 ) error {
-	listener, err := serverConfig.Listener()
+	listener, err := serverConfig.Listener(ctx)
 	if err != nil {
 		return err
 	}

utils/deliver/client_test.go

@@ -8,6 +8,7 @@ package deliver
 
 import (
 	"context"
+	"net"
 	"strings"
 	"testing"
 	"time"
@@ -50,7 +51,7 @@ func TestBroadcastDeliver(t *testing.T) {
 	require.NoError(t, err)
 	t.Cleanup(client.CloseConnections)
 
-	ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
 	t.Cleanup(cancel)
 
 	outputBlocksChan := make(chan *common.Block, 100)
@@ -77,20 +78,21 @@ func TestBroadcastDeliver(t *testing.T) {
 
 	t.Log("One server down")
 	servers[2].Servers[0].Stop()
-	waitUntilGrpcServerIsDown(ctx, t, &servers[2].Configs[0].Endpoint)
+	listener1 := holdPort(ctx, t, servers[2].Configs[0])
 	submit(t, &conf, outputBlocks, expectedSubmit{
 		success: 3,
 	})
 
 	t.Log("Two servers down")
 	servers[2].Servers[1].Stop()
-	waitUntilGrpcServerIsDown(ctx, t, &servers[2].Configs[1].Endpoint)
+	listener2 := holdPort(ctx, t, servers[2].Configs[1])
 	submit(t, &conf, outputBlocks, expectedSubmit{
 		success:     2,
 		unavailable: 1,
 	})
 
 	t.Log("One incorrect server")
+	_ = listener1.Close()
 	fakeServer := test.RunGrpcServerForTest(ctx, t, servers[2].Configs[0], nil)
 	waitUntilGrpcServerIsReady(ctx, t, &servers[2].Configs[0].Endpoint)
 	submit(t, &conf, outputBlocks, expectedSubmit{
@@ -100,22 +102,22 @@ func TestBroadcastDeliver(t *testing.T) {
 
 	t.Log("All good again")
 	fakeServer.Stop()
-	waitUntilGrpcServerIsDown(ctx, t, &servers[2].Configs[0].Endpoint)
 	servers[2].Servers[0] = test.RunGrpcServerForTest(ctx, t, servers[2].Configs[0], ordererService.RegisterService)
 	waitUntilGrpcServerIsReady(ctx, t, &servers[2].Configs[0].Endpoint)
 	submit(t, &conf, outputBlocks, expectedSubmit{
 		success: 3,
 	})
 
 	t.Log("Insufficient quorum")
+	_ = listener2.Close()
 	for _, gs := range servers[:2] {
 		for _, s := range gs.Servers[:2] {
 			s.Stop()
 		}
 	}
 	for _, gs := range servers[:2] {
 		for _, c := range gs.Configs[:2] {
-			waitUntilGrpcServerIsDown(ctx, t, &c.Endpoint)
+			holdPort(ctx, t, c)
 		}
 	}
 	submit(t, &conf, outputBlocks, expectedSubmit{
@@ -231,11 +233,17 @@ func waitUntilGrpcServerIsReady(ctx context.Context, t *testing.T, endpoint *con
 	t.Logf("%v is ready", endpoint)
 }
 
-func waitUntilGrpcServerIsDown(ctx context.Context, t *testing.T, endpoint *connection.Endpoint) {
+// holdPort attempts to bind to the specified server port and holds it until the listener is closed.
+// It serves two purposes:
+//  1. A successful bind indicates the port is free, meaning the server previously using it is down.
+//  2. It prevents other tests from binding to the same port, ensuring this test correctly detects the server as
+//     unavailable.
+func holdPort(ctx context.Context, t *testing.T, c *connection.ServerConfig) net.Listener {
 	t.Helper()
-	newConn, err := connection.Connect(test.NewInsecureDialConfig(endpoint))
+	listener, err := c.Listener(ctx)
 	require.NoError(t, err)
-	defer connection.CloseConnectionsLog(newConn)
-	test.WaitUntilGrpcServerIsDown(ctx, t, newConn)
-	t.Logf("%v is down", endpoint)
+	t.Cleanup(func() {
+		_ = listener.Close()
+	})
+	return listener
 }

utils/monitoring/provider.go

@@ -62,7 +62,7 @@ func (p *Provider) StartPrometheusServer(
 		Handler:     mux,
 	}
 
-	l, err := serverConfig.Listener()
+	l, err := serverConfig.Listener(ctx)
 	if err != nil {
 		return err
 	}

utils/test/utils.go

@@ -80,7 +80,7 @@ func RunGrpcServerForTest(
 	ctx context.Context, tb testing.TB, serverConfig *connection.ServerConfig, register func(server *grpc.Server),
 ) *grpc.Server {
 	tb.Helper()
-	listener, err := serverConfig.Listener()
+	listener, err := serverConfig.Listener(ctx)
 	require.NoError(tb, err)
 	server, err := serverConfig.GrpcServer()
 	require.NoError(tb, err)
@@ -221,25 +221,6 @@ func WaitUntilGrpcServerIsReady(
 	require.Equal(t, healthgrpc.HealthCheckResponse_SERVING, res.Status)
 }
 
-// WaitUntilGrpcServerIsDown uses the health check API to check a service is down.
-func WaitUntilGrpcServerIsDown(
-	ctx context.Context,
-	t *testing.T,
-	conn grpc.ClientConnInterface,
-) {
-	t.Helper()
-	if conn == nil {
-		return
-	}
-	healthClient := healthgrpc.NewHealthClient(conn)
-	require.EventuallyWithT(t, func(ct *assert.CollectT) {
-		checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
-		_, err := healthClient.Check(checkCtx, nil)
-		require.Error(ct, err)
-	}, time.Minute, 50*time.Millisecond)
-}
-
 // StatusRetriever provides implementation retrieve status of given transaction identifiers.
 type StatusRetriever interface {
 	GetTransactionsStatus(context.Context, *protoblocktx.QueryStatus, ...grpc.CallOption) (