rpc: configure DPRC conn pool with idle timeout

DPRC pool holds a set of DRPC connections to a peer node. Until now, pool was not configired with
any connection timeout under the assumption that cancellnig a stream would also free its underlying
connection. However, our benchmarks where QPS fluctuates over the course of a run relevealed that
streams are continously opened and closed. Because each stream in the pool corresponds to its own
TCP connection (unlike gRPC, which multiplexes streams over a single connection), this constant
opening and closing of a connection is undesireable. To address this, the PR adds a connection
expiration mechanism so that idle connections are closed independently of the streams.

Fixes: #140670
Epic: CRDB-48929
Release note: none

Author: cthumuluru-crdb

pkg/rpc/drpc.go

@@ -10,6 +10,7 @@ import (
 	"crypto/tls"
 	"math"
 	"net"
+	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/errors"
@@ -28,6 +29,9 @@ import (
 // have the DRPC server enabled.
 var ErrDRPCDisabled = errors.New("DRPC is not enabled")
 
+// Default idle connection timeout for DRPC connections in the pool.
+var defaultDRPCConnIdleTimeout = 5 * time.Minute
+
 type drpcServerI interface {
 	Serve(ctx context.Context, lis net.Listener) error
 }
@@ -93,7 +97,9 @@ func newDRPCServer(_ context.Context, rpcCtx *Context) (*DRPCServer, error) {
 func dialDRPC(rpcCtx *Context) func(ctx context.Context, target string) (drpcpool.Conn, error) {
 	return func(ctx context.Context, target string) (drpcpool.Conn, error) {
 		// TODO(server): could use connection class instead of empty key here.
-		pool := drpcpool.New[struct{}, drpcpool.Conn](drpcpool.Options{})
+		pool := drpcpool.New[struct{}, drpcpool.Conn](drpcpool.Options{
+			Expiration: defaultDRPCConnIdleTimeout,
+		})
 		pooledConn := pool.Get(ctx /* unused */, struct{}{}, func(ctx context.Context,
 			_ struct{}) (drpcpool.Conn, error) {