From 6ba2360cc94b6022d9344c1744ed72b3df5a1eac Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 21 Oct 2025 17:59:47 +0300 Subject: [PATCH 01/19] Added suppoer for YugabyteDB or PostgreSQL secure db node for the release-image-integration-test. Signed-off-by: Dean Amar --- cmd/config/app_config_test.go | 24 ++-- cmd/config/cobra_test_exports.go | 2 +- cmd/config/create_config_file.go | 2 + cmd/config/samples/coordinator.yaml | 12 +- cmd/config/samples/loadgen.yaml | 12 +- cmd/config/samples/query.yaml | 10 +- cmd/config/samples/sidecar.yaml | 12 +- cmd/config/samples/vc.yaml | 10 +- cmd/config/samples/verifier.yaml | 6 +- cmd/config/templates/queryexecutor.yaml | 5 +- cmd/config/templates/validatorpersister.yaml | 5 +- docker/images/test_node/Dockerfile | 4 + docker/test/common.go | 2 +- docker/test/container_release_image_test.go | 115 +++++++++++++++--- integration/runner/cluster_controller.go | 2 - integration/runner/postgres.go | 7 +- integration/runner/runtime.go | 17 ++- integration/runner/yugabyte.go | 4 +- .../test/db_node_failure_handling_test.go | 2 +- loadgen/client_test.go | 2 +- service/vc/config.go | 11 +- service/vc/dbtest/connection.go | 18 ++- service/vc/dbtest/container.go | 91 ++++++++++++-- service/vc/dbtest/database_setup.go | 8 +- service/vc/test_exports.go | 5 +- utils/connection/config.go | 11 ++ utils/test/secure_connection.go | 104 +++++++++++++--- utils/utils.go | 11 ++ 28 files changed, 406 insertions(+), 108 deletions(-) diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index 691d24ca..15679417 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -34,18 +34,18 @@ import ( var ( defaultServerTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/server-certs/public-key", - KeyPath: "/server-certs/private-key", + CertPath: "/server-certs/public-key.crt", + KeyPath: "/server-certs/private-key.key", CACertPaths: []string{ - "/server-certs/ca-certificate", + "/server-certs/ca-certificate.crt", }, } defaultClientTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/client-certs/public-key", - KeyPath: "/client-certs/private-key", + CertPath: "/client-certs/public-key.crt", + KeyPath: "/client-certs/private-key.key", CACertPaths: []string{ - "/client-certs/ca-certificate", + "/client-certs/ca-certificate.crt", }, } ) @@ -443,10 +443,14 @@ func defaultDBConfig() *vc.DatabaseConfig { func defaultSampleDBConfig() *vc.DatabaseConfig { return &vc.DatabaseConfig{ - Endpoints: []*connection.Endpoint{newEndpoint("db", 5433)}, - Username: "yugabyte", - Password: "yugabyte", - Database: "yugabyte", + Endpoints: []*connection.Endpoint{newEndpoint("db", 5433)}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + TLS: connection.DatabaseTLS{ + Activate: true, + CACertPath: "/server-certs/ca-certificate.crt", + }, MaxConnections: 10, MinConnections: 5, LoadBalance: false, diff --git a/cmd/config/cobra_test_exports.go b/cmd/config/cobra_test_exports.go index beaed2ae..82299f47 100644 --- a/cmd/config/cobra_test_exports.go +++ b/cmd/config/cobra_test_exports.go @@ -64,8 +64,8 @@ func StartDefaultSystem(t *testing.T) SystemConfig { }, DB: DatabaseConfig{ Name: conn.Database, - LoadBalance: false, Endpoints: conn.Endpoints, + LoadBalance: false, }, Policy: &workload.PolicyProfile{ ChannelID: "channel1", diff --git a/cmd/config/create_config_file.go b/cmd/config/create_config_file.go index 1a7516d5..3d886a31 100644 --- a/cmd/config/create_config_file.go +++ b/cmd/config/create_config_file.go @@ -72,8 +72,10 @@ type ( // DatabaseConfig represents the used DB. DatabaseConfig struct { Name string + Password string LoadBalance bool Endpoints []*connection.Endpoint + TLS connection.DatabaseTLS } ) diff --git a/cmd/config/samples/coordinator.yaml b/cmd/config/samples/coordinator.yaml index 9a401ac2..4bc70ed0 100644 --- a/cmd/config/samples/coordinator.yaml +++ b/cmd/config/samples/coordinator.yaml @@ -6,10 +6,10 @@ server: endpoint: :9001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2119 @@ -19,10 +19,10 @@ verifier: - verifier:5001 tls: &ClientTLS mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt validator-committer: endpoints: - vc:6001 diff --git a/cmd/config/samples/loadgen.yaml b/cmd/config/samples/loadgen.yaml index ef97fe63..4fef972b 100644 --- a/cmd/config/samples/loadgen.yaml +++ b/cmd/config/samples/loadgen.yaml @@ -6,10 +6,10 @@ server: endpoint: :8001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2118 @@ -26,10 +26,10 @@ orderer-client: endpoint: sidecar:4001 tls: mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt orderer: connection: endpoints: diff --git a/cmd/config/samples/query.yaml b/cmd/config/samples/query.yaml index b2011ddc..2c57e9f0 100644 --- a/cmd/config/samples/query.yaml +++ b/cmd/config/samples/query.yaml @@ -8,11 +8,10 @@ server: endpoint: :7001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate - # Credentials for the server + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2117 @@ -24,6 +23,9 @@ database: # TODO: pass password via environment variable password: "yugabyte" # The password for the database database: "yugabyte" # The database name + tls: + activate: true + ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool load-balance: false # Should be enabled for DB cluster diff --git a/cmd/config/samples/sidecar.yaml b/cmd/config/samples/sidecar.yaml index bb7b722f..ba1d5690 100644 --- a/cmd/config/samples/sidecar.yaml +++ b/cmd/config/samples/sidecar.yaml @@ -6,10 +6,10 @@ server: endpoint: :4001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt keep-alive: params: time: 300s @@ -40,10 +40,10 @@ committer: endpoint: coordinator:9001 tls: mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt ledger: path: /root/sc/ledger notification: diff --git a/cmd/config/samples/vc.yaml b/cmd/config/samples/vc.yaml index 54afaf09..2f61054d 100644 --- a/cmd/config/samples/vc.yaml +++ b/cmd/config/samples/vc.yaml @@ -8,11 +8,10 @@ server: endpoint: :6001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate - # Credentials for the server + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2116 @@ -23,6 +22,9 @@ database: # TODO: pass password via environment variable password: "yugabyte" # The password for the database database: "yugabyte" # The database name + tls: + activate: true + ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool. load-balance: false # Should be enabled for DB cluster. diff --git a/cmd/config/samples/verifier.yaml b/cmd/config/samples/verifier.yaml index 99fdb8c5..7f48b981 100644 --- a/cmd/config/samples/verifier.yaml +++ b/cmd/config/samples/verifier.yaml @@ -6,10 +6,10 @@ server: endpoint: :5001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2115 diff --git a/cmd/config/templates/queryexecutor.yaml b/cmd/config/templates/queryexecutor.yaml index 46b5ecba..6e538a2c 100644 --- a/cmd/config/templates/queryexecutor.yaml +++ b/cmd/config/templates/queryexecutor.yaml @@ -24,9 +24,12 @@ database: {{- end }} username: "yugabyte" # TODO: pass password via environment variable - password: "yugabyte" + password: {{ .DB.Password }} database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} + tls: + activate: {{ .DB.TLS.Activate }} + ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 retry: diff --git a/cmd/config/templates/validatorpersister.yaml b/cmd/config/templates/validatorpersister.yaml index 6e229546..ef6d7eb8 100644 --- a/cmd/config/templates/validatorpersister.yaml +++ b/cmd/config/templates/validatorpersister.yaml @@ -23,9 +23,12 @@ database: {{- end }} username: "yugabyte" # TODO: pass password via environment variable - password: "yugabyte" + password: {{ .DB.Password }} database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} + tls: + activate: { { .DB.TLS.Activate } } + ca-cert-path: { { .DB.TLS.CACertPath } } max-connections: 10 min-connections: 5 retry: diff --git a/docker/images/test_node/Dockerfile b/docker/images/test_node/Dockerfile index 8fef9ab4..189763fe 100644 --- a/docker/images/test_node/Dockerfile +++ b/docker/images/test_node/Dockerfile @@ -35,6 +35,10 @@ ENV SC_SIDECAR_COMMITTER_TLS_MODE="none" ENV SC_VC_SERVER_TLS_MODE="none" ENV SC_VERIFIER_SERVER_TLS_MODE="none" +# Disable TLS usage for db. +ENV SC_VC_DATABASE_TLS_ACTIVATE=false +ENV SC_QUERY_DATABASE_TLS_ACTIVATE=false + COPY ${ARCHBIN_PATH}/${TARGETOS}-${TARGETARCH}/* ${BINS_PATH}/ COPY ./docker/images/test_node/run ${BINS_PATH}/ COPY ./cmd/config/samples $CONFIGS_PATH diff --git a/docker/test/common.go b/docker/test/common.go index c30e2455..ea4e5cf6 100644 --- a/docker/test/common.go +++ b/docker/test/common.go @@ -134,7 +134,7 @@ func getContainerMappedHostPort( info, err := createDockerClient(t).ContainerInspect(ctx, containerName) require.NoError(t, err) require.NotNil(t, info) - portKey := nat.Port(fmt.Sprintf("%s/%s", containerPort, "tcp")) + portKey := nat.Port(fmt.Sprintf("%s/tcp", containerPort)) bindings, ok := info.NetworkSettings.Ports[portKey] require.True(t, ok) require.NotEmpty(t, bindings) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index e2bb2661..e78e2875 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -21,6 +21,8 @@ import ( "github.com/hyperledger/fabric-x-committer/cmd/config" "github.com/hyperledger/fabric-x-committer/loadgen/workload" + "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" + "github.com/hyperledger/fabric-x-committer/utils/connection" testutils "github.com/hyperledger/fabric-x-committer/utils/test" ) @@ -30,6 +32,8 @@ type startNodeParameters struct { networkName string tlsMode string configBlockPath string + dbType string + dbPassword string } const ( @@ -42,6 +46,28 @@ const ( containerConfigPath = "/root/config" // localConfigPath is the path to the sample YAML configuration of each service. localConfigPath = "../../cmd/config/samples" + + // containerPathForYugabytePassword holds the path to the database credentials inside the docker container. + // This work-around is needed due to a Yugabyte behavior that prevents using default passwords in secure mode. + // Instead, Yugabyte generates a random password, and this path points to the output file containing it. + containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec + + // yugabytedReadinessOutput is the output indicating that a Yugabyte node is ready. + yugabytedReadinessOutput = "Data placement constraint successfully verified" +) + +var ( + // enforcePostgresSSLScript enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. + enforcePostgresSSLScript = []string{ + "sh", "-c", + `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + + `/var/lib/postgresql/data/pg_hba.conf`, + } + + // reloadPostgresConfigScript reloads the PostgreSQL server configuration without restarting the instance. + reloadPostgresConfigScript = []string{ + "psql", "-U", "yugabyte", "-c", "SELECT pg_reload_conf();", + } ) // TestCommitterReleaseImagesWithTLS runs the committer components in different Docker containers with different TLS @@ -71,22 +97,25 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { testutils.RemoveDockerNetwork(t, networkName) }) + params := startNodeParameters{ + credsFactory: credsFactory, + networkName: networkName, + tlsMode: mode, + configBlockPath: configBlockPath, + dbType: testutils.YugaDBType, + } + for _, node := range []string{ "db", "verifier", "vc", "query", "coordinator", "sidecar", "orderer", "loadgen", } { - params := startNodeParameters{ - credsFactory: credsFactory, - node: node, - networkName: networkName, - tlsMode: mode, - configBlockPath: configBlockPath, - } - + params.node = node // stop and remove the container if it already exists. stopAndRemoveContainersByName(ctx, t, createDockerClient(t), assembleContainerName(node, mode)) switch node { - case "db", "orderer": + case "db": + params.dbPassword = startSecuredDatabaseNode(ctx, t, params).Password + case "orderer": startCommitterNodeWithTestImage(ctx, t, params) case "loadgen": startLoadgenNodeWithReleaseImage(ctx, t, params) @@ -101,12 +130,64 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { } } +// CreateAndStartSecuredDatabaseNode creates a containerized Yugabyte or PostgreSQL database instance in a secure mode. +func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNodeParameters) *dbtest.Connection { + t.Helper() + + tlsConfig, credsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.dbType, params.node) + + node := &dbtest.DatabaseContainer{ + DatabaseType: params.dbType, + Network: params.networkName, + Hostname: params.node, + TLSConfig: tlsConfig, + CredsPathDir: credsPath, + UseTLS: true, + } + + node.StartContainer(ctx, t) + conn := node.GetConnectionOptions(ctx, t) + + // this is relevant if we used different CA to create the DB's tls certificates. + require.NotEmpty(t, node.TLSConfig.CACertPaths) + conn.TLS = connection.DatabaseTLS{ + Activate: true, + CACertPath: node.TLSConfig.CACertPaths[0], + } + + // post start container tweaking + switch node.DatabaseType { + case testutils.YugaDBType: + node.FixFilesPermissions(t, + "root:root", + fmt.Sprintf("/creds/node.%s.crt", node.Hostname), + fmt.Sprintf("/creds/node.%s.key", node.Hostname), + ) + node.EnsureNodeReadiness(t, yugabytedReadinessOutput) + conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) + case testutils.PostgresDBType: + node.FixFilesPermissions(t, + "postgres:postgres", + "/creds/server.crt", + "/creds/server.key", + ) + node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) + node.ExecuteCommand(t, enforcePostgresSSLScript) + node.ExecuteCommand(t, reloadPostgresConfigScript) + default: + t.Fatalf("Unsupported database type: %s", node.DatabaseType) + } + + t.Cleanup( + func() { + node.StopAndRemoveContainer(t) + }) + + return conn +} + // startCommitterNodeWithReleaseImage starts a committer node using the release image. -func startCommitterNodeWithReleaseImage( - ctx context.Context, - t *testing.T, - params startNodeParameters, -) { +func startCommitterNodeWithReleaseImage(ctx context.Context, t *testing.T, params startNodeParameters) { t.Helper() configPath := filepath.Join(containerConfigPath, params.node) @@ -129,6 +210,8 @@ func startCommitterNodeWithReleaseImage( "SC_SIDECAR_COMMITTER_TLS_MODE=" + params.tlsMode, "SC_VC_SERVER_TLS_MODE=" + params.tlsMode, "SC_VERIFIER_SERVER_TLS_MODE=" + params.tlsMode, + "SC_VC_DATABASE_PASSWORD=" + params.dbPassword, + "SC_QUERY_DATABASE_PASSWORD=" + params.dbPassword, }, Tty: true, }, @@ -222,7 +305,9 @@ func assembleContainerName(node, tlsMode string) string { func assembleBinds(t *testing.T, params startNodeParameters, additionalBinds ...string) []string { t.Helper() - _, serverCredsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.node) + _, serverCredsPath := params.credsFactory.CreateServerCredentials( + t, params.tlsMode, testutils.DefaultCertStyle, params.node, + ) require.NotEmpty(t, serverCredsPath) _, clientCredsPath := params.credsFactory.CreateClientCredentials(t, params.tlsMode) require.NotEmpty(t, clientCredsPath) diff --git a/integration/runner/cluster_controller.go b/integration/runner/cluster_controller.go index 8268096c..84455c40 100644 --- a/integration/runner/cluster_controller.go +++ b/integration/runner/cluster_controller.go @@ -22,8 +22,6 @@ type DBClusterController struct { nodes []*dbtest.DatabaseContainer } -const linuxOS = "linux" - // StopAndRemoveSingleNodeByRole stops and removes a node given a role. func (cc *DBClusterController) StopAndRemoveSingleNodeByRole(t *testing.T, role string) { t.Helper() diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 60c1b6db..9c586fb0 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" + "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -65,7 +66,7 @@ type ( func StartPostgresCluster(ctx context.Context, t *testing.T) (*PostgresClusterController, *dbtest.Connection) { t.Helper() - if runtime.GOOS != linuxOS { + if runtime.GOOS != "linux" { t.Skip("Container IP access not supported on non-linux Docker") } @@ -99,7 +100,7 @@ func (cc *PostgresClusterController) addPrimaryNode(ctx context.Context, t *test }, }) node.StartContainer(ctx, t) - node.EnsureNodeReadiness(t, "database system is ready to accept connections") + node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) return node } @@ -132,7 +133,7 @@ func (cc *PostgresClusterController) createNode( Role: nodeCreationOpts.role, Image: postgresImage, Tag: defaultBitnamiPostgresTag, - DatabaseType: dbtest.PostgresDBType, + DatabaseType: test.PostgresDBType, Env: append([]string{ "POSTGRESQL_REPLICATION_USER=repl_user", "POSTGRESQL_REPLICATION_PASSWORD=repl_password", diff --git a/integration/runner/runtime.go b/integration/runner/runtime.go index 2ce3e33e..8676ac0e 100644 --- a/integration/runner/runtime.go +++ b/integration/runner/runtime.go @@ -95,8 +95,8 @@ type ( BlockTimeout time.Duration LoadgenBlockLimit uint64 - // DBCluster configures the cluster to operate in DB cluster mode. - DBCluster *dbtest.Connection + // DBConnection configures the runtime to operate with a custom database connection. + DBConnection *dbtest.Connection // TLS configures the secure level between the components: none | tls | mtls TLSMode string @@ -159,16 +159,18 @@ func NewRuntime(t *testing.T, conf *Config) *CommitterRuntime { c.AddOrUpdateNamespaces(t, types.MetaNamespaceID, workload.GeneratedNamespaceID, "1", "2", "3") t.Log("Making DB env") - if conf.DBCluster == nil { + if conf.DBConnection == nil { c.dbEnv = vc.NewDatabaseTestEnv(t) } else { - c.dbEnv = vc.NewDatabaseTestEnvWithCluster(t, conf.DBCluster) + c.dbEnv = vc.NewDatabaseTestEnvWithCustomConnection(t, conf.DBConnection) } s := &c.SystemConfig s.DB.Name = c.dbEnv.DBConf.Database + s.DB.Password = c.dbEnv.DBConf.Password s.DB.LoadBalance = c.dbEnv.DBConf.LoadBalance s.DB.Endpoints = c.dbEnv.DBConf.Endpoints + s.DB.TLS = c.dbEnv.DBConf.TLS s.LedgerPath = t.TempDir() s.ConfigBlockPath = filepath.Join(t.TempDir(), "config-block.pb.bin") @@ -581,7 +583,12 @@ func (c *CommitterRuntime) createSystemConfigWithServerTLS( ) *config.SystemConfig { t.Helper() serviceCfg := c.SystemConfig - serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials(t, c.config.TLSMode, endpoints.Server.Host) + serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials( + t, + c.config.TLSMode, + test.DefaultCertStyle, + endpoints.Server.Host, + ) serviceCfg.ServiceEndpoints = endpoints return &serviceCfg } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index 7b0d9bf9..5ae7d3e9 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -77,7 +77,7 @@ func StartYugaCluster(ctx context.Context, t *testing.T, numberOfMasters, number ) { t.Helper() - if runtime.GOOS != linuxOS { + if runtime.GOOS != "linux" { t.Skip("Container IP access not supported on non-linux Docker") } @@ -125,7 +125,7 @@ func (cc *YugaClusterController) createNode(role string) { Name: fmt.Sprintf("yuga-%s-%s", role, uuid.New().String()), Image: defaultImage, Role: role, - DatabaseType: dbtest.YugaDBType, + DatabaseType: test.YugaDBType, Network: cc.networkName, } cc.nodes = append(cc.nodes, node) diff --git a/integration/test/db_node_failure_handling_test.go b/integration/test/db_node_failure_handling_test.go index 5a111d5f..1794b6bc 100644 --- a/integration/test/db_node_failure_handling_test.go +++ b/integration/test/db_node_failure_handling_test.go @@ -109,7 +109,7 @@ func registerAndCreateRuntime(t *testing.T, clusterConnection *dbtest.Connection NumVCService: 2, BlockTimeout: 2 * time.Second, BlockSize: 500, - DBCluster: clusterConnection, + DBConnection: clusterConnection, }) c.Start(t, runner.FullTxPathWithLoadGen) diff --git a/loadgen/client_test.go b/loadgen/client_test.go index 4d39c427..8afa7074 100644 --- a/loadgen/client_test.go +++ b/loadgen/client_test.go @@ -453,7 +453,7 @@ func createServerAndClientTLSConfig(t *testing.T, tlsMode string) ( ) { t.Helper() credsFactory := test.NewCredentialsFactory(t) - clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, defaultServerSAN) + clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, test.DefaultCertStyle, defaultServerSAN) serverTLSConfig, _ = credsFactory.CreateClientCredentials(t, tlsMode) return clientTLSConfig, serverTLSConfig } diff --git a/service/vc/config.go b/service/vc/config.go index 21a91d72..c5467d91 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -32,18 +32,27 @@ type DatabaseConfig struct { MinConnections int32 `mapstructure:"min-connections"` LoadBalance bool `mapstructure:"load-balance"` Retry *connection.RetryProfile `mapstructure:"retry"` + TLS connection.DatabaseTLS `mapstructure:"tls"` } // DataSourceName returns the data source name of the database. func (d *DatabaseConfig) DataSourceName() string { - ret := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", d.Username, d.Password, d.EndpointsString(), d.Database) + if d.TLS.UseTLS() { + ret += "sslmode=verify-full" + } else { + ret += "sslmode=disable" + } // The load balancing flag is only available when the server supports it (having multiple nodes). // Thus, we only add it when explicitly required. Otherwise, an error will occur. if d.LoadBalance { ret += "&load_balance=true" } + if d.TLS.UseTLS() { + ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) + } return ret } diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index 8e02fbb8..087eaf6a 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -44,6 +44,7 @@ type Connection struct { Password string Database string LoadBalance bool + TLS connection.DatabaseTLS } // NewConnection returns a connection parameters with the specified host:port, and the default values @@ -58,8 +59,23 @@ func NewConnection(endpoints ...*connection.Endpoint) *Connection { // dataSourceName returns the dataSourceName to be used by the database/sql package. func (c *Connection) dataSourceName() string { - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", c.User, c.Password, c.endpointsString(), c.Database) + + if c.TLS.UseTLS() { + ret += "sslmode=verify-full" + } else { + ret += "sslmode=disable" + } + // The load balancing flag is only available when the server supports it (having multiple nodes). + // Thus, we only add it when explicitly required. Otherwise, an error will occur. + if c.LoadBalance { + ret += "&load_balance=true" + } + if c.TLS.UseTLS() { + ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) + } + return ret } // endpointsString returns the address:port as a string with comma as a separator between endpoints. diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 3260cbac..746502a7 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -7,10 +7,12 @@ SPDX-License-Identifier: Apache-2.0 package dbtest import ( + "bufio" "bytes" "context" "fmt" "os" + "regexp" "strconv" "strings" "testing" @@ -18,9 +20,11 @@ import ( "github.com/cockroachdb/errors" docker "github.com/fsouza/go-dockerclient" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/hyperledger/fabric-x-committer/utils" "github.com/hyperledger/fabric-x-committer/utils/connection" "github.com/hyperledger/fabric-x-committer/utils/test" ) @@ -36,6 +40,9 @@ const ( // container's Memory and CPU management. gb = 1 << 30 // gb is the number of bytes needed to represent 1 GB. memorySwap = -1 // memorySwap disable memory swaps (don't store data on disk) + + // PostgresReadinessOutput is the output indicating that a Postgres node is ready. + PostgresReadinessOutput = "database system is ready to accept connections" ) // YugabyteCMD starts yugabyte without SSL and fault tolerance (single server). @@ -59,17 +66,22 @@ type DatabaseContainer struct { Image string HostIP string Network string + Hostname string DatabaseType string Tag string Role string + CredsPathDir string Cmd []string Env []string + Binds []string HostPort int DbPort docker.Port PortMap docker.Port PortBinds map[docker.Port][]docker.PortBinding NetToIP map[string]*docker.EndpointConfig AutoRm bool + UseTLS bool + TLSConfig connection.TLSConfig client *docker.Client containerID string @@ -100,19 +112,27 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit t.Helper() switch dc.DatabaseType { - case YugaDBType: + case test.YugaDBType: if dc.Image == "" { dc.Image = defaultYugabyteImage } if dc.Cmd == nil { dc.Cmd = YugabyteCMD + + if dc.UseTLS { + dc.Cmd = append( + utils.ReplacePattern(dc.Cmd, func(s string) bool { return s == "--insecure" }, "--secure"), + "--certs_dir=/creds", + "--advertise_address", dc.Hostname, + ) + } } if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", yugaDBPort)) } - case PostgresDBType: + case test.PostgresDBType: if dc.Image == "" { dc.Image = defaultPostgresImage } @@ -127,6 +147,14 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", postgresDBPort)) } + + if dc.UseTLS { + dc.Cmd = []string{ + "-c", "ssl=on", + "-c", "ssl_cert_file=/creds/server.crt", + "-c", "ssl_key_file=/creds/server.key", + } + } default: t.Fatalf("Unsupported database type: %s", dc.DatabaseType) } @@ -154,6 +182,10 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.client == nil { dc.client = test.GetDockerClient(t) } + if dc.UseTLS { + dc.Binds = append(dc.Binds, fmt.Sprintf("%s:/creds", dc.CredsPathDir)) + dc.Name += fmt.Sprintf("_with_tls_%s", uuid.NewString()[0:8]) + } } // createContainer attempts to create a container instance, or attach to an existing one. @@ -179,14 +211,16 @@ func (dc *DatabaseContainer) createContainer(ctx context.Context, t *testing.T) Context: ctx, Name: dc.Name, Config: &docker.Config{ - Image: dc.Image, - Cmd: dc.Cmd, - Env: dc.Env, + Image: dc.Image, + Cmd: dc.Cmd, + Env: dc.Env, + Hostname: dc.Hostname, }, HostConfig: &docker.HostConfig{ AutoRemove: dc.AutoRm, PortBindings: dc.PortBinds, NetworkMode: dc.Network, + Binds: dc.Binds, Memory: 4 * gb, MemorySwap: memorySwap, }, @@ -223,8 +257,8 @@ func (dc *DatabaseContainer) findContainer(t *testing.T) error { return errors.Errorf("cannot find container '%s'. Containers: %v", dc.Name, names) } -// getConnectionOptions inspect the container and fetches the available connection options. -func (dc *DatabaseContainer) getConnectionOptions(ctx context.Context, t *testing.T) *Connection { +// GetConnectionOptions inspect the container and fetches the available connection options. +func (dc *DatabaseContainer) GetConnectionOptions(ctx context.Context, t *testing.T) *Connection { t.Helper() container, err := dc.client.InspectContainerWithOptions(docker.InspectContainerOptions{ Context: ctx, @@ -233,7 +267,7 @@ func (dc *DatabaseContainer) getConnectionOptions(ctx context.Context, t *testin require.NoError(t, err) endpoints := []*connection.Endpoint{ - connection.CreateEndpointHP(container.NetworkSettings.IPAddress, dc.DbPort.Port()), + dc.GetContainerConnectionDetails(t), } for _, p := range container.NetworkSettings.Ports[dc.DbPort] { endpoints = append(endpoints, connection.CreateEndpointHP(p.HostIP, p.HostPort)) @@ -317,6 +351,30 @@ func (dc *DatabaseContainer) ContainerID() string { return dc.containerID } +// ReadPasswordFromContainer extracts the randomly generated password from a file inside the container. +// This is required because YugabyteDB, when running in secure mode, doesn't allow default passwords +// and instead generates a random one at startup. +// If no password is found, the default one will be returned. +func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { + t.Helper() + output := dc.ExecuteCommand(t, []string{"cat", filePath}) + + scanner := bufio.NewScanner(strings.NewReader(output)) + re := regexp.MustCompile(`(?i)^password:\s*(.+)$`) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if matches := re.FindStringSubmatch(line); len(matches) == 2 { + return matches[1] + } + } + + require.NoError(t, scanner.Err(), "error scanning command output") + t.Log("password not found in output, returning default password.") + + return defaultPassword +} + // ExecuteCommand executes a command and returns the container output. func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { t.Helper() @@ -335,10 +393,6 @@ func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { RawTerminal: false, })) - inspect, err := dc.client.InspectExec(exec.ID) - require.NoError(t, err) - require.Equal(t, 0, inspect.ExitCode) - return stdout.String() } @@ -350,3 +404,16 @@ func (dc *DatabaseContainer) EnsureNodeReadiness(t *testing.T, requiredOutput st require.Contains(ct, output, requiredOutput) }, 45*time.Second, 250*time.Millisecond) } + +// FixFilesPermissions fixes the ownership of the given files in the container. +func (dc *DatabaseContainer) FixFilesPermissions(t *testing.T, user string, files ...string, +) { + t.Helper() + exec, err := dc.client.CreateExec(docker.CreateExecOptions{ + Container: dc.containerID, + Cmd: append([]string{"chown", user}, files...), + User: "root", // Run as root to change ownership + }) + require.NoError(t, err) + require.NoError(t, dc.client.StartExec(exec.ID, docker.StartExecOptions{})) +} diff --git a/service/vc/dbtest/database_setup.go b/service/vc/dbtest/database_setup.go index 78b16f05..2aa1efd1 100644 --- a/service/vc/dbtest/database_setup.go +++ b/service/vc/dbtest/database_setup.go @@ -22,6 +22,7 @@ import ( "github.com/hyperledger/fabric-x-committer/utils" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -31,9 +32,6 @@ const ( deploymentLocal = "local" deploymentContainer = "container" - YugaDBType = "yugabyte" //nolint:revive - PostgresDBType = "postgres" - yugaDBPort = "5433" postgresDBPort = "5432" @@ -73,7 +71,7 @@ func getDBTypeFromEnv() string { if found { return strings.ToLower(val) } - return YugaDBType + return test.YugaDBType } // PrepareTestEnv initializes a test environment for an existing or uncontrollable db instance. @@ -120,7 +118,7 @@ func StartAndConnect(ctx context.Context, t *testing.T) *Connection { DatabaseType: getDBTypeFromEnv(), } container.StartContainer(ctx, t) - connOptions = container.getConnectionOptions(ctx, t) + connOptions = container.GetConnectionOptions(ctx, t) case deploymentLocal: connOptions = NewConnection(connection.CreateEndpointHP("localhost", defaultLocalDBPort)) default: diff --git a/service/vc/test_exports.go b/service/vc/test_exports.go index 2e6c59d7..f0fdc2c1 100644 --- a/service/vc/test_exports.go +++ b/service/vc/test_exports.go @@ -131,8 +131,8 @@ func NewDatabaseTestEnv(t *testing.T) *DatabaseTestEnv { return newDatabaseTestEnv(t, dbtest.PrepareTestEnv(t), false) } -// NewDatabaseTestEnvWithCluster creates a new db cluster test environment. -func NewDatabaseTestEnvWithCluster(t *testing.T, dbConnections *dbtest.Connection) *DatabaseTestEnv { +// NewDatabaseTestEnvWithCustomConnection creates a new db test environment given a connection. +func NewDatabaseTestEnvWithCustomConnection(t *testing.T, dbConnections *dbtest.Connection) *DatabaseTestEnv { t.Helper() require.NotNil(t, dbConnections) return newDatabaseTestEnv(t, dbtest.PrepareTestEnvWithConnection(t, dbConnections), dbConnections.LoadBalance) @@ -148,6 +148,7 @@ func newDatabaseTestEnv(t *testing.T, cs *dbtest.Connection, loadBalance bool) * MaxConnections: 10, MinConnections: 1, LoadBalance: loadBalance, + TLS: cs.TLS, Retry: &connection.RetryProfile{ MaxElapsedTime: 5 * time.Minute, InitialInterval: time.Duration(rand.Intn(900)+100) * time.Millisecond, diff --git a/utils/connection/config.go b/utils/connection/config.go index a7fa5db3..cf7a01e5 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -77,6 +77,12 @@ type ( KeyPath string `mapstructure:"key-path"` CACertPaths []string `mapstructure:"ca-cert-paths"` } + + // DatabaseTLS holds the database connection credentials. + DatabaseTLS struct { + Activate bool `mapstructure:"activate"` + CACertPath string `mapstructure:"ca-cert-path"` + } ) const ( @@ -90,6 +96,11 @@ const ( DefaultTLSMinVersion = tls.VersionTLS12 ) +// UseTLS sets the option of using TLS configuration for database connection. +func (dc *DatabaseTLS) UseTLS() bool { + return dc.Activate && dc.CACertPath != "" +} + // ServerCredentials returns the gRPC transport credentials to be used by a server, // based on the provided TLS configuration. func (c TLSConfig) ServerCredentials() (credentials.TransportCredentials, error) { diff --git a/utils/test/secure_connection.go b/utils/test/secure_connection.go index a0123c2d..c7976153 100644 --- a/utils/test/secure_connection.go +++ b/utils/test/secure_connection.go @@ -36,6 +36,13 @@ type ( shouldFail bool } + createTLSConfigParameters struct { + connectionMode string + keyPair *tlsgen.CertKeyPair + namingStyle string + san string + } + // ServerStarter is a function that receives a TLS configuration, starts the server, // and returns a RPCAttempt function for initiating a client connection and attempting an RPC call. ServerStarter func(t *testing.T, serverTLS connection.TLSConfig) RPCAttempt @@ -45,7 +52,20 @@ type ( RPCAttempt func(ctx context.Context, t *testing.T, cfg connection.TLSConfig) error ) -const defaultHostName = "localhost" +const ( + defaultHostName = "localhost" + + YugaDBType = "yugabyte" //nolint:revive + PostgresDBType = "postgres" + + DefaultCertStyle = "default" + + KeyPrivate = "private-key" + KeyPublic = "public-key" + KeyCACert = "ca-certificate" + + caCertFileName = "ca.crt" +) // ServerModes is a list of server-side TLS modes used for testing. var ServerModes = []string{connection.MutualTLSMode, connection.OneSideTLSMode, connection.NoneTLSMode} @@ -65,12 +85,19 @@ func NewCredentialsFactory(t *testing.T) *CredentialsFactory { func (scm *CredentialsFactory) CreateServerCredentials( t *testing.T, tlsMode string, + namingStyle string, san ...string, ) (connection.TLSConfig, string) { t.Helper() + require.NotEmpty(t, san) serverKeypair, err := scm.CertificateAuthority.NewServerCertKeyPair(san...) require.NoError(t, err) - return createTLSConfig(t, tlsMode, serverKeypair, scm.CertificateAuthority.CertBytes()) + return scm.createTLSConfig(t, createTLSConfigParameters{ + connectionMode: tlsMode, + keyPair: serverKeypair, + san: san[0], + namingStyle: namingStyle, + }) } // CreateClientCredentials creates a client key pair, @@ -79,7 +106,11 @@ func (scm *CredentialsFactory) CreateClientCredentials(t *testing.T, tlsMode str t.Helper() clientKeypair, err := scm.CertificateAuthority.NewClientCertKeyPair() require.NoError(t, err) - return createTLSConfig(t, tlsMode, clientKeypair, scm.CertificateAuthority.CertBytes()) + return scm.createTLSConfig(t, createTLSConfigParameters{ + connectionMode: tlsMode, + keyPair: clientKeypair, + namingStyle: DefaultCertStyle, + }) } /* @@ -136,7 +167,7 @@ func RunSecureConnectionTest( t.Run(fmt.Sprintf("server-tls:%s", tc.serverMode), func(t *testing.T) { t.Parallel() // create server's tls config and start it according to the server tls mode. - serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, defaultHostName) + serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, DefaultCertStyle, defaultHostName) rpcAttemptFunc := starter(t, serverTLS) // for each server secure mode, build the client's test cases. for _, clientTestCase := range tc.cases { @@ -186,28 +217,71 @@ func CreateClientWithTLS[T any]( // createTLSConfig creates a TLS configuration based on the // given TLS mode and credential bytes, and returns it along with the certificates' path. -func createTLSConfig( +func (scm *CredentialsFactory) createTLSConfig( t *testing.T, - connectionMode string, - keyPair *tlsgen.CertKeyPair, - caCertificate []byte, + params createTLSConfigParameters, ) (connection.TLSConfig, string) { t.Helper() tmpDir := t.TempDir() + namingFunction := selectFileNames(params.namingStyle, params.san) - privateKeyPath := filepath.Join(tmpDir, "private-key") - require.NoError(t, os.WriteFile(privateKeyPath, keyPair.Key, 0o600)) + privateKeyPath := filepath.Join(tmpDir, namingFunction("private-key")) + require.NoError(t, os.WriteFile(privateKeyPath, params.keyPair.Key, 0o600)) - publicKeyPath := filepath.Join(tmpDir, "public-key") - require.NoError(t, os.WriteFile(publicKeyPath, keyPair.Cert, 0o600)) + publicKeyPath := filepath.Join(tmpDir, namingFunction("public-key")) + require.NoError(t, os.WriteFile(publicKeyPath, params.keyPair.Cert, 0o600)) - caCertificatePath := filepath.Join(tmpDir, "ca-certificate") - require.NoError(t, os.WriteFile(caCertificatePath, caCertificate, 0o600)) + caCertificatePath := filepath.Join(tmpDir, namingFunction("ca-certificate")) + require.NoError(t, os.WriteFile(caCertificatePath, scm.CertificateAuthority.CertBytes(), 0o600)) return connection.TLSConfig{ - Mode: connectionMode, + Mode: params.connectionMode, KeyPath: privateKeyPath, CertPath: publicKeyPath, CACertPaths: []string{caCertificatePath}, }, tmpDir } + +func selectFileNames(style, serverName string) func(string) string { + switch style { + case YugaDBType: + return func(key string) string { + switch key { + case KeyPublic: + return fmt.Sprintf("node.%s.crt", serverName) + case KeyPrivate: + return fmt.Sprintf("node.%s.key", serverName) + case KeyCACert: + return caCertFileName + default: + return "" + } + } + case PostgresDBType: + return func(key string) string { + switch key { + case KeyPublic: + return "server.crt" + case KeyPrivate: + return "server.key" + case KeyCACert: + return "ca-certificate.crt" + default: + return "" + } + } + default: + return func(key string) string { + switch key { + case KeyPublic: + return "public-key.crt" + case KeyPrivate: + return "private-key.key" + case KeyCACert: + return "ca-certificate.crt" + default: + return "" + } + } + } +} diff --git a/utils/utils.go b/utils/utils.go index 43853f05..44560b6f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -88,3 +88,14 @@ func CountAppearances[T comparable](items []T) map[T]int { } return countMap } + +// ReplacePattern replaces the first matching value in a slice using a predicate and a new value. +func ReplacePattern[T any](slice []T, match func(T) bool, newValue T) []T { + for i, val := range slice { + if match(val) { + slice[i] = newValue + break + } + } + return slice +} From 20668398888b7f6f2b0d62840999110b68aacf6e Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 21 Oct 2025 18:22:58 +0300 Subject: [PATCH 02/19] * fixed template issues. Signed-off-by: Dean Amar --- cmd/config/templates/validatorpersister.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/config/templates/validatorpersister.yaml b/cmd/config/templates/validatorpersister.yaml index ef6d7eb8..7d613627 100644 --- a/cmd/config/templates/validatorpersister.yaml +++ b/cmd/config/templates/validatorpersister.yaml @@ -27,8 +27,8 @@ database: database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} tls: - activate: { { .DB.TLS.Activate } } - ca-cert-path: { { .DB.TLS.CACertPath } } + activate: {{ .DB.TLS.Activate }} + ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 retry: From 30392aea6c960539de0075c93fad45979f0313d9 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Wed, 22 Oct 2025 16:09:59 +0300 Subject: [PATCH 03/19] * Semantic and code-hygiene changes. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 24 ++++++++-------- integration/runner/cluster_controller.go | 2 ++ integration/runner/postgres.go | 2 +- integration/runner/yugabyte.go | 2 +- service/vc/config.go | 7 +++-- service/vc/dbtest/connection.go | 7 +++-- service/vc/dbtest/container.go | 31 ++++++--------------- service/vc/test_exports.go | 2 +- utils/test/secure_connection.go | 26 ++++++++--------- utils/utils.go | 11 -------- 10 files changed, 46 insertions(+), 68 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index e78e2875..2270dd19 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -51,9 +51,6 @@ const ( // This work-around is needed due to a Yugabyte behavior that prevents using default passwords in secure mode. // Instead, Yugabyte generates a random password, and this path points to the output file containing it. containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec - - // yugabytedReadinessOutput is the output indicating that a Yugabyte node is ready. - yugabytedReadinessOutput = "Data placement constraint successfully verified" ) var ( @@ -130,7 +127,8 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { } } -// CreateAndStartSecuredDatabaseNode creates a containerized Yugabyte or PostgreSQL database instance in a secure mode. +// CreateAndStartSecuredDatabaseNode creates a containerized YugabyteDB or PostgreSQL +// database instance in a secure mode. func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNodeParameters) *dbtest.Connection { t.Helper() @@ -148,7 +146,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod node.StartContainer(ctx, t) conn := node.GetConnectionOptions(ctx, t) - // this is relevant if we used different CA to create the DB's tls certificates. + // This is relevant if a different CA was used to issue the DB's TLS certificates. require.NotEmpty(t, node.TLSConfig.CACertPaths) conn.TLS = connection.DatabaseTLS{ Activate: true, @@ -158,19 +156,21 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod // post start container tweaking switch node.DatabaseType { case testutils.YugaDBType: - node.FixFilesPermissions(t, - "root:root", + // Ensure proper root ownership and permissions for the TLS certificate files. + node.ExecuteCommand(t, []string{ + "chown", "root:root", fmt.Sprintf("/creds/node.%s.crt", node.Hostname), fmt.Sprintf("/creds/node.%s.key", node.Hostname), - ) - node.EnsureNodeReadiness(t, yugabytedReadinessOutput) + }) + node.EnsureNodeReadiness(t, "Data placement constraint successfully verified") conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - node.FixFilesPermissions(t, - "postgres:postgres", + // Ensure proper root ownership and permissions for the TLS certificate files. + node.ExecuteCommand(t, []string{ + "chown", "postgres:postgres", "/creds/server.crt", "/creds/server.key", - ) + }) node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) node.ExecuteCommand(t, enforcePostgresSSLScript) node.ExecuteCommand(t, reloadPostgresConfigScript) diff --git a/integration/runner/cluster_controller.go b/integration/runner/cluster_controller.go index 84455c40..8268096c 100644 --- a/integration/runner/cluster_controller.go +++ b/integration/runner/cluster_controller.go @@ -22,6 +22,8 @@ type DBClusterController struct { nodes []*dbtest.DatabaseContainer } +const linuxOS = "linux" + // StopAndRemoveSingleNodeByRole stops and removes a node given a role. func (cc *DBClusterController) StopAndRemoveSingleNodeByRole(t *testing.T, role string) { t.Helper() diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 9c586fb0..afb6798d 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -66,7 +66,7 @@ type ( func StartPostgresCluster(ctx context.Context, t *testing.T) (*PostgresClusterController, *dbtest.Connection) { t.Helper() - if runtime.GOOS != "linux" { + if runtime.GOOS != linuxOS { t.Skip("Container IP access not supported on non-linux Docker") } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index 5ae7d3e9..c8761900 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -77,7 +77,7 @@ func StartYugaCluster(ctx context.Context, t *testing.T, numberOfMasters, number ) { t.Helper() - if runtime.GOOS != "linux" { + if runtime.GOOS != linuxOS { t.Skip("Container IP access not supported on non-linux Docker") } diff --git a/service/vc/config.go b/service/vc/config.go index c5467d91..1de3e22d 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -41,7 +41,11 @@ func (d *DatabaseConfig) DataSourceName() string { d.Username, d.Password, d.EndpointsString(), d.Database) if d.TLS.UseTLS() { + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. ret += "sslmode=verify-full" + ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) } else { ret += "sslmode=disable" } @@ -50,9 +54,6 @@ func (d *DatabaseConfig) DataSourceName() string { if d.LoadBalance { ret += "&load_balance=true" } - if d.TLS.UseTLS() { - ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) - } return ret } diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index 087eaf6a..250bf330 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -63,7 +63,11 @@ func (c *Connection) dataSourceName() string { c.User, c.Password, c.endpointsString(), c.Database) if c.TLS.UseTLS() { + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. ret += "sslmode=verify-full" + ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) } else { ret += "sslmode=disable" } @@ -72,9 +76,6 @@ func (c *Connection) dataSourceName() string { if c.LoadBalance { ret += "&load_balance=true" } - if c.TLS.UseTLS() { - ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) - } return ret } diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 746502a7..bb9199b6 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/hyperledger/fabric-x-committer/utils" "github.com/hyperledger/fabric-x-committer/utils/connection" "github.com/hyperledger/fabric-x-committer/utils/test" ) @@ -45,7 +44,7 @@ const ( PostgresReadinessOutput = "database system is ready to accept connections" ) -// YugabyteCMD starts yugabyte without SSL and fault tolerance (single server). +// YugabyteCMD starts yugabyte without fault tolerance (single server). var YugabyteCMD = []string{ "bin/yugabyted", "start", "--callhome", "false", @@ -57,7 +56,6 @@ var YugabyteCMD = []string{ "yb_num_shards_per_tserver=1," + "minloglevel=3," + "yb_enable_read_committed_isolation=true", - "--insecure", } // DatabaseContainer manages the execution of an instance of a dockerized DB for tests. @@ -119,13 +117,10 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.Cmd == nil { dc.Cmd = YugabyteCMD - if dc.UseTLS { - dc.Cmd = append( - utils.ReplacePattern(dc.Cmd, func(s string) bool { return s == "--insecure" }, "--secure"), - "--certs_dir=/creds", - "--advertise_address", dc.Hostname, - ) + dc.Cmd = append(dc.Cmd, "--secure", "--certs_dir=/creds", "--advertise_address", dc.Hostname) + } else { + dc.Cmd = append(dc.Cmd, "--insecure") } } @@ -147,7 +142,6 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", postgresDBPort)) } - if dc.UseTLS { dc.Cmd = []string{ "-c", "ssl=on", @@ -393,6 +387,10 @@ func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { RawTerminal: false, })) + inspect, err := dc.client.InspectExec(exec.ID) + require.NoError(t, err) + require.Equal(t, 0, inspect.ExitCode) + return stdout.String() } @@ -404,16 +402,3 @@ func (dc *DatabaseContainer) EnsureNodeReadiness(t *testing.T, requiredOutput st require.Contains(ct, output, requiredOutput) }, 45*time.Second, 250*time.Millisecond) } - -// FixFilesPermissions fixes the ownership of the given files in the container. -func (dc *DatabaseContainer) FixFilesPermissions(t *testing.T, user string, files ...string, -) { - t.Helper() - exec, err := dc.client.CreateExec(docker.CreateExecOptions{ - Container: dc.containerID, - Cmd: append([]string{"chown", user}, files...), - User: "root", // Run as root to change ownership - }) - require.NoError(t, err) - require.NoError(t, dc.client.StartExec(exec.ID, docker.StartExecOptions{})) -} diff --git a/service/vc/test_exports.go b/service/vc/test_exports.go index f0fdc2c1..f39f9d8b 100644 --- a/service/vc/test_exports.go +++ b/service/vc/test_exports.go @@ -131,7 +131,7 @@ func NewDatabaseTestEnv(t *testing.T) *DatabaseTestEnv { return newDatabaseTestEnv(t, dbtest.PrepareTestEnv(t), false) } -// NewDatabaseTestEnvWithCustomConnection creates a new db test environment given a connection. +// NewDatabaseTestEnvWithCustomConnection creates a new db test environment given a db connection. func NewDatabaseTestEnvWithCustomConnection(t *testing.T, dbConnections *dbtest.Connection) *DatabaseTestEnv { t.Helper() require.NotNil(t, dbConnections) diff --git a/utils/test/secure_connection.go b/utils/test/secure_connection.go index c7976153..8f654c7a 100644 --- a/utils/test/secure_connection.go +++ b/utils/test/secure_connection.go @@ -40,7 +40,6 @@ type ( connectionMode string keyPair *tlsgen.CertKeyPair namingStyle string - san string } // ServerStarter is a function that receives a TLS configuration, starts the server, @@ -55,16 +54,16 @@ type ( const ( defaultHostName = "localhost" - YugaDBType = "yugabyte" //nolint:revive + // YugaDBType represents the usage of Yugabyte DB. + YugaDBType = "yugabyte" + // PostgresDBType represents the usage of PostgreSQL DB. PostgresDBType = "postgres" - + // DefaultCertStyle represents the default TLS certificate style creation. DefaultCertStyle = "default" - + //nolint:revive // KeyPrivate, KeyPublic and KeyCACert represents the chosen key in the naming function. KeyPrivate = "private-key" KeyPublic = "public-key" KeyCACert = "ca-certificate" - - caCertFileName = "ca.crt" ) // ServerModes is a list of server-side TLS modes used for testing. @@ -89,13 +88,11 @@ func (scm *CredentialsFactory) CreateServerCredentials( san ...string, ) (connection.TLSConfig, string) { t.Helper() - require.NotEmpty(t, san) serverKeypair, err := scm.CertificateAuthority.NewServerCertKeyPair(san...) require.NoError(t, err) return scm.createTLSConfig(t, createTLSConfigParameters{ connectionMode: tlsMode, keyPair: serverKeypair, - san: san[0], namingStyle: namingStyle, }) } @@ -223,7 +220,7 @@ func (scm *CredentialsFactory) createTLSConfig( ) (connection.TLSConfig, string) { t.Helper() tmpDir := t.TempDir() - namingFunction := selectFileNames(params.namingStyle, params.san) + namingFunction := selectFileNames(params.namingStyle) privateKeyPath := filepath.Join(tmpDir, namingFunction("private-key")) require.NoError(t, os.WriteFile(privateKeyPath, params.keyPair.Key, 0o600)) @@ -242,17 +239,20 @@ func (scm *CredentialsFactory) createTLSConfig( }, tmpDir } -func selectFileNames(style, serverName string) func(string) string { +func selectFileNames(style string) func(string) string { switch style { case YugaDBType: return func(key string) string { switch key { + // We currently use YugabyteDB with the hostname "db" only. + // To support additional instances with different hostnames, + // replace "db" with the desired hostname when creating the instance. case KeyPublic: - return fmt.Sprintf("node.%s.crt", serverName) + return "node.db.crt" case KeyPrivate: - return fmt.Sprintf("node.%s.key", serverName) + return "node.db.key" case KeyCACert: - return caCertFileName + return "ca.crt" default: return "" } diff --git a/utils/utils.go b/utils/utils.go index 44560b6f..43853f05 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -88,14 +88,3 @@ func CountAppearances[T comparable](items []T) map[T]int { } return countMap } - -// ReplacePattern replaces the first matching value in a slice using a predicate and a new value. -func ReplacePattern[T any](slice []T, match func(T) bool, newValue T) []T { - for i, val := range slice { - if match(val) { - slice[i] = newValue - break - } - } - return slice -} From b0d5ad651d3bd9a575c78fd12fc901593826a456 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Thu, 30 Oct 2025 13:56:27 +0200 Subject: [PATCH 04/19] * minor semantic change. Signed-off-by: Dean Amar --- cmd/config/app_config_test.go | 2 +- cmd/config/create_config_file.go | 2 +- docker/test/container_release_image_test.go | 2 +- service/vc/config.go | 18 +++++++++--------- service/vc/dbtest/connection.go | 2 +- utils/connection/config.go | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index 15679417..a32174ef 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -447,7 +447,7 @@ func defaultSampleDBConfig() *vc.DatabaseConfig { Username: "yugabyte", Password: "yugabyte", Database: "yugabyte", - TLS: connection.DatabaseTLS{ + TLS: connection.DatabaseTLSConfig{ Activate: true, CACertPath: "/server-certs/ca-certificate.crt", }, diff --git a/cmd/config/create_config_file.go b/cmd/config/create_config_file.go index a0e67629..830f9d6a 100644 --- a/cmd/config/create_config_file.go +++ b/cmd/config/create_config_file.go @@ -75,7 +75,7 @@ type ( Password string LoadBalance bool Endpoints []*connection.Endpoint - TLS connection.DatabaseTLS + TLS connection.DatabaseTLSConfig } ) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 2270dd19..7c0d73c4 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -148,7 +148,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod // This is relevant if a different CA was used to issue the DB's TLS certificates. require.NotEmpty(t, node.TLSConfig.CACertPaths) - conn.TLS = connection.DatabaseTLS{ + conn.TLS = connection.DatabaseTLSConfig{ Activate: true, CACertPath: node.TLSConfig.CACertPaths[0], } diff --git a/service/vc/config.go b/service/vc/config.go index 1de3e22d..a0ac2cb6 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -24,15 +24,15 @@ type Config struct { // DatabaseConfig is the configuration for the database. type DatabaseConfig struct { - Endpoints []*connection.Endpoint `mapstructure:"endpoints"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Database string `mapstructure:"database"` - MaxConnections int32 `mapstructure:"max-connections"` - MinConnections int32 `mapstructure:"min-connections"` - LoadBalance bool `mapstructure:"load-balance"` - Retry *connection.RetryProfile `mapstructure:"retry"` - TLS connection.DatabaseTLS `mapstructure:"tls"` + Endpoints []*connection.Endpoint `mapstructure:"endpoints"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + MaxConnections int32 `mapstructure:"max-connections"` + MinConnections int32 `mapstructure:"min-connections"` + LoadBalance bool `mapstructure:"load-balance"` + Retry *connection.RetryProfile `mapstructure:"retry"` + TLS connection.DatabaseTLSConfig `mapstructure:"tls"` } // DataSourceName returns the data source name of the database. diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index 250bf330..ec07b684 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -44,7 +44,7 @@ type Connection struct { Password string Database string LoadBalance bool - TLS connection.DatabaseTLS + TLS connection.DatabaseTLSConfig } // NewConnection returns a connection parameters with the specified host:port, and the default values diff --git a/utils/connection/config.go b/utils/connection/config.go index ee63e813..bd410e5a 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -78,8 +78,8 @@ type ( CACertPaths []string `mapstructure:"ca-cert-paths"` } - // DatabaseTLS holds the database connection credentials. - DatabaseTLS struct { + // DatabaseTLSConfig holds the database connection credentials. + DatabaseTLSConfig struct { Activate bool `mapstructure:"activate"` CACertPath string `mapstructure:"ca-cert-path"` } @@ -97,7 +97,7 @@ const ( ) // UseTLS sets the option of using TLS configuration for database connection. -func (dc *DatabaseTLS) UseTLS() bool { +func (dc *DatabaseTLSConfig) UseTLS() bool { return dc.Activate && dc.CACertPath != "" } From 3f8c875b0a0ad8509fa5f3ef9f51ec9531e07ace Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Thu, 30 Oct 2025 18:20:08 +0200 Subject: [PATCH 05/19] * changed tls field to 'mode' - match the services tls configs. Signed-off-by: Dean Amar --- cmd/config/app_config_test.go | 2 +- cmd/config/samples/query.yaml | 2 +- cmd/config/samples/vc.yaml | 2 +- cmd/config/templates/query.yaml | 2 +- cmd/config/templates/vc.yaml | 2 +- docker/images/test_node/Dockerfile | 4 ++-- docker/test/container_release_image_test.go | 2 +- service/vc/config.go | 18 ++++++++++++----- service/vc/config_test.go | 8 ++++++-- service/vc/dbinit.go | 8 ++++++-- service/vc/dbtest/connection.go | 22 +++++++++++++++------ utils/connection/config.go | 7 +------ 12 files changed, 50 insertions(+), 29 deletions(-) diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index a32174ef..e513aafb 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -448,7 +448,7 @@ func defaultSampleDBConfig() *vc.DatabaseConfig { Password: "yugabyte", Database: "yugabyte", TLS: connection.DatabaseTLSConfig{ - Activate: true, + Mode: connection.OneSideTLSMode, CACertPath: "/server-certs/ca-certificate.crt", }, MaxConnections: 10, diff --git a/cmd/config/samples/query.yaml b/cmd/config/samples/query.yaml index 2c57e9f0..2ca5ecd4 100644 --- a/cmd/config/samples/query.yaml +++ b/cmd/config/samples/query.yaml @@ -24,7 +24,7 @@ database: password: "yugabyte" # The password for the database database: "yugabyte" # The database name tls: - activate: true + mode: tls ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool diff --git a/cmd/config/samples/vc.yaml b/cmd/config/samples/vc.yaml index 2f61054d..68c6ec52 100644 --- a/cmd/config/samples/vc.yaml +++ b/cmd/config/samples/vc.yaml @@ -23,7 +23,7 @@ database: password: "yugabyte" # The password for the database database: "yugabyte" # The database name tls: - activate: true + mode: tls ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool. diff --git a/cmd/config/templates/query.yaml b/cmd/config/templates/query.yaml index 6e538a2c..8b182924 100644 --- a/cmd/config/templates/query.yaml +++ b/cmd/config/templates/query.yaml @@ -28,7 +28,7 @@ database: database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} tls: - activate: {{ .DB.TLS.Activate }} + mode: {{ .DB.TLS.Mode }} ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 diff --git a/cmd/config/templates/vc.yaml b/cmd/config/templates/vc.yaml index 7d613627..678e1dcc 100644 --- a/cmd/config/templates/vc.yaml +++ b/cmd/config/templates/vc.yaml @@ -27,7 +27,7 @@ database: database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} tls: - activate: {{ .DB.TLS.Activate }} + mode: {{ .DB.TLS.Mode }} ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 diff --git a/docker/images/test_node/Dockerfile b/docker/images/test_node/Dockerfile index 189763fe..9f66ff52 100644 --- a/docker/images/test_node/Dockerfile +++ b/docker/images/test_node/Dockerfile @@ -36,8 +36,8 @@ ENV SC_VC_SERVER_TLS_MODE="none" ENV SC_VERIFIER_SERVER_TLS_MODE="none" # Disable TLS usage for db. -ENV SC_VC_DATABASE_TLS_ACTIVATE=false -ENV SC_QUERY_DATABASE_TLS_ACTIVATE=false +ENV SC_VC_DATABASE_TLS_MODE="none" +ENV SC_QUERY_DATABASE_TLS_MODE="none" COPY ${ARCHBIN_PATH}/${TARGETOS}-${TARGETARCH}/* ${BINS_PATH}/ COPY ./docker/images/test_node/run ${BINS_PATH}/ diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 7c0d73c4..59dde3c1 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -149,7 +149,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod // This is relevant if a different CA was used to issue the DB's TLS certificates. require.NotEmpty(t, node.TLSConfig.CACertPaths) conn.TLS = connection.DatabaseTLSConfig{ - Activate: true, + Mode: connection.OneSideTLSMode, CACertPath: node.TLSConfig.CACertPaths[0], } diff --git a/service/vc/config.go b/service/vc/config.go index a0ac2cb6..3f5a631b 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -10,6 +10,8 @@ import ( "fmt" "time" + "github.com/cockroachdb/errors" + "github.com/hyperledger/fabric-x-committer/utils/connection" "github.com/hyperledger/fabric-x-committer/utils/monitoring" ) @@ -36,25 +38,31 @@ type DatabaseConfig struct { } // DataSourceName returns the data source name of the database. -func (d *DatabaseConfig) DataSourceName() string { +func (d *DatabaseConfig) DataSourceName() (string, error) { ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", d.Username, d.Password, d.EndpointsString(), d.Database) - if d.TLS.UseTLS() { + switch d.TLS.Mode { + case connection.NoneTLSMode, connection.UnmentionedTLSMode: + ret += "sslmode=disable" + case connection.OneSideTLSMode: // Enforce full SSL verification: // requires an encrypted connection (TLS), // and ensures the server hostname matches the certificate. ret += "sslmode=verify-full" ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) - } else { - ret += "sslmode=disable" + case connection.MutualTLSMode: + return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) + default: + return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", + d.TLS.Mode, connection.NoneTLSMode, connection.OneSideTLSMode, connection.MutualTLSMode) } // The load balancing flag is only available when the server supports it (having multiple nodes). // Thus, we only add it when explicitly required. Otherwise, an error will occur. if d.LoadBalance { ret += "&load_balance=true" } - return ret + return ret, nil } // EndpointsString returns the address:port as a string with comma as a separator between endpoints. diff --git a/service/vc/config_test.go b/service/vc/config_test.go index dda4ac6c..3646eba1 100644 --- a/service/vc/config_test.go +++ b/service/vc/config_test.go @@ -28,7 +28,9 @@ func TestDatasourceName(t *testing.T) { } e1 := "postgres://yugabyte_user:yugabyte_pass@localhost:5433/" + "yugabyte_db?sslmode=disable" - require.Equal(t, e1, c1.DataSourceName()) + connString, err := c1.DataSourceName() + require.NoError(t, err) + require.Equal(t, e1, connString) c2 := &DatabaseConfig{ Endpoints: []*connection.Endpoint{ @@ -42,5 +44,7 @@ func TestDatasourceName(t *testing.T) { } //nolint:gosec // fake password. e2 := "postgres://yugabyte:yugabyte@host1:1111,host2:2222/yugabyte?sslmode=disable&load_balance=true" - require.Equal(t, e2, c2.DataSourceName()) + connString, err = c2.DataSourceName() + require.NoError(t, err) + require.Equal(t, e2, connString) } diff --git a/service/vc/dbinit.go b/service/vc/dbinit.go index 37320b6a..001f2611 100644 --- a/service/vc/dbinit.go +++ b/service/vc/dbinit.go @@ -43,8 +43,12 @@ var ( // NewDatabasePool creates a new pool from a database config. func NewDatabasePool(ctx context.Context, config *DatabaseConfig) (*pgxpool.Pool, error) { - logger.Infof("DB source: %s", config.DataSourceName()) - poolConfig, err := pgxpool.ParseConfig(config.DataSourceName()) + connString, err := config.DataSourceName() + if err != nil { + return nil, errors.Wrapf(err, "could not build database connection string") + } + logger.Infof("DB source: %s", connString) + poolConfig, err := pgxpool.ParseConfig(connString) if err != nil { return nil, errors.Wrapf(err, "failed parsing datasource") } diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index ec07b684..d36f3cc2 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -58,25 +58,31 @@ func NewConnection(endpoints ...*connection.Endpoint) *Connection { } // dataSourceName returns the dataSourceName to be used by the database/sql package. -func (c *Connection) dataSourceName() string { +func (c *Connection) dataSourceName() (string, error) { ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", c.User, c.Password, c.endpointsString(), c.Database) - if c.TLS.UseTLS() { + switch c.TLS.Mode { + case connection.NoneTLSMode, connection.UnmentionedTLSMode: + ret += "sslmode=disable" + case connection.OneSideTLSMode: // Enforce full SSL verification: // requires an encrypted connection (TLS), // and ensures the server hostname matches the certificate. ret += "sslmode=verify-full" ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) - } else { - ret += "sslmode=disable" + case connection.MutualTLSMode: + return "", errors.Newf("unsupportted db tls mode: %s", c.TLS.Mode) + default: + return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", + c.TLS.Mode, connection.NoneTLSMode, connection.OneSideTLSMode, connection.MutualTLSMode) } // The load balancing flag is only available when the server supports it (having multiple nodes). // Thus, we only add it when explicitly required. Otherwise, an error will occur. if c.LoadBalance { ret += "&load_balance=true" } - return ret + return ret, nil } // endpointsString returns the address:port as a string with comma as a separator between endpoints. @@ -86,7 +92,11 @@ func (c *Connection) endpointsString() string { // open opens a connection pool to the database. func (c *Connection) open(ctx context.Context) (*pgxpool.Pool, error) { - poolConfig, err := pgxpool.ParseConfig(c.dataSourceName()) + connString, err := c.dataSourceName() + if err != nil { + return nil, errors.Wrapf(err, "could not build database connection string") + } + poolConfig, err := pgxpool.ParseConfig(connString) if err != nil { return nil, errors.Wrapf(err, "error parsing datasource: %s", c.endpointsString()) } diff --git a/utils/connection/config.go b/utils/connection/config.go index bd410e5a..f1f8033d 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -80,7 +80,7 @@ type ( // DatabaseTLSConfig holds the database connection credentials. DatabaseTLSConfig struct { - Activate bool `mapstructure:"activate"` + Mode string `mapstructure:"mode"` CACertPath string `mapstructure:"ca-cert-path"` } ) @@ -96,11 +96,6 @@ const ( DefaultTLSMinVersion = tls.VersionTLS12 ) -// UseTLS sets the option of using TLS configuration for database connection. -func (dc *DatabaseTLSConfig) UseTLS() bool { - return dc.Activate && dc.CACertPath != "" -} - // ServerCredentials returns the gRPC transport credentials to be used by a server, // based on the provided TLS configuration. func (c TLSConfig) ServerCredentials() (credentials.TransportCredentials, error) { From 61b0fc51ac25bac5e81cce9d542af306ff95978a Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Thu, 30 Oct 2025 19:21:28 +0200 Subject: [PATCH 06/19] * improved code readability. Signed-off-by: Dean Amar --- service/vc/config.go | 3 +-- service/vc/dbtest/connection.go | 3 +-- utils/connection/config.go | 2 +- utils/test/secure_connection.go | 30 +++++++++--------------------- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/service/vc/config.go b/service/vc/config.go index 3f5a631b..bbb2aaaf 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -49,8 +49,7 @@ func (d *DatabaseConfig) DataSourceName() (string, error) { // Enforce full SSL verification: // requires an encrypted connection (TLS), // and ensures the server hostname matches the certificate. - ret += "sslmode=verify-full" - ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) + ret += fmt.Sprintf("sslmode=%s&sslrootcert=%s", "verify-full", d.TLS.CACertPath) case connection.MutualTLSMode: return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) default: diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index d36f3cc2..de1d7fa0 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -69,8 +69,7 @@ func (c *Connection) dataSourceName() (string, error) { // Enforce full SSL verification: // requires an encrypted connection (TLS), // and ensures the server hostname matches the certificate. - ret += "sslmode=verify-full" - ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) + ret += fmt.Sprintf("sslmode=%s&sslrootcert=%s", "verify-full", c.TLS.CACertPath) case connection.MutualTLSMode: return "", errors.Newf("unsupportted db tls mode: %s", c.TLS.Mode) default: diff --git a/utils/connection/config.go b/utils/connection/config.go index f1f8033d..cc89c47f 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -78,7 +78,7 @@ type ( CACertPaths []string `mapstructure:"ca-cert-paths"` } - // DatabaseTLSConfig holds the database connection credentials. + // DatabaseTLSConfig holds the database TLS mode and its necessary credentials. DatabaseTLSConfig struct { Mode string `mapstructure:"mode"` CACertPath string `mapstructure:"ca-cert-path"` diff --git a/utils/test/secure_connection.go b/utils/test/secure_connection.go index 9bf41cbe..826a268c 100644 --- a/utils/test/secure_connection.go +++ b/utils/test/secure_connection.go @@ -36,12 +36,6 @@ type ( shouldFail bool } - createTLSConfigParameters struct { - connectionMode string - keyPair *tlsgen.CertKeyPair - namingStyle string - } - // ServerStarter is a function that receives a TLS configuration, starts the server, // and returns a RPCAttempt function for initiating a client connection and attempting an RPC call. ServerStarter func(t *testing.T, serverTLS connection.TLSConfig) RPCAttempt @@ -90,11 +84,7 @@ func (scm *CredentialsFactory) CreateServerCredentials( t.Helper() serverKeypair, err := scm.CertificateAuthority.NewServerCertKeyPair(san...) require.NoError(t, err) - return scm.createTLSConfig(t, createTLSConfigParameters{ - connectionMode: tlsMode, - keyPair: serverKeypair, - namingStyle: namingStyle, - }) + return scm.createTLSConfig(t, tlsMode, serverKeypair, namingStyle) } // CreateClientCredentials creates a client key pair, @@ -103,11 +93,7 @@ func (scm *CredentialsFactory) CreateClientCredentials(t *testing.T, tlsMode str t.Helper() clientKeypair, err := scm.CertificateAuthority.NewClientCertKeyPair() require.NoError(t, err) - return scm.createTLSConfig(t, createTLSConfigParameters{ - connectionMode: tlsMode, - keyPair: clientKeypair, - namingStyle: DefaultCertStyle, - }) + return scm.createTLSConfig(t, tlsMode, clientKeypair, DefaultCertStyle) } /* @@ -216,23 +202,25 @@ func CreateClientWithTLS[T any]( // given TLS mode and credential bytes, and returns it along with the certificates' path. func (scm *CredentialsFactory) createTLSConfig( t *testing.T, - params createTLSConfigParameters, + connectionMode string, + keyPair *tlsgen.CertKeyPair, + namingStyle string, ) (connection.TLSConfig, string) { t.Helper() tmpDir := t.TempDir() - namingFunction := selectFileNames(params.namingStyle) + namingFunction := selectFileNames(namingStyle) privateKeyPath := filepath.Join(tmpDir, namingFunction("private-key")) - require.NoError(t, os.WriteFile(privateKeyPath, params.keyPair.Key, 0o600)) + require.NoError(t, os.WriteFile(privateKeyPath, keyPair.Key, 0o600)) publicKeyPath := filepath.Join(tmpDir, namingFunction("public-key")) - require.NoError(t, os.WriteFile(publicKeyPath, params.keyPair.Cert, 0o600)) + require.NoError(t, os.WriteFile(publicKeyPath, keyPair.Cert, 0o600)) caCertificatePath := filepath.Join(tmpDir, namingFunction("ca-certificate")) require.NoError(t, os.WriteFile(caCertificatePath, scm.CertificateAuthority.CertBytes(), 0o600)) return connection.TLSConfig{ - Mode: params.connectionMode, + Mode: connectionMode, KeyPath: privateKeyPath, CertPath: publicKeyPath, CACertPaths: []string{caCertificatePath}, From dd329ec5cb2ddfe8fdd9e9d1956f2dfd8c9caa5b Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 3 Nov 2025 21:01:11 +0200 Subject: [PATCH 07/19] * Addressed PR comments. Signed-off-by: Dean Amar --- cmd/config/samples/coordinator.yaml | 12 +- cmd/config/samples/loadgen.yaml | 12 +- cmd/config/samples/query.yaml | 8 +- cmd/config/samples/sidecar.yaml | 12 +- cmd/config/samples/vc.yaml | 8 +- cmd/config/samples/verifier.yaml | 6 +- docker/test/container_release_image_test.go | 169 ++++++++++---------- integration/runner/postgres.go | 4 +- integration/runner/runtime.go | 7 +- integration/runner/yugabyte.go | 4 +- loadgen/client_test.go | 2 +- service/vc/config.go | 34 +--- service/vc/dbtest/connection.go | 32 +--- service/vc/dbtest/container.go | 117 +++++++++----- utils/connection/client_util.go | 37 +++++ utils/test/secure_connection.go | 76 ++------- utils/test/utils.go | 21 ++- 17 files changed, 272 insertions(+), 289 deletions(-) diff --git a/cmd/config/samples/coordinator.yaml b/cmd/config/samples/coordinator.yaml index 4bc70ed0..3aa573aa 100644 --- a/cmd/config/samples/coordinator.yaml +++ b/cmd/config/samples/coordinator.yaml @@ -6,10 +6,10 @@ server: endpoint: :9001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem monitoring: server: endpoint: :2119 @@ -19,10 +19,10 @@ verifier: - verifier:5001 tls: &ClientTLS mode: mtls - cert-path: /client-certs/public-key.crt - key-path: /client-certs/private-key.key + cert-path: /client-certs/public-key.pem + key-path: /client-certs/private-key.pem ca-cert-paths: - - /client-certs/ca-certificate.crt + - /client-certs/ca-certificate.pem validator-committer: endpoints: - vc:6001 diff --git a/cmd/config/samples/loadgen.yaml b/cmd/config/samples/loadgen.yaml index 4fef972b..386c4d90 100644 --- a/cmd/config/samples/loadgen.yaml +++ b/cmd/config/samples/loadgen.yaml @@ -6,10 +6,10 @@ server: endpoint: :8001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem monitoring: server: endpoint: :2118 @@ -26,10 +26,10 @@ orderer-client: endpoint: sidecar:4001 tls: mode: mtls - cert-path: /client-certs/public-key.crt - key-path: /client-certs/private-key.key + cert-path: /client-certs/public-key.pem + key-path: /client-certs/private-key.pem ca-cert-paths: - - /client-certs/ca-certificate.crt + - /client-certs/ca-certificate.pem orderer: connection: endpoints: diff --git a/cmd/config/samples/query.yaml b/cmd/config/samples/query.yaml index 2ca5ecd4..8e50d4ce 100644 --- a/cmd/config/samples/query.yaml +++ b/cmd/config/samples/query.yaml @@ -8,10 +8,10 @@ server: endpoint: :7001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem monitoring: server: endpoint: :2117 @@ -25,7 +25,7 @@ database: database: "yugabyte" # The database name tls: mode: tls - ca-cert-path: /server-certs/ca-certificate.crt + ca-cert-path: /server-certs/ca-certificate.pem max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool load-balance: false # Should be enabled for DB cluster diff --git a/cmd/config/samples/sidecar.yaml b/cmd/config/samples/sidecar.yaml index ba1d5690..f08ed201 100644 --- a/cmd/config/samples/sidecar.yaml +++ b/cmd/config/samples/sidecar.yaml @@ -6,10 +6,10 @@ server: endpoint: :4001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem keep-alive: params: time: 300s @@ -40,10 +40,10 @@ committer: endpoint: coordinator:9001 tls: mode: mtls - cert-path: /client-certs/public-key.crt - key-path: /client-certs/private-key.key + cert-path: /client-certs/public-key.pem + key-path: /client-certs/private-key.pem ca-cert-paths: - - /client-certs/ca-certificate.crt + - /client-certs/ca-certificate.pem ledger: path: /root/sc/ledger notification: diff --git a/cmd/config/samples/vc.yaml b/cmd/config/samples/vc.yaml index 68c6ec52..7bcc780c 100644 --- a/cmd/config/samples/vc.yaml +++ b/cmd/config/samples/vc.yaml @@ -8,10 +8,10 @@ server: endpoint: :6001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem monitoring: server: endpoint: :2116 @@ -24,7 +24,7 @@ database: database: "yugabyte" # The database name tls: mode: tls - ca-cert-path: /server-certs/ca-certificate.crt + ca-cert-path: /server-certs/ca-certificate.pem max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool. load-balance: false # Should be enabled for DB cluster. diff --git a/cmd/config/samples/verifier.yaml b/cmd/config/samples/verifier.yaml index 7f48b981..68c6330c 100644 --- a/cmd/config/samples/verifier.yaml +++ b/cmd/config/samples/verifier.yaml @@ -6,10 +6,10 @@ server: endpoint: :5001 tls: mode: mtls - cert-path: /server-certs/public-key.crt - key-path: /server-certs/private-key.key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate.crt + - /server-certs/ca-certificate.pem monitoring: server: endpoint: :2115 diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 59dde3c1..e224580e 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -36,6 +36,12 @@ type startNodeParameters struct { dbPassword string } +func (p *startNodeParameters) asNode(node string) startNodeParameters { + params := *p + params.node = node + return params +} + const ( committerReleaseImage = "icr.io/cbdc/committer:0.0.2" loadgenReleaseImage = "icr.io/cbdc/loadgen:0.0.2" @@ -53,20 +59,6 @@ const ( containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec ) -var ( - // enforcePostgresSSLScript enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. - enforcePostgresSSLScript = []string{ - "sh", "-c", - `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + - `/var/lib/postgresql/data/pg_hba.conf`, - } - - // reloadPostgresConfigScript reloads the PostgreSQL server configuration without restarting the instance. - reloadPostgresConfigScript = []string{ - "psql", "-U", "yugabyte", "-c", "SELECT pg_reload_conf();", - } -) - // TestCommitterReleaseImagesWithTLS runs the committer components in different Docker containers with different TLS // modes and verifies it starts and connect successfully. // This test uses the release images for all the components but 'db' and 'orderer'. @@ -83,74 +75,87 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { require.NoError(t, err) require.NoError(t, configtxgen.WriteOutputBlock(configBlock, configBlockPath)) + dbNode := "db" + ordererNode := "orderer" + loadgenNode := "loadgen" + committerNodes := []string{"verifier", "vc", "query", "coordinator", "sidecar"} + credsFactory := testutils.NewCredentialsFactory(t) - for _, mode := range testutils.ServerModes { - t.Run(fmt.Sprintf("tls-mode:%s", mode), func(t *testing.T) { + for _, dbType := range []string{testutils.YugaDBType, testutils.PostgresDBType} { + t.Run(fmt.Sprintf("database:%s", dbType), func(t *testing.T) { t.Parallel() - // Create an isolated network for each test with different tls mode. - networkName := fmt.Sprintf("%s_%s", networkPrefixName, uuid.NewString()) - testutils.CreateDockerNetwork(t, networkName) - t.Cleanup(func() { - testutils.RemoveDockerNetwork(t, networkName) - }) - - params := startNodeParameters{ - credsFactory: credsFactory, - networkName: networkName, - tlsMode: mode, - configBlockPath: configBlockPath, - dbType: testutils.YugaDBType, - } - - for _, node := range []string{ - "db", "verifier", "vc", "query", "coordinator", "sidecar", "orderer", "loadgen", - } { - params.node = node - // stop and remove the container if it already exists. - stopAndRemoveContainersByName(ctx, t, createDockerClient(t), assembleContainerName(node, mode)) - - switch node { - case "db": - params.dbPassword = startSecuredDatabaseNode(ctx, t, params).Password - case "orderer": - startCommitterNodeWithTestImage(ctx, t, params) - case "loadgen": - startLoadgenNodeWithReleaseImage(ctx, t, params) - default: - startCommitterNodeWithReleaseImage(ctx, t, params) - } + for _, mode := range testutils.ServerModes { + t.Run(fmt.Sprintf("tls-mode:%s", mode), func(t *testing.T) { + t.Parallel() + // Create an isolated network for each test with different tls mode. + networkName := fmt.Sprintf("%s_%s", networkPrefixName, uuid.NewString()) + testutils.CreateDockerNetwork(t, networkName) + t.Cleanup(func() { + testutils.RemoveDockerNetwork(t, networkName) + }) + + params := startNodeParameters{ + credsFactory: credsFactory, + networkName: networkName, + tlsMode: mode, + configBlockPath: configBlockPath, + dbType: dbType, + } + + for _, node := range append(committerNodes, dbNode, ordererNode, loadgenNode) { + // stop and remove the container if it already exists. + stopAndRemoveContainersByName( + ctx, t, createDockerClient(t), assembleContainerName(node, mode, dbType), + ) + } + + // start a secured database node and return the db password. + params.dbPassword = startSecuredDatabaseNode(ctx, t, params.asNode(dbNode)) + // start the orderer node. + startCommitterNodeWithTestImage(ctx, t, params.asNode(ordererNode)) + // start the committer nodes. + for _, node := range committerNodes { + startCommitterNodeWithReleaseImage(ctx, t, params.asNode(node)) + } + // start the load generator node. + startLoadgenNodeWithReleaseImage(ctx, t, params.asNode(loadgenNode)) + + monitorMetric(t, + getContainerMappedHostPort( + ctx, t, assembleContainerName("loadgen", mode, dbType), loadGenMetricsPort, + ), + ) + }) } - monitorMetric(t, - getContainerMappedHostPort(ctx, t, assembleContainerName("loadgen", mode), loadGenMetricsPort), - ) }) } } // CreateAndStartSecuredDatabaseNode creates a containerized YugabyteDB or PostgreSQL // database instance in a secure mode. -func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNodeParameters) *dbtest.Connection { +func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNodeParameters) string { t.Helper() - tlsConfig, credsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.dbType, params.node) + tlsConfig, _ := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.node) node := &dbtest.DatabaseContainer{ DatabaseType: params.dbType, Network: params.networkName, Hostname: params.node, - TLSConfig: tlsConfig, - CredsPathDir: credsPath, - UseTLS: true, + TLSConfig: &tlsConfig, } node.StartContainer(ctx, t) + t.Cleanup(func() { + node.StopAndRemoveContainer(t) + }) conn := node.GetConnectionOptions(ctx, t) // This is relevant if a different CA was used to issue the DB's TLS certificates. - require.NotEmpty(t, node.TLSConfig.CACertPaths) + require.NotEmpty(t, tlsConfig.CACertPaths) conn.TLS = connection.DatabaseTLSConfig{ Mode: connection.OneSideTLSMode, - CACertPath: node.TLSConfig.CACertPaths[0], + CACertPath: tlsConfig.CACertPaths[0], } // post start container tweaking @@ -158,32 +163,32 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod case testutils.YugaDBType: // Ensure proper root ownership and permissions for the TLS certificate files. node.ExecuteCommand(t, []string{ - "chown", "root:root", - fmt.Sprintf("/creds/node.%s.crt", node.Hostname), - fmt.Sprintf("/creds/node.%s.key", node.Hostname), + "chown", + "root:root", + filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), + filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), }) - node.EnsureNodeReadiness(t, "Data placement constraint successfully verified") + node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - // Ensure proper root ownership and permissions for the TLS certificate files. + node.EnsurePostgresNodeReadiness(t) node.ExecuteCommand(t, []string{ - "chown", "postgres:postgres", - "/creds/server.crt", - "/creds/server.key", + "sh", "-c", + `// Ensure proper root ownership and permissions for the TLS certificate files. + chown postgres:postgres /creds/server.crt /creds/server.key + + // Enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. + sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' + /var/lib/postgresql/data/pg_hba.conf + + // Reloads the PostgreSQL server configuration without restarting the instance. + psql -U yugabyte -c SELECT pg_reload_conf();`, }) - node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) - node.ExecuteCommand(t, enforcePostgresSSLScript) - node.ExecuteCommand(t, reloadPostgresConfigScript) default: t.Fatalf("Unsupported database type: %s", node.DatabaseType) } - t.Cleanup( - func() { - node.StopAndRemoveContainer(t) - }) - - return conn + return conn.Password } // startCommitterNodeWithReleaseImage starts a committer node using the release image. @@ -223,7 +228,7 @@ func startCommitterNodeWithReleaseImage(ctx context.Context, t *testing.T, param ), ), }, - name: assembleContainerName(params.node, params.tlsMode), + name: assembleContainerName(params.node, params.tlsMode, params.dbType), }) } @@ -269,7 +274,7 @@ func startLoadgenNodeWithReleaseImage( ), ), }, - name: assembleContainerName(params.node, params.tlsMode), + name: assembleContainerName(params.node, params.tlsMode, params.dbType), }) } @@ -294,20 +299,18 @@ func startCommitterNodeWithTestImage( fmt.Sprintf("%s:/%s", params.configBlockPath, filepath.Join(containerConfigPath, genBlockFile)), ), }, - name: assembleContainerName(params.node, params.tlsMode), + name: assembleContainerName(params.node, params.tlsMode, params.dbType), }) } -func assembleContainerName(node, tlsMode string) string { - return fmt.Sprintf("%s_%s_%s", containerPrefixName, node, tlsMode) +func assembleContainerName(node, tlsMode, dbType string) string { + return fmt.Sprintf("%s_%s_%s_%s", containerPrefixName, node, tlsMode, dbType) } func assembleBinds(t *testing.T, params startNodeParameters, additionalBinds ...string) []string { t.Helper() - _, serverCredsPath := params.credsFactory.CreateServerCredentials( - t, params.tlsMode, testutils.DefaultCertStyle, params.node, - ) + _, serverCredsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.node) require.NotEmpty(t, serverCredsPath) _, clientCredsPath := params.credsFactory.CreateClientCredentials(t, params.tlsMode) require.NotEmpty(t, clientCredsPath) diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index afb6798d..9f1afa34 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -100,7 +100,7 @@ func (cc *PostgresClusterController) addPrimaryNode(ctx context.Context, t *test }, }) node.StartContainer(ctx, t) - node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) + node.EnsurePostgresNodeReadiness(t) return node } @@ -121,7 +121,7 @@ func (cc *PostgresClusterController) addSecondaryNode(ctx context.Context, t *te }, }) node.StartContainer(ctx, t) - node.EnsureNodeReadiness(t, "started streaming WAL from primary") + node.EnsurePostgresNodeReadiness(t) return node } diff --git a/integration/runner/runtime.go b/integration/runner/runtime.go index 8676ac0e..c551adac 100644 --- a/integration/runner/runtime.go +++ b/integration/runner/runtime.go @@ -583,12 +583,7 @@ func (c *CommitterRuntime) createSystemConfigWithServerTLS( ) *config.SystemConfig { t.Helper() serviceCfg := c.SystemConfig - serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials( - t, - c.config.TLSMode, - test.DefaultCertStyle, - endpoints.Server.Host, - ) + serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials(t, c.config.TLSMode, endpoints.Server.Host) serviceCfg.ServiceEndpoints = endpoints return &serviceCfg } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index c8761900..83556f43 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -161,7 +161,7 @@ func (cc *YugaClusterController) startNodes(ctx context.Context, t *testing.T) { } for _, n := range cc.IterNodesByRole(TabletNode) { - n.EnsureNodeReadiness(t, "syncing data to disk ... ok") + n.EnsureNodeReadinessByLogs(t, "syncing data to disk ... ok") } } @@ -194,7 +194,7 @@ func (cc *YugaClusterController) listAllMasters(t *testing.T) string { "list_all_masters", } for _, n := range cc.nodes { - if output = n.ExecuteCommand(t, cmd); output != "" { + if output, _ = n.ExecuteCommand(t, cmd); output != "" { break } } diff --git a/loadgen/client_test.go b/loadgen/client_test.go index 8afa7074..4d39c427 100644 --- a/loadgen/client_test.go +++ b/loadgen/client_test.go @@ -453,7 +453,7 @@ func createServerAndClientTLSConfig(t *testing.T, tlsMode string) ( ) { t.Helper() credsFactory := test.NewCredentialsFactory(t) - clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, test.DefaultCertStyle, defaultServerSAN) + clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, defaultServerSAN) serverTLSConfig, _ = credsFactory.CreateClientCredentials(t, tlsMode) return clientTLSConfig, serverTLSConfig } diff --git a/service/vc/config.go b/service/vc/config.go index bbb2aaaf..8a37fc5e 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -7,11 +7,8 @@ SPDX-License-Identifier: Apache-2.0 package vc import ( - "fmt" "time" - "github.com/cockroachdb/errors" - "github.com/hyperledger/fabric-x-committer/utils/connection" "github.com/hyperledger/fabric-x-committer/utils/monitoring" ) @@ -39,29 +36,14 @@ type DatabaseConfig struct { // DataSourceName returns the data source name of the database. func (d *DatabaseConfig) DataSourceName() (string, error) { - ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", - d.Username, d.Password, d.EndpointsString(), d.Database) - - switch d.TLS.Mode { - case connection.NoneTLSMode, connection.UnmentionedTLSMode: - ret += "sslmode=disable" - case connection.OneSideTLSMode: - // Enforce full SSL verification: - // requires an encrypted connection (TLS), - // and ensures the server hostname matches the certificate. - ret += fmt.Sprintf("sslmode=%s&sslrootcert=%s", "verify-full", d.TLS.CACertPath) - case connection.MutualTLSMode: - return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) - default: - return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", - d.TLS.Mode, connection.NoneTLSMode, connection.OneSideTLSMode, connection.MutualTLSMode) - } - // The load balancing flag is only available when the server supports it (having multiple nodes). - // Thus, we only add it when explicitly required. Otherwise, an error will occur. - if d.LoadBalance { - ret += "&load_balance=true" - } - return ret, nil + return connection.DataSourceName(connection.DataSourceNameParams{ + Username: d.Username, + Password: d.Password, + Database: d.Database, + EndpointsString: d.EndpointsString(), + LoadBalance: d.LoadBalance, + TLS: d.TLS, + }) } // EndpointsString returns the address:port as a string with comma as a separator between endpoints. diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index de1d7fa0..29818a87 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -8,7 +8,6 @@ package dbtest import ( "context" - "fmt" "math/rand" "time" @@ -59,29 +58,14 @@ func NewConnection(endpoints ...*connection.Endpoint) *Connection { // dataSourceName returns the dataSourceName to be used by the database/sql package. func (c *Connection) dataSourceName() (string, error) { - ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", - c.User, c.Password, c.endpointsString(), c.Database) - - switch c.TLS.Mode { - case connection.NoneTLSMode, connection.UnmentionedTLSMode: - ret += "sslmode=disable" - case connection.OneSideTLSMode: - // Enforce full SSL verification: - // requires an encrypted connection (TLS), - // and ensures the server hostname matches the certificate. - ret += fmt.Sprintf("sslmode=%s&sslrootcert=%s", "verify-full", c.TLS.CACertPath) - case connection.MutualTLSMode: - return "", errors.Newf("unsupportted db tls mode: %s", c.TLS.Mode) - default: - return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", - c.TLS.Mode, connection.NoneTLSMode, connection.OneSideTLSMode, connection.MutualTLSMode) - } - // The load balancing flag is only available when the server supports it (having multiple nodes). - // Thus, we only add it when explicitly required. Otherwise, an error will occur. - if c.LoadBalance { - ret += "&load_balance=true" - } - return ret, nil + return connection.DataSourceName(connection.DataSourceNameParams{ + Username: c.User, + Password: c.Password, + Database: c.Database, + EndpointsString: c.endpointsString(), + LoadBalance: c.LoadBalance, + TLS: c.TLS, + }) } // endpointsString returns the address:port as a string with comma as a separator between endpoints. diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index bb9199b6..519be96e 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -7,11 +7,11 @@ SPDX-License-Identifier: Apache-2.0 package dbtest import ( - "bufio" "bytes" "context" "fmt" "os" + "path/filepath" "regexp" "strconv" "strings" @@ -40,23 +40,35 @@ const ( gb = 1 << 30 // gb is the number of bytes needed to represent 1 GB. memorySwap = -1 // memorySwap disable memory swaps (don't store data on disk) - // PostgresReadinessOutput is the output indicating that a Postgres node is ready. - PostgresReadinessOutput = "database system is ready to accept connections" + // YugabytedReadinessOutput is the output indicating that a Yugabyted node is ready. + YugabytedReadinessOutput = "Data placement constraint successfully verified" + + YugabytePublicKeyFileName = "node.db.crt" + YugabytePrivateKeyFileName = "node.db.key" + YugabyteCACertificateFileName = "ca.crt" + PostgresPublicKeyFileName = "server.crt" + PostgresPrivateKeyFileName = "server.key" ) -// YugabyteCMD starts yugabyte without fault tolerance (single server). -var YugabyteCMD = []string{ - "bin/yugabyted", "start", - "--callhome", "false", - "--background", "false", - "--ui", "false", - "--tserver_flags", - "ysql_max_connections=500," + - "tablet_replicas_per_gib_limit=4000," + - "yb_num_shards_per_tserver=1," + - "minloglevel=3," + - "yb_enable_read_committed_isolation=true", -} +var ( + // YugabyteCMD starts yugabyte without fault tolerance (single server). + YugabyteCMD = []string{ + "bin/yugabyted", "start", + "--callhome", "false", + "--background", "false", + "--ui", "false", + "--tserver_flags", + "ysql_max_connections=500," + + "tablet_replicas_per_gib_limit=4000," + + "yb_num_shards_per_tserver=1," + + "minloglevel=3," + + "yb_enable_read_committed_isolation=true", + } + + // passwordRegex is the compiled regular expression. + // to efficiently extract the password of the Yugabyted node. + passwordRegex = regexp.MustCompile(`(?i)(?m)^password:\s*(.+)$`) +) // DatabaseContainer manages the execution of an instance of a dockerized DB for tests. type DatabaseContainer struct { @@ -68,7 +80,6 @@ type DatabaseContainer struct { DatabaseType string Tag string Role string - CredsPathDir string Cmd []string Env []string Binds []string @@ -78,8 +89,9 @@ type DatabaseContainer struct { PortBinds map[docker.Port][]docker.PortBinding NetToIP map[string]*docker.EndpointConfig AutoRm bool - UseTLS bool - TLSConfig connection.TLSConfig + // TLSConfig holds the node TLS certificates. + // If TLSConfig isn't available (is nil), we fallback to insecure mode. + TLSConfig *connection.TLSConfig client *docker.Client containerID string @@ -117,7 +129,8 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.Cmd == nil { dc.Cmd = YugabyteCMD - if dc.UseTLS { + if dc.TLSConfig != nil { + require.NotEmpty(t, dc.Hostname) dc.Cmd = append(dc.Cmd, "--secure", "--certs_dir=/creds", "--advertise_address", dc.Hostname) } else { dc.Cmd = append(dc.Cmd, "--insecure") @@ -127,6 +140,14 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", yugaDBPort)) } + if dc.TLSConfig != nil { + require.NotEmpty(t, dc.TLSConfig.CACertPaths) + dc.Binds = append(dc.Binds, + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), + ) + } case test.PostgresDBType: if dc.Image == "" { dc.Image = defaultPostgresImage @@ -142,11 +163,18 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", postgresDBPort)) } - if dc.UseTLS { + if dc.TLSConfig != nil { + dc.Binds = append(dc.Binds, + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), + ) dc.Cmd = []string{ + // Configure PostgreSQL to run in secure mode + // and listen for incoming connections on port 5433. + "-c", "port=5433", "-c", "ssl=on", - "-c", "ssl_cert_file=/creds/server.crt", - "-c", "ssl_key_file=/creds/server.key", + "-c", fmt.Sprintf("ssl_cert_file=%s", filepath.Join("/creds", PostgresPublicKeyFileName)), + "-c", fmt.Sprintf("ssl_key_file=%s", filepath.Join("/creds", PostgresPrivateKeyFileName)), } } default: @@ -176,8 +204,7 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.client == nil { dc.client = test.GetDockerClient(t) } - if dc.UseTLS { - dc.Binds = append(dc.Binds, fmt.Sprintf("%s:/creds", dc.CredsPathDir)) + if dc.TLSConfig != nil { dc.Name += fmt.Sprintf("_with_tls_%s", uuid.NewString()[0:8]) } } @@ -351,26 +378,17 @@ func (dc *DatabaseContainer) ContainerID() string { // If no password is found, the default one will be returned. func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { t.Helper() - output := dc.ExecuteCommand(t, []string{"cat", filePath}) - - scanner := bufio.NewScanner(strings.NewReader(output)) - re := regexp.MustCompile(`(?i)^password:\s*(.+)$`) - - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if matches := re.FindStringSubmatch(line); len(matches) == 2 { - return matches[1] - } + output, _ := dc.ExecuteCommand(t, []string{"cat", filePath}) + found := passwordRegex.FindStringSubmatch(output) + if len(found) > 1 { + return found[1] } - - require.NoError(t, scanner.Err(), "error scanning command output") t.Log("password not found in output, returning default password.") - return defaultPassword } // ExecuteCommand executes a command and returns the container output. -func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { +func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) (string, int) { t.Helper() require.NotNil(t, dc.client) t.Logf("executing %s", strings.Join(cmd, " ")) @@ -389,16 +407,29 @@ func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { inspect, err := dc.client.InspectExec(exec.ID) require.NoError(t, err) - require.Equal(t, 0, inspect.ExitCode) - return stdout.String() + return stdout.String(), inspect.ExitCode } -// EnsureNodeReadiness checks the container's readiness by monitoring its logs and ensure its running correctly. -func (dc *DatabaseContainer) EnsureNodeReadiness(t *testing.T, requiredOutput string) { +// EnsureNodeReadinessByLogs checks the container's readiness by monitoring its logs and ensure its running correctly. +func (dc *DatabaseContainer) EnsureNodeReadinessByLogs(t *testing.T, requiredOutput string) { t.Helper() require.EventuallyWithT(t, func(ct *assert.CollectT) { output := dc.GetContainerLogs(t) require.Contains(ct, output, requiredOutput) }, 45*time.Second, 250*time.Millisecond) } + +// EnsurePostgresNodeReadiness verifies that the PostgreSQL node +// is ready to accept connections. +// It repeatedly executes `pg_isready` until the command +// returns a successful exit code (0) or the timeout is reached. +func (dc *DatabaseContainer) EnsurePostgresNodeReadiness(t *testing.T) { + t.Helper() + require.EventuallyWithT(t, func(ct *assert.CollectT) { + _, exitCode := dc.ExecuteCommand(t, []string{ + "bash", "-c", "pg_isready -U yugabyte -d yugabyte -h localhost -p 5433", + }) + require.Equal(ct, 0, exitCode) + }, 45*time.Second, 250*time.Millisecond) +} diff --git a/utils/connection/client_util.go b/utils/connection/client_util.go index 9bb57404..2c1b7f9e 100644 --- a/utils/connection/client_util.go +++ b/utils/connection/client_util.go @@ -61,6 +61,16 @@ type ( Resolver *manual.Resolver AdditionalOpts []grpc.DialOption } + + // DataSourceNameParams defines the parameters required to construct a database connection string. + DataSourceNameParams struct { + Username string + Password string + Database string + EndpointsString string + LoadBalance bool + TLS DatabaseTLSConfig + } ) var logger = logging.New("connection") @@ -282,3 +292,30 @@ func AddressString[T WithAddress](addresses ...T) string { } return strings.Join(listOfAddresses, ",") } + +// DataSourceName returns the data source name of the database. +func DataSourceName(d DataSourceNameParams) (string, error) { + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", + d.Username, d.Password, d.EndpointsString, d.Database) + + switch d.TLS.Mode { + case NoneTLSMode, UnmentionedTLSMode: + ret += "sslmode=disable" + case OneSideTLSMode: + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. + ret += fmt.Sprintf("sslmode=verify-full&sslrootcert=%s", d.TLS.CACertPath) + case MutualTLSMode: + return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) + default: + return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", + d.TLS.Mode, NoneTLSMode, OneSideTLSMode, MutualTLSMode) + } + // The load balancing flag is only available when the server supports it (having multiple nodes). + // Thus, we only add it when explicitly required. Otherwise, an error will occur. + if d.LoadBalance { + ret += "&load_balance=true" + } + return ret, nil +} diff --git a/utils/test/secure_connection.go b/utils/test/secure_connection.go index 826a268c..8cbdd22c 100644 --- a/utils/test/secure_connection.go +++ b/utils/test/secure_connection.go @@ -48,16 +48,10 @@ type ( const ( defaultHostName = "localhost" - // YugaDBType represents the usage of Yugabyte DB. - YugaDBType = "yugabyte" - // PostgresDBType represents the usage of PostgreSQL DB. - PostgresDBType = "postgres" - // DefaultCertStyle represents the default TLS certificate style creation. - DefaultCertStyle = "default" - //nolint:revive // KeyPrivate, KeyPublic and KeyCACert represents the chosen key in the naming function. - KeyPrivate = "private-key" - KeyPublic = "public-key" - KeyCACert = "ca-certificate" + //nolint:revive // represents the default certificate's names. + PrivateKey = "private-key.pem" + PublicKey = "public-key.pem" + CACertificateKey = "ca-certificate.pem" ) // ServerModes is a list of server-side TLS modes used for testing. @@ -78,13 +72,12 @@ func NewCredentialsFactory(t *testing.T) *CredentialsFactory { func (scm *CredentialsFactory) CreateServerCredentials( t *testing.T, tlsMode string, - namingStyle string, san ...string, ) (connection.TLSConfig, string) { t.Helper() serverKeypair, err := scm.CertificateAuthority.NewServerCertKeyPair(san...) require.NoError(t, err) - return scm.createTLSConfig(t, tlsMode, serverKeypair, namingStyle) + return scm.createTLSConfig(t, tlsMode, serverKeypair) } // CreateClientCredentials creates a client key pair, @@ -93,7 +86,7 @@ func (scm *CredentialsFactory) CreateClientCredentials(t *testing.T, tlsMode str t.Helper() clientKeypair, err := scm.CertificateAuthority.NewClientCertKeyPair() require.NoError(t, err) - return scm.createTLSConfig(t, tlsMode, clientKeypair, DefaultCertStyle) + return scm.createTLSConfig(t, tlsMode, clientKeypair) } /* @@ -150,7 +143,7 @@ func RunSecureConnectionTest( t.Run(fmt.Sprintf("server-tls:%s", tc.serverMode), func(t *testing.T) { t.Parallel() // create server's tls config and start it according to the server tls mode. - serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, DefaultCertStyle, defaultHostName) + serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, defaultHostName) rpcAttemptFunc := starter(t, serverTLS) // for each server secure mode, build the client's test cases. for _, clientTestCase := range tc.cases { @@ -204,19 +197,17 @@ func (scm *CredentialsFactory) createTLSConfig( t *testing.T, connectionMode string, keyPair *tlsgen.CertKeyPair, - namingStyle string, ) (connection.TLSConfig, string) { t.Helper() tmpDir := t.TempDir() - namingFunction := selectFileNames(namingStyle) - privateKeyPath := filepath.Join(tmpDir, namingFunction("private-key")) + privateKeyPath := filepath.Join(tmpDir, PrivateKey) require.NoError(t, os.WriteFile(privateKeyPath, keyPair.Key, 0o600)) - publicKeyPath := filepath.Join(tmpDir, namingFunction("public-key")) + publicKeyPath := filepath.Join(tmpDir, PublicKey) require.NoError(t, os.WriteFile(publicKeyPath, keyPair.Cert, 0o600)) - caCertificatePath := filepath.Join(tmpDir, namingFunction("ca-certificate")) + caCertificatePath := filepath.Join(tmpDir, CACertificateKey) require.NoError(t, os.WriteFile(caCertificatePath, scm.CertificateAuthority.CertBytes(), 0o600)) return connection.TLSConfig{ @@ -226,50 +217,3 @@ func (scm *CredentialsFactory) createTLSConfig( CACertPaths: []string{caCertificatePath}, }, tmpDir } - -func selectFileNames(style string) func(string) string { - switch style { - case YugaDBType: - return func(key string) string { - switch key { - // We currently use YugabyteDB with the hostname "db" only. - // To support additional instances with different hostnames, - // replace "db" with the desired hostname when creating the instance. - case KeyPublic: - return "node.db.crt" - case KeyPrivate: - return "node.db.key" - case KeyCACert: - return "ca.crt" - default: - return "" - } - } - case PostgresDBType: - return func(key string) string { - switch key { - case KeyPublic: - return "server.crt" - case KeyPrivate: - return "server.key" - case KeyCACert: - return "ca-certificate.crt" - default: - return "" - } - } - default: - return func(key string) string { - switch key { - case KeyPublic: - return "public-key.crt" - case KeyPrivate: - return "private-key.key" - case KeyCACert: - return "ca-certificate.crt" - default: - return "" - } - } - } -} diff --git a/utils/test/utils.go b/utils/test/utils.go index a720fc5a..b8ee231f 100644 --- a/utils/test/utils.go +++ b/utils/test/utils.go @@ -34,6 +34,20 @@ import ( "github.com/hyperledger/fabric-x-committer/utils/logging" ) +var ( + // InsecureTLSConfig defines an empty tls config. + InsecureTLSConfig connection.TLSConfig + // defaultGrpcRetryProfile defines the retry policy for a gRPC client connection. + defaultGrpcRetryProfile connection.RetryProfile +) + +const ( + // YugaDBType represents the usage of Yugabyte DB. + YugaDBType = "yugabyte" + // PostgresDBType represents the usage of PostgreSQL DB. + PostgresDBType = "postgres" +) + type ( // GrpcServers holds the server instances and their respective configurations. GrpcServers struct { @@ -52,13 +66,6 @@ func FailHandler(t *testing.T) { }) } -var ( - // InsecureTLSConfig defines an empty tls config. - InsecureTLSConfig connection.TLSConfig - // defaultGrpcRetryProfile defines the retry policy for a gRPC client connection. - defaultGrpcRetryProfile connection.RetryProfile -) - // ServerToMultiClientConfig is used to create a multi client configuration from existing server(s). func ServerToMultiClientConfig(servers ...*connection.ServerConfig) *connection.MultiClientConfig { endpoints := make([]*connection.Endpoint, len(servers)) From 6171523b58e15ea09ae31a9fa8d20ee3392c0d23 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 3 Nov 2025 21:47:09 +0200 Subject: [PATCH 08/19] * Minor issues fixed. Signed-off-by: Dean Amar --- cmd/config/app_config_test.go | 14 +++++++------- docker/test/container_release_image_test.go | 2 +- integration/runner/postgres.go | 4 ++-- service/vc/dbtest/container.go | 5 +++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index e513aafb..886f79cf 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -34,18 +34,18 @@ import ( var ( defaultServerTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/server-certs/public-key.crt", - KeyPath: "/server-certs/private-key.key", + CertPath: "/server-certs/public-key.pem", + KeyPath: "/server-certs/private-key.pem", CACertPaths: []string{ - "/server-certs/ca-certificate.crt", + "/server-certs/ca-certificate.pem", }, } defaultClientTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/client-certs/public-key.crt", - KeyPath: "/client-certs/private-key.key", + CertPath: "/client-certs/public-key.pem", + KeyPath: "/client-certs/private-key.pem", CACertPaths: []string{ - "/client-certs/ca-certificate.crt", + "/client-certs/ca-certificate.pem", }, } ) @@ -449,7 +449,7 @@ func defaultSampleDBConfig() *vc.DatabaseConfig { Database: "yugabyte", TLS: connection.DatabaseTLSConfig{ Mode: connection.OneSideTLSMode, - CACertPath: "/server-certs/ca-certificate.crt", + CACertPath: "/server-certs/ca-certificate.pem", }, MaxConnections: 10, MinConnections: 5, diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index e224580e..3abba522 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -171,7 +171,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - node.EnsurePostgresNodeReadiness(t) + node.EnsurePostgresNodeReadiness(t, "5433") node.ExecuteCommand(t, []string{ "sh", "-c", `// Ensure proper root ownership and permissions for the TLS certificate files. diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 9f1afa34..6e3bc423 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -100,7 +100,7 @@ func (cc *PostgresClusterController) addPrimaryNode(ctx context.Context, t *test }, }) node.StartContainer(ctx, t) - node.EnsurePostgresNodeReadiness(t) + node.EnsurePostgresNodeReadiness(t, "5432") return node } @@ -121,7 +121,7 @@ func (cc *PostgresClusterController) addSecondaryNode(ctx context.Context, t *te }, }) node.StartContainer(ctx, t) - node.EnsurePostgresNodeReadiness(t) + node.EnsurePostgresNodeReadiness(t, "5432") return node } diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 519be96e..15d73bc5 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -43,6 +43,7 @@ const ( // YugabytedReadinessOutput is the output indicating that a Yugabyted node is ready. YugabytedReadinessOutput = "Data placement constraint successfully verified" + //nolint:revive // Represents the required TLS certificate files name. YugabytePublicKeyFileName = "node.db.crt" YugabytePrivateKeyFileName = "node.db.key" YugabyteCACertificateFileName = "ca.crt" @@ -424,11 +425,11 @@ func (dc *DatabaseContainer) EnsureNodeReadinessByLogs(t *testing.T, requiredOut // is ready to accept connections. // It repeatedly executes `pg_isready` until the command // returns a successful exit code (0) or the timeout is reached. -func (dc *DatabaseContainer) EnsurePostgresNodeReadiness(t *testing.T) { +func (dc *DatabaseContainer) EnsurePostgresNodeReadiness(t *testing.T, port string) { t.Helper() require.EventuallyWithT(t, func(ct *assert.CollectT) { _, exitCode := dc.ExecuteCommand(t, []string{ - "bash", "-c", "pg_isready -U yugabyte -d yugabyte -h localhost -p 5433", + "bash", "-c", fmt.Sprintf("pg_isready -U yugabyte -d yugabyte -h localhost -p %s", port), }) require.Equal(ct, 0, exitCode) }, 45*time.Second, 250*time.Millisecond) From b35994d559d9c149a98bd5110fe1eb6d29c6022b Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 3 Nov 2025 23:13:11 +0200 Subject: [PATCH 09/19] * Revert to serial scripts execution. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 31 +++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 3abba522..5b32dd5d 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -59,6 +59,20 @@ const ( containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec ) +var ( + // enforcePostgresSSLScript enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. + enforcePostgresSSLScript = []string{ + "sh", "-c", + `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + + `/var/lib/postgresql/data/pg_hba.conf`, + } + + // reloadPostgresConfigScript reloads the PostgreSQL server configuration without restarting the instance. + reloadPostgresConfigScript = []string{ + "psql", "-U", "yugabyte", "-c", "SELECT pg_reload_conf();", + } +) + // TestCommitterReleaseImagesWithTLS runs the committer components in different Docker containers with different TLS // modes and verifies it starts and connect successfully. // This test uses the release images for all the components but 'db' and 'orderer'. @@ -171,19 +185,14 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - node.EnsurePostgresNodeReadiness(t, "5433") node.ExecuteCommand(t, []string{ - "sh", "-c", - `// Ensure proper root ownership and permissions for the TLS certificate files. - chown postgres:postgres /creds/server.crt /creds/server.key - - // Enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. - sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' - /var/lib/postgresql/data/pg_hba.conf - - // Reloads the PostgreSQL server configuration without restarting the instance. - psql -U yugabyte -c SELECT pg_reload_conf();`, + "chown", "postgres:postgres", + "/creds/server.crt", + "/creds/server.key", }) + node.EnsurePostgresNodeReadiness(t, "5433") + node.ExecuteCommand(t, enforcePostgresSSLScript) + node.ExecuteCommand(t, reloadPostgresConfigScript) default: t.Fatalf("Unsupported database type: %s", node.DatabaseType) } From 8ee290b52881140c495a09737b730f999abf194c Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 3 Nov 2025 23:25:54 +0200 Subject: [PATCH 10/19] * preset permissions to certificates Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 22 ++++++++++----------- service/vc/dbtest/container.go | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 5b32dd5d..bbb5dcd9 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -176,20 +176,20 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod switch node.DatabaseType { case testutils.YugaDBType: // Ensure proper root ownership and permissions for the TLS certificate files. - node.ExecuteCommand(t, []string{ - "chown", - "root:root", - filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), - filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), - }) + //node.ExecuteCommand(t, []string{ + // "chown", + // "root:root", + // filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), + // filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), + //}) node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - node.ExecuteCommand(t, []string{ - "chown", "postgres:postgres", - "/creds/server.crt", - "/creds/server.key", - }) + //node.ExecuteCommand(t, []string{ + // "chown", "postgres:postgres", + // "/creds/server.crt", + // "/creds/server.key", + //}) node.EnsurePostgresNodeReadiness(t, "5433") node.ExecuteCommand(t, enforcePostgresSSLScript) node.ExecuteCommand(t, reloadPostgresConfigScript) diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 15d73bc5..da698d16 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -144,9 +144,9 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.TLSConfig != nil { require.NotEmpty(t, dc.TLSConfig.CACertPaths) dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), + fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), + fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), ) } case test.PostgresDBType: @@ -166,8 +166,8 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit } if dc.TLSConfig != nil { dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), + fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), ) dc.Cmd = []string{ // Configure PostgreSQL to run in secure mode From 21af1240fce504b67099357167cf0903f951fd18 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 3 Nov 2025 23:33:17 +0200 Subject: [PATCH 11/19] * revert latest changes. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 22 ++++++++++----------- service/vc/dbtest/container.go | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index bbb5dcd9..5b32dd5d 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -176,20 +176,20 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod switch node.DatabaseType { case testutils.YugaDBType: // Ensure proper root ownership and permissions for the TLS certificate files. - //node.ExecuteCommand(t, []string{ - // "chown", - // "root:root", - // filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), - // filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), - //}) + node.ExecuteCommand(t, []string{ + "chown", + "root:root", + filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), + filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), + }) node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - //node.ExecuteCommand(t, []string{ - // "chown", "postgres:postgres", - // "/creds/server.crt", - // "/creds/server.key", - //}) + node.ExecuteCommand(t, []string{ + "chown", "postgres:postgres", + "/creds/server.crt", + "/creds/server.key", + }) node.EnsurePostgresNodeReadiness(t, "5433") node.ExecuteCommand(t, enforcePostgresSSLScript) node.ExecuteCommand(t, reloadPostgresConfigScript) diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index da698d16..15d73bc5 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -144,9 +144,9 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.TLSConfig != nil { require.NotEmpty(t, dc.TLSConfig.CACertPaths) dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), - fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), - fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), ) } case test.PostgresDBType: @@ -166,8 +166,8 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit } if dc.TLSConfig != nil { dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), - fmt.Sprintf("%s:/creds/%s:rw", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), ) dc.Cmd = []string{ // Configure PostgreSQL to run in secure mode From 8818efdb48a75565087f5adb02b1a48c2ef295d6 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 4 Nov 2025 14:00:27 +0200 Subject: [PATCH 12/19] * Addressed PR comments. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 20 +++++++++++++------- integration/runner/postgres.go | 3 ++- integration/runner/yugabyte.go | 7 +++++-- service/vc/dbtest/container.go | 7 +++++-- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 5b32dd5d..3919b483 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -57,6 +57,8 @@ const ( // This work-around is needed due to a Yugabyte behavior that prevents using default passwords in secure mode. // Instead, Yugabyte generates a random password, and this path points to the output file containing it. containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec + + defaultDBPort = "5433" ) var ( @@ -156,6 +158,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod DatabaseType: params.dbType, Network: params.networkName, Hostname: params.node, + DbPort: defaultDBPort, TLSConfig: &tlsConfig, } @@ -175,24 +178,27 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod // post start container tweaking switch node.DatabaseType { case testutils.YugaDBType: - // Ensure proper root ownership and permissions for the TLS certificate files. - node.ExecuteCommand(t, []string{ + // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. + _, exitCode := node.ExecuteCommand(t, []string{ "chown", "root:root", filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), }) + require.Zero(t, exitCode) node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case testutils.PostgresDBType: - node.ExecuteCommand(t, []string{ + // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. + _, exitCode := node.ExecuteCommand(t, []string{ "chown", "postgres:postgres", - "/creds/server.crt", - "/creds/server.key", + filepath.Join("/creds", dbtest.PostgresPublicKeyFileName), + filepath.Join("/creds", dbtest.PostgresPrivateKeyFileName), }) + require.Zero(t, exitCode) node.EnsurePostgresNodeReadiness(t, "5433") - node.ExecuteCommand(t, enforcePostgresSSLScript) - node.ExecuteCommand(t, reloadPostgresConfigScript) + _, exitCode = node.ExecuteCommand(t, append(enforcePostgresSSLScript, reloadPostgresConfigScript...)) + require.Zero(t, exitCode) default: t.Fatalf("Unsupported database type: %s", node.DatabaseType) } diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 6e3bc423..14f8fa99 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -151,5 +151,6 @@ func (cc *PostgresClusterController) PromoteSecondaryNode(t *testing.T) { t.Helper() secondaryNode, _ := cc.GetSingleNodeByRole(SecondaryNode) require.NotNil(t, secondaryNode) - secondaryNode.ExecuteCommand(t, postgresSecondaryPromotionCommand) + _, exitCode := secondaryNode.ExecuteCommand(t, postgresSecondaryPromotionCommand) + require.Zero(t, exitCode) } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index 83556f43..516ae030 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -187,14 +187,17 @@ func (cc *YugaClusterController) getNumberOfAliveMasters(t *testing.T) int { func (cc *YugaClusterController) listAllMasters(t *testing.T) string { t.Helper() - var output string cmd := []string{ "/home/yugabyte/bin/yb-admin", "-master_addresses", cc.getMasterAddresses(), "list_all_masters", } + var ( + output string + exitCode int + ) for _, n := range cc.nodes { - if output, _ = n.ExecuteCommand(t, cmd); output != "" { + if output, exitCode = n.ExecuteCommand(t, cmd); output != "" && exitCode == 0 { break } } diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 15d73bc5..f880e96f 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -141,6 +141,7 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", yugaDBPort)) } + if dc.TLSConfig != nil { require.NotEmpty(t, dc.TLSConfig.CACertPaths) dc.Binds = append(dc.Binds, @@ -164,6 +165,7 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", postgresDBPort)) } + if dc.TLSConfig != nil { dc.Binds = append(dc.Binds, fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), @@ -379,7 +381,8 @@ func (dc *DatabaseContainer) ContainerID() string { // If no password is found, the default one will be returned. func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { t.Helper() - output, _ := dc.ExecuteCommand(t, []string{"cat", filePath}) + output, exitCode := dc.ExecuteCommand(t, []string{"cat", filePath}) + require.Zero(t, exitCode) found := passwordRegex.FindStringSubmatch(output) if len(found) > 1 { return found[1] @@ -431,6 +434,6 @@ func (dc *DatabaseContainer) EnsurePostgresNodeReadiness(t *testing.T, port stri _, exitCode := dc.ExecuteCommand(t, []string{ "bash", "-c", fmt.Sprintf("pg_isready -U yugabyte -d yugabyte -h localhost -p %s", port), }) - require.Equal(ct, 0, exitCode) + require.Zero(ct, exitCode) }, 45*time.Second, 250*time.Millisecond) } From e391bc39b8c039331ce3853b5ecdddcba0a27905 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 4 Nov 2025 14:05:20 +0200 Subject: [PATCH 13/19] * Code hygiene. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 3919b483..e9c17bb4 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -180,8 +180,7 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod case testutils.YugaDBType: // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. _, exitCode := node.ExecuteCommand(t, []string{ - "chown", - "root:root", + "chown", "root:root", filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), }) From 64d8575b4eb5ad9807963aeba456e8a88a69b6bb Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 10 Nov 2025 18:06:36 +0200 Subject: [PATCH 14/19] * Addressed PR comments. Signed-off-by: Dean Amar --- cmd/config/app_config_test.go | 3 +- cmd/config/create_config_file.go | 3 +- docker/test/container_release_image_test.go | 41 ++++++-------- integration/runner/postgres.go | 10 ++-- integration/runner/yugabyte.go | 11 ++-- service/vc/config.go | 21 ++++---- service/vc/dbtest/connection.go | 5 +- service/vc/dbtest/container.go | 50 ++++++++--------- service/vc/dbtest/database_setup.go | 8 ++- utils/connection/client_util.go | 37 ------------- utils/connection/config.go | 6 --- utils/dbconn/database_connection.go | 60 +++++++++++++++++++++ utils/test/utils.go | 7 --- 13 files changed, 131 insertions(+), 131 deletions(-) create mode 100644 utils/dbconn/database_connection.go diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index 886f79cf..90e5c449 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -26,6 +26,7 @@ import ( "github.com/hyperledger/fabric-x-committer/service/vc" "github.com/hyperledger/fabric-x-committer/service/verifier" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/dbconn" "github.com/hyperledger/fabric-x-committer/utils/monitoring" "github.com/hyperledger/fabric-x-committer/utils/ordererconn" "github.com/hyperledger/fabric-x-committer/utils/signature" @@ -447,7 +448,7 @@ func defaultSampleDBConfig() *vc.DatabaseConfig { Username: "yugabyte", Password: "yugabyte", Database: "yugabyte", - TLS: connection.DatabaseTLSConfig{ + TLS: dbconn.DatabaseTLSConfig{ Mode: connection.OneSideTLSMode, CACertPath: "/server-certs/ca-certificate.pem", }, diff --git a/cmd/config/create_config_file.go b/cmd/config/create_config_file.go index 830f9d6a..7b427ba7 100644 --- a/cmd/config/create_config_file.go +++ b/cmd/config/create_config_file.go @@ -22,6 +22,7 @@ import ( "github.com/hyperledger/fabric-x-committer/loadgen/workload" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/dbconn" "github.com/hyperledger/fabric-x-committer/utils/logging" ) @@ -75,7 +76,7 @@ type ( Password string LoadBalance bool Endpoints []*connection.Endpoint - TLS connection.DatabaseTLSConfig + TLS dbconn.DatabaseTLSConfig } ) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index e9c17bb4..26178fcc 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -23,6 +23,7 @@ import ( "github.com/hyperledger/fabric-x-committer/loadgen/workload" "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/dbconn" testutils "github.com/hyperledger/fabric-x-committer/utils/test" ) @@ -61,19 +62,14 @@ const ( defaultDBPort = "5433" ) -var ( - // enforcePostgresSSLScript enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. - enforcePostgresSSLScript = []string{ - "sh", "-c", - `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + - `/var/lib/postgresql/data/pg_hba.conf`, - } - - // reloadPostgresConfigScript reloads the PostgreSQL server configuration without restarting the instance. - reloadPostgresConfigScript = []string{ - "psql", "-U", "yugabyte", "-c", "SELECT pg_reload_conf();", - } -) +// enforcePostgresSSLAndReloadConfigScript enforces SSL-only client connections to a PostgreSQL +// instance by updating pg_hba.conf and reloads its server configuration without restarting the instance. +var enforcePostgresSSLAndReloadConfigScript = []string{ + "sh", "-c", + `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + + `/var/lib/postgresql/data/pg_hba.conf`, + `psql -U yugabyte -c "SELECT pg_reload_conf();"`, +} // TestCommitterReleaseImagesWithTLS runs the committer components in different Docker containers with different TLS // modes and verifies it starts and connect successfully. @@ -97,7 +93,7 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { committerNodes := []string{"verifier", "vc", "query", "coordinator", "sidecar"} credsFactory := testutils.NewCredentialsFactory(t) - for _, dbType := range []string{testutils.YugaDBType, testutils.PostgresDBType} { + for _, dbType := range []string{dbtest.YugaDBType, dbtest.PostgresDBType} { t.Run(fmt.Sprintf("database:%s", dbType), func(t *testing.T) { t.Parallel() for _, mode := range testutils.ServerModes { @@ -170,34 +166,31 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod // This is relevant if a different CA was used to issue the DB's TLS certificates. require.NotEmpty(t, tlsConfig.CACertPaths) - conn.TLS = connection.DatabaseTLSConfig{ + conn.TLS = dbconn.DatabaseTLSConfig{ Mode: connection.OneSideTLSMode, CACertPath: tlsConfig.CACertPaths[0], } // post start container tweaking switch node.DatabaseType { - case testutils.YugaDBType: + case dbtest.YugaDBType: // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. - _, exitCode := node.ExecuteCommand(t, []string{ + node.ExecuteCommand(t, []string{ "chown", "root:root", filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), }) - require.Zero(t, exitCode) node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) - case testutils.PostgresDBType: + case dbtest.PostgresDBType: // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. - _, exitCode := node.ExecuteCommand(t, []string{ + node.ExecuteCommand(t, []string{ "chown", "postgres:postgres", filepath.Join("/creds", dbtest.PostgresPublicKeyFileName), filepath.Join("/creds", dbtest.PostgresPrivateKeyFileName), }) - require.Zero(t, exitCode) - node.EnsurePostgresNodeReadiness(t, "5433") - _, exitCode = node.ExecuteCommand(t, append(enforcePostgresSSLScript, reloadPostgresConfigScript...)) - require.Zero(t, exitCode) + node.EnsureNodeReadinessByLogs(t, dbtest.PostgresReadinesssOutput) + node.ExecuteCommand(t, enforcePostgresSSLAndReloadConfigScript) default: t.Fatalf("Unsupported database type: %s", node.DatabaseType) } diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 14f8fa99..f75453e9 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -16,7 +16,6 @@ import ( "github.com/stretchr/testify/require" "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" - "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -100,7 +99,7 @@ func (cc *PostgresClusterController) addPrimaryNode(ctx context.Context, t *test }, }) node.StartContainer(ctx, t) - node.EnsurePostgresNodeReadiness(t, "5432") + node.EnsureNodeReadinessByLogs(t, dbtest.PostgresReadinesssOutput) return node } @@ -121,7 +120,7 @@ func (cc *PostgresClusterController) addSecondaryNode(ctx context.Context, t *te }, }) node.StartContainer(ctx, t) - node.EnsurePostgresNodeReadiness(t, "5432") + node.EnsureNodeReadinessByLogs(t, dbtest.SecondaryPostgresNodeReadinessOutput) return node } @@ -133,7 +132,7 @@ func (cc *PostgresClusterController) createNode( Role: nodeCreationOpts.role, Image: postgresImage, Tag: defaultBitnamiPostgresTag, - DatabaseType: test.PostgresDBType, + DatabaseType: dbtest.PostgresDBType, Env: append([]string{ "POSTGRESQL_REPLICATION_USER=repl_user", "POSTGRESQL_REPLICATION_PASSWORD=repl_password", @@ -151,6 +150,5 @@ func (cc *PostgresClusterController) PromoteSecondaryNode(t *testing.T) { t.Helper() secondaryNode, _ := cc.GetSingleNodeByRole(SecondaryNode) require.NotNil(t, secondaryNode) - _, exitCode := secondaryNode.ExecuteCommand(t, postgresSecondaryPromotionCommand) - require.Zero(t, exitCode) + secondaryNode.ExecuteCommand(t, postgresSecondaryPromotionCommand) } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index 516ae030..1d680dff 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -125,7 +125,7 @@ func (cc *YugaClusterController) createNode(role string) { Name: fmt.Sprintf("yuga-%s-%s", role, uuid.New().String()), Image: defaultImage, Role: role, - DatabaseType: test.YugaDBType, + DatabaseType: dbtest.YugaDBType, Network: cc.networkName, } cc.nodes = append(cc.nodes, node) @@ -161,7 +161,7 @@ func (cc *YugaClusterController) startNodes(ctx context.Context, t *testing.T) { } for _, n := range cc.IterNodesByRole(TabletNode) { - n.EnsureNodeReadinessByLogs(t, "syncing data to disk ... ok") + n.EnsureNodeReadinessByLogs(t, dbtest.YugabyteTabletNodeReadinessOutput) } } @@ -192,12 +192,9 @@ func (cc *YugaClusterController) listAllMasters(t *testing.T) string { "-master_addresses", cc.getMasterAddresses(), "list_all_masters", } - var ( - output string - exitCode int - ) + var output string for _, n := range cc.nodes { - if output, exitCode = n.ExecuteCommand(t, cmd); output != "" && exitCode == 0 { + if output = n.ExecuteCommand(t, cmd); output != "" { break } } diff --git a/service/vc/config.go b/service/vc/config.go index 8a37fc5e..ec4b59ba 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/dbconn" "github.com/hyperledger/fabric-x-committer/utils/monitoring" ) @@ -23,20 +24,20 @@ type Config struct { // DatabaseConfig is the configuration for the database. type DatabaseConfig struct { - Endpoints []*connection.Endpoint `mapstructure:"endpoints"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Database string `mapstructure:"database"` - MaxConnections int32 `mapstructure:"max-connections"` - MinConnections int32 `mapstructure:"min-connections"` - LoadBalance bool `mapstructure:"load-balance"` - Retry *connection.RetryProfile `mapstructure:"retry"` - TLS connection.DatabaseTLSConfig `mapstructure:"tls"` + Endpoints []*connection.Endpoint `mapstructure:"endpoints"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + MaxConnections int32 `mapstructure:"max-connections"` + MinConnections int32 `mapstructure:"min-connections"` + LoadBalance bool `mapstructure:"load-balance"` + Retry *connection.RetryProfile `mapstructure:"retry"` + TLS dbconn.DatabaseTLSConfig `mapstructure:"tls"` } // DataSourceName returns the data source name of the database. func (d *DatabaseConfig) DataSourceName() (string, error) { - return connection.DataSourceName(connection.DataSourceNameParams{ + return dbconn.DataSourceName(dbconn.DataSourceNameParams{ Username: d.Username, Password: d.Password, Database: d.Database, diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index 29818a87..02ba201a 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -15,6 +15,7 @@ import ( "github.com/yugabyte/pgx/v4/pgxpool" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/dbconn" "github.com/hyperledger/fabric-x-committer/utils/logging" ) @@ -43,7 +44,7 @@ type Connection struct { Password string Database string LoadBalance bool - TLS connection.DatabaseTLSConfig + TLS dbconn.DatabaseTLSConfig } // NewConnection returns a connection parameters with the specified host:port, and the default values @@ -58,7 +59,7 @@ func NewConnection(endpoints ...*connection.Endpoint) *Connection { // dataSourceName returns the dataSourceName to be used by the database/sql package. func (c *Connection) dataSourceName() (string, error) { - return connection.DataSourceName(connection.DataSourceNameParams{ + return dbconn.DataSourceName(dbconn.DataSourceNameParams{ Username: c.User, Password: c.Password, Database: c.Database, diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index f880e96f..c11bfc5a 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -42,13 +42,19 @@ const ( // YugabytedReadinessOutput is the output indicating that a Yugabyted node is ready. YugabytedReadinessOutput = "Data placement constraint successfully verified" + // YugabyteTabletNodeReadinessOutput is the output indicating that a yugabyte's tablet node is ready. + YugabyteTabletNodeReadinessOutput = "syncing data to disk ... ok" + // PostgresReadinesssOutput is the output indicating that a PostgreSQL node is ready. + PostgresReadinesssOutput = "database system is ready to accept connections" + // SecondaryPostgresNodeReadinessOutput is the output indicating that a secondary PostgreSQL node is ready. + SecondaryPostgresNodeReadinessOutput = "started streaming WAL from primary" //nolint:revive // Represents the required TLS certificate files name. YugabytePublicKeyFileName = "node.db.crt" YugabytePrivateKeyFileName = "node.db.key" - YugabyteCACertificateFileName = "ca.crt" PostgresPublicKeyFileName = "server.crt" PostgresPrivateKeyFileName = "server.key" + yugabyteCACertificateFileName = "ca.crt" ) var ( @@ -123,7 +129,7 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit t.Helper() switch dc.DatabaseType { - case test.YugaDBType: + case YugaDBType: if dc.Image == "" { dc.Image = defaultYugabyteImage } @@ -147,10 +153,10 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit dc.Binds = append(dc.Binds, fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], YugabyteCACertificateFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], yugabyteCACertificateFileName), ) } - case test.PostgresDBType: + case PostgresDBType: if dc.Image == "" { dc.Image = defaultPostgresImage } @@ -173,8 +179,8 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit ) dc.Cmd = []string{ // Configure PostgreSQL to run in secure mode - // and listen for incoming connections on port 5433. - "-c", "port=5433", + // and listen for incoming connections on a chosen port. + "-c", fmt.Sprintf("port=%s", dc.DbPort), "-c", "ssl=on", "-c", fmt.Sprintf("ssl_cert_file=%s", filepath.Join("/creds", PostgresPublicKeyFileName)), "-c", fmt.Sprintf("ssl_key_file=%s", filepath.Join("/creds", PostgresPrivateKeyFileName)), @@ -186,6 +192,9 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.Name == "" { dc.Name = fmt.Sprintf(defaultDBDeploymentTemplateName, dc.DatabaseType) + if dc.TLSConfig != nil { + dc.Name += fmt.Sprintf("_with_tls_%s", uuid.NewString()[0:8]) + } } if dc.HostIP == "" { @@ -207,9 +216,6 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.client == nil { dc.client = test.GetDockerClient(t) } - if dc.TLSConfig != nil { - dc.Name += fmt.Sprintf("_with_tls_%s", uuid.NewString()[0:8]) - } } // createContainer attempts to create a container instance, or attach to an existing one. @@ -378,11 +384,12 @@ func (dc *DatabaseContainer) ContainerID() string { // ReadPasswordFromContainer extracts the randomly generated password from a file inside the container. // This is required because YugabyteDB, when running in secure mode, doesn't allow default passwords // and instead generates a random one at startup. -// If no password is found, the default one will be returned. +// This method being called only when a secured Yugabyted node is started. +// If the file doesn’t exist, the test should fail. +// If the file exists but doesn’t contain a password (no password found), we fall back to the default password. func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { t.Helper() - output, exitCode := dc.ExecuteCommand(t, []string{"cat", filePath}) - require.Zero(t, exitCode) + output := dc.ExecuteCommand(t, []string{"cat", filePath}) found := passwordRegex.FindStringSubmatch(output) if len(found) > 1 { return found[1] @@ -392,7 +399,7 @@ func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath st } // ExecuteCommand executes a command and returns the container output. -func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) (string, int) { +func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { t.Helper() require.NotNil(t, dc.client) t.Logf("executing %s", strings.Join(cmd, " ")) @@ -411,8 +418,9 @@ func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) (string, inspect, err := dc.client.InspectExec(exec.ID) require.NoError(t, err) + require.Zero(t, inspect.ExitCode) - return stdout.String(), inspect.ExitCode + return stdout.String() } // EnsureNodeReadinessByLogs checks the container's readiness by monitoring its logs and ensure its running correctly. @@ -423,17 +431,3 @@ func (dc *DatabaseContainer) EnsureNodeReadinessByLogs(t *testing.T, requiredOut require.Contains(ct, output, requiredOutput) }, 45*time.Second, 250*time.Millisecond) } - -// EnsurePostgresNodeReadiness verifies that the PostgreSQL node -// is ready to accept connections. -// It repeatedly executes `pg_isready` until the command -// returns a successful exit code (0) or the timeout is reached. -func (dc *DatabaseContainer) EnsurePostgresNodeReadiness(t *testing.T, port string) { - t.Helper() - require.EventuallyWithT(t, func(ct *assert.CollectT) { - _, exitCode := dc.ExecuteCommand(t, []string{ - "bash", "-c", fmt.Sprintf("pg_isready -U yugabyte -d yugabyte -h localhost -p %s", port), - }) - require.Zero(ct, exitCode) - }, 45*time.Second, 250*time.Millisecond) -} diff --git a/service/vc/dbtest/database_setup.go b/service/vc/dbtest/database_setup.go index 2aa1efd1..dbe7912b 100644 --- a/service/vc/dbtest/database_setup.go +++ b/service/vc/dbtest/database_setup.go @@ -22,7 +22,6 @@ import ( "github.com/hyperledger/fabric-x-committer/utils" "github.com/hyperledger/fabric-x-committer/utils/connection" - "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -32,6 +31,11 @@ const ( deploymentLocal = "local" deploymentContainer = "container" + // YugaDBType represents the usage of Yugabyte DB. + YugaDBType = "yugabyte" + // PostgresDBType represents the usage of PostgreSQL DB. + PostgresDBType = "postgres" + yugaDBPort = "5433" postgresDBPort = "5432" @@ -71,7 +75,7 @@ func getDBTypeFromEnv() string { if found { return strings.ToLower(val) } - return test.YugaDBType + return YugaDBType } // PrepareTestEnv initializes a test environment for an existing or uncontrollable db instance. diff --git a/utils/connection/client_util.go b/utils/connection/client_util.go index 2c1b7f9e..9bb57404 100644 --- a/utils/connection/client_util.go +++ b/utils/connection/client_util.go @@ -61,16 +61,6 @@ type ( Resolver *manual.Resolver AdditionalOpts []grpc.DialOption } - - // DataSourceNameParams defines the parameters required to construct a database connection string. - DataSourceNameParams struct { - Username string - Password string - Database string - EndpointsString string - LoadBalance bool - TLS DatabaseTLSConfig - } ) var logger = logging.New("connection") @@ -292,30 +282,3 @@ func AddressString[T WithAddress](addresses ...T) string { } return strings.Join(listOfAddresses, ",") } - -// DataSourceName returns the data source name of the database. -func DataSourceName(d DataSourceNameParams) (string, error) { - ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", - d.Username, d.Password, d.EndpointsString, d.Database) - - switch d.TLS.Mode { - case NoneTLSMode, UnmentionedTLSMode: - ret += "sslmode=disable" - case OneSideTLSMode: - // Enforce full SSL verification: - // requires an encrypted connection (TLS), - // and ensures the server hostname matches the certificate. - ret += fmt.Sprintf("sslmode=verify-full&sslrootcert=%s", d.TLS.CACertPath) - case MutualTLSMode: - return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) - default: - return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", - d.TLS.Mode, NoneTLSMode, OneSideTLSMode, MutualTLSMode) - } - // The load balancing flag is only available when the server supports it (having multiple nodes). - // Thus, we only add it when explicitly required. Otherwise, an error will occur. - if d.LoadBalance { - ret += "&load_balance=true" - } - return ret, nil -} diff --git a/utils/connection/config.go b/utils/connection/config.go index cc89c47f..8194d510 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -77,12 +77,6 @@ type ( KeyPath string `mapstructure:"key-path"` CACertPaths []string `mapstructure:"ca-cert-paths"` } - - // DatabaseTLSConfig holds the database TLS mode and its necessary credentials. - DatabaseTLSConfig struct { - Mode string `mapstructure:"mode"` - CACertPath string `mapstructure:"ca-cert-path"` - } ) const ( diff --git a/utils/dbconn/database_connection.go b/utils/dbconn/database_connection.go new file mode 100644 index 00000000..d09a6315 --- /dev/null +++ b/utils/dbconn/database_connection.go @@ -0,0 +1,60 @@ +/* +Copyright IBM Corp. All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package dbconn + +import ( + "fmt" + + "github.com/cockroachdb/errors" + + "github.com/hyperledger/fabric-x-committer/utils/connection" +) + +type ( + // DatabaseTLSConfig holds the database TLS mode and its necessary credentials. + DatabaseTLSConfig struct { + Mode string `mapstructure:"mode"` + CACertPath string `mapstructure:"ca-cert-path"` + } + + // DataSourceNameParams defines the parameters required to construct a database connection string. + DataSourceNameParams struct { + Username string + Password string + Database string + EndpointsString string + LoadBalance bool + TLS DatabaseTLSConfig + } +) + +// DataSourceName returns the data source name of the database. +func DataSourceName(d DataSourceNameParams) (string, error) { + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", + d.Username, d.Password, d.EndpointsString, d.Database) + + switch d.TLS.Mode { + case connection.NoneTLSMode, connection.UnmentionedTLSMode: + ret += "sslmode=disable" + case connection.OneSideTLSMode: + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. + ret += fmt.Sprintf("sslmode=verify-full&sslrootcert=%s", d.TLS.CACertPath) + case connection.MutualTLSMode: + return "", errors.Newf("unsupportted db tls mode: %s", d.TLS.Mode) + default: + return "", errors.Newf("unknown TLS mode: %s (valid modes: %s, %s, %s)", + d.TLS.Mode, connection.NoneTLSMode, connection.OneSideTLSMode, connection.MutualTLSMode) + } + // The load balancing flag is only available when the server supports it (having multiple nodes). + // Thus, we only add it when explicitly required. Otherwise, an error will occur. + if d.LoadBalance { + ret += "&load_balance=true" + } + return ret, nil +} diff --git a/utils/test/utils.go b/utils/test/utils.go index b8ee231f..a5268ea0 100644 --- a/utils/test/utils.go +++ b/utils/test/utils.go @@ -41,13 +41,6 @@ var ( defaultGrpcRetryProfile connection.RetryProfile ) -const ( - // YugaDBType represents the usage of Yugabyte DB. - YugaDBType = "yugabyte" - // PostgresDBType represents the usage of PostgreSQL DB. - PostgresDBType = "postgres" -) - type ( // GrpcServers holds the server instances and their respective configurations. GrpcServers struct { From a43e199578a957389fa27d464d541f61b9da936b Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 10 Nov 2025 18:17:03 +0200 Subject: [PATCH 15/19] * Addressed PR comments. Signed-off-by: Dean Amar --- docker/test/container_release_image_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index 26178fcc..c7645de7 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -175,20 +175,12 @@ func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNod switch node.DatabaseType { case dbtest.YugaDBType: // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. - node.ExecuteCommand(t, []string{ - "chown", "root:root", - filepath.Join("/creds", dbtest.YugabytePublicKeyFileName), - filepath.Join("/creds", dbtest.YugabytePrivateKeyFileName), - }) + node.ExecuteCommand(t, []string{"bash", "-c", "chown root:root /creds/*"}) node.EnsureNodeReadinessByLogs(t, dbtest.YugabytedReadinessOutput) conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) case dbtest.PostgresDBType: // Must run after node startup to ensure proper root ownership and permissions for the TLS certificate files. - node.ExecuteCommand(t, []string{ - "chown", "postgres:postgres", - filepath.Join("/creds", dbtest.PostgresPublicKeyFileName), - filepath.Join("/creds", dbtest.PostgresPrivateKeyFileName), - }) + node.ExecuteCommand(t, []string{"bash", "-c", "chown postgres:postgres /creds/*"}) node.EnsureNodeReadinessByLogs(t, dbtest.PostgresReadinesssOutput) node.ExecuteCommand(t, enforcePostgresSSLAndReloadConfigScript) default: From 389f4eea5821384c43972fdc0aec50fc3fce8ec6 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Mon, 10 Nov 2025 19:23:55 +0200 Subject: [PATCH 16/19] * minor documentation update. Signed-off-by: Dean Amar --- utils/connection/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/connection/config.go b/utils/connection/config.go index 8194d510..87ab94bc 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -19,7 +19,7 @@ import ( ) type ( - // MultiClientConfig contains the endpoints, CAs, and retry profile. + // MultiClientConfig contains the endpoints, TLS config, and retry profile. // This config allows the support of number of different endpoints to multiple service instances. MultiClientConfig struct { Endpoints []*Endpoint `mapstructure:"endpoints" yaml:"endpoints"` @@ -27,7 +27,7 @@ type ( Retry *RetryProfile `mapstructure:"reconnect" yaml:"reconnect"` } - // ClientConfig contains a single endpoint, CAs, and retry profile. + // ClientConfig contains a single endpoint, TLS config, and retry profile. ClientConfig struct { Endpoint *Endpoint `mapstructure:"endpoint" yaml:"endpoint"` TLS TLSConfig `mapstructure:"tls" yaml:"tls"` From 0f6e2591479bbfdf4b8040d4e332c2c2124fbbc3 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 11 Nov 2025 12:56:23 +0200 Subject: [PATCH 17/19] * Addressed PR's comments. Signed-off-by: Dean Amar --- service/vc/dbtest/container.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index c11bfc5a..d831e95e 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -385,17 +385,13 @@ func (dc *DatabaseContainer) ContainerID() string { // This is required because YugabyteDB, when running in secure mode, doesn't allow default passwords // and instead generates a random one at startup. // This method being called only when a secured Yugabyted node is started. -// If the file doesn’t exist, the test should fail. -// If the file exists but doesn’t contain a password (no password found), we fall back to the default password. +// If the file doesn’t exist or doesn't contain a password, the test should fail. func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { t.Helper() output := dc.ExecuteCommand(t, []string{"cat", filePath}) found := passwordRegex.FindStringSubmatch(output) - if len(found) > 1 { - return found[1] - } - t.Log("password not found in output, returning default password.") - return defaultPassword + require.Greater(t, len(found), 1) + return found[1] } // ExecuteCommand executes a command and returns the container output. From de0d451d6e296134093e0c90ebc72fbd01a2badd Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 11 Nov 2025 13:28:48 +0200 Subject: [PATCH 18/19] * minor changes and mock-orderer yaml correction. Signed-off-by: Dean Amar --- cmd/config/samples/mock-orderer.yaml | 6 +++--- service/vc/dbtest/container.go | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/config/samples/mock-orderer.yaml b/cmd/config/samples/mock-orderer.yaml index 01c1ed77..b1692b14 100644 --- a/cmd/config/samples/mock-orderer.yaml +++ b/cmd/config/samples/mock-orderer.yaml @@ -6,10 +6,10 @@ server: endpoint: :7050 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.pem + key-path: /server-certs/private-key.pem ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.pem block-size: 1024 block-timeout: 30s diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index d831e95e..43a38389 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -50,10 +50,10 @@ const ( SecondaryPostgresNodeReadinessOutput = "started streaming WAL from primary" //nolint:revive // Represents the required TLS certificate files name. - YugabytePublicKeyFileName = "node.db.crt" - YugabytePrivateKeyFileName = "node.db.key" - PostgresPublicKeyFileName = "server.crt" - PostgresPrivateKeyFileName = "server.key" + yugabytePublicKeyFileName = "node.db.crt" + yugabytePrivateKeyFileName = "node.db.key" + postgresPublicKeyFileName = "server.crt" + postgresPrivateKeyFileName = "server.key" yugabyteCACertificateFileName = "ca.crt" ) @@ -151,8 +151,8 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.TLSConfig != nil { require.NotEmpty(t, dc.TLSConfig.CACertPaths) dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, YugabytePublicKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, YugabytePrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, yugabytePublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, yugabytePrivateKeyFileName), fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CACertPaths[0], yugabyteCACertificateFileName), ) } @@ -174,16 +174,16 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.TLSConfig != nil { dc.Binds = append(dc.Binds, - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, PostgresPublicKeyFileName), - fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, PostgresPrivateKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.CertPath, postgresPublicKeyFileName), + fmt.Sprintf("%s:/creds/%s", dc.TLSConfig.KeyPath, postgresPrivateKeyFileName), ) dc.Cmd = []string{ // Configure PostgreSQL to run in secure mode // and listen for incoming connections on a chosen port. "-c", fmt.Sprintf("port=%s", dc.DbPort), "-c", "ssl=on", - "-c", fmt.Sprintf("ssl_cert_file=%s", filepath.Join("/creds", PostgresPublicKeyFileName)), - "-c", fmt.Sprintf("ssl_key_file=%s", filepath.Join("/creds", PostgresPrivateKeyFileName)), + "-c", fmt.Sprintf("ssl_cert_file=%s", filepath.Join("/creds", postgresPublicKeyFileName)), + "-c", fmt.Sprintf("ssl_key_file=%s", filepath.Join("/creds", postgresPrivateKeyFileName)), } } default: From 5a8c41f764546d9acd0fcdd6ce6194699d04baa5 Mon Sep 17 00:00:00 2001 From: Dean Amar Date: Tue, 11 Nov 2025 13:33:45 +0200 Subject: [PATCH 19/19] * linter issue fixed. Signed-off-by: Dean Amar --- service/vc/dbtest/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 43a38389..18fa83e3 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -49,7 +49,7 @@ const ( // SecondaryPostgresNodeReadinessOutput is the output indicating that a secondary PostgreSQL node is ready. SecondaryPostgresNodeReadinessOutput = "started streaming WAL from primary" - //nolint:revive // Represents the required TLS certificate files name. + // Represents the required database TLS certificate files name. yugabytePublicKeyFileName = "node.db.crt" yugabytePrivateKeyFileName = "node.db.key" postgresPublicKeyFileName = "server.crt"