Driver Configuration Parameters (spacecloud-io#1374)

Co-authored-by: Noorain Panjwani <[email protected]>

Author: KhushalPShah

.github/workflows/go.yml

@@ -5,10 +5,10 @@ jobs:
     name: Build and test gateway
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.13
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.13
+          go-version: 1.15
         id: go
 
       - name: Check out code into the Go module directory
@@ -36,10 +36,10 @@ jobs:
     name: Build and test Runner
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.13
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.13
+          go-version: 1.15
         id: go
 
       - name: Check out code into the Go module directory
@@ -67,10 +67,10 @@ jobs:
     name: Build and test space cli
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.13
+      - name: Set up Go 1.15
         uses: actions/setup-go@v1
         with:
-          go-version: 1.13
+          go-version: 1.15
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v1

gateway/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.9-alpine3.12
+FROM golang:1.15.3-alpine3.12
 WORKDIR /build
 
 # Take the current space cloud version as a argument

gateway/config/config.go

@@ -88,17 +88,26 @@ type ProjectConfig struct {
 	ContextTimeGraphQL int       `json:"contextTimeGraphQL,omitempty" yaml:"contextTimeGraphQL,omitempty"` // contextTime sets the timeout of query
 }
 
+// DriverConfig stores the parameters for drivers of Databases.
+type DriverConfig struct {
+	MaxConn        int    `json:"maxConn,omitempty" yaml:"maxConn,omitempty"`               //for SQL and Mongo
+	MaxIdleTimeout int    `json:"maxIdleTimeout,omitempty" yaml:"maxIdleTimeout,omitempty"` //for SQL and Mongo
+	MinConn        uint64 `json:"minConn,omitempty" yaml:"minConn,omitempty"`               //only for Mongo
+	MaxIdleConn    int    `json:"maxIdleConn,omitempty" yaml:"maxIdleConn,omitempty"`       //only for SQL
+}
+
 // DatabaseConfig stores information of database config
 type DatabaseConfig struct {
-	DbAlias      string `json:"dbAlias,omitempty" yaml:"dbAlias"`
-	Type         string `json:"type,omitempty" yaml:"type"` // database type
-	DBName       string `json:"name,omitempty" yaml:"name"` // name of the logical database or schema name according to the database type
-	Conn         string `json:"conn,omitempty" yaml:"conn"`
-	IsPrimary    bool   `json:"isPrimary" yaml:"isPrimary"`
-	Enabled      bool   `json:"enabled" yaml:"enabled"`
-	BatchTime    int    `json:"batchTime,omitempty" yaml:"batchTime"`       // time in milli seconds
-	BatchRecords int    `json:"batchRecords,omitempty" yaml:"batchRecords"` // indicates number of records per batch
-	Limit        int64  `json:"limit,omitempty" yaml:"limit"`               // indicates number of records to send per request
+	DbAlias      string       `json:"dbAlias,omitempty" yaml:"dbAlias"`
+	Type         string       `json:"type,omitempty" yaml:"type"` // database type
+	DBName       string       `json:"name,omitempty" yaml:"name"` // name of the logical database or schema name according to the database type
+	Conn         string       `json:"conn,omitempty" yaml:"conn"`
+	IsPrimary    bool         `json:"isPrimary" yaml:"isPrimary"`
+	Enabled      bool         `json:"enabled" yaml:"enabled"`
+	BatchTime    int          `json:"batchTime,omitempty" yaml:"batchTime"`       // time in milli seconds
+	BatchRecords int          `json:"batchRecords,omitempty" yaml:"batchRecords"` // indicates number of records per batch
+	Limit        int64        `json:"limit,omitempty" yaml:"limit"`               // indicates number of records to send per request
+	DriverConf   DriverConfig `json:"driverConf,omitempty" yaml:"driverConf"`
 }
 
 // DatabaseSchema stores information of db schemas

gateway/go.mod

@@ -59,4 +59,4 @@ require (
 
 )
 
-go 1.14
+go 1.15

gateway/modules/crud/bolt/bolt.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spaceuptech/helpers"
 	"go.etcd.io/bbolt"
 
+	"github.com/spaceuptech/space-cloud/gateway/config"
 	"github.com/spaceuptech/space-cloud/gateway/model"
 	"github.com/spaceuptech/space-cloud/gateway/utils"
 )
@@ -40,8 +41,8 @@ func (b *Bolt) Close() error {
 }
 
 // IsSame checks if we've got the same connection string
-func (b *Bolt) IsSame(conn, dbName string) bool {
-	return b.connection == conn && dbName == b.bucketName
+func (b *Bolt) IsSame(conn, dbName string, driverConf config.DriverConfig) bool {
+	return b.connection == conn && dbName == b.bucketName //DriverConfig is not used for now.
 }
 
 // IsClientSafe checks whether database is enabled and connected

gateway/modules/crud/crud.go

@@ -63,7 +63,7 @@ type Crud interface {
 	RawBatch(ctx context.Context, batchedQueries []string) error
 	GetDBType() model.DBType
 	IsClientSafe(ctx context.Context) error
-	IsSame(conn, dbName string) bool
+	IsSame(conn, dbName string, driverConf config.DriverConfig) bool
 	Close() error
 	GetConnectionState(ctx context.Context) bool
 	SetQueryFetchLimit(limit int64)
@@ -74,14 +74,14 @@ func Init() *Module {
 	return &Module{batchMapTableToChan: make(batchMap), dataLoader: loader{loaderMap: map[string]*dataloader.Loader{}}}
 }
 
-func (m *Module) initBlock(dbType model.DBType, enabled bool, connection, dbName string) (Crud, error) {
+func (m *Module) initBlock(dbType model.DBType, enabled bool, connection, dbName string, driverConf config.DriverConfig) (Crud, error) {
 	switch dbType {
 	case model.Mongo:
-		return mgo.Init(enabled, connection, dbName)
+		return mgo.Init(enabled, connection, dbName, driverConf)
 	case model.EmbeddedDB:
 		return bolt.Init(enabled, connection, dbName)
 	case model.MySQL, model.Postgres, model.SQLServer:
-		c, err := sql.Init(dbType, enabled, connection, dbName)
+		c, err := sql.Init(dbType, enabled, connection, dbName, driverConf)
 		if err == nil && enabled {
 			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
@@ -90,7 +90,7 @@ func (m *Module) initBlock(dbType model.DBType, enabled bool, connection, dbName
 			}
 		}
 		if dbType == model.MySQL {
-			return sql.Init(dbType, enabled, fmt.Sprintf("%s%s", connection, dbName), dbName)
+			return sql.Init(dbType, enabled, fmt.Sprintf("%s%s", connection, dbName), dbName, driverConf)
 		}
 		return c, err
 	default:

gateway/modules/crud/mgo/mongo.go

@@ -10,6 +10,7 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 
+	"github.com/spaceuptech/space-cloud/gateway/config"
 	"github.com/spaceuptech/space-cloud/gateway/model"
 	"github.com/spaceuptech/space-cloud/gateway/utils"
 )
@@ -21,11 +22,12 @@ type Mongo struct {
 	connection      string
 	dbName          string
 	client          *mongo.Client
+	driverConf      config.DriverConfig
 }
 
 // Init initialises a new mongo instance
-func Init(enabled bool, connection, dbName string) (mongoStub *Mongo, err error) {
-	mongoStub = &Mongo{dbName: dbName, enabled: enabled, connection: connection, client: nil}
+func Init(enabled bool, connection, dbName string, driverConf config.DriverConfig) (mongoStub *Mongo, err error) {
+	mongoStub = &Mongo{dbName: dbName, enabled: enabled, connection: connection, client: nil, driverConf: driverConf}
 
 	if mongoStub.enabled {
 		err = mongoStub.connect()
@@ -44,8 +46,8 @@ func (m *Mongo) Close() error {
 }
 
 // IsSame checks if we've got the same connection string
-func (m *Mongo) IsSame(conn, dbName string) bool {
-	return strings.HasPrefix(m.connection, conn) && dbName == m.dbName
+func (m *Mongo) IsSame(conn, dbName string, driverConf config.DriverConfig) bool {
+	return strings.HasPrefix(m.connection, conn) && dbName == m.dbName && driverConf.MaxConn == m.driverConf.MaxConn && driverConf.MaxIdleTimeout == m.driverConf.MaxIdleTimeout && driverConf.MinConn == m.driverConf.MinConn
 }
 
 // IsClientSafe checks whether database is enabled and connected
@@ -70,7 +72,29 @@ func (m *Mongo) connect() error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeOut)
 	defer cancel()
 
-	client, err := mongo.NewClient(options.Client().ApplyURI(m.connection))
+	opts := options.Client().ApplyURI(m.connection)
+
+	maxConn := m.driverConf.MaxConn
+	if maxConn == 0 {
+		maxConn = 100
+	}
+
+	maxIdleTimeout := m.driverConf.MaxIdleTimeout
+	if maxIdleTimeout == 0 {
+		maxIdleTimeout = 60 * 5 * 1000
+	}
+
+	minConn := m.driverConf.MinConn
+	if minConn == 0 {
+		minConn = 10
+	}
+
+	opts = opts.SetMaxPoolSize((uint64)(maxConn))
+	duration := time.Duration(maxIdleTimeout) * time.Millisecond
+	opts = opts.SetMaxConnIdleTime(duration)
+	opts = opts.SetMinPoolSize(minConn)
+	client, err := mongo.NewClient(opts)
+
 	if err != nil {
 		return err
 	}

gateway/modules/crud/setters.go

@@ -56,7 +56,7 @@ func (m *Module) SetConfig(project string, crud config.DatabaseConfigs) error {
 		if m.block != nil {
 			m.block.SetQueryFetchLimit(v.Limit)
 			// Skip if the connection string is the same
-			if m.block.IsSame(connectionString, v.DBName) {
+			if m.block.IsSame(connectionString, v.DBName, v.DriverConf) {
 				continue
 			}
 			// Close the previous database connection
@@ -69,7 +69,7 @@ func (m *Module) SetConfig(project string, crud config.DatabaseConfigs) error {
 		var err error
 
 		v.Type = strings.TrimPrefix(v.Type, "sql-")
-		c, err = m.initBlock(model.DBType(v.Type), v.Enabled, connectionString, v.DBName)
+		c, err = m.initBlock(model.DBType(v.Type), v.Enabled, connectionString, v.DBName, v.DriverConf)
 
 		if v.Enabled {
 			if err != nil {

gateway/modules/crud/sql/sql.go

@@ -14,6 +14,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"   // Import for MySQL
 	_ "github.com/lib/pq"                // Import for postgres
 
+	"github.com/spaceuptech/space-cloud/gateway/config"
 	"github.com/spaceuptech/space-cloud/gateway/model"
 	"github.com/spaceuptech/space-cloud/gateway/utils"
 )
@@ -26,11 +27,12 @@ type SQL struct {
 	client          *sqlx.DB
 	dbType          string
 	name            string // logical db name or schema name according to the database type
+	driverConf      config.DriverConfig
 }
 
 // Init initialises a new sql instance
-func Init(dbType model.DBType, enabled bool, connection string, dbName string) (s *SQL, err error) {
-	s = &SQL{enabled: enabled, connection: connection, name: dbName, client: nil}
+func Init(dbType model.DBType, enabled bool, connection string, dbName string, driverConf config.DriverConfig) (s *SQL, err error) {
+	s = &SQL{enabled: enabled, connection: connection, name: dbName, client: nil, driverConf: driverConf}
 
 	switch dbType {
 	case model.Postgres:
@@ -55,8 +57,8 @@ func Init(dbType model.DBType, enabled bool, connection string, dbName string) (
 }
 
 // IsSame checks if we've got the same connection string
-func (s *SQL) IsSame(conn, dbName string) bool {
-	return strings.HasPrefix(s.connection, conn) && dbName == s.name
+func (s *SQL) IsSame(conn, dbName string, driverConf config.DriverConfig) bool {
+	return strings.HasPrefix(s.connection, conn) && dbName == s.name && driverConf.MaxConn == s.driverConf.MaxConn && driverConf.MaxIdleTimeout == s.driverConf.MaxIdleTimeout && driverConf.MaxIdleConn == s.driverConf.MaxIdleConn
 }
 
 // Close gracefully the SQL client
@@ -115,8 +117,25 @@ func (s *SQL) connect() error {
 
 	s.client = sql
 
-	s.client.SetMaxOpenConns(10)
-	s.client.SetMaxIdleConns(0)
+	maxConn := s.driverConf.MaxConn
+	if maxConn == 0 {
+		maxConn = 100
+	}
+
+	maxIdleConn := s.driverConf.MaxIdleConn
+	if maxIdleConn == 0 {
+		maxIdleConn = 50
+	}
+
+	maxIdleTimeout := s.driverConf.MaxIdleTimeout
+	if maxIdleTimeout == 0 {
+		maxIdleTimeout = 60 * 5 * 1000
+	}
+
+	s.client.SetMaxOpenConns(maxConn)
+	s.client.SetMaxIdleConns(maxIdleConn)
+	duration := time.Duration(maxIdleTimeout) * time.Millisecond
+	s.client.SetConnMaxIdleTime(duration)
 	return sql.PingContext(ctx)
 }
 

runner/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.9-alpine3.12
+FROM golang:1.15.3-alpine3.12
 WORKDIR /build
 COPY . .
 #RUN apk --no-cache add build-base

runner/go.mod

@@ -1,6 +1,6 @@
 module github.com/spaceuptech/space-cloud/runner
 
-go 1.14
+go 1.15
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.0.7

space-cli/go.mod

@@ -1,6 +1,6 @@
 module github.com/spaceuptech/space-cloud/space-cli
 
-go 1.13
+go 1.15
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.0.7