Merge branch 'crud-optimize' into 'ee-v0.10.0'

added crud block optimisation to share db drivers

See merge request spaceuptech/space-cloud-ee!7

Author: YourTechBud

Diff for: modules/crud/crud.go

@@ -1,57 +1,30 @@
 package crud
 
 import (
-	"context"
 	"errors"
 	"sync"
 
 	"github.com/spaceuptech/space-cloud/config"
-	"github.com/spaceuptech/space-cloud/model"
 	"github.com/spaceuptech/space-cloud/utils"
 
-	"github.com/spaceuptech/space-cloud/modules/crud/mgo"
-	"github.com/spaceuptech/space-cloud/modules/crud/sql"
+	"github.com/spaceuptech/space-cloud/modules/crud/driver"
 )
 
 // Module is the root block providing convenient wrappers
 type Module struct {
 	sync.RWMutex
-	blocks    map[string]Crud
+	blocks    map[string]*stub
 	primaryDB string
-}
-
-// Crud abstracts the implementation crud operations of databases
-type Crud interface {
-	Create(ctx context.Context, project, col string, req *model.CreateRequest) error
-	Read(ctx context.Context, project, col string, req *model.ReadRequest) (interface{}, error)
-	Update(ctx context.Context, project, col string, req *model.UpdateRequest) error
-	Delete(ctx context.Context, project, col string, req *model.DeleteRequest) error
-	Aggregate(ctx context.Context, project, col string, req *model.AggregateRequest) (interface{}, error)
-	Batch(ctx context.Context, project string, req *model.BatchRequest) error
-	GetDBType() utils.DBType
-	Close() error
+	h         *driver.Handler
 }
 
 // Init create a new instance of the Module object
-func Init() *Module {
-	return &Module{blocks: make(map[string]Crud)}
-}
-
-func initBlock(dbType utils.DBType, connection string) (Crud, error) {
-	switch dbType {
-	case utils.Mongo:
-		return mgo.Init(connection)
-
-	case utils.MySQL, utils.Postgres:
-		return sql.Init(dbType, connection)
-
-	default:
-		return nil, utils.ErrInvalidParams
-	}
+func Init(h *driver.Handler) *Module {
+	return &Module{blocks: make(map[string]*stub), h: h}
 }
 
 // GetPrimaryDB get the database configured as primary
-func (m *Module) GetPrimaryDB() (Crud, error) {
+func (m *Module) GetPrimaryDB() (driver.Crud, error) {
 	m.RLock()
 	defer m.RUnlock()
 
@@ -60,15 +33,7 @@ func (m *Module) GetPrimaryDB() (Crud, error) {
 		return nil, errors.New("CRUD: Primary DB not configured")
 	}
 
-	return c, nil
-}
-
-func (m *Module) getCrudBlock(dbType string) (Crud, error) {
-	if crud, p := m.blocks[dbType]; p {
-		return crud, nil
-	}
-
-	return nil, errors.New("CRUD: No crud block present for db")
+	return c.c, nil
 }
 
 // SetConfig set the rules adn secret key required by the crud block
@@ -78,26 +43,43 @@ func (m *Module) SetConfig(crud config.Crud) error {
 
 	// Close the previous database connections
 	for _, v := range m.blocks {
-		v.Close()
+		m.h.RemoveBlock(v.dbType, v.conn)
 	}
-	m.blocks = make(map[string]Crud, len(crud))
+
+	m.blocks = make(map[string]*stub, len(crud))
 
 	// Create a new crud blocks
-	for k, v := range crud {
+	for dbType, v := range crud {
 		// Skip this block if it is not enabled
 		if !v.Enabled {
 			continue
 		}
 
-		c, err := initBlock(utils.DBType(k), v.Conn)
+		// Initialise a new block
+		c, err := m.h.InitBlock(utils.DBType(dbType), v.Conn)
 		if err != nil {
-			return errors.New("CURD: Error - " + k + " could not be initialised")
+			return errors.New("CURD: Error - " + dbType + " could not be initialised")
 		}
+
 		if v.IsPrimary {
-			m.primaryDB = k
+			m.primaryDB = dbType
 		}
-		m.blocks[k] = c
+		m.blocks[dbType] = &stub{c: c, conn: v.Conn, dbType: utils.DBType(dbType)}
 	}
 
 	return nil
 }
+
+type stub struct {
+	conn   string
+	c      driver.Crud
+	dbType utils.DBType
+}
+
+func (m *Module) getCrudBlock(dbType string) (driver.Crud, error) {
+	if crud, p := m.blocks[dbType]; p {
+		return crud.c, nil
+	}
+
+	return nil, errors.New("CRUD: No crud block present for db")
+}

Diff for: modules/crud/driver/driver.go

@@ -0,0 +1,103 @@
+package driver
+
+import (
+	"context"
+	"sync"
+
+	"github.com/spaceuptech/space-cloud/model"
+	"github.com/spaceuptech/space-cloud/modules/crud/mgo"
+	"github.com/spaceuptech/space-cloud/modules/crud/sql"
+	"github.com/spaceuptech/space-cloud/utils"
+)
+
+// Crud abstracts the implementation crud operations of databases
+type Crud interface {
+	Create(ctx context.Context, project, col string, req *model.CreateRequest) error
+	Read(ctx context.Context, project, col string, req *model.ReadRequest) (interface{}, error)
+	Update(ctx context.Context, project, col string, req *model.UpdateRequest) error
+	Delete(ctx context.Context, project, col string, req *model.DeleteRequest) error
+	Aggregate(ctx context.Context, project, col string, req *model.AggregateRequest) (interface{}, error)
+	Batch(ctx context.Context, project string, req *model.BatchRequest) error
+	GetDBType() utils.DBType
+	Close() error
+}
+
+// Handler is the object managing the database connections
+type Handler struct {
+	lock    sync.Mutex
+	drivers map[string]*stub
+}
+
+// New creates a new driver handler
+func New() *Handler {
+	return &Handler{drivers: map[string]*stub{}}
+}
+
+// InitBlock creates and returns a new crud object. If the driver already exists, it returns that instead
+func (h *Handler) InitBlock(dbType utils.DBType, connection string) (Crud, error) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	// See if driver is present in cache
+	s, p := h.getBlock(dbType, connection)
+	if p {
+		s.addCount()
+		return s.getCrud(), nil
+	}
+
+	return h.addBlock(dbType, connection)
+}
+
+// RemoveBlock removes a crud object if other module is referencing it
+func (h *Handler) RemoveBlock(dbType utils.DBType, connection string) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	// See if driver is present in cache
+	s, p := h.getBlock(dbType, connection)
+	if !p {
+		return
+	}
+
+	// Subtract the count of references
+	s.subtractCount()
+
+	// Close then delete the driver if there are no references
+	if s.getCount() == 0 {
+		s.getCrud().Close()
+		delete(h.drivers, generateKey(dbType, connection))
+	}
+}
+
+func (h *Handler) getBlock(dbType utils.DBType, connection string) (s *stub, p bool) {
+	s, p = h.drivers[generateKey(dbType, connection)]
+	return
+}
+
+func (h *Handler) addBlock(dbType utils.DBType, connection string) (Crud, error) {
+	var c Crud
+	var err error
+
+	switch dbType {
+	case utils.Mongo:
+		c, err = mgo.Init(connection)
+
+	case utils.MySQL, utils.Postgres:
+		c, err = sql.Init(dbType, connection)
+
+	default:
+		c, err = nil, utils.ErrInvalidParams
+	}
+
+	// Return the error of exists
+	if err != nil {
+		return nil, err
+	}
+
+	h.drivers[generateKey(dbType, connection)] = newStub(c)
+	return c, nil
+}
+
+func generateKey(dbType utils.DBType, connection string) string {
+	return string(dbType) + ":" + connection
+}

Diff for: modules/crud/driver/stub.go

@@ -0,0 +1,26 @@
+package driver
+
+type stub struct {
+	c     Crud
+	count int
+}
+
+func newStub(c Crud) *stub {
+	return &stub{c: c, count: 1}
+}
+
+func (s *stub) addCount() {
+	s.count++
+}
+
+func (s *stub) subtractCount() {
+	s.count--
+}
+
+func (s *stub) getCount() int {
+	return s.count
+}
+
+func (s *stub) getCrud() Crud {
+	return s.c
+}

Diff for: modules/crud/operations.go

@@ -71,7 +71,7 @@ func (m *Module) Aggregate(ctx context.Context, dbType, project, col string, req
 	return crud.Aggregate(ctx, project, col, req)
 }
 
-// Create inserts a document (or multiple when op is "all") into the database based on dbType
+// Batch performs an transactional batch
 func (m *Module) Batch(ctx context.Context, dbType, project string, req *model.BatchRequest) error {
 	m.RLock()
 	defer m.RUnlock()

Diff for: utils/projects/projects.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/spaceuptech/space-cloud/modules/auth"
 	"github.com/spaceuptech/space-cloud/modules/crud"
+	"github.com/spaceuptech/space-cloud/modules/crud/driver"
 	"github.com/spaceuptech/space-cloud/modules/filestore"
 	"github.com/spaceuptech/space-cloud/modules/functions"
 	"github.com/spaceuptech/space-cloud/modules/realtime"
@@ -29,11 +30,12 @@ type ProjectState struct {
 type Projects struct {
 	lock     sync.RWMutex
 	projects map[string]*ProjectState
+	h        *driver.Handler
 }
 
 // New creates a new Projects instance
-func New() *Projects {
-	return &Projects{projects: map[string]*ProjectState{}}
+func New(h *driver.Handler) *Projects {
+	return &Projects{projects: map[string]*ProjectState{}, h: h}
 }
 
 // LoadProject returns the state of the project specified
@@ -77,7 +79,7 @@ func (p *Projects) NewProject(project string) *ProjectState {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
-	c := crud.Init()
+	c := crud.Init(p.h)
 	f := functions.Init()
 	a := auth.Init(c, nil)
 	u := userman.Init(c, a)

Diff for: utils/server/server.go

@@ -15,6 +15,7 @@ import (
 	"google.golang.org/grpc/credentials"
 
 	"github.com/spaceuptech/space-cloud/config"
+	"github.com/spaceuptech/space-cloud/modules/crud/driver"
 	"github.com/spaceuptech/space-cloud/proto"
 	"github.com/spaceuptech/space-cloud/utils/projects"
 )
@@ -33,7 +34,7 @@ type Server struct {
 // New creates a new server instance
 func New(isProd bool) *Server {
 	r := mux.NewRouter()
-	projects := projects.New()
+	projects := projects.New(driver.New())
 	id := uuid.NewV1().String()
 	return &Server{id: id, router: r, projects: projects, isProd: isProd}
 }