feat: add DAL for MySQL (#62)

* feat: add DAL for MySQL

* Update CI&CD.yml

* Update CI&CD.yml

* Update CI&CD.yml

* Update CI&CD.yml

Author: powerman

.circleci/config.yml

@@ -7,6 +7,7 @@ setup_env: &setup_env
     command: |
       env | sort > /tmp/env.old
 
+      export DOCKERIZE_VER=0.12.0
       export HADOLINT_VER=1.18.0
       export SHELLCHECK_VER=0.7.1
       export GOLANGCI_LINT_VER=1.31.0
@@ -32,6 +33,13 @@ jobs:
         environment:
           GOFLAGS: "-mod=readonly"
           EXAMPLE_APIKEY_ADMIN: "admin"
+          EXAMPLE_MYSQL_ADDR_HOST: "localhost"
+          EXAMPLE_MYSQL_AUTH_LOGIN: "root"
+          EXAMPLE_MYSQL_AUTH_PASS: ""
+          NARADA4D_TEST_MYSQL: "goose-mysql://[email protected]"
+      - image: "mysql:5.6"
+        environment:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
     steps:
       - checkout
       - *setup_env
@@ -46,6 +54,8 @@ jobs:
             GO111MODULE: "on"
           command: |
             cd /tmp # Protect go.mod for modifications by `go get`.
+            dockerize --version | tee /dev/stderr | grep -wq v$DOCKERIZE_VER ||
+              curl -sSfL https://github.com/powerman/dockerize/releases/download/v${DOCKERIZE_VER}/dockerize-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/dockerize
             hadolint --version | tee /dev/stderr | grep -wq v$HADOLINT_VER ||
               curl -sSfL https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VER}/hadolint-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/hadolint
             shellcheck --version | tee /dev/stderr | grep -wq $SHELLCHECK_VER ||

.dockerignore

@@ -4,3 +4,4 @@
 # a*b?c/d[0-9]e[^a-z\]]f\[g     - pattern
 *
 !bin
+!internal/migrations/mysql/*.sql

.github/workflows/CI&CD.yml

@@ -16,14 +16,26 @@ jobs:
   test:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    services:
+      mysql:
+        image: 'mysql:5.6'
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+        ports:
+          - 3306:3306
     env:
+      DOCKERIZE_VER: '0.12.0'
       HADOLINT_VER: '1.18.0'
       SHELLCHECK_VER: '0.7.1'
       GOLANGCI_LINT_VER: '1.31.0'
       GOTESTSUM_VER: '0.5.3'
       GOSWAGGER_VER: '0.25.0'
       GOVERALLS_VER: '0.0.7'
       EXAMPLE_APIKEY_ADMIN: 'admin'
+      EXAMPLE_MYSQL_ADDR_HOST: 'localhost'
+      EXAMPLE_MYSQL_AUTH_LOGIN: 'root'
+      EXAMPLE_MYSQL_AUTH_PASS: ''
+      NARADA4D_TEST_MYSQL: 'goose-mysql://[email protected]'
     steps:
       - uses: actions/setup-go@v2
         with:
@@ -52,6 +64,8 @@ jobs:
           GO111MODULE: 'on'
         run: |
           cd /tmp # Protect go.mod for modifications by `go get`.
+          dockerize --version | tee /dev/stderr | grep -wq v$DOCKERIZE_VER ||
+            curl -sSfL https://github.com/powerman/dockerize/releases/download/v${DOCKERIZE_VER}/dockerize-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/dockerize
           hadolint --version | tee /dev/stderr | grep -wq v$HADOLINT_VER ||
             curl -sSfL https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VER}/hadolint-$(uname)-x86_64 | install /dev/stdin $(go env GOPATH)/bin/hadolint
           shellcheck --version | tee /dev/stderr | grep -wq $SHELLCHECK_VER ||

.golangci.yml

@@ -41,6 +41,7 @@ run:
   # on Windows.
   skip-files:
     - "\\.[\\w-]+\\.go$"
+    - "/migrations/(.*/)?[0-9]{5}_"
 
   # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
   # If invoked with -mod=readonly, the go command is disallowed from the implicit

README.md

@@ -271,7 +271,6 @@ $ ./bin/address-book serve
 ## TODO
 
 - [ ] Update JSON Schema support cheatsheet to latest go-swagger version.
-- [ ] Replace trivial in-memory DAL with more complete one based on
-  Postgresql with metrics and migrations support.
+- [ ] Add alternative DAL implementation for Postgresql.
 - [ ] Add cookie-based auth with CSRF middleware.
 - [ ] Add an example of adapter for external service in `svc/something`.

cmd/address-book/init_test.go

@@ -10,13 +10,15 @@ import (
 	"github.com/powerman/go-service-example/api/openapi/model"
 	"github.com/powerman/go-service-example/internal/app"
 	"github.com/powerman/go-service-example/internal/config"
+	dal "github.com/powerman/go-service-example/internal/dal/mysql"
 	"github.com/powerman/go-service-example/internal/srv/openapi"
 	"github.com/powerman/go-service-example/pkg/def"
 )
 
 func TestMain(m *testing.M) {
 	def.Init()
 	initMetrics(reg, "test")
+	dal.InitMetrics(reg, "test")
 	app.InitMetrics(reg)
 	openapi.InitMetrics(reg, "test")
 	cfg = config.MustGetServeTest()
@@ -32,3 +34,7 @@ var (
 	apiKeyUser  = oapiclient.APIKeyAuth("API-Key", "header", "user")
 	apiContact1 = &model.Contact{ID: 1, Name: swag.String("A")}
 )
+
+type tLogger check.C
+
+func (t tLogger) Print(args ...interface{}) { t.Log(args...) }

cmd/address-book/service.go

@@ -7,8 +7,10 @@ import (
 	"github.com/powerman/go-service-example/api/openapi/restapi"
 	"github.com/powerman/go-service-example/internal/app"
 	"github.com/powerman/go-service-example/internal/config"
-	dal "github.com/powerman/go-service-example/internal/dal/memory"
+	dal "github.com/powerman/go-service-example/internal/dal/mysql"
+	migrations_mysql "github.com/powerman/go-service-example/internal/migrations/mysql"
 	"github.com/powerman/go-service-example/internal/srv/openapi"
+	"github.com/powerman/go-service-example/pkg/cobrax"
 	"github.com/powerman/go-service-example/pkg/concurrent"
 	"github.com/powerman/go-service-example/pkg/def"
 	"github.com/powerman/go-service-example/pkg/serve"
@@ -29,14 +31,19 @@ type service struct {
 	srv  *restapi.Server
 }
 
-func initService(_, serveCmd *cobra.Command) error {
+func initService(cmd, serveCmd *cobra.Command) error {
 	namespace := regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(def.ProgName, "_")
 	initMetrics(reg, namespace)
+	dal.InitMetrics(reg, namespace)
 	app.InitMetrics(reg)
 	openapi.InitMetrics(reg, namespace)
 
+	gooseMySQLCmd := cobrax.NewGooseMySQLCmd(migrations_mysql.Goose(), config.GetGooseMySQL)
+	cmd.AddCommand(gooseMySQLCmd)
+
 	return config.Init(config.FlagSets{
-		Serve: serveCmd.Flags(),
+		Serve:      serveCmd.Flags(),
+		GooseMySQL: gooseMySQLCmd.Flags(),
 	})
 }
 
@@ -78,7 +85,7 @@ func (s *service) runServe(ctxStartup, ctxShutdown Ctx, shutdown func()) (err er
 }
 
 func (s *service) connectRepo(ctx Ctx) (interface{}, error) {
-	return dal.New(ctx)
+	return dal.New(ctx, s.cfg.MySQLGooseDir, s.cfg.MySQL)
 }
 
 func (s *service) serveMetrics(ctx Ctx) error {

cmd/address-book/service_integration_test.go

@@ -14,11 +14,17 @@ import (
 	"github.com/powerman/go-service-example/internal/srv/openapi"
 	"github.com/powerman/go-service-example/pkg/def"
 	"github.com/powerman/go-service-example/pkg/netx"
+	"github.com/powerman/mysqlx"
 )
 
 func TestSmoke(tt *testing.T) {
 	t := check.T(tt)
 
+	tempDBCfg, cleanup, err := mysqlx.EnsureTempDB(tLogger(*t), "", cfg.MySQL)
+	cfg.MySQL = tempDBCfg
+	t.Must(t.Nil(err))
+	defer cleanup()
+
 	s := &service{cfg: cfg}
 
 	ctxStartup, cancel := context.WithTimeout(ctx, def.TestTimeout)
@@ -29,6 +35,9 @@ func TestSmoke(tt *testing.T) {
 	defer func() {
 		shutdown()
 		t.Nil(<-errc, "RunServe")
+		if s.repo != nil {
+			s.repo.Close()
+		}
 	}()
 	t.Must(t.Nil(netx.WaitTCPPort(ctxStartup, cfg.Addr), "connect to service"))
 

docker-compose.yml

@@ -1,7 +1,21 @@
 version: "3.8"
 
+volumes:
+  mysql:
+
 services:
 
+  mysql:
+    image: "mysql:5.6"
+    container_name: example_mysql
+    restart: always
+    ports:
+      - "${example_mysql_addr_port:-0}:3306"
+    volumes:
+      - "mysql:/var/lib/mysql"
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+
   address-book:
     build:
       context: .
@@ -14,3 +28,6 @@ services:
       - "${EXAMPLE_METRICS_ADDR_PORT:-0}:9000"
     environment:
       EXAMPLE_APIKEY_ADMIN: "${EXAMPLE_APIKEY_ADMIN:?}"
+      EXAMPLE_MYSQL_ADDR_HOST: "mysql"
+      EXAMPLE_MYSQL_AUTH_LOGIN: "root"
+      EXAMPLE_MYSQL_AUTH_PASS: ""

env.sh.dist

@@ -5,11 +5,18 @@
 
 # - Set all _PORT vars to port numbers not used by your system.
 
+export example_mysql_addr_port="3306"
+
 export EXAMPLE_APIKEY_ADMIN="admin"
 export EXAMPLE_ADDR_HOST="localhost"
 export EXAMPLE_ADDR_PORT="8000"
 export EXAMPLE_METRICS_ADDR_PORT="9000"
+export EXAMPLE_MYSQL_ADDR_HOST="127.0.0.1"
+export EXAMPLE_MYSQL_ADDR_PORT="${example_mysql_addr_port}"
+export EXAMPLE_MYSQL_AUTH_LOGIN="root"
+export EXAMPLE_MYSQL_AUTH_PASS=""
 export GO_TEST_TIME_FACTOR="1.0" # Increase if tests fail because of slow CPU.
+export NARADA4D_TEST_MYSQL="goose-mysql://[email protected]:${example_mysql_addr_port}"
 
 # DO NOT MODIFY BELOW THIS LINE!
 env1="$(sed -e '/^$/d' -e '/^#/d' -e 's/=.*//' env.sh.dist)"

go.mod

@@ -11,12 +11,18 @@ require (
 	github.com/go-openapi/strfmt v0.19.5
 	github.com/go-openapi/swag v0.19.9
 	github.com/go-openapi/validate v0.19.11
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/mock v1.4.4
 	github.com/jessevdk/go-flags v1.4.0
+	github.com/jmoiron/sqlx v1.2.0
 	github.com/powerman/appcfg v0.5.0
 	github.com/powerman/check v1.2.1
 	github.com/powerman/getenv v0.1.0
+	github.com/powerman/goose/v2 v2.7.0
 	github.com/powerman/must v0.1.0
+	github.com/powerman/mysqlx v0.3.3
+	github.com/powerman/narada4d v1.7.0
+	github.com/powerman/sqlxx v0.2.0
 	github.com/powerman/structlog v0.7.1
 	github.com/prometheus/client_golang v1.7.1
 	github.com/rs/cors v1.7.0