make storage tests synchronous

Author: sechmann

.gitignore

@@ -1,5 +1,4 @@
 .idea
-go.sum
 bin
 testapp
 !testapp/

cmd/testapp/main.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"os/signal"
@@ -15,13 +16,15 @@ import (
 
 	"github.com/gorilla/mux"
 	_ "github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+	flag "github.com/spf13/pflag"
+
 	"github.com/nais/testapp/pkg/bigquery"
 	"github.com/nais/testapp/pkg/bucket"
 	"github.com/nais/testapp/pkg/database"
 	"github.com/nais/testapp/pkg/metrics"
+	"github.com/nais/testapp/pkg/testable"
 	"github.com/nais/testapp/pkg/version"
-	log "github.com/sirupsen/logrus"
-	flag "github.com/spf13/pflag"
 )
 
 var (
@@ -107,6 +110,9 @@ func timeSinceDeploy() float64 {
 }
 
 func main() {
+	programContext, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	interrupt := make(chan os.Signal, 1)
 	signal.Notify(interrupt, syscall.SIGTERM, syscall.SIGINT)
 	hostname, _ := os.Hostname()
@@ -207,32 +213,56 @@ func main() {
 		w.WriteHeader(http.StatusOK)
 		_, _ = fmt.Fprintf(w, "HTTP status: %d, body:\n%s", resp.StatusCode, string(b))
 	})
-	// Bucket set-ups and endpoints
-	r.HandleFunc("/readbucket", bucket.ReadBucketHandler(bucketName, bucketObjectName))
-	r.HandleFunc("/writebucket", bucket.WriteBucketHandler(bucketName, bucketObjectName)).Methods(http.MethodPost)
 
-	// Ceph bucket set-ups and endpoints
+	// Holds all tests. Possible feature: /test to run all tests
+	var tests []testable.Testable
+
+	// Set up google bucket test
+	bucketTest, err := bucket.NewGoogleBucketTest(programContext, bucketName, bucketObjectName)
+	if err != nil {
+		log.Errorf("Error setting up bucket test: %v", err)
+	} else {
+		tests = append(tests, bucketTest)
+	}
+
+	// Set up ceph test
 	if rgwAddress != "" && rgwAccessKey != "" && rgwSecretKey != "" {
-		bucket.Ceph.CephInit(rgwAddress, bucketName, "us-east-1", rgwAccessKey, rgwSecretKey)
-		r.HandleFunc("/readrgwbucket", bucket.Ceph.ReadBucketHandler(bucketObjectName))
-		r.HandleFunc("/writergwbucket", bucket.Ceph.WriteBucketHandler(bucketObjectName)).Methods(http.MethodPost)
+		cephTest, err := bucket.NewCephBucketTest(rgwAddress, bucketName, "us-east-1", rgwAccessKey, rgwSecretKey, bucketObjectName)
+		if err != nil {
+			log.Errorf("Error setting up ceph bucket test: %v", err)
+		} else {
+			tests = append(tests, cephTest)
+		}
 	}
 
-	// DB set-ups and endpoints
-	r.HandleFunc("/writedb", database.WriteDatabaseHandler(dbUser, dbPassword, dbName, dbHost)).Methods(http.MethodPost)
-	r.HandleFunc("/readdb", database.ReadDatabaseHandler(dbUser, dbPassword, dbName, dbHost))
+	// Set up database test
+	databaseTest, err := database.NewDatabaseTest(dbUser, dbPassword, dbName, dbHost)
+	if err != nil {
+		log.Errorf("Error setting up bucket test: %v", err)
+	} else {
+		tests = append(tests, databaseTest)
+	}
 
-	// Bigquery set-ups and endpoints
+	// Set up bigquery test
 	if bigqueryName != "" && bigqueryTableName != "" {
-		// Use input parameters as feature toggle for bigquery
-		//  since testrig will crash apparently
-		err := bigquery.CreateDatasetAndTable(projectID, bigqueryName, bigqueryTableName)
-		switch {
-		case err != nil:
-			log.Errorf("Unable to create bigquery dataset and table, all tests will return HTTP STATUS 404: %v", err)
-		default:
-			r.HandleFunc("/writebigquery", bigquery.WriteBigQueryHandler(projectID, bigqueryName, bigqueryTableName)).Methods(http.MethodPost)
-			r.HandleFunc("/readbigquery", bigquery.ReadBigQueryHandler(projectID, bigqueryName, bigqueryTableName))
+		bq, err := bigquery.NewBigqueryTest(programContext, projectID, bigqueryName, bigqueryTableName)
+		err = bq.Init(programContext)
+		if err != nil {
+			log.Errorf("Error setting up bigquery test: %v", err)
+		} else {
+			tests = append(tests, bq)
+		}
+	}
+
+	for _, test := range tests {
+		err := test.Init(programContext)
+		if err != nil {
+			log.Errorf("Error initializing test: %s, will not set up handler.", test.Name())
+		} else {
+			setupTestHandler(r, test)
+
+			//goland:noinspection GoDeferInLoop
+			defer test.Cleanup()
 		}
 	}
 
@@ -243,7 +273,10 @@ func main() {
 	server := &http.Server{Addr: bindAddr, Handler: r}
 
 	go func() {
-		log.Fatal(server.ListenAndServe())
+		err := server.ListenAndServe()
+		if err != http.ErrServerClosed {
+			log.Errorf("closing http server: %v", err)
+		}
 	}()
 
 	<-interrupt
@@ -252,5 +285,35 @@ func main() {
 	time.Sleep(time.Duration(gracefulShutdownPeriodSeconds) * time.Second)
 	log.Print("shutting down")
 
-	_ = server.Shutdown(context.Background())
+	_ = server.Shutdown(programContext)
+}
+
+func setupTestHandler(router *mux.Router, test testable.Testable) {
+	log.Infof("setting up %s test handler", test.Name())
+
+	path := fmt.Sprintf("%s/test", test.Name())
+	router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
+		expected := fmt.Sprintf("%x", 9999+rand.Intn(999999))[:4]
+		result, err := test.Test(r.Context(), expected)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, respondErr := fmt.Fprintf(w, "%s test: error: %v", test.Name(), err)
+			if respondErr != nil {
+				log.Errorf("%s test: write response err: %v (test err: %v)", test.Name(), respondErr, err)
+			} else {
+				log.Warnf("%s test: failed, %v", test.Name(), err)
+			}
+			return
+		}
+
+		if expected != result {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, err := fmt.Fprintf(w, "%s test: data missmatch, epected: %s got: %s", test.Name(), expected, result)
+			if err != nil {
+				log.Errorf("%s test: unable to write response: %v", test.Name(), err)
+			}
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	})
 }

go.mod

@@ -13,4 +13,33 @@ require (
 	google.golang.org/api v0.9.0
 )
 
-go 1.16
+require (
+	cloud.google.com/go v0.46.3 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
+	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 // indirect
+	github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect
+	go.opencensus.io v0.22.0 // indirect
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+	golang.org/x/exp v0.0.0-20190912063710-ac5d2bfcbfe0 // indirect
+	golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac // indirect
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
+	golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect
+	golang.org/x/text v0.3.3 // indirect
+	golang.org/x/tools v0.0.0-20190917162342-3b4f30a44f3b // indirect
+	google.golang.org/appengine v1.6.1 // indirect
+	google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51 // indirect
+	google.golang.org/grpc v1.21.1 // indirect
+	honnef.co/go/tools v0.0.1-2019.2.3 // indirect
+)
+
+go 1.17