feat(secrets): support for csi directories (#565)

Co-authored-by: Ted Dorfeuille <[email protected]>
Co-authored-by: Matt Terwilliger <[email protected]>

Author: 3 people

secrets/secrets.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/fs"
 
 	"github.com/reddit/baseplate.go/errorsbp"
 )
@@ -28,6 +29,11 @@ func (s Secret) IsEmpty() bool {
 	return len(s) == 0
 }
 
+// CSIFile represents the raw parsed object of a file made by the Vault CSI provider
+type CSIFile struct {
+	Secret GenericSecret `json:"data"`
+}
+
 // Secrets allows to access secrets based on their different type.
 type Secrets struct {
 	simpleSecrets     map[string]SimpleSecret
@@ -258,16 +264,32 @@ func NewSecrets(r io.Reader) (*Secrets, error) {
 		return nil, err
 	}
 
-	err = secretsDocument.Validate()
+	return secretsValidate(secretsDocument)
+}
+
+// FromDir parses a directory and returns its secrets
+func FromDir(dir fs.FS) (*Secrets, error) {
+	secretsDocument := Document{
+		Secrets: make(map[string]GenericSecret),
+	}
+	secretsDocument, err := walkCSIDirectory(dir)
 	if err != nil {
 		return nil, err
 	}
+	return secretsValidate(secretsDocument)
+
+}
+
+func secretsValidate(secretsDocument Document) (*Secrets, error) {
 	secrets := &Secrets{
 		simpleSecrets:     make(map[string]SimpleSecret),
 		versionedSecrets:  make(map[string]VersionedSecret),
 		credentialSecrets: make(map[string]CredentialSecret),
 		vault:             secretsDocument.Vault,
 	}
+	if err := secretsDocument.Validate(); err != nil {
+		return nil, err
+	}
 	for key, secret := range secretsDocument.Secrets {
 		switch secret.Type {
 		case "simple":
@@ -298,3 +320,39 @@ func NewSecrets(r io.Reader) (*Secrets, error) {
 	}
 	return secrets, nil
 }
+
+// walkCSIDirectory parses a directory for vault secrets and merges them into one object
+func walkCSIDirectory(dir fs.FS) (Document, error) {
+	secretsDocument := Document{
+		Secrets: make(map[string]GenericSecret),
+	}
+	err := fs.WalkDir(
+		dir,
+		".",
+		func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() {
+				return nil
+			}
+			file, err := dir.Open(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+
+			var secretFile CSIFile
+			err = json.NewDecoder(file).Decode(&secretFile)
+			if err != nil {
+				return fmt.Errorf("decoding %q: %w", path, err)
+			}
+			secretsDocument.Secrets[path] = secretFile.Secret
+			return nil
+		},
+	)
+	if err != nil {
+		return Document{}, fmt.Errorf("Error during walkCSIDirectory: %w", err)
+	}
+	return secretsDocument, nil
+}

secrets/store.go

@@ -3,6 +3,8 @@ package secrets
 import (
 	"context"
 	"io"
+	"io/fs"
+	"os"
 
 	"github.com/reddit/baseplate.go/filewatcher"
 	"github.com/reddit/baseplate.go/log"
@@ -42,12 +44,21 @@ func NewStore(ctx context.Context, path string, logger log.Wrapper, middlewares
 		secretHandlerFunc: nopSecretHandlerFunc,
 	}
 	store.secretHandler(middlewares...)
+	fileInfo, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	parser := store.parser
+	if fileInfo.IsDir() {
+		parser = filewatcher.WrapDirParser(store.dirParser)
+	}
 
 	result, err := filewatcher.New(
 		ctx,
 		filewatcher.Config{
 			Path:   path,
-			Parser: store.parser,
+			Parser: parser,
 			Logger: logger,
 		},
 	)
@@ -59,7 +70,7 @@ func NewStore(ctx context.Context, path string, logger log.Wrapper, middlewares
 	return store, nil
 }
 
-func (s *Store) parser(r io.Reader) (interface{}, error) {
+func (s *Store) parser(r io.Reader) (any, error) {
 	secrets, err := NewSecrets(r)
 	if err != nil {
 		return nil, err
@@ -70,6 +81,17 @@ func (s *Store) parser(r io.Reader) (interface{}, error) {
 	return secrets, nil
 }
 
+func (s *Store) dirParser(dir fs.FS) (any, error) {
+	secrets, err := FromDir(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	s.secretHandlerFunc(secrets)
+
+	return secrets, nil
+}
+
 // secretHandler creates the middleware chain.
 func (s *Store) secretHandler(middlewares ...SecretMiddleware) {
 	for _, m := range middlewares {

secrets/store_test.go

@@ -37,12 +37,72 @@ const specificationExample = `
 	}
 }`
 
+var externalAccountKey = `
+{
+  "request_id": "1afc3036-2282-d483-c2d4-6d483efdf16c",
+  "lease_id": "",
+  "lease_duration": 2764800,
+  "renewable": false,
+  "data": {
+    "type": "versioned",
+    "current": "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU=",
+    "previous": "aHVudGVyMg=="
+  },
+  "warnings": null
+}
+`
+
+var someAPIKey = `
+{
+  "request_id": "1afc3036-2282-d483-c2d4-6d483efdf16c",
+  "lease_id": "",
+  "lease_duration": 2764800,
+  "renewable": false,
+  "data": {
+    "type": "simple",
+    "value": "Y2RvVXhNMVdsTXJma3BDaHRGZ0dPYkVGSg==",
+    "encoding": "base64"
+  },
+  "warnings": null
+}
+`
+var someDatabaseCredentials = `
+{
+  "request_id": "1afc3036-2282-d483-c2d4-6d483efdf16c",
+  "lease_id": "",
+  "lease_duration": 2764800,
+  "renewable": false,
+  "data": {
+    "type": "credential",
+    "username": "spez",
+    "password": "hunter2"
+  },
+  "warnings": null
+}
+`
+
 func TestGetSimpleSecret(t *testing.T) {
 	dir := t.TempDir()
+	dirCSI := t.TempDir()
 	tmpFile, err := os.CreateTemp(dir, "secrets.json")
 	if err != nil {
 		t.Fatal(err)
 	}
+	if err := os.Mkdir(dirCSI+"/secret", 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(dirCSI+"/secret/myservice", 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(dirCSI+"/secret/myservice/external-account-key", []byte(externalAccountKey), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(dirCSI+"/secret/myservice/some-api-key", []byte(someAPIKey), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(dirCSI+"/secret/myservice/some-database-credentials", []byte(someDatabaseCredentials), 0777); err != nil {
+		t.Fatal(err)
+	}
 	tmpPath := tmpFile.Name()
 	tmpFile.Write([]byte(specificationExample))
 	if err := tmpFile.Close(); err != nil {
@@ -71,21 +131,28 @@ func TestGetSimpleSecret(t *testing.T) {
 		t.Run(
 			tt.name,
 			func(t *testing.T) {
-				store, err := secrets.NewStore(context.Background(), tmpPath, log.TestWrapper(t))
-				if err != nil {
-					t.Fatal(err)
-				}
-				defer store.Close()
-
-				secret, err := store.GetSimpleSecret(tt.key)
-				if tt.expectedError == nil && err != nil {
-					t.Fatal(err)
-				}
-				if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
-					t.Fatalf("expected error %v, actual: %v", tt.expectedError, err)
-				}
-				if !reflect.DeepEqual(secret, tt.expected) {
-					t.Fatalf("expected %+v, actual: %+v", tt.expected, secret)
+				for label, path := range map[string]string{
+					"file": tmpPath,
+					"dir":  dirCSI,
+				} {
+					t.Run(label, func(t *testing.T) {
+						store, err := secrets.NewStore(context.Background(), path, log.TestWrapper(t))
+						if err != nil {
+							t.Fatal(err)
+						}
+						t.Cleanup(func() { store.Close() })
+
+						secret, err := store.GetSimpleSecret(tt.key)
+						if tt.expectedError == nil && err != nil {
+							t.Fatal(err)
+						}
+						if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
+							t.Fatalf("expected error %v, actual: %v", tt.expectedError, err)
+						}
+						if !reflect.DeepEqual(secret, tt.expected) {
+							t.Fatalf("expected %+v, actual: %+v", tt.expected, secret)
+						}
+					})
 				}
 			},
 		)