From 5700ef56f0819014f49c56a2746cd791f2e6ae7f Mon Sep 17 00:00:00 2001 From: Matt Wilder Date: Tue, 4 Mar 2025 14:53:13 -0800 Subject: [PATCH 1/4] feat(jsonnet): add support for importing all files in a directory Add native function `importFiles` which reads and evaluates all jsonnet files in a directory. Returns an object of with key/value pairs of `: ` Directory path is found using a `calledFrom` option, similar to `helmTemplate`. Includes support for: - Choosing files by extension using the `extension` option (default is `.libsonnet`) - Excluding files using the `exclude` option, which takes an array of file names to skip --- pkg/jsonnet/implementations/goimpl/vm.go | 4 ++ pkg/jsonnet/native/funcs.go | 84 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/pkg/jsonnet/implementations/goimpl/vm.go b/pkg/jsonnet/implementations/goimpl/vm.go index cafb79d7b..80e17e66f 100644 --- a/pkg/jsonnet/implementations/goimpl/vm.go +++ b/pkg/jsonnet/implementations/goimpl/vm.go @@ -25,6 +25,10 @@ func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[stri vm.NativeFunction(nf) } + for _, nvf := range native.VMFuncs(vm) { + vm.NativeFunction(nvf) + } + if maxStack > 0 { vm.MaxStack = maxStack } diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go index d42fc33d3..368fe0c92 100644 --- a/pkg/jsonnet/native/funcs.go +++ b/pkg/jsonnet/native/funcs.go @@ -6,7 +6,10 @@ import ( "encoding/json" "fmt" "io" + "os" + "path/filepath" "regexp" + "slices" "strings" jsonnet "github.com/google/go-jsonnet" @@ -14,6 +17,7 @@ import ( "github.com/grafana/tanka/pkg/helm" "github.com/grafana/tanka/pkg/kustomize" "github.com/pkg/errors" + "github.com/rs/zerolog/log" yaml "gopkg.in/yaml.v3" ) @@ -42,6 +46,14 @@ func Funcs() []*jsonnet.NativeFunction { } } +// VMFuncs returns a slice of functions similar to Funcs but are passed the jsonnet VM +// for in-line evaluation +func VMFuncs(vm *jsonnet.VM) []*jsonnet.NativeFunction { + return []*jsonnet.NativeFunction{ + importFiles(vm), + } +} + // parseJSON wraps `json.Unmarshal` to convert a json string into a dict func parseJSON() *jsonnet.NativeFunction { return &jsonnet.NativeFunction{ @@ -178,3 +190,75 @@ func regexSubst() *jsonnet.NativeFunction { }, } } + +type importFilesOpts struct { + CalledFrom string `json:"calledFrom"` + Exclude []string `json:"exclude"` + Extension string `json:"extension"` +} + +func parseImportOpts(data interface{}) (*importFilesOpts, error) { + c, err := json.Marshal(data) + if err != nil { + return nil, err + } + + // default extension to `.libsonnet` + opts := importFilesOpts{ + Extension: ".libsonnet", + } + if err := json.Unmarshal(c, &opts); err != nil { + return nil, err + } + if opts.CalledFrom == "" { + return nil, fmt.Errorf("importFiles: `opts.calledFrom` is unset or empty\nTanka needs this to find your directory.") + } + return &opts, nil +} + +// importFiles imports and evaluates all matching jsonnet files in the given relative directory +func importFiles(vm *jsonnet.VM) *jsonnet.NativeFunction { + return &jsonnet.NativeFunction{ + Name: "importFiles", + Params: ast.Identifiers{"directory", "opts"}, + Func: func(data []interface{}) (interface{}, error) { + dir, ok := data[0].(string) + if !ok { + return nil, fmt.Errorf("first argument 'directory' must be of 'string' type, got '%T' instead", data[0]) + } + opts, err := parseImportOpts(data[1]) + if err != nil { + return nil, err + } + dirPath := filepath.Join(filepath.Dir(opts.CalledFrom), dir) + imports := make(map[string]interface{}) + err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !strings.HasSuffix(info.Name(), opts.Extension) { + return nil + } + if slices.Contains(opts.Exclude, info.Name()) { + return nil + } + log.Debug().Msgf("importFiles: parsing file %s", info.Name()) + resultStr, err := vm.EvaluateFile(path) + if err != nil { + return fmt.Errorf("importFiles: failed to evaluate %s: %s", path, err) + } + var result interface{} + err = json.Unmarshal([]byte(resultStr), &result) + if err != nil { + return err + } + imports[info.Name()] = result + return nil + }) + if err != nil { + return nil, err + } + return imports, nil + }, + } +} From af47706344f9a4c30a60bdf82429b7de4c41dd21 Mon Sep 17 00:00:00 2001 From: Matt Wilder Date: Wed, 12 Mar 2025 12:34:11 -0700 Subject: [PATCH 2/4] add unit test --- pkg/jsonnet/native/funcs_test.go | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pkg/jsonnet/native/funcs_test.go b/pkg/jsonnet/native/funcs_test.go index afa13a8f3..7fcd2b655 100644 --- a/pkg/jsonnet/native/funcs_test.go +++ b/pkg/jsonnet/native/funcs_test.go @@ -3,8 +3,11 @@ package native import ( "encoding/json" "fmt" + "os" + "path/filepath" "testing" + jsonnet "github.com/google/go-jsonnet" "github.com/stretchr/testify/assert" ) @@ -21,6 +24,20 @@ func callNative(name string, data []interface{}) (res interface{}, err error, ca return nil, nil, fmt.Errorf("could not find native function %s", name) } +// callVMNative calls a native function used by jsonnet VM that requires access to the VM resource +func callVMNative(name string, data []interface{}) (res interface{}, err error, callerr error) { + vm := jsonnet.MakeVM() + for _, fun := range VMFuncs(vm) { + if fun.Name == name { + // Call the function + ret, err := fun.Func(data) + return ret, err, nil + } + } + + return nil, nil, fmt.Errorf("could not find VM native function %s", name) +} + func TestSha256(t *testing.T) { ret, err, callerr := callNative("sha256", []interface{}{"foo"}) @@ -208,3 +225,41 @@ func TestRegexSubstInvalid(t *testing.T) { assert.Empty(t, ret) assert.NotEmpty(t, err) } + +func TestImportFiles(t *testing.T) { + tempDir, err := os.MkdirTemp("", "importFilesTest") + assert.Nil(t, err) + defer os.RemoveAll(tempDir) + importDirName := "imports" + importDir := filepath.Join(tempDir, importDirName) + err = os.Mkdir(importDir, 0750) + assert.Nil(t, err) + importFiles := []string{"test1.libsonnet", "test2.libsonnet"} + excludeFiles := []string{"skip1.libsonnet", "skip2.libsonnet"} + for i, fName := range append(importFiles, excludeFiles...) { + fPath := filepath.Join(importDir, fName) + content := fmt.Sprintf("{ test: %d }", i) + err = os.WriteFile(fPath, []byte(content), 0644) + assert.Nil(t, err) + } + opts := make(map[string]interface{}) + opts["calledFrom"] = filepath.Join(tempDir, "main.jsonnet") + opts["exclude"] = excludeFiles + ret, err, callerr := callVMNative("importFiles", []interface{}{importDirName, opts}) + assert.Nil(t, err) + assert.Nil(t, callerr) + importMap, ok := ret.(map[string]interface{}) + assert.True(t, ok) + for i, fName := range importFiles { + content, ok := importMap[fName] + assert.True(t, ok) + cMap, ok := content.(map[string]interface{}) + assert.True(t, ok) + assert.Equal(t, cMap["test"], float64(i)) + } + // Make sure excluded files were not imported + for _, fName := range excludeFiles { + _, ok = importMap[fName] + assert.False(t, ok) + } +} From 692ef56744267e5483e7b369732daf7b97a3bc15 Mon Sep 17 00:00:00 2001 From: Matt Wilder Date: Wed, 19 Mar 2025 12:49:20 -0700 Subject: [PATCH 3/4] require `calledFrom` to be inside Tankas current working directory --- pkg/jsonnet/native/funcs.go | 12 ++++++++++++ pkg/jsonnet/native/funcs_test.go | 19 ++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go index 368fe0c92..e60902568 100644 --- a/pkg/jsonnet/native/funcs.go +++ b/pkg/jsonnet/native/funcs.go @@ -213,6 +213,18 @@ func parseImportOpts(data interface{}) (*importFilesOpts, error) { if opts.CalledFrom == "" { return nil, fmt.Errorf("importFiles: `opts.calledFrom` is unset or empty\nTanka needs this to find your directory.") } + // Make sure calledFrom is inside the current working directory + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("importFiles: failed to get current working directory: %s", err) + } + calledFromAbs, err := filepath.Abs(opts.CalledFrom) + if err != nil { + return nil, fmt.Errorf("importFiles: failed to get absolute path to `opts.calledFrom`: %s", err) + } + if !strings.HasPrefix(calledFromAbs, cwd) { + return nil, fmt.Errorf("importFiles: `opts.calledFrom` must be a subdirectory of the current working directory: %s", cwd) + } return &opts, nil } diff --git a/pkg/jsonnet/native/funcs_test.go b/pkg/jsonnet/native/funcs_test.go index 7fcd2b655..2ac7375b2 100644 --- a/pkg/jsonnet/native/funcs_test.go +++ b/pkg/jsonnet/native/funcs_test.go @@ -227,26 +227,35 @@ func TestRegexSubstInvalid(t *testing.T) { } func TestImportFiles(t *testing.T) { + cwd, err := os.Getwd() + assert.NoError(t, err) tempDir, err := os.MkdirTemp("", "importFilesTest") - assert.Nil(t, err) - defer os.RemoveAll(tempDir) + assert.NoError(t, err) + defer func() { + if err := os.Chdir(cwd); err != nil { + panic(err) + } + os.RemoveAll(tempDir) + }() + err = os.Chdir(tempDir) + assert.NoError(t, err) importDirName := "imports" importDir := filepath.Join(tempDir, importDirName) err = os.Mkdir(importDir, 0750) - assert.Nil(t, err) + assert.NoError(t, err) importFiles := []string{"test1.libsonnet", "test2.libsonnet"} excludeFiles := []string{"skip1.libsonnet", "skip2.libsonnet"} for i, fName := range append(importFiles, excludeFiles...) { fPath := filepath.Join(importDir, fName) content := fmt.Sprintf("{ test: %d }", i) err = os.WriteFile(fPath, []byte(content), 0644) - assert.Nil(t, err) + assert.NoError(t, err) } opts := make(map[string]interface{}) opts["calledFrom"] = filepath.Join(tempDir, "main.jsonnet") opts["exclude"] = excludeFiles ret, err, callerr := callVMNative("importFiles", []interface{}{importDirName, opts}) - assert.Nil(t, err) + assert.NoError(t, err) assert.Nil(t, callerr) importMap, ok := ret.(map[string]interface{}) assert.True(t, ok) From dfba8d405cc10178bb6f614ec6fb23f65e7cccfc Mon Sep 17 00:00:00 2001 From: Matt Wilder Date: Thu, 20 Mar 2025 16:25:40 -0700 Subject: [PATCH 4/4] refactor TestImportFiles to compare rendered json with static data --- pkg/jsonnet/native/funcs_test.go | 42 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/pkg/jsonnet/native/funcs_test.go b/pkg/jsonnet/native/funcs_test.go index 2ac7375b2..df7a42c3e 100644 --- a/pkg/jsonnet/native/funcs_test.go +++ b/pkg/jsonnet/native/funcs_test.go @@ -3,8 +3,10 @@ package native import ( "encoding/json" "fmt" + "maps" "os" "path/filepath" + "slices" "testing" jsonnet "github.com/google/go-jsonnet" @@ -229,13 +231,11 @@ func TestRegexSubstInvalid(t *testing.T) { func TestImportFiles(t *testing.T) { cwd, err := os.Getwd() assert.NoError(t, err) - tempDir, err := os.MkdirTemp("", "importFilesTest") - assert.NoError(t, err) + tempDir := t.TempDir() defer func() { if err := os.Chdir(cwd); err != nil { panic(err) } - os.RemoveAll(tempDir) }() err = os.Chdir(tempDir) assert.NoError(t, err) @@ -243,32 +243,30 @@ func TestImportFiles(t *testing.T) { importDir := filepath.Join(tempDir, importDirName) err = os.Mkdir(importDir, 0750) assert.NoError(t, err) - importFiles := []string{"test1.libsonnet", "test2.libsonnet"} - excludeFiles := []string{"skip1.libsonnet", "skip2.libsonnet"} - for i, fName := range append(importFiles, excludeFiles...) { + importMap := map[string]string{ + "test1.libsonnet": "{ test: 1 }", + "test2.libsonnet": "{ test: 2 }", + } + excludeMap := map[string]string{ + "skip1.libsonnet": `{ test: error "should not be included" }`, + "skip2.libsonnet": `{ test: error "should not be included" }`, + } + expectedJson := `{"test1.libsonnet":{"test":1},"test2.libsonnet":{"test":2}}` + allMap := make(map[string]string) + maps.Copy(allMap, importMap) + maps.Copy(allMap, excludeMap) + for fName, content := range allMap { fPath := filepath.Join(importDir, fName) - content := fmt.Sprintf("{ test: %d }", i) err = os.WriteFile(fPath, []byte(content), 0644) assert.NoError(t, err) } opts := make(map[string]interface{}) opts["calledFrom"] = filepath.Join(tempDir, "main.jsonnet") - opts["exclude"] = excludeFiles + opts["exclude"] = slices.Collect(maps.Keys(excludeMap)) ret, err, callerr := callVMNative("importFiles", []interface{}{importDirName, opts}) assert.NoError(t, err) assert.Nil(t, callerr) - importMap, ok := ret.(map[string]interface{}) - assert.True(t, ok) - for i, fName := range importFiles { - content, ok := importMap[fName] - assert.True(t, ok) - cMap, ok := content.(map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, cMap["test"], float64(i)) - } - // Make sure excluded files were not imported - for _, fName := range excludeFiles { - _, ok = importMap[fName] - assert.False(t, ok) - } + retJson, err := json.Marshal(ret) + assert.NoError(t, err) + assert.Equal(t, expectedJson, string(retJson)) }