feat(jsonnet): add support for importing all files in a directory

Add native function `importFiles` which reads and evaluates all jsonnet files in a directory.

Returns an object of with key/value pairs of `<file name>: <evaluated object>`

Directory path is found using a `calledFrom` option, similar to `helmTemplate`.

Includes support for:
 - Choosing files by extension using the `extension` option (default is `.libsonnet`)
 - Excluding files using the `exclude` option, which takes an array of file names to skip

Author: partcyborg

pkg/jsonnet/implementations/goimpl/vm.go

@@ -25,6 +25,10 @@ func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[stri
 		vm.NativeFunction(nf)
 	}
 
+	for _, nvf := range native.VMFuncs(vm) {
+		vm.NativeFunction(nvf)
+	}
+
 	if maxStack > 0 {
 		vm.MaxStack = maxStack
 	}

pkg/jsonnet/native/funcs.go

@@ -6,14 +6,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"regexp"
+	"slices"
 	"strings"
 
 	jsonnet "github.com/google/go-jsonnet"
 	"github.com/google/go-jsonnet/ast"
 	"github.com/grafana/tanka/pkg/helm"
 	"github.com/grafana/tanka/pkg/kustomize"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -42,6 +46,14 @@ func Funcs() []*jsonnet.NativeFunction {
 	}
 }
 
+// VMFuncs returns a slice of functions similar to Funcs but are passed the jsonnet VM
+// for in-line evaluation
+func VMFuncs(vm *jsonnet.VM) []*jsonnet.NativeFunction {
+	return []*jsonnet.NativeFunction{
+		importFiles(vm),
+	}
+}
+
 // parseJSON wraps `json.Unmarshal` to convert a json string into a dict
 func parseJSON() *jsonnet.NativeFunction {
 	return &jsonnet.NativeFunction{
@@ -178,3 +190,75 @@ func regexSubst() *jsonnet.NativeFunction {
 		},
 	}
 }
+
+type importFilesOpts struct {
+	CalledFrom string   `json:"calledFrom"`
+	Exclude    []string `json:"exclude"`
+	Extension  string   `json:"extension"`
+}
+
+func parseImportOpts(data interface{}) (*importFilesOpts, error) {
+	c, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// default extension to `.libsonnet`
+	opts := importFilesOpts{
+		Extension: ".libsonnet",
+	}
+	if err := json.Unmarshal(c, &opts); err != nil {
+		return nil, err
+	}
+	if opts.CalledFrom == "" {
+		return nil, fmt.Errorf("importFiles: `opts.calledFrom` is unset or empty\nTanka needs this to find your directory.")
+	}
+	return &opts, nil
+}
+
+// importFiles imports and evaluates all matching jsonnet files in the given relative directory
+func importFiles(vm *jsonnet.VM) *jsonnet.NativeFunction {
+	return &jsonnet.NativeFunction{
+		Name:   "importFiles",
+		Params: ast.Identifiers{"directory", "opts"},
+		Func: func(data []interface{}) (interface{}, error) {
+			dir, ok := data[0].(string)
+			if !ok {
+				return nil, fmt.Errorf("first argument 'directory' must be of 'string' type, got '%T' instead", data[0])
+			}
+			opts, err := parseImportOpts(data[1])
+			if err != nil {
+				return nil, err
+			}
+			dirPath := filepath.Join(filepath.Dir(opts.CalledFrom), dir)
+			imports := make(map[string]interface{})
+			err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() || !strings.HasSuffix(info.Name(), opts.Extension) {
+					return nil
+				}
+				if slices.Contains(opts.Exclude, info.Name()) {
+					return nil
+				}
+				log.Debug().Msgf("importFiles: parsing file %s", info.Name())
+				resultStr, err := vm.EvaluateFile(path)
+				if err != nil {
+					return fmt.Errorf("importFiles: failed to evaluate %s: %s", path, err)
+				}
+				var result interface{}
+				err = json.Unmarshal([]byte(resultStr), &result)
+				if err != nil {
+					return err
+				}
+				imports[info.Name()] = result
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+			return imports, nil
+		},
+	}
+}