filecache as part of evaluation state

Author: mandelsoft

dynaml/expression.go

@@ -35,6 +35,7 @@ type Binding interface {
 	NoMerge() bool
 
 	GetTempName(data []byte) (string, error)
+	GetFileContent(file string, cached bool) ([]byte, error)
 
 	Flow(source yaml.Node, shouldOverride bool) (yaml.Node, Status)
 	Cascade(outer Binding, template yaml.Node, partial bool, templates ...yaml.Node) (yaml.Node, error)

dynaml/fake_binding_helper_test.go

@@ -20,6 +20,10 @@ func (c FakeBinding) GetTempName([]byte) (string, error) {
 	return "", fmt.Errorf("no temp names")
 }
 
+func (c FakeBinding) GetFileContent(file string, cached bool) ([]byte, error) {
+	return nil, fmt.Errorf("no file access")
+}
+
 func (c FakeBinding) Outer() Binding {
 	return nil
 }

dynaml/read.go

@@ -1,8 +1,6 @@
 package dynaml
 
 import (
-	"io/ioutil"
-	"net/http"
 	"path"
 	"regexp"
 	"strings"
@@ -11,8 +9,6 @@ import (
 	"github.com/mandelsoft/spiff/yaml"
 )
 
-var fileCache = map[string][]byte{}
-
 var templ_pattern = regexp.MustCompile(".*\\s+&template(\\(?|\\s+).*")
 
 func func_read(cached bool, arguments []interface{}, binding Binding) (interface{}, EvaluationInfo, bool) {
@@ -39,30 +35,9 @@ func func_read(cached bool, arguments []interface{}, binding Binding) (interface
 
 	}
 
-	var err error
-
-	data := fileCache[file]
-	if !cached || data == nil {
-		debug.Debug("reading %s file %s\n", t, file)
-		if strings.HasPrefix(file, "http:") || strings.HasPrefix(file, "https:") {
-			response, err := http.Get(file)
-			if err != nil {
-				return info.Error("error getting [%s]: %s", file, err)
-			} else {
-				defer response.Body.Close()
-				contents, err := ioutil.ReadAll(response.Body)
-				if err != nil {
-					return info.Error("error getting body [%s]: %s", file, err)
-				}
-				data = contents
-			}
-		} else {
-			data, err = ioutil.ReadFile(file)
-			if err != nil {
-				return info.Error("error reading [%s]: %s", path.Clean(file), err)
-			}
-		}
-		fileCache[file] = data
+	data, err := binding.GetFileContent(file, cached)
+	if err != nil {
+		return info.Error("read: %s", err)
 	}
 	return parse(file, data, t, binding)
 }

flow/environment.go

@@ -78,6 +78,13 @@ func (e DefaultEnvironment) GetTempName(data []byte) (string, error) {
 	return e.state.GetTempName(data)
 }
 
+func (e DefaultEnvironment) GetFileContent(file string, cached bool) ([]byte, error) {
+	if e.outer != nil {
+		return e.outer.GetFileContent(file, cached)
+	}
+	return e.state.GetFileContent(file, cached)
+}
+
 func (e DefaultEnvironment) Outer() dynaml.Binding {
 	return e.outer
 }

flow/state.go

@@ -3,16 +3,22 @@ package flow
 import (
 	"crypto/sha512"
 	"encoding/base64"
+	"fmt"
+	"github.com/mandelsoft/spiff/debug"
 	"io/ioutil"
+	"net/http"
 	"os"
+	"path"
+	"strings"
 )
 
 type State struct {
-	files map[string]string
+	files     map[string]string // content hash to temp file name
+	fileCache map[string][]byte // file content cache
 }
 
 func NewState() *State {
-	return &State{map[string]string{}}
+	return &State{map[string]string{}, map[string][]byte{}}
 }
 
 func (s *State) GetTempName(data []byte) (string, error) {
@@ -37,3 +43,32 @@ func (s *State) Cleanup() {
 	}
 	s.files = map[string]string{}
 }
+
+func (s *State) GetFileContent(file string, cached bool) ([]byte, error) {
+	var err error
+
+	data := s.fileCache[file]
+	if !cached || data == nil {
+		debug.Debug("reading file %s\n", file)
+		if strings.HasPrefix(file, "http:") || strings.HasPrefix(file, "https:") {
+			response, err := http.Get(file)
+			if err != nil {
+				return nil, fmt.Errorf("error getting [%s]: %s", file, err)
+			} else {
+				defer response.Body.Close()
+				contents, err := ioutil.ReadAll(response.Body)
+				if err != nil {
+					return nil, fmt.Errorf("error getting body [%s]: %s", file, err)
+				}
+				data = contents
+			}
+		} else {
+			data, err = ioutil.ReadFile(file)
+			if err != nil {
+				return nil, fmt.Errorf("error reading [%s]: %s", path.Clean(file), err)
+			}
+		}
+		s.fileCache[file] = data
+	}
+	return data, nil
+}