Merge branch 'master' into inoz_experiment

Author: facchinm

Diff for: .travis.yml

@@ -14,6 +14,7 @@ install:
   - go get github.com/arduino/go-timeutils
 
 script:
+  - go get github.com/arduino/arduino-builder/arduino-builder
   - go build -o $HOME/arduino-builder -v github.com/arduino/arduino-builder/arduino-builder
   - export TEST_PACKAGES=`go list github.com/arduino/arduino-builder/...`
   - RES=0; I=0; for PKG in $TEST_PACKAGES; do go test -v -timeout 30m -covermode=count -coverprofile=coverage.$I.out $PKG; ((RES=RES+$?)); ((I++)); done; ( exit $RES )

Diff for: README.md

@@ -63,8 +63,12 @@ To build, run the following commands:
 go get github.com/go-errors/errors
 go get github.com/stretchr/testify
 go get github.com/jstemmer/go-junit-report
-go get github.com/arduino/go-properties-map
-go get github.com/arduino/go-timeutils
+go get -u github.com/arduino/go-properties-map
+go get -u github.com/arduino/go-timeutils
+go get google.golang.org/grpc
+go get github.com/golang/protobuf/proto
+go get golang.org/x/net/context
+go get github.com/fsnotify/fsnotify
 go get github.com/arduino/arduino-builder
 go build github.com/arduino/arduino-builder/arduino-builder
 ```
@@ -74,7 +78,7 @@ go build github.com/arduino/arduino-builder/arduino-builder
 In order to run the tests, type:
 
 ```
-go test github.com/arduino/arduino-builder/arduino-builder/...
+go test github.com/arduino/arduino-builder/...
 ```
 
 This runs all tests, showing any failures and a summary at the end.
@@ -88,13 +92,13 @@ To run a single test, use the -run option, which accepts a regular
 expression (see also go help testflag).
 
 ```
-go test github.com/arduino/arduino-builder/arduino-builder/... -run 'TestBuilderEmptySketch'
-go test github.com/arduino/arduino-builder/arduino-builder/... -run 'TestPrototypesAdder.*'
+go test github.com/arduino/arduino-builder/... -run 'TestBuilderEmptySketch'
+go test github.com/arduino/arduino-builder/... -run 'TestPrototypesAdder.*'
 ```
 
 In jenkins, use
 ```
-go test -v github.com/arduino/arduino-builder/arduino-builder/... | bin/go-junit-report > report.xml
+go test -v github.com/arduino/arduino-builder/... | bin/go-junit-report > report.xml
 ```
 
 The first time you run the tests, some needed files (toolchains and

Diff for: arduino-builder/main.go

@@ -37,23 +37,30 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strings"
 	"syscall"
 
+	"runtime/pprof"
+	"runtime/trace"
+
 	"github.com/arduino/arduino-builder"
 	"github.com/arduino/arduino-builder/gohasissues"
+	"github.com/arduino/arduino-builder/grpc"
 	"github.com/arduino/arduino-builder/i18n"
 	"github.com/arduino/arduino-builder/types"
 	"github.com/arduino/arduino-builder/utils"
 	"github.com/arduino/go-properties-map"
 	"github.com/go-errors/errors"
 )
 
-const VERSION = "1.3.25"
+const VERSION = "1.4.2"
 
 const FLAG_ACTION_COMPILE = "compile"
 const FLAG_ACTION_PREPROCESS = "preprocess"
 const FLAG_ACTION_DUMP_PREFS = "dump-prefs"
+const FLAG_ACTION_CODE_COMPLETE_AT = "code-complete-at"
 const FLAG_BUILD_OPTIONS_FILE = "build-options-file"
 const FLAG_HARDWARE = "hardware"
 const FLAG_TOOLS = "tools"
@@ -78,27 +85,20 @@ const FLAG_LOGGER_HUMAN = "human"
 const FLAG_LOGGER_HUMANTAGS = "humantags"
 const FLAG_LOGGER_MACHINE = "machine"
 const FLAG_VERSION = "version"
+const FLAG_DAEMON = "daemon"
 const FLAG_VID_PID = "vid-pid"
+const FLAG_JOBS = "jobs"
+const FLAG_TRACE = "trace"
+const FLAG_EXPERIMENTAL = "experimental"
 
 type foldersFlag []string
 
 func (h *foldersFlag) String() string {
 	return fmt.Sprint(*h)
 }
 
-func (h *foldersFlag) Set(csv string) error {
-	var values []string
-	if strings.Contains(csv, string(os.PathListSeparator)) {
-		values = strings.Split(csv, string(os.PathListSeparator))
-	} else {
-		values = strings.Split(csv, ",")
-	}
-
-	for _, value := range values {
-		value = strings.TrimSpace(value)
-		*h = append(*h, value)
-	}
-
+func (h *foldersFlag) Set(folder string) error {
+	*h = append(*h, folder)
 	return nil
 }
 
@@ -118,6 +118,7 @@ func (h *propertiesFlag) Set(value string) error {
 var compileFlag *bool
 var preprocessFlag *bool
 var dumpPrefsFlag *bool
+var codeCompleteAtFlag *string
 var buildOptionsFileFlag *string
 var hardwareFoldersFlag foldersFlag
 var toolsFoldersFlag foldersFlag
@@ -135,12 +136,17 @@ var debugLevelFlag *int
 var warningsLevelFlag *string
 var loggerFlag *string
 var versionFlag *bool
+var daemonFlag *bool
 var vidPidFlag *string
+var jobsFlag *int
+var traceFlag *bool
+var experimentalFeatures *bool
 
 func init() {
 	compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
 	preprocessFlag = flag.Bool(FLAG_ACTION_PREPROCESS, false, "preprocess the given sketch")
 	dumpPrefsFlag = flag.Bool(FLAG_ACTION_DUMP_PREFS, false, "dumps build properties used when compiling")
+	codeCompleteAtFlag = flag.String(FLAG_ACTION_CODE_COMPLETE_AT, "", "output code completions for sketch at a specific location. Location format is \"file:line:col\"")
 	buildOptionsFileFlag = flag.String(FLAG_BUILD_OPTIONS_FILE, "", "Instead of specifying --"+FLAG_HARDWARE+", --"+FLAG_TOOLS+" etc every time, you can load all such options from a file")
 	flag.Var(&hardwareFoldersFlag, FLAG_HARDWARE, "Specify a 'hardware' folder. Can be added multiple times for specifying multiple 'hardware' folders")
 	flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders")
@@ -158,12 +164,40 @@ func init() {
 	warningsLevelFlag = flag.String(FLAG_WARNINGS, "", "Sets warnings level. Available values are '"+FLAG_WARNINGS_NONE+"', '"+FLAG_WARNINGS_DEFAULT+"', '"+FLAG_WARNINGS_MORE+"' and '"+FLAG_WARNINGS_ALL+"'")
 	loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'")
 	versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
+	daemonFlag = flag.Bool(FLAG_DAEMON, false, "daemonizes and serves its functions via rpc")
 	vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
+	jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
+	traceFlag = flag.Bool(FLAG_TRACE, false, "traces the whole process lifecycle")
+	experimentalFeatures = flag.Bool(FLAG_EXPERIMENTAL, false, "enables experimental features")
 }
 
 func main() {
+
 	flag.Parse()
 
+	if *traceFlag {
+		f, err := os.Create("trace.out")
+		if err != nil {
+			panic(err)
+		}
+		defer f.Close()
+
+		f2, err := os.Create("profile.out")
+		if err != nil {
+			panic(err)
+		}
+		defer f2.Close()
+
+		pprof.StartCPUProfile(f2)
+		defer pprof.StopCPUProfile()
+
+		err = trace.Start(f)
+		if err != nil {
+			panic(err)
+		}
+		defer trace.Stop()
+	}
+
 	if *versionFlag {
 		fmt.Println("Arduino Builder " + VERSION)
 		fmt.Println("Copyright (C) 2015 Arduino LLC and contributors")
@@ -173,8 +207,28 @@ func main() {
 		return
 	}
 
+	if *jobsFlag > 0 {
+		runtime.GOMAXPROCS(*jobsFlag)
+	} else {
+		runtime.GOMAXPROCS(runtime.NumCPU())
+	}
+
 	ctx := &types.Context{}
 
+	// place here all experimental features that should live under this flag
+	if *experimentalFeatures {
+		ctx.UseArduinoPreprocessor = true
+	}
+
+	if *daemonFlag {
+		var loggerBuffer []string
+		logger := i18n.AccumulatorLogger{}
+		logger.Buffer = &loggerBuffer
+		//logger := i18n.HumanLogger{}
+		ctx.SetLogger(logger)
+		jsonrpc.RegisterAndServeJsonRPC(ctx)
+	}
+
 	if *buildOptionsFileFlag != "" {
 		buildOptions := make(properties.Map)
 		if _, err := os.Stat(*buildOptionsFileFlag); err == nil {
@@ -258,7 +312,7 @@ func main() {
 			printCompleteError(err)
 		}
 	}
-	ctx.BuildPath = buildPath
+	ctx.BuildPath, _ = filepath.Abs(buildPath)
 
 	// FLAG_BUILD_CACHE
 	buildCachePath, err := gohasissues.Unquote(*buildCachePathFlag)
@@ -330,7 +384,8 @@ func main() {
 
 	if *dumpPrefsFlag {
 		err = builder.RunParseHardwareAndDumpBuildProperties(ctx)
-	} else if *preprocessFlag {
+	} else if *preprocessFlag || *codeCompleteAtFlag != "" {
+		ctx.CodeCompleteAt = *codeCompleteAtFlag
 		err = builder.RunPreprocess(ctx)
 	} else {
 		if flag.NArg() == 0 {

Diff for: builder.go

@@ -35,6 +35,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/arduino/arduino-builder/builder_utils"
 	"github.com/arduino/arduino-builder/constants"
 	"github.com/arduino/arduino-builder/i18n"
 	"github.com/arduino/arduino-builder/phases"
@@ -89,7 +90,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		&WarnAboutArchIncompatibleLibraries{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Generating function prototypes..."),
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Compiling sketch..."),
 		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
@@ -128,6 +129,8 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 		&PrintUsedLibrariesIfVerbose{},
 
+		&ExportProjectCMake{SketchError: mainErr != nil},
+
 		&phases.Sizer{SketchError: mainErr != nil},
 	}
 	otherErr := runCommands(ctx, commands, false)
@@ -139,6 +142,18 @@ func (s *Builder) Run(ctx *types.Context) error {
 	return otherErr
 }
 
+type PreprocessSketch struct{}
+
+func (s *PreprocessSketch) Run(ctx *types.Context) error {
+	var commands []types.Command
+	if ctx.UseArduinoPreprocessor {
+		commands = append(commands, &PreprocessSketchArduino{})
+	} else {
+		commands = append(commands, &ContainerAddPrototypes{})
+	}
+	return runCommands(ctx, commands, true)
+}
+
 type Preprocess struct{}
 
 func (s *Preprocess) Run(ctx *types.Context) error {
@@ -158,7 +173,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		&PrintPreprocessedSource{},
 	}
@@ -181,36 +196,22 @@ func (s *ParseHardwareAndDumpBuildProperties) Run(ctx *types.Context) error {
 }
 
 func runCommands(ctx *types.Context, commands []types.Command, progressEnabled bool) error {
-	commandsLength := len(commands)
-	progressForEachCommand := float32(100) / float32(commandsLength)
 
-	progress := float32(0)
+	ctx.Progress.PrintEnabled = progressEnabled
+	ctx.Progress.Progress = 0
+
 	for _, command := range commands {
 		PrintRingNameIfDebug(ctx, command)
-		printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, progress)
+		ctx.Progress.Steps = 100.0 / float64(len(commands))
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
 		err := command.Run(ctx)
 		if err != nil {
 			return i18n.WrapError(err)
 		}
-		progress += progressForEachCommand
 	}
-
-	printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, 100)
-
 	return nil
 }
 
-func printProgressIfProgressEnabledAndMachineLogger(progressEnabled bool, ctx *types.Context, progress float32) {
-	if !progressEnabled {
-		return
-	}
-
-	log := ctx.GetLogger()
-	if log.Name() == "machine" {
-		log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(float64(progress), 'f', 2, 32))
-	}
-}
-
 func PrintRingNameIfDebug(ctx *types.Context, command types.Command) {
 	if ctx.DebugLevel >= 10 {
 		ctx.GetLogger().Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_COMMAND, strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name())