[EXPERIMENTAL] RPC support + paraller compilation + arduino-preprocessor

.travis.yml

@@ -14,6 +14,7 @@ install:
   - go get github.com/arduino/go-timeutils
 
 script:
+  - go get github.com/arduino/arduino-builder/arduino-builder
   - go build -o $HOME/arduino-builder -v github.com/arduino/arduino-builder/arduino-builder
   - export TEST_PACKAGES=`go list github.com/arduino/arduino-builder/...`
   - RES=0; I=0; for PKG in $TEST_PACKAGES; do go test -v -timeout 30m -covermode=count -coverprofile=coverage.$I.out $PKG; ((RES=RES+$?)); ((I++)); done; ( exit $RES )

arduino-builder/main.go

@@ -37,11 +37,16 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 	"syscall"
 
+	"runtime/pprof"
+	"runtime/trace"
+
 	"github.com/arduino/arduino-builder"
 	"github.com/arduino/arduino-builder/gohasissues"
+	"github.com/arduino/arduino-builder/grpc"
 	"github.com/arduino/arduino-builder/i18n"
 	"github.com/arduino/arduino-builder/types"
 	"github.com/arduino/arduino-builder/utils"
@@ -54,6 +59,7 @@ const VERSION = "1.3.25"
 const FLAG_ACTION_COMPILE = "compile"
 const FLAG_ACTION_PREPROCESS = "preprocess"
 const FLAG_ACTION_DUMP_PREFS = "dump-prefs"
+const FLAG_ACTION_CODE_COMPLETE_AT = "code-complete-at"
 const FLAG_BUILD_OPTIONS_FILE = "build-options-file"
 const FLAG_HARDWARE = "hardware"
 const FLAG_TOOLS = "tools"
@@ -78,7 +84,11 @@ const FLAG_LOGGER_HUMAN = "human"
 const FLAG_LOGGER_HUMANTAGS = "humantags"
 const FLAG_LOGGER_MACHINE = "machine"
 const FLAG_VERSION = "version"
+const FLAG_DAEMON = "daemon"
 const FLAG_VID_PID = "vid-pid"
+const FLAG_JOBS = "jobs"
+const FLAG_TRACE = "trace"
+const FLAG_EXPERIMENTAL = "experimental"
 
 type foldersFlag []string
 
@@ -118,6 +128,7 @@ func (h *propertiesFlag) Set(value string) error {
 var compileFlag *bool
 var preprocessFlag *bool
 var dumpPrefsFlag *bool
+var codeCompleteAtFlag *string
 var buildOptionsFileFlag *string
 var hardwareFoldersFlag foldersFlag
 var toolsFoldersFlag foldersFlag
@@ -135,12 +146,17 @@ var debugLevelFlag *int
 var warningsLevelFlag *string
 var loggerFlag *string
 var versionFlag *bool
+var daemonFlag *bool
 var vidPidFlag *string
+var jobsFlag *int
+var traceFlag *bool
+var experimentalFeatures *bool
 
 func init() {
 	compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch")
 	preprocessFlag = flag.Bool(FLAG_ACTION_PREPROCESS, false, "preprocess the given sketch")
 	dumpPrefsFlag = flag.Bool(FLAG_ACTION_DUMP_PREFS, false, "dumps build properties used when compiling")
+	codeCompleteAtFlag = flag.String(FLAG_ACTION_CODE_COMPLETE_AT, "", "output code completions for sketch at a specific location. Location format is \"file:line:col\"")
 	buildOptionsFileFlag = flag.String(FLAG_BUILD_OPTIONS_FILE, "", "Instead of specifying --"+FLAG_HARDWARE+", --"+FLAG_TOOLS+" etc every time, you can load all such options from a file")
 	flag.Var(&hardwareFoldersFlag, FLAG_HARDWARE, "Specify a 'hardware' folder. Can be added multiple times for specifying multiple 'hardware' folders")
 	flag.Var(&toolsFoldersFlag, FLAG_TOOLS, "Specify a 'tools' folder. Can be added multiple times for specifying multiple 'tools' folders")
@@ -158,12 +174,40 @@ func init() {
 	warningsLevelFlag = flag.String(FLAG_WARNINGS, "", "Sets warnings level. Available values are '"+FLAG_WARNINGS_NONE+"', '"+FLAG_WARNINGS_DEFAULT+"', '"+FLAG_WARNINGS_MORE+"' and '"+FLAG_WARNINGS_ALL+"'")
 	loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'")
 	versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits")
+	daemonFlag = flag.Bool(FLAG_DAEMON, false, "daemonizes and serves its functions via rpc")
 	vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt")
+	jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine")
+	traceFlag = flag.Bool(FLAG_TRACE, false, "traces the whole process lifecycle")
+	experimentalFeatures = flag.Bool(FLAG_EXPERIMENTAL, false, "enables experimental features")
 }
 
 func main() {
+
 	flag.Parse()
 
+	if *traceFlag {
+		f, err := os.Create("trace.out")
+		if err != nil {
+			panic(err)
+		}
+		defer f.Close()
+
+		f2, err := os.Create("profile.out")
+		if err != nil {
+			panic(err)
+		}
+		defer f2.Close()
+
+		pprof.StartCPUProfile(f2)
+		defer pprof.StopCPUProfile()
+
+		err = trace.Start(f)
+		if err != nil {
+			panic(err)
+		}
+		defer trace.Stop()
+	}
+
 	if *versionFlag {
 		fmt.Println("Arduino Builder " + VERSION)
 		fmt.Println("Copyright (C) 2015 Arduino LLC and contributors")
@@ -173,8 +217,28 @@ func main() {
 		return
 	}
 
+	if *jobsFlag > 0 {
+		runtime.GOMAXPROCS(*jobsFlag)
+	} else {
+		runtime.GOMAXPROCS(runtime.NumCPU())
+	}
+
 	ctx := &types.Context{}
 
+	// place here all experimental features that should live under this flag
+	if *experimentalFeatures {
+		ctx.UseArduinoPreprocessor = true
+	}
+
+	if *daemonFlag {
+		var loggerBuffer []string
+		logger := i18n.AccumulatorLogger{}
+		logger.Buffer = &loggerBuffer
+		//logger := i18n.HumanLogger{}
+		ctx.SetLogger(logger)
+		jsonrpc.RegisterAndServeJsonRPC(ctx)
+	}
+
 	if *buildOptionsFileFlag != "" {
 		buildOptions := make(properties.Map)
 		if _, err := os.Stat(*buildOptionsFileFlag); err == nil {
@@ -330,7 +394,8 @@ func main() {
 
 	if *dumpPrefsFlag {
 		err = builder.RunParseHardwareAndDumpBuildProperties(ctx)
-	} else if *preprocessFlag {
+	} else if *preprocessFlag || *codeCompleteAtFlag != "" {
+		ctx.CodeCompleteAt = *codeCompleteAtFlag
 		err = builder.RunPreprocess(ctx)
 	} else {
 		if flag.NArg() == 0 {

builder.go

@@ -35,6 +35,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/arduino/arduino-builder/builder_utils"
 	"github.com/arduino/arduino-builder/constants"
 	"github.com/arduino/arduino-builder/i18n"
 	"github.com/arduino/arduino-builder/phases"
@@ -89,7 +90,7 @@ func (s *Builder) Run(ctx *types.Context) error {
 		&WarnAboutArchIncompatibleLibraries{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Generating function prototypes..."),
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		utils.LogIfVerbose(constants.LOG_LEVEL_INFO, "Compiling sketch..."),
 		&RecipeByPrefixSuffixRunner{Prefix: constants.HOOKS_SKETCH_PREBUILD, Suffix: constants.HOOKS_PATTERN_SUFFIX},
@@ -128,6 +129,8 @@ func (s *Builder) Run(ctx *types.Context) error {
 
 		&PrintUsedLibrariesIfVerbose{},
 
+		&ExportProjectCMake{SketchError: mainErr != nil},
+
 		&phases.Sizer{SketchError: mainErr != nil},
 	}
 	otherErr := runCommands(ctx, commands, false)
@@ -139,6 +142,18 @@ func (s *Builder) Run(ctx *types.Context) error {
 	return otherErr
 }
 
+type PreprocessSketch struct{}
+
+func (s *PreprocessSketch) Run(ctx *types.Context) error {
+	var commands []types.Command
+	if ctx.UseArduinoPreprocessor {
+		commands = append(commands, &PreprocessSketchArduino{})
+	} else {
+		commands = append(commands, &ContainerAddPrototypes{})
+	}
+	return runCommands(ctx, commands, true)
+}
+
 type Preprocess struct{}
 
 func (s *Preprocess) Run(ctx *types.Context) error {
@@ -158,7 +173,7 @@ func (s *Preprocess) Run(ctx *types.Context) error {
 
 		&WarnAboutArchIncompatibleLibraries{},
 
-		&ContainerAddPrototypes{},
+		&PreprocessSketch{},
 
 		&PrintPreprocessedSource{},
 	}
@@ -181,36 +196,22 @@ func (s *ParseHardwareAndDumpBuildProperties) Run(ctx *types.Context) error {
 }
 
 func runCommands(ctx *types.Context, commands []types.Command, progressEnabled bool) error {
-	commandsLength := len(commands)
-	progressForEachCommand := float32(100) / float32(commandsLength)
 
-	progress := float32(0)
+	ctx.Progress.PrintEnabled = progressEnabled
+	ctx.Progress.Progress = 0
+
 	for _, command := range commands {
 		PrintRingNameIfDebug(ctx, command)
-		printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, progress)
+		ctx.Progress.Steps = 100.0 / float64(len(commands))
+		builder_utils.PrintProgressIfProgressEnabledAndMachineLogger(ctx)
 		err := command.Run(ctx)
 		if err != nil {
 			return i18n.WrapError(err)
 		}
-		progress += progressForEachCommand
 	}
-
-	printProgressIfProgressEnabledAndMachineLogger(progressEnabled, ctx, 100)
-
 	return nil
 }
 
-func printProgressIfProgressEnabledAndMachineLogger(progressEnabled bool, ctx *types.Context, progress float32) {
-	if !progressEnabled {
-		return
-	}
-
-	log := ctx.GetLogger()
-	if log.Name() == "machine" {
-		log.Println(constants.LOG_LEVEL_INFO, constants.MSG_PROGRESS, strconv.FormatFloat(float64(progress), 'f', 2, 32))
-	}
-}
-
 func PrintRingNameIfDebug(ctx *types.Context, command types.Command) {
 	if ctx.DebugLevel >= 10 {
 		ctx.GetLogger().Fprintln(os.Stdout, constants.LOG_LEVEL_DEBUG, constants.MSG_RUNNING_COMMAND, strconv.FormatInt(time.Now().Unix(), 10), reflect.Indirect(reflect.ValueOf(command)).Type().Name())