From c7f55b54de36f973dbae6f7454b2b5f3a024d651 Mon Sep 17 00:00:00 2001 From: copercini Date: Mon, 14 Aug 2017 16:17:43 -0300 Subject: [PATCH] Parallel compiling: arduino-builder will leverage your multi core pc If -jobs X param is specified and X is greater than 0, arduino-builder won't spawn more than X processes. Fixes https://github.com/arduino/arduino-builder/issues/17 - Rebased from Federico Fissore branch https://github.com/arduino/arduino-builder/commit/0a3d888439ee369d7f4cc2333619e66fc7dcf672#diff-8f074e2b3fc911dbe8255891fc1a9b13R96 - Add HumanTagsLogger compatibility - Remove an if which suppress some GCC error messages here: https://github.com/arduino/arduino-builder/commit/0a3d888439ee369d7f4cc2333619e66fc7dcf672#diff-8f074e2b3fc911dbe8255891fc1a9b13R96 --- src/arduino.cc/arduino-builder/main.go | 10 +++ src/arduino.cc/builder/builder_utils/utils.go | 61 +++++++++++++--- src/arduino.cc/builder/ctags_runner.go | 4 +- src/arduino.cc/builder/i18n/i18n.go | 72 ++++++++++++++++--- src/arduino.cc/builder/types/accessories.go | 34 +++++++++ 5 files changed, 159 insertions(+), 22 deletions(-) diff --git a/src/arduino.cc/arduino-builder/main.go b/src/arduino.cc/arduino-builder/main.go index 39a65c09..ae4cea95 100644 --- a/src/arduino.cc/arduino-builder/main.go +++ b/src/arduino.cc/arduino-builder/main.go @@ -37,6 +37,7 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strings" "syscall" @@ -79,6 +80,7 @@ const FLAG_LOGGER_HUMANTAGS = "humantags" const FLAG_LOGGER_MACHINE = "machine" const FLAG_VERSION = "version" const FLAG_VID_PID = "vid-pid" +const FLAG_JOBS = "jobs" type foldersFlag []string @@ -136,6 +138,7 @@ var warningsLevelFlag *string var loggerFlag *string var versionFlag *bool var vidPidFlag *string +var jobsFlag *int func init() { compileFlag = flag.Bool(FLAG_ACTION_COMPILE, false, "compiles the given sketch") @@ -159,6 +162,7 @@ func init() { loggerFlag = flag.String(FLAG_LOGGER, FLAG_LOGGER_HUMAN, "Sets type of logger. Available values are '"+FLAG_LOGGER_HUMAN+"', '"+FLAG_LOGGER_HUMANTAGS+"', '"+FLAG_LOGGER_MACHINE+"'") versionFlag = flag.Bool(FLAG_VERSION, false, "prints version and exits") vidPidFlag = flag.String(FLAG_VID_PID, "", "specify to use vid/pid specific build properties, as defined in boards.txt") + jobsFlag = flag.Int(FLAG_JOBS, 0, "specify how many concurrent gcc processes should run at the same time. Defaults to the number of available cores on the running machine") } func main() { @@ -173,6 +177,12 @@ func main() { return } + if *jobsFlag > 0 { + runtime.GOMAXPROCS(*jobsFlag) + } else { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + ctx := &types.Context{} if *buildOptionsFileFlag != "" { diff --git a/src/arduino.cc/builder/builder_utils/utils.go b/src/arduino.cc/builder/builder_utils/utils.go index 97b6e355..8636abe3 100644 --- a/src/arduino.cc/builder/builder_utils/utils.go +++ b/src/arduino.cc/builder/builder_utils/utils.go @@ -31,15 +31,16 @@ package builder_utils import ( "bytes" - "fmt" "io" "os" "os/exec" "path/filepath" "strings" + "sync" "arduino.cc/builder/constants" "arduino.cc/builder/i18n" + "arduino.cc/builder/types" "arduino.cc/builder/utils" "arduino.cc/properties" ) @@ -146,15 +147,47 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { } func compileFilesWithRecipe(objectFiles []string, sourcePath string, sources []string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) ([]string, error) { + if len(sources) == 0 { + return objectFiles, nil + } + objectFilesChan := make(chan string) + errorsChan := make(chan error) + doneChan := make(chan struct{}) + + var wg sync.WaitGroup + wg.Add(len(sources)) + for _, source := range sources { - objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) - if err != nil { + go func(source string) { + defer wg.Done() + objectFile, err := compileFileWithRecipe(sourcePath, source, buildPath, buildProperties, includes, recipe, verbose, warningsLevel, logger) + if err != nil { + errorsChan <- err + } else { + objectFilesChan <- objectFile + } + }(source) + } + + go func() { + wg.Wait() + doneChan <- struct{}{} + }() + + for { + select { + case objectFile := <-objectFilesChan: + objectFiles = append(objectFiles, objectFile) + case err := <-errorsChan: return nil, i18n.WrapError(err) + case <-doneChan: + close(objectFilesChan) + for objectFile := range objectFilesChan { + objectFiles = append(objectFiles, objectFile) + } + return objectFiles, nil } - - objectFiles = append(objectFiles, objectFile) } - return objectFiles, nil } func compileFileWithRecipe(sourcePath string, source string, buildPath string, buildProperties properties.Map, includes []string, recipe string, verbose bool, warningsLevel string, logger i18n.Logger) (string, error) { @@ -361,10 +394,20 @@ func ExecRecipe(properties properties.Map, recipe string, removeUnsetProperties } if echoOutput { - command.Stdout = os.Stdout + printToStdOut := func(data []byte) { + logger.UnformattedWrite(os.Stdout, data) + } + stdout := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdOut, Buffer: bytes.Buffer{}} + defer stdout.Flush() + command.Stdout = stdout } - command.Stderr = os.Stderr + printToStdErr := func(data []byte) { + logger.UnformattedWrite(os.Stderr, data) + } + stderr := &types.BufferedUntilNewLineWriter{PrintFunc: printToStdErr, Buffer: bytes.Buffer{}} + defer stderr.Flush() + command.Stderr = stderr if echoOutput { err := command.Run() @@ -396,7 +439,7 @@ func PrepareCommandForRecipe(buildProperties properties.Map, recipe string, remo } if echoCommandLine { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } return command, nil diff --git a/src/arduino.cc/builder/ctags_runner.go b/src/arduino.cc/builder/ctags_runner.go index 2ab75638..f1e08a7b 100644 --- a/src/arduino.cc/builder/ctags_runner.go +++ b/src/arduino.cc/builder/ctags_runner.go @@ -30,7 +30,7 @@ package builder import ( - "fmt" + "os" "arduino.cc/builder/constants" "arduino.cc/builder/ctags" @@ -63,7 +63,7 @@ func (s *CTagsRunner) Run(ctx *types.Context) error { verbose := ctx.Verbose if verbose { - fmt.Println(commandLine) + logger.UnformattedFprintln(os.Stdout, commandLine) } sourceBytes, err := command.Output() diff --git a/src/arduino.cc/builder/i18n/i18n.go b/src/arduino.cc/builder/i18n/i18n.go index ce85d22c..6532ccb1 100644 --- a/src/arduino.cc/builder/i18n/i18n.go +++ b/src/arduino.cc/builder/i18n/i18n.go @@ -38,12 +38,15 @@ import ( "regexp" "strconv" "strings" + "sync" ) var PLACEHOLDER = regexp.MustCompile("{(\\d)}") type Logger interface { Fprintln(w io.Writer, level string, format string, a ...interface{}) + UnformattedFprintln(w io.Writer, s string) + UnformattedWrite(w io.Writer, data []byte) Println(level string, format string, a ...interface{}) Name() string } @@ -52,6 +55,10 @@ type NoopLogger struct{} func (s NoopLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) {} +func (s NoopLogger) UnformattedFprintln(w io.Writer, str string) {} + +func (s NoopLogger) UnformattedWrite(w io.Writer, data []byte) {} + func (s NoopLogger) Println(level string, format string, a ...interface{}) {} func (s NoopLogger) Name() string { @@ -62,13 +69,21 @@ type HumanTagsLogger struct{} func (s HumanTagsLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { format = "[" + level + "] " + format - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanTagsLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanTagsLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanTagsLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanTagsLogger) Name() string { return "humantags" } @@ -76,20 +91,48 @@ func (s HumanTagsLogger) Name() string { type HumanLogger struct{} func (s HumanLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - fmt.Fprintln(w, Format(format, a...)) + fprintln(w, Format(format, a...)) } func (s HumanLogger) Println(level string, format string, a ...interface{}) { s.Fprintln(os.Stdout, level, format, a...) } +func (s HumanLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s HumanLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + func (s HumanLogger) Name() string { return "human" } type MachineLogger struct{} -func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format string, a []interface{}) { +func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { + printMachineFormattedLogLine(w, level, format, a) +} + +func (s MachineLogger) Println(level string, format string, a ...interface{}) { + printMachineFormattedLogLine(os.Stdout, level, format, a) +} + +func (s MachineLogger) UnformattedFprintln(w io.Writer, str string) { + fprintln(w, str) +} + +func (s MachineLogger) Name() string { + return "machine" +} + +func (s MachineLogger) UnformattedWrite(w io.Writer, data []byte) { + write(w, data) +} + +func printMachineFormattedLogLine(w io.Writer, level string, format string, a []interface{}) { a = append([]interface{}(nil), a...) for idx, value := range a { typeof := reflect.Indirect(reflect.ValueOf(value)).Kind() @@ -97,20 +140,27 @@ func (s MachineLogger) printWithoutFormatting(w io.Writer, level string, format a[idx] = url.QueryEscape(value.(string)) } } - fmt.Fprintf(w, "===%s ||| %s ||| %s", level, format, a) - fmt.Fprintln(w) + fprintf(w, "===%s ||| %s ||| %s\n", level, format, a) } -func (s MachineLogger) Fprintln(w io.Writer, level string, format string, a ...interface{}) { - s.printWithoutFormatting(w, level, format, a) +var lock sync.Mutex + +func fprintln(w io.Writer, s string) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintln(w, s) } -func (s MachineLogger) Println(level string, format string, a ...interface{}) { - s.printWithoutFormatting(os.Stdout, level, format, a) +func write(w io.Writer, data []byte) { + lock.Lock() + defer lock.Unlock() + w.Write(data) } -func (s MachineLogger) Name() string { - return "machine" +func fprintf(w io.Writer, format string, a ...interface{}) { + lock.Lock() + defer lock.Unlock() + fmt.Fprintf(w, format, a...) } func FromJavaToGoSyntax(s string) string { diff --git a/src/arduino.cc/builder/types/accessories.go b/src/arduino.cc/builder/types/accessories.go index 6601bbaa..4c87c30c 100644 --- a/src/arduino.cc/builder/types/accessories.go +++ b/src/arduino.cc/builder/types/accessories.go @@ -29,6 +29,11 @@ package types +import ( + "bytes" + "sync" +) + type UniqueStringQueue []string func (queue UniqueStringQueue) Len() int { return len(queue) } @@ -74,3 +79,32 @@ func (queue *UniqueSourceFileQueue) Pop() SourceFile { func (queue *UniqueSourceFileQueue) Empty() bool { return queue.Len() == 0 } + +type BufferedUntilNewLineWriter struct { + PrintFunc PrintFunc + Buffer bytes.Buffer + lock sync.Mutex +} + +type PrintFunc func([]byte) + +func (w *BufferedUntilNewLineWriter) Write(p []byte) (n int, err error) { + w.lock.Lock() + defer w.lock.Unlock() + + writtenToBuffer, err := w.Buffer.Write(p) + return writtenToBuffer, err +} + +func (w *BufferedUntilNewLineWriter) Flush() { + w.lock.Lock() + defer w.lock.Unlock() + + remainingBytes := w.Buffer.Bytes() + if len(remainingBytes) > 0 { + if remainingBytes[len(remainingBytes)-1] != '\n' { + remainingBytes = append(remainingBytes, '\n') + } + w.PrintFunc(remainingBytes) + } +}