Add configuration to support free concurrent tasks and optimize git library update performance issues

db_lib/GitRepository.go

@@ -4,13 +4,16 @@ import (
 	"github.com/semaphoreui/semaphore/util"
 	"os"
 	"path"
+	"time"
 
 	"github.com/semaphoreui/semaphore/db"
 	"github.com/semaphoreui/semaphore/pkg/task_logger"
 )
 
 type GitRepositoryDirType int
 
+var GitUpdateTimeCache = map[string]int64{}
+
 const (
 	GitRepositoryTmpPath GitRepositoryDirType = iota
 	GitRepositoryFullPath
@@ -34,6 +37,28 @@ type GitRepository struct {
 	Client     GitClient
 }
 
+func (r GitRepository) isInCache() bool {
+	if r.TmpDirName != "" {
+		return false
+	}
+	cacheTime, ok := GitUpdateTimeCache[r.GetFullPath()]
+	if !ok {
+		return false
+	}
+	if util.Config.GitCacheTime == 0 {
+		return false
+	}
+	if (time.Now().Unix() - cacheTime) < int64(util.Config.GitCacheTime) {
+		return true
+	}
+	return false
+}
+func (r GitRepository) setCache() {
+	if r.TmpDirName != "" {
+		return
+	}
+	GitUpdateTimeCache[r.GetFullPath()] = time.Now().Unix()
+}
 func (r GitRepository) GetFullPath() string {
 	if r.TmpDirName != "" {
 		return path.Join(util.Config.TmpPath, r.TmpDirName)
@@ -47,10 +72,15 @@ func (r GitRepository) ValidateRepo() error {
 }
 
 func (r GitRepository) Clone() error {
+	r.setCache()
 	return r.Client.Clone(r)
 }
 
 func (r GitRepository) Pull() error {
+	if r.isInCache() {
+		return nil
+	}
+	r.setCache()
 	return r.Client.Pull(r)
 }
 
@@ -59,6 +89,9 @@ func (r GitRepository) Checkout(target string) error {
 }
 
 func (r GitRepository) CanBePulled() bool {
+	if r.isInCache() {
+		return true
+	}
 	return r.Client.CanBePulled(r)
 }
 

services/tasks/TaskPool.go

@@ -89,7 +89,11 @@ func (p *TaskPool) GetTask(id int) (task *TaskRunner) {
 
 // nolint: gocyclo
 func (p *TaskPool) Run() {
-	ticker := time.NewTicker(5 * time.Second)
+	tickerDuration := time.Duration(int64(util.Config.TaskPoolTickerSec)) * time.Second
+	if tickerDuration.Seconds() == 0 {
+		tickerDuration = 5 * time.Second
+	}
+	ticker := time.NewTicker(tickerDuration)
 
 	defer func() {
 		close(p.resourceLocker)
@@ -193,12 +197,14 @@ func (p *TaskPool) blocks(t *TaskRunner) bool {
 		return false
 	}
 
-	for _, r := range p.activeProj[t.Task.ProjectID] {
-		if r.Task.Status.IsFinished() {
-			continue
-		}
-		if r.Template.ID == t.Task.TemplateID {
-			return true
+	if !util.Config.TaskFreeConcurrencyMode {
+		for _, r := range p.activeProj[t.Task.ProjectID] {
+			if r.Task.Status.IsFinished() {
+				continue
+			}
+			if r.Template.ID == t.Task.TemplateID {
+				return true
+			}
 		}
 	}
 

util/config.go

@@ -137,6 +137,8 @@ type ConfigType struct {
 
 	GitClientId string `json:"git_client,omitempty" rule:"^go_git|cmd_git$" env:"SEMAPHORE_GIT_CLIENT" default:"cmd_git"`
 
+	GitCacheTime int `json:"git_cache_time" default:"0" env:"SEMAPHORE_GIT_CACHE_TIME"`
+
 	// web host
 	WebHost string `json:"web_host,omitempty" env:"SEMAPHORE_WEB_ROOT"`
 
@@ -191,6 +193,9 @@ type ConfigType struct {
 	// task concurrency
 	MaxParallelTasks int `json:"max_parallel_tasks,omitempty" default:"10" rule:"^[0-9]{1,10}$" env:"SEMAPHORE_MAX_PARALLEL_TASKS"`
 
+	TaskFreeConcurrencyMode bool `json:"task_free_concurrency_mode" env:"SEMAPHORE_TASK_FREE_CONCURRENCY_MODE"`
+	TaskPoolTickerSec       int  `json:"task_pool_ticker_sec" default:"5" env:"SEMAPHORE_TASK_POOL_TICKER_SEC"`
+
 	RunnerRegistrationToken string `json:"runner_registration_token,omitempty" env:"SEMAPHORE_RUNNER_REGISTRATION_TOKEN"`
 
 	// feature switches