Skip to content
This repository was archived by the owner on May 8, 2024. It is now read-only.

CIRRUS_CLONE_DIR #130

Closed
wants to merge 10 commits into from
Closed
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ task:
task:
name: Test (Linux)
alias: test-linux
env:
CIRRUS_WORKING_DIR: /tmp/custom-working-dir
modules_cache:
fingerprint_script: cat go.sum
folder: $GOPATH/pkg/mod
Expand All @@ -40,6 +42,8 @@ docker_builder:
task:
name: Test (macOS)
alias: test-macos
env:
CIRRUS_WORKING_DIR: /tmp/custom-working-dir
macos_instance:
image: big-sur-base
test_script:
Expand Down
75 changes: 55 additions & 20 deletions internal/executor/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ func NewExecutor(
}
}

func (executor *Executor) SetPreCreatedWorkingDir(dir string) {
executor.preCreatedWorkingDir = dir
}

func (executor *Executor) RunBuild() {
log.Println("Getting initial commands...")
response, err := client.CirrusClient.InitialCommands(context.Background(), &api.InitialCommandsRequest{
Expand All @@ -99,7 +103,7 @@ func (executor *Executor) RunBuild() {
return
}

environment := getExpandedScriptEnvironment(executor, response.Environment)
environment := executor.GetExpandedScriptEnvironment(response.Environment)

workingDir, ok := environment["CIRRUS_WORKING_DIR"]
if ok {
Expand Down Expand Up @@ -178,10 +182,8 @@ func BoundedCommands(commands []*api.Command, fromName, toName string) []*api.Co
return commands[left:right]
}

func getExpandedScriptEnvironment(executor *Executor, responseEnvironment map[string]string) map[string]string {
if responseEnvironment == nil {
responseEnvironment = make(map[string]string)
}
func (executor *Executor) GetExpandedScriptEnvironment(responseEnvironment map[string]string) map[string]string {
responseEnvironment = executor.PopulateCloneAndWorkingDirEnvironmentVariables(responseEnvironment)

if _, ok := responseEnvironment["OS"]; !ok {
if _, ok := os.LookupEnv("OS"); !ok {
Expand All @@ -190,28 +192,60 @@ func getExpandedScriptEnvironment(executor *Executor, responseEnvironment map[st
}
responseEnvironment["CIRRUS_OS"] = runtime.GOOS

return expandEnvironmentRecursively(responseEnvironment)
}

func (executor *Executor) PopulateCloneAndWorkingDirEnvironmentVariables(environment map[string]string) map[string]string {
result := make(map[string]string)
for key, value := range environment {
result[key] = value
}

_, hasCloneDir := result["CIRRUS_CLONE_DIR"]
_, hasWorkingDir := result["CIRRUS_WORKING_DIR"]

// Use directory created by the persistent worker if CIRRUS_WORKING_DIR
// was not overridden in the task specification by the user
_, hasWorkingDir := responseEnvironment["CIRRUS_WORKING_DIR"]
if !hasWorkingDir && executor.preCreatedWorkingDir != "" {
responseEnvironment["CIRRUS_WORKING_DIR"] = executor.preCreatedWorkingDir
if executor.preCreatedWorkingDir != "" {
result["CIRRUS_CLONE_DIR"] = executor.preCreatedWorkingDir
hasCloneDir = true
// Need to override working dir too
if hasWorkingDir && !strings.Contains(result["CIRRUS_WORKING_DIR"], "CIRRUS_CLONE_DIR") {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if the $CIRRUS_CLONE_DIR variable occurrence in the middle of CIRRUS_WORKING_DIR should be considered as the CIRRUS_WORKING_DIR won't be rooted in CIRRUS_CLONE_DIR this way.

See #129 for a more strict check.

// If working dir doesn't depend on clone dir then we can default clone dir to the one provided by user
result["CIRRUS_WORKING_DIR"] = "$CIRRUS_CLONE_DIR"
}
}

if hasCloneDir && !hasWorkingDir {
// Only clone was overridden. Make sure $CIRRUS_WORKING_DIR is set
result["CIRRUS_WORKING_DIR"] = "$CIRRUS_CLONE_DIR"
}
if !hasCloneDir && hasWorkingDir {
// User did override working dir
if !strings.Contains(result["CIRRUS_WORKING_DIR"], "CIRRUS_CLONE_DIR") {
// If working dir doesn't depend on clone dir then we can default clone dir to the one provided by user
result["CIRRUS_CLONE_DIR"] = "$CIRRUS_WORKING_DIR"
}
}

if !hasCloneDir && !hasWorkingDir {
// none of the dirs are explicitly set. Make sure they'll be the same
result["CIRRUS_WORKING_DIR"] = "$CIRRUS_CLONE_DIR"
}

if _, ok := responseEnvironment["CIRRUS_WORKING_DIR"]; !ok {
if _, ok := result["CIRRUS_CLONE_DIR"]; !ok {
defaultTempDirPath := filepath.Join(os.TempDir(), "cirrus-ci-build")
if _, err := os.Stat(defaultTempDirPath); os.IsNotExist(err) {
responseEnvironment["CIRRUS_WORKING_DIR"] = filepath.ToSlash(defaultTempDirPath)
result["CIRRUS_CLONE_DIR"] = filepath.ToSlash(defaultTempDirPath)
} else if executor.commandFrom != "" {
// Default folder exists and we continue execution. Therefore we need to use it.
responseEnvironment["CIRRUS_WORKING_DIR"] = filepath.ToSlash(defaultTempDirPath)
result["CIRRUS_CLONE_DIR"] = filepath.ToSlash(defaultTempDirPath)
} else {
uniqueTempDirPath, _ := ioutil.TempDir(os.TempDir(), fmt.Sprintf("cirrus-task-%d", executor.taskIdentification.TaskId))
responseEnvironment["CIRRUS_WORKING_DIR"] = filepath.ToSlash(uniqueTempDirPath)
result["CIRRUS_CLONE_DIR"] = filepath.ToSlash(uniqueTempDirPath)
}
}

result := expandEnvironmentRecursively(responseEnvironment)

return result
}

Expand Down Expand Up @@ -386,7 +420,7 @@ func (executor *Executor) CloneRepository(env map[string]string) bool {

logUploader.Write([]byte("Using built-in Git...\n"))

working_dir := env["CIRRUS_WORKING_DIR"]
cloneDir := env["CIRRUS_CLONE_DIR"]
change := env["CIRRUS_CHANGE_IN_REPO"]
branch := env["CIRRUS_BRANCH"]
pr_number, is_pr := env["CIRRUS_PR"]
Expand Down Expand Up @@ -426,7 +460,7 @@ func (executor *Executor) CloneRepository(env map[string]string) bool {
var repo *git.Repository

if is_pr {
repo, err = git.PlainInit(working_dir, false)
repo, err = git.PlainInit(cloneDir, false)
if err != nil {
logUploader.Write([]byte(fmt.Sprintf("\nFailed to init repository: %s!", err)))
return false
Expand Down Expand Up @@ -498,13 +532,14 @@ func (executor *Executor) CloneRepository(env map[string]string) bool {
}
logUploader.Write([]byte(fmt.Sprintf("\nCloning %s...\n", cloneOptions.ReferenceName)))

repo, err = git.PlainClone(working_dir, false, &cloneOptions)
EnsureFolderExists(cloneDir)
repo, err = git.PlainClone(cloneDir, false, &cloneOptions)

if err != nil && retryableCloneError(err) {
logUploader.Write([]byte(fmt.Sprintf("\nRetryable error '%s' while cloning! Trying again...", err)))
os.RemoveAll(working_dir)
EnsureFolderExists(working_dir)
repo, err = git.PlainClone(working_dir, false, &cloneOptions)
os.RemoveAll(cloneDir)
EnsureFolderExists(cloneDir)
repo, err = git.PlainClone(cloneDir, false, &cloneOptions)
}

if err != nil {
Expand Down
162 changes: 162 additions & 0 deletions internal/executor/executor_linux_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
// +build !windows !freebsd

package executor

import (
"github.com/stretchr/testify/require"
"os"
"testing"
)

func TestPopulateCloneAndWorkingDirEnvironmentVariables(t *testing.T) {
tmpDirToRestore := os.Getenv("TMPDIR")
_ = os.Setenv("TMPDIR", "/tmp")
e := &Executor{}
ePreCreate := &Executor{}
ePreCreate.SetPreCreatedWorkingDir("/tmp/precreated-build")
examples := []struct {
Executor *Executor
Description string
Given, Expected map[string]string
}{
{
e,
"empty",
map[string]string{},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/cirrus-ci-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
ePreCreate,
"empty (precreated)",
map[string]string{},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
e,
"only working",
map[string]string{
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "$CIRRUS_WORKING_DIR",
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
},
{
ePreCreate,
"only working (precreated)",
map[string]string{
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
e,
"only working (monorepo)",
map[string]string{
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/cirrus-ci-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/foo",
},
},
{
ePreCreate,
"only working (monorepo + precreated)",
map[string]string{
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/foo",
},
},
{
e,
"only clone",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
ePreCreate,
"only clone (precreated)",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
e,
"both",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
},
{
ePreCreate,
"both (precreated)",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "/tmp/foo",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR",
},
},
{
e,
"both (monorepo)",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/bar",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/bar",
},
},
{
ePreCreate,
"both (monorepo + precreated)",
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/foo",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/bar",
},
map[string]string{
"CIRRUS_CLONE_DIR": "/tmp/precreated-build",
"CIRRUS_WORKING_DIR": "$CIRRUS_CLONE_DIR/bar",
},
},
}

for _, example := range examples {
t.Run(example.Description, func(t *testing.T) {
require.Equal(t, example.Expected, example.Executor.PopulateCloneAndWorkingDirEnvironmentVariables(example.Given))
})
}
_ = os.Setenv("TMPDIR", tmpDirToRestore)
}
5 changes: 2 additions & 3 deletions internal/executor/executor_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package executor_test
package executor

import (
"github.com/cirruslabs/cirrus-ci-agent/api"
"github.com/cirruslabs/cirrus-ci-agent/internal/executor"
"github.com/stretchr/testify/require"
"testing"
)
Expand Down Expand Up @@ -51,7 +50,7 @@ func TestLimitCommands(t *testing.T) {

for _, example := range examples {
t.Run(example.Description, func(t *testing.T) {
require.Equal(t, example.Expected, executor.BoundedCommands(commands, example.FromName, example.ToName))
require.Equal(t, example.Expected, BoundedCommands(commands, example.FromName, example.ToName))
})
}
}