Merge pull request #14 from swaroopar/feature/integrateGoAnsible

add first integration of ansible client

Author: swaroopar

.github/workflows/ci.yml

@@ -21,6 +21,9 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ">=1.22"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
       - run: go version
       - name: Format code
         run: |

go.mod

@@ -3,6 +3,7 @@ module xpanse-agent
 go 1.22.5
 
 require (
+	github.com/apenella/go-ansible/v2 v2.0.1
 	github.com/go-co-op/gocron/v2 v2.11.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/spf13/cobra v1.8.1
@@ -14,6 +15,8 @@ require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df // indirect
+	github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -32,6 +35,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
@@ -42,6 +46,7 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
@@ -56,5 +61,6 @@ require (
 	golang.org/x/tools v0.22.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -7,6 +7,12 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
+github.com/apenella/go-ansible/v2 v2.0.1 h1:9o3805u4NAIMf1px5EKpRS2LEa2aeQbMtHB7vilt7XE=
+github.com/apenella/go-ansible/v2 v2.0.1/go.mod h1:ifhiX4d0bpynb8yhdzLTmGl/38HqTYr/26PfjB1enXQ=
+github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df h1:sEikY2P+NZK/7VZUwIsnXIGElhsuFDSxh1bZYwHxdcI=
+github.com/apenella/go-common-utils/data v0.0.0-20220913191136-86daaa87e7df/go.mod h1:cLVL6GjUiKG/WyBzX+KD6h/XRV/HnNZIZbMNNiBgQ9o=
+github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df h1:SvlYbjlsSQDS7hbVT1h012/zdgvcwWJ+Yd9XRiiY/8s=
+github.com/apenella/go-common-utils/error v0.0.0-20220913191136-86daaa87e7df/go.mod h1:+3dyIlHX350xJIUIffwMLswZXU+N2FwDE05VuKqxYdw=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
@@ -28,10 +34,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
-github.com/go-co-op/gocron/v2 v2.11.0 h1:IOowNA6SzwdRFnD4/Ol3Kj6G2xKfsoiiGq2Jhhm9bvE=
-github.com/go-co-op/gocron/v2 v2.11.0/go.mod h1:xY7bJxGazKam1cz04EebrlP4S9q4iWdiAylMGP3jY9w=
 github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
 github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
+github.com/go-co-op/gocron/v2 v2.11.0 h1:IOowNA6SzwdRFnD4/Ol3Kj6G2xKfsoiiGq2Jhhm9bvE=
+github.com/go-co-op/gocron/v2 v2.11.0/go.mod h1:xY7bJxGazKam1cz04EebrlP4S9q4iWdiAylMGP3jY9w=
+github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
+github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
@@ -92,6 +100,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A=
 github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo=
+github.com/sosedoff/ansible-vault-go v0.2.0 h1:XqkBdqbXgTuFQ++NdrZvSdUTNozeb6S3V5x7FVs17vg=
+github.com/sosedoff/ansible-vault-go v0.2.0/go.mod h1:wMU54HNJfY0n0KIgbpA9m15NBfaUDlJrAsaZp0FwzkI=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
@@ -107,6 +117,7 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -201,6 +212,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/ansible/manage_virtual_env.go

@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *  SPDX-FileCopyrightText: Huawei Inc.
+ */
+
+package ansible
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"xpanse-agent/pkg/config"
+	"xpanse-agent/pkg/logger"
+)
+
+func CreateVirtualEnv(virtualEnvDir string, pythonVersion float32, moduleRequirementsFile string) error {
+	var cmd *exec.Cmd
+	var err error
+	// Check if the virtualenv already exists
+	if _, err = os.Stat(virtualEnvDir); !os.IsNotExist(err) {
+		logger.Logger.Info(fmt.Sprintf("Virtual environment '%s' already exists. Installing required modules on the same.", virtualEnvDir))
+	} else {
+		cmd = exec.Command(fmt.Sprintf("python%.2f", pythonVersion), "-m", "venv", filepath.Dir(virtualEnvDir))
+		err = cmd.Run()
+	}
+	if err != nil {
+		logger.Logger.Error(fmt.Sprintf("Error creating virtual environment: %s", err))
+		return err
+	}
+	err = installRequirements(virtualEnvDir, moduleRequirementsFile)
+	logger.Logger.Info(fmt.Sprintf("Virtual environment '%s' prepared successfully.", virtualEnvDir))
+	return err
+}
+
+func installRequirements(virtualEnvDir string, requirementsFile string) error {
+	logger.Logger.Info(fmt.Sprintf("Installing required modules in %s", virtualEnvDir))
+	var err error
+	originalWorkingDir, err := os.Getwd()
+	if err != nil {
+		logger.Logger.Error(fmt.Sprintf("Error getting current working directory: %s", err))
+		return err
+	}
+	err = os.Chdir(config.LoadedConfig.RepoCheckoutLocation)
+	logger.Logger.Info(fmt.Sprintf("running pip install from %s", config.LoadedConfig.RepoCheckoutLocation))
+	if err != nil {
+		return err
+	}
+	if _, err = os.Stat(requirementsFile); os.IsNotExist(err) {
+		logger.Logger.Error(fmt.Sprintf("Requirements file '%s' does not exist.", requirementsFile))
+		return err
+	}
+
+	// Prepare the pip install command
+	cmd := exec.Command(fmt.Sprintf("%s/bin/pip", virtualEnvDir), "install", "-r", requirementsFile)
+
+	// Run the command and capture output
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		logger.Logger.Error(fmt.Sprintf("Error installing packages: %s", err))
+		return err
+	}
+
+	logger.Logger.Info("Required modules installed successfully")
+	logger.Logger.Info(string(output))
+
+	// return back to original directory.
+	err = os.Chdir(originalWorkingDir)
+	return err
+}

pkg/ansible/run_playbook.go

@@ -0,0 +1,101 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *  SPDX-FileCopyrightText: Huawei Inc.
+ */
+
+package ansible
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/apenella/go-ansible/v2/pkg/execute"
+	results "github.com/apenella/go-ansible/v2/pkg/execute/result/json"
+	"github.com/apenella/go-ansible/v2/pkg/execute/stdoutcallback"
+	"github.com/apenella/go-ansible/v2/pkg/playbook"
+	"io"
+	"xpanse-agent/pkg/config"
+	"xpanse-agent/pkg/logger"
+)
+
+func RunPlaybook(playbookName string,
+	extraVars map[string]interface{},
+	inventory string,
+	virtualEnvRootDir string,
+	pythonVersion float32,
+	manageVirtualEnv bool,
+	requirementsFileNameInRepo string) error {
+	var res *results.AnsiblePlaybookJSONResults
+	var err error
+	buff := new(bytes.Buffer)
+	buffError := new(bytes.Buffer)
+
+	if manageVirtualEnv {
+		logger.Logger.Info("preparing virtual environment in " + virtualEnvRootDir)
+		err = CreateVirtualEnv(virtualEnvRootDir, pythonVersion, requirementsFileNameInRepo)
+		if err != nil {
+			return err
+		}
+	}
+
+	logger.Logger.Info("Running ansible task using ansible installed in venv " + virtualEnvRootDir)
+
+	ansiblePlaybookOptions := &playbook.AnsiblePlaybookOptions{
+		Become:    true,
+		Inventory: inventory,
+		ExtraVars: extraVars,
+	}
+
+	// constructs the ansible command to be executed.
+	playbookCmd := playbook.NewAnsiblePlaybookCmd(
+		playbook.WithPlaybooks(playbookName),
+		playbook.WithPlaybookOptions(ansiblePlaybookOptions),
+		playbook.WithBinary(fmt.Sprintf("%s/bin/ansible-playbook", virtualEnvRootDir)),
+	)
+
+	// execute the ansible command constructed above.
+	exec := stdoutcallback.NewJSONStdoutCallbackExecute(
+		execute.NewDefaultExecute(
+			execute.WithCmd(playbookCmd),
+			execute.WithCmdRunDir(config.LoadedConfig.RepoCheckoutLocation),
+			execute.WithErrorEnrich(playbook.NewAnsiblePlaybookErrorEnrich()),
+			execute.WithWrite(io.Writer(buff)),
+			execute.WithWriteError(io.Writer(buffError)),
+		),
+	)
+	err = exec.Execute(context.TODO())
+	if err != nil {
+		logger.Logger.Error(err.Error())
+		return err
+	}
+
+	// all warnings from Ansible are written to stderr stream.
+	if buffError.Len() > 0 {
+		for _, line := range bytes.Split(buffError.Bytes(), []byte("\n")) {
+			if len(line) > 0 {
+				logger.Logger.Warn(string(line))
+			}
+		}
+	}
+
+	res, err = results.ParseJSONResultsStream(io.Reader(buff))
+	if err != nil {
+		logger.Logger.Error(err.Error())
+		return err
+	}
+
+	parseAndLogAnsibleOutputForResults(res)
+
+	return err
+}
+
+func parseAndLogAnsibleOutputForResults(ansibleOutput *results.AnsiblePlaybookJSONResults) {
+	for _, play := range ansibleOutput.Plays {
+		for _, task := range play.Tasks {
+			name := task.Task.Name
+			for host, result := range task.Hosts {
+				logger.Logger.Info("|" + host + "|" + play.Play.Name + "|" + name + "|" + fmt.Sprintf("%v", result.StdoutLines) + "|" + fmt.Sprintf("%t", result.Changed) + "|" + fmt.Sprintf("%t", result.Failed))
+			}
+		}
+	}
+}

pkg/ansible/run_playbook_test.go

@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *  SPDX-FileCopyrightText: Huawei Inc.
+ */
+
+package ansible
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"xpanse-agent/pkg/config"
+)
+
+func TestRunPlaybook(t *testing.T) {
+	err := config.LoadConfig("test_data/test-xpanse-agent-config.yml")
+	assert.Nil(t, err)
+	runError := RunPlaybook(
+		"kafka-container-manage.yml", nil, "", "/tmp/kafka-test/", 3.10, true, "requirements.txt")
+	assert.Nil(t, runError)
+}

pkg/ansible/test_data/kafka-container-manage.yml

@@ -0,0 +1,45 @@
+---
+- name: Manage containers
+  hosts: localhost
+  connection: local
+  become: true
+
+  tasks:
+    - name: Install packages from requirements.txt
+      pip:
+        requirements: requirements.txt
+        state: present
+        chdir: "/opt/repos/forked/osc/samples/ansible"
+
+    - name: Ensure Docker service is running
+      systemd:
+        name: docker
+        state: started
+        enabled: yes
+
+    - name: Pull latest container image
+      docker_image:
+        name: "bitnami/kafka:3.3.2"
+        source: pull
+        force_source: yes
+
+    - name: Run container
+      docker_container:
+        name: "kafka-server"
+        image: "bitnami/kafka:3.3.2"
+        env:
+          KAFKA_BROKER_ID: "1"
+          KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://192.168.10.103:9092"
+          KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092"
+          ALLOW_PLAINTEXT_LISTENER: "yes"
+          KAFKA_CFG_ZOOKEEPER_CONNECT: "192.168.10.103:2181"
+        state: started
+        restart_policy: always
+        network_mode: host
+        ports:
+          - "9092:9092"
+          - "9093:9093"
+
+    - name: Remove unused images
+      docker_prune:
+        images: yes

pkg/ansible/test_data/requirements.txt

@@ -0,0 +1,2 @@
+requests==2.25.1
+ansible==10.3.0

pkg/ansible/test_data/test-xpanse-agent-config.yml

@@ -0,0 +1,3 @@
+---
+polling_frequency: 40
+repo_checkout_location: test_data

pkg/git/git_clone.go

@@ -21,6 +21,7 @@ func CloneProject(projectUrl string, serviceId string, branch string) error {
 	}
 	var repository *git.Repository
 	var err error
+	var w *git.Worktree
 
 	repository, err = git.PlainClone(fmt.Sprintf("/tmp/%s", serviceId), false, gitConfig)
 
@@ -47,7 +48,7 @@ func CloneProject(projectUrl string, serviceId string, branch string) error {
 		}
 
 		// load default main/master worktree
-		w, workTreeError := repository.Worktree()
+		w, err = repository.Worktree()
 
 		checkoutOptions := &git.CheckoutOptions{Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), Force: true}
 		if w != nil {
@@ -56,8 +57,8 @@ func CloneProject(projectUrl string, serviceId string, branch string) error {
 				return checkoutErr
 			}
 		}
-		if workTreeError != nil {
-			return workTreeError
+		if err != nil {
+			return err
 		}
 	}