Client/Server mode: sending configuration profile to a remote server

.github/workflows/build.yml

@@ -12,7 +12,6 @@ on:
       - 'docs/**'
 
 jobs:
-
   build:
     name: Build and test
     runs-on: ${{ matrix.os }}
@@ -25,7 +24,6 @@ jobs:
       GO: ${{ matrix.go_version }}
 
     steps:
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
 
@@ -71,6 +69,25 @@ jobs:
           name: code-coverage-report-${{ matrix.os }}
           path: coverage.out
 
+  test-ssh:
+    name: Test SSH client
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.24
+
+      - name: Run tests
+        run: |
+          make start-ssh-server
+          make ssh-test
+          make stop-ssh-server
+
   sonarCloudTrigger:
     needs: build
     name: SonarCloud Trigger

.vscode/settings.json

@@ -18,5 +18,6 @@
     "go.lintTool": "golangci-lint",
     "githubPullRequests.ignoredPullRequestBranches": [
         "master"
-    ]
+    ],
+    "go.buildTags": "ssh"
 }

Makefile

@@ -54,6 +54,9 @@ ifeq ($(UNAME),Darwin)
 	TMP_MOUNT=${TMP_MOUNT_DARWIN}
 endif
 
+TMPDIR ?= /tmp
+SSH_TESTS_TMPDIR=$(shell echo "$(TMPDIR)/resticprofile-ssh-tests" | tr -s /)
+
 TOC_START=<\!--ts-->
 TOC_END=<\!--te-->
 TOC_PATH=toc.md
@@ -336,3 +339,25 @@ deploy-current: build-linux build-pi
 		rsync -avz --progress $(BINARY_PI) $$server: ; \
 		ssh $$server "sudo -S install $(BINARY_PI) /usr/local/bin/resticprofile" ; \
 	done
+
+.PHONY: start-ssh-server
+start-ssh-server:
+	@echo "[*] $@"
+	@mkdir -p $(SSH_TESTS_TMPDIR) && rm -f $(SSH_TESTS_TMPDIR)/id_rsa* || echo "Failed to create temporary directory"
+	@ssh-keygen -t rsa -b 2048 -f $(SSH_TESTS_TMPDIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)"
+	@cd ./ssh/test && \
+		USER_ID=$(shell id -u) GROUP_ID=$(shell id -g) SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) \
+		docker compose up -d --force-recreate
+	@sleep 1
+	@ssh-keyscan -p 2222 -H localhost > $(SSH_TESTS_TMPDIR)/known_hosts
+
+.PHONY: stop-ssh-server
+stop-ssh-server:
+	@echo "[*] $@"
+	cd ./ssh/test && SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) docker compose down --remove-orphans
+	@test -d "$(SSH_TESTS_TMPDIR)" && rm -rf "$(SSH_TESTS_TMPDIR)" || echo "temporary directory not found, nothing to remove"
+
+.PHONY: ssh-test
+ssh-test:
+	@echo "[*] $@"
+	@go test -run TestSSHClient -v -tags ssh ./ssh

commands.go

@@ -168,6 +168,24 @@ func getOwnCommands() []ownCommand {
 			needConfiguration: false,
 			hide:              true,
 		},
+		{
+			name:              "send",
+			description:       "send a configuration profile to a remote client and execute a command",
+			action:            sendProfileCommand,
+			needConfiguration: true,
+			noProfile:         true,
+			hide:              true,
+			experimental:      true,
+		},
+		{
+			name:              "serve",
+			description:       "serve configuration profiles to remote clients",
+			action:            serveCommand,
+			needConfiguration: true,
+			noProfile:         true,
+			hide:              true,
+			experimental:      true,
+		},
 	}
 }
 

config/config.go

@@ -716,6 +716,22 @@ func (c *Config) getProfilePath(key string) string {
 	return c.flatKey(constants.SectionConfigurationProfiles, key)
 }
 
+// HasRemote returns true if the remote exists in the configuration
+func (c *Config) HasRemote(remoteName string) bool {
+	return c.IsSet(c.flatKey(constants.SectionConfigurationRemotes, remoteName))
+}
+
+func (c *Config) GetRemote(remoteName string) (*Remote, error) {
+	// we don't need to check the file version: the remotes can be in a separate configuration file
+
+	remote := NewRemote(c, remoteName)
+	err := c.unmarshalKey(c.flatKey(constants.SectionConfigurationRemotes, remoteName), remote)
+
+	rootPath := filepath.Dir(c.GetConfigFile())
+	remote.SetRootPath(rootPath)
+	return remote, err
+}
+
 // unmarshalConfig returns the decoder config options depending on the configuration version and format
 func (c *Config) unmarshalConfig() viper.DecoderConfigOption {
 	if c.GetVersion() == Version01 {

config/config_test.go

@@ -366,15 +366,6 @@ x=0
 }
 
 func TestIncludes(t *testing.T) {
-	files := []string{}
-	cleanFiles := func() {
-		for _, file := range files {
-			os.Remove(file)
-		}
-		files = files[:0]
-	}
-	defer cleanFiles()
-
 	createFile := func(t *testing.T, suffix, content string) string {
 		t.Helper()
 		name := ""
@@ -383,7 +374,9 @@ func TestIncludes(t *testing.T) {
 			defer file.Close()
 			_, err = file.WriteString(content)
 			name = file.Name()
-			files = append(files, name)
+			t.Cleanup(func() {
+				_ = os.Remove(name)
+			})
 		}
 		require.NoError(t, err)
 		return name
@@ -399,7 +392,6 @@ func TestIncludes(t *testing.T) {
 	testID := fmt.Sprintf("%d", time.Now().Unix())
 
 	t.Run("multiple-includes", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`includes=['*%[1]s.inc.toml','*%[1]s.inc.yaml','*%[1]s.inc.json']`, testID)
 
 		configFile := createFile(t, "profiles.conf", content)
@@ -415,8 +407,6 @@ func TestIncludes(t *testing.T) {
 	})
 
 	t.Run("overrides", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.conf", `
 includes = "*`+testID+`.inc.toml"
 [default]
@@ -435,8 +425,6 @@ repository = "overridden-repo"`)
 	})
 
 	t.Run("mixins", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.conf", `
 version = 2
 includes = "*`+testID+`.inc.toml"
@@ -463,8 +451,6 @@ use = "another-run-before2"`)
 	})
 
 	t.Run("hcl-includes-only-hcl", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.hcl", `includes = "*`+testID+`.inc.*"`)
 		createFile(t, "pass-"+testID+".inc.hcl", `one { }`)
 
@@ -473,13 +459,11 @@ use = "another-run-before2"`)
 
 		createFile(t, "fail-"+testID+".inc.toml", `[two]`)
 		_, err := LoadFile(configFile, "")
-		assert.Error(t, err)
+		require.Error(t, err)
 		assert.Regexp(t, ".+ is in hcl format, includes must use the same format", err.Error())
 	})
 
 	t.Run("non-hcl-include-no-hcl", func(t *testing.T) {
-		defer cleanFiles()
-
 		configFile := createFile(t, "profiles.toml", `includes = "*`+testID+`.inc.*"`)
 		createFile(t, "pass-"+testID+".inc.toml", "[one]\nk='v'")
 
@@ -488,12 +472,11 @@ use = "another-run-before2"`)
 
 		createFile(t, "fail-"+testID+".inc.hcl", `one { }`)
 		_, err := LoadFile(configFile, "")
-		assert.Error(t, err)
+		require.Error(t, err)
 		assert.Regexp(t, "hcl format .+ cannot be used in includes from toml", err.Error())
 	})
 
 	t.Run("cannot-load-different-versions", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`includes=['*%s.inc.json']`, testID)
 
 		configFile := createFile(t, "profiles.conf", content)
@@ -505,7 +488,6 @@ use = "another-run-before2"`)
 	})
 
 	t.Run("cannot-load-different-versions", func(t *testing.T) {
-		defer cleanFiles()
 		content := fmt.Sprintf(`{"version": 2, "includes":["*%s.inc.json"]}`, testID)
 
 		configFile := createFile(t, "profiles.json", content)

config/error.go

@@ -3,5 +3,6 @@ package config
 import "errors"
 
 var (
-	ErrNotFound = errors.New("not found")
+	ErrNotFound               = errors.New("not found")
+	ErrNotSupportedInVersion1 = errors.New("not supported in configuration version 1")
 )

config/remote.go

@@ -0,0 +1,36 @@
+package config
+
+type Remote struct {
+	name              string
+	config            *Config
+	Connection        string   `mapstructure:"connection" default:"ssh" description:"Connection type to use to connect to the remote client"`
+	Host              string   `mapstructure:"host" description:"Address of the remote client. Format: <host>:<port>"`
+	Username          string   `mapstructure:"username" description:"User to connect to the remote client"`
+	PrivateKeyPath    string   `mapstructure:"private-key" description:"Path to the private key to use for authentication"`
+	KnownHostsPath    string   `mapstructure:"known-hosts" description:"Path to the known hosts file"`
+	BinaryPath        string   `mapstructure:"binary-path" description:"Path to the resticprofile binary to use on the remote client"`
+	ConfigurationFile string   `mapstructure:"configuration-file" description:"Path to the configuration file to transfer to the remote client"`
+	ProfileName       string   `mapstructure:"profile-name" description:"Name of the profile to use on the remote client"`
+	SendFiles         []string `mapstructure:"send-files" description:"Other configuration files to transfer to the remote client"`
+	SSHConfig         string   `mapstructure:"ssh-config" description:"Path to the OpenSSH config file to use for the connection"`
+}
+
+func NewRemote(config *Config, name string) *Remote {
+	remote := &Remote{
+		name:   name,
+		config: config,
+	}
+	return remote
+}
+
+// SetRootPath changes the path of all the relative paths and files in the configuration
+func (r *Remote) SetRootPath(rootPath string) {
+	r.PrivateKeyPath = fixPath(r.PrivateKeyPath, expandEnv, absolutePrefix(rootPath))
+	r.KnownHostsPath = fixPath(r.KnownHostsPath, expandEnv, absolutePrefix(rootPath))
+	r.ConfigurationFile = fixPath(r.ConfigurationFile, expandEnv, absolutePrefix(rootPath))
+	r.SSHConfig = fixPath(r.SSHConfig, expandEnv, absolutePrefix(rootPath))
+
+	for i := range r.SendFiles {
+		r.SendFiles[i] = fixPath(r.SendFiles[i], expandEnv, absolutePrefix(rootPath))
+	}
+}

constants/exit_code.go

@@ -0,0 +1,10 @@
+package constants
+
+const (
+	ExitSuccess = iota
+	ExitGeneralError
+	ExitErrorInvalidFlags
+	ExitRunningOnBattery
+	ExitCannotSetupRemoteConfiguration
+	ExitErrorChildHasNoParentPort = 10
+)

constants/other.go

@@ -3,4 +3,5 @@ package constants
 const (
 	TemporaryDirMarker = "temp:"
 	JSONSchema         = "$schema"
+	ManifestFilename   = ".manifest.json"
 )

constants/section.go

@@ -13,6 +13,7 @@ const (
 	SectionConfigurationMixins      = "mixins"
 	SectionConfigurationMixinUse    = "use"
 	SectionConfigurationSchedule    = "schedule"
+	SectionConfigurationRemotes     = "remotes"
 
 	SectionDefinitionCommon = "common"
 	SectionDefinitionForget = "forget"

examples/linux.yaml

@@ -106,7 +106,15 @@ self:
             at: "*:15,20,25"
             permission: system
             after-network-online: true
+        exclude-file:
+            - root-excludes
+            - excludes
     check:
+        schedule:
+          at: "*:15"
+          permission: system
+    copy:
+        initialize: true
         schedule-permission: user
         schedule:
             - "*:15"
@@ -131,9 +139,8 @@ src:
         run-after: echo All Done!
         run-before:
             - echo Starting!
-            - ls -al ~/go
         source:
-            - ~/go
+            - ~/go/src/github.com/creativeprojects/resticprofile
         tag:
             - test
             - dev

filesearch/filesearch_test.go

@@ -282,6 +282,7 @@ func TestFindResticBinaryWithTilde(t *testing.T) {
 		t.Skip("not supported on Windows")
 		return
 	}
+
 	home, err := os.UserHomeDir()
 	require.NoError(t, err)
 
@@ -336,7 +337,6 @@ func TestShellExpand(t *testing.T) {
 func TestFindConfigurationIncludes(t *testing.T) {
 	t.Parallel()
 
-	fs := afero.NewMemMapFs()
 	testID := fmt.Sprintf("%x", time.Now().UnixNano())
 	tempDir := os.TempDir()
 	files := []string{
@@ -346,6 +346,7 @@ func TestFindConfigurationIncludes(t *testing.T) {
 		filepath.Join(tempDir, "inc3."+testID+".conf"),
 	}
 
+	fs := afero.NewMemMapFs()
 	for _, file := range files {
 		require.NoError(t, afero.WriteFile(fs, file, []byte{}, iofs.ModePerm))
 	}

flags.go

@@ -38,6 +38,7 @@ type commandLineFlags struct {
 	noPriority      bool
 	ignoreOnBattery int
 	usagesHelp      string
+	remote          string // url of the remote server to download configuration files from
 }
 
 func envValueOverride[T any](defaultValue T, keys ...string) T {
@@ -92,6 +93,7 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) {
 		noPriority:      envValueOverride(false, "RESTICPROFILE_NO_PRIORITY"),
 		wait:            envValueOverride(false, "RESTICPROFILE_WAIT"),
 		ignoreOnBattery: envValueOverride(0, "RESTICPROFILE_IGNORE_ON_BATTERY"),
+		remote:          envValueOverride("", "RESTICPROFILE_REMOTE"),
 	}
 
 	flagset.BoolVarP(&flags.help, "help", "h", flags.help, "display this help")
@@ -113,6 +115,9 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) {
 	flagset.BoolVarP(&flags.wait, "wait", "w", flags.wait, "wait at the end until the user presses the enter key")
 	flagset.IntVar(&flags.ignoreOnBattery, "ignore-on-battery", flags.ignoreOnBattery, "don't start the profile when the computer is running on battery. You can specify a value to ignore only when the % charge left is less or equal than the value")
 	flagset.Lookup("ignore-on-battery").NoOptDefVal = "100" // 0 is flag not set, 100 is for a flag with no value (meaning just battery discharge)
+	flagset.StringVarP(&flags.remote, "remote", "r", flags.remote, "remote server to download configuration files from")
+	// keep the "remote" flag hidden for now
+	_ = flagset.MarkHidden("remote")
 
 	flagset.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName {
 		switch name {