Add opendal as backup source

Author: aawsome

config/README.md

@@ -277,15 +277,22 @@ See [Global Metrics labels](#global-metrics-labels-globalmetrics-labels).
 **Note**: All of the backup options mentioned before can also be used as
 snapshot-specific option and then only apply to this snapshot.
 
-| Attribute | Description                                                            | Default Value | Example Value                                                          |
-| --------- | ---------------------------------------------------------------------- | ------------- | ---------------------------------------------------------------------- |
-| name      | Name to identify this snapshot (to be used with the --name CLI option) | ""            | "myid"                                                                 |
-| sources   | Array of source directories or file(s) to back up.                     | []            | ["/dir1", "/dir2"]                                                     |
-| hooks     | Hooks to run before and after backing up the defined sources.          | Not set       | { run-before = [], run-after = [], run-failed = [], run-finally = [] } |
+| Attribute | Description                                                                                                              | Default Value | Example Value                                                          |
+| --------- | ------------------------------------------------------------------------------------------------------------------------ | ------------- | ---------------------------------------------------------------------- |
+| name      | Name to identify this snapshot (to be used with the --name CLI option)                                                   | ""            | "myid"                                                                 |
+| sources   | Array of source directories or file(s) to back up. Allows "opendal:" for a remote source if only a single source is used | []            | ["/dir1", "/dir2"], ["opendal:s3"]                                     |
+| hooks     | Hooks to run before and after backing up the defined sources.                                                            | Not set       | { run-before = [], run-after = [], run-failed = [], run-finally = [] } |
 
 Source-specific hooks are called additionally to global, repository and backup
 hooks when backing up the defined sources into a snapshot.
 
+### Snapshot Sourcey Options `[backup.snapshots.options]`
+
+Additional source options - depending on the used source. This is currently only
+relevant for an opendal source. These can be only set in the config file.
+
+see [Repository Options](#repository-options-repository)
+
 ### Forget Options `[forget]`
 
 **Note**: At least one of the `keep-*` options must be given. Use

config/full.toml

@@ -172,10 +172,19 @@ label-a = "xxx"
 
 # Backup options for specific sources - all above options are also available here and replace them for the given source
 [[backup.snapshots]]
-sources = ["/path/to/source1"]
+sources = [
+  "opendal:s3",
+] # Note: For opendal, currently only a single source is supported; see next [[backup.snapshots]] section for a local path example
 label = "label" # Default: not set
 # .. and so on. see [backup]
 
+# Additional backup source options - depending on source; currently only relevant for a "opendal:" source.
+# These can be only set in the config file.
+[backup.snapshots.options]
+connections = "20" # Default: Not set
+# Note that an opendal source use several service-dependent options which may be specified here, see
+# https://opendal.apache.org/docs/rust/opendal/services/index.html
+
 # Source-specific hooks: The given commands when backing up the defined source
 [backup.snapshots.hooks]
 run-before = ["echo before"] # Default: []

config/opendal-source.toml

@@ -0,0 +1,16 @@
+# rustic config file to backup a s3 storage to a local repository
+[repository]
+repository = "/path/to/repo"
+password = "password"
+
+[[backup.snapshots]]
+# Here only a single opendal source is allowed, the used service is given here
+sources = ["opendal:s3"]
+host = "s3" # we manually set the hostname, so we can easily distinguish the snapshot(s) created from here
+
+# Other options can be given here - note that opendal also support reading config from env files or AWS config dirs, see the opendal S3 docu
+[backup.snapshots.options]
+access_key_id = "xxx" # this can be omitted, when AWS config is used
+secret_access_key = "xxx" # this can be omitted, when AWS config is used
+bucket = "bucket_name"
+root = "/path/to/repo"

src/commands/backup.rs

@@ -21,7 +21,8 @@ use clap::ValueHint;
 use comfy_table::Cell;
 use conflate::{Merge, MergeFrom};
 use log::{debug, error, info, warn};
-use rustic_core::{Excludes, StringList};
+use rustic_backend::OpenDALBackend;
+use rustic_core::{ChildStdoutSource, Excludes, LocalSource, StdinSource, StringList};
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
@@ -145,6 +146,11 @@ pub struct BackupCmd {
     #[merge(skip)]
     sources: Vec<String>,
 
+    /// Other options for this source
+    #[clap(skip)]
+    #[merge(strategy = conflate::btreemap::append_or_ignore)]
+    options: BTreeMap<String, String>,
+
     /// Job name for the metrics. Default: rustic-backup
     #[clap(long, value_name = "JOB_NAME", env = "RUSTIC_METRICS_JOB")]
     #[merge(strategy=conflate::option::overwrite_none)]
@@ -327,6 +333,51 @@ impl BackupCmd {
         hooks.with_env(&hooks_variables)
     }
 
+    fn backup_source(
+        source: &PathList,
+        options: BTreeMap<String, String>,
+        backup_opts: BackupOptions,
+        snap: SnapshotFile,
+        repo: &IndexedIdsRepo,
+    ) -> Result<SnapshotFile> {
+        let backup_stdin = PathList::from_string("-")?;
+        let source = source
+            .clone()
+            .sanitize()
+            .with_context(|| format!("error sanitizing source=s\"{:?}\"", source))?
+            .merge();
+
+        let snap = if source.len() == 1
+                // TODO: This check should not be done on PathList, but in the sources list directly
+                && let Some(path) = source[0].to_string_lossy().strip_prefix("opendal:")
+        {
+            let source = OpenDALBackend::new(path, options)?.as_source()?;
+            repo.archive(&backup_opts, &source, snap, &[PathBuf::new()])?
+        } else if source == backup_stdin {
+            let path = PathBuf::from(&backup_opts.stdin_filename);
+            let backup_paths = vec![path.clone()];
+            if let Some(command) = &backup_opts.stdin_command {
+                let src = ChildStdoutSource::new(command, path)?;
+                let res = repo.archive(&backup_opts, &src, snap, &backup_paths)?;
+                src.finish()?;
+                res
+            } else {
+                let src = StdinSource::new(path);
+                repo.archive(&backup_opts, &src, snap, &backup_paths)?
+            }
+        } else {
+            let backup_path = source.paths();
+            let src = LocalSource::new(
+                backup_opts.ignore_save_opts,
+                &backup_opts.excludes,
+                &backup_opts.ignore_filter_opts,
+                &backup_path,
+            )?;
+            repo.archive(&backup_opts, &src, snap, &backup_path)?
+        };
+        Ok(snap)
+    }
+
     fn backup_snapshot(mut self, source: PathList, repo: &IndexedIdsRepo) -> Result<()> {
         let config = RUSTIC_APP.config();
         let snapshot_opts = &config.backup.snapshots;
@@ -369,13 +420,9 @@ impl BackupCmd {
             .no_scan(self.no_scan)
             .dry_run(config.global.dry_run);
 
+        let snap = self.snap_opts.to_snapshot()?;
         let snap = hooks.use_with(|| -> Result<_> {
-            let source = source
-                .clone()
-                .sanitize()
-                .with_context(|| format!("error sanitizing source=s\"{:?}\"", source))?
-                .merge();
-            Ok(repo.backup(&backup_opts, &source, self.snap_opts.to_snapshot()?)?)
+            Self::backup_source(&source, self.options, backup_opts, snap, repo)
         })?;
 
         if config.global.progress_options.json_progress {

src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap

@@ -78,6 +78,8 @@ run-after = []
 run-failed = []
 run-finally = []
 
+[backup.options]
+
 [backup.metrics-labels]
 
 [copy]

src/snapshots/rustic_rs__config__tests__default_config_passes.snap

@@ -171,6 +171,7 @@ RusticConfig {
         },
         snapshots: [],
         sources: [],
+        options: {},
         metrics_job: None,
         metrics_labels: {},
     },

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap

@@ -88,6 +88,8 @@ run-after = []
 run-failed = []
 run-finally = []
 
+[backup.options]
+
 [backup.metrics-labels]
 
 [copy]

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap

@@ -182,6 +182,7 @@ RusticConfig {
         },
         snapshots: [],
         sources: [],
+        options: {},
         metrics_job: None,
         metrics_labels: {},
     },

src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap

@@ -88,6 +88,8 @@ run-after = []
 run-failed = []
 run-finally = []
 
+[backup.options]
+
 [backup.metrics-labels]
 
 [copy]

tests/snapshots/show_config__show_config_passes.snap

@@ -78,6 +78,8 @@ run-after = []
 run-failed = []
 run-finally = []
 
+[backup.options]
+
 [backup.metrics-labels]
 
 [copy]