Add support to parse a filelist specified within a Bender.yml

CHANGELOG.md

@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 - `script`: Allow flist formats to use `only-sources`/`only-includes`/`only-defines` flags.
 - Add check to ensure files referenced in all manifests exist.
 - Add warnings for unknown fields in manifest.
+- Add support to indicate and read in external file lists in manifest.
 
 ### Changed
 - `update`: Clean up alignment of manual resolution output.

README.md

@@ -150,6 +150,11 @@ sources:
   # Source files can use glob patterns to include all matching files:
   - src/more_stuff/**/*.sv
 
+  # File list in another external file, supporting simple file names, `+define+` and `+incdir+`
+  - external_flists:
+      - other_file_list.f
+    files: []
+
 # A list of include directories which should implicitly be added to source
 # file groups of packages that have the current package as a dependency.
 # Optional.

src/config.rs

@@ -11,7 +11,9 @@
 use std;
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::fmt;
+use std::fs::File;
 use std::hash::Hash;
+use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
@@ -600,6 +602,8 @@ pub struct PartialSources {
     pub defines: Option<IndexMap<String, Option<String>>>,
     /// The source file paths.
     pub files: Vec<PartialSourceFile>,
+    /// The list of external flists to include.
+    pub external_flists: Option<Vec<String>>,
     /// Unknown extra fields
     #[serde(flatten)]
     extra: HashMap<String, Value>,
@@ -614,6 +618,7 @@ impl PartialSources {
             include_dirs: None,
             defines: None,
             files: Vec::new(),
+            external_flists: None,
             extra: HashMap::new(),
         }
     }
@@ -626,6 +631,7 @@ impl PrefixPaths for PartialSources {
             include_dirs: self.include_dirs.prefix_paths(prefix)?,
             defines: self.defines,
             files: self.files.prefix_paths(prefix)?,
+            external_flists: self.external_flists.prefix_paths(prefix)?,
             extra: self.extra,
         })
     }
@@ -638,6 +644,7 @@ impl From<Vec<PartialSourceFile>> for PartialSources {
             include_dirs: None,
             defines: None,
             files: v,
+            external_flists: None,
             extra: HashMap::new(),
         }
     }
@@ -647,9 +654,121 @@ impl Validate for PartialSources {
     type Output = Sources;
     type Error = Error;
     fn validate(self, package_name: &str, pre_output: bool) -> Result<Sources> {
+        let external_flists: Result<Vec<_>> = self
+            .external_flists
+            .clone()
+            .unwrap_or_default()
+            .iter()
+            .map(|path| env_path_from_string(path.to_string()))
+            .collect();
+
+        let external_flist_list: Result<Vec<(PathBuf, Vec<String>)>> = external_flists?
+            .into_iter()
+            .map(|filename| {
+                let file = File::open(&filename).map_err(|cause| {
+                    Error::chain(
+                        format!("Unable to open external flist file {:?}", filename),
+                        cause,
+                    )
+                })?;
+                let reader = BufReader::new(file);
+                let lines: Vec<String> = reader
+                    .lines()
+                    .map(|line| {
+                        line.map_err(|cause| {
+                            Error::chain(
+                                format!("Error reading external flist file {:?}", filename),
+                                cause,
+                            )
+                        })
+                    })
+                    .collect::<Result<Vec<String>>>()?;
+                let lines = lines
+                    .iter()
+                    .filter_map(|line| {
+                        let line = line.trim();
+                        if line.is_empty() || line.starts_with('#') || line.starts_with("//") {
+                            None
+                        } else {
+                            Some(
+                                line.split_whitespace()
+                                    .map(|s| s.to_string())
+                                    .collect::<Vec<_>>(),
+                            )
+                        }
+                    })
+                    .flatten()
+                    .collect();
+                Ok((filename.parent().unwrap().to_path_buf(), lines))
+            })
+            .collect();
+
+        let external_flist_groups: Result<Vec<PartialSourceFile>> = external_flist_list?
+            .into_iter()
+            .map(|(flist_dir, flist)| {
+                Ok(PartialSourceFile::Group(Box::new(PartialSources {
+                    target: None,
+                    include_dirs: Some(
+                        flist
+                            .clone()
+                            .into_iter()
+                            .filter_map(|file| {
+                                if file.starts_with("+incdir+") {
+                                    Some(file.trim_start_matches("+incdir+").to_string())
+                                } else {
+                                    None
+                                }
+                            })
+                            .flat_map(|s| s.split('+').map(|s| s.to_string()).collect::<Vec<_>>())
+                            .map(|dir| dir.prefix_paths(&flist_dir))
+                            .collect::<Result<_>>()?,
+                    ),
+                    defines: Some(
+                        flist
+                            .clone()
+                            .into_iter()
+                            .filter_map(|file| {
+                                if file.starts_with("+define+") {
+                                    Some(file.trim_start_matches("+define+").to_string())
+                                } else {
+                                    None
+                                }
+                            })
+                            .flat_map(|s| s.split('+').map(|s| s.to_string()).collect::<Vec<_>>())
+                            .map(|file| {
+                                if let Some(eq_idx) = file.find("=") {
+                                    (
+                                        file[..eq_idx].to_string(),
+                                        Some(file[eq_idx + 1..].to_string()),
+                                    )
+                                } else {
+                                    (file.to_string(), None)
+                                }
+                            })
+                            .collect(),
+                    ),
+                    files: flist
+                        .into_iter()
+                        .filter_map(|file| {
+                            if file.starts_with("+") {
+                                None
+                            } else {
+                                // prefix path
+                                Some(PartialSourceFile::File(file))
+                            }
+                        })
+                        .map(|file| file.prefix_paths(&flist_dir))
+                        .collect::<Result<Vec<_>>>()?,
+                    external_flists: None,
+                    extra: HashMap::new(),
+                })))
+            })
+            .collect();
+
         let post_env_files: Vec<PartialSourceFile> = self
             .files
             .into_iter()
+            .chain(external_flist_groups?.into_iter())
             .map(|file| match file {
                 PartialSourceFile::File(file) => {
                     Ok(PartialSourceFile::File(env_string_from_string(file)?))
@@ -680,6 +799,7 @@ impl Validate for PartialSources {
             .iter()
             .map(|path| env_path_from_string(path.to_string()))
             .collect();
+
         let defines = self.defines.unwrap_or_default();
         let files: Result<Vec<_>> = post_glob_files
             .into_iter()

src/sess.rs

@@ -22,7 +22,7 @@ use std::fs::canonicalize;
 use dunce::canonicalize;
 
 use async_recursion::async_recursion;
-use futures::future::{self, join_all};
+use futures::future::join_all;
 use futures::TryFutureExt;
 use indexmap::{IndexMap, IndexSet};
 use semver::Version;
@@ -539,35 +539,36 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> {
             // Initialize.
             self.sess.stats.num_database_init.increment();
             // TODO MICHAERO: May need throttle
-            future::lazy(|_| {
-                stageln!("Cloning", "{} ({})", name2, url2);
-                Ok(())
-            })
-            .and_then(|_| git.clone().spawn_with(|c| c.arg("init").arg("--bare")))
-            .and_then(|_| {
-                git.clone()
-                    .spawn_with(|c| c.arg("remote").arg("add").arg("origin").arg(url))
-            })
-            .and_then(|_| git.clone().fetch("origin"))
-            .and_then(|_| async {
-                if let Some(reference) = fetch_ref {
-                    git.clone().fetch_ref("origin", reference).await
-                } else {
-                    Ok(())
-                }
-            })
-            .await
-            .map_err(move |cause| {
-                if url3.contains("git@") {
-                    warnln!("Please ensure your public ssh key is added to the git server.");
-                }
-                warnln!("Please ensure the url is correct and you have access to the repository.");
-                Error::chain(
-                    format!("Failed to initialize git database in {:?}.", db_dir),
-                    cause,
-                )
-            })
-            .map(move |_| git)
+            stageln!("Cloning", "{} ({})", name2, url2);
+            git.clone()
+                .spawn_with(|c| c.arg("init").arg("--bare"))
+                .await?;
+            git.clone()
+                .spawn_with(|c| c.arg("remote").arg("add").arg("origin").arg(url))
+                .await?;
+            git.clone()
+                .fetch("origin")
+                .and_then(|_| async {
+                    if let Some(reference) = fetch_ref {
+                        git.clone().fetch_ref("origin", reference).await
+                    } else {
+                        Ok(())
+                    }
+                })
+                .await
+                .map_err(move |cause| {
+                    if url3.contains("git@") {
+                        warnln!("Please ensure your public ssh key is added to the git server.");
+                    }
+                    warnln!(
+                        "Please ensure the url is correct and you have access to the repository."
+                    );
+                    Error::chain(
+                        format!("Failed to initialize git database in {:?}.", db_dir),
+                        cause,
+                    )
+                })
+                .map(move |_| git)
         } else {
             // Update if the manifest has been modified since the last fetch.
             let db_mtime = try_modification_time(db_dir.join("FETCH_HEAD"));
@@ -577,30 +578,30 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> {
             }
             self.sess.stats.num_database_fetch.increment();
             // TODO MICHAERO: May need throttle
-            future::lazy(|_| {
-                stageln!("Fetching", "{} ({})", name2, url2);
-                Ok(())
-            })
-            .and_then(|_| git.clone().fetch("origin"))
-            .and_then(|_| async {
-                if let Some(reference) = fetch_ref {
-                    git.clone().fetch_ref("origin", reference).await
-                } else {
-                    Ok(())
-                }
-            })
-            .await
-            .map_err(move |cause| {
-                if url3.contains("git@") {
-                    warnln!("Please ensure your public ssh key is added to the git server.");
-                }
-                warnln!("Please ensure the url is correct and you have access to the repository.");
-                Error::chain(
-                    format!("Failed to update git database in {:?}.", db_dir),
-                    cause,
-                )
-            })
-            .map(move |_| git)
+            stageln!("Fetching", "{} ({})", name2, url2);
+            git.clone()
+                .fetch("origin")
+                .and_then(|_| async {
+                    if let Some(reference) = fetch_ref {
+                        git.clone().fetch_ref("origin", reference).await
+                    } else {
+                        Ok(())
+                    }
+                })
+                .await
+                .map_err(move |cause| {
+                    if url3.contains("git@") {
+                        warnln!("Please ensure your public ssh key is added to the git server.");
+                    }
+                    warnln!(
+                        "Please ensure the url is correct and you have access to the repository."
+                    );
+                    Error::chain(
+                        format!("Failed to update git database in {:?}.", db_dir),
+                        cause,
+                    )
+                })
+                .map(move |_| git)
         }
     }
 
@@ -1612,7 +1613,7 @@ pub struct DependencyEntry {
 
 impl DependencyEntry {
     /// Obtain the dependency version for this entry.
-    pub fn version(&self) -> DependencyVersion {
+    pub fn version(&self) -> DependencyVersion<'_> {
         match self.source {
             DependencySource::Registry => unimplemented!(),
             DependencySource::Path(_) => DependencyVersion::Path,