Skip to content

Commit c159a77

Browse files
micprogmicprog
authored
Merge pull request #220 from tenstorrent/ext_flist
Add support to parse a filelist specified within a Bender.yml
2 parents 8249305 + 3b1295c commit c159a77

File tree

4 files changed

+182
-55
lines changed

4 files changed

+182
-55
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
2828
- `script`: Allow flist formats to use `only-sources`/`only-includes`/`only-defines` flags.
2929
- Add check to ensure files referenced in all manifests exist.
3030
- Add warnings for unknown fields in manifest.
31+
- Add support to indicate and read in external file lists in manifest.
3132

3233
### Changed
3334
- `update`: Clean up alignment of manual resolution output.

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,11 @@ sources:
150150
# Source files can use glob patterns to include all matching files:
151151
- src/more_stuff/**/*.sv
152152

153+
# File list in another external file, supporting simple file names, `+define+` and `+incdir+`
154+
- external_flists:
155+
- other_file_list.f
156+
files: []
157+
153158
# A list of include directories which should implicitly be added to source
154159
# file groups of packages that have the current package as a dependency.
155160
# Optional.

‎src/config.rs‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
use std;
1212
use std::collections::{BTreeMap, BTreeSet, HashMap};
1313
use std::fmt;
14+
use std::fs::File;
1415
use std::hash::Hash;
16+
use std::io::{BufRead, BufReader};
1517
use std::path::{Path, PathBuf};
1618
use std::str::FromStr;
1719

@@ -600,6 +602,8 @@ pub struct PartialSources {
600602
pub defines: Option<IndexMap<String, Option<String>>>,
601603
/// The source file paths.
602604
pub files: Vec<PartialSourceFile>,
605+
/// The list of external flists to include.
606+
pub external_flists: Option<Vec<String>>,
603607
/// Unknown extra fields
604608
#[serde(flatten)]
605609
extra: HashMap<String, Value>,
@@ -614,6 +618,7 @@ impl PartialSources {
614618
include_dirs: None,
615619
defines: None,
616620
files: Vec::new(),
621+
external_flists: None,
617622
extra: HashMap::new(),
618623
}
619624
}
@@ -626,6 +631,7 @@ impl PrefixPaths for PartialSources {
626631
include_dirs: self.include_dirs.prefix_paths(prefix)?,
627632
defines: self.defines,
628633
files: self.files.prefix_paths(prefix)?,
634+
external_flists: self.external_flists.prefix_paths(prefix)?,
629635
extra: self.extra,
630636
})
631637
}
@@ -638,6 +644,7 @@ impl From<Vec<PartialSourceFile>> for PartialSources {
638644
include_dirs: None,
639645
defines: None,
640646
files: v,
647+
external_flists: None,
641648
extra: HashMap::new(),
642649
}
643650
}
@@ -647,9 +654,121 @@ impl Validate for PartialSources {
647654
type Output = Sources;
648655
type Error = Error;
649656
fn validate(self, package_name: &str, pre_output: bool) -> Result<Sources> {
657+
let external_flists: Result<Vec<_>> = self
658+
.external_flists
659+
.clone()
660+
.unwrap_or_default()
661+
.iter()
662+
.map(|path| env_path_from_string(path.to_string()))
663+
.collect();
664+
665+
let external_flist_list: Result<Vec<(PathBuf, Vec<String>)>> = external_flists?
666+
.into_iter()
667+
.map(|filename| {
668+
let file = File::open(&filename).map_err(|cause| {
669+
Error::chain(
670+
format!("Unable to open external flist file {:?}", filename),
671+
cause,
672+
)
673+
})?;
674+
let reader = BufReader::new(file);
675+
let lines: Vec<String> = reader
676+
.lines()
677+
.map(|line| {
678+
line.map_err(|cause| {
679+
Error::chain(
680+
format!("Error reading external flist file {:?}", filename),
681+
cause,
682+
)
683+
})
684+
})
685+
.collect::<Result<Vec<String>>>()?;
686+
let lines = lines
687+
.iter()
688+
.filter_map(|line| {
689+
let line = line.trim();
690+
if line.is_empty() || line.starts_with('#') || line.starts_with("//") {
691+
None
692+
} else {
693+
Some(
694+
line.split_whitespace()
695+
.map(|s| s.to_string())
696+
.collect::<Vec<_>>(),
697+
)
698+
}
699+
})
700+
.flatten()
701+
.collect();
702+
Ok((filename.parent().unwrap().to_path_buf(), lines))
703+
})
704+
.collect();
705+
706+
let external_flist_groups: Result<Vec<PartialSourceFile>> = external_flist_list?
707+
.into_iter()
708+
.map(|(flist_dir, flist)| {
709+
Ok(PartialSourceFile::Group(Box::new(PartialSources {
710+
target: None,
711+
include_dirs: Some(
712+
flist
713+
.clone()
714+
.into_iter()
715+
.filter_map(|file| {
716+
if file.starts_with("+incdir+") {
717+
Some(file.trim_start_matches("+incdir+").to_string())
718+
} else {
719+
None
720+
}
721+
})
722+
.flat_map(|s| s.split('+').map(|s| s.to_string()).collect::<Vec<_>>())
723+
.map(|dir| dir.prefix_paths(&flist_dir))
724+
.collect::<Result<_>>()?,
725+
),
726+
defines: Some(
727+
flist
728+
.clone()
729+
.into_iter()
730+
.filter_map(|file| {
731+
if file.starts_with("+define+") {
732+
Some(file.trim_start_matches("+define+").to_string())
733+
} else {
734+
None
735+
}
736+
})
737+
.flat_map(|s| s.split('+').map(|s| s.to_string()).collect::<Vec<_>>())
738+
.map(|file| {
739+
if let Some(eq_idx) = file.find("=") {
740+
(
741+
file[..eq_idx].to_string(),
742+
Some(file[eq_idx + 1..].to_string()),
743+
)
744+
} else {
745+
(file.to_string(), None)
746+
}
747+
})
748+
.collect(),
749+
),
750+
files: flist
751+
.into_iter()
752+
.filter_map(|file| {
753+
if file.starts_with("+") {
754+
None
755+
} else {
756+
// prefix path
757+
Some(PartialSourceFile::File(file))
758+
}
759+
})
760+
.map(|file| file.prefix_paths(&flist_dir))
761+
.collect::<Result<Vec<_>>>()?,
762+
external_flists: None,
763+
extra: HashMap::new(),
764+
})))
765+
})
766+
.collect();
767+
650768
let post_env_files: Vec<PartialSourceFile> = self
651769
.files
652770
.into_iter()
771+
.chain(external_flist_groups?.into_iter())
653772
.map(|file| match file {
654773
PartialSourceFile::File(file) => {
655774
Ok(PartialSourceFile::File(env_string_from_string(file)?))
@@ -680,6 +799,7 @@ impl Validate for PartialSources {
680799
.iter()
681800
.map(|path| env_path_from_string(path.to_string()))
682801
.collect();
802+
683803
let defines = self.defines.unwrap_or_default();
684804
let files: Result<Vec<_>> = post_glob_files
685805
.into_iter()

‎src/sess.rs‎

Lines changed: 56 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use std::fs::canonicalize;
2222
use dunce::canonicalize;
2323

2424
use async_recursion::async_recursion;
25-
use futures::future::{self, join_all};
25+
use futures::future::join_all;
2626
use futures::TryFutureExt;
2727
use indexmap::{IndexMap, IndexSet};
2828
use semver::Version;
@@ -539,35 +539,36 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> {
539539
// Initialize.
540540
self.sess.stats.num_database_init.increment();
541541
// TODO MICHAERO: May need throttle
542-
future::lazy(|_| {
543-
stageln!("Cloning", "{} ({})", name2, url2);
544-
Ok(())
545-
})
546-
.and_then(|_| git.clone().spawn_with(|c| c.arg("init").arg("--bare")))
547-
.and_then(|_| {
548-
git.clone()
549-
.spawn_with(|c| c.arg("remote").arg("add").arg("origin").arg(url))
550-
})
551-
.and_then(|_| git.clone().fetch("origin"))
552-
.and_then(|_| async {
553-
if let Some(reference) = fetch_ref {
554-
git.clone().fetch_ref("origin", reference).await
555-
} else {
556-
Ok(())
557-
}
558-
})
559-
.await
560-
.map_err(move |cause| {
561-
if url3.contains("git@") {
562-
warnln!("Please ensure your public ssh key is added to the git server.");
563-
}
564-
warnln!("Please ensure the url is correct and you have access to the repository.");
565-
Error::chain(
566-
format!("Failed to initialize git database in {:?}.", db_dir),
567-
cause,
568-
)
569-
})
570-
.map(move |_| git)
542+
stageln!("Cloning", "{} ({})", name2, url2);
543+
git.clone()
544+
.spawn_with(|c| c.arg("init").arg("--bare"))
545+
.await?;
546+
git.clone()
547+
.spawn_with(|c| c.arg("remote").arg("add").arg("origin").arg(url))
548+
.await?;
549+
git.clone()
550+
.fetch("origin")
551+
.and_then(|_| async {
552+
if let Some(reference) = fetch_ref {
553+
git.clone().fetch_ref("origin", reference).await
554+
} else {
555+
Ok(())
556+
}
557+
})
558+
.await
559+
.map_err(move |cause| {
560+
if url3.contains("git@") {
561+
warnln!("Please ensure your public ssh key is added to the git server.");
562+
}
563+
warnln!(
564+
"Please ensure the url is correct and you have access to the repository."
565+
);
566+
Error::chain(
567+
format!("Failed to initialize git database in {:?}.", db_dir),
568+
cause,
569+
)
570+
})
571+
.map(move |_| git)
571572
} else {
572573
// Update if the manifest has been modified since the last fetch.
573574
let db_mtime = try_modification_time(db_dir.join("FETCH_HEAD"));
@@ -577,30 +578,30 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> {
577578
}
578579
self.sess.stats.num_database_fetch.increment();
579580
// TODO MICHAERO: May need throttle
580-
future::lazy(|_| {
581-
stageln!("Fetching", "{} ({})", name2, url2);
582-
Ok(())
583-
})
584-
.and_then(|_| git.clone().fetch("origin"))
585-
.and_then(|_| async {
586-
if let Some(reference) = fetch_ref {
587-
git.clone().fetch_ref("origin", reference).await
588-
} else {
589-
Ok(())
590-
}
591-
})
592-
.await
593-
.map_err(move |cause| {
594-
if url3.contains("git@") {
595-
warnln!("Please ensure your public ssh key is added to the git server.");
596-
}
597-
warnln!("Please ensure the url is correct and you have access to the repository.");
598-
Error::chain(
599-
format!("Failed to update git database in {:?}.", db_dir),
600-
cause,
601-
)
602-
})
603-
.map(move |_| git)
581+
stageln!("Fetching", "{} ({})", name2, url2);
582+
git.clone()
583+
.fetch("origin")
584+
.and_then(|_| async {
585+
if let Some(reference) = fetch_ref {
586+
git.clone().fetch_ref("origin", reference).await
587+
} else {
588+
Ok(())
589+
}
590+
})
591+
.await
592+
.map_err(move |cause| {
593+
if url3.contains("git@") {
594+
warnln!("Please ensure your public ssh key is added to the git server.");
595+
}
596+
warnln!(
597+
"Please ensure the url is correct and you have access to the repository."
598+
);
599+
Error::chain(
600+
format!("Failed to update git database in {:?}.", db_dir),
601+
cause,
602+
)
603+
})
604+
.map(move |_| git)
604605
}
605606
}
606607

@@ -1612,7 +1613,7 @@ pub struct DependencyEntry {
16121613

16131614
impl DependencyEntry {
16141615
/// Obtain the dependency version for this entry.
1615-
pub fn version(&self) -> DependencyVersion {
1616+
pub fn version(&self) -> DependencyVersion<'_> {
16161617
match self.source {
16171618
DependencySource::Registry => unimplemented!(),
16181619
DependencySource::Path(_) => DependencyVersion::Path,

0 commit comments

Comments
 (0)