first version of Forest (#1)

Author: KennedyTedesco

Cargo.toml

@@ -14,11 +14,15 @@ keywords = ["ara", "php", "programming-language", "parser"]
 categories = ["compilers", "development-tools::build-utils"]
 
 [dependencies]
-ara_parser = { version = "0.6.0" }
+ara_parser = { version = "0.6.5" }
 ara_source = { version = "0.2.0" }
-ara_reporting = { version = "0.6.0" }
-serde = { version = "1.0.152", features = ["derive"] }
-num_cpus = { version = "1.15.0"}
+ara_reporting = { version = "0.6.1" }
+num_cpus = { version = "1.15.0" }
+rustc-hash = { version = "1.1.0" }
+walkdir = { version = "2.3.2" }
+bincode = { version = "2.0.0-rc.2" }
+log = { version = "0.4.17" }
+simplelog = { version = "0.12.0" }
 
 [profile.release]
 opt-level = 3

src/config.rs

@@ -1,42 +1,70 @@
-use std::hash::Hasher;
+use std::path::PathBuf;
 
+use crate::logger::Logger;
+
+#[derive(Debug)]
 pub struct Config {
-    pub root: String,
-    pub source: String,
-    pub definitions: Vec<String>,
-    pub cache: String,
+    pub root: PathBuf,
+    pub source: PathBuf,
+    pub definitions: Vec<PathBuf>,
+    pub cache: Option<PathBuf>,
     pub threads: usize,
-    pub hasher: Box<dyn Hasher>,
+    pub logger: Option<Logger>,
 }
 
 impl Config {
-    pub fn new<R, S, D, C>(
-        root: R,
-        source: S,
-        definitions: Vec<D>,
-        cache: Option<C>,
-        threads: Option<usize>,
-        hasher: Option<Box<dyn Hasher>>,
-    ) -> Self
-    where
-        R: Into<String>,
-        S: Into<String>,
-        D: Into<String>,
-        C: Into<String>,
-    {
-        let root = root.into();
-        let cache = cache
-            .map(|c| c.into())
-            .unwrap_or_else(|| format!("{}/.cache", root));
-
+    pub fn new<R: Into<String>>(root: R) -> Self {
         Self {
-            root,
-            source: source.into(),
-            definitions: definitions.into_iter().map(|d| d.into()).collect(),
-            cache,
-            threads: threads.unwrap_or_else(num_cpus::get),
-            hasher: hasher
-                .unwrap_or_else(|| Box::new(std::collections::hash_map::DefaultHasher::new())),
+            root: PathBuf::from(root.into()),
+            source: PathBuf::from(String::default()),
+            definitions: Vec::new(),
+            cache: None,
+            threads: num_cpus::get(),
+            logger: None,
         }
     }
+
+    #[must_use]
+    pub fn with_source<S: Into<String>>(mut self, source: S) -> Self {
+        self.source = PathBuf::from(source.into());
+
+        self
+    }
+
+    #[must_use]
+    pub fn with_definitions<D: Into<String>>(mut self, definitions: Vec<D>) -> Self {
+        self.definitions = definitions
+            .into_iter()
+            .map(|definition| PathBuf::from(definition.into()))
+            .collect();
+
+        self
+    }
+
+    #[must_use]
+    pub fn with_cache_directory<C: Into<String>>(mut self, cache_dir: C) -> Self {
+        let path = PathBuf::from(cache_dir.into());
+
+        if path.is_relative() {
+            self.cache = Some(self.root.join(path));
+        } else {
+            self.cache = Some(path);
+        }
+
+        self
+    }
+
+    #[must_use]
+    pub fn with_threads(mut self, threads: usize) -> Self {
+        self.threads = threads;
+
+        self
+    }
+
+    #[must_use]
+    pub fn with_logger(mut self, logger: Logger) -> Self {
+        self.logger = Some(logger);
+
+        self
+    }
 }

src/error.rs

@@ -0,0 +1,61 @@
+use ara_reporting::issue::Issue;
+use ara_reporting::Report;
+
+#[derive(Debug)]
+pub enum Error {
+    EncodeError(String),
+    DecodeError(String),
+    InvalidPath(String),
+    IoError(std::io::Error),
+    ParseError(Box<Report>),
+    LogError(log::SetLoggerError),
+}
+
+impl From<walkdir::Error> for Error {
+    fn from(error: walkdir::Error) -> Self {
+        Error::IoError(error.into())
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(error: std::io::Error) -> Self {
+        Error::IoError(error)
+    }
+}
+
+impl From<Error> for Report {
+    fn from(error: Error) -> Self {
+        Report::new().with_issue(Issue::from_string(error.to_string()))
+    }
+}
+
+impl From<bincode::error::EncodeError> for Error {
+    fn from(error: bincode::error::EncodeError) -> Self {
+        Error::EncodeError(error.to_string())
+    }
+}
+
+impl From<bincode::error::DecodeError> for Error {
+    fn from(error: bincode::error::DecodeError) -> Self {
+        Error::DecodeError(error.to_string())
+    }
+}
+
+impl From<log::SetLoggerError> for Error {
+    fn from(error: log::SetLoggerError) -> Self {
+        Error::LogError(error)
+    }
+}
+
+impl std::fmt::Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Error::IoError(error) => write!(f, "io error: {error}"),
+            Error::InvalidPath(message) => write!(f, "invalid source: {message}"),
+            Error::EncodeError(message) => write!(f, "encode error: {message}"),
+            Error::DecodeError(message) => write!(f, "decode error: {message}"),
+            Error::ParseError(report) => write!(f, "parse error: {report}"),
+            Error::LogError(error) => write!(f, "log error: {error}"),
+        }
+    }
+}

src/hash.rs

@@ -0,0 +1,21 @@
+use std::hash::Hasher;
+
+pub trait ContentHasher: Send + Sync {
+    fn hash(&self, content: &str) -> u64;
+}
+
+pub struct FxHasher;
+
+impl FxHasher {
+    pub fn new() -> Self {
+        Self
+    }
+}
+
+impl ContentHasher for FxHasher {
+    fn hash(&self, content: &str) -> u64 {
+        let mut hasher = rustc_hash::FxHasher::default();
+        hasher.write(content.as_bytes());
+        hasher.finish()
+    }
+}

src/lib.rs

@@ -1,74 +1,139 @@
-use ara_parser::parser;
+use std::fs;
+use std::path::PathBuf;
+use std::thread;
+
+use ara_parser::tree::Tree;
 use ara_parser::tree::TreeMap;
-use ara_source::loader;
+use ara_reporting::Report;
+use ara_source::source::Source;
 use ara_source::SourceMap;
 
 use crate::config::Config;
+use crate::error::Error;
+use crate::hash::FxHasher;
+use crate::serializer::BincodeSerializer;
+use crate::source::SourceFilesCollector;
+use crate::tree::TreeBuilder;
 
 pub mod config;
+pub mod error;
+pub(crate) mod hash;
+pub mod logger;
+pub(crate) mod serializer;
+pub mod source;
+pub(crate) mod tree;
+
+pub(crate) const ARA_SOURCE_EXTENSION: &str = "ara";
+pub(crate) const ARA_DEFINITION_EXTENSION: &str = "d.ara";
+pub(crate) const ARA_CACHED_SOURCE_EXTENSION: &str = "ara.cache";
 
+#[derive(Debug)]
 pub struct Forest {
     pub source: SourceMap,
     pub tree: TreeMap,
 }
 
-pub struct Parser {
-    pub config: Config,
+impl Forest {
+    pub fn new(source: SourceMap, tree: TreeMap) -> Self {
+        Self { source, tree }
+    }
+}
+
+pub struct Parser<'a> {
+    pub config: &'a Config,
+    tree_builder: TreeBuilder<'a>,
 }
 
-impl Parser {
-    pub fn new(config: Config) -> Self {
-        Self { config }
+impl<'a> Parser<'a> {
+    pub fn new(config: &'a Config) -> Self {
+        let tree_builder = TreeBuilder::new(
+            config,
+            Box::new(FxHasher::new()),
+            Box::new(BincodeSerializer::new()),
+        );
+
+        Self {
+            config,
+            tree_builder,
+        }
     }
 
-    pub fn parse(&self) -> Result<Forest, String> {
-        let mut threads = Vec::with_capacity(self.config.threads);
-
-        let source_map = loader::load_directories(&self.config.root, {
-            let mut directories = self.config.definitions.clone();
-            directories.push(self.config.source.clone());
-
-            directories
-        })
-        .expect("Failed to load source map");
-
-        // split the sources into N chunks, where N is the number of threads
-        let chunk_size = source_map.sources.len() / self.config.threads;
-        let chunks: Vec<Vec<ara_source::source::Source>> = source_map
-            .sources
-            .chunks(chunk_size)
-            .map(|chunk| chunk.to_vec())
-            .collect();
-
-        for chunk in chunks {
-            threads.push(std::thread::spawn(move || {
-                let map = SourceMap::new(chunk);
-                parser::parse_map(&map)
-            }));
+    pub fn parse(&self) -> Result<Forest, Box<Report>> {
+        self.init_logger().map_err(|error| Box::new(error.into()))?;
+
+        let (sources, trees) =
+            thread::scope(|scope| -> Result<(Vec<Source>, Vec<Tree>), Box<Report>> {
+                self.create_cache_dir()
+                    .map_err(|error| Box::new(error.into()))?;
+
+                let files = SourceFilesCollector::new(self.config)
+                    .collect()
+                    .map_err(|error| Box::new(error.into()))?;
+
+                if files.is_empty() {
+                    return Ok((Vec::new(), Vec::new()));
+                }
+
+                let threads_count = self.threads_count(files.len());
+                let chunks = files
+                    .chunks(files.len() / threads_count)
+                    .map(Vec::from)
+                    .collect::<Vec<Vec<PathBuf>>>();
+
+                let mut threads = Vec::with_capacity(threads_count);
+                for chunk in chunks.into_iter() {
+                    threads.push(scope.spawn(
+                        move || -> Result<Vec<(Source, Tree)>, Box<Report>> {
+                            let mut source_tree = Vec::with_capacity(chunk.len());
+                            for source_path in chunk {
+                                let (source, tree) = self
+                                    .tree_builder
+                                    .build(&source_path)
+                                    .map_err(|error| match error {
+                                        Error::ParseError(report) => report,
+                                        _ => Box::new(error.into()),
+                                    })?;
+                                source_tree.push((source, tree));
+                            }
+
+                            Ok(source_tree)
+                        },
+                    ));
+                }
+
+                let mut result = Vec::new();
+                for handle in threads {
+                    result.extend(handle.join().unwrap()?);
+                }
+                let (sources, trees) = result.into_iter().unzip();
+
+                Ok((sources, trees))
+            })?;
+
+        Ok(Forest::new(SourceMap::new(sources), TreeMap::new(trees)))
+    }
+
+    fn threads_count(&self, files_len: usize) -> usize {
+        if self.config.threads > files_len {
+            files_len
+        } else {
+            self.config.threads
         }
+    }
+
+    fn create_cache_dir(&self) -> Result<(), Error> {
+        if self.config.cache.is_some() {
+            fs::create_dir_all(self.config.cache.as_ref().unwrap())?;
+        }
+
+        Ok(())
+    }
 
-        let mut results = vec![];
-        for thread in threads {
-            results.push(thread.join().unwrap());
+    fn init_logger(&self) -> Result<(), Error> {
+        if self.config.logger.is_some() {
+            self.config.logger.as_ref().unwrap().init()?
         }
 
-        todo!("
-            the implementation above is just a placeholder
-
-            the idea is to:
-              1. load the source map
-              2. split the source map into N chunks, where N is the number of threads
-              3. spawn N threads, each of which parses a chunk of the source map
-              4. in each thread, iterate over the sources in the chunk and:
-                first we need to check if the source is present in the cache, if yes, load the cached tree,
-                and check if the hash of the source matches the hash of the cached tree, if yes, return the cached tree,
-                otherwise, parse the source and save the tree to the cache
-                if the source is not present in the cache, parse the source and save the tree to the cache.
-                If the parser failed, return the report immediately and do not continue
-              5. join the threads and collect the results
-                If any of the threads failed, return the report immediately and do not continue
-              6. merge the results into a single forest
-              7. return the forest
-        ");
+        Ok(())
     }
 }