Implement loading shared libraries for proc macro plugins (#1093)

Resolves #1128

commit-id:a3155bbf

---

**Stack**:
- #1159
- #1157
- #1143
- #1151
- #1150
- #1148
- #1100
- #1155
- #1110
- #1093 ⬅


⚠️ *Part of a stack created by [spr](https://github.com/ejoffe/spr). Do
not merge manually using the UI - doing so may have unexpected results.*

Author: maciektr

Cargo.lock

@@ -79,6 +79,7 @@ indoc = "2"
 io_tee = "0.1"
 itertools = "0.12"
 libc = "0.2"
+libloading = "0.8.1"
 log = "0.4"
 ntest = "0.9"
 num-bigint = { version = "0.4", features = ["rand"] }

Cargo.toml

@@ -46,6 +46,7 @@ ignore.workspace = true
 include_dir.workspace = true
 indoc.workspace = true
 itertools.workspace = true
+libloading.workspace = true
 once_cell.workspace = true
 pathdiff.workspace = true
 petgraph.workspace = true

scarb/Cargo.toml

@@ -43,7 +43,7 @@ fn load_plugins(
             let instance = plugin.instantiate()?;
             builder.with_plugin_suite(instance.plugin_suite());
         } else {
-            proc_macros.register(plugin_info.package.clone())?;
+            proc_macros.register(plugin_info.package.clone(), ws.config())?;
         }
     }
     builder.with_plugin_suite(proc_macros.into_plugin_suite());

scarb/src/compiler/db.rs

@@ -0,0 +1,40 @@
+use crate::compiler::plugin::proc_macro::PROC_MACRO_BUILD_PROFILE;
+use crate::core::{Config, Package};
+use crate::flock::Filesystem;
+use camino::Utf8PathBuf;
+use libloading::library_filename;
+
+/// This trait is used to define the shared library path for a package.
+pub trait SharedLibraryProvider {
+    /// Location of Cargo `target` directory.
+    fn target_path(&self, config: &Config) -> Filesystem;
+    /// Location of the shared library for the package.
+    fn shared_lib_path(&self, config: &Config) -> Utf8PathBuf;
+}
+
+impl SharedLibraryProvider for Package {
+    fn target_path(&self, config: &Config) -> Filesystem {
+        let ident = format!("{}-{}", self.id.name, self.id.source_id.ident());
+        // Defines the Cargo target directory in cache, as:
+        // `/(..)/SCARB_CACHE/plugins/proc_macro/<package_name>-<source_id_ident>/v<version>/target/`
+        config
+            .dirs()
+            .procedural_macros_dir()
+            .into_child(ident)
+            .into_child(format!("v{}", self.id.version))
+            .into_child("target")
+    }
+
+    fn shared_lib_path(&self, config: &Config) -> Utf8PathBuf {
+        let lib_name = library_filename(self.id.name.to_string());
+        let lib_name = lib_name
+            .into_string()
+            .expect("library name must be valid UTF-8");
+        // Defines the shared library path inside the target directory, as:
+        // `/(..)/target/release/[lib]<package_name>.[so|dll|dylib]`
+        self.target_path(config)
+            .into_child(PROC_MACRO_BUILD_PROFILE)
+            .path_unchecked()
+            .join(lib_name)
+    }
+}

scarb/src/compiler/plugin/proc_macro/compilation.rs

@@ -1,11 +1,22 @@
-use crate::core::{Package, PackageId};
-use anyhow::Result;
+use crate::core::{Config, Package, PackageId};
+use anyhow::{Context, Result};
 use cairo_lang_defs::patcher::PatchBuilder;
+use cairo_lang_macro::stable_abi::{StableProcMacroResult, StableTokenStream};
 use cairo_lang_macro::{ProcMacroResult, TokenStream};
 use cairo_lang_syntax::node::db::SyntaxGroup;
 use cairo_lang_syntax::node::{ast, TypedSyntaxNode};
+use camino::Utf8PathBuf;
+use libloading::{Library, Symbol};
 use std::fmt::Debug;
 
+use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider;
+#[cfg(not(windows))]
+use libloading::os::unix::Symbol as RawSymbol;
+#[cfg(windows)]
+use libloading::os::windows::Symbol as RawSymbol;
+
+pub const PROC_MACRO_BUILD_PROFILE: &str = "release";
+
 pub trait FromItemAst {
     fn from_item_ast(db: &dyn SyntaxGroup, item_ast: ast::ModuleItem) -> Self;
 }
@@ -22,31 +33,72 @@ impl FromItemAst for TokenStream {
 ///
 /// This struct is a wrapper around a shared library containing the procedural macro implementation.
 /// It is responsible for loading the shared library and providing a safe interface for code expansion.
-#[derive(Debug, Clone)]
 pub struct ProcMacroInstance {
     package_id: PackageId,
+    plugin: Plugin,
+}
+
+impl Debug for ProcMacroInstance {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("ProcMacroInstance")
+            .field("package_id", &self.package_id)
+            .finish()
+    }
 }
 
 impl ProcMacroInstance {
     pub fn package_id(&self) -> PackageId {
         self.package_id
     }
 
-    pub fn try_new(package: Package) -> Result<Self> {
-        // Load shared library
-        // TODO(maciektr): Implement
+    /// Load shared library
+    pub fn try_new(package: Package, config: &Config) -> Result<Self> {
+        let lib_path = package.shared_lib_path(config);
+        let plugin = unsafe { Plugin::try_new(lib_path.to_path_buf())? };
         Ok(Self {
+            plugin,
             package_id: package.id,
         })
     }
-
     pub fn declared_attributes(&self) -> Vec<String> {
         vec![self.package_id.name.to_string()]
     }
 
-    pub(crate) fn generate_code(&self, _token_stream: TokenStream) -> ProcMacroResult {
-        // Apply expansion to token stream.
-        // TODO(maciektr): Implement
-        ProcMacroResult::Leave
+    /// Apply expansion to token stream.
+    pub(crate) fn generate_code(&self, token_stream: TokenStream) -> ProcMacroResult {
+        let ffi_token_stream = unsafe { StableTokenStream::from_token_stream(token_stream) };
+        let result = (self.plugin.vtable.expand)(ffi_token_stream);
+        unsafe { result.into_proc_macro_result() }
+    }
+}
+
+type ExpandCode = extern "C" fn(StableTokenStream) -> StableProcMacroResult;
+
+struct VTableV0 {
+    expand: RawSymbol<ExpandCode>,
+}
+
+impl VTableV0 {
+    unsafe fn try_new(library: &Library) -> Result<VTableV0> {
+        let expand: Symbol<'_, ExpandCode> = library
+            .get(b"expand\0")
+            .context("failed to load expand function for procedural macro")?;
+        let expand = expand.into_raw();
+        Ok(VTableV0 { expand })
+    }
+}
+
+struct Plugin {
+    #[allow(dead_code)]
+    library: Library,
+    vtable: VTableV0,
+}
+
+impl Plugin {
+    unsafe fn try_new(library_path: Utf8PathBuf) -> Result<Plugin> {
+        let library = Library::new(library_path)?;
+        let vtable = VTableV0::try_new(&library)?;
+
+        Ok(Plugin { library, vtable })
     }
 }

scarb/src/compiler/plugin/proc_macro/ffi.rs

@@ -1,5 +1,5 @@
 use crate::compiler::plugin::proc_macro::{FromItemAst, ProcMacroInstance};
-use crate::core::{Package, PackageId};
+use crate::core::{Config, Package, PackageId};
 use anyhow::Result;
 use cairo_lang_defs::plugin::{
     MacroPlugin, MacroPluginMetadata, PluginGeneratedFile, PluginResult,
@@ -178,10 +178,8 @@ pub struct ProcMacroHost {
 }
 
 impl ProcMacroHost {
-    pub fn register(&mut self, package: Package) -> Result<()> {
-        // Create instance
-        // Register instance in hash map
-        let instance = ProcMacroInstance::try_new(package)?;
+    pub fn register(&mut self, package: Package, config: &Config) -> Result<()> {
+        let instance = ProcMacroInstance::try_new(package, config)?;
         self.macros.push(Arc::new(instance));
         Ok(())
     }

scarb/src/compiler/plugin/proc_macro/host.rs

@@ -1,3 +1,4 @@
+pub mod compilation;
 mod ffi;
 mod host;
 

scarb/src/compiler/plugin/proc_macro/mod.rs

@@ -70,6 +70,10 @@ impl AppDirs {
     pub fn registry_dir(&self) -> Filesystem {
         self.cache_dir.child("registry")
     }
+
+    pub fn procedural_macros_dir(&self) -> Filesystem {
+        self.cache_dir.child("plugins").child("proc_macro")
+    }
 }
 
 impl fmt::Display for AppDirs {