Support in-memory Syncables imports in browser WASM

Author: Michiel de Jong

.github/workflows/rust.yml

@@ -12,8 +12,10 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
+          targets: wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
       - run: cargo fmt --all --check
       - run: cargo clippy --all-targets --all-features -- -D warnings
       - run: cargo test --all-features
       - run: cargo build --release
+      - run: cargo check --lib --target wasm32-unknown-unknown

Cargo.toml

@@ -20,10 +20,12 @@ serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["preserve_order"] }
 serde_yaml_ng = "0.10"
 thiserror = "2"
-tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros", "net", "sync", "time"] }
-uuid = { version = "1", features = ["v4"] }
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tokio = { version = "1", features = ["fs"] }
 
 [dev-dependencies]
+uuid = { version = "1", features = ["v4"] }
 tokio = { version = "1", features = ["full"] }
 
 [lints.rust]

README.md

@@ -196,3 +196,14 @@ As an NLnet-funded project, this follows
 ## License
 
 Apache-2.0, matching the original.
+
+## Browser / WASM
+
+`cargo check --target wasm32-unknown-unknown --lib` builds the engine without
+Tokio filesystem or native runtime dependencies. Load catalog text with
+`openapi::load::parse_yaml`, pass the resulting value to
+`load_open_api_document`, then call `SyncClient::sync_document(&doc, &storage)`.
+This path never opens `ClientConfig.document` or `overlays`; apply overlays in
+memory before calling it. File sources return an explicit unsupported error
+in WASM. Implement browser `Fetch` with `#[async_trait(?Send)]`; it may hold
+a JS callback. Native `Fetch` keeps its Send + Sync contract.

src/client/client.rs

@@ -73,8 +73,9 @@ pub struct HttpResponse {
 /// The client's only extension point for how requests reach the network.
 ///
 /// This is the Rust equivalent of the original's injectable `fetch`.
-#[async_trait]
-pub trait Fetch: Send + Sync {
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+pub trait Fetch: FetchBounds {
     /// Sends one request and returns the response.
     async fn fetch(&self, request: HttpRequest) -> Result<HttpResponse>;
 }
@@ -355,3 +356,14 @@ pub fn create_api_client(document: OpenApiDocument, options: ApiClientOptions) -
         options: Arc::new(options),
     }
 }
+
+/// Native transports must be thread safe; browser transports run on one JS thread.
+#[cfg(not(target_arch = "wasm32"))]
+pub trait FetchBounds: Send + Sync {}
+#[cfg(not(target_arch = "wasm32"))]
+impl<T: Send + Sync> FetchBounds for T {}
+/// Browser transports may own JavaScript callbacks.
+#[cfg(target_arch = "wasm32")]
+pub trait FetchBounds {}
+#[cfg(target_arch = "wasm32")]
+impl<T> FetchBounds for T {}

src/client/storage.rs

@@ -13,7 +13,8 @@ use crate::error::Result;
 ///
 /// Implement this to persist somewhere other than memory;
 /// [`InMemoryStorageAdapter`] is the default.
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait StorageAdapter: Send + Sync {
     /// Every record held for `resource`.
     async fn list(&self, resource: &str) -> Result<Vec<Map<String, Value>>>;
@@ -41,7 +42,8 @@ impl InMemoryStorageAdapter {
     }
 }
 
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl StorageAdapter for InMemoryStorageAdapter {
     async fn list(&self, resource: &str) -> Result<Vec<Map<String, Value>>> {
         let mut collections = self.collections.lock().expect("storage mutex poisoned");

src/openapi/load.rs

@@ -63,6 +63,7 @@ pub fn parse_yaml(text: &str) -> Result<Value> {
 /// [`Error::FileLoad`] so it names the offending path — a bare
 /// [`std::io::Error`] from a failed read doesn't otherwise mention which
 /// file was missing or unreadable.
+#[cfg(not(target_arch = "wasm32"))]
 pub(crate) async fn load_yaml_file(path: &Path) -> Result<Value> {
     async {
         let text = tokio::fs::read_to_string(path).await?;
@@ -74,3 +75,17 @@ pub(crate) async fn load_yaml_file(path: &Path) -> Result<Value> {
         source: Box::new(source),
     })
 }
+
+#[cfg(target_arch = "wasm32")]
+pub(crate) async fn load_yaml_file(path: &Path) -> Result<Value> {
+    Err(Error::FileLoad {
+        path: path.to_path_buf(),
+        source: Box::new(
+            std::io::Error::new(
+                std::io::ErrorKind::Unsupported,
+                "Use an in-memory OpenAPI document in the browser",
+            )
+            .into(),
+        ),
+    })
+}

src/sync/client.rs

@@ -165,15 +165,24 @@ impl SyncClient {
             &self.config.overlays,
         )
         .await?;
-        let model = discover_resource_model(&document)?;
-        validate_constants(&document, &model, &self.config.constants)?;
+        self.sync_document(&document, storage).await
+    }
+
+    /// Sync an already loaded, resolved document without filesystem access.
+    pub async fn sync_document(
+        &self,
+        document: &OpenApiDocument,
+        storage: &dyn Storage,
+    ) -> Result<SyncReport, SyncError> {
+        let model = discover_resource_model(document)?;
+        validate_constants(document, &model, &self.config.constants)?;
 
         // Fail before writing anything if there's nowhere to sync from —
         // no point minting an ontology for a sync that can't run at all.
-        let base = base_url(&document)
+        let base = base_url(document)
             .ok_or_else(|| SyncError::Document("document declares no servers".to_string()))?;
 
-        let ontology = derive_ontology(&document)?;
+        let ontology = derive_ontology(document)?;
         storage
             .put_ontology(&ontology)
             .await
@@ -183,7 +192,7 @@ impl SyncClient {
             ontology_terms: ontology.terms.len(),
             ..SyncReport::default()
         };
-        self.walk_all(&document, &model, base, storage, &mut report)
+        self.walk_all(document, &model, base, storage, &mut report)
             .await;
         Ok(report)
     }

src/sync/storage.rs

@@ -102,7 +102,8 @@ impl StorageError {
 /// it. That ordering is the engine's responsibility
 /// ([issue #9](https://github.com/localthought/syncables-rs/issues/9));
 /// `reflector-rs`'s `AtomicStorage` is written against it.
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait Storage: Send + Sync {
     /// Inserts or replaces a record, keyed by its `namespace`, `resource`
     /// and `id`.
@@ -159,7 +160,8 @@ impl InMemoryStorage {
     }
 }
 
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl Storage for InMemoryStorage {
     async fn put(&self, record: &Record) -> Result<(), StorageError> {
         let key = (

tests/unit/sync/client.rs

@@ -570,3 +570,51 @@ async fn tracks_every_url_it_requested() {
         vec!["https://api.example.com/repos/acme/widgets/issues".to_string()]
     );
 }
+
+#[tokio::test]
+async fn syncs_in_memory_without_opening_the_configured_path() {
+    let document = syncables::load_open_api_document(flat_document(Some(
+        json!([{ "url": "https://api.example.com" }]),
+    )))
+    .await
+    .unwrap();
+    let fetch = MockFetch::default().respond_json(
+        "https://api.example.com/repos/acme/widgets/issues",
+        200,
+        &[],
+        json!([
+            { "number": 1, "title": "First issue" },
+            { "number": 2, "title": "Second issue" }
+        ]),
+    );
+    let client = SyncClient::new(
+        config(
+            Path::new("/this-file-must-never-be-opened"),
+            &[("owner", "acme"), ("repo", "widgets")],
+        ),
+        Arc::new(fetch),
+    )
+    .expect("valid config");
+
+    let storage = InMemoryStorage::new();
+    let report = client
+        .sync_document(&document, &storage)
+        .await
+        .expect("sync succeeds");
+
+    assert!(
+        report.errors.is_empty(),
+        "unexpected errors: {:?}",
+        report.errors
+    );
+    assert_eq!(report.read.get("issue"), Some(&2));
+    assert_eq!(report.ontology_terms, 1);
+    assert_eq!(storage.ontologies().len(), 1);
+
+    let first = storage
+        .get("acme/widgets", "issue", "1")
+        .await
+        .expect("get succeeds")
+        .expect("record present");
+    assert_eq!(first.value.get("title"), Some(&json!("First issue")));
+}