Support in-memory Syncables imports in browser WASM

Author: Michiel de Jong

.github/workflows/rust.yml

@@ -12,13 +12,12 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
+          targets: wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
       - run: cargo fmt --all --check
       - run: cargo clippy --all-targets --all-features -- -D warnings
       - run: cargo test --all-features
       - run: cargo build --release
-      - name: Install wasm32-unknown-unknown target
-        run: rustup target add wasm32-unknown-unknown
       - name: Build for wasm32 (library only — tests use dev-dependencies that aren't wasm-compatible)
         run: cargo build --target wasm32-unknown-unknown --lib
       - run: cargo clippy --target wasm32-unknown-unknown --lib -- -D warnings

Cargo.toml

@@ -20,7 +20,6 @@ serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["preserve_order"] }
 serde_yaml_ng = "0.10"
 thiserror = "2"
-uuid = { version = "1", features = ["v4", "js"] }
 
 # `tokio::fs` (used only by `openapi::load::load_yaml_file`) has no wasm32
 # support at all — see that module's wasm32 stub. Every other tokio use in
@@ -30,6 +29,7 @@ uuid = { version = "1", features = ["v4", "js"] }
 tokio = { version = "1", features = ["fs"] }
 
 [dev-dependencies]
+uuid = { version = "1", features = ["v4"] }
 tokio = { version = "1", features = ["full"] }
 
 [lints.rust]

README.md

@@ -95,9 +95,8 @@ networking:
   [`OpenApiSource::Value`] instead (the mock server and sync engine's own
   document loading only ever need a path when a *native* host chooses to
   read one off disk).
-- `uuid`'s `v4` feature needs a source of randomness, and the `js`
-  feature (backed by `getrandom`'s wasm-bindgen support) supplies one on
-  wasm32 without affecting other targets.
+- `uuid` is only needed by native test fixtures and is a dev-dependency;
+  browser library builds do not pull in its randomness backend.
 
 A wasm32 host implementing [`Fetch`](client::client::Fetch) or
 [`Storage`](sync::storage::Storage) — e.g. wrapping a JS `Promise` or a
@@ -233,3 +232,14 @@ As an NLnet-funded project, this follows
 ## License
 
 Apache-2.0, matching the original.
+
+## Browser / WASM
+
+`cargo check --target wasm32-unknown-unknown --lib` builds the engine without
+Tokio filesystem or native runtime dependencies. Load catalog text with
+`openapi::load::parse_yaml`, pass the resulting value to
+`load_open_api_document`, then call `SyncClient::sync_document(&doc, &storage)`.
+This path never opens `ClientConfig.document` or `overlays`; apply overlays in
+memory before calling it. File sources return an explicit unsupported error
+in WASM. Implement browser `Fetch` with `#[async_trait(?Send)]`; it may hold
+a JS callback. Native `Fetch` keeps its Send + Sync contract.

src/client/client.rs

@@ -80,7 +80,7 @@ pub struct HttpResponse {
 /// dependency in `reflector-rs`) uses for its own `Storelike` trait.
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
-pub trait Fetch: Send + Sync {
+pub trait Fetch: FetchBounds {
     /// Sends one request and returns the response.
     async fn fetch(&self, request: HttpRequest) -> Result<HttpResponse>;
 }
@@ -346,6 +346,9 @@ impl ApiClient {
 }
 
 /// Builds a client for `document` against the server in `options`.
+// Browser options may own a JS callback. Keep the shared Arc-shaped API on
+// both targets; WASM hosts never send it across threads.
+#[cfg_attr(target_arch = "wasm32", allow(clippy::arc_with_non_send_sync))]
 pub fn create_api_client(document: OpenApiDocument, options: ApiClientOptions) -> ApiClient {
     let routes = discover_resources(&document.paths);
     let storage = options
@@ -361,3 +364,14 @@ pub fn create_api_client(document: OpenApiDocument, options: ApiClientOptions) -
         options: Arc::new(options),
     }
 }
+
+/// Native transports must be thread safe; browser transports run on one JS thread.
+#[cfg(not(target_arch = "wasm32"))]
+pub trait FetchBounds: Send + Sync {}
+#[cfg(not(target_arch = "wasm32"))]
+impl<T: Send + Sync> FetchBounds for T {}
+/// Browser transports may own JavaScript callbacks.
+#[cfg(target_arch = "wasm32")]
+pub trait FetchBounds {}
+#[cfg(target_arch = "wasm32")]
+impl<T> FetchBounds for T {}

src/client/storage.rs

@@ -13,7 +13,8 @@ use crate::error::Result;
 ///
 /// Implement this to persist somewhere other than memory;
 /// [`InMemoryStorageAdapter`] is the default.
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait StorageAdapter: Send + Sync {
     /// Every record held for `resource`.
     async fn list(&self, resource: &str) -> Result<Vec<Map<String, Value>>>;
@@ -41,7 +42,8 @@ impl InMemoryStorageAdapter {
     }
 }
 
-#[async_trait]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl StorageAdapter for InMemoryStorageAdapter {
     async fn list(&self, resource: &str) -> Result<Vec<Map<String, Value>>> {
         let mut collections = self.collections.lock().expect("storage mutex poisoned");

src/sync/client.rs

@@ -165,15 +165,24 @@ impl SyncClient {
             &self.config.overlays,
         )
         .await?;
-        let model = discover_resource_model(&document)?;
-        validate_constants(&document, &model, &self.config.constants)?;
+        self.sync_document(&document, storage).await
+    }
+
+    /// Sync an already loaded, resolved document without filesystem access.
+    pub async fn sync_document(
+        &self,
+        document: &OpenApiDocument,
+        storage: &dyn Storage,
+    ) -> Result<SyncReport, SyncError> {
+        let model = discover_resource_model(document)?;
+        validate_constants(document, &model, &self.config.constants)?;
 
         // Fail before writing anything if there's nowhere to sync from —
         // no point minting an ontology for a sync that can't run at all.
-        let base = base_url(&document)
+        let base = base_url(document)
             .ok_or_else(|| SyncError::Document("document declares no servers".to_string()))?;
 
-        let ontology = derive_ontology(&document)?;
+        let ontology = derive_ontology(document)?;
         storage
             .put_ontology(&ontology)
             .await
@@ -183,7 +192,7 @@ impl SyncClient {
             ontology_terms: ontology.terms.len(),
             ..SyncReport::default()
         };
-        self.walk_all(&document, &model, base, storage, &mut report)
+        self.walk_all(document, &model, base, storage, &mut report)
             .await;
         Ok(report)
     }

tests/unit/sync/client.rs

@@ -570,3 +570,51 @@ async fn tracks_every_url_it_requested() {
         vec!["https://api.example.com/repos/acme/widgets/issues".to_string()]
     );
 }
+
+#[tokio::test]
+async fn syncs_in_memory_without_opening_the_configured_path() {
+    let document = syncables::load_open_api_document(flat_document(Some(
+        json!([{ "url": "https://api.example.com" }]),
+    )))
+    .await
+    .unwrap();
+    let fetch = MockFetch::default().respond_json(
+        "https://api.example.com/repos/acme/widgets/issues",
+        200,
+        &[],
+        json!([
+            { "number": 1, "title": "First issue" },
+            { "number": 2, "title": "Second issue" }
+        ]),
+    );
+    let client = SyncClient::new(
+        config(
+            Path::new("/this-file-must-never-be-opened"),
+            &[("owner", "acme"), ("repo", "widgets")],
+        ),
+        Arc::new(fetch),
+    )
+    .expect("valid config");
+
+    let storage = InMemoryStorage::new();
+    let report = client
+        .sync_document(&document, &storage)
+        .await
+        .expect("sync succeeds");
+
+    assert!(
+        report.errors.is_empty(),
+        "unexpected errors: {:?}",
+        report.errors
+    );
+    assert_eq!(report.read.get("issue"), Some(&2));
+    assert_eq!(report.ontology_terms, 1);
+    assert_eq!(storage.ontologies().len(), 1);
+
+    let first = storage
+        .get("acme/widgets", "issue", "1")
+        .await
+        .expect("get succeeds")
+        .expect("record present");
+    assert_eq!(first.value.get("title"), Some(&json!("First issue")));
+}