Implement support for input method editors and screen readers for web runner

Author: white-axe

crates/launcher/Cargo.toml

@@ -114,13 +114,15 @@ web-sys = { workspace = true, features = [
     "BeforeUnloadEvent",
     "console",
     "ClipboardEvent",
+    "CompositionEvent",
     "CssStyleDeclaration",
     "DedicatedWorkerGlobalScope",
     "Document",
     "Event",
     "HtmlElement",
     "HtmlCanvasElement",
     "HtmlImageElement",
+    "InputEvent",
     "KeyboardEvent",
     "MediaQueryList",
     "MediaQueryListEvent",
@@ -130,6 +132,8 @@ web-sys = { workspace = true, features = [
     "MutationRecord",
     "OffscreenCanvas",
     "Performance",
+    "SpeechSynthesis",
+    "SpeechSynthesisUtterance",
     "Touch",
     "TouchEvent",
     "WheelEvent",

crates/launcher/src/entrypoint/web/runner/events.rs

@@ -45,9 +45,8 @@ where
     E: AsRef<web_sys::Event> + wasm_bindgen::JsCast,
 {
     let closure = wasm_bindgen::closure::Closure::new(move |event: web_sys::Event| {
-        let mut state = state_cell.borrow_mut();
         if !super::has_panicked() {
-            listener(&mut state, event.unchecked_into());
+            listener(&mut state_cell.borrow_mut(), event.unchecked_into());
         }
     });
 
@@ -142,6 +141,7 @@ pub(super) fn register_events(
     output_rx: flume::Receiver<super::Output>,
     panic_rx: oneshot::Receiver<()>,
     window: web_sys::Window,
+    document: web_sys::Document,
 ) -> Result<(), wasm_bindgen::JsValue> {
     // Set up panic handler to unsubscribe all event handlers on panic
     wasm_bindgen_futures::spawn_local(async move {
@@ -156,10 +156,11 @@ pub(super) fn register_events(
         });
     });
 
-    let document = window.document().unwrap();
-
     let state_cell = std::rc::Rc::new(std::cell::RefCell::new(state));
-    let canvas = state_cell.borrow().canvas.clone();
+    let state = state_cell.borrow();
+    let canvas = state.canvas.clone();
+    let input = state.input.clone();
+    drop(state);
 
     // Register event listener for screen resizing
     let listener = {
@@ -451,14 +452,13 @@ pub(super) fn register_events(
     // Register event listener for keyboard key presses and releases
     let listener_builder = |pressed| {
         move |state: &mut super::MainState, event: web_sys::KeyboardEvent| {
-            // TODO: handle input method editors
             let modifiers = event.egui_modifiers();
             state
                 .event_tx
                 .send(super::Event::Modifiers(modifiers))
                 .unwrap();
             let key = event.key();
-            if pressed && !modifiers.command && key.len() == 1 {
+            if pressed && !modifiers.command && state.ime.is_none() && key.len() == 1 {
                 state
                     .event_tx
                     .send(super::Event::Egui(egui::Event::Text(key.clone())))
@@ -516,6 +516,68 @@ pub(super) fn register_events(
         listener_builder(false),
     )?;
 
+    // Register event listener for inputting text while the egui app wants text input
+    let listener = move |state: &mut super::MainState, event: web_sys::InputEvent| {
+        if state.ime.is_none() || event.is_composing() {
+            return;
+        }
+        let text = state.input.value();
+        if text.is_empty() {
+            return;
+        }
+        state
+            .event_tx
+            .send(super::Event::Egui(egui::Event::Text(text)))
+            .unwrap();
+        state.input.set_value("");
+    };
+    add_event_listener(state_cell.clone(), &input, "input", listener)?;
+
+    // Register event listener for starting to compose text an input method editor while the egui
+    // app wants text input
+    let listener = move |state: &mut super::MainState, _event: web_sys::CompositionEvent| {
+        state
+            .event_tx
+            .send(super::Event::Egui(egui::Event::Ime(
+                egui::ImeEvent::Enabled,
+            )))
+            .unwrap();
+        state.input.set_value("");
+    };
+    add_event_listener(state_cell.clone(), &input, "compositionstart", listener)?;
+
+    // Register event listener for submitting composed text from an input method editor while the
+    // egui app wants text input
+    let listener = move |state: &mut super::MainState, event: web_sys::CompositionEvent| {
+        let Some(text) = event.data() else {
+            return;
+        };
+        state
+            .event_tx
+            .send(super::Event::Egui(egui::Event::Ime(
+                egui::ImeEvent::Commit(text),
+            )))
+            .unwrap();
+        state.input.set_value("");
+    };
+    add_event_listener(state_cell.clone(), &input, "compositionend", listener)?;
+
+    // Register event listener for when the preview of the text in an input method editor updates
+    // while the egui wapp wants text input
+    let listener = move |state: &mut super::MainState, event: web_sys::CompositionEvent| {
+        let Some(text) = event.data() else {
+            return;
+        };
+        state
+            .event_tx
+            .send(super::Event::Egui(egui::Event::Ime(
+                egui::ImeEvent::Preedit(text),
+            )))
+            .unwrap();
+        state.input.set_value("");
+    };
+    add_event_listener(state_cell.clone(), &input, "compositionupdate", listener)?;
+
     // Register event listener for text pasting
     let listener = move |state: &mut super::MainState, event: web_sys::ClipboardEvent| {
         if let Some(data) = event.clipboard_data() {
@@ -608,6 +670,7 @@ pub(super) fn register_events(
                     zoom_factor,
                     screen_reader_enabled,
                 } => {
+                    // Update the cursor icon to the one specified by the app for this frame
                     let _ = body_style.set_property(
                         "cursor",
                         match output.cursor_icon {
@@ -654,8 +717,47 @@ pub(super) fn register_events(
                         },
                     );
 
-                    // TODO: handle input method editors and screen readers here
-                    let _ = screen_reader_enabled;
+                    // Speak the text description output by the app, if applicable
+                    if screen_reader_enabled {
+                        let text = output.events_description();
+                        if !text.is_empty() {
+                            if let (Ok(speech_synthesis), Ok(utterance)) = (
+                                window.speech_synthesis(),
+                                web_sys::SpeechSynthesisUtterance::new_with_text(&text),
+                            ) {
+                                speech_synthesis.cancel();
+                                speech_synthesis.speak(&utterance);
+                            }
+                        }
+                    }
+
+                    {
+                        let mut state = state_cell.borrow_mut();
+
+                        // Focus the input handler if the app wants keyboard input, otherwise focus
+                        // the canvas
+                        if let Some(ime) = output.ime {
+                            if state.ime != Some(ime) {
+                                // Move the input handler to the position the app wants input method
+                                // editors to show up at
+                                let pos = ime.cursor_rect.center();
+                                let style = state.input.style();
+                                let _ = style.set_property("left", &format!("{}px", pos.x));
+                                let _ = style.set_property("top", &format!("{}px", pos.y));
+                                // Workaround for the input method editor sometimes opening in the
+                                // location the input handler was previously located at instead of
+                                // the new location we just moved it to when using Firefox
+                                let _ = state.input.focus();
+                                let _ = state.input.blur();
+                            }
+                            let _ = state.input.focus();
+                        } else {
+                            let _ = state.canvas.focus();
+                        }
+                        state.ime = output.ime;
+
+                        state.zoom_factor = zoom_factor;
+                    }
 
                     if !output.copied_text.is_empty() {
                         if let Err(e) = wasm_bindgen_futures::JsFuture::from(
@@ -684,8 +786,6 @@ pub(super) fn register_events(
                             );
                         }
                     }
-
-                    state_cell.borrow_mut().zoom_factor = zoom_factor;
                 }
 
                 super::Output::StorageGet { key, oneshot_tx } => {

crates/launcher/src/entrypoint/web/runner/mod.rs

@@ -22,6 +22,8 @@
 // terms of the Steamworks API by Valve Corporation, the licensors of this
 // Program grant you additional permission to convey the resulting work.
 
+use wasm_bindgen::JsCast;
+
 mod events;
 mod worker;
 
@@ -148,6 +150,10 @@ struct MainState {
     event_tx: flume::Sender<Event>,
     /// The HTML canvas element that the web runner is rendering to
     canvas: web_sys::HtmlCanvasElement,
+    /// The HTML input element that handles input method editors and mobile keyboards
+    input: web_sys::HtmlInputElement,
+    /// Value of the `ime` field of the egui platform output from the previous frame
+    ime: Option<egui::output::IMEOutput>,
     /// The current egui zoom factor (`ctx.zoom_factor()`)
     zoom_factor: f32,
     /// JavaScript touch ID currently being tracked by the web runner
@@ -203,6 +209,12 @@ impl Runner {
         let Some(window) = web_sys::window() else {
             panic!("cannot create a `luminol_launcher::Runner` outside of the main thread");
         };
+        let Some(document) = window.document() else {
+            panic!("cannot create a `luminol_launcher::Runner` outside of the main thread");
+        };
+        let Some(body) = document.body() else {
+            panic!("cannot create a `luminol_launcher::Runner` outside of the main thread");
+        };
 
         // Install a hook to set `HAS_PANICKED` to true when a panic occurs on any thread
         if !PANIC_HOOK_INSTALLED.load(portable_atomic::Ordering::Relaxed) {
@@ -211,6 +223,9 @@ impl Runner {
                 HAS_PANICKED.store(true, portable_atomic::Ordering::Release);
                 old_panic_hook(info);
             }));
+            if let Some(input) = document.get_element_by_id("luminol-ime") {
+                input.remove();
+            }
             PANIC_HOOK_INSTALLED.store(true, portable_atomic::Ordering::Relaxed);
         }
 
@@ -220,6 +235,26 @@ impl Runner {
             .flatten()
             .map(|query| query.matches());
 
+        // Initialize handler for input method editors
+        let input = if let Some(input) = document.get_element_by_id("luminol-ime") {
+            input.dyn_into()?
+        } else {
+            let input = document
+                .create_element("input")?
+                .unchecked_into::<web_sys::HtmlInputElement>();
+            input.set_id("luminol-ime");
+            input.set_type("text");
+            let style = input.style();
+            style.set_property("opacity", "0")?;
+            style.set_property("width", "1px")?;
+            style.set_property("height", "1px")?;
+            style.set_property("position", "absolute")?;
+            style.set_property("top", "0")?;
+            style.set_property("left", "0")?;
+            body.append_child(&input)?;
+            input
+        };
+
         let (event_tx, event_rx) = flume::unbounded();
         let (output_tx, output_rx) = flume::unbounded();
         let (panic_tx, panic_rx) = oneshot::channel();
@@ -228,12 +263,15 @@ impl Runner {
             MainState {
                 event_tx,
                 canvas,
+                input,
+                ime: None,
                 zoom_factor: 1.,
                 touch_id: None,
             },
             output_rx,
             panic_rx,
             window,
+            document,
         )?;
 
         Ok(Self {