fix(ar): prevent WebXR select events on DOM overlay inputs and adapt visualViewport layout for virtual keyboards (#5153)

Author: diegoteran

packages/model-viewer/src/test/three-components/ARRenderer-spec.ts

@@ -241,6 +241,44 @@ suite('ARRenderer', () => {
       expect(scale.z).to.be.equal(1);
     });
 
+    test('prevents default WebXR select when beforexrselect is triggered on interactive elements', () => {
+      const button = document.createElement('button');
+      const event = new CustomEvent('beforexrselect', {
+        bubbles: true,
+        cancelable: true
+      });
+      Object.defineProperty(event, 'composedPath', {
+        value: () => [button, (arRenderer as any).overlay]
+      });
+
+      let defaultPrevented = false;
+      event.preventDefault = () => {
+        defaultPrevented = true;
+      };
+
+      (arRenderer as any).overlay.dispatchEvent(event);
+      expect(defaultPrevented).to.be.equal(true);
+    });
+
+    test('does not prevent default WebXR select on non-interactive elements', () => {
+      const div = document.createElement('div');
+      const event = new CustomEvent('beforexrselect', {
+        bubbles: true,
+        cancelable: true
+      });
+      Object.defineProperty(event, 'composedPath', {
+        value: () => [div, (arRenderer as any).overlay]
+      });
+
+      let defaultPrevented = false;
+      event.preventDefault = () => {
+        defaultPrevented = true;
+      };
+
+      (arRenderer as any).overlay.dispatchEvent(event);
+      expect(defaultPrevented).to.be.equal(false);
+    });
+
     suite('presentation ends', () => {
       setup(async () => {
         await arRenderer.stopPresenting();

packages/model-viewer/src/three-components/ARRenderer.ts

@@ -261,6 +261,9 @@ export class ARRenderer extends EventDispatcher<
     // This sets isPresenting to true
     this._presentedScene = scene;
     this.overlay = scene.element.shadowRoot!.querySelector('div.default');
+    if (this.overlay != null) {
+      this.overlay.addEventListener('beforexrselect', this.onBeforeXRSelect);
+    }
 
     if (environmentEstimation === true) {
       this.xrLight = new XREstimatedLight(this.threeRenderer);
@@ -712,7 +715,10 @@ export class ARRenderer extends EventDispatcher<
     this.oldShadowIntensity = null;
     this.frame = null;
     this.inputSource = null;
-    this.overlay = null;
+    if (this.overlay != null) {
+      this.overlay.removeEventListener('beforexrselect', this.onBeforeXRSelect);
+      this.overlay = null;
+    }
     this.worldSpaceInitialPlacementDone = false;
 
     if (this.resolveCleanup != null) {
@@ -945,6 +951,22 @@ export class ARRenderer extends EventDispatcher<
     this.placementBox!.show = false
   };
 
+  private onBeforeXRSelect = (event: Event) => {
+    const path = event.composedPath();
+    for (const element of path) {
+      if (element instanceof HTMLElement) {
+        const tagName = element.tagName.toLowerCase();
+        if (tagName === 'input' || tagName === 'button' ||
+            tagName === 'select' || tagName === 'textarea' ||
+            tagName === 'a' || element.hasAttribute('data-pointer-coalesce') ||
+            element.classList.contains('interactive')) {
+          event.preventDefault();
+          break;
+        }
+      }
+    }
+  };
+
   private fingerPolar(fingers: XRTransientInputHitTestResult[]):
       {separation: number, deltaYaw: number, angle: number} {
     const fingerOne = fingers[0].inputSource.gamepad!.axes;

packages/modelviewer.dev/data/faq.json

@@ -124,6 +124,14 @@
           "<a href='../docs/#entrydocs-loading-staticproperties-dracoDecoderLocation'>Decoder location static properties</a>",
           "<a href='../examples/loading/#dracoSupport'>Details on setting static properties</a>"
         ]
+      },
+      {
+        "name": "How do I make custom inputs or overlays work reliably in AR DOM Overlay mode?",
+        "htmlName": "ar-dom-overlay-inputs",
+        "description": "Inside a WebXR full-screen immersive AR session, mobile browsers do not shrink the layout viewport when a soft keyboard is shown, which can cover your custom input overlays. Additionally, tap events on your overlays can propagate as WebXR screen taps and trigger unexpected 3D gestures (like model movements). To prevent these issues, follow two best practices:<br/><br/>1. <b>Event Suppression:</b> The core &lt;model-viewer&gt; now automatically intercepts the <code>beforexrselect</code> event on your interactive DOM elements and calls <code>event.preventDefault()</code> to stop WebXR from turning DOM taps into 3D session interactions.<br/>2. <b>Viewport Resizing:</b> Use the <code>visualViewport</code> API to listen for keyboard height overlays (when <code>ar-status='session-started'</code>) and translate your elements upward (e.g., via <code>transform: translateY(-offset px);</code>) to keep them visible above the keyboard, as demonstrated in our scenegraph example.",
+        "links": [
+          "<a href='../examples/scenegraph/#transforms'>Model Transformations example</a>"
+        ]
       }
     ]
   },

packages/modelviewer.dev/examples/scenegraph/index.html

@@ -157,6 +157,7 @@ <h2 class="demo-title">Model Transformations</h2>
                 will be generated on the fly from the current state.
               </p>
               <p><b>Tip:</b> When building custom interactive overlay controls with text or numeric input fields on mobile, ensure your inputs are styled with a CSS rule setting <code>font-size: 16px;</code> (or larger). If a focused text field's computed font size is below 16px, mobile WebKit browsers (like iOS Safari and Chrome) will automatically zoom in the entire viewport, which does not automatically reset when the input is blurred and can break the page layout.</p>
+              <p><b>AR Keyboard Overlays:</b> Inside a WebXR full-screen immersive AR session, mobile browsers do not shrink the layout viewport when a software virtual keyboard is open. To prevent inputs from being covered by the keyboard, you can register a <code>visualViewport</code> resize listener (only active when <code>ar-status="session-started"</code>) to calculate the soft keyboard's overlay height and dynamically translate your overlay container upward (e.g., via <code>transform: translateY(-offset px);</code>), as demonstrated in the script below.</p>
             </div>
             <example-snippet stamp-to="transforms" highlight-as="html">
               <template>
@@ -215,6 +216,28 @@ <h2 class="demo-title">Model Transformations</h2>
                   z.addEventListener('input', () => {
                     updateScale();
                   });
+
+                  const controls = modelViewerTransform.querySelector(".controls");
+                  if (window.visualViewport) {
+                    const updateControls = () => {
+                      if (modelViewerTransform.getAttribute('ar-status') !== 'session-started') {
+                        controls.style.transform = 'none';
+                        return;
+                      }
+                      const layoutHeight = window.innerHeight;
+                      const visualHeight = window.visualViewport.height;
+                      const offsetTop = window.visualViewport.offsetTop;
+                      const offset = layoutHeight - (visualHeight + offsetTop);
+                      if (offset > 0) {
+                        controls.style.transform = `translateY(-${offset}px)`;
+                      } else {
+                        controls.style.transform = 'none';
+                      }
+                    };
+                    window.visualViewport.addEventListener('resize', updateControls);
+                    window.visualViewport.addEventListener('scroll', updateControls);
+                    modelViewerTransform.addEventListener('ar-status', updateControls);
+                  }
                 </script>
               </template>
             </example-snippet>