Few improvements to unified gsplat rendering (#8065)

* Few improvements to unified gsplat rendering

* Update src/framework/components/gsplat/component.js

Co-authored-by: Will Eastcott <[email protected]>

* feedback changes, thanks

* additional change

---------

Co-authored-by: Martin Valigursky <[email protected]>
Co-authored-by: Will Eastcott <[email protected]>

Author: 3 people

src/framework/components/gsplat/component.js

@@ -221,12 +221,17 @@ class GSplatComponent extends Component {
     /**
      * Sets the material used to render the gsplat.
      *
+     * **Note:** This setter is only supported when {@link unified} is `false`. When it's true, multiple
+     * gsplat components share a single material per camera/layer combination. To access materials in
+     * unified mode, use {@link GsplatComponentSystem#getGSplatMaterial}.
+     *
      * @param {ShaderMaterial} value - The material instance.
      */
     set material(value) {
-
-        Debug.assert(!this.unified);
-
+        if (this.unified) {
+            Debug.warn('GSplatComponent#material setter is not supported when unified true. Use app.systems.gsplat.getGSplatMaterial(camera, layer) to access materials.');
+            return;
+        }
         if (this._instance) {
             this._instance.material = value;
         } else {
@@ -237,9 +242,17 @@ class GSplatComponent extends Component {
     /**
      * Gets the material used to render the gsplat.
      *
+     * **Note:** This getter returns `null` when {@link unified} is `true`. In unified mode, materials are
+     * organized per camera/layer combination rather than per component. To access materials in
+     * unified mode, use {@link GsplatComponentSystem#getGSplatMaterial}.
+     *
      * @type {ShaderMaterial|null}
      */
     get material() {
+        if (this.unified) {
+            Debug.warnOnce('GSplatComponent#material getter returns null when unified=true. Use app.systems.gsplat.getGSplatMaterial(camera, layer) instead.');
+            return null;
+        }
         return this._instance?.material ?? this._materialTmp ?? null;
     }
 

src/framework/components/gsplat/system.js

@@ -9,6 +9,9 @@ import { GSplatAssetLoader } from './gsplat-asset-loader.js';
 
 /**
  * @import { AppBase } from '../../app-base.js'
+ * @import { Camera } from '../../../scene/camera.js'
+ * @import { Layer } from '../../../scene/layer.js'
+ * @import { ShaderMaterial } from '../../../scene/materials/shader-material.js'
  */
 
 const _schema = [
@@ -32,6 +35,24 @@ const _properties = [
  * @category Graphics
  */
 class GSplatComponentSystem extends ComponentSystem {
+    /**
+     * Fired when a GSplat material is created for a camera and layer combination. In unified
+     * mode, materials are created during the first frame update when the GSplat is rendered.
+     * The handler is passed the {@link ShaderMaterial}, the {@link Camera}, and the {@link Layer}.
+     *
+     * This event is useful for setting up custom material chunks and parameters before the
+     * first render.
+     *
+     * @event
+     * @example
+     * app.systems.gsplat.on('material:created', (material, camera, layer) => {
+     *     console.log(`Material created for camera ${camera.entity.name} on layer ${layer.name}`);
+     *     // Set custom material parameters before first render
+     *     material.setParameter('myParam', value);
+     * });
+     */
+    static EVENT_MATERIALCREATED = 'material:created';
+
     /**
      * Create a new GSplatComponentSystem.
      *
@@ -50,7 +71,7 @@ class GSplatComponentSystem extends ComponentSystem {
 
         // loader for splat LOD assets, as asset system is not available on the scene level
         const gsplatAssetLoader = new GSplatAssetLoader(app.assets);
-        app.renderer.gsplatDirector = new GSplatDirector(app.graphicsDevice, app.renderer, app.scene, gsplatAssetLoader);
+        app.renderer.gsplatDirector = new GSplatDirector(app.graphicsDevice, app.renderer, app.scene, gsplatAssetLoader, this);
 
         this.on('beforeremove', this.onRemove, this);
     }
@@ -105,6 +126,34 @@ class GSplatComponentSystem extends ComponentSystem {
     onRemove(entity, component) {
         component.onRemove();
     }
+
+    /**
+     * Gets the GSplat material used by unified GSplat rendering for the given camera and layer.
+     *
+     * Returns null if the material hasn't been created yet. In unified mode, materials are created
+     * during the first frame update when the GSplat is rendered. To be notified immediately when
+     * materials are created, listen to the 'material:created' event on GSplatComponentSystem:
+     *
+     * @example
+     * app.systems.gsplat.on('material:created', (material, camera, layer) => {
+     *     // Material is now available
+     *     material.setParameter('myParam', value);
+     * });
+     *
+     * @param {Camera} camera - The camera instance.
+     * @param {Layer} layer - The layer instance.
+     * @returns {ShaderMaterial|null} The material, or null if not created yet.
+     */
+    getGSplatMaterial(camera, layer) {
+        const director = this.app.renderer.gsplatDirector;
+        if (!director) return null;
+
+        const cameraData = director.camerasMap.get(camera);
+        if (!cameraData) return null;
+
+        const layerData = cameraData.layersMap.get(layer);
+        return layerData?.gsplatManager?.material ?? null;
+    }
 }
 
 Component._buildAccessors(GSplatComponent.prototype, _schema);

src/scene/gsplat-unified/gsplat-director.js

@@ -9,6 +9,7 @@ import { GSplatManager } from './gsplat-manager.js';
  * @import { GSplatAssetLoaderBase } from './gsplat-asset-loader-base.js'
  * @import { Scene } from '../scene.js'
  * @import { Renderer } from '../renderer/renderer.js'
+ * @import { EventHandler } from '../../core/event-handler.js'
  */
 
 /**
@@ -66,6 +67,12 @@ class GSplatCameraData {
         if (!layerData) {
             layerData = new GSplatLayerData(device, director, layer, cameraNode);
             this.layersMap.set(layer, layerData);
+
+            // Fire event that material was created
+            const material = layerData.gsplatManager.material;
+            if (material && director.eventHandler) {
+                director.eventHandler.fire('material:created', material, cameraNode.camera, layer);
+            }
         }
         return layerData;
     }
@@ -99,17 +106,24 @@ class GSplatDirector {
      */
     scene;
 
+    /**
+     * @type {EventHandler}
+     */
+    eventHandler;
+
     /**
      * @param {GraphicsDevice} device - The graphics device.
      * @param {Renderer} renderer - The renderer.
      * @param {Scene} scene - The scene.
      * @param {GSplatAssetLoaderBase} assetLoader - The asset loader.
+     * @param {EventHandler} eventHandler - Event handler for firing events.
      */
-    constructor(device, renderer, scene, assetLoader) {
+    constructor(device, renderer, scene, assetLoader, eventHandler) {
         this.device = device;
         this.renderer = renderer;
         this.assetLoader = assetLoader;
         this.scene = scene;
+        this.eventHandler = eventHandler;
     }
 
     destroy() {

src/scene/gsplat-unified/gsplat-manager.js

@@ -122,6 +122,13 @@ class GSplatManager {
      */
     octreeInstancesToDestroy = [];
 
+    /**
+     * Flag set when new octree instances are added, to trigger immediate LOD evaluation.
+     *
+     * @type {boolean}
+     */
+    hasNewOctreeInstances = false;
+
     constructor(device, director, layer, cameraNode) {
         this.device = device;
         this.scene = director.scene;
@@ -139,6 +146,10 @@ class GSplatManager {
         this.sorter.destroy();
     }
 
+    get material() {
+        return this.renderer.material;
+    }
+
     createSorter() {
         // create sorter
         const sorter = new GSplatUnifiedSorter();
@@ -163,6 +174,9 @@ class GSplatManager {
                 // make sure octree instance exists for placement
                 if (!this.octreeInstances.has(p)) {
                     this.octreeInstances.set(p, new GSplatOctreeInstance(p.resource.octree, p, this.director.assetLoader));
+
+                    // mark that we have new instances that need initial LOD evaluation
+                    this.hasNewOctreeInstances = true;
                 }
                 tempOctreePlacements.add(p);
             } else {
@@ -378,6 +392,10 @@ class GSplatManager {
             }
         }
 
+        // when new octree instances are added, we need to evaluate their LOD immediately
+        const hasNewInstances = this.hasNewOctreeInstances && this.sorter.jobsInFlight < 3;
+        if (hasNewInstances) this.hasNewOctreeInstances = false;
+
         let anyInstanceNeedsLodUpdate = false;
         let anyOctreeMoved = false;
         let cameraMovedOrRotated = false;
@@ -416,8 +434,8 @@ class GSplatManager {
             this.renderer.updateOverdrawMode(this.scene.gsplat);
         }
 
-        // when camera or octree need LOD evaluated, or params are dirty, or resources completed
-        if (cameraMovedOrRotated || anyOctreeMoved || this.scene.gsplat.dirty || anyInstanceNeedsLodUpdate) {
+        // when camera or octree need LOD evaluated, or params are dirty, or resources completed, or new instances added
+        if (cameraMovedOrRotated || anyOctreeMoved || this.scene.gsplat.dirty || anyInstanceNeedsLodUpdate || hasNewInstances) {
 
             // update the previous position where LOD was evaluated for octree instances
             for (const [, inst] of this.octreeInstances) {

src/scene/gsplat-unified/gsplat-octree-instance.js

@@ -291,7 +291,7 @@ class GSplatOctreeInstance {
 
         // calculate max LOD once for all nodes
         const maxLod = this.octree.lodLevels - 1;
-        const lodDistances = this.placement.lodDistances || [5, 10, 15, 20, 25];
+        const lodDistances = this.placement.lodDistances || [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60];
 
         // parameters
         const { lodBehindPenalty, lodRangeMin, lodRangeMax, lodUnderfillLimit = 0 } = params;

src/scene/gsplat-unified/gsplat-renderer.js

@@ -60,6 +60,25 @@ class GSplatRenderer {
             }
         });
 
+        this.configureMaterial();
+
+        this.meshInstance = this.createMeshInstance();
+        layer.addMeshInstances([this.meshInstance]);
+    }
+
+    destroy() {
+        this.layer.removeMeshInstances([this.meshInstance]);
+        this._material.destroy();
+        this.meshInstance.destroy();
+    }
+
+    get material() {
+        return this._material;
+    }
+
+    configureMaterial() {
+        const { device, workBuffer } = this;
+
         // input format
         this._material.setDefine('GSPLAT_WORKBUFFER_DATA', true);
         this._material.setDefine('STORAGE_ORDER', device.isWebGPU);
@@ -85,15 +104,6 @@ class GSplatRenderer {
         this._material.blendType = dither ? BLEND_NONE : BLEND_PREMULTIPLIED;
         this._material.depthWrite = !!dither;
         this._material.update();
-
-        this.meshInstance = this.createMeshInstance();
-        layer.addMeshInstances([this.meshInstance]);
-    }
-
-    destroy() {
-        this.layer.removeMeshInstances([this.meshInstance]);
-        this._material.destroy();
-        this.meshInstance.destroy();
     }
 
     update(count, textureSize) {

src/scene/shader-lib/glsl/chunks/gsplat/vert/gsplatCustomize.js

@@ -13,6 +13,13 @@ void modifyCovariance(vec3 originalCenter, vec3 modifiedCenter, inout vec3 covA,
     // float size = gsplatExtractSize(covA, covB);
     // float newSize = clamp(size, 0.01, 0.5);
     // gsplatApplyUniformScale(covA, covB, newSize / size);
+    //
+    // Example to make splats round/spherical:
+    // float size = gsplatExtractSize(covA, covB);
+    // gsplatMakeRound(covA, covB, size * 0.5);
+    //
+    // To hide a splat:
+    // gsplatMakeRound(covA, covB, 0.0);
 }
 
 void modifyColor(vec3 center, inout vec4 color) {

src/scene/shader-lib/glsl/chunks/gsplat/vert/gsplatHelpers.js

@@ -11,4 +11,12 @@ void gsplatApplyUniformScale(inout vec3 covA, inout vec3 covB, float scale) {
     covA *= s2;
     covB *= s2;
 }
+
+// Helper function to make splat spherical/round with given size
+// Use size = 0.0 to hide the splat
+void gsplatMakeRound(inout vec3 covA, inout vec3 covB, float size) {
+    float s2 = size * size;
+    covA = vec3(s2, 0.0, 0.0);
+    covB = vec3(s2, 0.0, s2);
+}
 `;

src/scene/shader-lib/wgsl/chunks/gsplat/vert/gsplatCustomize.js

@@ -13,6 +13,13 @@ fn modifyCovariance(originalCenter: vec3f, modifiedCenter: vec3f, covA: ptr<func
     // let size = gsplatExtractSize(*covA, *covB);
     // let newSize = clamp(size, 0.01, 0.5);
     // gsplatApplyUniformScale(covA, covB, newSize / size);
+    //
+    // Example to make splats round/spherical:
+    // let size = gsplatExtractSize(*covA, *covB);
+    // gsplatMakeRound(covA, covB, size * 0.5);
+    //
+    // To hide a splat:
+    // gsplatMakeRound(covA, covB, 0.0);
 }
 
 fn modifyColor(center: vec3f, color: ptr<function, vec4f>) {

src/scene/shader-lib/wgsl/chunks/gsplat/vert/gsplatHelpers.js

@@ -11,4 +11,12 @@ fn gsplatApplyUniformScale(covA: ptr<function, vec3f>, covB: ptr<function, vec3f
     *covA = *covA * s2;
     *covB = *covB * s2;
 }
+
+// Helper function to make splat spherical/round with given size
+// Use size = 0.0 to hide the splat
+fn gsplatMakeRound(covA: ptr<function, vec3f>, covB: ptr<function, vec3f>, size: f32) {
+    let s2 = size * size;
+    *covA = vec3f(s2, 0.0, 0.0);
+    *covB = vec3f(s2, 0.0, s2);
+}
 `;