Feat/animation api (#20)

Co-authored-by: Marc Rousavy <[email protected]>

package/android/CMakeLists.txt

@@ -37,6 +37,7 @@ add_library(
         ../cpp/core/ViewWrapper.cpp
         ../cpp/core/SwapChainWrapper.cpp
         ../cpp/core/FilamentAssetWrapper.cpp
+        ../cpp/core/AnimatorWrapper.cpp
 
         # Filament Utils
         ../cpp/core/utils/EntityWrapper.cpp

package/cpp/core/AnimatorWrapper.cpp

@@ -0,0 +1,61 @@
+//
+// Created by Hanno Gödecke on 29.02.24.
+//
+
+#include "AnimatorWrapper.h"
+
+namespace margelo {
+void AnimatorWrapper::loadHybridMethods() {
+  registerHybridMethod("applyAnimation", &AnimatorWrapper::applyAnimation, this);
+  registerHybridMethod("updateBoneMatrices", &AnimatorWrapper::updateBoneMatrices, this);
+  registerHybridMethod("applyCrossFade", &AnimatorWrapper::applyCrossFade, this);
+  registerHybridMethod("resetBoneMatrices", &AnimatorWrapper::resetBoneMatrices, this);
+  registerHybridMethod("getAnimationCount", &AnimatorWrapper::getAnimationCount, this);
+  registerHybridMethod("getAnimationDuration", &AnimatorWrapper::getAnimationDuration, this);
+  registerHybridMethod("getAnimationName", &AnimatorWrapper::getAnimationName, this);
+}
+
+Animator* AnimatorWrapper::getAnimator() {
+  FilamentInstance* instance = _asset->getInstance();
+  if (instance == nullptr) {
+    [[unlikely]];
+    throw std::runtime_error("Filament Asset does not contain a valid FilamentInstance!");
+  }
+  return instance->getAnimator();
+}
+
+void AnimatorWrapper::applyAnimation(int animationIndex, double time) {
+  Animator* animator = getAnimator();
+  animator->applyAnimation(animationIndex, time);
+}
+
+void AnimatorWrapper::updateBoneMatrices() {
+  Animator* animator = getAnimator();
+  animator->updateBoneMatrices();
+}
+
+void AnimatorWrapper::applyCrossFade(int previousAnimationIndex, double previousAnimationTime, double alpha) {
+  Animator* animator = getAnimator();
+  animator->applyCrossFade(previousAnimationIndex, previousAnimationTime, alpha);
+}
+
+void AnimatorWrapper::resetBoneMatrices() {
+  Animator* animator = getAnimator();
+  animator->resetBoneMatrices();
+}
+
+int AnimatorWrapper::getAnimationCount() {
+  Animator* animator = getAnimator();
+  return animator->getAnimationCount();
+}
+
+double AnimatorWrapper::getAnimationDuration(int animationIndex) {
+  Animator* animator = getAnimator();
+  return animator->getAnimationDuration(animationIndex);
+}
+
+std::string AnimatorWrapper::getAnimationName(int animationIndex) {
+  Animator* animator = getAnimator();
+  return animator->getAnimationName(animationIndex);
+}
+} // namespace margelo

package/cpp/core/AnimatorWrapper.h

@@ -0,0 +1,35 @@
+//
+// Created by Hanno Gödecke on 29.02.24.
+//
+
+#pragma once
+
+#include "jsi/HybridObject.h"
+#include <gltfio/Animator.h>
+
+namespace margelo {
+
+using namespace filament::gltfio;
+
+class AnimatorWrapper : public HybridObject {
+public:
+  explicit AnimatorWrapper(const std::shared_ptr<FilamentAsset>& asset) : HybridObject("AnimatorWrapper"), _asset(std::move(asset)) {}
+
+  void loadHybridMethods() override;
+
+private:
+  void applyAnimation(int animationIndex, double time);
+  void applyCrossFade(int previousAnimationIndex, double previousAnimationTime, double alpha);
+  void updateBoneMatrices();
+  void resetBoneMatrices();
+  int getAnimationCount();
+  double getAnimationDuration(int animationIndex);
+  std::string getAnimationName(int animationIndex);
+
+  Animator* getAnimator();
+
+private:
+  std::shared_ptr<FilamentAsset> _asset;
+};
+
+} // namespace margelo

package/cpp/core/EngineWrapper.cpp

@@ -179,7 +179,7 @@ void EngineWrapper::destroySurface() {
   _swapChain = nullptr;
 }
 
-void EngineWrapper::setRenderCallback(std::function<void(std::shared_ptr<EngineWrapper>)> callback) {
+void EngineWrapper::setRenderCallback(std::function<void(double, double, double)> callback) {
   _renderCallback = std::move(callback);
 }
 
@@ -209,7 +209,8 @@ void EngineWrapper::renderFrame(double timestamp) {
 
   if (_renderCallback) {
     // Call JS callback with scene information
-    _renderCallback(nullptr);
+    double passedSeconds = (timestamp - _startTime) / 1e9;
+    _renderCallback(timestamp, _startTime, passedSeconds);
   }
 
   std::shared_ptr<Renderer> renderer = _renderer->getRenderer();

package/cpp/core/EngineWrapper.h

@@ -60,7 +60,7 @@ class EngineWrapper : public HybridObject {
   void setSurface(std::shared_ptr<Surface> surface);
   void destroySurface();
   void surfaceSizeChanged(int width, int height);
-  void setRenderCallback(std::function<void(std::shared_ptr<EngineWrapper>)> callback);
+  void setRenderCallback(std::function<void(double, double, double)> callback);
   void renderFrame(double timestamp);
 
   void transformToUnitCube(std::shared_ptr<FilamentAssetWrapper> asset);
@@ -82,7 +82,7 @@ class EngineWrapper : public HybridObject {
   std::shared_ptr<Engine> _engine;
   std::shared_ptr<SurfaceProvider> _surfaceProvider;
   std::shared_ptr<Listener> _listener;
-  std::function<void(std::shared_ptr<EngineWrapper>)> _renderCallback;
+  std::function<void(double, double, double)> _renderCallback;
   std::function<std::shared_ptr<FilamentBuffer>(std::string)> _getAssetBytes;
   std::shared_ptr<Choreographer> _choreographer;
   std::shared_ptr<Listener> _choreographerListener;

package/cpp/core/FilamentAssetWrapper.cpp

@@ -9,6 +9,8 @@ using namespace utils;
 
 void FilamentAssetWrapper::loadHybridMethods() {
   registerHybridMethod("getRoot", &FilamentAssetWrapper::getRoot, this);
+  registerHybridMethod("releaseSourceData", &FilamentAssetWrapper::releaseSourceData, this);
+  registerHybridMethod("getAnimator", &FilamentAssetWrapper::getAnimator, this);
 }
 
 /**
@@ -30,4 +32,12 @@ std::shared_ptr<EntityWrapper> FilamentAssetWrapper::getRoot() {
   return std::make_shared<EntityWrapper>(rootEntity);
 }
 
+void FilamentAssetWrapper::releaseSourceData() {
+  _asset->releaseSourceData();
+}
+
+std::shared_ptr<AnimatorWrapper> FilamentAssetWrapper::getAnimator() {
+  return std::make_shared<AnimatorWrapper>(_asset);
+}
+
 } // namespace margelo

package/cpp/core/FilamentAssetWrapper.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "AnimatorWrapper.h"
 #include "core/utils/EntityWrapper.h"
 #include "jsi/HybridObject.h"
 #include <filament/TransformManager.h>
@@ -17,12 +18,10 @@ class FilamentAssetWrapper : public HybridObject {
 
   void transformToUnitCube(TransformManager& transformManager);
 
-  const std::shared_ptr<gltfio::FilamentAsset> getAsset() {
-    return _asset;
-  }
-
 private:
   std::shared_ptr<EntityWrapper> getRoot();
+  void releaseSourceData();
+  std::shared_ptr<AnimatorWrapper> getAnimator();
 
 private:
   std::shared_ptr<gltfio::FilamentAsset> _asset;

package/example/ios/FilamentExample.xcodeproj/project.pbxproj

@@ -16,6 +16,8 @@
 		59249653BB174593BEDB9F3D /* pengu.glb in Resources */ = {isa = PBXBuildFile; fileRef = 3E2439B417DA497E83EB1F05 /* pengu.glb */; };
 		7699B88040F8A987B510C191 /* libPods-FilamentExample-FilamentExampleTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 19F6CBCC0A4E27FBF8BF4A61 /* libPods-FilamentExample-FilamentExampleTests.a */; };
 		81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; };
+		0B65F1CE0E1D4DBB9E25D0BD /* default_env_ibl.ktx in Resources */ = {isa = PBXBuildFile; fileRef = 7A69823713B34897AF31CAF0 /* default_env_ibl.ktx */; };
+		59249653BB174593BEDB9F3D /* pengu.glb in Resources */ = {isa = PBXBuildFile; fileRef = 3E2439B417DA497E83EB1F05 /* pengu.glb */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -48,6 +50,8 @@
 		81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = FilamentExample/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		89C6BE57DB24E9ADA2F236DE /* Pods-FilamentExample-FilamentExampleTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-FilamentExample-FilamentExampleTests.release.xcconfig"; path = "Target Support Files/Pods-FilamentExample-FilamentExampleTests/Pods-FilamentExample-FilamentExampleTests.release.xcconfig"; sourceTree = "<group>"; };
 		ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
+		7A69823713B34897AF31CAF0 /* default_env_ibl.ktx */ = {isa = PBXFileReference; name = "default_env_ibl.ktx"; path = "../assets/default_env_ibl.ktx"; sourceTree = "<group>"; fileEncoding = undefined; lastKnownFileType = unknown; explicitFileType = undefined; includeInIndex = 0; };
+		3E2439B417DA497E83EB1F05 /* pengu.glb */ = {isa = PBXFileReference; name = "pengu.glb"; path = "../assets/pengu.glb"; sourceTree = "<group>"; fileEncoding = undefined; lastKnownFileType = unknown; explicitFileType = undefined; includeInIndex = 0; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -163,6 +167,16 @@
 			path = Pods;
 			sourceTree = "<group>";
 		};
+		995DFB4F30584326AD631A9F /* Resources */ = {
+			isa = "PBXGroup";
+			children = (
+				7A69823713B34897AF31CAF0 /* default_env_ibl.ktx */,
+				3E2439B417DA497E83EB1F05 /* pengu.glb */,
+			);
+			name = Resources;
+			sourceTree = "<group>";
+			path = "";
+		};
 /* End PBXGroup section */
 
 /* Begin PBXNativeTarget section */
@@ -599,10 +613,7 @@
 					"-DFOLLY_USE_LIBCPP=1",
 					"-DFOLLY_CFG_NO_COROUTINES=1",
 				);
-				OTHER_LDFLAGS = (
-					"$(inherited)",
-					" ",
-				);
+				OTHER_LDFLAGS = "$(inherited)  ";
 				REACT_NATIVE_PATH = "${PODS_ROOT}/../../node_modules/react-native";
 				SDKROOT = iphoneos;
 				USE_HERMES = true;
@@ -670,10 +681,7 @@
 					"-DFOLLY_USE_LIBCPP=1",
 					"-DFOLLY_CFG_NO_COROUTINES=1",
 				);
-				OTHER_LDFLAGS = (
-					"$(inherited)",
-					" ",
-				);
+				OTHER_LDFLAGS = "$(inherited)  ";
 				REACT_NATIVE_PATH = "${PODS_ROOT}/../../node_modules/react-native";
 				SDKROOT = iphoneos;
 				USE_HERMES = true;

package/src/FilamentView.tsx

@@ -1,8 +1,9 @@
 import React from 'react'
-import { findNodeHandle, NativeMethods, Platform } from 'react-native'
+import { Button, findNodeHandle, NativeMethods, Platform, ScrollView, View } from 'react-native'
 import { FilamentProxy } from './native/FilamentProxy'
 import { FilamentNativeView, NativeProps } from './native/FilamentNativeView'
 import type { Float3 } from './types/float3'
+import { Animator } from './types/Animator'
 
 type FilamentViewProps = NativeProps
 
@@ -18,14 +19,36 @@ const indirectLightPath = Platform.select({
   ios: 'default_env_ibl.ktx',
 })!
 
-export class FilamentView extends React.PureComponent<FilamentViewProps> {
+// Temporarily
+type _State = {
+  animationNames: string[]
+  currentAnimation: number
+}
+
+const animationInterpolationTime = 5 // 1 second
+
+export class FilamentView extends React.PureComponent<FilamentViewProps, _State> {
   private readonly ref: React.RefObject<RefType>
   private readonly engine = FilamentProxy.createEngine()
+  private animator: Animator
+
+  private prevAnimationIndex: number | null = null
+  private prevAnimationStarted: number | null = null
+  private animationInterpolation = 0
 
   constructor(props: FilamentViewProps) {
     super(props)
     this.ref = React.createRef<RefType>()
 
+    const pengu = this.loadPengu()
+    this.animator = pengu.getAnimator()
+    this.state = {
+      animationNames: Array.from({ length: this.animator.getAnimationCount() }, (_, i) => this.animator.getAnimationName(i)),
+      currentAnimation: 0,
+    }
+    pengu.releaseSourceData() // Cleanup memory after loading the asset
+    console.log('animationNames', this.state.animationNames)
+
     this.setup3dScene()
   }
 
@@ -46,18 +69,19 @@ export class FilamentView extends React.PureComponent<FilamentViewProps> {
     }, 100)
   }
 
-  setup3dScene = () => {
+  loadPengu = () => {
     // Load a model into the scene:
     const modelBuffer = FilamentProxy.getAssetByteBuffer(penguModelPath)
     const penguAsset = this.engine.loadAsset(modelBuffer)
+
     // By default all assets get added to the origin at 0,0,0,
     // we transform it to fit into a unit cube at the origin using this utility:
     this.engine.transformToUnitCube(penguAsset)
 
-    // We can also change the pengus position, rotation and scale:
-    const penguEntity = penguAsset.getRoot()
-    this.engine.setEntityPosition(penguEntity, [0, 2, 0], true) // Move the pengu up by 2 units
+    return penguAsset
+  }
 
+  setup3dScene = () => {
     // Create a default light:
     const indirectLightBuffer = FilamentProxy.getAssetByteBuffer(indirectLightPath)
     this.engine.setIndirectLight(indirectLightBuffer)
@@ -67,11 +91,33 @@ export class FilamentView extends React.PureComponent<FilamentViewProps> {
     this.engine.getScene().addEntity(light)
   }
 
-  renderCallback = () => {
+  renderCallback = (_timestamp: number, _startTime: number, passedSeconds: number) => {
     const cameraPosition: Float3 = [0, 0, 5]
     const cameraTarget: Float3 = [0, 0, 0]
     const cameraUp: Float3 = [0, 1, 0]
 
+    if (this.animator) {
+      this.animator.applyAnimation(this.state.currentAnimation, passedSeconds)
+      if (this.prevAnimationIndex !== null) {
+        if (this.prevAnimationStarted === null) {
+          this.prevAnimationStarted = passedSeconds
+        }
+        this.animationInterpolation += passedSeconds - this.prevAnimationStarted
+        const alpha = this.animationInterpolation / animationInterpolationTime
+
+        this.animator.applyCrossFade(this.prevAnimationIndex, this.prevAnimationStarted!, alpha)
+        console.log('this.animationInterpolation', this.animationInterpolation)
+        console.log('passedSeconds', passedSeconds)
+        if (this.animationInterpolation >= animationInterpolationTime) {
+          this.prevAnimationIndex = null
+          this.prevAnimationStarted = null
+          this.animationInterpolation = 0
+        }
+      }
+
+      this.animator.updateBoneMatrices()
+    }
+
     this.engine.getCamera().lookAt(cameraPosition, cameraTarget, cameraUp)
   }
 
@@ -99,6 +145,23 @@ export class FilamentView extends React.PureComponent<FilamentViewProps> {
 
   /** @internal */
   public render(): React.ReactNode {
-    return <FilamentNativeView ref={this.ref} {...this.props} />
+    return (
+      <View style={this.props.style}>
+        <FilamentNativeView ref={this.ref} {...this.props} />
+
+        <ScrollView style={{ position: 'absolute', bottom: 0, maxHeight: 200, width: '100%' }}>
+          {this.state.animationNames.map((name, i) => (
+            <Button
+              key={i}
+              onPress={() => {
+                this.prevAnimationIndex = this.state.currentAnimation
+                this.setState({ currentAnimation: i })
+              }}
+              title={name}
+            />
+          ))}
+        </ScrollView>
+      </View>
+    )
   }
 }