Fingertracking, Input and driver version

server/core/src/main/java/dev/slimevr/VRServer.kt

@@ -10,6 +10,8 @@ import dev.slimevr.firmware.SerialFlashingHandler
 import dev.slimevr.games.vrchat.VRCConfigHandler
 import dev.slimevr.games.vrchat.VRCConfigHandlerStub
 import dev.slimevr.games.vrchat.VRChatConfigManager
+import dev.slimevr.inputs.Input
+import dev.slimevr.inputs.TapInputManager
 import dev.slimevr.osc.OSCHandler
 import dev.slimevr.osc.OSCRouter
 import dev.slimevr.osc.VMCHandler
@@ -117,6 +119,9 @@ class VRServer @JvmOverloads constructor(
 	@JvmField
 	val handshakeHandler = HandshakeHandler()
 
+	val tapInputManager: TapInputManager
+	val inputs: MutableList<Input> = FastList()
+
 	init {
 		// UwU
 		configManager = ConfigManager(configPath)
@@ -147,7 +152,9 @@ class VRServer @JvmOverloads constructor(
 		for (bridge in bridgeProvider(this, computedTrackers) + sequenceOf(WebSocketVRBridge(computedTrackers, this))) {
 			tasks.add(Runnable { bridge.startBridge() })
 			bridges.add(bridge)
+			bridge.addFingerBones(humanPoseManager.shareableFingerBones)
 		}
+		tapInputManager = TapInputManager(humanPoseManager.skeleton, this)
 
 		// Initialize OSC handlers
 		vrcOSCHandler = VRCOSCHandler(
@@ -245,11 +252,23 @@ class VRServer @JvmOverloads constructor(
 				tracker.tick(fpsTimer.timePerFrame)
 			}
 			humanPoseManager.update()
+
+			// Check for tap inputs
+			tapInputManager.update()
+			// Update bridges
 			for (bridge in bridges) {
+				// Send inputs to each bridge
+				for (input in inputs) {
+					bridge.sendInput(input)
+				}
 				bridge.dataWrite()
 			}
+			// Don't forget to clear inputs so that we don't send the same ones again
+			inputs.clear()
+
 			vrcOSCHandler.update()
 			vMCHandler.update()
+
 			// final long time = System.currentTimeMillis() - start;
 			try {
 				sleep(1) // 1000Hz

server/core/src/main/java/dev/slimevr/bridge/Bridge.kt

@@ -1,5 +1,7 @@
 package dev.slimevr.bridge
 
+import dev.slimevr.inputs.Input
+import dev.slimevr.tracking.processor.ShareableBone
 import dev.slimevr.tracking.trackers.Tracker
 import dev.slimevr.tracking.trackers.TrackerRole
 import dev.slimevr.util.ann.VRServerThread
@@ -38,6 +40,24 @@ interface Bridge {
 	@VRServerThread
 	fun removeSharedTracker(tracker: Tracker?)
 
+	/**
+	 * Adds a list of finger bones to the bridge. This should only be set
+	 * once the skeleton has initialized. Bridge will send finger data for
+	 * the hand trackers if it serves them.
+	 *
+	 * @param bones
+	 */
+	@VRServerThread
+	fun addFingerBones(bones: List<ShareableBone>)
+
+	/**
+	 * Reports a virtual controller input to the driver.
+	 *
+	 * @param input the object containing the data about the input
+	 */
+	@VRServerThread
+	fun sendInput(input: Input)
+
 	@VRServerThread
 	fun startBridge()
 

server/core/src/main/java/dev/slimevr/inputs/Input.kt

@@ -0,0 +1,8 @@
+package dev.slimevr.inputs
+
+class Input(val rightHand: Boolean, val type: InputType)
+
+enum class InputType {
+	DOUBLE_TAP,
+	TRIPLE_TAP,
+}

server/core/src/main/java/dev/slimevr/inputs/TapInputManager.kt

@@ -0,0 +1,98 @@
+package dev.slimevr.inputs
+
+import dev.slimevr.VRServer
+import dev.slimevr.tracking.processor.skeleton.HumanSkeleton
+import dev.slimevr.tracking.processor.skeleton.TapDetection
+import dev.slimevr.tracking.trackers.Tracker
+import dev.slimevr.util.ann.VRServerThread
+
+/**
+ * Handles tap detection for SteamVR virtual controller input
+ */
+class TapInputManager(
+	private val skeleton: HumanSkeleton,
+	private val vrServer: VRServer,
+) {
+
+	// tap detectors
+	private lateinit var leftDoubleDetector: TapDetection
+	private lateinit var leftTripleDetector: TapDetection
+	private lateinit var rightDoubleDetector: TapDetection
+	private lateinit var rightTripleDetector: TapDetection
+
+	val NS_CONVERTER: Float = 1.0e9f
+	private var doubleTapDelay = 0.25f * NS_CONVERTER
+
+	init {
+		reinit()
+	}
+
+	fun reinit() {
+		// Create tap detectors for both hands
+		leftDoubleDetector = TapDetection(skeleton, trackerToWatchLeftHand)
+		leftDoubleDetector.enabled = true
+		leftDoubleDetector.setMaxTaps(2)
+
+		leftTripleDetector = TapDetection(skeleton, trackerToWatchLeftHand)
+		leftTripleDetector.enabled = true
+		leftTripleDetector.setMaxTaps(3)
+
+		rightDoubleDetector = TapDetection(skeleton, trackerToWatchRightHand)
+		rightDoubleDetector.enabled = true
+		rightDoubleDetector.setMaxTaps(2)
+
+		rightTripleDetector = TapDetection(skeleton, trackerToWatchRightHand)
+		rightTripleDetector.enabled = true
+		rightTripleDetector.setMaxTaps(3)
+	}
+
+	@VRServerThread
+	fun update() {
+		// update the tap detectors
+		leftDoubleDetector.update()
+		leftTripleDetector.update()
+		rightDoubleDetector.update()
+		rightTripleDetector.update()
+
+		// check if any tap detectors have detected taps
+		if (3 <= leftTripleDetector.taps) {
+			leftTripleDetector.resetDetector()
+			leftDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(false, InputType.TRIPLE_TAP))
+		} else if (2 <= leftDoubleDetector.taps && System.nanoTime() - rightDoubleDetector.detectionTime > doubleTapDelay) {
+			leftTripleDetector.resetDetector()
+			leftDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(false, InputType.DOUBLE_TAP))
+		}
+
+		if (3 <= rightTripleDetector.taps) {
+			rightTripleDetector.resetDetector()
+			rightDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(true, InputType.TRIPLE_TAP))
+		} else if (2 <= rightDoubleDetector.taps && System.nanoTime() - rightDoubleDetector.detectionTime > doubleTapDelay) {
+			rightTripleDetector.resetDetector()
+			rightDoubleDetector.resetDetector()
+			vrServer.inputs.add(Input(true, InputType.DOUBLE_TAP))
+		}
+	}
+
+	private val trackerToWatchLeftHand: Tracker?
+		get() = listOfNotNull(
+			skeleton.leftHandTracker,
+			skeleton.leftLowerArmTracker,
+			skeleton.leftUpperArmTracker,
+			skeleton.leftThumbDistalTracker,
+			skeleton.leftThumbProximalTracker,
+			skeleton.leftThumbMetacarpalTracker,
+		).firstOrNull()
+
+	private val trackerToWatchRightHand: Tracker?
+		get() = listOfNotNull(
+			skeleton.rightHandTracker,
+			skeleton.rightLowerArmTracker,
+			skeleton.rightUpperArmTracker,
+			skeleton.rightThumbDistalTracker,
+			skeleton.rightThumbProximalTracker,
+			skeleton.rightThumbMetacarpalTracker,
+		).firstOrNull()
+}

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsBuilder.java

@@ -61,8 +61,8 @@ public static int createVRCOSCSettings(
 					&& config.getOSCTrackerRole(TrackerRole.RIGHT_FOOT, false),
 				config.getOSCTrackerRole(TrackerRole.LEFT_ELBOW, false)
 					&& config.getOSCTrackerRole(TrackerRole.RIGHT_ELBOW, false),
-				config.getOSCTrackerRole(TrackerRole.LEFT_HAND, false)
-					&& config.getOSCTrackerRole(TrackerRole.RIGHT_HAND, false)
+				config.getOSCTrackerRole(TrackerRole.LEFT_CONTROLLER, false)
+					&& config.getOSCTrackerRole(TrackerRole.RIGHT_CONTROLLER, false)
 			);
 		VRCOSCSettings.startVRCOSCSettings(fbb);
 		VRCOSCSettings.addOscSettings(fbb, generalSettingOffset);
@@ -164,8 +164,8 @@ public static int createSteamVRSettings(FlatBufferBuilder fbb, ISteamVRBridge br
 					bridge.getShareSetting(TrackerRole.RIGHT_KNEE),
 					bridge.getShareSetting(TrackerRole.LEFT_ELBOW),
 					bridge.getShareSetting(TrackerRole.RIGHT_ELBOW),
-					bridge.getShareSetting(TrackerRole.LEFT_HAND),
-					bridge.getShareSetting(TrackerRole.RIGHT_HAND)
+					bridge.getShareSetting(TrackerRole.LEFT_CONTROLLER),
+					bridge.getShareSetting(TrackerRole.RIGHT_CONTROLLER)
 				);
 		}
 		return steamvrTrackerSettings;

server/core/src/main/java/dev/slimevr/protocol/rpc/settings/RPCSettingsHandler.kt

@@ -59,8 +59,8 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) {
 				bridge.changeShareSettings(TrackerRole.RIGHT_KNEE, req.steamVrTrackers().rightKnee())
 				bridge.changeShareSettings(TrackerRole.LEFT_ELBOW, req.steamVrTrackers().leftElbow())
 				bridge.changeShareSettings(TrackerRole.RIGHT_ELBOW, req.steamVrTrackers().rightElbow())
-				bridge.changeShareSettings(TrackerRole.LEFT_HAND, req.steamVrTrackers().leftHand())
-				bridge.changeShareSettings(TrackerRole.RIGHT_HAND, req.steamVrTrackers().rightHand())
+				bridge.changeShareSettings(TrackerRole.LEFT_CONTROLLER, req.steamVrTrackers().leftHand())
+				bridge.changeShareSettings(TrackerRole.RIGHT_CONTROLLER, req.steamVrTrackers().rightHand())
 				bridge.setAutomaticSharedTrackers(req.steamVrTrackers().automaticTrackerToggle())
 			}
 		}
@@ -128,8 +128,8 @@ class RPCSettingsHandler(var rpcHandler: RPCHandler, var api: ProtocolAPI) {
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_FOOT, trackers.feet())
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_ELBOW, trackers.elbows())
 				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_ELBOW, trackers.elbows())
-				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_HAND, trackers.hands())
-				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_HAND, trackers.hands())
+				vrcOSCConfig.setOSCTrackerRole(TrackerRole.LEFT_CONTROLLER, trackers.hands())
+				vrcOSCConfig.setOSCTrackerRole(TrackerRole.RIGHT_CONTROLLER, trackers.hands())
 			}
 			vrcOSCConfig.oscqueryEnabled = req.vrcOsc().oscqueryEnabled()
 

server/core/src/main/java/dev/slimevr/tracking/processor/BoneType.java → server/core/src/main/java/dev/slimevr/tracking/processor/BoneType.kt

@@ -1,14 +1,58 @@
-package dev.slimevr.tracking.processor;
+@file:Suppress("ktlint:standard:no-wildcard-imports")
 
-import solarxr_protocol.datatypes.BodyPart;
+package dev.slimevr.tracking.processor
 
+import dev.slimevr.tracking.processor.BoneType.*
+import solarxr_protocol.datatypes.BodyPart
+
+fun BoneType?.isLeftFinger(): Boolean {
+	this?.let {
+		return it == LEFT_THUMB_METACARPAL ||
+			it == LEFT_THUMB_PROXIMAL ||
+			it == LEFT_THUMB_DISTAL ||
+			it == LEFT_INDEX_PROXIMAL ||
+			it == LEFT_INDEX_INTERMEDIATE ||
+			it == LEFT_INDEX_DISTAL ||
+			it == LEFT_MIDDLE_PROXIMAL ||
+			it == LEFT_MIDDLE_INTERMEDIATE ||
+			it == LEFT_MIDDLE_DISTAL ||
+			it == LEFT_RING_PROXIMAL ||
+			it == LEFT_RING_INTERMEDIATE ||
+			it == LEFT_RING_DISTAL ||
+			it == LEFT_LITTLE_PROXIMAL ||
+			it == LEFT_LITTLE_INTERMEDIATE ||
+			it == LEFT_LITTLE_DISTAL
+	}
+	return false
+}
+
+fun BoneType?.isRightFinger(): Boolean {
+	this?.let {
+		return it == RIGHT_THUMB_METACARPAL ||
+			it == RIGHT_THUMB_PROXIMAL ||
+			it == RIGHT_THUMB_DISTAL ||
+			it == RIGHT_INDEX_PROXIMAL ||
+			it == RIGHT_INDEX_INTERMEDIATE ||
+			it == RIGHT_INDEX_DISTAL ||
+			it == RIGHT_MIDDLE_PROXIMAL ||
+			it == RIGHT_MIDDLE_INTERMEDIATE ||
+			it == RIGHT_MIDDLE_DISTAL ||
+			it == RIGHT_RING_PROXIMAL ||
+			it == RIGHT_RING_INTERMEDIATE ||
+			it == RIGHT_RING_DISTAL ||
+			it == RIGHT_LITTLE_PROXIMAL ||
+			it == RIGHT_LITTLE_INTERMEDIATE ||
+			it == RIGHT_LITTLE_DISTAL
+	}
+	return false
+}
 
 /**
  * Keys for all the bones in the skeleton.
  */
-public enum BoneType {
+enum class BoneType {
 	HEAD(BodyPart.HEAD),
-	HEAD_TRACKER(),
+	HEAD_TRACKER,
 	NECK(BodyPart.NECK),
 	UPPER_CHEST(BodyPart.UPPER_CHEST),
 	CHEST_TRACKER,
@@ -71,17 +115,21 @@ public enum BoneType {
 	RIGHT_RING_DISTAL(BodyPart.RIGHT_RING_DISTAL),
 	RIGHT_LITTLE_PROXIMAL(BodyPart.RIGHT_LITTLE_PROXIMAL),
 	RIGHT_LITTLE_INTERMEDIATE(BodyPart.RIGHT_LITTLE_INTERMEDIATE),
-	RIGHT_LITTLE_DISTAL(BodyPart.RIGHT_LITTLE_DISTAL);
+	RIGHT_LITTLE_DISTAL(BodyPart.RIGHT_LITTLE_DISTAL),
+	;
 
-	public static final BoneType[] values = values();
+	@JvmField
+	val bodyPart: Int
 
-	public final int bodyPart;
+	constructor() {
+		this.bodyPart = BodyPart.NONE
+	}
 
-	BoneType() {
-		this.bodyPart = BodyPart.NONE;
+	constructor(associatedBodyPart: Int) {
+		this.bodyPart = associatedBodyPart
 	}
 
-	BoneType(int associatedBodyPart) {
-		this.bodyPart = associatedBodyPart;
+	companion object {
+		val values: Array<BoneType> = entries.toTypedArray()
 	}
 }

server/core/src/main/java/dev/slimevr/tracking/processor/HumanPoseManager.kt

@@ -34,6 +34,7 @@ import kotlin.math.*
  */
 class HumanPoseManager(val server: VRServer?) {
 	val computedTrackers: MutableList<Tracker> = FastList()
+	val shareableFingerBones: MutableList<ShareableBone> = FastList()
 	private val onSkeletonUpdated: MutableList<Consumer<HumanSkeleton>> = FastList()
 	private val skeletonConfigManager = SkeletonConfigManager(true, this)
 
@@ -45,6 +46,7 @@ class HumanPoseManager(val server: VRServer?) {
 	// #region Constructors
 	init {
 		initializeComputedHumanPoseTracker()
+		initializeShareableFingerBones()
 	}
 
 	init {
@@ -230,6 +232,39 @@ class HumanPoseManager(val server: VRServer?) {
 		connectComputedHumanPoseTrackers()
 	}
 
+	private fun initializeShareableFingerBones() {
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_METACARPAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_THUMB_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_INDEX_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_MIDDLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_RING_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.LEFT_LITTLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_METACARPAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_THUMB_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_INDEX_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_MIDDLE_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_RING_DISTAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_PROXIMAL))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_INTERMEDIATE))
+		shareableFingerBones.add(ShareableBone(BoneType.RIGHT_LITTLE_DISTAL))
+	}
+
 	fun loadFromConfig(configManager: ConfigManager) {
 		skeletonConfigManager.loadFromConfig(configManager)
 	}