math-pixel/La-Fabrik
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix hand tracking glove rendering

Browse Source
This commit is contained in:
Tom Boullay
2026-05-02 11:32:00 +02:00
parent bdc06f772f
commit fe662ebe7d
16 changed files with 319 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/three/handTracking/HandTrackingGlove.tsx
+235
View File
@@ -0,0 +1,235 @@
import type { ReactNode } from "react";
import { Component, useEffect, useMemo, useRef } from "react";
import { useFrame, useThree } from "@react-three/fiber";
import { useGLTF } from "@react-three/drei";
import * as THREE from "three";
import { clone } from "three/addons/utils/SkeletonUtils.js";
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
import {
useHandTrackingGloveStatus,
type HandTrackingGloveHandedness,
} from "@/hooks/handTracking/useHandTrackingGloveStatus";
import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
import { logModelLoadError } from "@/utils/three/modelLoadLogger";
const GLOVE_MODEL_PATHS: Record<HandTrackingGloveHandedness, string> = {
left: "/models/gant_l/model.gltf",
right: "/models/gant_r/model.gltf",
};
const GLOVE_ROOT_NODE_NAMES: Record<HandTrackingGloveHandedness, string> = {
left: "Hand_l",
right: "Hand_r",
};
const HAND_SPACE_DISTANCE = 2.4;
const HAND_DEPTH_SCALE = 0.45;
const GLOVE_SCALE = 0.17;
const _cameraPosition = new THREE.Vector3();
const _direction = new THREE.Vector3();
const _xAxis = new THREE.Vector3();
const _yAxis = new THREE.Vector3();
const _zAxis = new THREE.Vector3();
const _matrix = new THREE.Matrix4();
const _targetQuaternion = new THREE.Quaternion();
const _targetPosition = new THREE.Vector3();
const _wristPosition = new THREE.Vector3();
const _indexPosition = new THREE.Vector3();
const _middlePosition = new THREE.Vector3();
const _ringPosition = new THREE.Vector3();
const _pinkyPosition = new THREE.Vector3();
interface HandTrackingGloveProps {
handedness: HandTrackingGloveHandedness;
}
interface HandTrackingGloveErrorBoundaryProps {
children: ReactNode;
handedness: HandTrackingGloveHandedness;
modelPath: string;
}
class HandTrackingGloveErrorBoundary extends Component<
HandTrackingGloveErrorBoundaryProps,
{ hasError: boolean }
> {
constructor(props: HandTrackingGloveErrorBoundaryProps) {
super(props);
this.state = { hasError: false };
}
static getDerivedStateFromError(): { hasError: boolean } {
return { hasError: true };
}
componentDidCatch(error: Error): void {
useHandTrackingGloveStatus
.getState()
.setGloveStatus(this.props.handedness, "error");
logModelLoadError(
{
modelPath: this.props.modelPath,
scope: `HandTrackingGlove.${this.props.handedness}`,
scale: GLOVE_SCALE,
},
error,
);
}
render(): ReactNode {
if (this.state.hasError) return null;
return this.props.children;
}
}
function landmarkToWorldPoint(
landmark: HandTrackingLandmark,
camera: THREE.Camera,
target: THREE.Vector3,
): THREE.Vector3 {
_cameraPosition.setFromMatrixPosition(camera.matrixWorld);
target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5);
target.unproject(camera);
_direction.copy(target).sub(_cameraPosition).normalize();
target
.copy(_cameraPosition)
.addScaledVector(
_direction,
HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE,
);
return target;
}
function matchesHandedness(
handHandedness: string,
targetHandedness: HandTrackingGloveHandedness,
): boolean {
return handHandedness.toLowerCase() === targetHandedness;
}
function HandTrackingGloveModel({
handedness,
}: HandTrackingGloveProps): React.JSX.Element | null {
const groupRef = useRef<THREE.Group>(null);
const { camera } = useThree();
const { hands } = useHandTrackingSnapshot();
const setGloveStatus = useHandTrackingGloveStatus(
(state) => state.setGloveStatus,
);
const modelPath = GLOVE_MODEL_PATHS[handedness];
const gltf = useLoggedGLTF(modelPath, {
scope: `HandTrackingGlove.${handedness}`,
scale: GLOVE_SCALE,
});
const gloveScene = useMemo(() => {
const rootNode = gltf.scene.getObjectByName(
GLOVE_ROOT_NODE_NAMES[handedness],
);
if (!rootNode) {
throw new Error(
`Missing glove root node ${GLOVE_ROOT_NODE_NAMES[handedness]}`,
);
}
return clone(rootNode);
}, [gltf.scene, handedness]);
const hand = hands.find((candidate) =>
matchesHandedness(candidate.handedness, handedness),
);
useEffect(() => {
setGloveStatus(handedness, "loaded");
}, [handedness, setGloveStatus]);
useFrame((_, delta) => {
const group = groupRef.current;
const trackedHand = hands.find((candidate) =>
matchesHandedness(candidate.handedness, handedness),
);
if (!group || !trackedHand || trackedHand.landmarks.length < 21) {
if (group) group.visible = false;
return;
}
group.visible = true;
const wrist = trackedHand.landmarks[0];
const indexMcp = trackedHand.landmarks[5];
const middleMcp = trackedHand.landmarks[9];
const ringMcp = trackedHand.landmarks[13];
const pinkyMcp = trackedHand.landmarks[17];
if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) {
group.visible = false;
return;
}
landmarkToWorldPoint(wrist, camera, _wristPosition);
landmarkToWorldPoint(indexMcp, camera, _indexPosition);
landmarkToWorldPoint(middleMcp, camera, _middlePosition);
landmarkToWorldPoint(ringMcp, camera, _ringPosition);
landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition);
_targetPosition
.copy(_wristPosition)
.add(_indexPosition)
.add(_middlePosition)
.add(_ringPosition)
.add(_pinkyPosition)
.multiplyScalar(0.2);
_yAxis.copy(_middlePosition).sub(_wristPosition).normalize();
_xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize();
_zAxis.crossVectors(_xAxis, _yAxis).normalize();
if (
_xAxis.lengthSq() === 0 ||
_yAxis.lengthSq() === 0 ||
_zAxis.lengthSq() === 0
) {
return;
}
_xAxis.crossVectors(_yAxis, _zAxis).normalize();
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
_targetQuaternion.setFromRotationMatrix(_matrix);
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
const palmLength = _wristPosition.distanceTo(_middlePosition);
const scale = palmLength * GLOVE_SCALE;
group.scale.setScalar(scale);
});
if (!hand) return null;
return <primitive ref={groupRef} object={gloveScene} />;
}
export function HandTrackingGlove({
handedness,
}: HandTrackingGloveProps): React.JSX.Element {
const modelPath = GLOVE_MODEL_PATHS[handedness];
return (
<HandTrackingGloveErrorBoundary
handedness={handedness}
modelPath={modelPath}
>
<HandTrackingGloveModel handedness={handedness} />
</HandTrackingGloveErrorBoundary>
);
}
useGLTF.preload(GLOVE_MODEL_PATHS.left);
useGLTF.preload(GLOVE_MODEL_PATHS.right);
src/components/three/handTracking/HandTrackingLeftGlove.tsx
-130
View File
@@ -1,130 +0,0 @@
import { useMemo, useRef } from "react";
import { useFrame, useThree } from "@react-three/fiber";
import { useGLTF } from "@react-three/drei";
import * as THREE from "three";
import { clone } from "three/addons/utils/SkeletonUtils.js";
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
const LEFT_GLOVE_MODEL_URL = "/models/gant_l/model.gltf";
const HAND_SPACE_DISTANCE = 2.4;
const HAND_DEPTH_SCALE = 0.45;
const GLOVE_SCALE = 0.34;
const _cameraPosition = new THREE.Vector3();
const _direction = new THREE.Vector3();
const _xAxis = new THREE.Vector3();
const _yAxis = new THREE.Vector3();
const _zAxis = new THREE.Vector3();
const _matrix = new THREE.Matrix4();
const _targetQuaternion = new THREE.Quaternion();
const _targetPosition = new THREE.Vector3();
const _wristPosition = new THREE.Vector3();
const _indexPosition = new THREE.Vector3();
const _middlePosition = new THREE.Vector3();
const _ringPosition = new THREE.Vector3();
const _pinkyPosition = new THREE.Vector3();
function landmarkToWorldPoint(
landmark: HandTrackingLandmark,
camera: THREE.Camera,
target: THREE.Vector3,
): THREE.Vector3 {
_cameraPosition.setFromMatrixPosition(camera.matrixWorld);
target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5);
target.unproject(camera);
_direction.copy(target).sub(_cameraPosition).normalize();
target
.copy(_cameraPosition)
.addScaledVector(
_direction,
HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE,
);
return target;
}
function isLeftHand(handedness: string): boolean {
return handedness.toLowerCase() === "left";
}
export function HandTrackingLeftGlove(): React.JSX.Element | null {
const groupRef = useRef<THREE.Group>(null);
const { camera } = useThree();
const { hands } = useHandTrackingSnapshot();
const gltf = useLoggedGLTF(LEFT_GLOVE_MODEL_URL, {
scope: "HandTrackingLeftGlove",
scale: GLOVE_SCALE,
});
const gloveScene = useMemo(() => clone(gltf.scene), [gltf.scene]);
const leftHand = hands.find((hand) => isLeftHand(hand.handedness));
useFrame((_, delta) => {
const group = groupRef.current;
const hand = hands.find((candidate) => isLeftHand(candidate.handedness));
if (!group || !hand || hand.landmarks.length < 21) {
if (group) group.visible = false;
return;
}
group.visible = true;
const wrist = hand.landmarks[0];
const indexMcp = hand.landmarks[5];
const middleMcp = hand.landmarks[9];
const ringMcp = hand.landmarks[13];
const pinkyMcp = hand.landmarks[17];
if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) {
group.visible = false;
return;
}
landmarkToWorldPoint(wrist, camera, _wristPosition);
landmarkToWorldPoint(indexMcp, camera, _indexPosition);
landmarkToWorldPoint(middleMcp, camera, _middlePosition);
landmarkToWorldPoint(ringMcp, camera, _ringPosition);
landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition);
_targetPosition
.copy(_wristPosition)
.add(_indexPosition)
.add(_middlePosition)
.add(_ringPosition)
.add(_pinkyPosition)
.multiplyScalar(0.2);
_yAxis.copy(_middlePosition).sub(_wristPosition).normalize();
_xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize();
_zAxis.crossVectors(_xAxis, _yAxis).normalize();
if (
_xAxis.lengthSq() === 0 ||
_yAxis.lengthSq() === 0 ||
_zAxis.lengthSq() === 0
) {
return;
}
_xAxis.crossVectors(_yAxis, _zAxis).normalize();
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
_targetQuaternion.setFromRotationMatrix(_matrix);
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
const palmLength = _wristPosition.distanceTo(_middlePosition);
const scale = palmLength * GLOVE_SCALE;
group.scale.setScalar(scale);
});
if (!leftHand) return null;
return <primitive ref={groupRef} object={gloveScene} />;
}
useGLTF.preload(LEFT_GLOVE_MODEL_URL);
src/components/ui/HandTrackingVisualizer.tsx
+6 -1
View File
@@ -1,4 +1,5 @@
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus";
const HAND_CONNECTIONS: Array<[number, number]> = [
[0, 1],
@@ -26,8 +27,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [
export function HandTrackingVisualizer(): React.JSX.Element | null {
const { hands, status } = useHandTrackingSnapshot();
const gloves = useHandTrackingGloveStatus((state) => state.gloves);
const shouldShowSvgFallback = Object.values(gloves).some(
(gloveStatus) => gloveStatus === "error" || gloveStatus === "idle",
);
if (status === "idle" || hands.length === 0) {
if (status === "idle" || hands.length === 0 || !shouldShowSvgFallback) {
return null;
}
src/components/ui/debug/HandTrackingDebugPanel.tsx
+13 -7
View File
@@ -1,4 +1,5 @@
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus";
import type { HandTrackingStatus } from "@/types/handTracking/handTracking";
const STATUS_LABELS: Record<HandTrackingStatus, string> = {
@@ -15,22 +16,23 @@ const STATUS_LABELS: Record<HandTrackingStatus, string> = {
export function HandTrackingDebugPanel(): React.JSX.Element | null {
const { hands, status, usageStatus, serverStatus, error } =
useHandTrackingSnapshot();
const gloves = useHandTrackingGloveStatus((state) => state.gloves);
if (status === "idle") {
return null;
}
const fist = hands.some((hand) => hand.isFist);
const hasLeftHand = hands.some(
(hand) => hand.handedness.toLowerCase() === "left",
);
const hasRightHand = hands.some(
(hand) => hand.handedness.toLowerCase() === "right",
);
const modelLoaded =
[hasLeftHand ? "gant_l" : null, hasRightHand ? "gant_r" : null]
[
gloves.left === "loaded" ? "gant_l" : null,
gloves.right === "loaded" ? "gant_r" : null,
]
.filter(Boolean)
.join(", ") || "none";
const modelFallback = Object.values(gloves).some(
(gloveStatus) => gloveStatus === "error",
);
return (
<section
@@ -51,6 +53,10 @@ export function HandTrackingDebugPanel(): React.JSX.Element | null {
<dt>Model loaded</dt>
<dd>{modelLoaded}</dd>
</div>
<div>
<dt>SVG fallback</dt>
<dd>{modelFallback ? "yes" : "no"}</dd>
</div>
{serverStatus ? (
<div>
<dt>Server</dt>