From 8fbb2e942846e09c7b2f4694c7f7bcb5ec3137de Mon Sep 17 00:00:00 2001 From: Tom Boullay Date: Sat, 2 May 2026 00:14:56 +0200 Subject: [PATCH] feat add left hand tracking glove model --- README.md | 1 + docs/technical/architecture.md | 4 +- docs/technical/hand-tracking.md | 13 +- public/models/gant_l/model.gltf | 2 +- .../handTracking/HandTrackingLeftGlove.tsx | 130 ++++++++++++++++++ .../ui/debug/HandTrackingDebugPanel.tsx | 7 +- src/data/docs/docsTranslations.ts | 3 +- src/world/World.tsx | 2 + 8 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 src/components/three/handTracking/HandTrackingLeftGlove.tsx diff --git a/README.md b/README.md index 6abc3be..d36be11 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ la-fabrik/ ├── components/ │ ├── three/ # Shared R3F components by domain │ │ ├── gameplay/ # Core repair gameplay prototype + │ │ ├── handTracking/ # R3F hand tracking debug models │ │ ├── interaction/ # Trigger, grab, focus wrappers │ │ ├── models/ # GLTF model components │ │ └── world/ # Environment-specific 3D objects diff --git a/docs/technical/architecture.md b/docs/technical/architecture.md index aea9388..ec7c194 100644 --- a/docs/technical/architecture.md +++ b/docs/technical/architecture.md @@ -41,7 +41,8 @@ This document describes the code that exists today in the repository. - `src/components/debug/DebugPerf.tsx` lazily mounts `r3f-perf` in debug mode. - `src/components/ui/debug/DebugOverlayLayout.tsx` mounts the compact HTML debug overlay when enabled from `lil-gui`. - `src/components/ui/debug/GameStateDebugPanel.tsx` exposes current game state, main/sub-state switching, previous/next step controls, and reset. -- `src/components/ui/debug/HandTrackingDebugPanel.tsx` shows hand tracking status, usage, model-loaded placeholder, hand count, and fist state while hand tracking is active. +- `src/components/ui/debug/HandTrackingDebugPanel.tsx` shows hand tracking status, usage, loaded glove model, hand count, and fist state while hand tracking is active. +- `src/components/three/handTracking/HandTrackingLeftGlove.tsx` places the rigged `gant_l` model on the detected left hand in the debug physics scene. - `src/components/debug/scene/DebugHelpers.tsx` mounts debug helpers. - `src/components/debug/scene/DebugCameraControls.tsx` mounts the free debug camera. - `lil-gui` global debug controls include camera mode, scene mode, `R3F Perf`, and `Debug Overlay`; interaction-specific controls live in the `Interaction` folder. @@ -50,6 +51,7 @@ This document describes the code that exists today in the repository. - `src/components/three/models/` contains reusable model helpers such as `ExplodableModel`. - `src/components/three/interaction/` contains reusable interaction wrappers such as `InteractableObject`, `TriggerObject`, and `GrabbableObject`. +- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the left glove overlay. - `src/components/three/gameplay/` contains the current core repair gameplay prototype: the repair case, repair game zone, and module slots. - `src/components/three/world/` contains reusable world/environment objects such as `SkyModel`. diff --git a/docs/technical/hand-tracking.md b/docs/technical/hand-tracking.md index b5c0f31..0bf28d1 100644 --- a/docs/technical/hand-tracking.md +++ b/docs/technical/hand-tracking.md @@ -16,6 +16,7 @@ The feature is scoped to the debug physics scene rather than production gameplay 4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`. 5. React stores the latest snapshot in the hand tracking provider. 6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects. +7. `HandTrackingLeftGlove` reads the same snapshot and places the rigged `gant_l` model on the detected left hand in the debug physics scene. ## Activation Rules @@ -104,12 +105,19 @@ The final hold distance is clamped between the configured grab minimum and maxim The current debug UI includes: -- `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, model-loaded placeholder, server state, hand count, and fist state +- `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, loaded glove model, server state, hand count, and fist state - `HandTrackingVisualizer` for the SVG landmark wireframe +- `HandTrackingLeftGlove` for the left-hand `gant_l` model in the R3F scene - `r3f-perf` for render performance - `lil-gui` for scene, camera, lighting, interaction, and grab controls -The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` is currently hardcoded to `none` until model-loading information is wired into the hand tracking flow. The hand wireframe is also HTML/SVG, not a 3D hand model. +The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays `gant_l` when a left hand is detected, otherwise `none`. The hand wireframe is also HTML/SVG, not a 3D hand model. + +## Left Glove Model + +The current left glove MVP uses `public/models/gant_l/model.gltf`, which contains a GLTF skin and armature. For now the model is positioned, oriented, and scaled as a whole from palm landmarks instead of driving individual finger bones. + +The right hand is intentionally ignored in this MVP. The available right-hand models are static in the current assets and are not mapped to MediaPipe bones yet. ## Known Limitations @@ -118,3 +126,4 @@ The hand tracking debug panel is a compact HTML grid outside the canvas. `Model - The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider. - There is no smoothing layer for hand position or depth yet. - The hand visualization is an SVG landmark wireframe. +- The left glove follows the palm as a whole; finger-by-finger bone animation still requires a verified landmark-to-bone mapping and smoothing. diff --git a/public/models/gant_l/model.gltf b/public/models/gant_l/model.gltf index da03db6..06919b0 100644 --- a/public/models/gant_l/model.gltf +++ b/public/models/gant_l/model.gltf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:01cdac724dfe936d112bd19202694a16196f54b6726b9e7e4e5102235fd41980 +oid sha256:cb31d7f73070f30f152c974afe0557bbd4c8b1468a5247c71eaf82afd6cc67bb size 10303 diff --git a/src/components/three/handTracking/HandTrackingLeftGlove.tsx b/src/components/three/handTracking/HandTrackingLeftGlove.tsx new file mode 100644 index 0000000..544639e --- /dev/null +++ b/src/components/three/handTracking/HandTrackingLeftGlove.tsx @@ -0,0 +1,130 @@ +import { useMemo, useRef } from "react"; +import { useFrame, useThree } from "@react-three/fiber"; +import { useGLTF } from "@react-three/drei"; +import * as THREE from "three"; +import { clone } from "three/addons/utils/SkeletonUtils.js"; +import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; +import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF"; +import type { HandTrackingLandmark } from "@/types/handTracking/handTracking"; + +const LEFT_GLOVE_MODEL_URL = "/models/gant_l/model.gltf"; +const HAND_SPACE_DISTANCE = 2.4; +const HAND_DEPTH_SCALE = 0.45; +const GLOVE_SCALE = 0.34; + +const _cameraPosition = new THREE.Vector3(); +const _direction = new THREE.Vector3(); +const _xAxis = new THREE.Vector3(); +const _yAxis = new THREE.Vector3(); +const _zAxis = new THREE.Vector3(); +const _matrix = new THREE.Matrix4(); +const _targetQuaternion = new THREE.Quaternion(); +const _targetPosition = new THREE.Vector3(); +const _wristPosition = new THREE.Vector3(); +const _indexPosition = new THREE.Vector3(); +const _middlePosition = new THREE.Vector3(); +const _ringPosition = new THREE.Vector3(); +const _pinkyPosition = new THREE.Vector3(); + +function landmarkToWorldPoint( + landmark: HandTrackingLandmark, + camera: THREE.Camera, + target: THREE.Vector3, +): THREE.Vector3 { + _cameraPosition.setFromMatrixPosition(camera.matrixWorld); + target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5); + target.unproject(camera); + + _direction.copy(target).sub(_cameraPosition).normalize(); + target + .copy(_cameraPosition) + .addScaledVector( + _direction, + HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE, + ); + + return target; +} + +function isLeftHand(handedness: string): boolean { + return handedness.toLowerCase() === "left"; +} + +export function HandTrackingLeftGlove(): React.JSX.Element | null { + const groupRef = useRef(null); + const { camera } = useThree(); + const { hands } = useHandTrackingSnapshot(); + const gltf = useLoggedGLTF(LEFT_GLOVE_MODEL_URL, { + scope: "HandTrackingLeftGlove", + scale: GLOVE_SCALE, + }); + const gloveScene = useMemo(() => clone(gltf.scene), [gltf.scene]); + + const leftHand = hands.find((hand) => isLeftHand(hand.handedness)); + + useFrame((_, delta) => { + const group = groupRef.current; + const hand = hands.find((candidate) => isLeftHand(candidate.handedness)); + + if (!group || !hand || hand.landmarks.length < 21) { + if (group) group.visible = false; + return; + } + + group.visible = true; + + const wrist = hand.landmarks[0]; + const indexMcp = hand.landmarks[5]; + const middleMcp = hand.landmarks[9]; + const ringMcp = hand.landmarks[13]; + const pinkyMcp = hand.landmarks[17]; + + if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) { + group.visible = false; + return; + } + + landmarkToWorldPoint(wrist, camera, _wristPosition); + landmarkToWorldPoint(indexMcp, camera, _indexPosition); + landmarkToWorldPoint(middleMcp, camera, _middlePosition); + landmarkToWorldPoint(ringMcp, camera, _ringPosition); + landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition); + + _targetPosition + .copy(_wristPosition) + .add(_indexPosition) + .add(_middlePosition) + .add(_ringPosition) + .add(_pinkyPosition) + .multiplyScalar(0.2); + + _yAxis.copy(_middlePosition).sub(_wristPosition).normalize(); + _xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize(); + _zAxis.crossVectors(_xAxis, _yAxis).normalize(); + + if ( + _xAxis.lengthSq() === 0 || + _yAxis.lengthSq() === 0 || + _zAxis.lengthSq() === 0 + ) { + return; + } + + _xAxis.crossVectors(_yAxis, _zAxis).normalize(); + _matrix.makeBasis(_xAxis, _yAxis, _zAxis); + _targetQuaternion.setFromRotationMatrix(_matrix); + + group.position.lerp(_targetPosition, Math.min(1, delta * 18)); + group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18)); + + const palmLength = _wristPosition.distanceTo(_middlePosition); + const scale = palmLength * GLOVE_SCALE; + group.scale.setScalar(scale); + }); + + if (!leftHand) return null; + + return ; +} + +useGLTF.preload(LEFT_GLOVE_MODEL_URL); diff --git a/src/components/ui/debug/HandTrackingDebugPanel.tsx b/src/components/ui/debug/HandTrackingDebugPanel.tsx index d156bbe..df2f9b5 100644 --- a/src/components/ui/debug/HandTrackingDebugPanel.tsx +++ b/src/components/ui/debug/HandTrackingDebugPanel.tsx @@ -21,6 +21,11 @@ export function HandTrackingDebugPanel(): React.JSX.Element | null { } const fist = hands.some((hand) => hand.isFist); + const modelLoaded = hands.some( + (hand) => hand.handedness.toLowerCase() === "left", + ) + ? "gant_l" + : "none"; return (
Model loaded
-
none
+
{modelLoaded}
{serverStatus ? (
diff --git a/src/data/docs/docsTranslations.ts b/src/data/docs/docsTranslations.ts index 05064d3..88dd675 100644 --- a/src/data/docs/docsTranslations.ts +++ b/src/data/docs/docsTranslations.ts @@ -124,7 +124,8 @@ Ce document décrit le code réellement présent aujourd'hui dans le dépôt. - \`src/components/debug/DebugPerf.tsx\` monte \`r3f-perf\` en lazy uniquement en mode debug. - \`src/components/ui/debug/DebugOverlayLayout.tsx\` monte l'overlay HTML debug compact quand il est activé depuis \`lil-gui\`. - \`src/components/ui/debug/GameStateDebugPanel.tsx\` expose l'état de jeu courant, le changement de main/sub-state, les contrôles previous/next step et le reset. -- \`src/components/ui/debug/HandTrackingDebugPanel.tsx\` affiche le statut hand tracking, l'usage, le placeholder de modèle chargé, le nombre de mains et l'état fist pendant l'activation du hand tracking. +- \`src/components/ui/debug/HandTrackingDebugPanel.tsx\` affiche le statut hand tracking, l'usage, le modèle de gant chargé, le nombre de mains et l'état fist pendant l'activation du hand tracking. +- \`src/components/three/handTracking/HandTrackingLeftGlove.tsx\` place le modèle riggé \`gant_l\` sur la main gauche détectée dans la scène physics debug. - \`src/components/debug/scene/DebugHelpers.tsx\` monte les helpers debug. - \`src/components/debug/scene/DebugCameraControls.tsx\` monte la caméra libre debug. - Les contrôles globaux \`lil-gui\` incluent camera mode, scene mode, \`R3F Perf\` et \`Debug Overlay\`; les contrôles d'interaction vivent dans le dossier \`Interaction\`. diff --git a/src/world/World.tsx b/src/world/World.tsx index 5a772e2..5ff5a7a 100644 --- a/src/world/World.tsx +++ b/src/world/World.tsx @@ -8,6 +8,7 @@ import { useCameraMode } from "@/hooks/debug/useCameraMode"; import { useSceneMode } from "@/hooks/debug/useSceneMode"; import { DebugCameraControls } from "@/components/debug/scene/DebugCameraControls"; import { DebugHelpers } from "@/components/debug/scene/DebugHelpers"; +import { HandTrackingLeftGlove } from "@/components/three/handTracking/HandTrackingLeftGlove"; import { Environment } from "@/world/Environment"; import { GameMusic } from "@/world/GameMusic"; import { Lighting } from "@/world/Lighting"; @@ -30,6 +31,7 @@ export function World(): React.JSX.Element { + {sceneMode === "physics" ? : null} {cameraMode === "debug" ? : null} {sceneMode === "game" ? (