From 0b950a4557c7fb6ac40561274ec37c9f8c952383 Mon Sep 17 00:00:00 2001 From: Tom Boullay Date: Sat, 2 May 2026 11:32:00 +0200 Subject: [PATCH] fix hand tracking glove rendering --- docs/technical/architecture.md | 4 +- docs/technical/hand-tracking.md | 18 +- public/models/gant_l/model.gltf | 2 +- public/models/gant_r/galet_baseColor.png | 3 + public/models/gant_r/galet_normal.png | 3 + .../galet_occlusionRoughnessMetallic.png | 3 + public/models/gant_r/model.bin | 3 + public/models/gant_r/model.glb | 3 + public/models/gant_r/model.gltf | 4 +- .../three/handTracking/HandTrackingGlove.tsx | 235 ++++++++++++++++++ .../handTracking/HandTrackingLeftGlove.tsx | 130 ---------- src/components/ui/HandTrackingVisualizer.tsx | 7 +- .../ui/debug/HandTrackingDebugPanel.tsx | 20 +- src/data/docs/docsTranslations.ts | 2 +- .../useHandTrackingGloveStatus.ts | 28 +++ src/world/World.tsx | 9 +- 16 files changed, 319 insertions(+), 155 deletions(-) create mode 100644 public/models/gant_r/galet_baseColor.png create mode 100644 public/models/gant_r/galet_normal.png create mode 100644 public/models/gant_r/galet_occlusionRoughnessMetallic.png create mode 100644 public/models/gant_r/model.bin create mode 100644 public/models/gant_r/model.glb create mode 100644 src/components/three/handTracking/HandTrackingGlove.tsx delete mode 100644 src/components/three/handTracking/HandTrackingLeftGlove.tsx create mode 100644 src/hooks/handTracking/useHandTrackingGloveStatus.ts diff --git a/docs/technical/architecture.md b/docs/technical/architecture.md index ec7c194..f8fdbda 100644 --- a/docs/technical/architecture.md +++ b/docs/technical/architecture.md @@ -42,7 +42,7 @@ This document describes the code that exists today in the repository. - `src/components/ui/debug/DebugOverlayLayout.tsx` mounts the compact HTML debug overlay when enabled from `lil-gui`. - `src/components/ui/debug/GameStateDebugPanel.tsx` exposes current game state, main/sub-state switching, previous/next step controls, and reset. - `src/components/ui/debug/HandTrackingDebugPanel.tsx` shows hand tracking status, usage, loaded glove model, hand count, and fist state while hand tracking is active. -- `src/components/three/handTracking/HandTrackingLeftGlove.tsx` places the rigged `gant_l` model on the detected left hand in the debug physics scene. +- `src/components/three/handTracking/HandTrackingGlove.tsx` places the rigged `gant_l` and `gant_r` models on detected hands in the debug physics scene. - `src/components/debug/scene/DebugHelpers.tsx` mounts debug helpers. - `src/components/debug/scene/DebugCameraControls.tsx` mounts the free debug camera. - `lil-gui` global debug controls include camera mode, scene mode, `R3F Perf`, and `Debug Overlay`; interaction-specific controls live in the `Interaction` folder. @@ -51,7 +51,7 @@ This document describes the code that exists today in the repository. - `src/components/three/models/` contains reusable model helpers such as `ExplodableModel`. - `src/components/three/interaction/` contains reusable interaction wrappers such as `InteractableObject`, `TriggerObject`, and `GrabbableObject`. -- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the left glove overlay. +- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the glove overlays. - `src/components/three/gameplay/` contains the current core repair gameplay prototype: the repair case, repair game zone, and module slots. - `src/components/three/world/` contains reusable world/environment objects such as `SkyModel`. diff --git a/docs/technical/hand-tracking.md b/docs/technical/hand-tracking.md index 0bf28d1..0335425 100644 --- a/docs/technical/hand-tracking.md +++ b/docs/technical/hand-tracking.md @@ -16,7 +16,7 @@ The feature is scoped to the debug physics scene rather than production gameplay 4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`. 5. React stores the latest snapshot in the hand tracking provider. 6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects. -7. `HandTrackingLeftGlove` reads the same snapshot and places the rigged `gant_l` model on the detected left hand in the debug physics scene. +7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands in the debug physics scene. ## Activation Rules @@ -106,18 +106,18 @@ The final hold distance is clamped between the configured grab minimum and maxim The current debug UI includes: - `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, loaded glove model, server state, hand count, and fist state -- `HandTrackingVisualizer` for the SVG landmark wireframe -- `HandTrackingLeftGlove` for the left-hand `gant_l` model in the R3F scene +- `HandTrackingVisualizer` for the SVG landmark wireframe fallback +- `HandTrackingGlove` for the left-hand `gant_l` and right-hand `gant_r` models in the R3F scene - `r3f-perf` for render performance - `lil-gui` for scene, camera, lighting, interaction, and grab controls -The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays `gant_l` when a left hand is detected, otherwise `none`. The hand wireframe is also HTML/SVG, not a 3D hand model. +The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays the successfully loaded glove models. The SVG hand wireframe is only a fallback while models are loading or if a glove model fails to load. -## Left Glove Model +## Glove Models -The current left glove MVP uses `public/models/gant_l/model.gltf`, which contains a GLTF skin and armature. For now the model is positioned, oriented, and scaled as a whole from palm landmarks instead of driving individual finger bones. +The current glove MVP uses `public/models/gant_l/model.gltf` and `public/models/gant_r/model.gltf`, which contain GLTF skins and armatures. For now each model is positioned, oriented, and scaled as a whole from palm landmarks instead of driving individual finger bones. -The right hand is intentionally ignored in this MVP. The available right-hand models are static in the current assets and are not mapped to MediaPipe bones yet. +The glove models are intentionally smaller than the raw SVG overlay so they do not dominate the camera view. ## Known Limitations @@ -125,5 +125,5 @@ The right hand is intentionally ignored in this MVP. The available right-hand mo - MediaPipe depth is relative and can be noisy. - The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider. - There is no smoothing layer for hand position or depth yet. -- The hand visualization is an SVG landmark wireframe. -- The left glove follows the palm as a whole; finger-by-finger bone animation still requires a verified landmark-to-bone mapping and smoothing. +- The SVG hand visualization is a fallback, not the primary display when glove models load correctly. +- Each glove follows the palm as a whole; finger-by-finger bone animation still requires a verified landmark-to-bone mapping and smoothing. diff --git a/public/models/gant_l/model.gltf b/public/models/gant_l/model.gltf index 06919b0..da03db6 100644 --- a/public/models/gant_l/model.gltf +++ b/public/models/gant_l/model.gltf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb31d7f73070f30f152c974afe0557bbd4c8b1468a5247c71eaf82afd6cc67bb +oid sha256:01cdac724dfe936d112bd19202694a16196f54b6726b9e7e4e5102235fd41980 size 10303 diff --git a/public/models/gant_r/galet_baseColor.png b/public/models/gant_r/galet_baseColor.png new file mode 100644 index 0000000..33f9861 --- /dev/null +++ b/public/models/gant_r/galet_baseColor.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8230f08a07b4a5cad22e68c563192a664741db2eaa9c6ac481b7b6e2c1f00a5 +size 492994 diff --git a/public/models/gant_r/galet_normal.png b/public/models/gant_r/galet_normal.png new file mode 100644 index 0000000..d8f29af --- /dev/null +++ b/public/models/gant_r/galet_normal.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:044bed9c465d55e5be571862e46aaf5f48172a8572665286068bd005847a5f17 +size 2613811 diff --git a/public/models/gant_r/galet_occlusionRoughnessMetallic.png b/public/models/gant_r/galet_occlusionRoughnessMetallic.png new file mode 100644 index 0000000..f262ab4 --- /dev/null +++ b/public/models/gant_r/galet_occlusionRoughnessMetallic.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:137ce434f528bd56a3a5eabc339173c5e8fb4f9306fb0ca624d996ee90b8d1f8 +size 16902 diff --git a/public/models/gant_r/model.bin b/public/models/gant_r/model.bin new file mode 100644 index 0000000..ddaf087 --- /dev/null +++ b/public/models/gant_r/model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94caa72fe32b36c373174f83aad4658df02ddd55de862b5f62357e879597b592 +size 1384136 diff --git a/public/models/gant_r/model.glb b/public/models/gant_r/model.glb new file mode 100644 index 0000000..bd52ef8 --- /dev/null +++ b/public/models/gant_r/model.glb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09384178466e1dc10ae2db681655117d34ee8010a06a469cc07c9078bfaf512b +size 7815820 diff --git a/public/models/gant_r/model.gltf b/public/models/gant_r/model.gltf index 655b7b3..3618596 100644 --- a/public/models/gant_r/model.gltf +++ b/public/models/gant_r/model.gltf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fa8dc2473ec95c3548f03c2f094328302f4e1ff4a6be7c94d6ac941392d912c4 -size 3607 +oid sha256:c82eb9596e0829193b8c860670ff9cad959dfcced8d17183c2347346870d267b +size 31499 diff --git a/src/components/three/handTracking/HandTrackingGlove.tsx b/src/components/three/handTracking/HandTrackingGlove.tsx new file mode 100644 index 0000000..f124239 --- /dev/null +++ b/src/components/three/handTracking/HandTrackingGlove.tsx @@ -0,0 +1,235 @@ +import type { ReactNode } from "react"; +import { Component, useEffect, useMemo, useRef } from "react"; +import { useFrame, useThree } from "@react-three/fiber"; +import { useGLTF } from "@react-three/drei"; +import * as THREE from "three"; +import { clone } from "three/addons/utils/SkeletonUtils.js"; +import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; +import { + useHandTrackingGloveStatus, + type HandTrackingGloveHandedness, +} from "@/hooks/handTracking/useHandTrackingGloveStatus"; +import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF"; +import type { HandTrackingLandmark } from "@/types/handTracking/handTracking"; +import { logModelLoadError } from "@/utils/three/modelLoadLogger"; + +const GLOVE_MODEL_PATHS: Record = { + left: "/models/gant_l/model.gltf", + right: "/models/gant_r/model.gltf", +}; + +const GLOVE_ROOT_NODE_NAMES: Record = { + left: "Hand_l", + right: "Hand_r", +}; + +const HAND_SPACE_DISTANCE = 2.4; +const HAND_DEPTH_SCALE = 0.45; +const GLOVE_SCALE = 0.17; + +const _cameraPosition = new THREE.Vector3(); +const _direction = new THREE.Vector3(); +const _xAxis = new THREE.Vector3(); +const _yAxis = new THREE.Vector3(); +const _zAxis = new THREE.Vector3(); +const _matrix = new THREE.Matrix4(); +const _targetQuaternion = new THREE.Quaternion(); +const _targetPosition = new THREE.Vector3(); +const _wristPosition = new THREE.Vector3(); +const _indexPosition = new THREE.Vector3(); +const _middlePosition = new THREE.Vector3(); +const _ringPosition = new THREE.Vector3(); +const _pinkyPosition = new THREE.Vector3(); + +interface HandTrackingGloveProps { + handedness: HandTrackingGloveHandedness; +} + +interface HandTrackingGloveErrorBoundaryProps { + children: ReactNode; + handedness: HandTrackingGloveHandedness; + modelPath: string; +} + +class HandTrackingGloveErrorBoundary extends Component< + HandTrackingGloveErrorBoundaryProps, + { hasError: boolean } +> { + constructor(props: HandTrackingGloveErrorBoundaryProps) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(): { hasError: boolean } { + return { hasError: true }; + } + + componentDidCatch(error: Error): void { + useHandTrackingGloveStatus + .getState() + .setGloveStatus(this.props.handedness, "error"); + + logModelLoadError( + { + modelPath: this.props.modelPath, + scope: `HandTrackingGlove.${this.props.handedness}`, + scale: GLOVE_SCALE, + }, + error, + ); + } + + render(): ReactNode { + if (this.state.hasError) return null; + + return this.props.children; + } +} + +function landmarkToWorldPoint( + landmark: HandTrackingLandmark, + camera: THREE.Camera, + target: THREE.Vector3, +): THREE.Vector3 { + _cameraPosition.setFromMatrixPosition(camera.matrixWorld); + target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5); + target.unproject(camera); + + _direction.copy(target).sub(_cameraPosition).normalize(); + target + .copy(_cameraPosition) + .addScaledVector( + _direction, + HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE, + ); + + return target; +} + +function matchesHandedness( + handHandedness: string, + targetHandedness: HandTrackingGloveHandedness, +): boolean { + return handHandedness.toLowerCase() === targetHandedness; +} + +function HandTrackingGloveModel({ + handedness, +}: HandTrackingGloveProps): React.JSX.Element | null { + const groupRef = useRef(null); + const { camera } = useThree(); + const { hands } = useHandTrackingSnapshot(); + const setGloveStatus = useHandTrackingGloveStatus( + (state) => state.setGloveStatus, + ); + const modelPath = GLOVE_MODEL_PATHS[handedness]; + const gltf = useLoggedGLTF(modelPath, { + scope: `HandTrackingGlove.${handedness}`, + scale: GLOVE_SCALE, + }); + const gloveScene = useMemo(() => { + const rootNode = gltf.scene.getObjectByName( + GLOVE_ROOT_NODE_NAMES[handedness], + ); + + if (!rootNode) { + throw new Error( + `Missing glove root node ${GLOVE_ROOT_NODE_NAMES[handedness]}`, + ); + } + + return clone(rootNode); + }, [gltf.scene, handedness]); + + const hand = hands.find((candidate) => + matchesHandedness(candidate.handedness, handedness), + ); + + useEffect(() => { + setGloveStatus(handedness, "loaded"); + }, [handedness, setGloveStatus]); + + useFrame((_, delta) => { + const group = groupRef.current; + const trackedHand = hands.find((candidate) => + matchesHandedness(candidate.handedness, handedness), + ); + + if (!group || !trackedHand || trackedHand.landmarks.length < 21) { + if (group) group.visible = false; + return; + } + + group.visible = true; + + const wrist = trackedHand.landmarks[0]; + const indexMcp = trackedHand.landmarks[5]; + const middleMcp = trackedHand.landmarks[9]; + const ringMcp = trackedHand.landmarks[13]; + const pinkyMcp = trackedHand.landmarks[17]; + + if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) { + group.visible = false; + return; + } + + landmarkToWorldPoint(wrist, camera, _wristPosition); + landmarkToWorldPoint(indexMcp, camera, _indexPosition); + landmarkToWorldPoint(middleMcp, camera, _middlePosition); + landmarkToWorldPoint(ringMcp, camera, _ringPosition); + landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition); + + _targetPosition + .copy(_wristPosition) + .add(_indexPosition) + .add(_middlePosition) + .add(_ringPosition) + .add(_pinkyPosition) + .multiplyScalar(0.2); + + _yAxis.copy(_middlePosition).sub(_wristPosition).normalize(); + _xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize(); + _zAxis.crossVectors(_xAxis, _yAxis).normalize(); + + if ( + _xAxis.lengthSq() === 0 || + _yAxis.lengthSq() === 0 || + _zAxis.lengthSq() === 0 + ) { + return; + } + + _xAxis.crossVectors(_yAxis, _zAxis).normalize(); + _matrix.makeBasis(_xAxis, _yAxis, _zAxis); + _targetQuaternion.setFromRotationMatrix(_matrix); + + group.position.lerp(_targetPosition, Math.min(1, delta * 18)); + group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18)); + + const palmLength = _wristPosition.distanceTo(_middlePosition); + const scale = palmLength * GLOVE_SCALE; + group.scale.setScalar(scale); + }); + + if (!hand) return null; + + return ; +} + +export function HandTrackingGlove({ + handedness, +}: HandTrackingGloveProps): React.JSX.Element { + const modelPath = GLOVE_MODEL_PATHS[handedness]; + + return ( + + + + ); +} + +useGLTF.preload(GLOVE_MODEL_PATHS.left); +useGLTF.preload(GLOVE_MODEL_PATHS.right); diff --git a/src/components/three/handTracking/HandTrackingLeftGlove.tsx b/src/components/three/handTracking/HandTrackingLeftGlove.tsx deleted file mode 100644 index 544639e..0000000 --- a/src/components/three/handTracking/HandTrackingLeftGlove.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { useMemo, useRef } from "react"; -import { useFrame, useThree } from "@react-three/fiber"; -import { useGLTF } from "@react-three/drei"; -import * as THREE from "three"; -import { clone } from "three/addons/utils/SkeletonUtils.js"; -import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; -import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF"; -import type { HandTrackingLandmark } from "@/types/handTracking/handTracking"; - -const LEFT_GLOVE_MODEL_URL = "/models/gant_l/model.gltf"; -const HAND_SPACE_DISTANCE = 2.4; -const HAND_DEPTH_SCALE = 0.45; -const GLOVE_SCALE = 0.34; - -const _cameraPosition = new THREE.Vector3(); -const _direction = new THREE.Vector3(); -const _xAxis = new THREE.Vector3(); -const _yAxis = new THREE.Vector3(); -const _zAxis = new THREE.Vector3(); -const _matrix = new THREE.Matrix4(); -const _targetQuaternion = new THREE.Quaternion(); -const _targetPosition = new THREE.Vector3(); -const _wristPosition = new THREE.Vector3(); -const _indexPosition = new THREE.Vector3(); -const _middlePosition = new THREE.Vector3(); -const _ringPosition = new THREE.Vector3(); -const _pinkyPosition = new THREE.Vector3(); - -function landmarkToWorldPoint( - landmark: HandTrackingLandmark, - camera: THREE.Camera, - target: THREE.Vector3, -): THREE.Vector3 { - _cameraPosition.setFromMatrixPosition(camera.matrixWorld); - target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5); - target.unproject(camera); - - _direction.copy(target).sub(_cameraPosition).normalize(); - target - .copy(_cameraPosition) - .addScaledVector( - _direction, - HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE, - ); - - return target; -} - -function isLeftHand(handedness: string): boolean { - return handedness.toLowerCase() === "left"; -} - -export function HandTrackingLeftGlove(): React.JSX.Element | null { - const groupRef = useRef(null); - const { camera } = useThree(); - const { hands } = useHandTrackingSnapshot(); - const gltf = useLoggedGLTF(LEFT_GLOVE_MODEL_URL, { - scope: "HandTrackingLeftGlove", - scale: GLOVE_SCALE, - }); - const gloveScene = useMemo(() => clone(gltf.scene), [gltf.scene]); - - const leftHand = hands.find((hand) => isLeftHand(hand.handedness)); - - useFrame((_, delta) => { - const group = groupRef.current; - const hand = hands.find((candidate) => isLeftHand(candidate.handedness)); - - if (!group || !hand || hand.landmarks.length < 21) { - if (group) group.visible = false; - return; - } - - group.visible = true; - - const wrist = hand.landmarks[0]; - const indexMcp = hand.landmarks[5]; - const middleMcp = hand.landmarks[9]; - const ringMcp = hand.landmarks[13]; - const pinkyMcp = hand.landmarks[17]; - - if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) { - group.visible = false; - return; - } - - landmarkToWorldPoint(wrist, camera, _wristPosition); - landmarkToWorldPoint(indexMcp, camera, _indexPosition); - landmarkToWorldPoint(middleMcp, camera, _middlePosition); - landmarkToWorldPoint(ringMcp, camera, _ringPosition); - landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition); - - _targetPosition - .copy(_wristPosition) - .add(_indexPosition) - .add(_middlePosition) - .add(_ringPosition) - .add(_pinkyPosition) - .multiplyScalar(0.2); - - _yAxis.copy(_middlePosition).sub(_wristPosition).normalize(); - _xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize(); - _zAxis.crossVectors(_xAxis, _yAxis).normalize(); - - if ( - _xAxis.lengthSq() === 0 || - _yAxis.lengthSq() === 0 || - _zAxis.lengthSq() === 0 - ) { - return; - } - - _xAxis.crossVectors(_yAxis, _zAxis).normalize(); - _matrix.makeBasis(_xAxis, _yAxis, _zAxis); - _targetQuaternion.setFromRotationMatrix(_matrix); - - group.position.lerp(_targetPosition, Math.min(1, delta * 18)); - group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18)); - - const palmLength = _wristPosition.distanceTo(_middlePosition); - const scale = palmLength * GLOVE_SCALE; - group.scale.setScalar(scale); - }); - - if (!leftHand) return null; - - return ; -} - -useGLTF.preload(LEFT_GLOVE_MODEL_URL); diff --git a/src/components/ui/HandTrackingVisualizer.tsx b/src/components/ui/HandTrackingVisualizer.tsx index 6ce2f47..1da25e1 100644 --- a/src/components/ui/HandTrackingVisualizer.tsx +++ b/src/components/ui/HandTrackingVisualizer.tsx @@ -1,4 +1,5 @@ import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; +import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus"; const HAND_CONNECTIONS: Array<[number, number]> = [ [0, 1], @@ -26,8 +27,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [ export function HandTrackingVisualizer(): React.JSX.Element | null { const { hands, status } = useHandTrackingSnapshot(); + const gloves = useHandTrackingGloveStatus((state) => state.gloves); + const shouldShowSvgFallback = Object.values(gloves).some( + (gloveStatus) => gloveStatus === "error" || gloveStatus === "idle", + ); - if (status === "idle" || hands.length === 0) { + if (status === "idle" || hands.length === 0 || !shouldShowSvgFallback) { return null; } diff --git a/src/components/ui/debug/HandTrackingDebugPanel.tsx b/src/components/ui/debug/HandTrackingDebugPanel.tsx index 0822497..d4dedba 100644 --- a/src/components/ui/debug/HandTrackingDebugPanel.tsx +++ b/src/components/ui/debug/HandTrackingDebugPanel.tsx @@ -1,4 +1,5 @@ import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; +import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus"; import type { HandTrackingStatus } from "@/types/handTracking/handTracking"; const STATUS_LABELS: Record = { @@ -15,22 +16,23 @@ const STATUS_LABELS: Record = { export function HandTrackingDebugPanel(): React.JSX.Element | null { const { hands, status, usageStatus, serverStatus, error } = useHandTrackingSnapshot(); + const gloves = useHandTrackingGloveStatus((state) => state.gloves); if (status === "idle") { return null; } const fist = hands.some((hand) => hand.isFist); - const hasLeftHand = hands.some( - (hand) => hand.handedness.toLowerCase() === "left", - ); - const hasRightHand = hands.some( - (hand) => hand.handedness.toLowerCase() === "right", - ); const modelLoaded = - [hasLeftHand ? "gant_l" : null, hasRightHand ? "gant_r" : null] + [ + gloves.left === "loaded" ? "gant_l" : null, + gloves.right === "loaded" ? "gant_r" : null, + ] .filter(Boolean) .join(", ") || "none"; + const modelFallback = Object.values(gloves).some( + (gloveStatus) => gloveStatus === "error", + ); return (
Model loaded
{modelLoaded}
+
+
SVG fallback
+
{modelFallback ? "yes" : "no"}
+
{serverStatus ? (
Server
diff --git a/src/data/docs/docsTranslations.ts b/src/data/docs/docsTranslations.ts index 88dd675..166cfc0 100644 --- a/src/data/docs/docsTranslations.ts +++ b/src/data/docs/docsTranslations.ts @@ -125,7 +125,7 @@ Ce document décrit le code réellement présent aujourd'hui dans le dépôt. - \`src/components/ui/debug/DebugOverlayLayout.tsx\` monte l'overlay HTML debug compact quand il est activé depuis \`lil-gui\`. - \`src/components/ui/debug/GameStateDebugPanel.tsx\` expose l'état de jeu courant, le changement de main/sub-state, les contrôles previous/next step et le reset. - \`src/components/ui/debug/HandTrackingDebugPanel.tsx\` affiche le statut hand tracking, l'usage, le modèle de gant chargé, le nombre de mains et l'état fist pendant l'activation du hand tracking. -- \`src/components/three/handTracking/HandTrackingLeftGlove.tsx\` place le modèle riggé \`gant_l\` sur la main gauche détectée dans la scène physics debug. +- \`src/components/three/handTracking/HandTrackingGlove.tsx\` place les modèles riggés \`gant_l\` et \`gant_r\` sur les mains détectées dans la scène physics debug. - \`src/components/debug/scene/DebugHelpers.tsx\` monte les helpers debug. - \`src/components/debug/scene/DebugCameraControls.tsx\` monte la caméra libre debug. - Les contrôles globaux \`lil-gui\` incluent camera mode, scene mode, \`R3F Perf\` et \`Debug Overlay\`; les contrôles d'interaction vivent dans le dossier \`Interaction\`. diff --git a/src/hooks/handTracking/useHandTrackingGloveStatus.ts b/src/hooks/handTracking/useHandTrackingGloveStatus.ts new file mode 100644 index 0000000..fb9dd0e --- /dev/null +++ b/src/hooks/handTracking/useHandTrackingGloveStatus.ts @@ -0,0 +1,28 @@ +import { create } from "zustand"; + +export type HandTrackingGloveHandedness = "left" | "right"; + +type HandTrackingGloveLoadState = "idle" | "loaded" | "error"; + +interface HandTrackingGloveStatusState { + gloves: Record; + setGloveStatus: ( + handedness: HandTrackingGloveHandedness, + status: HandTrackingGloveLoadState, + ) => void; +} + +export const useHandTrackingGloveStatus = + create()((set) => ({ + gloves: { + left: "idle", + right: "idle", + }, + setGloveStatus: (handedness, status) => + set((state) => ({ + gloves: { + ...state.gloves, + [handedness]: status, + }, + })), + })); diff --git a/src/world/World.tsx b/src/world/World.tsx index 5ff5a7a..4737abb 100644 --- a/src/world/World.tsx +++ b/src/world/World.tsx @@ -8,7 +8,7 @@ import { useCameraMode } from "@/hooks/debug/useCameraMode"; import { useSceneMode } from "@/hooks/debug/useSceneMode"; import { DebugCameraControls } from "@/components/debug/scene/DebugCameraControls"; import { DebugHelpers } from "@/components/debug/scene/DebugHelpers"; -import { HandTrackingLeftGlove } from "@/components/three/handTracking/HandTrackingLeftGlove"; +import { HandTrackingGlove } from "@/components/three/handTracking/HandTrackingGlove"; import { Environment } from "@/world/Environment"; import { GameMusic } from "@/world/GameMusic"; import { Lighting } from "@/world/Lighting"; @@ -31,7 +31,12 @@ export function World(): React.JSX.Element { - {sceneMode === "physics" ? : null} + {sceneMode === "physics" ? ( + <> + + + + ) : null} {cameraMode === "debug" ? : null} {sceneMode === "game" ? (