fix hand tracking glove rendering
This commit is contained in:
@@ -42,7 +42,7 @@ This document describes the code that exists today in the repository.
|
||||
- `src/components/ui/debug/DebugOverlayLayout.tsx` mounts the compact HTML debug overlay when enabled from `lil-gui`.
|
||||
- `src/components/ui/debug/GameStateDebugPanel.tsx` exposes current game state, main/sub-state switching, previous/next step controls, and reset.
|
||||
- `src/components/ui/debug/HandTrackingDebugPanel.tsx` shows hand tracking status, usage, loaded glove model, hand count, and fist state while hand tracking is active.
|
||||
- `src/components/three/handTracking/HandTrackingLeftGlove.tsx` places the rigged `gant_l` model on the detected left hand in the debug physics scene.
|
||||
- `src/components/three/handTracking/HandTrackingGlove.tsx` places the rigged `gant_l` and `gant_r` models on detected hands in the debug physics scene.
|
||||
- `src/components/debug/scene/DebugHelpers.tsx` mounts debug helpers.
|
||||
- `src/components/debug/scene/DebugCameraControls.tsx` mounts the free debug camera.
|
||||
- `lil-gui` global debug controls include camera mode, scene mode, `R3F Perf`, and `Debug Overlay`; interaction-specific controls live in the `Interaction` folder.
|
||||
@@ -51,7 +51,7 @@ This document describes the code that exists today in the repository.
|
||||
|
||||
- `src/components/three/models/` contains reusable model helpers such as `ExplodableModel`.
|
||||
- `src/components/three/interaction/` contains reusable interaction wrappers such as `InteractableObject`, `TriggerObject`, and `GrabbableObject`.
|
||||
- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the left glove overlay.
|
||||
- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the glove overlays.
|
||||
- `src/components/three/gameplay/` contains the current core repair gameplay prototype: the repair case, repair game zone, and module slots.
|
||||
- `src/components/three/world/` contains reusable world/environment objects such as `SkyModel`.
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ The feature is scoped to the debug physics scene rather than production gameplay
|
||||
4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`.
|
||||
5. React stores the latest snapshot in the hand tracking provider.
|
||||
6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects.
|
||||
7. `HandTrackingLeftGlove` reads the same snapshot and places the rigged `gant_l` model on the detected left hand in the debug physics scene.
|
||||
7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands in the debug physics scene.
|
||||
|
||||
## Activation Rules
|
||||
|
||||
@@ -106,18 +106,18 @@ The final hold distance is clamped between the configured grab minimum and maxim
|
||||
The current debug UI includes:
|
||||
|
||||
- `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, loaded glove model, server state, hand count, and fist state
|
||||
- `HandTrackingVisualizer` for the SVG landmark wireframe
|
||||
- `HandTrackingLeftGlove` for the left-hand `gant_l` model in the R3F scene
|
||||
- `HandTrackingVisualizer` for the SVG landmark wireframe fallback
|
||||
- `HandTrackingGlove` for the left-hand `gant_l` and right-hand `gant_r` models in the R3F scene
|
||||
- `r3f-perf` for render performance
|
||||
- `lil-gui` for scene, camera, lighting, interaction, and grab controls
|
||||
|
||||
The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays `gant_l` when a left hand is detected, otherwise `none`. The hand wireframe is also HTML/SVG, not a 3D hand model.
|
||||
The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays the successfully loaded glove models. The SVG hand wireframe is only a fallback while models are loading or if a glove model fails to load.
|
||||
|
||||
## Left Glove Model
|
||||
## Glove Models
|
||||
|
||||
The current left glove MVP uses `public/models/gant_l/model.gltf`, which contains a GLTF skin and armature. For now the model is positioned, oriented, and scaled as a whole from palm landmarks instead of driving individual finger bones.
|
||||
The current glove MVP uses `public/models/gant_l/model.gltf` and `public/models/gant_r/model.gltf`, which contain GLTF skins and armatures. For now each model is positioned, oriented, and scaled as a whole from palm landmarks instead of driving individual finger bones.
|
||||
|
||||
The right hand is intentionally ignored in this MVP. The available right-hand models are static in the current assets and are not mapped to MediaPipe bones yet.
|
||||
The glove models are intentionally smaller than the raw SVG overlay so they do not dominate the camera view.
|
||||
|
||||
## Known Limitations
|
||||
|
||||
@@ -125,5 +125,5 @@ The right hand is intentionally ignored in this MVP. The available right-hand mo
|
||||
- MediaPipe depth is relative and can be noisy.
|
||||
- The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider.
|
||||
- There is no smoothing layer for hand position or depth yet.
|
||||
- The hand visualization is an SVG landmark wireframe.
|
||||
- The left glove follows the palm as a whole; finger-by-finger bone animation still requires a verified landmark-to-bone mapping and smoothing.
|
||||
- The SVG hand visualization is a fallback, not the primary display when glove models load correctly.
|
||||
- Each glove follows the palm as a whole; finger-by-finger bone animation still requires a verified landmark-to-bone mapping and smoothing.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,235 @@
|
||||
import type { ReactNode } from "react";
|
||||
import { Component, useEffect, useMemo, useRef } from "react";
|
||||
import { useFrame, useThree } from "@react-three/fiber";
|
||||
import { useGLTF } from "@react-three/drei";
|
||||
import * as THREE from "three";
|
||||
import { clone } from "three/addons/utils/SkeletonUtils.js";
|
||||
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
|
||||
import {
|
||||
useHandTrackingGloveStatus,
|
||||
type HandTrackingGloveHandedness,
|
||||
} from "@/hooks/handTracking/useHandTrackingGloveStatus";
|
||||
import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
|
||||
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
|
||||
import { logModelLoadError } from "@/utils/three/modelLoadLogger";
|
||||
|
||||
const GLOVE_MODEL_PATHS: Record<HandTrackingGloveHandedness, string> = {
|
||||
left: "/models/gant_l/model.gltf",
|
||||
right: "/models/gant_r/model.gltf",
|
||||
};
|
||||
|
||||
const GLOVE_ROOT_NODE_NAMES: Record<HandTrackingGloveHandedness, string> = {
|
||||
left: "Hand_l",
|
||||
right: "Hand_r",
|
||||
};
|
||||
|
||||
const HAND_SPACE_DISTANCE = 2.4;
|
||||
const HAND_DEPTH_SCALE = 0.45;
|
||||
const GLOVE_SCALE = 0.17;
|
||||
|
||||
const _cameraPosition = new THREE.Vector3();
|
||||
const _direction = new THREE.Vector3();
|
||||
const _xAxis = new THREE.Vector3();
|
||||
const _yAxis = new THREE.Vector3();
|
||||
const _zAxis = new THREE.Vector3();
|
||||
const _matrix = new THREE.Matrix4();
|
||||
const _targetQuaternion = new THREE.Quaternion();
|
||||
const _targetPosition = new THREE.Vector3();
|
||||
const _wristPosition = new THREE.Vector3();
|
||||
const _indexPosition = new THREE.Vector3();
|
||||
const _middlePosition = new THREE.Vector3();
|
||||
const _ringPosition = new THREE.Vector3();
|
||||
const _pinkyPosition = new THREE.Vector3();
|
||||
|
||||
interface HandTrackingGloveProps {
|
||||
handedness: HandTrackingGloveHandedness;
|
||||
}
|
||||
|
||||
interface HandTrackingGloveErrorBoundaryProps {
|
||||
children: ReactNode;
|
||||
handedness: HandTrackingGloveHandedness;
|
||||
modelPath: string;
|
||||
}
|
||||
|
||||
class HandTrackingGloveErrorBoundary extends Component<
|
||||
HandTrackingGloveErrorBoundaryProps,
|
||||
{ hasError: boolean }
|
||||
> {
|
||||
constructor(props: HandTrackingGloveErrorBoundaryProps) {
|
||||
super(props);
|
||||
this.state = { hasError: false };
|
||||
}
|
||||
|
||||
static getDerivedStateFromError(): { hasError: boolean } {
|
||||
return { hasError: true };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error): void {
|
||||
useHandTrackingGloveStatus
|
||||
.getState()
|
||||
.setGloveStatus(this.props.handedness, "error");
|
||||
|
||||
logModelLoadError(
|
||||
{
|
||||
modelPath: this.props.modelPath,
|
||||
scope: `HandTrackingGlove.${this.props.handedness}`,
|
||||
scale: GLOVE_SCALE,
|
||||
},
|
||||
error,
|
||||
);
|
||||
}
|
||||
|
||||
render(): ReactNode {
|
||||
if (this.state.hasError) return null;
|
||||
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
|
||||
function landmarkToWorldPoint(
|
||||
landmark: HandTrackingLandmark,
|
||||
camera: THREE.Camera,
|
||||
target: THREE.Vector3,
|
||||
): THREE.Vector3 {
|
||||
_cameraPosition.setFromMatrixPosition(camera.matrixWorld);
|
||||
target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5);
|
||||
target.unproject(camera);
|
||||
|
||||
_direction.copy(target).sub(_cameraPosition).normalize();
|
||||
target
|
||||
.copy(_cameraPosition)
|
||||
.addScaledVector(
|
||||
_direction,
|
||||
HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE,
|
||||
);
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
function matchesHandedness(
|
||||
handHandedness: string,
|
||||
targetHandedness: HandTrackingGloveHandedness,
|
||||
): boolean {
|
||||
return handHandedness.toLowerCase() === targetHandedness;
|
||||
}
|
||||
|
||||
function HandTrackingGloveModel({
|
||||
handedness,
|
||||
}: HandTrackingGloveProps): React.JSX.Element | null {
|
||||
const groupRef = useRef<THREE.Group>(null);
|
||||
const { camera } = useThree();
|
||||
const { hands } = useHandTrackingSnapshot();
|
||||
const setGloveStatus = useHandTrackingGloveStatus(
|
||||
(state) => state.setGloveStatus,
|
||||
);
|
||||
const modelPath = GLOVE_MODEL_PATHS[handedness];
|
||||
const gltf = useLoggedGLTF(modelPath, {
|
||||
scope: `HandTrackingGlove.${handedness}`,
|
||||
scale: GLOVE_SCALE,
|
||||
});
|
||||
const gloveScene = useMemo(() => {
|
||||
const rootNode = gltf.scene.getObjectByName(
|
||||
GLOVE_ROOT_NODE_NAMES[handedness],
|
||||
);
|
||||
|
||||
if (!rootNode) {
|
||||
throw new Error(
|
||||
`Missing glove root node ${GLOVE_ROOT_NODE_NAMES[handedness]}`,
|
||||
);
|
||||
}
|
||||
|
||||
return clone(rootNode);
|
||||
}, [gltf.scene, handedness]);
|
||||
|
||||
const hand = hands.find((candidate) =>
|
||||
matchesHandedness(candidate.handedness, handedness),
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
setGloveStatus(handedness, "loaded");
|
||||
}, [handedness, setGloveStatus]);
|
||||
|
||||
useFrame((_, delta) => {
|
||||
const group = groupRef.current;
|
||||
const trackedHand = hands.find((candidate) =>
|
||||
matchesHandedness(candidate.handedness, handedness),
|
||||
);
|
||||
|
||||
if (!group || !trackedHand || trackedHand.landmarks.length < 21) {
|
||||
if (group) group.visible = false;
|
||||
return;
|
||||
}
|
||||
|
||||
group.visible = true;
|
||||
|
||||
const wrist = trackedHand.landmarks[0];
|
||||
const indexMcp = trackedHand.landmarks[5];
|
||||
const middleMcp = trackedHand.landmarks[9];
|
||||
const ringMcp = trackedHand.landmarks[13];
|
||||
const pinkyMcp = trackedHand.landmarks[17];
|
||||
|
||||
if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) {
|
||||
group.visible = false;
|
||||
return;
|
||||
}
|
||||
|
||||
landmarkToWorldPoint(wrist, camera, _wristPosition);
|
||||
landmarkToWorldPoint(indexMcp, camera, _indexPosition);
|
||||
landmarkToWorldPoint(middleMcp, camera, _middlePosition);
|
||||
landmarkToWorldPoint(ringMcp, camera, _ringPosition);
|
||||
landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition);
|
||||
|
||||
_targetPosition
|
||||
.copy(_wristPosition)
|
||||
.add(_indexPosition)
|
||||
.add(_middlePosition)
|
||||
.add(_ringPosition)
|
||||
.add(_pinkyPosition)
|
||||
.multiplyScalar(0.2);
|
||||
|
||||
_yAxis.copy(_middlePosition).sub(_wristPosition).normalize();
|
||||
_xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize();
|
||||
_zAxis.crossVectors(_xAxis, _yAxis).normalize();
|
||||
|
||||
if (
|
||||
_xAxis.lengthSq() === 0 ||
|
||||
_yAxis.lengthSq() === 0 ||
|
||||
_zAxis.lengthSq() === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
_xAxis.crossVectors(_yAxis, _zAxis).normalize();
|
||||
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
|
||||
_targetQuaternion.setFromRotationMatrix(_matrix);
|
||||
|
||||
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
|
||||
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
|
||||
|
||||
const palmLength = _wristPosition.distanceTo(_middlePosition);
|
||||
const scale = palmLength * GLOVE_SCALE;
|
||||
group.scale.setScalar(scale);
|
||||
});
|
||||
|
||||
if (!hand) return null;
|
||||
|
||||
return <primitive ref={groupRef} object={gloveScene} />;
|
||||
}
|
||||
|
||||
export function HandTrackingGlove({
|
||||
handedness,
|
||||
}: HandTrackingGloveProps): React.JSX.Element {
|
||||
const modelPath = GLOVE_MODEL_PATHS[handedness];
|
||||
|
||||
return (
|
||||
<HandTrackingGloveErrorBoundary
|
||||
handedness={handedness}
|
||||
modelPath={modelPath}
|
||||
>
|
||||
<HandTrackingGloveModel handedness={handedness} />
|
||||
</HandTrackingGloveErrorBoundary>
|
||||
);
|
||||
}
|
||||
|
||||
useGLTF.preload(GLOVE_MODEL_PATHS.left);
|
||||
useGLTF.preload(GLOVE_MODEL_PATHS.right);
|
||||
@@ -1,130 +0,0 @@
|
||||
import { useMemo, useRef } from "react";
|
||||
import { useFrame, useThree } from "@react-three/fiber";
|
||||
import { useGLTF } from "@react-three/drei";
|
||||
import * as THREE from "three";
|
||||
import { clone } from "three/addons/utils/SkeletonUtils.js";
|
||||
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
|
||||
import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
|
||||
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
|
||||
|
||||
const LEFT_GLOVE_MODEL_URL = "/models/gant_l/model.gltf";
|
||||
const HAND_SPACE_DISTANCE = 2.4;
|
||||
const HAND_DEPTH_SCALE = 0.45;
|
||||
const GLOVE_SCALE = 0.34;
|
||||
|
||||
const _cameraPosition = new THREE.Vector3();
|
||||
const _direction = new THREE.Vector3();
|
||||
const _xAxis = new THREE.Vector3();
|
||||
const _yAxis = new THREE.Vector3();
|
||||
const _zAxis = new THREE.Vector3();
|
||||
const _matrix = new THREE.Matrix4();
|
||||
const _targetQuaternion = new THREE.Quaternion();
|
||||
const _targetPosition = new THREE.Vector3();
|
||||
const _wristPosition = new THREE.Vector3();
|
||||
const _indexPosition = new THREE.Vector3();
|
||||
const _middlePosition = new THREE.Vector3();
|
||||
const _ringPosition = new THREE.Vector3();
|
||||
const _pinkyPosition = new THREE.Vector3();
|
||||
|
||||
function landmarkToWorldPoint(
|
||||
landmark: HandTrackingLandmark,
|
||||
camera: THREE.Camera,
|
||||
target: THREE.Vector3,
|
||||
): THREE.Vector3 {
|
||||
_cameraPosition.setFromMatrixPosition(camera.matrixWorld);
|
||||
target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5);
|
||||
target.unproject(camera);
|
||||
|
||||
_direction.copy(target).sub(_cameraPosition).normalize();
|
||||
target
|
||||
.copy(_cameraPosition)
|
||||
.addScaledVector(
|
||||
_direction,
|
||||
HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE,
|
||||
);
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
function isLeftHand(handedness: string): boolean {
|
||||
return handedness.toLowerCase() === "left";
|
||||
}
|
||||
|
||||
export function HandTrackingLeftGlove(): React.JSX.Element | null {
|
||||
const groupRef = useRef<THREE.Group>(null);
|
||||
const { camera } = useThree();
|
||||
const { hands } = useHandTrackingSnapshot();
|
||||
const gltf = useLoggedGLTF(LEFT_GLOVE_MODEL_URL, {
|
||||
scope: "HandTrackingLeftGlove",
|
||||
scale: GLOVE_SCALE,
|
||||
});
|
||||
const gloveScene = useMemo(() => clone(gltf.scene), [gltf.scene]);
|
||||
|
||||
const leftHand = hands.find((hand) => isLeftHand(hand.handedness));
|
||||
|
||||
useFrame((_, delta) => {
|
||||
const group = groupRef.current;
|
||||
const hand = hands.find((candidate) => isLeftHand(candidate.handedness));
|
||||
|
||||
if (!group || !hand || hand.landmarks.length < 21) {
|
||||
if (group) group.visible = false;
|
||||
return;
|
||||
}
|
||||
|
||||
group.visible = true;
|
||||
|
||||
const wrist = hand.landmarks[0];
|
||||
const indexMcp = hand.landmarks[5];
|
||||
const middleMcp = hand.landmarks[9];
|
||||
const ringMcp = hand.landmarks[13];
|
||||
const pinkyMcp = hand.landmarks[17];
|
||||
|
||||
if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) {
|
||||
group.visible = false;
|
||||
return;
|
||||
}
|
||||
|
||||
landmarkToWorldPoint(wrist, camera, _wristPosition);
|
||||
landmarkToWorldPoint(indexMcp, camera, _indexPosition);
|
||||
landmarkToWorldPoint(middleMcp, camera, _middlePosition);
|
||||
landmarkToWorldPoint(ringMcp, camera, _ringPosition);
|
||||
landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition);
|
||||
|
||||
_targetPosition
|
||||
.copy(_wristPosition)
|
||||
.add(_indexPosition)
|
||||
.add(_middlePosition)
|
||||
.add(_ringPosition)
|
||||
.add(_pinkyPosition)
|
||||
.multiplyScalar(0.2);
|
||||
|
||||
_yAxis.copy(_middlePosition).sub(_wristPosition).normalize();
|
||||
_xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize();
|
||||
_zAxis.crossVectors(_xAxis, _yAxis).normalize();
|
||||
|
||||
if (
|
||||
_xAxis.lengthSq() === 0 ||
|
||||
_yAxis.lengthSq() === 0 ||
|
||||
_zAxis.lengthSq() === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
_xAxis.crossVectors(_yAxis, _zAxis).normalize();
|
||||
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
|
||||
_targetQuaternion.setFromRotationMatrix(_matrix);
|
||||
|
||||
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
|
||||
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
|
||||
|
||||
const palmLength = _wristPosition.distanceTo(_middlePosition);
|
||||
const scale = palmLength * GLOVE_SCALE;
|
||||
group.scale.setScalar(scale);
|
||||
});
|
||||
|
||||
if (!leftHand) return null;
|
||||
|
||||
return <primitive ref={groupRef} object={gloveScene} />;
|
||||
}
|
||||
|
||||
useGLTF.preload(LEFT_GLOVE_MODEL_URL);
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
|
||||
import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus";
|
||||
|
||||
const HAND_CONNECTIONS: Array<[number, number]> = [
|
||||
[0, 1],
|
||||
@@ -26,8 +27,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [
|
||||
|
||||
export function HandTrackingVisualizer(): React.JSX.Element | null {
|
||||
const { hands, status } = useHandTrackingSnapshot();
|
||||
const gloves = useHandTrackingGloveStatus((state) => state.gloves);
|
||||
const shouldShowSvgFallback = Object.values(gloves).some(
|
||||
(gloveStatus) => gloveStatus === "error" || gloveStatus === "idle",
|
||||
);
|
||||
|
||||
if (status === "idle" || hands.length === 0) {
|
||||
if (status === "idle" || hands.length === 0 || !shouldShowSvgFallback) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
|
||||
import { useHandTrackingGloveStatus } from "@/hooks/handTracking/useHandTrackingGloveStatus";
|
||||
import type { HandTrackingStatus } from "@/types/handTracking/handTracking";
|
||||
|
||||
const STATUS_LABELS: Record<HandTrackingStatus, string> = {
|
||||
@@ -15,22 +16,23 @@ const STATUS_LABELS: Record<HandTrackingStatus, string> = {
|
||||
export function HandTrackingDebugPanel(): React.JSX.Element | null {
|
||||
const { hands, status, usageStatus, serverStatus, error } =
|
||||
useHandTrackingSnapshot();
|
||||
const gloves = useHandTrackingGloveStatus((state) => state.gloves);
|
||||
|
||||
if (status === "idle") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const fist = hands.some((hand) => hand.isFist);
|
||||
const hasLeftHand = hands.some(
|
||||
(hand) => hand.handedness.toLowerCase() === "left",
|
||||
);
|
||||
const hasRightHand = hands.some(
|
||||
(hand) => hand.handedness.toLowerCase() === "right",
|
||||
);
|
||||
const modelLoaded =
|
||||
[hasLeftHand ? "gant_l" : null, hasRightHand ? "gant_r" : null]
|
||||
[
|
||||
gloves.left === "loaded" ? "gant_l" : null,
|
||||
gloves.right === "loaded" ? "gant_r" : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(", ") || "none";
|
||||
const modelFallback = Object.values(gloves).some(
|
||||
(gloveStatus) => gloveStatus === "error",
|
||||
);
|
||||
|
||||
return (
|
||||
<section
|
||||
@@ -51,6 +53,10 @@ export function HandTrackingDebugPanel(): React.JSX.Element | null {
|
||||
<dt>Model loaded</dt>
|
||||
<dd>{modelLoaded}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>SVG fallback</dt>
|
||||
<dd>{modelFallback ? "yes" : "no"}</dd>
|
||||
</div>
|
||||
{serverStatus ? (
|
||||
<div>
|
||||
<dt>Server</dt>
|
||||
|
||||
@@ -125,7 +125,7 @@ Ce document décrit le code réellement présent aujourd'hui dans le dépôt.
|
||||
- \`src/components/ui/debug/DebugOverlayLayout.tsx\` monte l'overlay HTML debug compact quand il est activé depuis \`lil-gui\`.
|
||||
- \`src/components/ui/debug/GameStateDebugPanel.tsx\` expose l'état de jeu courant, le changement de main/sub-state, les contrôles previous/next step et le reset.
|
||||
- \`src/components/ui/debug/HandTrackingDebugPanel.tsx\` affiche le statut hand tracking, l'usage, le modèle de gant chargé, le nombre de mains et l'état fist pendant l'activation du hand tracking.
|
||||
- \`src/components/three/handTracking/HandTrackingLeftGlove.tsx\` place le modèle riggé \`gant_l\` sur la main gauche détectée dans la scène physics debug.
|
||||
- \`src/components/three/handTracking/HandTrackingGlove.tsx\` place les modèles riggés \`gant_l\` et \`gant_r\` sur les mains détectées dans la scène physics debug.
|
||||
- \`src/components/debug/scene/DebugHelpers.tsx\` monte les helpers debug.
|
||||
- \`src/components/debug/scene/DebugCameraControls.tsx\` monte la caméra libre debug.
|
||||
- Les contrôles globaux \`lil-gui\` incluent camera mode, scene mode, \`R3F Perf\` et \`Debug Overlay\`; les contrôles d'interaction vivent dans le dossier \`Interaction\`.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { create } from "zustand";
|
||||
|
||||
export type HandTrackingGloveHandedness = "left" | "right";
|
||||
|
||||
type HandTrackingGloveLoadState = "idle" | "loaded" | "error";
|
||||
|
||||
interface HandTrackingGloveStatusState {
|
||||
gloves: Record<HandTrackingGloveHandedness, HandTrackingGloveLoadState>;
|
||||
setGloveStatus: (
|
||||
handedness: HandTrackingGloveHandedness,
|
||||
status: HandTrackingGloveLoadState,
|
||||
) => void;
|
||||
}
|
||||
|
||||
export const useHandTrackingGloveStatus =
|
||||
create<HandTrackingGloveStatusState>()((set) => ({
|
||||
gloves: {
|
||||
left: "idle",
|
||||
right: "idle",
|
||||
},
|
||||
setGloveStatus: (handedness, status) =>
|
||||
set((state) => ({
|
||||
gloves: {
|
||||
...state.gloves,
|
||||
[handedness]: status,
|
||||
},
|
||||
})),
|
||||
}));
|
||||
+7
-2
@@ -8,7 +8,7 @@ import { useCameraMode } from "@/hooks/debug/useCameraMode";
|
||||
import { useSceneMode } from "@/hooks/debug/useSceneMode";
|
||||
import { DebugCameraControls } from "@/components/debug/scene/DebugCameraControls";
|
||||
import { DebugHelpers } from "@/components/debug/scene/DebugHelpers";
|
||||
import { HandTrackingLeftGlove } from "@/components/three/handTracking/HandTrackingLeftGlove";
|
||||
import { HandTrackingGlove } from "@/components/three/handTracking/HandTrackingGlove";
|
||||
import { Environment } from "@/world/Environment";
|
||||
import { GameMusic } from "@/world/GameMusic";
|
||||
import { Lighting } from "@/world/Lighting";
|
||||
@@ -31,7 +31,12 @@ export function World(): React.JSX.Element {
|
||||
<Environment />
|
||||
<Lighting />
|
||||
<DebugHelpers />
|
||||
{sceneMode === "physics" ? <HandTrackingLeftGlove /> : null}
|
||||
{sceneMode === "physics" ? (
|
||||
<>
|
||||
<HandTrackingGlove handedness="left" />
|
||||
<HandTrackingGlove handedness="right" />
|
||||
</>
|
||||
) : null}
|
||||
{cameraMode === "debug" ? <DebugCameraControls /> : null}
|
||||
|
||||
{sceneMode === "game" ? (
|
||||
|
||||
Reference in New Issue
Block a user