diff --git a/public/models/gant_l/model.gltf b/public/models/gant_l/model.gltf index 30a1036..61ecc6e 100644 --- a/public/models/gant_l/model.gltf +++ b/public/models/gant_l/model.gltf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:51299d4fc4df3466e0367e9f4371ab3f71f4a0fd476518ef61f86ec10f7a302e -size 10304 +oid sha256:0fcc5a63d512ec6bb0fd047f3a3d799f680c4ef5ed88b1ce65b5b73c6201e4f3 +size 10327 diff --git a/src/components/three/handTracking/HandTrackingGlove.tsx b/src/components/three/handTracking/HandTrackingGlove.tsx index d5c8904..015bf10 100644 --- a/src/components/three/handTracking/HandTrackingGlove.tsx +++ b/src/components/three/handTracking/HandTrackingGlove.tsx @@ -12,6 +12,11 @@ import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF"; import type { HandTrackingLandmark } from "@/types/handTracking/handTracking"; import { logModelLoadError } from "@/utils/three/modelLoadLogger"; +// Both gloves share the same source mesh (gant_l). The right glove is +// rendered by mirroring scale.x at the group level — this is more +// reliable than the historical gant_r GLTF, which embeds multiple +// skeletons (Hand_l, Hand_l_pad, Hand_r) and was breaking the finger +// rig. const GLOVE_CONFIGS: Record< HandTrackingGloveHandedness, { @@ -24,8 +29,8 @@ const GLOVE_CONFIGS: Record< rootNodeName: "Armature", }, right: { - modelPath: "/models/gant_r/model.gltf", - rootNodeName: "Hand_r", + modelPath: "/models/gant_l/model.gltf", + rootNodeName: "Armature", }, }; @@ -226,7 +231,10 @@ function applyFingerPose( _boneTargetQuaternion .copy(_boneDeltaQuaternion) .multiply(pose.restQuaternion); - pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.45); + // Lower slerp factor = smoother but more latency. MediaPipe at + // ~10fps produces noisy landmark frames; smoothing cuts the + // jitter the user sees on every finger bone. + pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.3); } } } @@ -334,12 +342,18 @@ function HandTrackingGloveModel({ _matrix.makeBasis(_xAxis, _yAxis, _zAxis); _targetQuaternion.setFromRotationMatrix(_matrix); - group.position.lerp(_targetPosition, Math.min(1, delta * 18)); - group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18)); + // Lower factor (was 18) damps the glove jitter caused by noisy + // landmarks while keeping a responsive feel. + group.position.lerp(_targetPosition, Math.min(1, delta * 12)); + group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 12)); const palmLength = _wristPosition.distanceTo(_middlePosition); const scale = palmLength * GLOVE_MODEL_SCALE; - group.scale.setScalar(scale); + // Both gloves use the gant_l mesh; flip X for the right hand so the + // thumb ends up on the correct side instead of being a left-glove + // clone on the right hand. + const mirrorSignX = handedness === "right" ? -1 : 1; + group.scale.set(scale * mirrorSignX, scale, scale); group.updateMatrixWorld(true); applyFingerPose(fingerPoseChains, trackedHand.landmarks, camera); }); diff --git a/src/components/ui/GameUI.tsx b/src/components/ui/GameUI.tsx index 526c3b6..c7a1b49 100644 --- a/src/components/ui/GameUI.tsx +++ b/src/components/ui/GameUI.tsx @@ -1,6 +1,7 @@ import { Crosshair } from "@/components/ui/Crosshair"; import { DebugOverlayLayout } from "@/components/ui/debug/DebugOverlayLayout"; import { GameSettingsMenu } from "@/components/ui/GameSettingsMenu"; +import { HandTrackingFallback } from "@/components/ui/HandTrackingFallback"; import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer"; import { InteractPrompt } from "@/components/ui/InteractPrompt"; import { RepairMovementLockIndicator } from "@/components/ui/RepairMovementLockIndicator"; @@ -15,6 +16,7 @@ export function GameUI(): React.JSX.Element { + diff --git a/src/components/ui/HandTrackingFallback.tsx b/src/components/ui/HandTrackingFallback.tsx new file mode 100644 index 0000000..b5d2d53 --- /dev/null +++ b/src/components/ui/HandTrackingFallback.tsx @@ -0,0 +1,82 @@ +import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; +import { + useHandTrackingGloveStatus, + type HandTrackingGloveHandedness, +} from "@/hooks/handTracking/useHandTrackingGloveStatus"; + +// Simple schematic silhouettes used as a last-resort fallback when the +// rigged glove model has failed to load. Both icons share the same +// 48x48 viewBox and the same stroke/fill rules from the .css. + +const OpenHandShape = (): React.JSX.Element => ( + <> + + + + + + + +); + +const FistShape = (): React.JSX.Element => ( + <> + + + + + + + +); + +function getHandedness(raw: string): HandTrackingGloveHandedness | null { + const lower = raw.toLowerCase(); + if (lower === "left" || lower === "right") return lower; + return null; +} + +export function HandTrackingFallback(): React.JSX.Element | null { + const { hands } = useHandTrackingSnapshot(); + const gloveStatus = useHandTrackingGloveStatus((state) => state.gloves); + + const visibleHands = hands.flatMap((hand, index) => { + const handedness = getHandedness(hand.handedness); + if (!handedness) return []; + if (gloveStatus[handedness] !== "error") return []; + + const wrist = hand.landmarks[0]; + if (!wrist) return []; + + return [{ hand, handedness, wrist, index }]; + }); + + if (visibleHands.length === 0) return null; + + return ( + + ); +} diff --git a/src/components/ui/HandTrackingVisualizer.tsx b/src/components/ui/HandTrackingVisualizer.tsx index 2238554..283a870 100644 --- a/src/components/ui/HandTrackingVisualizer.tsx +++ b/src/components/ui/HandTrackingVisualizer.tsx @@ -26,6 +26,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [ [0, 17], ]; +const LANDMARK_FILL = "#67e8f9"; // cyan-300, opaque interior +const LANDMARK_STROKE = "#0c4a6e"; // sky-900, dark blue outline +const LANDMARK_STROKE_FIST = "#1e3a8a"; // blue-900, thicker accent when fist +const CONNECTION_STROKE = "#ffffff"; // white bones +const INDEX_TIP_LANDMARK = 8; + export function HandTrackingVisualizer(): React.JSX.Element | null { const { hands, status } = useHandTrackingSnapshot(); const showHandTrackingSvg = useDebugStore((debug) => @@ -50,7 +56,9 @@ export function HandTrackingVisualizer(): React.JSX.Element | null { const landmarks = hand.landmarks; if (landmarks.length === 0) return null; - const color = hand.isFist ? "#facc15" : "#38bdf8"; + const landmarkStroke = hand.isFist + ? LANDMARK_STROKE_FIST + : LANDMARK_STROKE; return ( @@ -66,8 +74,8 @@ export function HandTrackingVisualizer(): React.JSX.Element | null { y1={`${fromPoint.y * 100}%`} x2={`${(1 - toPoint.x) * 100}%`} y2={`${toPoint.y * 100}%`} - stroke={color} - strokeWidth="2" + stroke={CONNECTION_STROKE} + strokeWidth="2.5" strokeLinecap="round" /> ); @@ -78,8 +86,10 @@ export function HandTrackingVisualizer(): React.JSX.Element | null { key={landmarkIndex} cx={`${(1 - landmark.x) * 100}%`} cy={`${landmark.y * 100}%`} - r={landmarkIndex === 8 ? 5 : 3} - fill={landmarkIndex === 8 ? "#ffffff" : color} + r={landmarkIndex === INDEX_TIP_LANDMARK ? 6 : 4} + fill={LANDMARK_FILL} + stroke={landmarkStroke} + strokeWidth={hand.isFist ? 2.5 : 2} /> ))} diff --git a/src/data/handTrackingConfig.ts b/src/data/handTrackingConfig.ts index 9537124..e07df5a 100644 --- a/src/data/handTrackingConfig.ts +++ b/src/data/handTrackingConfig.ts @@ -14,7 +14,7 @@ export const HAND_TRACKING_BROWSER_WASM_URL = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm"; export const HAND_TRACKING_BROWSER_MODEL_URL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"; -export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "CPU"; +export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "GPU"; // Delay before the runtime actually starts after `enabled` flips to true. // Absorbs React StrictMode's mount/unmount/mount cycle in dev and rapid @@ -26,3 +26,9 @@ export const HAND_TRACKING_RUNTIME_START_DELAY_MS = 80; // to initialize webcam + model + first frame inference before we cleanup, // so the user actually sees their hands when entering a zone briefly. export const HAND_TRACKING_LINGER_MS = 2000; + +// EMA weight applied to the latest landmark frame. Lower = smoother but +// laggier; higher = more responsive but more jitter from raw MediaPipe +// noise. 0.4 keeps the glove and grabbed objects from trembling without +// feeling sluggish. +export const HAND_TRACKING_LANDMARK_SMOOTHING = 0.4; diff --git a/src/hooks/handTracking/useBrowserHandTracking.ts b/src/hooks/handTracking/useBrowserHandTracking.ts index 9364dcf..8c5dbc5 100644 --- a/src/hooks/handTracking/useBrowserHandTracking.ts +++ b/src/hooks/handTracking/useBrowserHandTracking.ts @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react"; import { HAND_TRACKING_BROWSER_CAMERA_HEIGHT, HAND_TRACKING_BROWSER_CAMERA_WIDTH, + HAND_TRACKING_LANDMARK_SMOOTHING, HAND_TRACKING_RUNTIME_START_DELAY_MS, HAND_TRACKING_TARGET_FPS, } from "@/data/handTrackingConfig"; @@ -10,11 +11,15 @@ import { getBrowserHandLandmarker, releaseBrowserHandLandmarker, } from "@/lib/handTracking/browserHandTracking"; +import { smoothHands } from "@/lib/handTracking/handSmoothing"; import { INITIAL_HAND_TRACKING_SNAPSHOT, getCameraStreamWithTimeout, } from "@/lib/handTracking/handTrackingSession"; -import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking"; +import type { + HandTrackingHand, + HandTrackingSnapshot, +} from "@/types/handTracking/handTracking"; import { logger } from "@/utils/core/Logger"; interface UseBrowserHandTrackingOptions { @@ -30,6 +35,7 @@ export function useBrowserHandTracking({ const videoRef = useRef(null); const streamRef = useRef(null); const intervalRef = useRef(null); + const previousHandsRef = useRef([]); useEffect(() => { if (!enabled) { @@ -51,6 +57,7 @@ export function useBrowserHandTracking({ streamRef.current?.getTracks().forEach((track) => track.stop()); streamRef.current = null; videoRef.current = null; + previousHandsRef.current = []; releaseBrowserHandLandmarker(); }; @@ -124,7 +131,13 @@ export function useBrowserHandTracking({ video, performance.now(), ); - const hands = convertBrowserHandResult(result); + const rawHands = convertBrowserHandResult(result); + const hands = smoothHands( + previousHandsRef.current, + rawHands, + HAND_TRACKING_LANDMARK_SMOOTHING, + ); + previousHandsRef.current = hands; setSnapshot((current) => ({ ...current, diff --git a/src/lib/handTracking/handSmoothing.ts b/src/lib/handTracking/handSmoothing.ts new file mode 100644 index 0000000..70ff33a --- /dev/null +++ b/src/lib/handTracking/handSmoothing.ts @@ -0,0 +1,64 @@ +import type { + HandTrackingHand, + HandTrackingLandmark, +} from "@/types/handTracking/handTracking"; + +function lerp(previous: number, next: number, factor: number): number { + return previous * (1 - factor) + next * factor; +} + +function smoothLandmark( + previous: HandTrackingLandmark, + next: HandTrackingLandmark, + factor: number, +): HandTrackingLandmark { + return { + x: lerp(previous.x, next.x, factor), + y: lerp(previous.y, next.y, factor), + z: lerp(previous.z, next.z, factor), + }; +} + +function smoothHand( + previous: HandTrackingHand, + next: HandTrackingHand, + factor: number, +): HandTrackingHand { + return { + ...next, + x: lerp(previous.x, next.x, factor), + y: lerp(previous.y, next.y, factor), + z: lerp(previous.z, next.z, factor), + landmarks: next.landmarks.map((landmark, index) => { + const previousLandmark = previous.landmarks[index]; + if (!previousLandmark) return landmark; + return smoothLandmark(previousLandmark, landmark, factor); + }), + }; +} + +/** + * Apply an exponential moving average to the landmark positions of each + * detected hand. MediaPipe lands per-frame positions with noticeable + * jitter (especially at ~10fps), and feeding those raw values into the + * scene makes both the glove rig and any grabbed object tremble. + * + * `factor` is the weight given to the latest sample (0 = previous frame + * only, 1 = no smoothing). Hands are matched between frames by + * handedness so left/right don't bleed into each other. + */ +export function smoothHands( + previousHands: HandTrackingHand[], + nextHands: HandTrackingHand[], + factor: number, +): HandTrackingHand[] { + if (factor >= 1) return nextHands; + + return nextHands.map((next) => { + const previous = previousHands.find( + (candidate) => candidate.handedness === next.handedness, + ); + if (!previous) return next; + return smoothHand(previous, next, factor); + }); +}