feat(handtracking): restyle svg visualizer and add silhouette fallback

src/components/three/handTracking/HandTrackingGlove.tsx

@@ -12,6 +12,11 @@ import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
 import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
 import { logModelLoadError } from "@/utils/three/modelLoadLogger";
 
+// Both gloves share the same source mesh (gant_l). The right glove is
+// rendered by mirroring scale.x at the group level — this is more
+// reliable than the historical gant_r GLTF, which embeds multiple
+// skeletons (Hand_l, Hand_l_pad, Hand_r) and was breaking the finger
+// rig.
 const GLOVE_CONFIGS: Record<
   HandTrackingGloveHandedness,
   {
@@ -24,8 +29,8 @@ const GLOVE_CONFIGS: Record<
     rootNodeName: "Armature",
   },
   right: {
-    modelPath: "/models/gant_r/model.gltf",
-    rootNodeName: "Hand_r",
+    modelPath: "/models/gant_l/model.gltf",
+    rootNodeName: "Armature",
   },
 };
 
@@ -226,7 +231,10 @@ function applyFingerPose(
       _boneTargetQuaternion
         .copy(_boneDeltaQuaternion)
         .multiply(pose.restQuaternion);
-      pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.45);
+      // Lower slerp factor = smoother but more latency. MediaPipe at
+      // ~10fps produces noisy landmark frames; smoothing cuts the
+      // jitter the user sees on every finger bone.
+      pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.3);
     }
   }
 }
@@ -334,12 +342,18 @@ function HandTrackingGloveModel({
     _matrix.makeBasis(_xAxis, _yAxis, _zAxis);
     _targetQuaternion.setFromRotationMatrix(_matrix);
 
-    group.position.lerp(_targetPosition, Math.min(1, delta * 18));
-    group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
+    // Lower factor (was 18) damps the glove jitter caused by noisy
+    // landmarks while keeping a responsive feel.
+    group.position.lerp(_targetPosition, Math.min(1, delta * 12));
+    group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 12));
 
     const palmLength = _wristPosition.distanceTo(_middlePosition);
     const scale = palmLength * GLOVE_MODEL_SCALE;
-    group.scale.setScalar(scale);
+    // Both gloves use the gant_l mesh; flip X for the right hand so the
+    // thumb ends up on the correct side instead of being a left-glove
+    // clone on the right hand.
+    const mirrorSignX = handedness === "right" ? -1 : 1;
+    group.scale.set(scale * mirrorSignX, scale, scale);
     group.updateMatrixWorld(true);
     applyFingerPose(fingerPoseChains, trackedHand.landmarks, camera);
   });

src/components/ui/GameUI.tsx

@@ -1,6 +1,7 @@
 import { Crosshair } from "@/components/ui/Crosshair";
 import { DebugOverlayLayout } from "@/components/ui/debug/DebugOverlayLayout";
 import { GameSettingsMenu } from "@/components/ui/GameSettingsMenu";
+import { HandTrackingFallback } from "@/components/ui/HandTrackingFallback";
 import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer";
 import { InteractPrompt } from "@/components/ui/InteractPrompt";
 import { RepairMovementLockIndicator } from "@/components/ui/RepairMovementLockIndicator";
@@ -15,6 +16,7 @@ export function GameUI(): React.JSX.Element {
       <RepairMovementLockIndicator />
       <InteractPrompt />
       <HandTrackingVisualizer />
+      <HandTrackingFallback />
       <Subtitles />
       <TalkieDialogueOverlay />
       <GameSettingsMenu />

src/components/ui/HandTrackingFallback.tsx

@@ -0,0 +1,82 @@
+import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
+import {
+  useHandTrackingGloveStatus,
+  type HandTrackingGloveHandedness,
+} from "@/hooks/handTracking/useHandTrackingGloveStatus";
+
+// Simple schematic silhouettes used as a last-resort fallback when the
+// rigged glove model has failed to load. Both icons share the same
+// 48x48 viewBox and the same stroke/fill rules from the .css.
+
+const OpenHandShape = (): React.JSX.Element => (
+  <>
+    <ellipse cx="9" cy="30" rx="3" ry="6" transform="rotate(-25 9 30)" />
+    <rect x="14" y="8" width="4" height="22" rx="2" />
+    <rect x="20" y="4" width="4" height="26" rx="2" />
+    <rect x="26" y="6" width="4" height="24" rx="2" />
+    <rect x="32" y="10" width="4" height="20" rx="2" />
+    <rect x="10" y="26" width="28" height="18" rx="6" />
+  </>
+);
+
+const FistShape = (): React.JSX.Element => (
+  <>
+    <ellipse cx="8" cy="26" rx="3" ry="5" />
+    <rect x="10" y="14" width="28" height="30" rx="10" />
+    <circle cx="15" cy="14" r="3" />
+    <circle cx="21" cy="13" r="3" />
+    <circle cx="27" cy="13" r="3" />
+    <circle cx="33" cy="14" r="3" />
+  </>
+);
+
+function getHandedness(raw: string): HandTrackingGloveHandedness | null {
+  const lower = raw.toLowerCase();
+  if (lower === "left" || lower === "right") return lower;
+  return null;
+}
+
+export function HandTrackingFallback(): React.JSX.Element | null {
+  const { hands } = useHandTrackingSnapshot();
+  const gloveStatus = useHandTrackingGloveStatus((state) => state.gloves);
+
+  const visibleHands = hands.flatMap((hand, index) => {
+    const handedness = getHandedness(hand.handedness);
+    if (!handedness) return [];
+    if (gloveStatus[handedness] !== "error") return [];
+
+    const wrist = hand.landmarks[0];
+    if (!wrist) return [];
+
+    return [{ hand, handedness, wrist, index }];
+  });
+
+  if (visibleHands.length === 0) return null;
+
+  return (
+    <div className="hand-tracking-fallback" aria-hidden="true">
+      {visibleHands.map(({ hand, handedness, wrist, index }) => {
+        // MediaPipe coords are mirrored (selfie cam), keep the same
+        // mapping the SVG visualizer uses.
+        const leftPercent = (1 - wrist.x) * 100;
+        const topPercent = wrist.y * 100;
+        const flipX = handedness === "right" ? -1 : 1;
+
+        return (
+          <svg
+            key={`${handedness}-${index}`}
+            className="hand-tracking-fallback__icon"
+            viewBox="0 0 48 48"
+            style={{
+              left: `${leftPercent}%`,
+              top: `${topPercent}%`,
+              transform: `translate(-50%, -50%) scaleX(${flipX})`,
+            }}
+          >
+            {hand.isFist ? <FistShape /> : <OpenHandShape />}
+          </svg>
+        );
+      })}
+    </div>
+  );
+}

src/components/ui/HandTrackingVisualizer.tsx

@@ -26,6 +26,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [
   [0, 17],
 ];
 
+const LANDMARK_FILL = "#67e8f9"; // cyan-300, opaque interior
+const LANDMARK_STROKE = "#0c4a6e"; // sky-900, dark blue outline
+const LANDMARK_STROKE_FIST = "#1e3a8a"; // blue-900, thicker accent when fist
+const CONNECTION_STROKE = "#ffffff"; // white bones
+const INDEX_TIP_LANDMARK = 8;
+
 export function HandTrackingVisualizer(): React.JSX.Element | null {
   const { hands, status } = useHandTrackingSnapshot();
   const showHandTrackingSvg = useDebugStore((debug) =>
@@ -50,7 +56,9 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
         const landmarks = hand.landmarks;
         if (landmarks.length === 0) return null;
 
-        const color = hand.isFist ? "#facc15" : "#38bdf8";
+        const landmarkStroke = hand.isFist
+          ? LANDMARK_STROKE_FIST
+          : LANDMARK_STROKE;
 
         return (
           <g key={`${hand.handedness}-${handIndex}`}>
@@ -66,8 +74,8 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
                   y1={`${fromPoint.y * 100}%`}
                   x2={`${(1 - toPoint.x) * 100}%`}
                   y2={`${toPoint.y * 100}%`}
-                  stroke={color}
-                  strokeWidth="2"
+                  stroke={CONNECTION_STROKE}
+                  strokeWidth="2.5"
                   strokeLinecap="round"
                 />
               );
@@ -78,8 +86,10 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
                 key={landmarkIndex}
                 cx={`${(1 - landmark.x) * 100}%`}
                 cy={`${landmark.y * 100}%`}
-                r={landmarkIndex === 8 ? 5 : 3}
-                fill={landmarkIndex === 8 ? "#ffffff" : color}
+                r={landmarkIndex === INDEX_TIP_LANDMARK ? 6 : 4}
+                fill={LANDMARK_FILL}
+                stroke={landmarkStroke}
+                strokeWidth={hand.isFist ? 2.5 : 2}
               />
             ))}
           </g>

src/data/handTrackingConfig.ts

@@ -14,7 +14,7 @@ export const HAND_TRACKING_BROWSER_WASM_URL =
   "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm";
 export const HAND_TRACKING_BROWSER_MODEL_URL =
   "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task";
-export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "CPU";
+export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "GPU";
 
 // Delay before the runtime actually starts after `enabled` flips to true.
 // Absorbs React StrictMode's mount/unmount/mount cycle in dev and rapid
@@ -26,3 +26,9 @@ export const HAND_TRACKING_RUNTIME_START_DELAY_MS = 80;
 // to initialize webcam + model + first frame inference before we cleanup,
 // so the user actually sees their hands when entering a zone briefly.
 export const HAND_TRACKING_LINGER_MS = 2000;
+
+// EMA weight applied to the latest landmark frame. Lower = smoother but
+// laggier; higher = more responsive but more jitter from raw MediaPipe
+// noise. 0.4 keeps the glove and grabbed objects from trembling without
+// feeling sluggish.
+export const HAND_TRACKING_LANDMARK_SMOOTHING = 0.4;

src/hooks/handTracking/useBrowserHandTracking.ts

@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react";
 import {
   HAND_TRACKING_BROWSER_CAMERA_HEIGHT,
   HAND_TRACKING_BROWSER_CAMERA_WIDTH,
+  HAND_TRACKING_LANDMARK_SMOOTHING,
   HAND_TRACKING_RUNTIME_START_DELAY_MS,
   HAND_TRACKING_TARGET_FPS,
 } from "@/data/handTrackingConfig";
@@ -10,11 +11,15 @@ import {
   getBrowserHandLandmarker,
   releaseBrowserHandLandmarker,
 } from "@/lib/handTracking/browserHandTracking";
+import { smoothHands } from "@/lib/handTracking/handSmoothing";
 import {
   INITIAL_HAND_TRACKING_SNAPSHOT,
   getCameraStreamWithTimeout,
 } from "@/lib/handTracking/handTrackingSession";
-import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking";
+import type {
+  HandTrackingHand,
+  HandTrackingSnapshot,
+} from "@/types/handTracking/handTracking";
 import { logger } from "@/utils/core/Logger";
 
 interface UseBrowserHandTrackingOptions {
@@ -30,6 +35,7 @@ export function useBrowserHandTracking({
   const videoRef = useRef<HTMLVideoElement | null>(null);
   const streamRef = useRef<MediaStream | null>(null);
   const intervalRef = useRef<number | null>(null);
+  const previousHandsRef = useRef<HandTrackingHand[]>([]);
 
   useEffect(() => {
     if (!enabled) {
@@ -51,6 +57,7 @@ export function useBrowserHandTracking({
       streamRef.current?.getTracks().forEach((track) => track.stop());
       streamRef.current = null;
       videoRef.current = null;
+      previousHandsRef.current = [];
       releaseBrowserHandLandmarker();
     };
 
@@ -124,7 +131,13 @@ export function useBrowserHandTracking({
               video,
               performance.now(),
             );
-            const hands = convertBrowserHandResult(result);
+            const rawHands = convertBrowserHandResult(result);
+            const hands = smoothHands(
+              previousHandsRef.current,
+              rawHands,
+              HAND_TRACKING_LANDMARK_SMOOTHING,
+            );
+            previousHandsRef.current = hands;
 
             setSnapshot((current) => ({
               ...current,

src/lib/handTracking/handSmoothing.ts

@@ -0,0 +1,64 @@
+import type {
+  HandTrackingHand,
+  HandTrackingLandmark,
+} from "@/types/handTracking/handTracking";
+
+function lerp(previous: number, next: number, factor: number): number {
+  return previous * (1 - factor) + next * factor;
+}
+
+function smoothLandmark(
+  previous: HandTrackingLandmark,
+  next: HandTrackingLandmark,
+  factor: number,
+): HandTrackingLandmark {
+  return {
+    x: lerp(previous.x, next.x, factor),
+    y: lerp(previous.y, next.y, factor),
+    z: lerp(previous.z, next.z, factor),
+  };
+}
+
+function smoothHand(
+  previous: HandTrackingHand,
+  next: HandTrackingHand,
+  factor: number,
+): HandTrackingHand {
+  return {
+    ...next,
+    x: lerp(previous.x, next.x, factor),
+    y: lerp(previous.y, next.y, factor),
+    z: lerp(previous.z, next.z, factor),
+    landmarks: next.landmarks.map((landmark, index) => {
+      const previousLandmark = previous.landmarks[index];
+      if (!previousLandmark) return landmark;
+      return smoothLandmark(previousLandmark, landmark, factor);
+    }),
+  };
+}
+
+/**
+ * Apply an exponential moving average to the landmark positions of each
+ * detected hand. MediaPipe lands per-frame positions with noticeable
+ * jitter (especially at ~10fps), and feeding those raw values into the
+ * scene makes both the glove rig and any grabbed object tremble.
+ *
+ * `factor` is the weight given to the latest sample (0 = previous frame
+ * only, 1 = no smoothing). Hands are matched between frames by
+ * handedness so left/right don't bleed into each other.
+ */
+export function smoothHands(
+  previousHands: HandTrackingHand[],
+  nextHands: HandTrackingHand[],
+  factor: number,
+): HandTrackingHand[] {
+  if (factor >= 1) return nextHands;
+
+  return nextHands.map((next) => {
+    const previous = previousHands.find(
+      (candidate) => candidate.handedness === next.handedness,
+    );
+    if (!previous) return next;
+    return smoothHand(previous, next, factor);
+  });
+}