Date: Tue, 2 Jun 2026 19:05:39 +0200
Subject: [PATCH 11/14] feat(handtracking): restyle svg visualizer and add
silhouette fallback
---
public/models/gant_l/model.gltf | 4 +-
.../three/handTracking/HandTrackingGlove.tsx | 26 ++++--
src/components/ui/GameUI.tsx | 2 +
src/components/ui/HandTrackingFallback.tsx | 82 +++++++++++++++++++
src/components/ui/HandTrackingVisualizer.tsx | 20 +++--
src/data/handTrackingConfig.ts | 8 +-
.../handTracking/useBrowserHandTracking.ts | 17 +++-
src/lib/handTracking/handSmoothing.ts | 64 +++++++++++++++
8 files changed, 207 insertions(+), 16 deletions(-)
create mode 100644 src/components/ui/HandTrackingFallback.tsx
create mode 100644 src/lib/handTracking/handSmoothing.ts
diff --git a/public/models/gant_l/model.gltf b/public/models/gant_l/model.gltf
index 30a1036..61ecc6e 100644
--- a/public/models/gant_l/model.gltf
+++ b/public/models/gant_l/model.gltf
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:51299d4fc4df3466e0367e9f4371ab3f71f4a0fd476518ef61f86ec10f7a302e
-size 10304
+oid sha256:0fcc5a63d512ec6bb0fd047f3a3d799f680c4ef5ed88b1ce65b5b73c6201e4f3
+size 10327
diff --git a/src/components/three/handTracking/HandTrackingGlove.tsx b/src/components/three/handTracking/HandTrackingGlove.tsx
index d5c8904..015bf10 100644
--- a/src/components/three/handTracking/HandTrackingGlove.tsx
+++ b/src/components/three/handTracking/HandTrackingGlove.tsx
@@ -12,6 +12,11 @@ import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
import { logModelLoadError } from "@/utils/three/modelLoadLogger";
+// Both gloves share the same source mesh (gant_l). The right glove is
+// rendered by mirroring scale.x at the group level — this is more
+// reliable than the historical gant_r GLTF, which embeds multiple
+// skeletons (Hand_l, Hand_l_pad, Hand_r) and was breaking the finger
+// rig.
const GLOVE_CONFIGS: Record<
HandTrackingGloveHandedness,
{
@@ -24,8 +29,8 @@ const GLOVE_CONFIGS: Record<
rootNodeName: "Armature",
},
right: {
- modelPath: "/models/gant_r/model.gltf",
- rootNodeName: "Hand_r",
+ modelPath: "/models/gant_l/model.gltf",
+ rootNodeName: "Armature",
},
};
@@ -226,7 +231,10 @@ function applyFingerPose(
_boneTargetQuaternion
.copy(_boneDeltaQuaternion)
.multiply(pose.restQuaternion);
- pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.45);
+ // Lower slerp factor = smoother but more latency. MediaPipe at
+ // ~10fps produces noisy landmark frames; smoothing cuts the
+ // jitter the user sees on every finger bone.
+ pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.3);
}
}
}
@@ -334,12 +342,18 @@ function HandTrackingGloveModel({
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
_targetQuaternion.setFromRotationMatrix(_matrix);
- group.position.lerp(_targetPosition, Math.min(1, delta * 18));
- group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
+ // Lower factor (was 18) damps the glove jitter caused by noisy
+ // landmarks while keeping a responsive feel.
+ group.position.lerp(_targetPosition, Math.min(1, delta * 12));
+ group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 12));
const palmLength = _wristPosition.distanceTo(_middlePosition);
const scale = palmLength * GLOVE_MODEL_SCALE;
- group.scale.setScalar(scale);
+ // Both gloves use the gant_l mesh; flip X for the right hand so the
+ // thumb ends up on the correct side instead of being a left-glove
+ // clone on the right hand.
+ const mirrorSignX = handedness === "right" ? -1 : 1;
+ group.scale.set(scale * mirrorSignX, scale, scale);
group.updateMatrixWorld(true);
applyFingerPose(fingerPoseChains, trackedHand.landmarks, camera);
});
diff --git a/src/components/ui/GameUI.tsx b/src/components/ui/GameUI.tsx
index 526c3b6..c7a1b49 100644
--- a/src/components/ui/GameUI.tsx
+++ b/src/components/ui/GameUI.tsx
@@ -1,6 +1,7 @@
import { Crosshair } from "@/components/ui/Crosshair";
import { DebugOverlayLayout } from "@/components/ui/debug/DebugOverlayLayout";
import { GameSettingsMenu } from "@/components/ui/GameSettingsMenu";
+import { HandTrackingFallback } from "@/components/ui/HandTrackingFallback";
import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer";
import { InteractPrompt } from "@/components/ui/InteractPrompt";
import { RepairMovementLockIndicator } from "@/components/ui/RepairMovementLockIndicator";
@@ -15,6 +16,7 @@ export function GameUI(): React.JSX.Element {
+
diff --git a/src/components/ui/HandTrackingFallback.tsx b/src/components/ui/HandTrackingFallback.tsx
new file mode 100644
index 0000000..b5d2d53
--- /dev/null
+++ b/src/components/ui/HandTrackingFallback.tsx
@@ -0,0 +1,82 @@
+import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
+import {
+ useHandTrackingGloveStatus,
+ type HandTrackingGloveHandedness,
+} from "@/hooks/handTracking/useHandTrackingGloveStatus";
+
+// Simple schematic silhouettes used as a last-resort fallback when the
+// rigged glove model has failed to load. Both icons share the same
+// 48x48 viewBox and the same stroke/fill rules from the .css.
+
+const OpenHandShape = (): React.JSX.Element => (
+ <>
+
+
+
+
+
+
+
+);
+
+const FistShape = (): React.JSX.Element => (
+ <>
+
+
+
+
+
+
+
+);
+
+function getHandedness(raw: string): HandTrackingGloveHandedness | null {
+ const lower = raw.toLowerCase();
+ if (lower === "left" || lower === "right") return lower;
+ return null;
+}
+
+export function HandTrackingFallback(): React.JSX.Element | null {
+ const { hands } = useHandTrackingSnapshot();
+ const gloveStatus = useHandTrackingGloveStatus((state) => state.gloves);
+
+ const visibleHands = hands.flatMap((hand, index) => {
+ const handedness = getHandedness(hand.handedness);
+ if (!handedness) return [];
+ if (gloveStatus[handedness] !== "error") return [];
+
+ const wrist = hand.landmarks[0];
+ if (!wrist) return [];
+
+ return [{ hand, handedness, wrist, index }];
+ });
+
+ if (visibleHands.length === 0) return null;
+
+ return (
+
+ {visibleHands.map(({ hand, handedness, wrist, index }) => {
+ // MediaPipe coords are mirrored (selfie cam), keep the same
+ // mapping the SVG visualizer uses.
+ const leftPercent = (1 - wrist.x) * 100;
+ const topPercent = wrist.y * 100;
+ const flipX = handedness === "right" ? -1 : 1;
+
+ return (
+
+ );
+ })}
+
+ );
+}
diff --git a/src/components/ui/HandTrackingVisualizer.tsx b/src/components/ui/HandTrackingVisualizer.tsx
index 2238554..283a870 100644
--- a/src/components/ui/HandTrackingVisualizer.tsx
+++ b/src/components/ui/HandTrackingVisualizer.tsx
@@ -26,6 +26,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [
[0, 17],
];
+const LANDMARK_FILL = "#67e8f9"; // cyan-300, opaque interior
+const LANDMARK_STROKE = "#0c4a6e"; // sky-900, dark blue outline
+const LANDMARK_STROKE_FIST = "#1e3a8a"; // blue-900, thicker accent when fist
+const CONNECTION_STROKE = "#ffffff"; // white bones
+const INDEX_TIP_LANDMARK = 8;
+
export function HandTrackingVisualizer(): React.JSX.Element | null {
const { hands, status } = useHandTrackingSnapshot();
const showHandTrackingSvg = useDebugStore((debug) =>
@@ -50,7 +56,9 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
const landmarks = hand.landmarks;
if (landmarks.length === 0) return null;
- const color = hand.isFist ? "#facc15" : "#38bdf8";
+ const landmarkStroke = hand.isFist
+ ? LANDMARK_STROKE_FIST
+ : LANDMARK_STROKE;
return (
@@ -66,8 +74,8 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
y1={`${fromPoint.y * 100}%`}
x2={`${(1 - toPoint.x) * 100}%`}
y2={`${toPoint.y * 100}%`}
- stroke={color}
- strokeWidth="2"
+ stroke={CONNECTION_STROKE}
+ strokeWidth="2.5"
strokeLinecap="round"
/>
);
@@ -78,8 +86,10 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
key={landmarkIndex}
cx={`${(1 - landmark.x) * 100}%`}
cy={`${landmark.y * 100}%`}
- r={landmarkIndex === 8 ? 5 : 3}
- fill={landmarkIndex === 8 ? "#ffffff" : color}
+ r={landmarkIndex === INDEX_TIP_LANDMARK ? 6 : 4}
+ fill={LANDMARK_FILL}
+ stroke={landmarkStroke}
+ strokeWidth={hand.isFist ? 2.5 : 2}
/>
))}
diff --git a/src/data/handTrackingConfig.ts b/src/data/handTrackingConfig.ts
index 9537124..e07df5a 100644
--- a/src/data/handTrackingConfig.ts
+++ b/src/data/handTrackingConfig.ts
@@ -14,7 +14,7 @@ export const HAND_TRACKING_BROWSER_WASM_URL =
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm";
export const HAND_TRACKING_BROWSER_MODEL_URL =
"https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task";
-export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "CPU";
+export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "GPU";
// Delay before the runtime actually starts after `enabled` flips to true.
// Absorbs React StrictMode's mount/unmount/mount cycle in dev and rapid
@@ -26,3 +26,9 @@ export const HAND_TRACKING_RUNTIME_START_DELAY_MS = 80;
// to initialize webcam + model + first frame inference before we cleanup,
// so the user actually sees their hands when entering a zone briefly.
export const HAND_TRACKING_LINGER_MS = 2000;
+
+// EMA weight applied to the latest landmark frame. Lower = smoother but
+// laggier; higher = more responsive but more jitter from raw MediaPipe
+// noise. 0.4 keeps the glove and grabbed objects from trembling without
+// feeling sluggish.
+export const HAND_TRACKING_LANDMARK_SMOOTHING = 0.4;
diff --git a/src/hooks/handTracking/useBrowserHandTracking.ts b/src/hooks/handTracking/useBrowserHandTracking.ts
index 9364dcf..8c5dbc5 100644
--- a/src/hooks/handTracking/useBrowserHandTracking.ts
+++ b/src/hooks/handTracking/useBrowserHandTracking.ts
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react";
import {
HAND_TRACKING_BROWSER_CAMERA_HEIGHT,
HAND_TRACKING_BROWSER_CAMERA_WIDTH,
+ HAND_TRACKING_LANDMARK_SMOOTHING,
HAND_TRACKING_RUNTIME_START_DELAY_MS,
HAND_TRACKING_TARGET_FPS,
} from "@/data/handTrackingConfig";
@@ -10,11 +11,15 @@ import {
getBrowserHandLandmarker,
releaseBrowserHandLandmarker,
} from "@/lib/handTracking/browserHandTracking";
+import { smoothHands } from "@/lib/handTracking/handSmoothing";
import {
INITIAL_HAND_TRACKING_SNAPSHOT,
getCameraStreamWithTimeout,
} from "@/lib/handTracking/handTrackingSession";
-import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking";
+import type {
+ HandTrackingHand,
+ HandTrackingSnapshot,
+} from "@/types/handTracking/handTracking";
import { logger } from "@/utils/core/Logger";
interface UseBrowserHandTrackingOptions {
@@ -30,6 +35,7 @@ export function useBrowserHandTracking({
const videoRef = useRef(null);
const streamRef = useRef(null);
const intervalRef = useRef(null);
+ const previousHandsRef = useRef([]);
useEffect(() => {
if (!enabled) {
@@ -51,6 +57,7 @@ export function useBrowserHandTracking({
streamRef.current?.getTracks().forEach((track) => track.stop());
streamRef.current = null;
videoRef.current = null;
+ previousHandsRef.current = [];
releaseBrowserHandLandmarker();
};
@@ -124,7 +131,13 @@ export function useBrowserHandTracking({
video,
performance.now(),
);
- const hands = convertBrowserHandResult(result);
+ const rawHands = convertBrowserHandResult(result);
+ const hands = smoothHands(
+ previousHandsRef.current,
+ rawHands,
+ HAND_TRACKING_LANDMARK_SMOOTHING,
+ );
+ previousHandsRef.current = hands;
setSnapshot((current) => ({
...current,
diff --git a/src/lib/handTracking/handSmoothing.ts b/src/lib/handTracking/handSmoothing.ts
new file mode 100644
index 0000000..70ff33a
--- /dev/null
+++ b/src/lib/handTracking/handSmoothing.ts
@@ -0,0 +1,64 @@
+import type {
+ HandTrackingHand,
+ HandTrackingLandmark,
+} from "@/types/handTracking/handTracking";
+
+function lerp(previous: number, next: number, factor: number): number {
+ return previous * (1 - factor) + next * factor;
+}
+
+function smoothLandmark(
+ previous: HandTrackingLandmark,
+ next: HandTrackingLandmark,
+ factor: number,
+): HandTrackingLandmark {
+ return {
+ x: lerp(previous.x, next.x, factor),
+ y: lerp(previous.y, next.y, factor),
+ z: lerp(previous.z, next.z, factor),
+ };
+}
+
+function smoothHand(
+ previous: HandTrackingHand,
+ next: HandTrackingHand,
+ factor: number,
+): HandTrackingHand {
+ return {
+ ...next,
+ x: lerp(previous.x, next.x, factor),
+ y: lerp(previous.y, next.y, factor),
+ z: lerp(previous.z, next.z, factor),
+ landmarks: next.landmarks.map((landmark, index) => {
+ const previousLandmark = previous.landmarks[index];
+ if (!previousLandmark) return landmark;
+ return smoothLandmark(previousLandmark, landmark, factor);
+ }),
+ };
+}
+
+/**
+ * Apply an exponential moving average to the landmark positions of each
+ * detected hand. MediaPipe lands per-frame positions with noticeable
+ * jitter (especially at ~10fps), and feeding those raw values into the
+ * scene makes both the glove rig and any grabbed object tremble.
+ *
+ * `factor` is the weight given to the latest sample (0 = previous frame
+ * only, 1 = no smoothing). Hands are matched between frames by
+ * handedness so left/right don't bleed into each other.
+ */
+export function smoothHands(
+ previousHands: HandTrackingHand[],
+ nextHands: HandTrackingHand[],
+ factor: number,
+): HandTrackingHand[] {
+ if (factor >= 1) return nextHands;
+
+ return nextHands.map((next) => {
+ const previous = previousHands.find(
+ (candidate) => candidate.handedness === next.handedness,
+ );
+ if (!previous) return next;
+ return smoothHand(previous, next, factor);
+ });
+}
From 6edc5f7972bd98c477f756d67ca02df16d7de937 Mon Sep 17 00:00:00 2001
From: Tom Boullay
Date: Tue, 2 Jun 2026 19:06:32 +0200
Subject: [PATCH 12/14] docs: refresh hand-tracking notes and drop context-lost
investigation
---
docs/technical/hand-tracking.md | 88 ++++-
.../webgl-context-lost-investigation.md | 367 ------------------
.../handTracking/useRemoteHandTracking.ts | 14 +-
src/index.css | 20 +
4 files changed, 100 insertions(+), 389 deletions(-)
delete mode 100644 docs/technical/webgl-context-lost-investigation.md
diff --git a/docs/technical/hand-tracking.md b/docs/technical/hand-tracking.md
index 0ab9dd1..dc2ece6 100644
--- a/docs/technical/hand-tracking.md
+++ b/docs/technical/hand-tracking.md
@@ -10,17 +10,23 @@ It is now also available to the production repair flow when a mission reaches a
## Runtime Flow
-1. The browser captures webcam frames in `src/hooks/handTracking/useRemoteHandTracking.ts`.
-2. Frames are sent to the local Python backend over WebSocket.
-3. The backend runs MediaPipe hand landmark detection.
-4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`.
-5. React stores the latest snapshot in the hand tracking provider.
-6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects.
-7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands when hand tracking is active.
+The frontend can run hand tracking with two interchangeable sources, selected from the debug source controller:
+
+- **Browser JS** (`src/hooks/handTracking/useBrowserHandTracking.ts`) runs MediaPipe `hand_landmarker.task` directly in the browser via `@mediapipe/tasks-vision`. Default for debug.
+- **Backend** (`src/hooks/handTracking/useRemoteHandTracking.ts`) sends webcam frames as JPEG over WebSocket to a local Python process that runs MediaPipe and returns landmarks.
+
+Both sources funnel into the same `HandTrackingContext` so all consumers see one shared snapshot:
+
+1. The active source captures or receives landmarks.
+2. The hook applies an EMA smoothing pass on the landmarks before publishing the snapshot.
+3. `HandTrackingProvider` exposes that snapshot through React context.
+4. `GrabbableObject` reads the snapshot each frame and uses the fist state plus raycasting to grab objects.
+5. `HandTrackingGlove` reads the same snapshot and places a rigged glove on each detected hand.
+6. `HandTrackingVisualizer` paints an SVG wireframe overlay on top of the canvas.
## Activation Rules
-Hand tracking is intentionally gated so the webcam and backend are not used all the time.
+Hand tracking is gated so the webcam and runtime are only spun up when actually needed.
The debug activation conditions are:
@@ -28,16 +34,26 @@ The debug activation conditions are:
- scene mode is `physics`
- the player is near an interaction, is holding an object, or is hand-holding an object
-This keeps hand tracking active while the player is inside an interaction zone, even if the camera is not aimed directly at the object.
-
The production repair activation conditions are:
- active `mainState` is `ebike`, `pylon`, or `farm`
- the active mission step is `inspected`, `repairing`, `reassembling`, or `done`
-This keeps the webcam off during `waiting`, `fragmented`, and `scanning`, then enables hand input only when the repair flow is expected to use hands.
+This keeps the webcam off during `waiting`, `fragmented`, and `scanning`.
-In the current production repair flow, `inspected` uses a two-fists hold gesture to advance to `fragmented`. The hold must last one second and is independent from local object interaction distance once the mission is in the correct state. Keyboard input for the same transition is handled separately by the repair case trigger, so pressing `E` requires the case to be focused through the shared interaction system.
+### Linger
+
+Once activation turns off (player walks back out of a trigger zone, or a mission step transitions away), the runtime stays alive for `HAND_TRACKING_LINGER_MS` (2000 ms) before being torn down. This gives MediaPipe enough time to finish initializing the webcam and load the model on a fresh entry — without the linger, a quick walk-through of a trigger zone never produces a detected hand.
+
+## Provider Stability
+
+`HandTrackingProvider` always renders the same JSX root (`HandTrackingRuntime`) and exposes `enabled` as a prop. Returning two different element types (`` vs ``) used to be the historical shape and was the root cause of WebGL context loss: every `enabled` toggle forced React to remount the entire subtree, including the `