update: enable hand tracking for repair steps (not only when we are close to something)

docs/technical/hand-tracking.md

@@ -4,9 +4,9 @@ This document describes the hand tracking system that exists in the current code
 
 ## Purpose
 
-Hand tracking is a debug-stage interaction system used to test direct 3D object manipulation with a webcam. It allows a user to close their fist to grab a nearby object and move it in 3D space without relying on the center crosshair.
+Hand tracking started as a debug-stage interaction system used to test direct 3D object manipulation with a webcam. It allows a user to close their fist to grab a nearby object and move it in 3D space without relying on the center crosshair.
 
-The feature is scoped to the debug physics scene rather than production gameplay input.
+It is now also available to the production repair flow when a mission reaches a hand-driven step.
 
 ## Runtime Flow
 
@@ -16,13 +16,13 @@ The feature is scoped to the debug physics scene rather than production gameplay
 4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`.
 5. React stores the latest snapshot in the hand tracking provider.
 6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects.
-7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands in the debug physics scene.
+7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands when hand tracking is active.
 
 ## Activation Rules
 
 Hand tracking is intentionally gated so the webcam and backend are not used all the time.
 
-The current activation conditions are:
+The debug activation conditions are:
 
 - debug mode is active with `?debug`
 - scene mode is `physics`
@@ -30,6 +30,13 @@ The current activation conditions are:
 
 This keeps hand tracking active while the player is inside an interaction zone, even if the camera is not aimed directly at the object.
 
+The production repair activation conditions are:
+
+- active `mainState` is `bike`, `pylone`, or `ferme`
+- the active mission step is `inspected`, `repairing`, or `done`
+
+This keeps the webcam off during `waiting`, `fragmented`, and `scanning`, then enables hand input only when the repair flow is expected to use hands.
+
 ## Backend
 
 The backend lives in `backend/` and exposes:
@@ -121,7 +128,7 @@ The glove models are intentionally smaller than the raw SVG overlay so they do n
 
 ## Known Limitations
 
-- The feature is debug-only and focused on the physics test scene.
+- Production usage is currently limited to repair mission steps that explicitly need hands.
 - MediaPipe depth is relative and can be noisy.
 - The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider.
 - There is no smoothing layer for hand position or depth yet.

src/providers/gameplay/HandTrackingProvider.tsx

@@ -8,6 +8,14 @@ import {
 } from "@/hooks/handTracking/useHandTrackingSnapshot";
 import { useBrowserHandTracking } from "@/hooks/handTracking/useBrowserHandTracking";
 import { useRemoteHandTracking } from "@/hooks/handTracking/useRemoteHandTracking";
+import { useGameStore } from "@/managers/stores/useGameStore";
+import type { MissionStep } from "@/managers/stores/useGameStore";
+
+const REPAIR_HAND_TRACKING_STEPS = new Set<MissionStep>([
+  "inspected",
+  "repairing",
+  "done",
+]);
 
 export function HandTrackingProvider({
   children,
@@ -18,8 +26,23 @@ export function HandTrackingProvider({
   const handTrackingSource = useDebugStore((debug) =>
     debug.getHandTrackingSource(),
   );
+  const repairNeedsHands = useGameStore((state) => {
+    switch (state.mainState) {
+      case "bike":
+        return REPAIR_HAND_TRACKING_STEPS.has(state.bike.currentStep);
+      case "pylone":
+        return REPAIR_HAND_TRACKING_STEPS.has(state.pylone.currentStep);
+      case "ferme":
+        return REPAIR_HAND_TRACKING_STEPS.has(state.ferme.currentStep);
+      case "intro":
+      case "outro":
+        return false;
+    }
+  });
   const { nearby, holding, handHolding } = useInteraction();
-  const enabled = sceneMode === "physics" && (nearby || holding || handHolding);
+  const enabled =
+    repairNeedsHands ||
+    (sceneMode === "physics" && (nearby || holding || handHolding));
   const backendSnapshot = useRemoteHandTracking({
     enabled: enabled && handTrackingSource === "backend",
   });

src/world/World.tsx

@@ -7,6 +7,7 @@ import {
 } from "@/data/player/playerConfig";
 import { useCameraMode } from "@/hooks/debug/useCameraMode";
 import { useSceneMode } from "@/hooks/debug/useSceneMode";
+import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
 import { DebugCameraControls } from "@/components/debug/scene/DebugCameraControls";
 import { DebugHelpers } from "@/components/debug/scene/DebugHelpers";
 import { HandTrackingGlove } from "@/components/three/handTracking/HandTrackingGlove";
@@ -21,25 +22,28 @@ import { TestMap } from "@/world/debug/TestMap";
 export function World(): React.JSX.Element {
   const cameraMode = useCameraMode();
   const sceneMode = useSceneMode();
+  const { status, usageStatus } = useHandTrackingSnapshot();
   const [octree, setOctree] = useState<Octree | null>(null);
   const playerSpawnPosition =
     sceneMode === "game"
       ? PLAYER_SPAWN_POSITION_GAME
       : PLAYER_SPAWN_POSITION_PHYSICS;
+  const showHandTrackingGloves =
+    sceneMode === "physics" ||
+    (status !== "idle" && usageStatus !== "inactive");
 
   return (
     <>
       <Environment />
       <Lighting />
       <DebugHelpers />
-      {sceneMode === "physics" ? (
+      {showHandTrackingGloves ? (
         <>
           <HandTrackingGlove handedness="left" />
           <HandTrackingGlove handedness="right" />
         </>
       ) : null}
       {cameraMode === "debug" ? <DebugCameraControls /> : null}
-
       {sceneMode === "game" ? (
         <>
           <GameMusic />
@@ -51,7 +55,6 @@ export function World(): React.JSX.Element {
       ) : (
         <TestMap onOctreeReady={setOctree} />
       )}
-
       {cameraMode !== "debug" ? (
         <Player octree={octree} spawnPosition={playerSpawnPosition} />
       ) : null}