diff --git a/docs/technical/hand-tracking.md b/docs/technical/hand-tracking.md index 6482ff3..a844a81 100644 --- a/docs/technical/hand-tracking.md +++ b/docs/technical/hand-tracking.md @@ -4,9 +4,9 @@ This document describes the hand tracking system that exists in the current code ## Purpose -Hand tracking is a debug-stage interaction system used to test direct 3D object manipulation with a webcam. It allows a user to close their fist to grab a nearby object and move it in 3D space without relying on the center crosshair. +Hand tracking started as a debug-stage interaction system used to test direct 3D object manipulation with a webcam. It allows a user to close their fist to grab a nearby object and move it in 3D space without relying on the center crosshair. -The feature is scoped to the debug physics scene rather than production gameplay input. +It is now also available to the production repair flow when a mission reaches a hand-driven step. ## Runtime Flow @@ -16,13 +16,13 @@ The feature is scoped to the debug physics scene rather than production gameplay 4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`. 5. React stores the latest snapshot in the hand tracking provider. 6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects. -7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands in the debug physics scene. +7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands when hand tracking is active. ## Activation Rules Hand tracking is intentionally gated so the webcam and backend are not used all the time. -The current activation conditions are: +The debug activation conditions are: - debug mode is active with `?debug` - scene mode is `physics` @@ -30,6 +30,13 @@ The current activation conditions are: This keeps hand tracking active while the player is inside an interaction zone, even if the camera is not aimed directly at the object. +The production repair activation conditions are: + +- active `mainState` is `bike`, `pylone`, or `ferme` +- the active mission step is `inspected`, `repairing`, or `done` + +This keeps the webcam off during `waiting`, `fragmented`, and `scanning`, then enables hand input only when the repair flow is expected to use hands. + ## Backend The backend lives in `backend/` and exposes: @@ -121,7 +128,7 @@ The glove models are intentionally smaller than the raw SVG overlay so they do n ## Known Limitations -- The feature is debug-only and focused on the physics test scene. +- Production usage is currently limited to repair mission steps that explicitly need hands. - MediaPipe depth is relative and can be noisy. - The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider. - There is no smoothing layer for hand position or depth yet. diff --git a/src/providers/gameplay/HandTrackingProvider.tsx b/src/providers/gameplay/HandTrackingProvider.tsx index e17fbfd..4c00603 100644 --- a/src/providers/gameplay/HandTrackingProvider.tsx +++ b/src/providers/gameplay/HandTrackingProvider.tsx @@ -8,6 +8,14 @@ import { } from "@/hooks/handTracking/useHandTrackingSnapshot"; import { useBrowserHandTracking } from "@/hooks/handTracking/useBrowserHandTracking"; import { useRemoteHandTracking } from "@/hooks/handTracking/useRemoteHandTracking"; +import { useGameStore } from "@/managers/stores/useGameStore"; +import type { MissionStep } from "@/managers/stores/useGameStore"; + +const REPAIR_HAND_TRACKING_STEPS = new Set([ + "inspected", + "repairing", + "done", +]); export function HandTrackingProvider({ children, @@ -18,8 +26,23 @@ export function HandTrackingProvider({ const handTrackingSource = useDebugStore((debug) => debug.getHandTrackingSource(), ); + const repairNeedsHands = useGameStore((state) => { + switch (state.mainState) { + case "bike": + return REPAIR_HAND_TRACKING_STEPS.has(state.bike.currentStep); + case "pylone": + return REPAIR_HAND_TRACKING_STEPS.has(state.pylone.currentStep); + case "ferme": + return REPAIR_HAND_TRACKING_STEPS.has(state.ferme.currentStep); + case "intro": + case "outro": + return false; + } + }); const { nearby, holding, handHolding } = useInteraction(); - const enabled = sceneMode === "physics" && (nearby || holding || handHolding); + const enabled = + repairNeedsHands || + (sceneMode === "physics" && (nearby || holding || handHolding)); const backendSnapshot = useRemoteHandTracking({ enabled: enabled && handTrackingSource === "backend", }); diff --git a/src/world/World.tsx b/src/world/World.tsx index 27be858..fef41fa 100644 --- a/src/world/World.tsx +++ b/src/world/World.tsx @@ -7,6 +7,7 @@ import { } from "@/data/player/playerConfig"; import { useCameraMode } from "@/hooks/debug/useCameraMode"; import { useSceneMode } from "@/hooks/debug/useSceneMode"; +import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; import { DebugCameraControls } from "@/components/debug/scene/DebugCameraControls"; import { DebugHelpers } from "@/components/debug/scene/DebugHelpers"; import { HandTrackingGlove } from "@/components/three/handTracking/HandTrackingGlove"; @@ -21,25 +22,28 @@ import { TestMap } from "@/world/debug/TestMap"; export function World(): React.JSX.Element { const cameraMode = useCameraMode(); const sceneMode = useSceneMode(); + const { status, usageStatus } = useHandTrackingSnapshot(); const [octree, setOctree] = useState(null); const playerSpawnPosition = sceneMode === "game" ? PLAYER_SPAWN_POSITION_GAME : PLAYER_SPAWN_POSITION_PHYSICS; + const showHandTrackingGloves = + sceneMode === "physics" || + (status !== "idle" && usageStatus !== "inactive"); return ( <> - {sceneMode === "physics" ? ( + {showHandTrackingGloves ? ( <> ) : null} {cameraMode === "debug" ? : null} - {sceneMode === "game" ? ( <> @@ -51,7 +55,6 @@ export function World(): React.JSX.Element { ) : ( )} - {cameraMode !== "debug" ? ( ) : null}