diff --git a/docs/technical/hand-tracking.md b/docs/technical/hand-tracking.md index dc2ece6..e2a1b0b 100644 --- a/docs/technical/hand-tracking.md +++ b/docs/technical/hand-tracking.md @@ -20,9 +20,11 @@ Both sources funnel into the same `HandTrackingContext` so all consumers see one 1. The active source captures or receives landmarks. 2. The hook applies an EMA smoothing pass on the landmarks before publishing the snapshot. 3. `HandTrackingProvider` exposes that snapshot through React context. -4. `GrabbableObject` reads the snapshot each frame and uses the fist state plus raycasting to grab objects. -5. `HandTrackingGlove` reads the same snapshot and places a rigged glove on each detected hand. -6. `HandTrackingVisualizer` paints an SVG wireframe overlay on top of the canvas. +4. `GrabbableObject` reads the snapshot each frame and uses `hand.isFist` plus raycasting to grab objects. +5. `HandTrackingVisualizer` paints the SVG hand silhouette overlay on top of the canvas — the primary visualization. +6. `HandTrackingGlove` (opt-in, see UI And Debug) places a rigged 3D glove on each detected hand when enabled via the debug toggle. + +All consumers — fist detection, grab raycasting, SVG silhouette, optional 3D glove — read the **same** landmarks from the snapshot. None of them depend on the others. ## Activation Rules @@ -108,6 +110,17 @@ interface HandTrackingHand { `x` and `y` are normalized camera coordinates. `z` is a relative depth value from MediaPipe, not an absolute world-space distance. +## Fist Detection + +`isFist` is computed in `src/lib/handTracking/browserHandTracking.ts` (`isFist()` function) from landmarks alone — no model, no glove. The check is: + +1. Palm center = mean of landmarks `[0, 5, 9, 13, 17]` (wrist + 4 MCPs). +2. Palm size = distance from wrist (landmark 0) to middle MCP (landmark 9). +3. For each of the four fingertip landmarks `[8, 12, 16, 20]`, check whether its distance to the palm center is less than `1.05 × palmSize`. +4. `isFist === true` iff all four fingertips pass the check. + +The flag is attached to each hand on the snapshot at the publish step (`isFist: isFist(normalizedLandmarks)`) and read directly by `GrabbableObject.tsx` — the SVG visualizer and the 3D glove never participate in the gesture decision. + ## Grab Targeting The hand grab logic lives in `src/components/three/interaction/GrabbableObject.tsx`. @@ -142,18 +155,40 @@ This is less expressive than true depth-aware hand movement, but it is more stab The current debug UI includes: - `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, loaded glove model, server state, hand count, and fist state -- `HandTrackingVisualizer` for the SVG landmark overlay -- `HandTrackingFallback` for the last-resort hand silhouette overlay -- `HandTrackingGlove` for the per-hand rigged glove models in the R3F scene +- `HandTrackingVisualizer` for the SVG hand silhouette overlay (always on when tracking is active) +- `HandTrackingFallback` for the last-resort hand silhouette overlay (legacy, see below) +- `HandTrackingGlove` for the per-hand rigged glove models in the R3F scene, opt-in via the **Show Model** toggle - `r3f-perf` for render performance - `lil-gui` for scene, camera, lighting, interaction, and grab controls -The SVG visualizer uses a "blueish hand" style: white connection lines between landmarks, cyan circles with a dark blue outline. The outline gets thicker when the hand is detected as a fist, so the user gets a visual confirmation of the grab gesture without having to look at the debug panel. +### SVG Visualizer -The fallback overlay (`HandTrackingFallback`) draws a simple open-hand or fist silhouette positioned on the detected wrist landmark. It only renders for a hand whose matching glove is in the `"error"` state in `useHandTrackingGloveStatus`. This guarantees the user always sees something on their hand even when the 3D glove model fails to load. +`HandTrackingVisualizer` is the primary hand visualization. It draws a light-blue hand silhouette with a crisp dark-blue outline by: + +1. Filling a palm polygon (landmarks `[1, 5, 9, 13, 17]` plus two synthetic wrist corners) and five finger tubes (thick rounded `stroke` along each finger's joint chain). +2. Wrapping the whole thing in an SVG `` that uses `feMorphology` to dilate the merged alpha by 2 px and subtract the original, producing a single continuous outline around the union — no internal seams where the palm and finger tubes overlap. +3. Shrinking every landmark toward the hand centroid by `RENDER_SCALE = 0.65` so the silhouette stays compact and doesn't dominate the screen. +4. Overlaying the 21 raw landmarks and 21 bones as faint translucent lines and dots, so the user can still see the MediaPipe data feeding the silhouette. + +The SVG only displays when MediaPipe is active and the debug **Show Model** toggle is off (default). When the toggle is on, the SVG hides and `HandTrackingGlove` takes over. + +### Show Model Toggle + +The `Hand Tracking` debug folder exposes a single visualization switch: + +- `showHandTrackingModel = false` (default): SVG visualizer renders, 3D glove is not mounted at all. +- `showHandTrackingModel = true`: SVG visualizer hides, 3D glove gets mounted for the detected hand(s). + +The 3D glove is treated as opt-in legacy because it had bugs (WebGL context loss, finger rig artefacts) and its hit/grab role was never load-bearing — grab has always read landmarks directly. + +### Fallback Overlay (legacy) + +`HandTrackingFallback` draws a simple open-hand or fist silhouette positioned on the detected wrist landmark. It renders for any hand whose glove is in the `"error"` state in `useHandTrackingGloveStatus`. Now that the glove is opt-in and rarely mounted, the fallback effectively only fires in the rare case where the user enables `showHandTrackingModel` and the glove fails to load. It is kept on disk for that edge case but is not part of the default visual path. ## Glove Models +The 3D glove is **opt-in** via the `Show Model` debug toggle (see UI And Debug). It is not mounted by default; the SVG visualizer is the primary hand UI. The information below applies only when the toggle is enabled. + `HandTrackingGlove` loads `public/models/gant_l/model.gltf` for both hands. The right hand applies `scale.x = -1` at the group level to mirror the mesh, so the thumb ends up on the correct side. Both hands therefore share the same rig and the same material. The historical `public/models/gant_r/model.gltf` is kept as legacy but is not loaded by the frontend — its GLB embeds three skeletons (`Hand_l`, `Hand_l_pad`, `Hand_r`) plus a `galet` mesh, which made the finger rig unreliable. @@ -172,6 +207,8 @@ They are intended for future swap-by-state usage but are **not yet rigged**. The - Production usage is currently limited to repair mission steps that explicitly need hands. - MediaPipe depth is relative and currently not used for stable object depth control. - The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider. +- The 3D glove is opt-in only (see `Show Model` toggle). Default visual is the SVG silhouette. +- `HandTrackingFallback` is legacy and effectively unused unless the glove toggle is enabled and the glove fails to load. - The right glove is a mirrored copy of `gant_l` rather than its own mesh; in the future a dedicated right-hand model would give a better visual. - The `_pad` glove variants are not rigged yet, so swap-by-state (normal ↔ pad) is not wired in. - Finger bone animation is an approximate landmark-to-bone mapping; it still needs calibration for per-model twist, offsets, and smoothing. diff --git a/src/components/ebike/Ebike.tsx b/src/components/ebike/Ebike.tsx index e7de0dc..ef323d3 100644 --- a/src/components/ebike/Ebike.tsx +++ b/src/components/ebike/Ebike.tsx @@ -334,7 +334,7 @@ export function Ebike({ const interactionLabel = mainState === "ebike" - ? "Lancer le repair game" + ? "Lancer le Repair Game" : movementMode === "walk" ? "Monter sur le bike" : "Descendre du bike"; @@ -344,13 +344,19 @@ export function Ebike({ // pollute the view. The prompt comes back the moment the bike comes to // a stop. window.ebikeDriveInputActive is published every frame by // PlayerController based on whether a movement key is currently held. + // Also hide entirely while the breakdown sequence is active — the bike + // must read as inert and non-interactive while the panne dialogue plays + // and during the auto-dismount that follows. const [isEbikeDriving, setIsEbikeDriving] = useState(false); + const [isEbikeBreakdown, setIsEbikeBreakdown] = useState(false); useFrame(() => { const driving = movementMode === "ebike" && window.ebikeDriveInputActive === true; if (driving !== isEbikeDriving) setIsEbikeDriving(driving); + const breakdown = window.ebikeBreakdownActive === true; + if (breakdown !== isEbikeBreakdown) setIsEbikeBreakdown(breakdown); }); - const showInteractPrompt = !isEbikeDriving; + const showInteractPrompt = !isEbikeDriving && !isEbikeBreakdown; const handleInteract = useCallback((): void => { if (window.ebikeBreakdownActive === true) return; diff --git a/src/components/site/SiteCard.tsx b/src/components/site/SiteCard.tsx index 97ae7d1..2dba434 100644 --- a/src/components/site/SiteCard.tsx +++ b/src/components/site/SiteCard.tsx @@ -22,8 +22,6 @@ export function SiteCard({ return "#b8b8b8"; }; - const borderColor = selected ? "#a8d5a2" : "rgba(255, 255, 255, 0.55)"; - const textColor = disabled ? "rgba(77, 77, 77, 0.72)" : "#4d4d4d"; return ( @@ -41,7 +39,9 @@ export function SiteCard({ height: isSituation ? "clamp(48px, 6vw, 60px)" : "clamp(140px, 18vw, 180px)", - border: `3px solid ${borderColor}`, + border: "3px solid rgba(255, 255, 255, 0.55)", + outline: selected ? "3px solid #a8d5a2" : "none", + outlineOffset: 0, background: getBackground(), cursor: disabled ? "not-allowed" : "pointer", display: "flex", diff --git a/src/components/site/SiteNamingScreen.tsx b/src/components/site/SiteNamingScreen.tsx index 1f04212..b4e1bdf 100644 --- a/src/components/site/SiteNamingScreen.tsx +++ b/src/components/site/SiteNamingScreen.tsx @@ -1,59 +1,133 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useEffect, useState } from "react"; import { useGameStore } from "@/managers/stores/useGameStore"; import { useSiteStore } from "@/managers/stores/useSiteStore"; +import { useSettingsStore } from "@/managers/stores/useSettingsStore"; import { SiteButton } from "@/components/site/SiteButton"; import { SITE_CONFIG } from "@/data/site/siteConfig"; import { SITE_DIALOGUE_IDS } from "@/data/site/dialogueIds"; -import { loadDialogueManifest } from "@/utils/dialogues/loadDialogueManifest"; +import { + loadDialogueManifest, + loadDialogueSubtitleCues, +} from "@/utils/dialogues/loadDialogueManifest"; import { playDialogueById, stopCurrentDialogue, } from "@/utils/dialogues/playDialogue"; +const TYPEWRITER_CHAR_DELAY_MS = 70; +// Fallback in case nothing else triggers the typewriter (audio failed to +// load, no subtitles, "ended" never fires). Long enough not to fire +// before the narration on a slow load. +const AUDIO_END_FALLBACK_MS = 8000; + /** - * Screen 3: Name input - * The displayed name is forced to SITE_CONFIG.presetPlayerName — the - * field reveals one letter per keystroke until the preset name is complete. + * Screen 3: Name reveal + * The player's preset name is revealed letter-by-letter inside the input + * once the naming dialogue finishes playing. The confirm button stays + * locked until the reveal completes. No user typing — the input is + * read-only and just acts as a typewriter target. */ export function SiteNamingScreen(): React.JSX.Element { const setStep = useSiteStore((state) => state.setStep); const setPlayerName = useGameStore((state) => state.setPlayerName); - const [charIndex, setCharIndex] = useState(0); - const inputRef = useRef(null); + const [revealedChars, setRevealedChars] = useState(0); + const [typewriterStarted, setTypewriterStarted] = useState(false); const presetPlayerName = SITE_CONFIG.presetPlayerName; - const displayValue = presetPlayerName.slice(0, charIndex); - const isComplete = charIndex >= presetPlayerName.length; + const displayValue = presetPlayerName.slice(0, revealedChars); + const isComplete = revealedChars >= presetPlayerName.length; + // Play the dialogue, then trigger the typewriter so it FINISHES at the + // same moment the narration ends. We compute that moment from the SRT + // cues: the last cue's endTime is where the narrator stops speaking, + // so we start typing `typewriterDuration` before that. useEffect(() => { let cancelled = false; + let audioElement: HTMLAudioElement | null = null; + let onTimeUpdate: (() => void) | null = null; + let fallbackTimer: ReturnType | null = null; + + const start = (): void => { + if (cancelled) return; + setTypewriterStarted(true); + }; + + const typewriterDurationSec = + (TYPEWRITER_CHAR_DELAY_MS * presetPlayerName.length) / 1000; void (async () => { const manifest = await loadDialogueManifest(); - if (cancelled || !manifest) return; - await playDialogueById(manifest, SITE_DIALOGUE_IDS.naming); + if (cancelled) return; + if (!manifest) { + start(); + return; + } + + // Resolve the dialogue + its SRT cues for the active subtitle language. + const dialogue = manifest.dialogues.find( + (item) => item.id === SITE_DIALOGUE_IDS.naming, + ); + const language = useSettingsStore.getState().subtitleLanguage; + const subtitleData = dialogue + ? await loadDialogueSubtitleCues(manifest, dialogue, language) + : null; + if (cancelled) return; + + audioElement = await playDialogueById(manifest, SITE_DIALOGUE_IDS.naming); + if (cancelled) return; + if (!audioElement) { + start(); + return; + } + + const lastCue = subtitleData?.cues[subtitleData.cues.length - 1]; + if (lastCue) { + // Trigger so the typewriter ends at the narration's end. + const audio = audioElement; + const triggerAt = Math.max(0, lastCue.endTime - typewriterDurationSec); + onTimeUpdate = (): void => { + if (audio.currentTime >= triggerAt) { + audio.removeEventListener("timeupdate", onTimeUpdate!); + start(); + } + }; + audio.addEventListener("timeupdate", onTimeUpdate); + } else { + // No SRT data — fall back to the audio "ended" event. + audioElement.addEventListener("ended", start, { once: true }); + } + + fallbackTimer = setTimeout(start, AUDIO_END_FALLBACK_MS); })(); return () => { cancelled = true; + if (fallbackTimer !== null) clearTimeout(fallbackTimer); + if (audioElement) { + if (onTimeUpdate) { + audioElement.removeEventListener("timeupdate", onTimeUpdate); + } + audioElement.removeEventListener("ended", start); + } stopCurrentDialogue(); }; - }, []); + }, [presetPlayerName.length]); + // Reveal the preset name one character at a time once the typewriter + // has been triggered. useEffect(() => { - inputRef.current?.focus(); - }, []); - - const handleNameChange = useCallback( - (event: React.ChangeEvent): void => { - const nextLength = Math.min( - event.target.value.length, - presetPlayerName.length, - ); - setCharIndex(nextLength); - }, - [presetPlayerName.length], - ); + if (!typewriterStarted) return; + const interval = setInterval(() => { + setRevealedChars((current) => { + if (current >= presetPlayerName.length) { + clearInterval(interval); + return current; + } + return current + 1; + }); + }, TYPEWRITER_CHAR_DELAY_MS); + return () => clearInterval(interval); + }, [typewriterStarted, presetPlayerName.length]); const handleConfirm = (): void => { if (isComplete) { @@ -98,17 +172,16 @@ export function SiteNamingScreen(): React.JSX.Element { margin: 0, }} > - Quel est votre prénom ? + Je suis… - - Votre personnage s'appelle {presetPlayerName}. Tapez{" "} - {presetPlayerName.length} caractères pour révéler son nom. - + + diff --git a/src/components/ui/HandTrackingFallback.tsx b/src/components/ui/HandTrackingFallback.tsx index b5d2d53..e874992 100644 --- a/src/components/ui/HandTrackingFallback.tsx +++ b/src/components/ui/HandTrackingFallback.tsx @@ -4,29 +4,70 @@ import { type HandTrackingGloveHandedness, } from "@/hooks/handTracking/useHandTrackingGloveStatus"; -// Simple schematic silhouettes used as a last-resort fallback when the -// rigged glove model has failed to load. Both icons share the same -// 48x48 viewBox and the same stroke/fill rules from the .css. +// Hand silhouettes used as a last-resort fallback when the rigged glove +// model has failed to load. Both icons share a 100x120 viewBox so finger +// lengths and the thumb angle stay anatomically readable. const OpenHandShape = (): React.JSX.Element => ( - <> - - - - - - - + ); const FistShape = (): React.JSX.Element => ( <> - - - - - - + + + + + + ); @@ -66,7 +107,7 @@ export function HandTrackingFallback(): React.JSX.Element | null { = [ +// MediaPipe indexes the 21 hand landmarks predictably: +// 0 wrist, 1-4 thumb (base→tip), 5-8 index, 9-12 middle, 13-16 ring, 17-20 pinky. +const FINGER_LANDMARKS: Array = [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + [13, 14, 15, 16], + [17, 18, 19, 20], +]; +const SKELETON_BONES: Array<[number, number]> = [ [0, 1], [1, 2], [2, 3], @@ -26,70 +34,187 @@ const HAND_CONNECTIONS: Array<[number, number]> = [ [0, 17], ]; -const LANDMARK_FILL = "#67e8f9"; // cyan-300, opaque interior -const LANDMARK_STROKE = "#0c4a6e"; // sky-900, dark blue outline -const LANDMARK_STROKE_FIST = "#1e3a8a"; // blue-900, thicker accent when fist -const CONNECTION_STROKE = "#ffffff"; // white bones -const INDEX_TIP_LANDMARK = 8; +const HAND_FILL = "#bfdbfe"; // blue-200, light interior +const HAND_OUTLINE_COLOR = "#1e3a8a"; // blue-900, crisp dark outline +const HAND_OUTLINE_RADIUS = 2; // px +// Shrink the rendered hand around its centroid. Grab/physics keep using raw +// landmarks elsewhere, so the silhouette is just visually smaller. +const RENDER_SCALE = 0.65; +const FINGER_THICKNESS_FACTOR = 0.08; // fraction of (scaled) hand length +const WRIST_HALF_WIDTH = 0.28; +const SKELETON_STROKE = "rgba(30, 58, 138, 0.22)"; +const SKELETON_DOT_FILL = "rgba(30, 58, 138, 0.35)"; +const FILTER_ID = "hand-tracking-outline"; export function HandTrackingVisualizer(): React.JSX.Element | null { const { hands, status } = useHandTrackingSnapshot(); - const showHandTrackingSvg = useDebugStore((debug) => - debug.getShowHandTrackingSvg(), - ); - const gloves = useHandTrackingGloveStatus((state) => state.gloves); - const hasLoadedGlove = Object.values(gloves).some( - (gloveStatus) => gloveStatus === "loaded", + const showHandTrackingModel = useDebugStore((debug) => + debug.getShowHandTrackingModel(), ); - if ( - status === "idle" || - hands.length === 0 || - (hasLoadedGlove && !showHandTrackingSvg) - ) { + if (status === "idle" || hands.length === 0 || showHandTrackingModel) { return null; } + const viewportWidth = window.innerWidth; + const viewportHeight = window.innerHeight; + return ( {label}; +} + +/** + * First-time movement tutorial. Visible during the intro reveal and the + * walk-around step before the ebike mount, until the player presses any + * of Z, Q, S, D. Once dismissed it stays dismissed for the session. + */ +export function MovementTutorial(): React.JSX.Element | null { + const introStep = useGameStore((state) => state.intro.currentStep); + const [dismissed, setDismissed] = useState(false); + + const isInShowWindow = MOVEMENT_TUTORIAL_STEPS.has(introStep); + + useEffect(() => { + if (dismissed) return; + function onKeyDown(event: KeyboardEvent): void { + if (MOVEMENT_KEYS.has(event.key.toLowerCase())) { + setDismissed(true); + } + } + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [dismissed]); + + if (!isInShowWindow || dismissed) return null; + + return ( + +