Merge pull request #7 from La-Fabrik-Durable/feat/main-feature

Feat/main feature

.agent/AGENT.md

@@ -24,7 +24,8 @@ You are working on **La Fabrik**, an interactive 3D web experience built with Re
 
 ## Current Architecture Rules
 
-- Scene objects live in `src/world/` and `src/components/3d/`.
+- Scene objects live in `src/world/` and `src/components/three/`.
+- Shared 3D components are grouped by domain under `src/components/three/models/`, `src/components/three/interaction/`, `src/components/three/gameplay/`, and `src/components/three/world/`.
 - HTML overlays live in `src/components/ui/`.
 - Shared static config lives in `src/data/`.
 - Debug tooling lives in `src/utils/debug/` and `src/hooks/debug/`.

.agent/skills/debug.md

@@ -58,19 +58,18 @@ if (debug.active) {
 r3f-perf is loaded only in debug mode to avoid dependency issues in production:
 
 ```tsx
-// src/utils/debug/DebugPerf.tsx
 import { Suspense, lazy } from "react";
-import { Debug } from "@/utils/debug/Debug";
+import { useShowDebugPerf } from "@/hooks/debug/useShowDebugPerf";
 
 const Perf = lazy(() => import("r3f-perf").then((m) => ({ default: m.Perf })));
 
 export function DebugPerf() {
-  const debug = Debug.getInstance();
-  if (!debug.active) return null;
+  const showDebugPerf = useShowDebugPerf();
+  if (!showDebugPerf) return null;
 
   return (
     <Suspense fallback={null}>
-      <Perf position="top-left" />
+      <Perf position="top-right" />
     </Suspense>
   );
 }
@@ -89,6 +88,9 @@ Usage in Canvas:
 
 - All debug UI goes through `Debug.getInstance()` — never inline `if (isDev)` checks
 - r3f-perf is always lazy-imported, never a hard dependency in scene components
-- Debug folders should be organized by domain (Lighting, PostFX, Player, Zone)
+- Debug folders should be organized by domain (Lighting, Player, Zone, Interaction)
+- Global debug controls include camera mode, scene mode, `R3F Perf`, and `Debug Overlay`
+- Interaction-specific controls such as interaction spheres belong in the `Interaction` folder
+- HTML debug panels should be grouped under `src/components/ui/debug/DebugOverlayLayout.tsx`
 - Debug panel must not affect production builds — it simply doesn't mount when `?debug` is absent
 - Clean up debug folders in `destroy()` when relevant

.agent/skills/managers.md

@@ -29,13 +29,16 @@ export class SomeManager {
 ## Managers in this project
 
 | Manager              | File                                 | Role                                                                          |
-| ------------------ | -------------------------------------- | ---------------------------------------------------------------------------------------------------------- |
-| `GameManager`      | `src/stateManager/GameManager.ts`      | Single source of truth. Owns phase, zone, mission, input lock, dialogue. Has `subscribe()` + `getState()`. |
-| `CinematicManager` | `src/stateManager/CinematicManager.ts` | GSAP timelines. Locks/unlocks input via GameManager.                                                       |
-| `AudioManager`     | `src/stateManager/AudioManager.ts`     | Music, SFX, spatial audio. Reads phase from GameManager.                                                   |
-| `ZoneManager`      | `src/stateManager/ZoneManager.ts`      | Zone entry/exit detection, LOD triggers. Notifies GameManager of zone changes.                             |
+| -------------------- | ------------------------------------ | ----------------------------------------------------------------------------- |
+| `AudioManager`       | `src/managers/AudioManager.ts`       | Music and SFX playback.                                                       |
+| `InteractionManager` | `src/managers/InteractionManager.ts` | Focus, nearby, trigger, grab, and hand-grab interaction state.                |
+| `GameManager`        | target-state only                    | Future single source of truth for phase, zone, mission, input lock, dialogue. |
+| `CinematicManager`   | target-state only                    | Future GSAP timeline orchestrator.                                            |
+| `ZoneManager`        | target-state only                    | Future zone entry/exit detection and LOD triggers.                            |
 
-## GameManager is the orchestrator
+## Target-State GameManager
+
+`GameManager` does not exist in the current implementation. The following pattern is target-state guidance only and should not be applied until the manager exists in code.
 
 ```ts
 export class GameManager {
@@ -51,7 +54,7 @@ export class GameManager {
 }
 ```
 
-Components and hooks access other managers **through GameManager only**:
+When a `GameManager` exists, components and hooks should access other managers through it:
 
 ```ts
 // Correct
@@ -61,7 +64,7 @@ GameManager.getInstance().cinematic.play("intro");
 CinematicManager.getInstance().play("intro");
 ```
 
-## Subscribe pattern (GameManager only)
+## Target-State Subscribe Pattern
 
 ```ts
 private listeners = new Set<() => void>()
@@ -76,9 +79,9 @@ private emit(): void {
 }
 ```
 
-Every `set*()` method calls `this.emit()` to notify subscribers.
+In that target-state manager, every `set*()` method calls `this.emit()` to notify subscribers.
 
-## React bridge hook
+## Target-State React Bridge Hook
 
 ```ts
 // hooks/useGameState.ts
@@ -96,8 +99,8 @@ export function useGameState() {
 
 ## Rules
 
-- Max 4 managers total
-- Only `GameManager` holds durable state with `subscribe()`
-- Other managers are side-effect handlers — they do not store persistent state
-- Always call `destroy()` on cleanup (App unmount)
-- Never create manager instances with `new` — always use `.getInstance()`
+- Do not add a `GameManager` unless the feature requires a real shared gameplay state owner.
+- Current managers may be imported directly until the target-state orchestrator exists.
+- Keep singleton managers limited to side-effect services or shared interaction state.
+- Always call `destroy()` on cleanup when a manager owns external resources.
+- Never create manager instances with `new` — always use `.getInstance()`.

.agent/skills/r3f.md

@@ -66,21 +66,6 @@ import { RigidBody, CuboidCollider } from "@react-three/rapier";
 - `type="dynamic"` for movable objects
 - Player uses `type="dynamic"` with `lockRotations`
 
-## Postprocessing
-
-```tsx
-import { EffectComposer, Bloom, Vignette } from "@react-three/postprocessing";
-
-<EffectComposer>
-  <Bloom intensity={0.5} luminanceThreshold={0.9} />
-  <Vignette offset={0.3} darkness={0.5} />
-</EffectComposer>;
-```
-
-- Always wrap in `<EffectComposer>`
-- Keep effects minimal for performance
-- Disable heavy effects on low-end devices via Debug panel
-
 ## What NOT to do
 
 - Do not use `new THREE.Scene()` or `new THREE.WebGLRenderer()` — R3F handles this

.gitattributes

@@ -3,6 +3,7 @@
 
 *.glb filter=lfs diff=lfs merge=lfs -text
 *.gltf filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
 
 # Textures
 *.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,5 +1,9 @@
 # Dependencies
 node_modules/
+.venv/
+backend/.venv/
+__pycache__/
+*.pyc
 
 # Build
 dist/
@@ -38,8 +42,3 @@ Thumbs.db
 # 3D Assets Cache (drei, GLTFJSX)
 .drei/
 .glitchdrei-cache/
-
-# Temporaire
-.backend/
-backend/
-temp/

README.md

@@ -24,7 +24,6 @@ Built with React, Three.js, and Vite. Runs in the browser, no installation requi
 | [@react-three/fiber](https://docs.pmnd.rs/react-three-fiber/getting-started/introduction) |
 | [@react-three/drei](https://pmndrs.github.io/drei)                                        |
 | [@react-three/rapier](https://rapier.rs/docs/)                                            |
-| [@react-three/postprocessing](https://github.com/pmndrs/postprocessing)                   |
 | [GSAP](https://gsap.com/docs/v3/Installation/)                                            |
 
 ### Performance & Effects
@@ -48,74 +47,60 @@ la-fabrik/
 │   └── sounds/
 │
 └── src/
-    ├── world/                              # Single persistent 3D world
-    │   ├── World.tsx                       # Main scene composition
-    │   ├── Map.tsx                         # Base map, always mounted
+    ├── world/                              # Persistent 3D world composition
+    │   ├── World.tsx                       # Active scene composition
+    │   ├── GameMap.tsx                     # Map loading and octree collision
     │   ├── Lighting.tsx                    # Ambient, directional, point lights
-    │   ├── Environment.tsx                 # HDRI, fog, sky
-    │   ├── PostFX.tsx                      # Bloom, SSAO, chromatic aberration
-    │   ├── zones/                          # Spatial zones — LOD per zone
-    │   │   ├── WorkshopZone.tsx
-    │   │   ├── PowerGridZone.tsx
-    │   │   ├── FarmZone.tsx
-    │   │   ├── SchoolZone.tsx
-    │   │   └── ResidentialZone.tsx
+    │   ├── Environment.tsx                 # Scene background / sky model
+    │   ├── GameMusic.tsx                   # Game scene music lifecycle
+    │   ├── debug/                          # Debug-only test scene
+    │   │   └── TestMap.tsx
     │   └── player/
-    │       ├── FPSController.tsx           # PointerLockControls + Rapier movement
-    │       └── Crosshair.tsx
+    │       ├── Player.tsx                  # Player rig composition
+    │       ├── PlayerCamera.tsx            # Player camera mount
+    │       └── PlayerController.tsx        # Pointer lock movement and inputs
     │
     ├── components/
-    │   ├── 3d/                             # Shared reusable 3D elements
-    │   │   └── InteractiveObject.tsx       # Raycasting + outline wrapper
+    │   ├── three/                          # Shared R3F components by domain
+    │   │   ├── gameplay/                   # Core repair gameplay prototype
+    │   │   ├── handTracking/               # R3F hand tracking debug models
+    │   │   ├── interaction/                # Trigger, grab, focus wrappers
+    │   │   ├── models/                     # GLTF model components
+    │   │   └── world/                      # Environment-specific 3D objects
     │   └── ui/                             # HTML overlays — outside Canvas
-    │       ├── NarrativeOverlay.tsx        # Floating dialogues
-    │       ├── MissionHUD.tsx              # Current objective
-    │       ├── MapHUD.tsx                  # Minimap
-    │       ├── CinematicBars.tsx           # GSAP black bars
-    │       └── LoadingScreen.tsx           # Asset progress
+    │       ├── Crosshair.tsx
+    │       ├── debug/                       # Debug-only HTML overlay panels
+    │       │   ├── DebugOverlayLayout.tsx
+    │       │   ├── GameStateDebugPanel.tsx
+    │       │   └── HandTrackingDebugPanel.tsx
+    │       ├── HandTrackingVisualizer.tsx
+    │       └── InteractPrompt.tsx
     │
-    ├── stateManager/                       # All logic, state, orchestration
-    │   ├── GameManager.ts                  # Single source of truth: phase, zone, mission
-    │   ├── CinematicManager.ts             # GSAP timelines, camera lock/unlock
-    │   ├── AudioManager.ts                 # Music, SFX, spatial audio
-    │   └── ZoneManager.ts                  # Zone detection, LOD triggers
+    ├── managers/                           # Current singleton-style services
+    │   ├── AudioManager.ts                 # Music and SFX playback
+    │   └── InteractionManager.ts           # Focus, nearby, grab state
     │
-    ├── hooks/                              # React hooks — thin wrappers on managers
-    │   ├── useGameState.ts                 # Subscribes to GameManager
-    │   ├── useZoneDetection.ts
-    │   ├── useInteraction.ts
-    │   ├── useCinematic.ts
-    │   ├── useAudio.ts
-    │   └── useLOD.ts
+    ├── hooks/                              # React hooks by domain
+    │   ├── debug/                          # Debug state and GUI folders
+    │   ├── docs/                           # Docs language context access
+    │   ├── editor/                         # Editor loading and history
+    │   ├── gameplay/                       # Repair gameplay helpers
+    │   ├── handTracking/                   # Webcam/WebSocket hand tracking
+    │   ├── interaction/                    # Interaction manager subscriptions
+    │   └── three/                          # Three.js/R3F helpers
     │
     ├── data/
-    │   ├── zones.ts                        # { id, position, radius, missionId }
-    │   ├── dialogues.ts                    # Narrative scripts, PNJ states
-    │   └── missions.ts                     # Mission definitions, steps
-    │
-    ├── shaders/
-    │   └── hologram/
-    │       ├── vertex.glsl
-    │       └── fragment.glsl
+    │   ├── interaction/                    # Interaction tuning
+    │   ├── player/                         # Player tuning
+    │   ├── gameplay/                       # Repair gameplay static config
+    │   └── world/                          # Environment and lighting config
     │
     ├── utils/
-    │   ├── EventEmitter.ts                 # Simple typed pub/sub utility
-    │   ├── Sizes.ts                        # Viewport size tracking
-    │   ├── Time.ts                         # Animation frame timing utility
-    │   └── debug/                          # Dev-only tools and scene inspection
-    │       ├── Debug.ts                    # Global lil-gui manager
-    │       ├── DebugPerf.tsx               # r3f-perf overlay mounted in Canvas
-    │       ├── isDebugEnabled.ts           # Debug query-string helper
-    │       └── scene/
-    │           ├── DebugHelpers.tsx        # Grid + axes helpers shown in debug mode
-    │           └── DebugCameraControls.tsx # Free debug camera for map inspection
-    ├── hooks/
-    │   └── debug/
-    │       ├── useCameraMode.ts
-    │       ├── useDebugFolder.ts
-    │       ├── useDebugStore.ts
-    │       └── useSceneMode.ts
-    │
+    │   ├── core/                           # Logger and generic utilities
+    │   ├── debug/                          # Dev-only tools and scene inspection
+    │   ├── editor/                         # Editor-only parsing utilities
+    │   ├── map/                            # Map loading and validation
+    │   └── three/                          # Three.js helpers
     ├── App.tsx                             # Canvas bootstrap
     └── main.tsx
 ```

backend/README.md

@@ -0,0 +1,91 @@
+# Hand Tracking Backend
+
+Remote-compatible Python backend for La-Fabrik hand tracking.
+
+The browser captures webcam frames, downsizes them, sends JPEG frames to this backend over WebSocket, and receives hand landmarks plus closed-fist state.
+
+## Setup
+
+```bash
+python3.11 -m venv backend/.venv
+source backend/.venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install -r backend/requirements.txt
+python backend/download_model.py
+```
+
+## Run
+
+Run the Vite frontend and the Python backend in two separate terminals.
+
+Terminal 1:
+
+```bash
+npm run dev
+```
+
+Terminal 2:
+
+```bash
+source backend/.venv/bin/activate
+python -m backend.main
+```
+
+The WebSocket endpoint is:
+
+```txt
+ws://localhost:8000/ws
+```
+
+## Health Check
+
+```txt
+http://localhost:8000/health
+```
+
+## Message Flow
+
+Client sends a compressed frame:
+
+```json
+{
+  "type": "frame",
+  "timestamp": 1234567890,
+  "width": 320,
+  "height": 240,
+  "image": "base64-jpeg"
+}
+```
+
+Server responds with detected hands:
+
+```json
+{
+  "type": "hands",
+  "timestamp": 1234567890,
+  "hands": [
+    {
+      "x": 0.5,
+      "y": 0.3,
+      "z": 0.1,
+      "landmarks": [
+        {
+          "x": 0.48,
+          "y": 0.32,
+          "z": 0.02
+        }
+      ],
+      "handedness": "Right",
+      "isFist": true,
+      "score": 0.92
+    }
+  ]
+}
+```
+
+## Notes
+
+- The backend does not read `cv2.VideoCapture(0)`.
+- This keeps local development and production behavior aligned.
+- Each browser connection sends its own webcam frames.
+- The backend rate-limits frames per connection and drops work when a client is already being processed.

backend/connection_manager.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+from uuid import uuid4
+
+from fastapi import WebSocket
+
+
+@dataclass
+class ClientConnection:
+    id: str
+    websocket: WebSocket
+    is_processing: bool = False
+    last_frame_at: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class ConnectionManager:
+    def __init__(self) -> None:
+        self._connections: dict[str, ClientConnection] = {}
+
+    @property
+    def count(self) -> int:
+        return len(self._connections)
+
+    async def connect(self, websocket: WebSocket) -> ClientConnection:
+        await websocket.accept()
+        connection = ClientConnection(id=str(uuid4()), websocket=websocket)
+        self._connections[connection.id] = connection
+        return connection
+
+    def disconnect(self, connection: ClientConnection) -> None:
+        self._connections.pop(connection.id, None)
+
+    async def send(self, connection: ClientConnection, payload: dict[str, Any]) -> None:
+        await connection.websocket.send_json(payload)

backend/download_model.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.request import urlretrieve
+
+
+MODEL_URL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"
+MODEL_PATH = Path(__file__).with_name("hand_landmarker.task")
+
+
+def download_model() -> None:
+    if MODEL_PATH.exists():
+        print(f"Model already exists at {MODEL_PATH}")
+        return
+
+    print("Downloading MediaPipe Hand Landmarker model...")
+    urlretrieve(MODEL_URL, MODEL_PATH)
+    print(f"Model downloaded to {MODEL_PATH}")
+
+
+if __name__ == "__main__":
+    download_model()

backend/hand_tracker.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import base64
+import math
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import cv2
+import mediapipe as mp
+import numpy as np
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+
+
+@dataclass(frozen=True)
+class HandData:
+    x: float
+    y: float
+    z: float
+    landmarks: list[dict[str, float]]
+    handedness: str
+    is_fist: bool
+    score: float
+
+    def to_payload(self) -> dict[str, float | str | bool | list[dict[str, float]]]:
+        return {
+            "x": self.x,
+            "y": self.y,
+            "z": self.z,
+            "landmarks": self.landmarks,
+            "handedness": self.handedness,
+            "isFist": self.is_fist,
+            "score": self.score,
+        }
+
+
+class HandTracker:
+    def __init__(self, max_hands: int = 2) -> None:
+        model_path = Path(__file__).with_name("hand_landmarker.task")
+        if not model_path.exists():
+            raise FileNotFoundError(
+                "Missing hand_landmarker.task. Run `python backend/download_model.py`.",
+            )
+
+        base_options = python.BaseOptions(model_asset_path=str(model_path))
+        options = vision.HandLandmarkerOptions(
+            base_options=base_options,
+            running_mode=vision.RunningMode.IMAGE,
+            num_hands=max_hands,
+        )
+        self._detector = vision.HandLandmarker.create_from_options(options)
+
+    def detect_from_base64_jpeg(self, image_base64: str) -> list[HandData]:
+        image_data = base64.b64decode(image_base64, validate=True)
+        image_buffer = np.frombuffer(image_data, dtype=np.uint8)
+        frame = cv2.imdecode(image_buffer, cv2.IMREAD_COLOR)
+        if frame is None:
+            raise ValueError("Invalid JPEG frame")
+
+        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_frame)
+        result = self._detector.detect(mp_image)
+        return self._to_hands(result)
+
+    def close(self) -> None:
+        self._detector.close()
+
+    def _to_hands(self, result: vision.HandLandmarkerResult) -> list[HandData]:
+        hands: list[HandData] = []
+        if not result.hand_landmarks or not result.handedness:
+            return hands
+
+        for landmarks, handedness_categories in zip(
+            result.hand_landmarks,
+            result.handedness,
+        ):
+            palm_center = self._average_points(
+                [landmarks[0], landmarks[5], landmarks[9], landmarks[13], landmarks[17]],
+            )
+            is_fist = self._is_fist(landmarks)
+            handedness = handedness_categories[0]
+
+            hands.append(
+                HandData(
+                    x=palm_center["x"],
+                    y=palm_center["y"],
+                    z=palm_center["z"],
+                    landmarks=[
+                        {"x": point.x, "y": point.y, "z": point.z}
+                        for point in landmarks
+                    ],
+                    handedness=handedness.category_name,
+                    is_fist=is_fist,
+                    score=handedness.score,
+                ),
+            )
+
+        return hands
+
+    def _is_fist(self, landmarks: list[Any]) -> bool:
+        palm_center = self._average_points(
+            [landmarks[0], landmarks[5], landmarks[9], landmarks[13], landmarks[17]],
+        )
+        palm_size = self._calculate_distance(landmarks[0], landmarks[9])
+        if palm_size <= 0:
+            return False
+
+        folded_finger_count = sum(
+            self._calculate_distance(landmarks[index], palm_center) / palm_size < 1.05
+            for index in (8, 12, 16, 20)
+        )
+
+        return folded_finger_count >= 4
+
+    def _average_points(self, points: list[Any]) -> dict[str, float]:
+        return {
+            "x": sum(point.x for point in points) / len(points),
+            "y": sum(point.y for point in points) / len(points),
+            "z": sum(point.z for point in points) / len(points),
+        }
+
+    def _calculate_distance(self, point_a: Any, point_b: Any) -> float:
+        return math.sqrt(
+            (self._get_coordinate(point_a, "x") - self._get_coordinate(point_b, "x"))
+            ** 2
+            + (self._get_coordinate(point_a, "y") - self._get_coordinate(point_b, "y"))
+            ** 2
+            + (self._get_coordinate(point_a, "z") - self._get_coordinate(point_b, "z"))
+            ** 2,
+        )
+
+    def _get_coordinate(self, point: Any, axis: str) -> float:
+        if isinstance(point, dict):
+            return point[axis]
+
+        return getattr(point, axis)
+
+
+def now_ms() -> int:
+    return time.monotonic_ns() // 1_000_000

backend/main.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+from typing import Any
+
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import JSONResponse
+
+from backend.connection_manager import ClientConnection, ConnectionManager
+from backend.hand_tracker import HandTracker, now_ms
+
+
+MAX_FRAME_BYTES = 220_000
+MIN_FRAME_INTERVAL_SECONDS = 0.08
+
+manager = ConnectionManager()
+tracker: HandTracker | None = None
+detection_lock = asyncio.Lock()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global tracker
+    tracker = HandTracker(max_hands=2)
+    yield
+    if tracker:
+        tracker.close()
+
+
+app = FastAPI(title="La-Fabrik Hand Tracking", lifespan=lifespan)
+
+
+@app.get("/health")
+async def health() -> JSONResponse:
+    return JSONResponse(
+        {
+            "status": "ok",
+            "connections": manager.count,
+        },
+    )
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket) -> None:
+    connection = await manager.connect(websocket)
+    await manager.send(connection, status_payload("connected"))
+
+    try:
+        while True:
+            message = await websocket.receive_json()
+            response = await handle_message(connection, message)
+            await manager.send(connection, response)
+    except WebSocketDisconnect:
+        manager.disconnect(connection)
+    except Exception as error:
+        await manager.send(connection, error_payload(str(error)))
+        manager.disconnect(connection)
+
+
+async def handle_message(
+    connection: ClientConnection,
+    message: dict[str, Any],
+) -> dict[str, Any]:
+    if message.get("type") != "frame":
+        return error_payload("Unsupported message type")
+
+    current_time = asyncio.get_running_loop().time()
+    if current_time - connection.last_frame_at < MIN_FRAME_INTERVAL_SECONDS:
+        return status_payload("rate_limited")
+
+    if connection.is_processing:
+        return status_payload("busy")
+
+    image = message.get("image")
+    if not isinstance(image, str):
+        return error_payload("Missing image payload")
+
+    if len(image) > MAX_FRAME_BYTES:
+        return error_payload("Frame payload too large")
+
+    if tracker is None:
+        return error_payload("Hand tracker is not ready")
+
+    if detection_lock.locked():
+        return status_payload("busy")
+
+    connection.last_frame_at = current_time
+    connection.is_processing = True
+    try:
+        async with detection_lock:
+            hands = await asyncio.to_thread(tracker.detect_from_base64_jpeg, image)
+        return {
+            "type": "hands",
+            "timestamp": now_ms(),
+            "hands": [hand.to_payload() for hand in hands],
+        }
+    finally:
+        connection.is_processing = False
+
+
+def status_payload(status: str) -> dict[str, str | int]:
+    return {
+        "type": "status",
+        "timestamp": now_ms(),
+        "status": status,
+    }
+
+
+def error_payload(message: str) -> dict[str, str | int | list[Any]]:
+    return {
+        "type": "error",
+        "timestamp": now_ms(),
+        "hands": [],
+        "message": message,
+    }
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

backend/requirements.txt

@@ -0,0 +1,5 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+opencv-python-headless==4.10.0.84
+mediapipe==0.10.20
+numpy==1.26.4

docs/technical/animation.md

@@ -1,338 +1,51 @@
-# Animation & 3D Model System
+# Animation & 3D Components
 
-This document describes how to use the 3D model components and animation system in La-Fabrik.
+This document describes the 3D components that are currently used in the runtime.
 
-## Table of Contents
+## Runtime Components
 
-1. [Model Types Overview](#model-types-overview)
-2. [SimpleModel - Static Models](#simplemodel---static-models)
-3. [AnimatedModel - Animated Models](#animatedmodel---animated-models)
-4. [Animation Control](#animation-control)
-5. [Other 3D Components](#other-3d-components)
-6. [Technical Notes](#technical-notes)
+| Domain      | Component            | Role                                                                  |
+| ----------- | -------------------- | --------------------------------------------------------------------- |
+| Interaction | `InteractableObject` | Focus detection through distance and raycasting                       |
+| Interaction | `TriggerObject`      | Press-to-trigger interactions, optional sound, optional spawned model |
+| Interaction | `GrabbableObject`    | Physics grab and hand-tracking grab behavior                          |
+| Model       | `ExplodableModel`    | Split/reassemble a GLTF model into separated parts                    |
+| Gameplay    | `RepairCaseModel`    | Repair case lid animation, proximity float, and wobble                |
 
----
+## Continuous Animation
 
-## Model Types Overview
+Use `useFrame` for per-frame 3D behavior. Current examples:
 
-The project provides three main types of model instantiation:
+- `GrabbableObject` updates held object velocity every frame.
+- `ExplodableModel` updates split part positions every frame.
+- `RepairCaseModel` updates proximity float and rotation wobble every frame.
+- `SkyModel` follows the camera position every frame.
 
-| Type        | Component                                                | Use Case                                     |
-| ----------- | -------------------------------------------------------- | -------------------------------------------- |
-| Static      | `SimpleModel`                                            | Props, decoration, objects without animation |
-| Animated    | `AnimatedModel`                                          | Characters, animated objects with skeleton   |
-| Interactive | `GrabbableObject`, `TriggerObject`, `InteractableObject` | Objects player can interact with             |
+## Timeline Animation
 
----
+Use GSAP only for discrete timeline-style transitions. Current example:
 
-## SimpleModel - Static Models
+- `RepairCaseModel` animates the case lid between open and closed rotations.
 
-Use for GLTF models **without** skeleton/armature and no animations.
+## GLTF Reuse
 
-```tsx
-import { SimpleModel } from "@/components/3d";
-
-<SimpleModel
-  modelPath="/models/elecsimple/model.gltf"
-  position={[0, 0, -5]}
-  rotation={[0, 45, 0]}
-  scale={1}
-  castShadow={true}
-  receiveShadow={true}
-/>;
-```
-
-### Props
-
-| Prop            | Type                     | Default     | Description                       |
-| --------------- | ------------------------ | ----------- | --------------------------------- |
-| `modelPath`     | `string`                 | required    | Path to GLTF file in `/public`    |
-| `position`      | `Vector3Tuple`           | `[0, 0, 0]` | World position [x, y, z]          |
-| `rotation`      | `Vector3Tuple`           | `[0, 0, 0]` | Rotation in degrees [x, y, z]     |
-| `scale`         | `number \| Vector3Tuple` | `1`         | Scale factor or [x, y, z]         |
-| `castShadow`    | `boolean`                | `true`      | Enable shadow casting             |
-| `receiveShadow` | `boolean`                | `true`      | Enable shadow receiving           |
-| `children`      | `ReactNode`              | -           | Child components to render inside |
-
----
-
-## AnimatedModel - Animated Models
-
-Use for GLTF models **with** skeleton/armature and animations (like Mixamo characters).
-
-```tsx
-import { AnimatedModel, useAnimatedModel } from "@/components/3d";
-
-// Basic usage
-<AnimatedModel
-  modelPath="/models/elec/model.gltf"
-  defaultAnimation="Idle"
-  position={[0, 0, -5]}
-  rotation={[0, 0, 0]}
-  scale={0.01}
-  autoPlay={true}
-  speed={1}
-  fadeDuration={0.3}
-/>;
-```
-
-### Props
-
-| Prop               | Type                     | Default     | Description                                   |
-| ------------------ | ------------------------ | ----------- | --------------------------------------------- |
-| `modelPath`        | `string`                 | required    | Path to GLTF file in `/public`                |
-| `defaultAnimation` | `string`                 | `"Idle"`    | Animation name to play by default             |
-| `animations`       | `string[]`               | `[]`        | List of animation names (optional)            |
-| `position`         | `Vector3Tuple`           | `[0, 0, 0]` | World position [x, y, z]                      |
-| `rotation`         | `Vector3Tuple`           | `[0, 0, 0]` | Rotation in degrees [x, y, z]                 |
-| `scale`            | `number \| Vector3Tuple` | `1`         | Scale factor                                  |
-| `autoPlay`         | `boolean`                | `true`      | Auto-play default animation                   |
-| `speed`            | `number`                 | `1`         | Animation playback speed                      |
-| `fadeDuration`     | `number`                 | `0.3`       | Transition duration in seconds                |
-| `onLoaded`         | `() => void`             | -           | Callback when model loads                     |
-| `onAnimationEnd`   | `(name: string) => void` | -           | Callback when animation ends                  |
-| `children`         | `ReactNode`              | -           | Child components (can use `useAnimatedModel`) |
-
-### Important: Scale
-
-Animated models (like Mixamo exports) often need a small scale (e.g., `0.01`) because they are exported in meters while Three.js uses different units. Adjust until the model appears at the right size.
-
----
-
-## Animation Control
-
-To control animations from inside or outside the `AnimatedModel`, use the `useAnimatedModel` hook.
-
-### Basic Control
-
-```tsx
-import { AnimatedModel, useAnimatedModel } from "@/components/3d";
-
-// Create a controller component to use inside AnimatedModel
-function AnimationController() {
-  const { play, stop, fadeTo, currentAnimation, names, setSpeed, isReady } =
-    useAnimatedModel();
-
-  // names contains all available animation names
-  // currentAnimation is the name of the currently playing animation
-  // isReady is true when model and animations are loaded
-
-  return (
-    <mesh onClick={() => play("Run", 0.5)}>
-      <boxGeometry />
-    </mesh>
-  );
-}
-
-// Usage
-<AnimatedModel
-  modelPath="/models/elec/model.gltf"
-  defaultAnimation="Idle"
-  position={[0, 0, -5]}
-  scale={0.01}
->
-  <AnimationController />
-</AnimatedModel>;
-```
-
-### Available Methods
-
-| Method             | Signature                               | Description                          |
-| ------------------ | --------------------------------------- | ------------------------------------ |
-| `play`             | `(name: string, fade?: number) => void` | Play animation with optional fade    |
-| `fadeTo`           | `(name: string, fade?: number) => void` | Fade to another animation            |
-| `stop`             | `(fade?: number) => void`               | Stop and return to default animation |
-| `setSpeed`         | `(speed: number) => void`               | Set animation speed                  |
-| `currentAnimation` | `string`                                | Current animation name (getter)      |
-| `names`            | `string[]`                              | Available animation names            |
-| `isReady`          | `boolean`                               | Whether model is loaded              |
-
-### Transition Example
-
-```tsx
-function Character() {
-  const { play, fadeTo, currentAnimation } = useAnimatedModel();
-
-  const handleWalk = () => fadeTo("Walk", 0.5); // 0.5s fade
-  const handleRun = () => play("Run", 0.3); // 0.3s fade
-  const handleIdle = () => play("Idle", 0.5); // return to idle
-
-  return (
-    <group>
-      <mesh onClick={handleWalk} position={[-1, 0, 0]}>
-        <boxGeometry />
-      </mesh>
-      <mesh onClick={handleRun} position={[0, 0, 0]}>
-        <boxGeometry />
-      </mesh>
-      <mesh onClick={handleIdle} position={[1, 0, 0]}>
-        <boxGeometry />
-      </mesh>
-    </group>
-  );
-}
-```
-
-### Combined: GrabbableObject with Animation
-
-You can combine `AnimatedModel` inside `GrabbableObject` to create animated objects that can be picked up:
-
-```tsx
-import { AnimatedModel, GrabbableObject } from "@/components/3d";
-
-// Animated weapon/tool that player can pick up
-<GrabbableObject position={[0, 1, 0]} colliders="cuboid">
-  <AnimatedModel
-    modelPath="/models/sword/model.gltf"
-    defaultAnimation="Idle"
-    position={[0, 0, 0]}
-    scale={0.02}
-    autoPlay={true}
-  />
-</GrabbableObject>;
-```
-
-Or create an animated character that can be grabbed:
-
-```tsx
-import {
-  AnimatedModel,
-  GrabbableObject,
-  useAnimatedModel,
-} from "@/components/3d";
-
-// Controller that triggers animations when grabbed
-function AnimatedGrabber() {
-  const { play, fadeTo } = useAnimatedModel();
-
-  return (
-    <AnimatedModel
-      modelPath="/models/elec/model.gltf"
-      defaultAnimation="Idle"
-      position={[0, 0, 0]}
-      scale={0.01}
-      autoPlay={true}
-    />
-  );
-}
-
-// When grabbed, play "Grab" animation
-<GrabbableObject
-  position={[0, 1, 0]}
-  colliders="cuboid"
-  onGrab={() => {
-    // This would require a context or store to trigger
-    console.log("Object grabbed!");
-  }}
->
-  <AnimatedGrabber />
-</GrabbableObject>;
-```
-
-**Note:** For complex interactions (like playing specific animations when grabbing), you'll need to connect the grab events to animation controls via a state manager or context.
-
----
-
-## Other 3D Components
-
-### GrabbableObject
-
-Objects that can be picked up by the player.
-
-```tsx
-import { GrabbableObject } from "@/components/3d";
-
-<GrabbableObject position={[0, 1, 0]} colliders="cuboid">
-  <mesh>
-    <boxGeometry args={[0.5, 0.5, 0.5]} />
-    <meshStandardMaterial color="red" />
-  </mesh>
-</GrabbableObject>;
-```
-
-### TriggerObject
-
-Objects that trigger events when interacted with.
-
-```tsx
-import { TriggerObject } from "@/components/3d";
-
-<TriggerObject
-  position={[0, 1, 0]}
-  soundPath="/sounds/click.mp3"
-  onTrigger={() => console.log("Triggered!")}
->
-  <mesh>
-    <sphereGeometry />
-    <meshStandardMaterial color="blue" />
-  </mesh>
-</TriggerObject>;
-```
-
-### InteractableObject
-
-Base object for interactions.
-
-```tsx
-import { InteractableObject } from "@/components/3d";
-
-<InteractableObject
-  position={[0, 1, 0]}
-  onInteract={() => console.log("Interacted!")}
->
-  <mesh>
-    <cylinderGeometry />
-    <meshStandardMaterial color="green" />
-  </mesh>
-</InteractableObject>;
-```
-
----
-
-## Technical Notes
-
-### GLTF Models
-
-- Models should be placed in `/public/models/`
-- Supported formats: `.gltf`, `.glb`
-- Animated models must have an Armature/skeleton for animations to work
-
-### Model Scale Issue
-
-If animated models don't appear, they may be too small or too large. Try:
-
-- Scale `0.01` for Mixamo-exported models
-- Scale `1` for models in correct units
-
-### Cloning
-
-- `SimpleModel` uses `scene.clone()` for proper React lifecycle
-- `AnimatedModel` uses the original scene directly to preserve SkinnedMesh + Armature structure
-
-### Animation System
-
-The animation system uses:
-
-- `@react-three/drei`: `useGLTF` for loading, `useAnimations` for animation control
-- Three.js: `AnimationMixer` for playback
-
-### No State Machine
-
-This system intentionally avoids complex state machines (like Unity's Animator). For simple animation transitions, use the `play`, `fadeTo`, and `stop` methods directly.
-
----
+Use `useClonedObject` when a GLTF scene is reused by a component instance. It memoizes `scene.clone(true)` and keeps clone creation out of render churn.
 
 ## File Structure
 
-```
-src/
-├── components/3d/
-│   ├── AnimatedModel.tsx    # Animated model component + context
-│   ├── SimpleModel.tsx      # Static model component
-│   ├── GrabbableObject.tsx  # Pickable object
-│   ├── TriggerObject.tsx    # Trigger event object
+```txt
+src/components/three/
+├── gameplay/
+│   ├── RepairCaseModel.tsx
+│   ├── RepairCaseObject.tsx
+│   ├── RepairGameZone.tsx
+│   └── RepairModuleSlot.tsx
+├── interaction/
+│   ├── GrabbableObject.tsx
 │   ├── InteractableObject.tsx
-│   └── index.ts             # Central exports
-└── hooks/
-    └── useCharacterAnimation.ts  # Animation hook (legacy)
+│   └── TriggerObject.tsx
+├── models/
+│   └── ExplodableModel.tsx
+└── world/
+    └── SkyModel.tsx
 ```

docs/technical/architecture.md

@@ -4,8 +4,9 @@ This document describes the code that exists today in the repository.
 
 ## Runtime Structure
 
-- `src/main.tsx` mounts React and wraps the app in `BrowserRouter`.
-- `src/App.tsx` declares the top-level routes:
+- `src/main.tsx` mounts React.
+- `src/App.tsx` mounts the TanStack `RouterProvider`.
+- `src/router.tsx` declares the top-level routes:
   - `/` mounts the playable 3D scene, debug perf overlay, and HTML overlays.
   - `/editor` mounts the map editor page.
 - `src/world/World.tsx` composes the active scene, including:
@@ -14,22 +15,22 @@ This document describes the code that exists today in the repository.
   - either the map scene or the debug physics test scene
   - the player rig when the active camera mode is `player`
 - `src/world/GameMap.tsx` loads map nodes from `public/map.json`, resolves available models, and builds the collision octree.
-- `src/world/debug/TestScene.tsx` provides a debug-oriented interaction and physics scene.
-- `src/world/player/PlayerComponent.tsx` mounts the camera and controller.
+- `src/world/debug/TestMap.tsx` provides a debug-oriented interaction and physics map.
+- `src/world/player/Player.tsx` mounts the camera and controller.
 - `src/world/player/PlayerController.tsx` owns pointer lock movement, jump handling, and interaction input.
 
 ## Interaction Model
 
-- `src/stateManager/InteractionManager.ts` is the current interaction state source.
-- `src/components/3d/InteractableObject.tsx` handles focus detection through distance and raycasting.
-- `src/components/3d/TriggerObject.tsx` implements trigger-style interactions.
-- `src/components/3d/GrabbableObject.tsx` implements hold-and-release interactions.
-- `src/hooks/useInteraction.ts` exposes the interaction snapshot to React UI.
+- `src/managers/InteractionManager.ts` is the current interaction state source.
+- `src/components/three/interaction/InteractableObject.tsx` handles focus detection through distance and raycasting.
+- `src/components/three/interaction/TriggerObject.tsx` implements trigger-style interactions.
+- `src/components/three/interaction/GrabbableObject.tsx` implements hold-and-release interactions.
+- `src/hooks/interaction/useInteraction.ts` exposes the interaction snapshot to React UI.
 - `src/components/ui/InteractPrompt.tsx` shows the `E` prompt for trigger interactions.
 
 ## Audio
 
-- `src/stateManager/AudioManager.ts` currently provides pooled one-shot sound playback.
+- `src/managers/AudioManager.ts` currently provides pooled one-shot sound playback and looped music playback.
 - Trigger interactions may play audio directly through `AudioManager`.
 
 ## Debug System
@@ -37,13 +38,26 @@ This document describes the code that exists today in the repository.
 - Debug mode is enabled with `?debug`.
 - `src/utils/debug/Debug.ts` owns the `lil-gui` instance and debug controls.
 - `src/hooks/debug/useCameraMode.ts` and `src/hooks/debug/useSceneMode.ts` subscribe to debug state.
-- `src/utils/debug/DebugPerf.tsx` lazily mounts `r3f-perf` in debug mode.
-- `src/utils/debug/scene/DebugHelpers.tsx` mounts debug helpers.
-- `src/utils/debug/scene/DebugCameraControls.tsx` mounts the free debug camera.
+- `src/components/debug/DebugPerf.tsx` lazily mounts `r3f-perf` in debug mode.
+- `src/components/ui/debug/DebugOverlayLayout.tsx` mounts the compact HTML debug overlay when enabled from `lil-gui`.
+- `src/components/ui/debug/GameStateDebugPanel.tsx` exposes current game state, main/sub-state switching, previous/next step controls, and reset.
+- `src/components/ui/debug/HandTrackingDebugPanel.tsx` shows hand tracking status, usage, loaded glove model, hand count, and fist state while hand tracking is active.
+- `src/components/three/handTracking/HandTrackingGlove.tsx` places the rigged `gant_l` and `gant_r` models on detected hands in the debug physics scene.
+- `src/components/debug/scene/DebugHelpers.tsx` mounts debug helpers.
+- `src/components/debug/scene/DebugCameraControls.tsx` mounts the free debug camera.
+- `lil-gui` global debug controls include camera mode, scene mode, `R3F Perf`, and `Debug Overlay`; interaction-specific controls live in the `Interaction` folder.
+
+## 3D Component Domains
+
+- `src/components/three/models/` contains reusable model helpers such as `ExplodableModel`.
+- `src/components/three/interaction/` contains reusable interaction wrappers such as `InteractableObject`, `TriggerObject`, and `GrabbableObject`.
+- `src/components/three/handTracking/` contains R3F hand tracking debug models such as the glove overlays.
+- `src/components/three/gameplay/` contains the current core repair gameplay prototype: the repair case, repair game zone, and module slots.
+- `src/components/three/world/` contains reusable world/environment objects such as `SkyModel`.
 
 ## Editor System
 
-- `src/pages/editor/EditorPage.tsx` is the route-level editor page for `/editor`.
+- `src/pages/editor/page.tsx` is the route-level editor page for `/editor`.
 - `src/components/editor/EditorControls.tsx` renders the HTML editor control panel.
 - `src/components/editor/scene/EditorScene.tsx` composes the editor canvas scene, camera controls, lights, shortcuts, and map rendering.
 - `src/components/editor/scene/EditorMap.tsx` renders map nodes, fallback cubes, selection highlighting, and transform controls.
@@ -51,20 +65,20 @@ This document describes the code that exists today in the repository.
 - `src/hooks/editor/useEditorSceneData.ts` loads scene data and handles folder upload fallback.
 - `src/hooks/editor/useEditorHistory.ts` owns editor undo and redo state.
 - `src/utils/editor/loadEditorScene.ts` handles editor-only folder upload parsing.
-- `src/utils/loadMapSceneData.ts` is shared by the game scene and editor to load `public/map.json` and resolve model URLs.
-- `src/types/editor.ts` contains the shared `MapNode`, `SceneData`, and `TransformMode` types.
+- `src/utils/map/loadMapSceneData.ts` is shared by the game scene and editor to load `public/map.json` and resolve model URLs.
+- `src/types/editor/editor.ts` contains the shared `MapNode`, `SceneData`, and `TransformMode` types.
 
 ## Map Data
 
 - `public/map.json` is expected to be a `MapNode[]`.
-- Each map node `name` maps to `public/models/{name}/model.gltf`.
+- Each map node `name` maps to `public/models/{name}/model.glb` when available, with `public/models/{name}/model.gltf` kept as fallback.
 - The editor renders a fallback cube for missing models.
 - The game scene filters out nodes whose model cannot be resolved.
 
 ## Current Limitations
 
 - The repository is a prototype, not the full intended game runtime.
-- `src/world/debug/TestScene.tsx` is part of the active scene composition.
+- `src/world/debug/TestMap.tsx` is part of the active scene composition.
 - There is no central gameplay orchestrator such as `GameManager`.
 - Missions, zones, cinematics, and dialogue systems are not implemented.
 - The player uses octree collision and simple movement rules, not a complete gameplay physics stack.

docs/technical/editor.md

@@ -34,10 +34,12 @@ src/
 │       ├── useEditorHistory.ts
 │       └── useEditorSceneData.ts
 ├── types/
+│   └── editor/
 │       └── editor.ts
 └── utils/
     ├── editor/
     │   └── loadEditorScene.ts
+    └── map/
         └── loadMapSceneData.ts
 ```
 
@@ -57,13 +59,13 @@ src/
 
 `src/controls/editor/FlyController.tsx` provides editor movement controls for player-style navigation.
 
-`src/utils/loadMapSceneData.ts` is shared by the game map and editor. It loads `/map.json` and resolves available `public/models/{name}/model.gltf` files.
+`src/utils/map/loadMapSceneData.ts` is shared by the game map and editor. It loads `/map.json` and resolves available `public/models/{name}/model.glb` files first, then falls back to `public/models/{name}/model.gltf`.
 
 `src/utils/editor/loadEditorScene.ts` contains editor-only upload handling for user-selected folders.
 
 ## Data Format
 
-The shared editor type lives in `src/types/editor.ts`.
+The shared editor type lives in `src/types/editor/editor.ts`.
 
 ```ts
 interface MapNode {
@@ -96,10 +98,10 @@ public/
 ├── map.json
 └── models/
     └── pylone/
-        └── model.gltf
+        └── model.glb
 ```
 
-If a model is missing, the editor renders a fallback cube so the node can still be selected and transformed.
+If `model.glb` and `model.gltf` are both missing, the editor renders a fallback cube so the node can still be selected and transformed.
 
 ## Editor Flow
 
@@ -138,7 +140,7 @@ Editor styles are in `src/index.css` under the `/* Editor page */` section. Clas
 
 ## Known Limitations
 
-- Uploaded model object URLs are not currently revoked after replacement or unmount.
+- Uploaded model object URLs are not revoked after replacement or unmount.
 - Large `map.json` files are not virtualized, culled, or LOD-managed.
 - There is no snap-to-grid, duplication, material editing, or object creation workflow.
 - Save to Server is a Vite dev-server helper, not a production backend API.

docs/technical/hand-tracking.md

@@ -0,0 +1,129 @@
+# Hand Tracking Technical Notes
+
+This document describes the hand tracking system that exists in the current codebase.
+
+## Purpose
+
+Hand tracking is a debug-stage interaction system used to test direct 3D object manipulation with a webcam. It allows a user to close their fist to grab a nearby object and move it in 3D space without relying on the center crosshair.
+
+The feature is scoped to the debug physics scene rather than production gameplay input.
+
+## Runtime Flow
+
+1. The browser captures webcam frames in `src/hooks/handTracking/useRemoteHandTracking.ts`.
+2. Frames are sent to the local Python backend over WebSocket.
+3. The backend runs MediaPipe hand landmark detection.
+4. The backend returns hand data including landmarks, handedness, score, center point, and `isFist`.
+5. React stores the latest snapshot in the hand tracking provider.
+6. `GrabbableObject` reads that snapshot each frame and uses fist state plus raycasting to grab objects.
+7. `HandTrackingGlove` reads the same snapshot and places the rigged `gant_l` and `gant_r` models on the detected hands in the debug physics scene.
+
+## Activation Rules
+
+Hand tracking is intentionally gated so the webcam and backend are not used all the time.
+
+The current activation conditions are:
+
+- debug mode is active with `?debug`
+- scene mode is `physics`
+- the player is near an interaction, is holding an object, or is hand-holding an object
+
+This keeps hand tracking active while the player is inside an interaction zone, even if the camera is not aimed directly at the object.
+
+## Backend
+
+The backend lives in `backend/` and exposes:
+
+- `GET /health` for health checks
+- `WS /ws` for frame input and hand tracking output
+
+The Python process uses MediaPipe and the local model file:
+
+```txt
+backend/hand_landmarker.task
+```
+
+The backend sends normalized hand coordinates and landmarks. The frontend treats the values as screen-space inputs, then maps them into world space with the active Three.js camera.
+
+## Frontend Data Shape
+
+The shared types live in `src/types/handTracking/handTracking.ts`.
+
+```ts
+interface HandTrackingHand {
+  x: number;
+  y: number;
+  z: number;
+  landmarks: HandTrackingLandmark[];
+  handedness: string;
+  isFist: boolean;
+  score: number;
+}
+```
+
+`x` and `y` are normalized camera coordinates. `z` is a relative depth value from MediaPipe, not an absolute world-space distance.
+
+## Grab Targeting
+
+The hand grab logic lives in `src/components/three/interaction/GrabbableObject.tsx`.
+
+The object is moved toward the visual center of the hand. That center is computed from the bounding box of all landmarks:
+
+```txt
+centerX = (minX + maxX) / 2
+centerY = (minY + maxY) / 2
+```
+
+Starting a grab uses a slightly wider virtual hit zone. Instead of raycasting only from one point, the code casts several rays around the hand center:
+
+- center
+- left
+- right
+- up
+- down
+
+If any ray hits the object while the object is within `INTERACTION_RADIUS`, the object enters hand-holding mode.
+
+## Depth Handling
+
+Because MediaPipe `z` is relative, the frontend captures the starting depth when the grab begins:
+
+```txt
+initialHandZ = hand.z
+initialHoldDistance = hit.distance
+```
+
+While holding, the object distance from the camera is adjusted by the change in hand depth:
+
+```txt
+holdDistance = initialHoldDistance + (hand.z - initialHandZ) * sensitivity
+```
+
+The final hold distance is clamped between the configured grab minimum and maximum distances to avoid unstable movement.
+
+## UI And Debug
+
+The current debug UI includes:
+
+- `HandTrackingDebugPanel` inside `DebugOverlayLayout` for status, usage, loaded glove model, server state, hand count, and fist state
+- `HandTrackingVisualizer` for the SVG landmark wireframe fallback
+- `HandTrackingGlove` for the left-hand `gant_l` and right-hand `gant_r` models in the R3F scene
+- `r3f-perf` for render performance
+- `lil-gui` for scene, camera, lighting, interaction, and grab controls
+
+The hand tracking debug panel is a compact HTML grid outside the canvas. `Model loaded` displays the successfully loaded glove models. The SVG hand wireframe is only a fallback while models are loading or if a glove model fails to load.
+
+## Glove Models
+
+The current glove MVP uses `public/models/gant_l/model.gltf` and `public/models/gant_r/model.gltf`, which contain GLTF skins and armatures. Each model is positioned, oriented, and scaled from palm landmarks, then each finger bone chain is rotated toward the matching MediaPipe landmark chain.
+
+The glove models are intentionally smaller than the raw SVG overlay so they do not dominate the camera view.
+
+## Known Limitations
+
+- The feature is debug-only and focused on the physics test scene.
+- MediaPipe depth is relative and can be noisy.
+- The virtual hit zone is an approximation based on multiple raycasts, not a real 3D collider.
+- There is no smoothing layer for hand position or depth yet.
+- The SVG hand visualization is a fallback, not the primary display when glove models load correctly.
+- Finger bone animation is an approximate landmark-to-bone mapping; it still needs calibration for per-model twist, offsets, and smoothing.

docs/technical/zustand.md

@@ -143,7 +143,8 @@ In React Three Fiber, mounting and unmounting JSX controls what appears in the T
 
 Current overlays:
 
-- `GameStateHUD`: debug-only progression panel shown with `?debug`
+- `DebugOverlayLayout`: debug-only overlay shown with `?debug`, including the `GameStateDebugPanel` progression panel
+- `GameStateDebugPanel`: compact debug UI for viewing and switching main/sub states, stepping backward or forward, and resetting the store
 - `Crosshair`: player aiming helper
 - `InteractPrompt`: interaction prompt
 

docs/user/editor.md

@@ -9,7 +9,7 @@ Use the editor when you need to move, rotate, or scale existing map objects with
 The editor reads the same map data as the runtime scene:
 
 - `public/map.json` contains the object list.
-- `public/models/{name}/model.gltf` contains the matching 3D model for each object name.
+- `public/models/{name}/model.glb` contains the matching 3D model for each object name. `model.gltf` is still supported as a fallback during migration.
 - Missing models are displayed as gray fallback cubes, so incomplete maps remain editable.
 
 ## Map Node Format

docs/user/features.md

@@ -5,7 +5,7 @@ This document lists features that are implemented in the current codebase.
 ## Scene
 
 - Fullscreen React Three Fiber scene
-- Main map scene loaded from `public/map.json` and matching `public/models/{name}/model.gltf` assets
+- Main map scene loaded from `public/map.json` and matching `public/models/{name}/model.glb` or `model.gltf` assets
 - Debug physics test scene selectable from the debug panel
 - Ambient and directional lighting
 - Environment background setup
@@ -33,7 +33,8 @@ This document lists features that are implemented in the current codebase.
 ## Debug Tooling
 
 - `?debug` query param enables the debug panel
-- `lil-gui` controls for camera mode, scene mode, and interaction spheres
+- `lil-gui` controls for camera mode, scene mode, `R3F Perf`, `Debug Overlay`, and interaction tuning
+- Compact debug overlay for game state controls and hand tracking status
 - Debug scene helpers
 - Free debug camera
 - `r3f-perf` overlay
@@ -43,7 +44,7 @@ This document lists features that are implemented in the current codebase.
 - `/editor` route for inspecting and editing `public/map.json`
 - Automatic loading of `public/map.json` when available
 - Folder upload fallback when `map.json` is missing
-- Rendering of available `public/models/{name}/model.gltf` assets
+- Rendering of available `public/models/{name}/model.glb` or `model.gltf` assets
 - Fallback cubes for nodes whose model is missing
 - Object selection by click
 - Transform modes for translate, rotate, and scale

docs/user/main-feature.md

@@ -0,0 +1,80 @@
+# Main Feature
+
+This document explains the current repair-game prototype in La-Fabrik.
+
+## What It Does
+
+The main feature is a repair interaction sandbox mounted in the debug physics scene. It lets the player approach a repair case, open it, and interact with module slots that can show selectable models and exploded-model states.
+
+The current user flow is:
+
+1. Open the app with `?debug`.
+2. Switch the scene to `Physics` in the debug panel.
+3. Move close to the repair case.
+4. Press the interaction key when prompted.
+5. Watch the case open or close with sound feedback.
+6. Interact with repair module slots to cycle/select repair models.
+
+## Why It Matters
+
+This feature validates the core repair fantasy before a full mission system exists. It tests whether repair objects, physical proximity, model selection, audio feedback, and exploded model visualization can work together in the 3D scene.
+
+## Current Behavior
+
+The repair case reacts to player proximity. When the player is close enough, it floats upward and rotates gently to signal interactivity. When the player moves away, it returns to its resting transform.
+
+Interacting with the case toggles its open state. The lid animation is handled with GSAP because it is a discrete interaction animation, not a continuous per-frame loop.
+
+Repair module slots are configured from static gameplay data. They render selectable repair models and can use exploded model visualization to show parts separated from their original positions.
+
+## Key Files
+
+- `src/world/debug/TestMap.tsx` mounts the repair-game prototype in the debug physics scene.
+- `src/components/three/gameplay/RepairGameZone.tsx` composes the repair-game zone.
+- `src/components/three/gameplay/RepairCaseObject.tsx` connects the repair case to trigger interaction and audio.
+- `src/components/three/gameplay/RepairCaseModel.tsx` renders and animates the case model.
+- `src/components/three/gameplay/RepairModuleSlot.tsx` renders repair slots and model selection behavior.
+- `src/components/three/models/ExplodableModel.tsx` renders selectable models with split/exploded visualization.
+- `src/data/gameplay/repairCaseConfig.ts` stores repair case model, sound, and animation constants.
+- `src/data/gameplay/repairGameConfig.ts` stores repair zone and slot positions.
+- `src/data/gameplay/repairGameModelCatalog.ts` stores selectable repair models.
+
+## Debug Requirements
+
+The repair-game prototype currently requires:
+
+- the app opened with `?debug`
+- the debug scene set to `Physics`
+- model assets available under `public/models/`
+- sound assets available under `public/sounds/`
+
+Frontend command:
+
+```bash
+npm run dev
+```
+
+Debug URL:
+
+```txt
+http://localhost:5173/?debug
+```
+
+## Related Hand Tracking
+
+Hand tracking is a separate debug interaction layer. It can move grabbable physics objects with webcam input, but it is not yet integrated into the repair-game mission flow.
+
+For hand tracking, run the Python backend separately:
+
+```bash
+source backend/.venv/bin/activate
+python -m backend.main
+```
+
+## Current Limitations
+
+- It is mounted only in the debug physics scene.
+- There is no mission progression system yet.
+- There is no central `GameManager` or Zustand store in this branch.
+- Hand tracking is available as debug interaction input, not as final repair gameplay.
+- The repair-game content is configured statically in `src/data/gameplay/`.

eslint.config.js

@@ -2,6 +2,7 @@ import js from "@eslint/js";
 import globals from "globals";
 import reactHooks from "eslint-plugin-react-hooks";
 import reactRefresh from "eslint-plugin-react-refresh";
+import prettierRecommended from "eslint-plugin-prettier/recommended";
 import tseslint from "typescript-eslint";
 import { defineConfig, globalIgnores } from "eslint/config";
 
@@ -20,4 +21,5 @@ export default defineConfig([
       globals: globals.browser,
     },
   },
+  prettierRecommended,
 ]);

package-lock.json

@@ -8,9 +8,9 @@
       "name": "la-fabrik",
       "version": "0.0.1",
       "dependencies": {
+        "@mediapipe/tasks-vision": "^0.10.35",
         "@react-three/drei": "^10.7.7",
-        "@react-three/fiber": "^9.6.0",
-        "@react-three/postprocessing": "^3.0.4",
+        "@react-three/fiber": "^9.6.1",
         "@react-three/rapier": "^2.2.0",
         "@tanstack/react-router": "^1.168.25",
         "gsap": "^3.15.0",
@@ -21,7 +21,7 @@
         "react-dom": "^19.2.4",
         "react-markdown": "^10.1.0",
         "remark-gfm": "^4.0.1",
-        "three": "^0.183.2",
+        "three": "0.182.0",
         "zustand": "^5.0.12"
       },
       "devDependencies": {
@@ -294,12 +294,6 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@dimforge/rapier3d-compat": {
-      "version": "0.19.2",
-      "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.19.2.tgz",
-      "integrity": "sha512-AZHL1jqUF55QJkJyU1yKeh4ImX2J93bVLIezT1+o0FZqTix6O06MOaqpKoJ4MmbDCsoZmwO+qc471/SDMDm2AA==",
-      "license": "Apache-2.0"
-    },
     "node_modules/@emnapi/core": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz",
@@ -608,9 +602,9 @@
       }
     },
     "node_modules/@mediapipe/tasks-vision": {
-      "version": "0.10.17",
-      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
-      "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
+      "version": "0.10.35",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.35.tgz",
+      "integrity": "sha512-HOvadwVRE6JC+45nyYhmnywnr5h/J8KZvOeUNVOG9q/0875pZgItznFB9bRTvLc264YSJqiZ1NsIpCStJw/egg==",
       "license": "Apache-2.0"
     },
     "node_modules/@monogrid/gainmap-js": {
@@ -716,10 +710,16 @@
         }
       }
     },
+    "node_modules/@react-three/drei/node_modules/@mediapipe/tasks-vision": {
+      "version": "0.10.17",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz",
+      "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==",
+      "license": "Apache-2.0"
+    },
     "node_modules/@react-three/fiber": {
-      "version": "9.6.0",
-      "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.6.0.tgz",
-      "integrity": "sha512-90abYK2q5/qDM+GACs9zRvc5KhEEpEWqWlHSd64zTPNxg+9wCJvTfyD9x2so7hlQhjRYO1Fa6flR3BC/kpTFkA==",
+      "version": "9.6.1",
+      "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.6.1.tgz",
+      "integrity": "sha512-zF0rsKcVYpcJwbFEnv2HkHX9cvOEgsfQo/X8lwmR2dn13S4qEQJXir9fxf5js2LQFoXqxOY7MDkOkYx2uZ4gSg==",
       "license": "MIT",
       "dependencies": {
         "@babel/runtime": "^7.17.8",
@@ -764,32 +764,6 @@
         }
       }
     },
-    "node_modules/@react-three/postprocessing": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/@react-three/postprocessing/-/postprocessing-3.0.4.tgz",
-      "integrity": "sha512-e4+F5xtudDYvhxx3y0NtWXpZbwvQ0x1zdOXWTbXMK6fFLVDd4qucN90YaaStanZGS4Bd5siQm0lGL/5ogf8iDQ==",
-      "license": "MIT",
-      "dependencies": {
-        "maath": "^0.6.0",
-        "n8ao": "^1.9.4",
-        "postprocessing": "^6.36.6"
-      },
-      "peerDependencies": {
-        "@react-three/fiber": "^9.0.0",
-        "react": "^19.0",
-        "three": ">= 0.156.0"
-      }
-    },
-    "node_modules/@react-three/postprocessing/node_modules/maath": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/maath/-/maath-0.6.0.tgz",
-      "integrity": "sha512-dSb2xQuP7vDnaYqfoKzlApeRcR2xtN8/f7WV/TMAkBC8552TwTLtOO0JTcSygkYMjNDPoo6V01jTw/aPi4JrMw==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/three": ">=0.144.0",
-        "three": ">=0.144.0"
-      }
-    },
     "node_modules/@react-three/rapier": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@react-three/rapier/-/rapier-2.2.0.tgz",
@@ -805,6 +779,12 @@
         "three": ">=0.159.0"
       }
     },
+    "node_modules/@react-three/rapier/node_modules/@dimforge/rapier3d-compat": {
+      "version": "0.19.1",
+      "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.19.1.tgz",
+      "integrity": "sha512-xvFNtb/9xILxfvdFOa7NCnYUEF6cfn51R44C1xnKXtk5DpyAARqsC4sxZwiJAHRSzYT5FFe889t36iFnzb3vxg==",
+      "license": "Apache-2.0"
+    },
     "node_modules/@rolldown/binding-android-arm64": {
       "version": "1.0.0-rc.17",
       "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz",
@@ -4191,16 +4171,6 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
-    "node_modules/n8ao": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/n8ao/-/n8ao-1.10.1.tgz",
-      "integrity": "sha512-hhI1pC+BfOZBV1KMwynBrVlIm8wqLxj/abAWhF2nZ0qQKyzTSQa1QtLVS2veRiuoBQXojxobcnp0oe+PUoxf/w==",
-      "license": "ISC",
-      "peerDependencies": {
-        "postprocessing": ">=6.30.0",
-        "three": ">=0.137"
-      }
-    },
     "node_modules/nanoid": {
       "version": "3.3.11",
       "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
@@ -4390,15 +4360,6 @@
         "node": "^10 || ^12 || >=14"
       }
     },
-    "node_modules/postprocessing": {
-      "version": "6.39.1",
-      "resolved": "https://registry.npmjs.org/postprocessing/-/postprocessing-6.39.1.tgz",
-      "integrity": "sha512-R2dG2zy+BAx3USl5EHw+PvnrlbT5PKnZVp3se0HCR0pWH8WQdh742yNG4YWOsq6c0bFpffk0Gd2RqPeoP/wKng==",
-      "license": "Zlib",
-      "peerDependencies": {
-        "three": ">= 0.168.0 < 0.185.0"
-      }
-    },
     "node_modules/potpack": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz",
@@ -4909,9 +4870,9 @@
       }
     },
     "node_modules/three": {
-      "version": "0.183.2",
-      "resolved": "https://registry.npmjs.org/three/-/three-0.183.2.tgz",
-      "integrity": "sha512-di3BsL2FEQ1PA7Hcvn4fyJOlxRRgFYBpMTcyOgkwJIaDOdJMebEFPA+t98EvjuljDx4hNulAGwF6KIjtwI5jgQ==",
+      "version": "0.182.0",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.182.0.tgz",
+      "integrity": "sha512-GbHabT+Irv+ihI1/f5kIIsZ+Ef9Sl5A1Y7imvS5RQjWgtTPfPnZ43JmlYI7NtCRDK9zir20lQpfg8/9Yd02OvQ==",
       "license": "MIT"
     },
     "node_modules/three-mesh-bvh": {

package.json

@@ -14,12 +14,12 @@
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "preview": "vite preview",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc -b"
   },
   "dependencies": {
+    "@mediapipe/tasks-vision": "^0.10.35",
     "@react-three/drei": "^10.7.7",
-    "@react-three/fiber": "^9.6.0",
-    "@react-three/postprocessing": "^3.0.4",
+    "@react-three/fiber": "^9.6.1",
     "@react-three/rapier": "^2.2.0",
     "@tanstack/react-router": "^1.168.25",
     "gsap": "^3.15.0",
@@ -30,7 +30,7 @@
     "react-dom": "^19.2.4",
     "react-markdown": "^10.1.0",
     "remark-gfm": "^4.0.1",
-    "three": "^0.183.2",
+    "three": "0.182.0",
     "zustand": "^5.0.12"
   },
   "devDependencies": {
@@ -51,6 +51,9 @@
     "vite": "^8.0.4"
   },
   "overrides": {
+    "@react-three/rapier": {
+      "@dimforge/rapier3d-compat": "0.19.1"
+    },
     "r3f-perf": {
       "@react-three/drei": "$@react-three/drei"
     }