feat move debug cube with remote hand tracking

src/App.tsx

@@ -1,13 +1,15 @@
 import { Suspense } from "react";
 import { Canvas } from "@react-three/fiber";
 import { Crosshair } from "@/components/ui/Crosshair";
+import { HandTrackingOverlay } from "@/components/ui/HandTrackingOverlay";
+import { HandTrackingProvider } from "@/components/ui/HandTrackingProvider";
 import { InteractPrompt } from "@/components/ui/InteractPrompt";
 import { DebugPerf } from "@/utils/debug/DebugPerf";
 import { World } from "@/world/World";
 
 function App(): React.JSX.Element {
   return (
-    <>
+    <HandTrackingProvider>
       <Canvas camera={{ position: [85, 60, 85], fov: 42 }} shadows>
         <Suspense fallback={null}>
           <World />
@@ -16,7 +18,8 @@ function App(): React.JSX.Element {
       </Canvas>
       <Crosshair />
       <InteractPrompt />
-    </>
+      <HandTrackingOverlay />
+    </HandTrackingProvider>
   );
 }
 

src/components/3d/GrabbableObject.tsx

@@ -21,6 +21,7 @@ import {
   GRAB_THROW_BOOST_STEP,
 } from "@/data/grabConfig";
 import { useDebugFolder } from "@/hooks/debug/useDebugFolder";
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
 import type { ColliderShape, Vector3Tuple } from "@/types/3d";
 
 interface GrabbableObjectProps {
@@ -28,6 +29,7 @@ interface GrabbableObjectProps {
   children: React.ReactNode;
   colliders?: ColliderShape;
   label?: string;
+  handControlled?: boolean;
 }
 
 // Shared params let one debug folder drive every instance.
@@ -42,14 +44,18 @@ const ZERO_ANGULAR_VELOCITY = { x: 0, y: 0, z: 0 };
 const _holdTarget = new THREE.Vector3();
 const _currentPos = new THREE.Vector3();
 const _velocity = new THREE.Vector3();
+const _handNdc = new THREE.Vector3();
+const _handDirection = new THREE.Vector3();
 
 export function GrabbableObject({
   position,
   children,
   colliders = GRAB_DEFAULT_COLLIDERS,
   label = GRAB_DEFAULT_LABEL,
+  handControlled = false,
 }: GrabbableObjectProps): React.JSX.Element {
   const camera = useThree((state) => state.camera);
+  const { hands } = useHandTrackingSnapshot();
   const rbRef = useRef<RapierRigidBody>(null);
   const isHolding = useRef(false);
 
@@ -84,10 +90,25 @@ export function GrabbableObject({
   });
 
   useFrame(() => {
-    if (!isHolding.current || !rbRef.current) return;
+    if (!rbRef.current) return;
 
-    camera.getWorldDirection(_holdTarget);
+    const pinchingHand = handControlled
-    _holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
+      ? hands.find((hand) => hand.isPinch)
+      : undefined;
+
+    if (!isHolding.current && !pinchingHand) return;
+
+    if (pinchingHand) {
+      _handNdc.set((1 - pinchingHand.x) * 2 - 1, -pinchingHand.y * 2 + 1, 0.5);
+      _handNdc.unproject(camera);
+      _handDirection.subVectors(_handNdc, camera.position).normalize();
+      _holdTarget
+        .copy(camera.position)
+        .addScaledVector(_handDirection, params.holdDistance);
+    } else {
+      camera.getWorldDirection(_holdTarget);
+      _holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
+    }
 
     const t = rbRef.current.translation();
     _currentPos.set(t.x, t.y, t.z);

src/components/ui/HandTrackingOverlay.tsx

@@ -0,0 +1,24 @@
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
+
+export function HandTrackingOverlay(): React.JSX.Element | null {
+  const { hands, status, serverStatus, error } = useHandTrackingSnapshot();
+
+  if (status === "idle") {
+    return null;
+  }
+
+  const pinching = hands.some((hand) => hand.isPinch);
+
+  return (
+    <aside className="hand-tracking-overlay" aria-label="Hand tracking status">
+      <strong>Hand tracking</strong>
+      <span>Status: {status}</span>
+      {serverStatus ? <span>Server: {serverStatus}</span> : null}
+      <span>Hands: {hands.length}</span>
+      <span>Pinch: {pinching ? "yes" : "no"}</span>
+      {error ? (
+        <span className="hand-tracking-overlay__error">{error}</span>
+      ) : null}
+    </aside>
+  );
+}

src/components/ui/HandTrackingProvider.tsx

@@ -0,0 +1,26 @@
+import type { ReactNode } from "react";
+import { useSceneMode } from "@/hooks/debug/useSceneMode";
+import {
+  HAND_TRACKING_IDLE_SNAPSHOT,
+  HandTrackingContext,
+} from "@/hooks/useHandTrackingSnapshot";
+import { useRemoteHandTracking } from "@/hooks/useRemoteHandTracking";
+import { isDebugEnabled } from "@/utils/debug/isDebugEnabled";
+
+export function HandTrackingProvider({
+  children,
+}: {
+  children: ReactNode;
+}): React.JSX.Element {
+  const sceneMode = useSceneMode();
+  const enabled = isDebugEnabled() && sceneMode === "physics";
+  const snapshot = useRemoteHandTracking({ enabled });
+
+  return (
+    <HandTrackingContext
+      value={enabled ? snapshot : HAND_TRACKING_IDLE_SNAPSHOT}
+    >
+      {children}
+    </HandTrackingContext>
+  );
+}

src/data/handTrackingConfig.ts

@@ -0,0 +1,20 @@
+export const HAND_TRACKING_LOCAL_WS_URL = "ws://localhost:8000/ws";
+export const HAND_TRACKING_PROD_WS_URL = "wss://handtracking.la-fabrik.fr/ws";
+
+export const HAND_TRACKING_FRAME_WIDTH = 320;
+export const HAND_TRACKING_FRAME_HEIGHT = 240;
+export const HAND_TRACKING_TARGET_FPS = 10;
+export const HAND_TRACKING_JPEG_QUALITY = 0.55;
+export const HAND_TRACKING_RESPONSE_TIMEOUT_MS = 1_500;
+
+export function getHandTrackingWsUrl(): string {
+  const configuredUrl = import.meta.env.VITE_HAND_TRACKING_WS_URL;
+
+  if (configuredUrl) {
+    return configuredUrl;
+  }
+
+  return import.meta.env.DEV
+    ? HAND_TRACKING_LOCAL_WS_URL
+    : HAND_TRACKING_PROD_WS_URL;
+}

src/hooks/useHandTrackingSnapshot.ts

@@ -0,0 +1,17 @@
+import { createContext, useContext } from "react";
+import type { HandTrackingSnapshot } from "@/types/handTracking";
+
+export const HAND_TRACKING_IDLE_SNAPSHOT: HandTrackingSnapshot = {
+  hands: [],
+  status: "idle",
+  serverStatus: null,
+  error: null,
+};
+
+export const HandTrackingContext = createContext<HandTrackingSnapshot>(
+  HAND_TRACKING_IDLE_SNAPSHOT,
+);
+
+export function useHandTrackingSnapshot(): HandTrackingSnapshot {
+  return useContext(HandTrackingContext);
+}

src/hooks/useRemoteHandTracking.ts

@@ -0,0 +1,231 @@
+import { useEffect, useRef, useState } from "react";
+import {
+  HAND_TRACKING_FRAME_HEIGHT,
+  HAND_TRACKING_FRAME_WIDTH,
+  HAND_TRACKING_JPEG_QUALITY,
+  HAND_TRACKING_RESPONSE_TIMEOUT_MS,
+  HAND_TRACKING_TARGET_FPS,
+  getHandTrackingWsUrl,
+} from "@/data/handTrackingConfig";
+import type {
+  HandTrackingFrameMessage,
+  HandTrackingServerMessage,
+  HandTrackingSnapshot,
+} from "@/types/handTracking";
+
+interface UseRemoteHandTrackingOptions {
+  enabled: boolean;
+  websocketUrl?: string;
+}
+
+const INITIAL_SNAPSHOT: HandTrackingSnapshot = {
+  hands: [],
+  status: "idle",
+  serverStatus: null,
+  error: null,
+};
+
+function getBase64Payload(dataUrl: string): string {
+  return dataUrl.slice(dataUrl.indexOf(",") + 1);
+}
+
+export function useRemoteHandTracking({
+  enabled,
+  websocketUrl = getHandTrackingWsUrl(),
+}: UseRemoteHandTrackingOptions): HandTrackingSnapshot {
+  const [snapshot, setSnapshot] =
+    useState<HandTrackingSnapshot>(INITIAL_SNAPSHOT);
+  const videoRef = useRef<HTMLVideoElement | null>(null);
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const streamRef = useRef<MediaStream | null>(null);
+  const wsRef = useRef<WebSocket | null>(null);
+  const sendIntervalRef = useRef<number | null>(null);
+  const responseTimeoutRef = useRef<number | null>(null);
+  const waitingForResponseRef = useRef(false);
+
+  useEffect(() => {
+    if (!enabled) {
+      return undefined;
+    }
+
+    let cancelled = false;
+
+    const clearResponseTimeout = (): void => {
+      if (responseTimeoutRef.current === null) return;
+      window.clearTimeout(responseTimeoutRef.current);
+      responseTimeoutRef.current = null;
+    };
+
+    const cleanup = (): void => {
+      if (sendIntervalRef.current !== null) {
+        window.clearInterval(sendIntervalRef.current);
+        sendIntervalRef.current = null;
+      }
+
+      clearResponseTimeout();
+      waitingForResponseRef.current = false;
+      wsRef.current?.close();
+      wsRef.current = null;
+
+      streamRef.current?.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+      videoRef.current = null;
+      canvasRef.current = null;
+    };
+
+    const markResponseReceived = (): void => {
+      waitingForResponseRef.current = false;
+      clearResponseTimeout();
+    };
+
+    const sendFrame = (): void => {
+      const ws = wsRef.current;
+      const video = videoRef.current;
+      const canvas = canvasRef.current;
+      const context = canvas?.getContext("2d");
+
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      if (!video || !canvas || !context) return;
+      if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
+      if (waitingForResponseRef.current) return;
+
+      context.drawImage(video, 0, 0, canvas.width, canvas.height);
+      const dataUrl = canvas.toDataURL(
+        "image/jpeg",
+        HAND_TRACKING_JPEG_QUALITY,
+      );
+      const message: HandTrackingFrameMessage = {
+        type: "frame",
+        timestamp: Date.now(),
+        width: canvas.width,
+        height: canvas.height,
+        image: getBase64Payload(dataUrl),
+      };
+
+      waitingForResponseRef.current = true;
+      ws.send(JSON.stringify(message));
+      responseTimeoutRef.current = window.setTimeout(() => {
+        waitingForResponseRef.current = false;
+        responseTimeoutRef.current = null;
+      }, HAND_TRACKING_RESPONSE_TIMEOUT_MS);
+    };
+
+    const start = async (): Promise<void> => {
+      await Promise.resolve();
+      if (cancelled) return;
+
+      setSnapshot({
+        hands: [],
+        status: "connecting",
+        serverStatus: null,
+        error: null,
+      });
+
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          video: {
+            width: HAND_TRACKING_FRAME_WIDTH,
+            height: HAND_TRACKING_FRAME_HEIGHT,
+            facingMode: "user",
+          },
+          audio: false,
+        });
+
+        if (cancelled) {
+          stream.getTracks().forEach((track) => track.stop());
+          return;
+        }
+
+        const video = document.createElement("video");
+        video.muted = true;
+        video.playsInline = true;
+        video.srcObject = stream;
+        await video.play();
+
+        const canvas = document.createElement("canvas");
+        canvas.width = HAND_TRACKING_FRAME_WIDTH;
+        canvas.height = HAND_TRACKING_FRAME_HEIGHT;
+
+        const ws = new WebSocket(websocketUrl);
+        ws.onopen = () => {
+          setSnapshot((current) => ({
+            ...current,
+            status: "connected",
+            error: null,
+          }));
+        };
+        ws.onmessage = (event) => {
+          markResponseReceived();
+          const data = JSON.parse(event.data) as HandTrackingServerMessage;
+
+          if (data.type === "hands") {
+            setSnapshot((current) => ({
+              ...current,
+              hands: data.hands,
+              serverStatus: null,
+              error: null,
+            }));
+            return;
+          }
+
+          if (data.type === "status") {
+            setSnapshot((current) => ({
+              ...current,
+              serverStatus: data.status,
+            }));
+            return;
+          }
+
+          setSnapshot((current) => ({
+            ...current,
+            hands: [],
+            status: "error",
+            error: data.message,
+          }));
+        };
+        ws.onerror = () => {
+          markResponseReceived();
+          setSnapshot((current) => ({
+            ...current,
+            status: "error",
+            error: "Hand tracking WebSocket error",
+          }));
+        };
+        ws.onclose = () => {
+          markResponseReceived();
+          setSnapshot((current) => ({
+            ...current,
+            status: cancelled ? "idle" : "disconnected",
+          }));
+        };
+
+        streamRef.current = stream;
+        videoRef.current = video;
+        canvasRef.current = canvas;
+        wsRef.current = ws;
+        sendIntervalRef.current = window.setInterval(
+          sendFrame,
+          1_000 / HAND_TRACKING_TARGET_FPS,
+        );
+      } catch (error) {
+        if (cancelled) return;
+        setSnapshot({
+          hands: [],
+          status: "error",
+          serverStatus: null,
+          error:
+            error instanceof Error ? error.message : "Hand tracking failed",
+        });
+      }
+    };
+
+    void start();
+
+    return () => {
+      cancelled = true;
+      cleanup();
+    };
+  }, [enabled, websocketUrl]);
+
+  return snapshot;
+}

src/index.css

@@ -79,3 +79,30 @@ canvas {
   color: rgba(255, 255, 255, 0.85);
   letter-spacing: 0.03em;
 }
+
+.hand-tracking-overlay {
+  position: fixed;
+  right: 16px;
+  bottom: 16px;
+  z-index: 20;
+  display: flex;
+  flex-direction: column;
+  gap: 4px;
+  min-width: 180px;
+  padding: 12px;
+  color: rgba(255, 255, 255, 0.9);
+  background: rgba(4, 7, 13, 0.78);
+  border: 1px solid rgba(255, 255, 255, 0.18);
+  border-radius: 8px;
+  font-size: 12px;
+  pointer-events: none;
+}
+
+.hand-tracking-overlay strong {
+  color: white;
+  font-size: 13px;
+}
+
+.hand-tracking-overlay__error {
+  color: #fca5a5;
+}

src/types/handTracking.ts

@@ -0,0 +1,55 @@
+export interface HandTrackingHand {
+  x: number;
+  y: number;
+  z: number;
+  handedness: string;
+  isPinch: boolean;
+  pinchDistance: number;
+  score: number;
+}
+
+export type HandTrackingStatus =
+  | "idle"
+  | "connecting"
+  | "connected"
+  | "disconnected"
+  | "error";
+
+export interface HandTrackingSnapshot {
+  hands: HandTrackingHand[];
+  status: HandTrackingStatus;
+  serverStatus: string | null;
+  error: string | null;
+}
+
+export interface HandTrackingFrameMessage {
+  type: "frame";
+  timestamp: number;
+  width: number;
+  height: number;
+  image: string;
+}
+
+export interface HandTrackingHandsMessage {
+  type: "hands";
+  timestamp: number;
+  hands: HandTrackingHand[];
+}
+
+export interface HandTrackingStatusMessage {
+  type: "status";
+  timestamp: number;
+  status: string;
+}
+
+export interface HandTrackingErrorMessage {
+  type: "error";
+  timestamp: number;
+  hands: HandTrackingHand[];
+  message: string;
+}
+
+export type HandTrackingServerMessage =
+  | HandTrackingHandsMessage
+  | HandTrackingStatusMessage
+  | HandTrackingErrorMessage;

src/world/debug/TestScene.tsx

@@ -54,6 +54,7 @@ export function TestScene({
         <GrabbableObject
           position={TEST_SCENE_GRABBABLE_POSITION}
           colliders="cuboid"
+          handControlled
         >
           <mesh castShadow receiveShadow>
             <boxGeometry args={TEST_SCENE_GRABBABLE_BOX_SIZE} />