refactor: replace pinch gesture with fist gesture

backend/README.md

@@ -2,7 +2,7 @@
 
 Remote-compatible Python backend for La-Fabrik hand tracking.
 
-The browser captures webcam frames, downsizes them, sends JPEG frames to this backend over WebSocket, and receives hand landmarks plus pinch state.
+The browser captures webcam frames, downsizes them, sends JPEG frames to this backend over WebSocket, and receives hand landmarks plus closed-fist state.
 
 ## Setup
 
@@ -76,8 +76,7 @@ Server responds with detected hands:
         }
       ],
       "handedness": "Right",
-      "isPinch": true,
-      "pinchDistance": 0.05,
+      "isFist": true,
       "score": 0.92
     }
   ]

backend/hand_tracker.py

@@ -21,8 +21,7 @@ class HandData:
     z: float
     landmarks: list[dict[str, float]]
     handedness: str
-    is_pinch: bool
-    pinch_distance: float
+    is_fist: bool
     score: float
 
     def to_payload(self) -> dict[str, float | str | bool | list[dict[str, float]]]:
@@ -32,8 +31,7 @@ class HandData:
             "z": self.z,
             "landmarks": self.landmarks,
             "handedness": self.handedness,
-            "isPinch": self.is_pinch,
-            "pinchDistance": self.pinch_distance,
+            "isFist": self.is_fist,
             "score": self.score,
         }
 
@@ -79,8 +77,7 @@ class HandTracker:
             result.handedness,
         ):
             index_tip = landmarks[8]
-            thumb_tip = landmarks[4]
-            pinch_distance = self._calculate_distance(index_tip, thumb_tip)
+            is_fist = self._is_fist(landmarks)
             handedness = handedness_categories[0]
 
             hands.append(
@@ -93,21 +90,51 @@ class HandTracker:
                         for point in landmarks
                     ],
                     handedness=handedness.category_name,
-                    is_pinch=pinch_distance < 0.07,
-                    pinch_distance=pinch_distance,
+                    is_fist=is_fist,
                     score=handedness.score,
                 ),
             )
 
         return hands
 
+    def _is_fist(self, landmarks: list[Any]) -> bool:
+        palm_center = self._average_points(
+            [landmarks[0], landmarks[5], landmarks[9], landmarks[13], landmarks[17]],
+        )
+        palm_size = self._calculate_distance(landmarks[0], landmarks[9])
+        if palm_size <= 0:
+            return False
+
+        folded_finger_count = sum(
+            self._calculate_distance(landmarks[index], palm_center) / palm_size < 1.05
+            for index in (8, 12, 16, 20)
+        )
+
+        return folded_finger_count >= 4
+
+    def _average_points(self, points: list[Any]) -> dict[str, float]:
+        return {
+            "x": sum(point.x for point in points) / len(points),
+            "y": sum(point.y for point in points) / len(points),
+            "z": sum(point.z for point in points) / len(points),
+        }
+
     def _calculate_distance(self, point_a: Any, point_b: Any) -> float:
         return math.sqrt(
-            (point_a.x - point_b.x) ** 2
-            + (point_a.y - point_b.y) ** 2
-            + (point_a.z - point_b.z) ** 2,
+            (self._get_coordinate(point_a, "x") - self._get_coordinate(point_b, "x"))
+            ** 2
+            + (self._get_coordinate(point_a, "y") - self._get_coordinate(point_b, "y"))
+            ** 2
+            + (self._get_coordinate(point_a, "z") - self._get_coordinate(point_b, "z"))
+            ** 2,
         )
 
+    def _get_coordinate(self, point: Any, axis: str) -> float:
+        if isinstance(point, dict):
+            return point[axis]
+
+        return getattr(point, axis)
+
 
 def now_ms() -> int:
     return time.monotonic_ns() // 1_000_000

src/components/three/GrabbableObject.tsx

@@ -92,14 +92,14 @@ export function GrabbableObject({
   useFrame(() => {
     if (!rbRef.current) return;
 
-    const pinchingHand = handControlled
-      ? hands.find((hand) => hand.isPinch)
+    const fistHand = handControlled
+      ? hands.find((hand) => hand.isFist)
       : undefined;
 
-    if (!isHolding.current && !pinchingHand) return;
+    if (!isHolding.current && !fistHand) return;
 
-    if (pinchingHand) {
-      _handNdc.set((1 - pinchingHand.x) * 2 - 1, -pinchingHand.y * 2 + 1, 0.5);
+    if (fistHand) {
+      _handNdc.set((1 - fistHand.x) * 2 - 1, -fistHand.y * 2 + 1, 0.5);
       _handNdc.unproject(camera);
       _handDirection.subVectors(_handNdc, camera.position).normalize();
       _holdTarget

src/components/ui/HandTrackingOverlay.tsx

@@ -13,21 +13,23 @@ const STATUS_LABELS: Record<HandTrackingStatus, string> = {
 };
 
 export function HandTrackingOverlay(): React.JSX.Element | null {
-  const { hands, status, serverStatus, error } = useHandTrackingSnapshot();
+  const { hands, status, usageStatus, serverStatus, error } =
+    useHandTrackingSnapshot();
 
   if (status === "idle") {
     return null;
   }
 
-  const pinching = hands.some((hand) => hand.isPinch);
+  const fist = hands.some((hand) => hand.isFist);
 
   return (
     <aside className="hand-tracking-overlay" aria-label="Hand tracking status">
       <strong>Hand tracking</strong>
       <span>Status: {STATUS_LABELS[status]}</span>
+      <span>Usage: {usageStatus}</span>
       {serverStatus ? <span>Server: {serverStatus}</span> : null}
       <span>Hands: {hands.length}</span>
-      <span>Pinch: {pinching ? "yes" : "no"}</span>
+      <span>Fist: {fist ? "yes" : "no"}</span>
       {error ? (
         <span className="hand-tracking-overlay__error">{error}</span>
       ) : null}

src/components/ui/HandTrackingProvider.tsx

@@ -1,5 +1,6 @@
 import type { ReactNode } from "react";
 import { useSceneMode } from "@/hooks/debug/useSceneMode";
+import { useInteraction } from "@/hooks/useInteraction";
 import {
   HAND_TRACKING_IDLE_SNAPSHOT,
   HandTrackingContext,
@@ -13,7 +14,10 @@ export function HandTrackingProvider({
   children: ReactNode;
 }): React.JSX.Element {
   const sceneMode = useSceneMode();
-  const enabled = isDebugEnabled() && sceneMode === "physics";
+  const { focused, holding } = useInteraction();
+  const isInInteractionZone = focused !== null || holding;
+  const enabled =
+    isDebugEnabled() && sceneMode === "physics" && isInInteractionZone;
   const snapshot = useRemoteHandTracking({ enabled });
 
   return (

src/components/ui/HandTrackingVisualizer.tsx

@@ -37,7 +37,7 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
         const landmarks = hand.landmarks ?? [];
         if (landmarks.length === 0) return null;
 
-        const color = hand.isPinch ? "#facc15" : "#38bdf8";
+        const color = hand.isFist ? "#facc15" : "#38bdf8";
 
         return (
           <g key={`${hand.handedness}-${handIndex}`}>

src/hooks/useHandTrackingSnapshot.ts

@@ -4,6 +4,7 @@ import type { HandTrackingSnapshot } from "@/types/handTracking";
 export const HAND_TRACKING_IDLE_SNAPSHOT: HandTrackingSnapshot = {
   hands: [],
   status: "idle",
+  usageStatus: "inactive",
   serverStatus: null,
   error: null,
 };

src/hooks/useRemoteHandTracking.ts

@@ -22,6 +22,7 @@ interface UseRemoteHandTrackingOptions {
 const INITIAL_SNAPSHOT: HandTrackingSnapshot = {
   hands: [],
   status: "idle",
+  usageStatus: "inactive",
   serverStatus: null,
   error: null,
 };
@@ -144,6 +145,7 @@ export function useRemoteHandTracking({
       setSnapshot({
         hands: [],
         status: "requesting_camera",
+        usageStatus: "available",
         serverStatus: null,
         error: null,
       });
@@ -193,6 +195,7 @@ export function useRemoteHandTracking({
           setSnapshot((current) => ({
             ...current,
             status: "connected",
+            usageStatus: "available",
             error: null,
           }));
         };
@@ -204,6 +207,9 @@ export function useRemoteHandTracking({
             setSnapshot((current) => ({
               ...current,
               hands: data.hands,
+              usageStatus: data.hands.some((hand) => hand.isFist)
+                ? "active"
+                : "available",
               serverStatus: null,
               error: null,
             }));
@@ -222,6 +228,7 @@ export function useRemoteHandTracking({
             ...current,
             hands: [],
             status: "error",
+            usageStatus: "inactive",
             error: data.message,
           }));
         };
@@ -254,6 +261,7 @@ export function useRemoteHandTracking({
         setSnapshot({
           hands: [],
           status: "error",
+          usageStatus: "inactive",
           serverStatus: null,
           error:
             error instanceof Error ? error.message : "Hand tracking failed",

src/types/handTracking.ts

@@ -10,11 +10,12 @@ export interface HandTrackingHand {
   z: number;
   landmarks: HandTrackingLandmark[];
   handedness: string;
-  isPinch: boolean;
-  pinchDistance: number;
+  isFist: boolean;
   score: number;
 }
 
+export type HandTrackingUsageStatus = "inactive" | "available" | "active";
+
 export type HandTrackingStatus =
   | "idle"
   | "requesting_camera"
@@ -28,6 +29,7 @@ export type HandTrackingStatus =
 export interface HandTrackingSnapshot {
   hands: HandTrackingHand[];
   status: HandTrackingStatus;
+  usageStatus: HandTrackingUsageStatus;
   serverStatus: string | null;
   error: string | null;
 }