feat: improve fist grab depth tracking

backend/hand_tracker.py

@@ -76,15 +76,17 @@ class HandTracker:
             result.hand_landmarks,
             result.handedness,
         ):
-            index_tip = landmarks[8]
+            palm_center = self._average_points(
+                [landmarks[0], landmarks[5], landmarks[9], landmarks[13], landmarks[17]],
+            )
             is_fist = self._is_fist(landmarks)
             handedness = handedness_categories[0]
 
             hands.append(
                 HandData(
-                    x=index_tip.x,
+                    x=palm_center["x"],
-                    y=index_tip.y,
+                    y=palm_center["y"],
-                    z=index_tip.z,
+                    z=palm_center["z"],
                     landmarks=[
                         {"x": point.x, "y": point.y, "z": point.z}
                         for point in landmarks

src/components/three/GrabbableObject.tsx

@@ -64,6 +64,7 @@ export function GrabbableObject({
   const rbRef = useRef<RapierRigidBody>(null);
   const isHolding = useRef(false);
   const isHandHolding = useRef(false);
+  const handHoldDistance = useRef<number | null>(null);
 
   useDebugFolder("GrabbableObject", (folder) => {
     folder
@@ -123,17 +124,23 @@ export function GrabbableObject({
           : [];
 
         isHandHolding.current = isObjectInRange && hits.length > 0;
+        handHoldDistance.current = isHandHolding.current
+          ? hits[0].distance
+          : null;
       }
     } else {
       isHandHolding.current = false;
+      handHoldDistance.current = null;
     }
 
     if (!isHolding.current && !isHandHolding.current) return;
 
     if (fistHand && isHandHolding.current) {
+      const holdDistance = handHoldDistance.current ?? params.holdDistance;
+
       _holdTarget
         .copy(_cameraPos)
-        .addScaledVector(_handDirection, params.holdDistance);
+        .addScaledVector(_handDirection, holdDistance);
     } else {
       camera.getWorldDirection(_holdTarget);
       _holdTarget.multiplyScalar(params.holdDistance).add(camera.position);