diff --git a/backend/README.md b/backend/README.md index 05d33d2..18607aa 100644 --- a/backend/README.md +++ b/backend/README.md @@ -2,7 +2,7 @@ Remote-compatible Python backend for La-Fabrik hand tracking. -The browser captures webcam frames, downsizes them, sends JPEG frames to this backend over WebSocket, and receives hand landmarks plus pinch state. +The browser captures webcam frames, downsizes them, sends JPEG frames to this backend over WebSocket, and receives hand landmarks plus closed-fist state. ## Setup @@ -76,8 +76,7 @@ Server responds with detected hands: } ], "handedness": "Right", - "isPinch": true, - "pinchDistance": 0.05, + "isFist": true, "score": 0.92 } ] diff --git a/backend/hand_tracker.py b/backend/hand_tracker.py index 767dbf4..67182e9 100644 --- a/backend/hand_tracker.py +++ b/backend/hand_tracker.py @@ -21,8 +21,7 @@ class HandData: z: float landmarks: list[dict[str, float]] handedness: str - is_pinch: bool - pinch_distance: float + is_fist: bool score: float def to_payload(self) -> dict[str, float | str | bool | list[dict[str, float]]]: @@ -32,8 +31,7 @@ class HandData: "z": self.z, "landmarks": self.landmarks, "handedness": self.handedness, - "isPinch": self.is_pinch, - "pinchDistance": self.pinch_distance, + "isFist": self.is_fist, "score": self.score, } @@ -79,8 +77,7 @@ class HandTracker: result.handedness, ): index_tip = landmarks[8] - thumb_tip = landmarks[4] - pinch_distance = self._calculate_distance(index_tip, thumb_tip) + is_fist = self._is_fist(landmarks) handedness = handedness_categories[0] hands.append( @@ -93,21 +90,51 @@ class HandTracker: for point in landmarks ], handedness=handedness.category_name, - is_pinch=pinch_distance < 0.07, - pinch_distance=pinch_distance, + is_fist=is_fist, score=handedness.score, ), ) return hands + def _is_fist(self, landmarks: list[Any]) -> bool: + palm_center = self._average_points( + [landmarks[0], landmarks[5], landmarks[9], landmarks[13], landmarks[17]], + ) + palm_size = self._calculate_distance(landmarks[0], landmarks[9]) + if palm_size <= 0: + return False + + folded_finger_count = sum( + self._calculate_distance(landmarks[index], palm_center) / palm_size < 1.05 + for index in (8, 12, 16, 20) + ) + + return folded_finger_count >= 4 + + def _average_points(self, points: list[Any]) -> dict[str, float]: + return { + "x": sum(point.x for point in points) / len(points), + "y": sum(point.y for point in points) / len(points), + "z": sum(point.z for point in points) / len(points), + } + def _calculate_distance(self, point_a: Any, point_b: Any) -> float: return math.sqrt( - (point_a.x - point_b.x) ** 2 - + (point_a.y - point_b.y) ** 2 - + (point_a.z - point_b.z) ** 2, + (self._get_coordinate(point_a, "x") - self._get_coordinate(point_b, "x")) + ** 2 + + (self._get_coordinate(point_a, "y") - self._get_coordinate(point_b, "y")) + ** 2 + + (self._get_coordinate(point_a, "z") - self._get_coordinate(point_b, "z")) + ** 2, ) + def _get_coordinate(self, point: Any, axis: str) -> float: + if isinstance(point, dict): + return point[axis] + + return getattr(point, axis) + def now_ms() -> int: return time.monotonic_ns() // 1_000_000 diff --git a/src/components/three/GrabbableObject.tsx b/src/components/three/GrabbableObject.tsx index 12e1ca7..e58917f 100644 --- a/src/components/three/GrabbableObject.tsx +++ b/src/components/three/GrabbableObject.tsx @@ -92,14 +92,14 @@ export function GrabbableObject({ useFrame(() => { if (!rbRef.current) return; - const pinchingHand = handControlled - ? hands.find((hand) => hand.isPinch) + const fistHand = handControlled + ? hands.find((hand) => hand.isFist) : undefined; - if (!isHolding.current && !pinchingHand) return; + if (!isHolding.current && !fistHand) return; - if (pinchingHand) { - _handNdc.set((1 - pinchingHand.x) * 2 - 1, -pinchingHand.y * 2 + 1, 0.5); + if (fistHand) { + _handNdc.set((1 - fistHand.x) * 2 - 1, -fistHand.y * 2 + 1, 0.5); _handNdc.unproject(camera); _handDirection.subVectors(_handNdc, camera.position).normalize(); _holdTarget diff --git a/src/components/ui/HandTrackingOverlay.tsx b/src/components/ui/HandTrackingOverlay.tsx index e73d720..4a415a9 100644 --- a/src/components/ui/HandTrackingOverlay.tsx +++ b/src/components/ui/HandTrackingOverlay.tsx @@ -13,21 +13,23 @@ const STATUS_LABELS: Record = { }; export function HandTrackingOverlay(): React.JSX.Element | null { - const { hands, status, serverStatus, error } = useHandTrackingSnapshot(); + const { hands, status, usageStatus, serverStatus, error } = + useHandTrackingSnapshot(); if (status === "idle") { return null; } - const pinching = hands.some((hand) => hand.isPinch); + const fist = hands.some((hand) => hand.isFist); return (