From 3503ff52ed49348468205f8cb33c5077328cd77c Mon Sep 17 00:00:00 2001
From: Tom Boullay <tom@famille-boullay.fr>
Date: Wed, 29 Apr 2026 09:52:46 +0200
Subject: [PATCH] fix: guard hand landmark visualization

---
 backend/README.md                            |  7 ++
 backend/hand_tracker.py                      |  8 +-
 src/components/ui/HandTrackingVisualizer.tsx | 77 ++++++++++++++++++++
 src/index.css                                | 10 +++
 src/pages/page.tsx                           |  2 +
 src/types/handTracking.ts                    |  7 ++
 6 files changed, 110 insertions(+), 1 deletion(-)
 create mode 100644 src/components/ui/HandTrackingVisualizer.tsx

diff --git a/backend/README.md b/backend/README.md
index b436c5c..05d33d2 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -68,6 +68,13 @@ Server responds with detected hands:
       "x": 0.5,
       "y": 0.3,
       "z": 0.1,
+      "landmarks": [
+        {
+          "x": 0.48,
+          "y": 0.32,
+          "z": 0.02
+        }
+      ],
       "handedness": "Right",
       "isPinch": true,
       "pinchDistance": 0.05,
diff --git a/backend/hand_tracker.py b/backend/hand_tracker.py
index ff9cbcd..767dbf4 100644
--- a/backend/hand_tracker.py
+++ b/backend/hand_tracker.py
@@ -19,16 +19,18 @@ class HandData:
     x: float
     y: float
     z: float
+    landmarks: list[dict[str, float]]
     handedness: str
     is_pinch: bool
     pinch_distance: float
     score: float
 
-    def to_payload(self) -> dict[str, float | str | bool]:
+    def to_payload(self) -> dict[str, float | str | bool | list[dict[str, float]]]:
         return {
             "x": self.x,
             "y": self.y,
             "z": self.z,
+            "landmarks": self.landmarks,
             "handedness": self.handedness,
             "isPinch": self.is_pinch,
             "pinchDistance": self.pinch_distance,
@@ -86,6 +88,10 @@ class HandTracker:
                     x=index_tip.x,
                     y=index_tip.y,
                     z=index_tip.z,
+                    landmarks=[
+                        {"x": point.x, "y": point.y, "z": point.z}
+                        for point in landmarks
+                    ],
                     handedness=handedness.category_name,
                     is_pinch=pinch_distance < 0.07,
                     pinch_distance=pinch_distance,
diff --git a/src/components/ui/HandTrackingVisualizer.tsx b/src/components/ui/HandTrackingVisualizer.tsx
new file mode 100644
index 0000000..1b49d51
--- /dev/null
+++ b/src/components/ui/HandTrackingVisualizer.tsx
@@ -0,0 +1,77 @@
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
+
+const HAND_CONNECTIONS: Array<[number, number]> = [
+  [0, 1],
+  [1, 2],
+  [2, 3],
+  [3, 4],
+  [0, 5],
+  [5, 6],
+  [6, 7],
+  [7, 8],
+  [5, 9],
+  [9, 10],
+  [10, 11],
+  [11, 12],
+  [9, 13],
+  [13, 14],
+  [14, 15],
+  [15, 16],
+  [13, 17],
+  [17, 18],
+  [18, 19],
+  [19, 20],
+  [0, 17],
+];
+
+export function HandTrackingVisualizer(): React.JSX.Element | null {
+  const { hands, status } = useHandTrackingSnapshot();
+
+  if (status === "idle" || hands.length === 0) {
+    return null;
+  }
+
+  return (
+    <svg className="hand-tracking-visualizer" aria-hidden="true">
+      {hands.map((hand, handIndex) => {
+        const landmarks = hand.landmarks ?? [];
+        if (landmarks.length === 0) return null;
+
+        const color = hand.isPinch ? "#facc15" : "#38bdf8";
+
+        return (
+          <g key={`${hand.handedness}-${handIndex}`}>
+            {HAND_CONNECTIONS.map(([from, to]) => {
+              const fromPoint = landmarks[from];
+              const toPoint = landmarks[to];
+              if (!fromPoint || !toPoint) return null;
+
+              return (
+                <line
+                  key={`${from}-${to}`}
+                  x1={`${(1 - fromPoint.x) * 100}%`}
+                  y1={`${fromPoint.y * 100}%`}
+                  x2={`${(1 - toPoint.x) * 100}%`}
+                  y2={`${toPoint.y * 100}%`}
+                  stroke={color}
+                  strokeWidth="2"
+                  strokeLinecap="round"
+                />
+              );
+            })}
+
+            {landmarks.map((landmark, landmarkIndex) => (
+              <circle
+                key={landmarkIndex}
+                cx={`${(1 - landmark.x) * 100}%`}
+                cy={`${landmark.y * 100}%`}
+                r={landmarkIndex === 8 ? 5 : 3}
+                fill={landmarkIndex === 8 ? "#ffffff" : color}
+              />
+            ))}
+          </g>
+        );
+      })}
+    </svg>
+  );
+}
diff --git a/src/index.css b/src/index.css
index cd7d65f..fd1e87a 100644
--- a/src/index.css
+++ b/src/index.css
@@ -418,6 +418,16 @@ canvas {
   color: #fca5a5;
 }
 
+.hand-tracking-visualizer {
+  position: fixed;
+  inset: 0;
+  z-index: 15;
+  width: 100vw;
+  height: 100vh;
+  pointer-events: none;
+  filter: drop-shadow(0 0 8px rgba(56, 189, 248, 0.55));
+}
+
 /* Editor page */
 .editor-container {
   position: fixed;
diff --git a/src/pages/page.tsx b/src/pages/page.tsx
index 19ddb74..12c6688 100644
--- a/src/pages/page.tsx
+++ b/src/pages/page.tsx
@@ -3,6 +3,7 @@ import { Canvas } from "@react-three/fiber";
 import { Crosshair } from "@/components/ui/Crosshair";
 import { HandTrackingOverlay } from "@/components/ui/HandTrackingOverlay";
 import { HandTrackingProvider } from "@/components/ui/HandTrackingProvider";
+import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer";
 import { InteractPrompt } from "@/components/ui/InteractPrompt";
 import { DebugPerf } from "@/components/debug/DebugPerf";
 import { World } from "@/world/World";
@@ -18,6 +19,7 @@ export function HomePage(): React.JSX.Element {
       </Canvas>
       <Crosshair />
       <InteractPrompt />
+      <HandTrackingVisualizer />
       <HandTrackingOverlay />
     </HandTrackingProvider>
   );
diff --git a/src/types/handTracking.ts b/src/types/handTracking.ts
index 6d7e1e2..05d91b7 100644
--- a/src/types/handTracking.ts
+++ b/src/types/handTracking.ts
@@ -1,7 +1,14 @@
+export interface HandTrackingLandmark {
+  x: number;
+  y: number;
+  z: number;
+}
+
 export interface HandTrackingHand {
   x: number;
   y: number;
   z: number;
+  landmarks: HandTrackingLandmark[];
   handedness: string;
   isPinch: boolean;
   pinchDistance: number;