diff --git a/backend/README.md b/backend/README.md
index b436c5c..05d33d2 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -68,6 +68,13 @@ Server responds with detected hands:
"x": 0.5,
"y": 0.3,
"z": 0.1,
+ "landmarks": [
+ {
+ "x": 0.48,
+ "y": 0.32,
+ "z": 0.02
+ }
+ ],
"handedness": "Right",
"isPinch": true,
"pinchDistance": 0.05,
diff --git a/backend/hand_tracker.py b/backend/hand_tracker.py
index ff9cbcd..767dbf4 100644
--- a/backend/hand_tracker.py
+++ b/backend/hand_tracker.py
@@ -19,16 +19,18 @@ class HandData:
x: float
y: float
z: float
+ landmarks: list[dict[str, float]]
handedness: str
is_pinch: bool
pinch_distance: float
score: float
- def to_payload(self) -> dict[str, float | str | bool]:
+ def to_payload(self) -> dict[str, float | str | bool | list[dict[str, float]]]:
return {
"x": self.x,
"y": self.y,
"z": self.z,
+ "landmarks": self.landmarks,
"handedness": self.handedness,
"isPinch": self.is_pinch,
"pinchDistance": self.pinch_distance,
@@ -86,6 +88,10 @@ class HandTracker:
x=index_tip.x,
y=index_tip.y,
z=index_tip.z,
+ landmarks=[
+ {"x": point.x, "y": point.y, "z": point.z}
+ for point in landmarks
+ ],
handedness=handedness.category_name,
is_pinch=pinch_distance < 0.07,
pinch_distance=pinch_distance,
diff --git a/src/components/ui/HandTrackingVisualizer.tsx b/src/components/ui/HandTrackingVisualizer.tsx
new file mode 100644
index 0000000..1b49d51
--- /dev/null
+++ b/src/components/ui/HandTrackingVisualizer.tsx
@@ -0,0 +1,77 @@
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
+
+const HAND_CONNECTIONS: Array<[number, number]> = [
+ [0, 1],
+ [1, 2],
+ [2, 3],
+ [3, 4],
+ [0, 5],
+ [5, 6],
+ [6, 7],
+ [7, 8],
+ [5, 9],
+ [9, 10],
+ [10, 11],
+ [11, 12],
+ [9, 13],
+ [13, 14],
+ [14, 15],
+ [15, 16],
+ [13, 17],
+ [17, 18],
+ [18, 19],
+ [19, 20],
+ [0, 17],
+];
+
+export function HandTrackingVisualizer(): React.JSX.Element | null {
+ const { hands, status } = useHandTrackingSnapshot();
+
+ if (status === "idle" || hands.length === 0) {
+ return null;
+ }
+
+ return (
+
+ );
+}
diff --git a/src/index.css b/src/index.css
index cd7d65f..fd1e87a 100644
--- a/src/index.css
+++ b/src/index.css
@@ -418,6 +418,16 @@ canvas {
color: #fca5a5;
}
+.hand-tracking-visualizer {
+ position: fixed;
+ inset: 0;
+ z-index: 15;
+ width: 100vw;
+ height: 100vh;
+ pointer-events: none;
+ filter: drop-shadow(0 0 8px rgba(56, 189, 248, 0.55));
+}
+
/* Editor page */
.editor-container {
position: fixed;
diff --git a/src/pages/page.tsx b/src/pages/page.tsx
index 19ddb74..12c6688 100644
--- a/src/pages/page.tsx
+++ b/src/pages/page.tsx
@@ -3,6 +3,7 @@ import { Canvas } from "@react-three/fiber";
import { Crosshair } from "@/components/ui/Crosshair";
import { HandTrackingOverlay } from "@/components/ui/HandTrackingOverlay";
import { HandTrackingProvider } from "@/components/ui/HandTrackingProvider";
+import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer";
import { InteractPrompt } from "@/components/ui/InteractPrompt";
import { DebugPerf } from "@/components/debug/DebugPerf";
import { World } from "@/world/World";
@@ -18,6 +19,7 @@ export function HomePage(): React.JSX.Element {
+
);
diff --git a/src/types/handTracking.ts b/src/types/handTracking.ts
index 6d7e1e2..05d91b7 100644
--- a/src/types/handTracking.ts
+++ b/src/types/handTracking.ts
@@ -1,7 +1,14 @@
+export interface HandTrackingLandmark {
+ x: number;
+ y: number;
+ z: number;
+}
+
export interface HandTrackingHand {
x: number;
y: number;
z: number;
+ landmarks: HandTrackingLandmark[];
handedness: string;
isPinch: boolean;
pinchDistance: number;