add browser hand tracking source

src/lib/handTracking/browserHandTracking.ts

@@ -0,0 +1,117 @@
+import {
+  HAND_TRACKING_BROWSER_MODEL_URL,
+  HAND_TRACKING_BROWSER_WASM_URL,
+} from "@/data/handTrackingConfig";
+import type {
+  HandTrackingHand,
+  HandTrackingLandmark,
+} from "@/types/handTracking/handTracking";
+
+type HandLandmarkerModule = typeof import("@mediapipe/tasks-vision");
+type HandLandmarker = Awaited<
+  ReturnType<HandLandmarkerModule["HandLandmarker"]["createFromOptions"]>
+>;
+type HandLandmarkerResult = ReturnType<HandLandmarker["detectForVideo"]>;
+
+let handLandmarkerPromise: Promise<HandLandmarker> | null = null;
+
+function averageLandmarks(
+  landmarks: HandTrackingLandmark[],
+  indices: number[],
+): HandTrackingLandmark {
+  const point = indices.reduce(
+    (current, index) => {
+      const landmark = landmarks[index];
+      if (!landmark) return current;
+
+      return {
+        x: current.x + landmark.x,
+        y: current.y + landmark.y,
+        z: current.z + landmark.z,
+      };
+    },
+    { x: 0, y: 0, z: 0 },
+  );
+
+  return {
+    x: point.x / indices.length,
+    y: point.y / indices.length,
+    z: point.z / indices.length,
+  };
+}
+
+function distance(
+  pointA: HandTrackingLandmark,
+  pointB: HandTrackingLandmark,
+): number {
+  return Math.sqrt(
+    (pointA.x - pointB.x) ** 2 +
+      (pointA.y - pointB.y) ** 2 +
+      (pointA.z - pointB.z) ** 2,
+  );
+}
+
+function isFist(landmarks: HandTrackingLandmark[]): boolean {
+  const palmCenter = averageLandmarks(landmarks, [0, 5, 9, 13, 17]);
+  const wrist = landmarks[0];
+  const middleMcp = landmarks[9];
+
+  if (!wrist || !middleMcp) return false;
+
+  const palmSize = distance(wrist, middleMcp);
+  if (palmSize <= 0) return false;
+
+  const foldedFingerCount = [8, 12, 16, 20].filter((index) => {
+    const landmark = landmarks[index];
+    if (!landmark) return false;
+
+    return distance(landmark, palmCenter) / palmSize < 1.05;
+  }).length;
+
+  return foldedFingerCount >= 4;
+}
+
+export async function getBrowserHandLandmarker(): Promise<HandLandmarker> {
+  handLandmarkerPromise ??= import("@mediapipe/tasks-vision").then(
+    async ({ FilesetResolver, HandLandmarker }) => {
+      const vision = await FilesetResolver.forVisionTasks(
+        HAND_TRACKING_BROWSER_WASM_URL,
+      );
+
+      return HandLandmarker.createFromOptions(vision, {
+        baseOptions: {
+          modelAssetPath: HAND_TRACKING_BROWSER_MODEL_URL,
+          delegate: "GPU",
+        },
+        numHands: 2,
+        runningMode: "VIDEO",
+      });
+    },
+  );
+
+  return handLandmarkerPromise;
+}
+
+export function convertBrowserHandResult(
+  result: HandLandmarkerResult,
+): HandTrackingHand[] {
+  return result.landmarks.map((landmarks, index) => {
+    const normalizedLandmarks = landmarks.map((landmark) => ({
+      x: landmark.x,
+      y: landmark.y,
+      z: landmark.z,
+    }));
+    const palmCenter = averageLandmarks(normalizedLandmarks, [0, 5, 9, 13, 17]);
+    const handedness = result.handedness[index]?.[0];
+
+    return {
+      x: palmCenter.x,
+      y: palmCenter.y,
+      z: palmCenter.z,
+      landmarks: normalizedLandmarks,
+      handedness: handedness?.categoryName ?? "Unknown",
+      isFist: isFist(normalizedLandmarks),
+      score: handedness?.score ?? 0,
+    };
+  });
+}