From 73c6d7d50d16b629d69201f36f43a7508af9f0e7 Mon Sep 17 00:00:00 2001 From: Tom Boullay Date: Tue, 2 Jun 2026 17:39:43 +0200 Subject: [PATCH] fix(handtracking): bump browser camera to 640x480 for detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser MediaPipe model (hand_landmarker.task float16) was failing to detect hands at 320x240 — MediaPipe ran for 10s straight without ever returning a hand. Bumping the requested camera resolution to 640x480 (browser only — backend still ships 320x240 JPEGs over the WebSocket) makes the model reliably pick hands up. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/data/handTrackingConfig.ts | 6 ++++++ src/hooks/handTracking/useBrowserHandTracking.ts | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/data/handTrackingConfig.ts b/src/data/handTrackingConfig.ts index 5133463..9537124 100644 --- a/src/data/handTrackingConfig.ts +++ b/src/data/handTrackingConfig.ts @@ -1,5 +1,11 @@ export const HAND_TRACKING_FRAME_WIDTH = 320; export const HAND_TRACKING_FRAME_HEIGHT = 240; +// The browser MediaPipe model (hand_landmarker.task float16) is more +// sensitive than the backend Python model and needs a higher-resolution +// frame to detect hands reliably. The backend keeps 320x240 because that +// is the JPEG payload size sent over the WebSocket. +export const HAND_TRACKING_BROWSER_CAMERA_WIDTH = 640; +export const HAND_TRACKING_BROWSER_CAMERA_HEIGHT = 480; export const HAND_TRACKING_TARGET_FPS = 10; export const HAND_TRACKING_JPEG_QUALITY = 0.55; export const HAND_TRACKING_CAMERA_TIMEOUT_MS = 8_000; diff --git a/src/hooks/handTracking/useBrowserHandTracking.ts b/src/hooks/handTracking/useBrowserHandTracking.ts index 285fa2f..9364dcf 100644 --- a/src/hooks/handTracking/useBrowserHandTracking.ts +++ b/src/hooks/handTracking/useBrowserHandTracking.ts @@ -1,7 +1,7 @@ import { useEffect, useRef, useState } from "react"; import { - HAND_TRACKING_FRAME_HEIGHT, - HAND_TRACKING_FRAME_WIDTH, + HAND_TRACKING_BROWSER_CAMERA_HEIGHT, + HAND_TRACKING_BROWSER_CAMERA_WIDTH, HAND_TRACKING_RUNTIME_START_DELAY_MS, HAND_TRACKING_TARGET_FPS, } from "@/data/handTrackingConfig"; @@ -66,8 +66,8 @@ export function useBrowserHandTracking({ try { const stream = await getCameraStreamWithTimeout({ video: { - width: HAND_TRACKING_FRAME_WIDTH, - height: HAND_TRACKING_FRAME_HEIGHT, + width: HAND_TRACKING_BROWSER_CAMERA_WIDTH, + height: HAND_TRACKING_BROWSER_CAMERA_HEIGHT, facingMode: "user", }, audio: false,