fix(handtracking): reduce GPU pressure on WebGL context loss

Several mitigations against the WebGL context lost that fires when
hand tracking starts on a loaded scene:

- Canvas: fixed DPR [1,1], antialias off, scoped id="game-canvas",
  context-lost handler releases MediaPipe and logs GPU memory counters
- optimizeGLTFScene: cap anisotropy at 2 and stop forcing mipmaps /
  needsUpdate on every pass — avoids massive texture re-uploads
- MediaPipe: force CPU delegate (HAND_TRACKING_BROWSER_DELEGATE),
  cache the landmarker instance, and expose releaseBrowserHandLandmarker
- useBrowserHandTracking / useRemoteHandTracking: idempotent cleanup
  guarded by a cleanedUp flag, try/catch around the detect loop, and
  release of the landmarker on stop
- World: mount HandTrackingGlove only when the matching hand is
  actually present in the snapshot (status connected + hands.length > 0)
- HandTrackingGlove: drop the eager useGLTF.preload that was running
  at startup whether or not hand tracking was used

Does not yet absorb the React StrictMode double-mount — that is the
follow-up commit.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Boullay
2026-06-02 16:48:39 +02:00
parent 864e075b42
commit d217c3376b
8 changed files with 156 additions and 35 deletions
@@ -7,12 +7,14 @@ import {
import {
convertBrowserHandResult,
getBrowserHandLandmarker,
releaseBrowserHandLandmarker,
} from "@/lib/handTracking/browserHandTracking";
import {
INITIAL_HAND_TRACKING_SNAPSHOT,
getCameraStreamWithTimeout,
} from "@/lib/handTracking/handTrackingSession";
import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking";
import { logger } from "@/utils/core/Logger";
interface UseBrowserHandTrackingOptions {
enabled: boolean;
@@ -34,8 +36,12 @@ export function useBrowserHandTracking({
}
let cancelled = false;
let cleanedUp = false;
const cleanup = (): void => {
if (cleanedUp) return;
cleanedUp = true;
if (intervalRef.current !== null) {
window.clearInterval(intervalRef.current);
intervalRef.current = null;
@@ -44,6 +50,7 @@ export function useBrowserHandTracking({
streamRef.current?.getTracks().forEach((track) => track.stop());
streamRef.current = null;
videoRef.current = null;
releaseBrowserHandLandmarker();
};
const start = async (): Promise<void> => {
@@ -111,24 +118,44 @@ export function useBrowserHandTracking({
intervalRef.current = window.setInterval(() => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
const result = handLandmarker.detectForVideo(
video,
performance.now(),
);
const hands = convertBrowserHandResult(result);
try {
const result = handLandmarker.detectForVideo(
video,
performance.now(),
);
const hands = convertBrowserHandResult(result);
setSnapshot((current) => ({
...current,
hands,
usageStatus: hands.some((hand) => hand.isFist)
? "active"
: "available",
error: null,
}));
setSnapshot((current) => ({
...current,
hands,
usageStatus: hands.some((hand) => hand.isFist)
? "active"
: "available",
error: null,
}));
} catch (error) {
logger.error("HandTracking", "Browser JS runtime error", {
error: error instanceof Error ? error.message : String(error),
});
cleanup();
setSnapshot({
hands: [],
status: "error",
usageStatus: "inactive",
serverStatus: "Browser JS",
error:
error instanceof Error
? error.message
: "Browser hand tracking failed",
});
}
}, 1_000 / HAND_TRACKING_TARGET_FPS);
} catch (error) {
if (cancelled) return;
logger.error("HandTracking", "Browser JS runtime failed", {
error: error instanceof Error ? error.message : String(error),
});
setSnapshot({
hands: [],
status: "error",