fix(handtracking): reduce GPU pressure on WebGL context loss
Several mitigations against the WebGL context lost that fires when hand tracking starts on a loaded scene: - Canvas: fixed DPR [1,1], antialias off, scoped id="game-canvas", context-lost handler releases MediaPipe and logs GPU memory counters - optimizeGLTFScene: cap anisotropy at 2 and stop forcing mipmaps / needsUpdate on every pass — avoids massive texture re-uploads - MediaPipe: force CPU delegate (HAND_TRACKING_BROWSER_DELEGATE), cache the landmarker instance, and expose releaseBrowserHandLandmarker - useBrowserHandTracking / useRemoteHandTracking: idempotent cleanup guarded by a cleanedUp flag, try/catch around the detect loop, and release of the landmarker on stop - World: mount HandTrackingGlove only when the matching hand is actually present in the snapshot (status connected + hands.length > 0) - HandTrackingGlove: drop the eager useGLTF.preload that was running at startup whether or not hand tracking was used Does not yet absorb the React StrictMode double-mount — that is the follow-up commit. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,12 +7,14 @@ import {
|
||||
import {
|
||||
convertBrowserHandResult,
|
||||
getBrowserHandLandmarker,
|
||||
releaseBrowserHandLandmarker,
|
||||
} from "@/lib/handTracking/browserHandTracking";
|
||||
import {
|
||||
INITIAL_HAND_TRACKING_SNAPSHOT,
|
||||
getCameraStreamWithTimeout,
|
||||
} from "@/lib/handTracking/handTrackingSession";
|
||||
import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking";
|
||||
import { logger } from "@/utils/core/Logger";
|
||||
|
||||
interface UseBrowserHandTrackingOptions {
|
||||
enabled: boolean;
|
||||
@@ -34,8 +36,12 @@ export function useBrowserHandTracking({
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
let cleanedUp = false;
|
||||
|
||||
const cleanup = (): void => {
|
||||
if (cleanedUp) return;
|
||||
cleanedUp = true;
|
||||
|
||||
if (intervalRef.current !== null) {
|
||||
window.clearInterval(intervalRef.current);
|
||||
intervalRef.current = null;
|
||||
@@ -44,6 +50,7 @@ export function useBrowserHandTracking({
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
videoRef.current = null;
|
||||
releaseBrowserHandLandmarker();
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
@@ -111,24 +118,44 @@ export function useBrowserHandTracking({
|
||||
intervalRef.current = window.setInterval(() => {
|
||||
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
|
||||
|
||||
const result = handLandmarker.detectForVideo(
|
||||
video,
|
||||
performance.now(),
|
||||
);
|
||||
const hands = convertBrowserHandResult(result);
|
||||
try {
|
||||
const result = handLandmarker.detectForVideo(
|
||||
video,
|
||||
performance.now(),
|
||||
);
|
||||
const hands = convertBrowserHandResult(result);
|
||||
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands,
|
||||
usageStatus: hands.some((hand) => hand.isFist)
|
||||
? "active"
|
||||
: "available",
|
||||
error: null,
|
||||
}));
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands,
|
||||
usageStatus: hands.some((hand) => hand.isFist)
|
||||
? "active"
|
||||
: "available",
|
||||
error: null,
|
||||
}));
|
||||
} catch (error) {
|
||||
logger.error("HandTracking", "Browser JS runtime error", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
cleanup();
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "error",
|
||||
usageStatus: "inactive",
|
||||
serverStatus: "Browser JS",
|
||||
error:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Browser hand tracking failed",
|
||||
});
|
||||
}
|
||||
}, 1_000 / HAND_TRACKING_TARGET_FPS);
|
||||
} catch (error) {
|
||||
if (cancelled) return;
|
||||
|
||||
logger.error("HandTracking", "Browser JS runtime failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "error",
|
||||
|
||||
Reference in New Issue
Block a user