fix(handtracking): reduce GPU pressure on WebGL context loss

Several mitigations against the WebGL context lost that fires when
hand tracking starts on a loaded scene:

- Canvas: fixed DPR [1,1], antialias off, scoped id="game-canvas",
  context-lost handler releases MediaPipe and logs GPU memory counters
- optimizeGLTFScene: cap anisotropy at 2 and stop forcing mipmaps /
  needsUpdate on every pass — avoids massive texture re-uploads
- MediaPipe: force CPU delegate (HAND_TRACKING_BROWSER_DELEGATE),
  cache the landmarker instance, and expose releaseBrowserHandLandmarker
- useBrowserHandTracking / useRemoteHandTracking: idempotent cleanup
  guarded by a cleanedUp flag, try/catch around the detect loop, and
  release of the landmarker on stop
- World: mount HandTrackingGlove only when the matching hand is
  actually present in the snapshot (status connected + hands.length > 0)
- HandTrackingGlove: drop the eager useGLTF.preload that was running
  at startup whether or not hand tracking was used

Does not yet absorb the React StrictMode double-mount — that is the
follow-up commit.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Boullay
2026-06-02 16:48:39 +02:00
parent 864e075b42
commit d217c3376b
8 changed files with 156 additions and 35 deletions
+43 -4
View File
@@ -15,7 +15,9 @@ import {
} from "@/components/ui/intro";
import { SceneLoadingOverlay } from "@/components/ui/SceneLoadingOverlay";
import { INITIAL_SCENE_LOADING_STATE } from "@/data/world/sceneLoadingConfig";
import { useDebugStore } from "@/hooks/debug/useDebugStore";
import { useTransientLoadingIndicator } from "@/hooks/ui/useTransientLoadingIndicator";
import { releaseBrowserHandLandmarker } from "@/lib/handTracking/browserHandTracking";
import { AudioManager } from "@/managers/AudioManager";
import { useGameStore } from "@/managers/stores/useGameStore";
import { useWorldSettingsStore } from "@/managers/stores/useWorldSettingsStore";
@@ -26,6 +28,9 @@ import { logger } from "@/utils/core/Logger";
import { World } from "@/world/World";
const LOADING_TO_VIDEO_FADE_MS = 500;
const WEBGL_CONTEXT_RESTORE_DELAY_MS = 500;
const CANVAS_DPR: [number, number] = [1, 1];
const registeredWebglContextCanvases = new WeakSet<HTMLCanvasElement>();
export function HomePage(): React.JSX.Element | null {
const navigate = useNavigate();
@@ -38,6 +43,11 @@ export function HomePage(): React.JSX.Element | null {
const graphicsPreset = useWorldSettingsStore(
(state) => state.graphics.preset,
);
const cameraMode = useDebugStore((debug) => debug.getCameraMode());
const handTrackingSource = useDebugStore((debug) =>
debug.getHandTrackingSource(),
);
const sceneMode = useDebugStore((debug) => debug.getSceneMode());
const dialogMessage = useGameStore(
(state) => state.missionFlow.dialogMessage,
);
@@ -48,9 +58,18 @@ export function HomePage(): React.JSX.Element | null {
INITIAL_SCENE_LOADING_STATE,
);
const sceneReadyRef = useRef(false);
const cameraModeRef = useRef(cameraMode);
const handTrackingSourceRef = useRef(handTrackingSource);
const sceneModeRef = useRef(sceneMode);
const runtimeLoadingSignal = `${graphicsPreset}:${mainState}:${ebikeStep}:${pylonStep}:${farmStep}`;
const previousRuntimeLoadingSignalRef = useRef(runtimeLoadingSignal);
useEffect(() => {
cameraModeRef.current = cameraMode;
handTrackingSourceRef.current = handTrackingSource;
sceneModeRef.current = sceneMode;
}, [cameraMode, handTrackingSource, sceneMode]);
useEffect(() => {
sceneReadyRef.current = sceneLoadingState.status === "ready";
}, [sceneLoadingState.status]);
@@ -138,11 +157,25 @@ export function HomePage(): React.JSX.Element | null {
// page stays frozen on a black canvas until the user reloads.
const loseContextExt = gl.getContext().getExtension("WEBGL_lose_context");
if (registeredWebglContextCanvases.has(canvas)) return;
registeredWebglContextCanvases.add(canvas);
const handleContextLost = (event: Event) => {
event.preventDefault();
logger.error("WebGL", "Context lost - attempting auto-restore");
releaseBrowserHandLandmarker();
logger.error("WebGL", "Context lost - attempting auto-restore", {
cameraMode: cameraModeRef.current,
geometries: gl.info.memory.geometries,
handTrackingSource: handTrackingSourceRef.current,
sceneMode: sceneModeRef.current,
textures: gl.info.memory.textures,
});
// Give the GPU a moment to free resources before asking it back.
window.setTimeout(() => loseContextExt?.restoreContext(), 500);
window.setTimeout(
() => loseContextExt?.restoreContext(),
WEBGL_CONTEXT_RESTORE_DELAY_MS,
);
};
const handleContextRestored = () => {
@@ -150,7 +183,11 @@ export function HomePage(): React.JSX.Element | null {
gl.shadowMap.type = THREE.PCFShadowMap;
gl.shadowMap.autoUpdate = true;
gl.shadowMap.needsUpdate = true;
logger.info("WebGL", "Context restored");
logger.info("WebGL", "Context restored", {
cameraMode: cameraModeRef.current,
handTrackingSource: handTrackingSourceRef.current,
sceneMode: sceneModeRef.current,
});
};
canvas.addEventListener("webglcontextlost", handleContextLost);
@@ -191,10 +228,12 @@ export function HomePage(): React.JSX.Element | null {
<HandTrackingProvider>
<Canvas
camera={{ position: [85, 60, 85], fov: 42 }}
dpr={CANVAS_DPR}
id="game-canvas"
shadows={{ type: THREE.PCFShadowMap }}
gl={{
powerPreference: "high-performance",
antialias: true,
antialias: false,
stencil: false,
}}
onCreated={handleCanvasCreated}