math-pixel/La-Fabrik
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(handtracking): reduce GPU pressure on WebGL context loss

Browse Source 
Several mitigations against the WebGL context lost that fires when
hand tracking starts on a loaded scene:

- Canvas: fixed DPR [1,1], antialias off, scoped id="game-canvas",
  context-lost handler releases MediaPipe and logs GPU memory counters
- optimizeGLTFScene: cap anisotropy at 2 and stop forcing mipmaps /
  needsUpdate on every pass — avoids massive texture re-uploads
- MediaPipe: force CPU delegate (HAND_TRACKING_BROWSER_DELEGATE),
  cache the landmarker instance, and expose releaseBrowserHandLandmarker
- useBrowserHandTracking / useRemoteHandTracking: idempotent cleanup
  guarded by a cleanedUp flag, try/catch around the detect loop, and
  release of the landmarker on stop
- World: mount HandTrackingGlove only when the matching hand is
  actually present in the snapshot (status connected + hands.length > 0)
- HandTrackingGlove: drop the eager useGLTF.preload that was running
  at startup whether or not hand tracking was used

Does not yet absorb the React StrictMode double-mount — that is the
follow-up commit.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Boullay
2026-06-02 16:48:39 +02:00
parent 864e075b42
commit d217c3376b
8 changed files with 156 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/three/handTracking/HandTrackingGlove.tsx
-4
View File
@@ -1,7 +1,6 @@
import type { ReactNode } from "react";
import { Component, useEffect, useMemo, useRef } from "react";
import { useFrame, useThree } from "@react-three/fiber";
import { useGLTF } from "@react-three/drei";
import * as THREE from "three";
import { SkeletonUtils } from "three-stdlib";
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
@@ -362,6 +361,3 @@ export function HandTrackingGlove({
</HandTrackingGloveErrorBoundary>
);
}
useGLTF.preload(GLOVE_CONFIGS.left.modelPath);
useGLTF.preload(GLOVE_CONFIGS.right.modelPath);
src/data/handTrackingConfig.ts
+1
View File
@@ -8,3 +8,4 @@ export const HAND_TRACKING_BROWSER_WASM_URL =
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm";
export const HAND_TRACKING_BROWSER_MODEL_URL =
"https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task";
export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "CPU";
src/hooks/handTracking/useBrowserHandTracking.ts
+40 -13
View File
@@ -7,12 +7,14 @@ import {
import {
convertBrowserHandResult,
getBrowserHandLandmarker,
releaseBrowserHandLandmarker,
} from "@/lib/handTracking/browserHandTracking";
import {
INITIAL_HAND_TRACKING_SNAPSHOT,
getCameraStreamWithTimeout,
} from "@/lib/handTracking/handTrackingSession";
import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking";
import { logger } from "@/utils/core/Logger";
interface UseBrowserHandTrackingOptions {
enabled: boolean;
@@ -34,8 +36,12 @@ export function useBrowserHandTracking({
}
let cancelled = false;
let cleanedUp = false;
const cleanup = (): void => {
if (cleanedUp) return;
cleanedUp = true;
if (intervalRef.current !== null) {
window.clearInterval(intervalRef.current);
intervalRef.current = null;
@@ -44,6 +50,7 @@ export function useBrowserHandTracking({
streamRef.current?.getTracks().forEach((track) => track.stop());
streamRef.current = null;
videoRef.current = null;
releaseBrowserHandLandmarker();
};
const start = async (): Promise<void> => {
@@ -111,24 +118,44 @@ export function useBrowserHandTracking({
intervalRef.current = window.setInterval(() => {
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
const result = handLandmarker.detectForVideo(
video,
performance.now(),
);
const hands = convertBrowserHandResult(result);
try {
const result = handLandmarker.detectForVideo(
video,
performance.now(),
);
const hands = convertBrowserHandResult(result);
setSnapshot((current) => ({
...current,
hands,
usageStatus: hands.some((hand) => hand.isFist)
? "active"
: "available",
error: null,
}));
setSnapshot((current) => ({
...current,
hands,
usageStatus: hands.some((hand) => hand.isFist)
? "active"
: "available",
error: null,
}));
} catch (error) {
logger.error("HandTracking", "Browser JS runtime error", {
error: error instanceof Error ? error.message : String(error),
});
cleanup();
setSnapshot({
hands: [],
status: "error",
usageStatus: "inactive",
serverStatus: "Browser JS",
error:
error instanceof Error
? error.message
: "Browser hand tracking failed",
});
}
}, 1_000 / HAND_TRACKING_TARGET_FPS);
} catch (error) {
if (cancelled) return;
logger.error("HandTracking", "Browser JS runtime failed", {
error: error instanceof Error ? error.message : String(error),
});
setSnapshot({
hands: [],
status: "error",
src/hooks/handTracking/useRemoteHandTracking.ts
+12
View File
@@ -17,6 +17,7 @@ import type {
HandTrackingServerMessage,
HandTrackingSnapshot,
} from "@/types/handTracking/handTracking";
import { logger } from "@/utils/core/Logger";
interface UseRemoteHandTrackingOptions {
enabled: boolean;
@@ -100,6 +101,7 @@ export function useRemoteHandTracking({
}
let cancelled = false;
let cleanedUp = false;
const clearResponseTimeout = (): void => {
if (responseTimeoutRef.current === null) return;
@@ -108,6 +110,9 @@ export function useRemoteHandTracking({
};
const cleanup = (): void => {
if (cleanedUp) return;
cleanedUp = true;
if (sendIntervalRef.current !== null) {
window.clearInterval(sendIntervalRef.current);
sendIntervalRef.current = null;
@@ -283,6 +288,9 @@ export function useRemoteHandTracking({
};
ws.onerror = () => {
markResponseReceived();
logger.error("HandTracking", "Backend WebSocket error", {
websocketUrl,
});
setSnapshot((current) => ({
...current,
status: "error",
@@ -307,6 +315,10 @@ export function useRemoteHandTracking({
);
} catch (error) {
if (cancelled) return;
logger.error("HandTracking", "Backend runtime failed", {
error: error instanceof Error ? error.message : String(error),
websocketUrl,
});
setSnapshot({
hands: [],
status: "error",
src/lib/handTracking/browserHandTracking.ts
+31 -2
View File
@@ -1,4 +1,5 @@
import {
HAND_TRACKING_BROWSER_DELEGATE,
HAND_TRACKING_BROWSER_MODEL_URL,
HAND_TRACKING_BROWSER_WASM_URL,
} from "@/data/handTrackingConfig";
@@ -6,6 +7,7 @@ import type {
HandTrackingHand,
HandTrackingLandmark,
} from "@/types/handTracking/handTracking";
import { logger } from "@/utils/core/Logger";
type HandLandmarkerModule = typeof import("@mediapipe/tasks-vision");
type HandLandmarker = Awaited<
@@ -14,6 +16,7 @@ type HandLandmarker = Awaited<
type HandLandmarkerResult = ReturnType<HandLandmarker["detectForVideo"]>;
let handLandmarkerPromise: Promise<HandLandmarker> | null = null;
let handLandmarkerInstance: HandLandmarker | null = null;
function averageLandmarks(
landmarks: HandTrackingLandmark[],
@@ -78,20 +81,46 @@ export async function getBrowserHandLandmarker(): Promise<HandLandmarker> {
HAND_TRACKING_BROWSER_WASM_URL,
);
return HandLandmarker.createFromOptions(vision, {
const handLandmarker = await HandLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: HAND_TRACKING_BROWSER_MODEL_URL,
delegate: "GPU",
delegate: HAND_TRACKING_BROWSER_DELEGATE,
},
numHands: 2,
runningMode: "VIDEO",
});
handLandmarkerInstance = handLandmarker;
return handLandmarker;
},
);
return handLandmarkerPromise;
}
export function releaseBrowserHandLandmarker(): void {
const activeLandmarker = handLandmarkerInstance;
const pendingLandmarker = handLandmarkerPromise;
handLandmarkerInstance = null;
handLandmarkerPromise = null;
if (activeLandmarker) {
activeLandmarker.close();
return;
}
void pendingLandmarker
?.then((landmarker) => {
landmarker.close();
})
.catch((error: unknown) => {
logger.warn("HandTracking", "Browser JS landmarker release failed", {
error: error instanceof Error ? error.message : String(error),
});
});
}
export function convertBrowserHandResult(
result: HandLandmarkerResult,
): HandTrackingHand[] {
src/pages/page.tsx
+43 -4
View File
@@ -15,7 +15,9 @@ import {
} from "@/components/ui/intro";
import { SceneLoadingOverlay } from "@/components/ui/SceneLoadingOverlay";
import { INITIAL_SCENE_LOADING_STATE } from "@/data/world/sceneLoadingConfig";
import { useDebugStore } from "@/hooks/debug/useDebugStore";
import { useTransientLoadingIndicator } from "@/hooks/ui/useTransientLoadingIndicator";
import { releaseBrowserHandLandmarker } from "@/lib/handTracking/browserHandTracking";
import { AudioManager } from "@/managers/AudioManager";
import { useGameStore } from "@/managers/stores/useGameStore";
import { useWorldSettingsStore } from "@/managers/stores/useWorldSettingsStore";
@@ -26,6 +28,9 @@ import { logger } from "@/utils/core/Logger";
import { World } from "@/world/World";
const LOADING_TO_VIDEO_FADE_MS = 500;
const WEBGL_CONTEXT_RESTORE_DELAY_MS = 500;
const CANVAS_DPR: [number, number] = [1, 1];
const registeredWebglContextCanvases = new WeakSet<HTMLCanvasElement>();
export function HomePage(): React.JSX.Element | null {
const navigate = useNavigate();
@@ -38,6 +43,11 @@ export function HomePage(): React.JSX.Element | null {
const graphicsPreset = useWorldSettingsStore(
(state) => state.graphics.preset,
);
const cameraMode = useDebugStore((debug) => debug.getCameraMode());
const handTrackingSource = useDebugStore((debug) =>
debug.getHandTrackingSource(),
);
const sceneMode = useDebugStore((debug) => debug.getSceneMode());
const dialogMessage = useGameStore(
(state) => state.missionFlow.dialogMessage,
);
@@ -48,9 +58,18 @@ export function HomePage(): React.JSX.Element | null {
INITIAL_SCENE_LOADING_STATE,
);
const sceneReadyRef = useRef(false);
const cameraModeRef = useRef(cameraMode);
const handTrackingSourceRef = useRef(handTrackingSource);
const sceneModeRef = useRef(sceneMode);
const runtimeLoadingSignal = `${graphicsPreset}:${mainState}:${ebikeStep}:${pylonStep}:${farmStep}`;
const previousRuntimeLoadingSignalRef = useRef(runtimeLoadingSignal);
useEffect(() => {
cameraModeRef.current = cameraMode;
handTrackingSourceRef.current = handTrackingSource;
sceneModeRef.current = sceneMode;
}, [cameraMode, handTrackingSource, sceneMode]);
useEffect(() => {
sceneReadyRef.current = sceneLoadingState.status === "ready";
}, [sceneLoadingState.status]);
@@ -138,11 +157,25 @@ export function HomePage(): React.JSX.Element | null {
// page stays frozen on a black canvas until the user reloads.
const loseContextExt = gl.getContext().getExtension("WEBGL_lose_context");
if (registeredWebglContextCanvases.has(canvas)) return;
registeredWebglContextCanvases.add(canvas);
const handleContextLost = (event: Event) => {
event.preventDefault();
logger.error("WebGL", "Context lost - attempting auto-restore");
releaseBrowserHandLandmarker();
logger.error("WebGL", "Context lost - attempting auto-restore", {
cameraMode: cameraModeRef.current,
geometries: gl.info.memory.geometries,
handTrackingSource: handTrackingSourceRef.current,
sceneMode: sceneModeRef.current,
textures: gl.info.memory.textures,
});
// Give the GPU a moment to free resources before asking it back.
window.setTimeout(() => loseContextExt?.restoreContext(), 500);
window.setTimeout(
() => loseContextExt?.restoreContext(),
WEBGL_CONTEXT_RESTORE_DELAY_MS,
);
};
const handleContextRestored = () => {
@@ -150,7 +183,11 @@ export function HomePage(): React.JSX.Element | null {
gl.shadowMap.type = THREE.PCFShadowMap;
gl.shadowMap.autoUpdate = true;
gl.shadowMap.needsUpdate = true;
logger.info("WebGL", "Context restored");
logger.info("WebGL", "Context restored", {
cameraMode: cameraModeRef.current,
handTrackingSource: handTrackingSourceRef.current,
sceneMode: sceneModeRef.current,
});
};
canvas.addEventListener("webglcontextlost", handleContextLost);
@@ -191,10 +228,12 @@ export function HomePage(): React.JSX.Element | null {
<HandTrackingProvider>
<Canvas
camera={{ position: [85, 60, 85], fov: 42 }}
dpr={CANVAS_DPR}
id="game-canvas"
shadows={{ type: THREE.PCFShadowMap }}
gl={{
powerPreference: "high-performance",
antialias: true,
antialias: false,
stencil: false,
}}
onCreated={handleCanvasCreated}
src/utils/three/optimizeGLTFScene.ts
+8 -7
View File
@@ -19,20 +19,21 @@ type TexturedMaterial = THREE.Material &
Partial<Record<TextureKey, THREE.Texture>>;
const optimizedTextures = new WeakSet<THREE.Texture>();
const MAX_GLTF_TEXTURE_ANISOTROPY = 2;
function optimizeTexture(texture: THREE.Texture, maxAnisotropy: number): void {
if (optimizedTextures.has(texture)) return;
optimizedTextures.add(texture);
texture.anisotropy = Math.min(4, Math.max(1, maxAnisotropy));
const nextAnisotropy = Math.min(
MAX_GLTF_TEXTURE_ANISOTROPY,
Math.max(1, maxAnisotropy),
);
if (!(texture instanceof THREE.CompressedTexture)) {
texture.generateMipmaps = true;
texture.minFilter = THREE.LinearMipmapLinearFilter;
texture.magFilter = THREE.LinearFilter;
if (texture.anisotropy > nextAnisotropy) {
texture.anisotropy = nextAnisotropy;
texture.needsUpdate = true;
}
texture.needsUpdate = true;
}
function optimizeMaterialTextures(
src/world/World.tsx
+21 -5
View File
@@ -31,11 +31,20 @@ import { CharacterSystem } from "@/world/characters/CharacterSystem";
import { Player } from "@/world/player/Player";
import { TestMap } from "@/world/debug/TestMap";
import type { SceneLoadingChangeHandler } from "@/types/world/sceneLoading";
import type { HandTrackingGloveHandedness } from "@/hooks/handTracking/useHandTrackingGloveStatus";
import type { HandTrackingHand } from "@/types/handTracking/handTracking";
interface WorldProps {
onLoadingStateChange?: SceneLoadingChangeHandler | undefined;
}
function hasTrackedHand(
hands: HandTrackingHand[],
handedness: HandTrackingGloveHandedness,
): boolean {
return hands.some((hand) => hand.handedness.toLowerCase() === handedness);
}
export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element {
useEnvironmentDebug();
useMapPerformanceDebug();
@@ -49,7 +58,7 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element {
(state) => state.showPlayerModel,
);
const showDebugOctree = useDebugVisualsStore((state) => state.showOctree);
const { status, usageStatus } = useHandTrackingSnapshot();
const { hands, status, usageStatus } = useHandTrackingSnapshot();
const {
octree,
gameplayReady,
@@ -63,8 +72,11 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element {
? PLAYER_SPAWN_POSITION_GAME
: PLAYER_SPAWN_POSITION_PHYSICS;
const showHandTrackingGloves =
sceneMode === "physics" ||
(status !== "idle" && usageStatus !== "inactive");
status === "connected" && usageStatus !== "inactive" && hands.length > 0;
const showLeftHandTrackingGlove =
showHandTrackingGloves && hasTrackedHand(hands, "left");
const showRightHandTrackingGlove =
showHandTrackingGloves && hasTrackedHand(hands, "right");
const spawnPlayer =
cameraMode !== "debug" &&
(sceneMode === "game" ? gameplayReady : octree !== null);
@@ -82,8 +94,12 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element {
) : null}
{showHandTrackingGloves ? (
<Suspense fallback={null}>
<HandTrackingGlove handedness="left" />
<HandTrackingGlove handedness="right" />
{showLeftHandTrackingGlove ? (
<HandTrackingGlove handedness="left" />
) : null}
{showRightHandTrackingGlove ? (
<HandTrackingGlove handedness="right" />
) : null}
</Suspense>
) : null}
{cameraMode === "debug" ? <DebugCameraControls /> : null}