diff --git a/src/components/three/handTracking/HandTrackingGlove.tsx b/src/components/three/handTracking/HandTrackingGlove.tsx index 37e4518..d5c8904 100644 --- a/src/components/three/handTracking/HandTrackingGlove.tsx +++ b/src/components/three/handTracking/HandTrackingGlove.tsx @@ -1,7 +1,6 @@ import type { ReactNode } from "react"; import { Component, useEffect, useMemo, useRef } from "react"; import { useFrame, useThree } from "@react-three/fiber"; -import { useGLTF } from "@react-three/drei"; import * as THREE from "three"; import { SkeletonUtils } from "three-stdlib"; import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot"; @@ -362,6 +361,3 @@ export function HandTrackingGlove({ ); } - -useGLTF.preload(GLOVE_CONFIGS.left.modelPath); -useGLTF.preload(GLOVE_CONFIGS.right.modelPath); diff --git a/src/data/handTrackingConfig.ts b/src/data/handTrackingConfig.ts index 0b8c773..6027d12 100644 --- a/src/data/handTrackingConfig.ts +++ b/src/data/handTrackingConfig.ts @@ -8,3 +8,4 @@ export const HAND_TRACKING_BROWSER_WASM_URL = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.35/wasm"; export const HAND_TRACKING_BROWSER_MODEL_URL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"; +export const HAND_TRACKING_BROWSER_DELEGATE: "CPU" | "GPU" = "CPU"; diff --git a/src/hooks/handTracking/useBrowserHandTracking.ts b/src/hooks/handTracking/useBrowserHandTracking.ts index 73064fa..7310f99 100644 --- a/src/hooks/handTracking/useBrowserHandTracking.ts +++ b/src/hooks/handTracking/useBrowserHandTracking.ts @@ -7,12 +7,14 @@ import { import { convertBrowserHandResult, getBrowserHandLandmarker, + releaseBrowserHandLandmarker, } from "@/lib/handTracking/browserHandTracking"; import { INITIAL_HAND_TRACKING_SNAPSHOT, getCameraStreamWithTimeout, } from "@/lib/handTracking/handTrackingSession"; import type { HandTrackingSnapshot } from "@/types/handTracking/handTracking"; +import { logger } from "@/utils/core/Logger"; interface UseBrowserHandTrackingOptions { enabled: boolean; @@ -34,8 +36,12 @@ export function useBrowserHandTracking({ } let cancelled = false; + let cleanedUp = false; const cleanup = (): void => { + if (cleanedUp) return; + cleanedUp = true; + if (intervalRef.current !== null) { window.clearInterval(intervalRef.current); intervalRef.current = null; @@ -44,6 +50,7 @@ export function useBrowserHandTracking({ streamRef.current?.getTracks().forEach((track) => track.stop()); streamRef.current = null; videoRef.current = null; + releaseBrowserHandLandmarker(); }; const start = async (): Promise => { @@ -111,24 +118,44 @@ export function useBrowserHandTracking({ intervalRef.current = window.setInterval(() => { if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return; - const result = handLandmarker.detectForVideo( - video, - performance.now(), - ); - const hands = convertBrowserHandResult(result); + try { + const result = handLandmarker.detectForVideo( + video, + performance.now(), + ); + const hands = convertBrowserHandResult(result); - setSnapshot((current) => ({ - ...current, - hands, - usageStatus: hands.some((hand) => hand.isFist) - ? "active" - : "available", - error: null, - })); + setSnapshot((current) => ({ + ...current, + hands, + usageStatus: hands.some((hand) => hand.isFist) + ? "active" + : "available", + error: null, + })); + } catch (error) { + logger.error("HandTracking", "Browser JS runtime error", { + error: error instanceof Error ? error.message : String(error), + }); + cleanup(); + setSnapshot({ + hands: [], + status: "error", + usageStatus: "inactive", + serverStatus: "Browser JS", + error: + error instanceof Error + ? error.message + : "Browser hand tracking failed", + }); + } }, 1_000 / HAND_TRACKING_TARGET_FPS); } catch (error) { if (cancelled) return; + logger.error("HandTracking", "Browser JS runtime failed", { + error: error instanceof Error ? error.message : String(error), + }); setSnapshot({ hands: [], status: "error", diff --git a/src/hooks/handTracking/useRemoteHandTracking.ts b/src/hooks/handTracking/useRemoteHandTracking.ts index f53236d..e9dc1db 100644 --- a/src/hooks/handTracking/useRemoteHandTracking.ts +++ b/src/hooks/handTracking/useRemoteHandTracking.ts @@ -17,6 +17,7 @@ import type { HandTrackingServerMessage, HandTrackingSnapshot, } from "@/types/handTracking/handTracking"; +import { logger } from "@/utils/core/Logger"; interface UseRemoteHandTrackingOptions { enabled: boolean; @@ -100,6 +101,7 @@ export function useRemoteHandTracking({ } let cancelled = false; + let cleanedUp = false; const clearResponseTimeout = (): void => { if (responseTimeoutRef.current === null) return; @@ -108,6 +110,9 @@ export function useRemoteHandTracking({ }; const cleanup = (): void => { + if (cleanedUp) return; + cleanedUp = true; + if (sendIntervalRef.current !== null) { window.clearInterval(sendIntervalRef.current); sendIntervalRef.current = null; @@ -283,6 +288,9 @@ export function useRemoteHandTracking({ }; ws.onerror = () => { markResponseReceived(); + logger.error("HandTracking", "Backend WebSocket error", { + websocketUrl, + }); setSnapshot((current) => ({ ...current, status: "error", @@ -307,6 +315,10 @@ export function useRemoteHandTracking({ ); } catch (error) { if (cancelled) return; + logger.error("HandTracking", "Backend runtime failed", { + error: error instanceof Error ? error.message : String(error), + websocketUrl, + }); setSnapshot({ hands: [], status: "error", diff --git a/src/lib/handTracking/browserHandTracking.ts b/src/lib/handTracking/browserHandTracking.ts index 06b80b4..5daf76e 100644 --- a/src/lib/handTracking/browserHandTracking.ts +++ b/src/lib/handTracking/browserHandTracking.ts @@ -1,4 +1,5 @@ import { + HAND_TRACKING_BROWSER_DELEGATE, HAND_TRACKING_BROWSER_MODEL_URL, HAND_TRACKING_BROWSER_WASM_URL, } from "@/data/handTrackingConfig"; @@ -6,6 +7,7 @@ import type { HandTrackingHand, HandTrackingLandmark, } from "@/types/handTracking/handTracking"; +import { logger } from "@/utils/core/Logger"; type HandLandmarkerModule = typeof import("@mediapipe/tasks-vision"); type HandLandmarker = Awaited< @@ -14,6 +16,7 @@ type HandLandmarker = Awaited< type HandLandmarkerResult = ReturnType; let handLandmarkerPromise: Promise | null = null; +let handLandmarkerInstance: HandLandmarker | null = null; function averageLandmarks( landmarks: HandTrackingLandmark[], @@ -78,20 +81,46 @@ export async function getBrowserHandLandmarker(): Promise { HAND_TRACKING_BROWSER_WASM_URL, ); - return HandLandmarker.createFromOptions(vision, { + const handLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: HAND_TRACKING_BROWSER_MODEL_URL, - delegate: "GPU", + delegate: HAND_TRACKING_BROWSER_DELEGATE, }, numHands: 2, runningMode: "VIDEO", }); + + handLandmarkerInstance = handLandmarker; + return handLandmarker; }, ); return handLandmarkerPromise; } +export function releaseBrowserHandLandmarker(): void { + const activeLandmarker = handLandmarkerInstance; + const pendingLandmarker = handLandmarkerPromise; + + handLandmarkerInstance = null; + handLandmarkerPromise = null; + + if (activeLandmarker) { + activeLandmarker.close(); + return; + } + + void pendingLandmarker + ?.then((landmarker) => { + landmarker.close(); + }) + .catch((error: unknown) => { + logger.warn("HandTracking", "Browser JS landmarker release failed", { + error: error instanceof Error ? error.message : String(error), + }); + }); +} + export function convertBrowserHandResult( result: HandLandmarkerResult, ): HandTrackingHand[] { diff --git a/src/pages/page.tsx b/src/pages/page.tsx index 996a285..d6b42a5 100644 --- a/src/pages/page.tsx +++ b/src/pages/page.tsx @@ -15,7 +15,9 @@ import { } from "@/components/ui/intro"; import { SceneLoadingOverlay } from "@/components/ui/SceneLoadingOverlay"; import { INITIAL_SCENE_LOADING_STATE } from "@/data/world/sceneLoadingConfig"; +import { useDebugStore } from "@/hooks/debug/useDebugStore"; import { useTransientLoadingIndicator } from "@/hooks/ui/useTransientLoadingIndicator"; +import { releaseBrowserHandLandmarker } from "@/lib/handTracking/browserHandTracking"; import { AudioManager } from "@/managers/AudioManager"; import { useGameStore } from "@/managers/stores/useGameStore"; import { useWorldSettingsStore } from "@/managers/stores/useWorldSettingsStore"; @@ -26,6 +28,9 @@ import { logger } from "@/utils/core/Logger"; import { World } from "@/world/World"; const LOADING_TO_VIDEO_FADE_MS = 500; +const WEBGL_CONTEXT_RESTORE_DELAY_MS = 500; +const CANVAS_DPR: [number, number] = [1, 1]; +const registeredWebglContextCanvases = new WeakSet(); export function HomePage(): React.JSX.Element | null { const navigate = useNavigate(); @@ -38,6 +43,11 @@ export function HomePage(): React.JSX.Element | null { const graphicsPreset = useWorldSettingsStore( (state) => state.graphics.preset, ); + const cameraMode = useDebugStore((debug) => debug.getCameraMode()); + const handTrackingSource = useDebugStore((debug) => + debug.getHandTrackingSource(), + ); + const sceneMode = useDebugStore((debug) => debug.getSceneMode()); const dialogMessage = useGameStore( (state) => state.missionFlow.dialogMessage, ); @@ -48,9 +58,18 @@ export function HomePage(): React.JSX.Element | null { INITIAL_SCENE_LOADING_STATE, ); const sceneReadyRef = useRef(false); + const cameraModeRef = useRef(cameraMode); + const handTrackingSourceRef = useRef(handTrackingSource); + const sceneModeRef = useRef(sceneMode); const runtimeLoadingSignal = `${graphicsPreset}:${mainState}:${ebikeStep}:${pylonStep}:${farmStep}`; const previousRuntimeLoadingSignalRef = useRef(runtimeLoadingSignal); + useEffect(() => { + cameraModeRef.current = cameraMode; + handTrackingSourceRef.current = handTrackingSource; + sceneModeRef.current = sceneMode; + }, [cameraMode, handTrackingSource, sceneMode]); + useEffect(() => { sceneReadyRef.current = sceneLoadingState.status === "ready"; }, [sceneLoadingState.status]); @@ -138,11 +157,25 @@ export function HomePage(): React.JSX.Element | null { // page stays frozen on a black canvas until the user reloads. const loseContextExt = gl.getContext().getExtension("WEBGL_lose_context"); + if (registeredWebglContextCanvases.has(canvas)) return; + registeredWebglContextCanvases.add(canvas); + const handleContextLost = (event: Event) => { event.preventDefault(); - logger.error("WebGL", "Context lost - attempting auto-restore"); + releaseBrowserHandLandmarker(); + + logger.error("WebGL", "Context lost - attempting auto-restore", { + cameraMode: cameraModeRef.current, + geometries: gl.info.memory.geometries, + handTrackingSource: handTrackingSourceRef.current, + sceneMode: sceneModeRef.current, + textures: gl.info.memory.textures, + }); // Give the GPU a moment to free resources before asking it back. - window.setTimeout(() => loseContextExt?.restoreContext(), 500); + window.setTimeout( + () => loseContextExt?.restoreContext(), + WEBGL_CONTEXT_RESTORE_DELAY_MS, + ); }; const handleContextRestored = () => { @@ -150,7 +183,11 @@ export function HomePage(): React.JSX.Element | null { gl.shadowMap.type = THREE.PCFShadowMap; gl.shadowMap.autoUpdate = true; gl.shadowMap.needsUpdate = true; - logger.info("WebGL", "Context restored"); + logger.info("WebGL", "Context restored", { + cameraMode: cameraModeRef.current, + handTrackingSource: handTrackingSourceRef.current, + sceneMode: sceneModeRef.current, + }); }; canvas.addEventListener("webglcontextlost", handleContextLost); @@ -191,10 +228,12 @@ export function HomePage(): React.JSX.Element | null { >; const optimizedTextures = new WeakSet(); +const MAX_GLTF_TEXTURE_ANISOTROPY = 2; function optimizeTexture(texture: THREE.Texture, maxAnisotropy: number): void { if (optimizedTextures.has(texture)) return; optimizedTextures.add(texture); - texture.anisotropy = Math.min(4, Math.max(1, maxAnisotropy)); + const nextAnisotropy = Math.min( + MAX_GLTF_TEXTURE_ANISOTROPY, + Math.max(1, maxAnisotropy), + ); - if (!(texture instanceof THREE.CompressedTexture)) { - texture.generateMipmaps = true; - texture.minFilter = THREE.LinearMipmapLinearFilter; - texture.magFilter = THREE.LinearFilter; + if (texture.anisotropy > nextAnisotropy) { + texture.anisotropy = nextAnisotropy; + texture.needsUpdate = true; } - - texture.needsUpdate = true; } function optimizeMaterialTextures( diff --git a/src/world/World.tsx b/src/world/World.tsx index 0b45210..1c14e31 100644 --- a/src/world/World.tsx +++ b/src/world/World.tsx @@ -31,11 +31,20 @@ import { CharacterSystem } from "@/world/characters/CharacterSystem"; import { Player } from "@/world/player/Player"; import { TestMap } from "@/world/debug/TestMap"; import type { SceneLoadingChangeHandler } from "@/types/world/sceneLoading"; +import type { HandTrackingGloveHandedness } from "@/hooks/handTracking/useHandTrackingGloveStatus"; +import type { HandTrackingHand } from "@/types/handTracking/handTracking"; interface WorldProps { onLoadingStateChange?: SceneLoadingChangeHandler | undefined; } +function hasTrackedHand( + hands: HandTrackingHand[], + handedness: HandTrackingGloveHandedness, +): boolean { + return hands.some((hand) => hand.handedness.toLowerCase() === handedness); +} + export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element { useEnvironmentDebug(); useMapPerformanceDebug(); @@ -49,7 +58,7 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element { (state) => state.showPlayerModel, ); const showDebugOctree = useDebugVisualsStore((state) => state.showOctree); - const { status, usageStatus } = useHandTrackingSnapshot(); + const { hands, status, usageStatus } = useHandTrackingSnapshot(); const { octree, gameplayReady, @@ -63,8 +72,11 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element { ? PLAYER_SPAWN_POSITION_GAME : PLAYER_SPAWN_POSITION_PHYSICS; const showHandTrackingGloves = - sceneMode === "physics" || - (status !== "idle" && usageStatus !== "inactive"); + status === "connected" && usageStatus !== "inactive" && hands.length > 0; + const showLeftHandTrackingGlove = + showHandTrackingGloves && hasTrackedHand(hands, "left"); + const showRightHandTrackingGlove = + showHandTrackingGloves && hasTrackedHand(hands, "right"); const spawnPlayer = cameraMode !== "debug" && (sceneMode === "game" ? gameplayReady : octree !== null); @@ -82,8 +94,12 @@ export function World({ onLoadingStateChange }: WorldProps): React.JSX.Element { ) : null} {showHandTrackingGloves ? ( - - + {showLeftHandTrackingGlove ? ( + + ) : null} + {showRightHandTrackingGlove ? ( + + ) : null} ) : null} {cameraMode === "debug" ? : null}