feat move debug cube with remote hand tracking
This commit is contained in:
Tom Boullay
+5
-2
@@ -1,13 +1,15 @@
|
||||
import { Suspense } from "react";
|
||||
import { Canvas } from "@react-three/fiber";
|
||||
import { Crosshair } from "@/components/ui/Crosshair";
|
||||
import { HandTrackingOverlay } from "@/components/ui/HandTrackingOverlay";
|
||||
import { HandTrackingProvider } from "@/components/ui/HandTrackingProvider";
|
||||
import { InteractPrompt } from "@/components/ui/InteractPrompt";
|
||||
import { DebugPerf } from "@/utils/debug/DebugPerf";
|
||||
import { World } from "@/world/World";
|
||||
|
||||
function App(): React.JSX.Element {
|
||||
return (
|
||||
<>
|
||||
<HandTrackingProvider>
|
||||
<Canvas camera={{ position: [85, 60, 85], fov: 42 }} shadows>
|
||||
<Suspense fallback={null}>
|
||||
<World />
|
||||
@@ -16,7 +18,8 @@ function App(): React.JSX.Element {
|
||||
</Canvas>
|
||||
<Crosshair />
|
||||
<InteractPrompt />
|
||||
</>
|
||||
<HandTrackingOverlay />
|
||||
</HandTrackingProvider>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
GRAB_THROW_BOOST_STEP,
|
||||
} from "@/data/grabConfig";
|
||||
import { useDebugFolder } from "@/hooks/debug/useDebugFolder";
|
||||
import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
|
||||
import type { ColliderShape, Vector3Tuple } from "@/types/3d";
|
||||
|
||||
interface GrabbableObjectProps {
|
||||
@@ -28,6 +29,7 @@ interface GrabbableObjectProps {
|
||||
children: React.ReactNode;
|
||||
colliders?: ColliderShape;
|
||||
label?: string;
|
||||
handControlled?: boolean;
|
||||
}
|
||||
|
||||
// Shared params let one debug folder drive every instance.
|
||||
@@ -42,14 +44,18 @@ const ZERO_ANGULAR_VELOCITY = { x: 0, y: 0, z: 0 };
|
||||
const _holdTarget = new THREE.Vector3();
|
||||
const _currentPos = new THREE.Vector3();
|
||||
const _velocity = new THREE.Vector3();
|
||||
const _handNdc = new THREE.Vector3();
|
||||
const _handDirection = new THREE.Vector3();
|
||||
|
||||
export function GrabbableObject({
|
||||
position,
|
||||
children,
|
||||
colliders = GRAB_DEFAULT_COLLIDERS,
|
||||
label = GRAB_DEFAULT_LABEL,
|
||||
handControlled = false,
|
||||
}: GrabbableObjectProps): React.JSX.Element {
|
||||
const camera = useThree((state) => state.camera);
|
||||
const { hands } = useHandTrackingSnapshot();
|
||||
const rbRef = useRef<RapierRigidBody>(null);
|
||||
const isHolding = useRef(false);
|
||||
|
||||
@@ -84,10 +90,25 @@ export function GrabbableObject({
|
||||
});
|
||||
|
||||
useFrame(() => {
|
||||
if (!isHolding.current || !rbRef.current) return;
|
||||
if (!rbRef.current) return;
|
||||
|
||||
camera.getWorldDirection(_holdTarget);
|
||||
_holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
|
||||
const pinchingHand = handControlled
|
||||
? hands.find((hand) => hand.isPinch)
|
||||
: undefined;
|
||||
|
||||
if (!isHolding.current && !pinchingHand) return;
|
||||
|
||||
if (pinchingHand) {
|
||||
_handNdc.set((1 - pinchingHand.x) * 2 - 1, -pinchingHand.y * 2 + 1, 0.5);
|
||||
_handNdc.unproject(camera);
|
||||
_handDirection.subVectors(_handNdc, camera.position).normalize();
|
||||
_holdTarget
|
||||
.copy(camera.position)
|
||||
.addScaledVector(_handDirection, params.holdDistance);
|
||||
} else {
|
||||
camera.getWorldDirection(_holdTarget);
|
||||
_holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
|
||||
}
|
||||
|
||||
const t = rbRef.current.translation();
|
||||
_currentPos.set(t.x, t.y, t.z);
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
|
||||
|
||||
export function HandTrackingOverlay(): React.JSX.Element | null {
|
||||
const { hands, status, serverStatus, error } = useHandTrackingSnapshot();
|
||||
|
||||
if (status === "idle") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const pinching = hands.some((hand) => hand.isPinch);
|
||||
|
||||
return (
|
||||
<aside className="hand-tracking-overlay" aria-label="Hand tracking status">
|
||||
<strong>Hand tracking</strong>
|
||||
<span>Status: {status}</span>
|
||||
{serverStatus ? <span>Server: {serverStatus}</span> : null}
|
||||
<span>Hands: {hands.length}</span>
|
||||
<span>Pinch: {pinching ? "yes" : "no"}</span>
|
||||
{error ? (
|
||||
<span className="hand-tracking-overlay__error">{error}</span>
|
||||
) : null}
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { ReactNode } from "react";
|
||||
import { useSceneMode } from "@/hooks/debug/useSceneMode";
|
||||
import {
|
||||
HAND_TRACKING_IDLE_SNAPSHOT,
|
||||
HandTrackingContext,
|
||||
} from "@/hooks/useHandTrackingSnapshot";
|
||||
import { useRemoteHandTracking } from "@/hooks/useRemoteHandTracking";
|
||||
import { isDebugEnabled } from "@/utils/debug/isDebugEnabled";
|
||||
|
||||
export function HandTrackingProvider({
|
||||
children,
|
||||
}: {
|
||||
children: ReactNode;
|
||||
}): React.JSX.Element {
|
||||
const sceneMode = useSceneMode();
|
||||
const enabled = isDebugEnabled() && sceneMode === "physics";
|
||||
const snapshot = useRemoteHandTracking({ enabled });
|
||||
|
||||
return (
|
||||
<HandTrackingContext
|
||||
value={enabled ? snapshot : HAND_TRACKING_IDLE_SNAPSHOT}
|
||||
>
|
||||
{children}
|
||||
</HandTrackingContext>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
export const HAND_TRACKING_LOCAL_WS_URL = "ws://localhost:8000/ws";
|
||||
export const HAND_TRACKING_PROD_WS_URL = "wss://handtracking.la-fabrik.fr/ws";
|
||||
|
||||
export const HAND_TRACKING_FRAME_WIDTH = 320;
|
||||
export const HAND_TRACKING_FRAME_HEIGHT = 240;
|
||||
export const HAND_TRACKING_TARGET_FPS = 10;
|
||||
export const HAND_TRACKING_JPEG_QUALITY = 0.55;
|
||||
export const HAND_TRACKING_RESPONSE_TIMEOUT_MS = 1_500;
|
||||
|
||||
export function getHandTrackingWsUrl(): string {
|
||||
const configuredUrl = import.meta.env.VITE_HAND_TRACKING_WS_URL;
|
||||
|
||||
if (configuredUrl) {
|
||||
return configuredUrl;
|
||||
}
|
||||
|
||||
return import.meta.env.DEV
|
||||
? HAND_TRACKING_LOCAL_WS_URL
|
||||
: HAND_TRACKING_PROD_WS_URL;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { createContext, useContext } from "react";
|
||||
import type { HandTrackingSnapshot } from "@/types/handTracking";
|
||||
|
||||
export const HAND_TRACKING_IDLE_SNAPSHOT: HandTrackingSnapshot = {
|
||||
hands: [],
|
||||
status: "idle",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
};
|
||||
|
||||
export const HandTrackingContext = createContext<HandTrackingSnapshot>(
|
||||
HAND_TRACKING_IDLE_SNAPSHOT,
|
||||
);
|
||||
|
||||
export function useHandTrackingSnapshot(): HandTrackingSnapshot {
|
||||
return useContext(HandTrackingContext);
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
HAND_TRACKING_FRAME_HEIGHT,
|
||||
HAND_TRACKING_FRAME_WIDTH,
|
||||
HAND_TRACKING_JPEG_QUALITY,
|
||||
HAND_TRACKING_RESPONSE_TIMEOUT_MS,
|
||||
HAND_TRACKING_TARGET_FPS,
|
||||
getHandTrackingWsUrl,
|
||||
} from "@/data/handTrackingConfig";
|
||||
import type {
|
||||
HandTrackingFrameMessage,
|
||||
HandTrackingServerMessage,
|
||||
HandTrackingSnapshot,
|
||||
} from "@/types/handTracking";
|
||||
|
||||
interface UseRemoteHandTrackingOptions {
|
||||
enabled: boolean;
|
||||
websocketUrl?: string;
|
||||
}
|
||||
|
||||
const INITIAL_SNAPSHOT: HandTrackingSnapshot = {
|
||||
hands: [],
|
||||
status: "idle",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
};
|
||||
|
||||
function getBase64Payload(dataUrl: string): string {
|
||||
return dataUrl.slice(dataUrl.indexOf(",") + 1);
|
||||
}
|
||||
|
||||
export function useRemoteHandTracking({
|
||||
enabled,
|
||||
websocketUrl = getHandTrackingWsUrl(),
|
||||
}: UseRemoteHandTrackingOptions): HandTrackingSnapshot {
|
||||
const [snapshot, setSnapshot] =
|
||||
useState<HandTrackingSnapshot>(INITIAL_SNAPSHOT);
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const sendIntervalRef = useRef<number | null>(null);
|
||||
const responseTimeoutRef = useRef<number | null>(null);
|
||||
const waitingForResponseRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const clearResponseTimeout = (): void => {
|
||||
if (responseTimeoutRef.current === null) return;
|
||||
window.clearTimeout(responseTimeoutRef.current);
|
||||
responseTimeoutRef.current = null;
|
||||
};
|
||||
|
||||
const cleanup = (): void => {
|
||||
if (sendIntervalRef.current !== null) {
|
||||
window.clearInterval(sendIntervalRef.current);
|
||||
sendIntervalRef.current = null;
|
||||
}
|
||||
|
||||
clearResponseTimeout();
|
||||
waitingForResponseRef.current = false;
|
||||
wsRef.current?.close();
|
||||
wsRef.current = null;
|
||||
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
videoRef.current = null;
|
||||
canvasRef.current = null;
|
||||
};
|
||||
|
||||
const markResponseReceived = (): void => {
|
||||
waitingForResponseRef.current = false;
|
||||
clearResponseTimeout();
|
||||
};
|
||||
|
||||
const sendFrame = (): void => {
|
||||
const ws = wsRef.current;
|
||||
const video = videoRef.current;
|
||||
const canvas = canvasRef.current;
|
||||
const context = canvas?.getContext("2d");
|
||||
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
||||
if (!video || !canvas || !context) return;
|
||||
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
|
||||
if (waitingForResponseRef.current) return;
|
||||
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL(
|
||||
"image/jpeg",
|
||||
HAND_TRACKING_JPEG_QUALITY,
|
||||
);
|
||||
const message: HandTrackingFrameMessage = {
|
||||
type: "frame",
|
||||
timestamp: Date.now(),
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
image: getBase64Payload(dataUrl),
|
||||
};
|
||||
|
||||
waitingForResponseRef.current = true;
|
||||
ws.send(JSON.stringify(message));
|
||||
responseTimeoutRef.current = window.setTimeout(() => {
|
||||
waitingForResponseRef.current = false;
|
||||
responseTimeoutRef.current = null;
|
||||
}, HAND_TRACKING_RESPONSE_TIMEOUT_MS);
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
if (cancelled) return;
|
||||
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "connecting",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: {
|
||||
width: HAND_TRACKING_FRAME_WIDTH,
|
||||
height: HAND_TRACKING_FRAME_HEIGHT,
|
||||
facingMode: "user",
|
||||
},
|
||||
audio: false,
|
||||
});
|
||||
|
||||
if (cancelled) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.srcObject = stream;
|
||||
await video.play();
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = HAND_TRACKING_FRAME_WIDTH;
|
||||
canvas.height = HAND_TRACKING_FRAME_HEIGHT;
|
||||
|
||||
const ws = new WebSocket(websocketUrl);
|
||||
ws.onopen = () => {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "connected",
|
||||
error: null,
|
||||
}));
|
||||
};
|
||||
ws.onmessage = (event) => {
|
||||
markResponseReceived();
|
||||
const data = JSON.parse(event.data) as HandTrackingServerMessage;
|
||||
|
||||
if (data.type === "hands") {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands: data.hands,
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.type === "status") {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
serverStatus: data.status,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands: [],
|
||||
status: "error",
|
||||
error: data.message,
|
||||
}));
|
||||
};
|
||||
ws.onerror = () => {
|
||||
markResponseReceived();
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "error",
|
||||
error: "Hand tracking WebSocket error",
|
||||
}));
|
||||
};
|
||||
ws.onclose = () => {
|
||||
markResponseReceived();
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: cancelled ? "idle" : "disconnected",
|
||||
}));
|
||||
};
|
||||
|
||||
streamRef.current = stream;
|
||||
videoRef.current = video;
|
||||
canvasRef.current = canvas;
|
||||
wsRef.current = ws;
|
||||
sendIntervalRef.current = window.setInterval(
|
||||
sendFrame,
|
||||
1_000 / HAND_TRACKING_TARGET_FPS,
|
||||
);
|
||||
} catch (error) {
|
||||
if (cancelled) return;
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "error",
|
||||
serverStatus: null,
|
||||
error:
|
||||
error instanceof Error ? error.message : "Hand tracking failed",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
void start();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
cleanup();
|
||||
};
|
||||
}, [enabled, websocketUrl]);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
@@ -79,3 +79,30 @@ canvas {
|
||||
color: rgba(255, 255, 255, 0.85);
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
.hand-tracking-overlay {
|
||||
position: fixed;
|
||||
right: 16px;
|
||||
bottom: 16px;
|
||||
z-index: 20;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
min-width: 180px;
|
||||
padding: 12px;
|
||||
color: rgba(255, 255, 255, 0.9);
|
||||
background: rgba(4, 7, 13, 0.78);
|
||||
border: 1px solid rgba(255, 255, 255, 0.18);
|
||||
border-radius: 8px;
|
||||
font-size: 12px;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.hand-tracking-overlay strong {
|
||||
color: white;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.hand-tracking-overlay__error {
|
||||
color: #fca5a5;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
export interface HandTrackingHand {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
handedness: string;
|
||||
isPinch: boolean;
|
||||
pinchDistance: number;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export type HandTrackingStatus =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "connected"
|
||||
| "disconnected"
|
||||
| "error";
|
||||
|
||||
export interface HandTrackingSnapshot {
|
||||
hands: HandTrackingHand[];
|
||||
status: HandTrackingStatus;
|
||||
serverStatus: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface HandTrackingFrameMessage {
|
||||
type: "frame";
|
||||
timestamp: number;
|
||||
width: number;
|
||||
height: number;
|
||||
image: string;
|
||||
}
|
||||
|
||||
export interface HandTrackingHandsMessage {
|
||||
type: "hands";
|
||||
timestamp: number;
|
||||
hands: HandTrackingHand[];
|
||||
}
|
||||
|
||||
export interface HandTrackingStatusMessage {
|
||||
type: "status";
|
||||
timestamp: number;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface HandTrackingErrorMessage {
|
||||
type: "error";
|
||||
timestamp: number;
|
||||
hands: HandTrackingHand[];
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type HandTrackingServerMessage =
|
||||
| HandTrackingHandsMessage
|
||||
| HandTrackingStatusMessage
|
||||
| HandTrackingErrorMessage;
|
||||
@@ -54,6 +54,7 @@ export function TestScene({
|
||||
<GrabbableObject
|
||||
position={TEST_SCENE_GRABBABLE_POSITION}
|
||||
colliders="cuboid"
|
||||
handControlled
|
||||
>
|
||||
<mesh castShadow receiveShadow>
|
||||
<boxGeometry args={TEST_SCENE_GRABBABLE_BOX_SIZE} />
|
||||
|
||||
Reference in New Issue
Block a user