refactor: split hooks types and utils by domain
This commit is contained in:
Tom Boullay
@@ -0,0 +1,281 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
HAND_TRACKING_CAMERA_TIMEOUT_MS,
|
||||
HAND_TRACKING_FRAME_HEIGHT,
|
||||
HAND_TRACKING_FRAME_WIDTH,
|
||||
HAND_TRACKING_JPEG_QUALITY,
|
||||
HAND_TRACKING_RESPONSE_TIMEOUT_MS,
|
||||
HAND_TRACKING_TARGET_FPS,
|
||||
getHandTrackingWsUrl,
|
||||
} from "@/data/handTrackingConfig";
|
||||
import type {
|
||||
HandTrackingFrameMessage,
|
||||
HandTrackingServerMessage,
|
||||
HandTrackingSnapshot,
|
||||
} from "@/types/handTracking/handTracking";
|
||||
|
||||
interface UseRemoteHandTrackingOptions {
|
||||
enabled: boolean;
|
||||
websocketUrl?: string;
|
||||
}
|
||||
|
||||
const INITIAL_SNAPSHOT: HandTrackingSnapshot = {
|
||||
hands: [],
|
||||
status: "idle",
|
||||
usageStatus: "inactive",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
};
|
||||
|
||||
function getBase64Payload(dataUrl: string): string {
|
||||
return dataUrl.slice(dataUrl.indexOf(",") + 1);
|
||||
}
|
||||
|
||||
function getCameraStreamWithTimeout(
|
||||
constraints: MediaStreamConstraints,
|
||||
): Promise<MediaStream> {
|
||||
let didTimeout = false;
|
||||
const streamPromise = navigator.mediaDevices.getUserMedia(constraints);
|
||||
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
window.setTimeout(() => {
|
||||
didTimeout = true;
|
||||
reject(
|
||||
new Error(
|
||||
"Camera request timed out. Restart Arc or check camera permissions for localhost:5173.",
|
||||
),
|
||||
);
|
||||
}, HAND_TRACKING_CAMERA_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
streamPromise.then((stream) => {
|
||||
if (didTimeout) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
}
|
||||
});
|
||||
|
||||
return Promise.race([streamPromise, timeoutPromise]);
|
||||
}
|
||||
|
||||
export function useRemoteHandTracking({
|
||||
enabled,
|
||||
websocketUrl = getHandTrackingWsUrl(),
|
||||
}: UseRemoteHandTrackingOptions): HandTrackingSnapshot {
|
||||
const [snapshot, setSnapshot] =
|
||||
useState<HandTrackingSnapshot>(INITIAL_SNAPSHOT);
|
||||
const videoRef = useRef<HTMLVideoElement | null>(null);
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const sendIntervalRef = useRef<number | null>(null);
|
||||
const responseTimeoutRef = useRef<number | null>(null);
|
||||
const waitingForResponseRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const clearResponseTimeout = (): void => {
|
||||
if (responseTimeoutRef.current === null) return;
|
||||
window.clearTimeout(responseTimeoutRef.current);
|
||||
responseTimeoutRef.current = null;
|
||||
};
|
||||
|
||||
const cleanup = (): void => {
|
||||
if (sendIntervalRef.current !== null) {
|
||||
window.clearInterval(sendIntervalRef.current);
|
||||
sendIntervalRef.current = null;
|
||||
}
|
||||
|
||||
clearResponseTimeout();
|
||||
waitingForResponseRef.current = false;
|
||||
wsRef.current?.close();
|
||||
wsRef.current = null;
|
||||
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
videoRef.current = null;
|
||||
canvasRef.current = null;
|
||||
};
|
||||
|
||||
const markResponseReceived = (): void => {
|
||||
waitingForResponseRef.current = false;
|
||||
clearResponseTimeout();
|
||||
};
|
||||
|
||||
const sendFrame = (): void => {
|
||||
const ws = wsRef.current;
|
||||
const video = videoRef.current;
|
||||
const canvas = canvasRef.current;
|
||||
const context = canvas?.getContext("2d");
|
||||
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return;
|
||||
if (!video || !canvas || !context) return;
|
||||
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return;
|
||||
if (waitingForResponseRef.current) return;
|
||||
|
||||
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||||
const dataUrl = canvas.toDataURL(
|
||||
"image/jpeg",
|
||||
HAND_TRACKING_JPEG_QUALITY,
|
||||
);
|
||||
const message: HandTrackingFrameMessage = {
|
||||
type: "frame",
|
||||
timestamp: Date.now(),
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
image: getBase64Payload(dataUrl),
|
||||
};
|
||||
|
||||
waitingForResponseRef.current = true;
|
||||
ws.send(JSON.stringify(message));
|
||||
responseTimeoutRef.current = window.setTimeout(() => {
|
||||
waitingForResponseRef.current = false;
|
||||
responseTimeoutRef.current = null;
|
||||
}, HAND_TRACKING_RESPONSE_TIMEOUT_MS);
|
||||
};
|
||||
|
||||
const start = async (): Promise<void> => {
|
||||
await Promise.resolve();
|
||||
if (cancelled) return;
|
||||
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "requesting_camera",
|
||||
usageStatus: "available",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const stream = await getCameraStreamWithTimeout({
|
||||
video: {
|
||||
width: HAND_TRACKING_FRAME_WIDTH,
|
||||
height: HAND_TRACKING_FRAME_HEIGHT,
|
||||
facingMode: "user",
|
||||
},
|
||||
audio: false,
|
||||
});
|
||||
|
||||
if (cancelled) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "starting_camera",
|
||||
}));
|
||||
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
video.srcObject = stream;
|
||||
await video.play();
|
||||
|
||||
if (cancelled) {
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "connecting_server",
|
||||
}));
|
||||
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = HAND_TRACKING_FRAME_WIDTH;
|
||||
canvas.height = HAND_TRACKING_FRAME_HEIGHT;
|
||||
|
||||
const ws = new WebSocket(websocketUrl);
|
||||
ws.onopen = () => {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "connected",
|
||||
usageStatus: "available",
|
||||
error: null,
|
||||
}));
|
||||
};
|
||||
ws.onmessage = (event) => {
|
||||
markResponseReceived();
|
||||
const data = JSON.parse(event.data) as HandTrackingServerMessage;
|
||||
|
||||
if (data.type === "hands") {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands: data.hands,
|
||||
usageStatus: data.hands.some((hand) => hand.isFist)
|
||||
? "active"
|
||||
: "available",
|
||||
serverStatus: null,
|
||||
error: null,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.type === "status") {
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
serverStatus: data.status,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
hands: [],
|
||||
status: "error",
|
||||
usageStatus: "inactive",
|
||||
error: data.message,
|
||||
}));
|
||||
};
|
||||
ws.onerror = () => {
|
||||
markResponseReceived();
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: "error",
|
||||
error: "Hand tracking WebSocket error",
|
||||
}));
|
||||
};
|
||||
ws.onclose = () => {
|
||||
markResponseReceived();
|
||||
setSnapshot((current) => ({
|
||||
...current,
|
||||
status: cancelled ? "idle" : "disconnected",
|
||||
}));
|
||||
};
|
||||
|
||||
streamRef.current = stream;
|
||||
videoRef.current = video;
|
||||
canvasRef.current = canvas;
|
||||
wsRef.current = ws;
|
||||
sendIntervalRef.current = window.setInterval(
|
||||
sendFrame,
|
||||
1_000 / HAND_TRACKING_TARGET_FPS,
|
||||
);
|
||||
} catch (error) {
|
||||
if (cancelled) return;
|
||||
setSnapshot({
|
||||
hands: [],
|
||||
status: "error",
|
||||
usageStatus: "inactive",
|
||||
serverStatus: null,
|
||||
error:
|
||||
error instanceof Error ? error.message : "Hand tracking failed",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
void start();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
cleanup();
|
||||
};
|
||||
}, [enabled, websocketUrl]);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
Reference in New Issue
Block a user