feat move debug cube with remote hand tracking

src/components/3d/GrabbableObject.tsx

@@ -21,6 +21,7 @@ import {
   GRAB_THROW_BOOST_STEP,
 } from "@/data/grabConfig";
 import { useDebugFolder } from "@/hooks/debug/useDebugFolder";
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
 import type { ColliderShape, Vector3Tuple } from "@/types/3d";
 
 interface GrabbableObjectProps {
@@ -28,6 +29,7 @@ interface GrabbableObjectProps {
   children: React.ReactNode;
   colliders?: ColliderShape;
   label?: string;
+  handControlled?: boolean;
 }
 
 // Shared params let one debug folder drive every instance.
@@ -42,14 +44,18 @@ const ZERO_ANGULAR_VELOCITY = { x: 0, y: 0, z: 0 };
 const _holdTarget = new THREE.Vector3();
 const _currentPos = new THREE.Vector3();
 const _velocity = new THREE.Vector3();
+const _handNdc = new THREE.Vector3();
+const _handDirection = new THREE.Vector3();
 
 export function GrabbableObject({
   position,
   children,
   colliders = GRAB_DEFAULT_COLLIDERS,
   label = GRAB_DEFAULT_LABEL,
+  handControlled = false,
 }: GrabbableObjectProps): React.JSX.Element {
   const camera = useThree((state) => state.camera);
+  const { hands } = useHandTrackingSnapshot();
   const rbRef = useRef<RapierRigidBody>(null);
   const isHolding = useRef(false);
 
@@ -84,10 +90,25 @@ export function GrabbableObject({
   });
 
   useFrame(() => {
-    if (!isHolding.current || !rbRef.current) return;
+    if (!rbRef.current) return;
 
-    camera.getWorldDirection(_holdTarget);
-    _holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
+    const pinchingHand = handControlled
+      ? hands.find((hand) => hand.isPinch)
+      : undefined;
+
+    if (!isHolding.current && !pinchingHand) return;
+
+    if (pinchingHand) {
+      _handNdc.set((1 - pinchingHand.x) * 2 - 1, -pinchingHand.y * 2 + 1, 0.5);
+      _handNdc.unproject(camera);
+      _handDirection.subVectors(_handNdc, camera.position).normalize();
+      _holdTarget
+        .copy(camera.position)
+        .addScaledVector(_handDirection, params.holdDistance);
+    } else {
+      camera.getWorldDirection(_holdTarget);
+      _holdTarget.multiplyScalar(params.holdDistance).add(camera.position);
+    }
 
     const t = rbRef.current.translation();
     _currentPos.set(t.x, t.y, t.z);

src/components/ui/HandTrackingOverlay.tsx

@@ -0,0 +1,24 @@
+import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
+
+export function HandTrackingOverlay(): React.JSX.Element | null {
+  const { hands, status, serverStatus, error } = useHandTrackingSnapshot();
+
+  if (status === "idle") {
+    return null;
+  }
+
+  const pinching = hands.some((hand) => hand.isPinch);
+
+  return (
+    <aside className="hand-tracking-overlay" aria-label="Hand tracking status">
+      <strong>Hand tracking</strong>
+      <span>Status: {status}</span>
+      {serverStatus ? <span>Server: {serverStatus}</span> : null}
+      <span>Hands: {hands.length}</span>
+      <span>Pinch: {pinching ? "yes" : "no"}</span>
+      {error ? (
+        <span className="hand-tracking-overlay__error">{error}</span>
+      ) : null}
+    </aside>
+  );
+}

src/components/ui/HandTrackingProvider.tsx

@@ -0,0 +1,26 @@
+import type { ReactNode } from "react";
+import { useSceneMode } from "@/hooks/debug/useSceneMode";
+import {
+  HAND_TRACKING_IDLE_SNAPSHOT,
+  HandTrackingContext,
+} from "@/hooks/useHandTrackingSnapshot";
+import { useRemoteHandTracking } from "@/hooks/useRemoteHandTracking";
+import { isDebugEnabled } from "@/utils/debug/isDebugEnabled";
+
+export function HandTrackingProvider({
+  children,
+}: {
+  children: ReactNode;
+}): React.JSX.Element {
+  const sceneMode = useSceneMode();
+  const enabled = isDebugEnabled() && sceneMode === "physics";
+  const snapshot = useRemoteHandTracking({ enabled });
+
+  return (
+    <HandTrackingContext
+      value={enabled ? snapshot : HAND_TRACKING_IDLE_SNAPSHOT}
+    >
+      {children}
+    </HandTrackingContext>
+  );
+}