fix: decouple hand tracking from crosshair focus

src/components/three/GrabbableObject.tsx

@@ -23,6 +23,11 @@ import {
 import { INTERACTION_RADIUS } from "@/data/interaction/interactionConfig";
 import { useDebugFolder } from "@/hooks/debug/useDebugFolder";
 import { useHandTrackingSnapshot } from "@/hooks/useHandTrackingSnapshot";
+import { InteractionManager } from "@/managers/InteractionManager";
+import type {
+  HandTrackingHand,
+  HandTrackingLandmark,
+} from "@/types/handTracking";
 import type { ColliderShape, Vector3Tuple } from "@/types/three";
 
 interface GrabbableObjectProps {
@@ -51,6 +56,14 @@ const _cameraPos = new THREE.Vector3();
 const _objectPos = new THREE.Vector3();
 const _handRaycaster = new THREE.Raycaster();
 
+function getHandAnchorPoint(hand: HandTrackingHand): HandTrackingLandmark {
+  return hand.landmarks.reduce<HandTrackingLandmark>(
+    (lowestPoint, landmark) =>
+      landmark.y > lowestPoint.y ? landmark : lowestPoint,
+    { x: hand.x, y: hand.y, z: hand.z },
+  );
+}
+
 export function GrabbableObject({
   position,
   children,
@@ -107,7 +120,9 @@ export function GrabbableObject({
     _currentPos.set(t.x, t.y, t.z);
 
     if (fistHand) {
-      _handNdc.set((1 - fistHand.x) * 2 - 1, -fistHand.y * 2 + 1, 0.5);
+      const handAnchor = getHandAnchorPoint(fistHand);
+
+      _handNdc.set((1 - handAnchor.x) * 2 - 1, -handAnchor.y * 2 + 1, 0.5);
       _handNdc.unproject(camera);
       camera.getWorldPosition(_cameraPos);
       _handDirection.subVectors(_handNdc, _cameraPos).normalize();
@@ -127,10 +142,12 @@ export function GrabbableObject({
         handHoldDistance.current = isHandHolding.current
           ? hits[0].distance
           : null;
+        InteractionManager.getInstance().setHandHolding(isHandHolding.current);
       }
     } else {
       isHandHolding.current = false;
       handHoldDistance.current = null;
+      InteractionManager.getInstance().setHandHolding(false);
     }
 
     if (!isHolding.current && !isHandHolding.current) return;

src/components/three/InteractableObject.tsx

@@ -74,6 +74,7 @@ export function InteractableObject(
 
   useEffect(() => {
     const currentHandle = handle.current;
+    const manager = InteractionManager.getInstance();
 
     if (currentHandle.kind === kind) {
       currentHandle.label = label;
@@ -87,6 +88,8 @@ export function InteractableObject(
       return;
     }
 
+    manager.setNearby(currentHandle, false);
+
     if (kind === "grab") {
       if (!onRelease) return;
       handle.current = { kind, label, onPress, onRelease };
@@ -94,12 +97,23 @@ export function InteractableObject(
       handle.current = { kind, label, onPress };
     }
 
-    const manager = InteractionManager.getInstance();
     if (manager.getState().focused === currentHandle) {
       manager.setFocused(handle.current);
     }
   }, [kind, label, onPress, onRelease]);
 
+  useEffect(() => {
+    const currentHandle = handle.current;
+
+    return () => {
+      const manager = InteractionManager.getInstance();
+      manager.setNearby(currentHandle, false);
+      if (manager.getState().focused === currentHandle) {
+        manager.setFocused(null);
+      }
+    };
+  }, []);
+
   const setupInteractionDebugFolder = useCallback((folder: GUI) => {
     folder
       .add({ radius: INTERACTION_RADIUS }, "radius")
@@ -128,8 +142,11 @@ export function InteractableObject(
 
     camera.getWorldPosition(_cameraPos);
     const dist = _cameraPos.distanceTo(_objectPos);
+    const isNearby = dist <= INTERACTION_RADIUS;
 
-    if (dist > INTERACTION_RADIUS) {
+    manager.setNearby(handle.current, isNearby);
+
+    if (!isNearby) {
       if (manager.getState().focused === handle.current) {
         manager.setFocused(null);
       }

src/components/ui/HandTrackingProvider.tsx

@@ -14,10 +14,11 @@ export function HandTrackingProvider({
   children: ReactNode;
 }): React.JSX.Element {
   const sceneMode = useSceneMode();
-  const { focused, holding } = useInteraction();
-  const isInInteractionZone = focused !== null || holding;
+  const { nearby, holding, handHolding } = useInteraction();
   const enabled =
-    isDebugEnabled() && sceneMode === "physics" && isInInteractionZone;
+    isDebugEnabled() &&
+    sceneMode === "physics" &&
+    (nearby || holding || handHolding);
   const snapshot = useRemoteHandTracking({ enabled });
 
   return (

src/managers/InteractionManager.ts

@@ -8,11 +8,15 @@ export class InteractionManager {
   private static _instance: InteractionManager | null = null;
 
   private _focused: InteractableHandle | null = null;
+  private readonly _nearbyHandles = new Set<InteractableHandle>();
   private _holding = false;
+  private _handHolding = false;
   private _holdingHandle: GrabInteractableHandle | null = null;
   private _snapshot: InteractionSnapshot = {
     focused: null,
+    nearby: false,
     holding: false,
+    handHolding: false,
   };
   private readonly _listeners = new Set<() => void>();
 
@@ -38,6 +42,26 @@ export class InteractionManager {
     this._emit();
   }
 
+  setNearby(handle: InteractableHandle, nearby: boolean): void {
+    const hadHandle = this._nearbyHandles.has(handle);
+    if (nearby === hadHandle) return;
+
+    if (nearby) {
+      this._nearbyHandles.add(handle);
+    } else {
+      this._nearbyHandles.delete(handle);
+    }
+
+    this._emit();
+  }
+
+  setHandHolding(holding: boolean): void {
+    if (this._handHolding === holding) return;
+
+    this._handHolding = holding;
+    this._emit();
+  }
+
   pressInteract(): void {
     if (!this._focused) return;
 
@@ -73,11 +97,15 @@ export class InteractionManager {
 
   destroy(): void {
     this._focused = null;
+    this._nearbyHandles.clear();
     this._holding = false;
+    this._handHolding = false;
     this._holdingHandle = null;
     this._snapshot = {
       focused: null,
+      nearby: false,
       holding: false,
+      handHolding: false,
     };
     this._listeners.clear();
     InteractionManager._instance = null;
@@ -86,7 +114,9 @@ export class InteractionManager {
   private _emit(): void {
     this._snapshot = {
       focused: this._focused,
+      nearby: this._nearbyHandles.size > 0,
       holding: this._holding,
+      handHolding: this._handHolding,
     };
     this._listeners.forEach((cb) => cb());
   }

src/types/interaction.ts

@@ -19,5 +19,7 @@ export type InteractableHandle =
 
 export interface InteractionSnapshot {
   focused: InteractableHandle | null;
+  nearby: boolean;
   holding: boolean;
+  handHolding: boolean;
 }