Tom Boullay
131 lines
4.1 KiB
TypeScript
131 lines
4.1 KiB
TypeScript
import { useMemo, useRef } from "react";
|
|
import { useFrame, useThree } from "@react-three/fiber";
|
|
import { useGLTF } from "@react-three/drei";
|
|
import * as THREE from "three";
|
|
import { clone } from "three/addons/utils/SkeletonUtils.js";
|
|
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
|
|
import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
|
|
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
|
|
|
|
const LEFT_GLOVE_MODEL_URL = "/models/gant_l/model.gltf";
|
|
const HAND_SPACE_DISTANCE = 2.4;
|
|
const HAND_DEPTH_SCALE = 0.45;
|
|
const GLOVE_SCALE = 0.34;
|
|
|
|
const _cameraPosition = new THREE.Vector3();
|
|
const _direction = new THREE.Vector3();
|
|
const _xAxis = new THREE.Vector3();
|
|
const _yAxis = new THREE.Vector3();
|
|
const _zAxis = new THREE.Vector3();
|
|
const _matrix = new THREE.Matrix4();
|
|
const _targetQuaternion = new THREE.Quaternion();
|
|
const _targetPosition = new THREE.Vector3();
|
|
const _wristPosition = new THREE.Vector3();
|
|
const _indexPosition = new THREE.Vector3();
|
|
const _middlePosition = new THREE.Vector3();
|
|
const _ringPosition = new THREE.Vector3();
|
|
const _pinkyPosition = new THREE.Vector3();
|
|
|
|
function landmarkToWorldPoint(
|
|
landmark: HandTrackingLandmark,
|
|
camera: THREE.Camera,
|
|
target: THREE.Vector3,
|
|
): THREE.Vector3 {
|
|
_cameraPosition.setFromMatrixPosition(camera.matrixWorld);
|
|
target.set((1 - landmark.x) * 2 - 1, -landmark.y * 2 + 1, 0.5);
|
|
target.unproject(camera);
|
|
|
|
_direction.copy(target).sub(_cameraPosition).normalize();
|
|
target
|
|
.copy(_cameraPosition)
|
|
.addScaledVector(
|
|
_direction,
|
|
HAND_SPACE_DISTANCE - landmark.z * HAND_DEPTH_SCALE,
|
|
);
|
|
|
|
return target;
|
|
}
|
|
|
|
function isLeftHand(handedness: string): boolean {
|
|
return handedness.toLowerCase() === "left";
|
|
}
|
|
|
|
export function HandTrackingLeftGlove(): React.JSX.Element | null {
|
|
const groupRef = useRef<THREE.Group>(null);
|
|
const { camera } = useThree();
|
|
const { hands } = useHandTrackingSnapshot();
|
|
const gltf = useLoggedGLTF(LEFT_GLOVE_MODEL_URL, {
|
|
scope: "HandTrackingLeftGlove",
|
|
scale: GLOVE_SCALE,
|
|
});
|
|
const gloveScene = useMemo(() => clone(gltf.scene), [gltf.scene]);
|
|
|
|
const leftHand = hands.find((hand) => isLeftHand(hand.handedness));
|
|
|
|
useFrame((_, delta) => {
|
|
const group = groupRef.current;
|
|
const hand = hands.find((candidate) => isLeftHand(candidate.handedness));
|
|
|
|
if (!group || !hand || hand.landmarks.length < 21) {
|
|
if (group) group.visible = false;
|
|
return;
|
|
}
|
|
|
|
group.visible = true;
|
|
|
|
const wrist = hand.landmarks[0];
|
|
const indexMcp = hand.landmarks[5];
|
|
const middleMcp = hand.landmarks[9];
|
|
const ringMcp = hand.landmarks[13];
|
|
const pinkyMcp = hand.landmarks[17];
|
|
|
|
if (!wrist || !indexMcp || !middleMcp || !ringMcp || !pinkyMcp) {
|
|
group.visible = false;
|
|
return;
|
|
}
|
|
|
|
landmarkToWorldPoint(wrist, camera, _wristPosition);
|
|
landmarkToWorldPoint(indexMcp, camera, _indexPosition);
|
|
landmarkToWorldPoint(middleMcp, camera, _middlePosition);
|
|
landmarkToWorldPoint(ringMcp, camera, _ringPosition);
|
|
landmarkToWorldPoint(pinkyMcp, camera, _pinkyPosition);
|
|
|
|
_targetPosition
|
|
.copy(_wristPosition)
|
|
.add(_indexPosition)
|
|
.add(_middlePosition)
|
|
.add(_ringPosition)
|
|
.add(_pinkyPosition)
|
|
.multiplyScalar(0.2);
|
|
|
|
_yAxis.copy(_middlePosition).sub(_wristPosition).normalize();
|
|
_xAxis.copy(_indexPosition).sub(_pinkyPosition).normalize();
|
|
_zAxis.crossVectors(_xAxis, _yAxis).normalize();
|
|
|
|
if (
|
|
_xAxis.lengthSq() === 0 ||
|
|
_yAxis.lengthSq() === 0 ||
|
|
_zAxis.lengthSq() === 0
|
|
) {
|
|
return;
|
|
}
|
|
|
|
_xAxis.crossVectors(_yAxis, _zAxis).normalize();
|
|
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
|
|
_targetQuaternion.setFromRotationMatrix(_matrix);
|
|
|
|
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
|
|
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
|
|
|
|
const palmLength = _wristPosition.distanceTo(_middlePosition);
|
|
const scale = palmLength * GLOVE_SCALE;
|
|
group.scale.setScalar(scale);
|
|
});
|
|
|
|
if (!leftHand) return null;
|
|
|
|
return <primitive ref={groupRef} object={gloveScene} />;
|
|
}
|
|
|
|
useGLTF.preload(LEFT_GLOVE_MODEL_URL);
|