math-pixel/La-Fabrik
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(handtracking): restyle svg visualizer and add silhouette fallback

Browse Source
This commit is contained in:
Tom Boullay
2026-06-02 19:05:39 +02:00
parent 4de86f4e58
commit ae35eb1dfb
8 changed files with 207 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/three/handTracking/HandTrackingGlove.tsx
+20 -6
View File
@@ -12,6 +12,11 @@ import { useLoggedGLTF } from "@/hooks/three/useLoggedGLTF";
import type { HandTrackingLandmark } from "@/types/handTracking/handTracking";
import { logModelLoadError } from "@/utils/three/modelLoadLogger";
// Both gloves share the same source mesh (gant_l). The right glove is
// rendered by mirroring scale.x at the group level — this is more
// reliable than the historical gant_r GLTF, which embeds multiple
// skeletons (Hand_l, Hand_l_pad, Hand_r) and was breaking the finger
// rig.
const GLOVE_CONFIGS: Record<
HandTrackingGloveHandedness,
{
@@ -24,8 +29,8 @@ const GLOVE_CONFIGS: Record<
rootNodeName: "Armature",
},
right: {
modelPath: "/models/gant_r/model.gltf",
rootNodeName: "Hand_r",
modelPath: "/models/gant_l/model.gltf",
rootNodeName: "Armature",
},
};
@@ -226,7 +231,10 @@ function applyFingerPose(
_boneTargetQuaternion
.copy(_boneDeltaQuaternion)
.multiply(pose.restQuaternion);
pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.45);
// Lower slerp factor = smoother but more latency. MediaPipe at
// ~10fps produces noisy landmark frames; smoothing cuts the
// jitter the user sees on every finger bone.
pose.bone.quaternion.slerp(_boneTargetQuaternion, 0.3);
}
}
}
@@ -334,12 +342,18 @@ function HandTrackingGloveModel({
_matrix.makeBasis(_xAxis, _yAxis, _zAxis);
_targetQuaternion.setFromRotationMatrix(_matrix);
group.position.lerp(_targetPosition, Math.min(1, delta * 18));
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 18));
// Lower factor (was 18) damps the glove jitter caused by noisy
// landmarks while keeping a responsive feel.
group.position.lerp(_targetPosition, Math.min(1, delta * 12));
group.quaternion.slerp(_targetQuaternion, Math.min(1, delta * 12));
const palmLength = _wristPosition.distanceTo(_middlePosition);
const scale = palmLength * GLOVE_MODEL_SCALE;
group.scale.setScalar(scale);
// Both gloves use the gant_l mesh; flip X for the right hand so the
// thumb ends up on the correct side instead of being a left-glove
// clone on the right hand.
const mirrorSignX = handedness === "right" ? -1 : 1;
group.scale.set(scale * mirrorSignX, scale, scale);
group.updateMatrixWorld(true);
applyFingerPose(fingerPoseChains, trackedHand.landmarks, camera);
});
src/components/ui/GameUI.tsx
+2
View File
@@ -1,6 +1,7 @@
import { Crosshair } from "@/components/ui/Crosshair";
import { DebugOverlayLayout } from "@/components/ui/debug/DebugOverlayLayout";
import { GameSettingsMenu } from "@/components/ui/GameSettingsMenu";
import { HandTrackingFallback } from "@/components/ui/HandTrackingFallback";
import { HandTrackingVisualizer } from "@/components/ui/HandTrackingVisualizer";
import { InteractPrompt } from "@/components/ui/InteractPrompt";
import { RepairMovementLockIndicator } from "@/components/ui/RepairMovementLockIndicator";
@@ -15,6 +16,7 @@ export function GameUI(): React.JSX.Element {
<RepairMovementLockIndicator />
<InteractPrompt />
<HandTrackingVisualizer />
<HandTrackingFallback />
<Subtitles />
<TalkieDialogueOverlay />
<GameSettingsMenu />
src/components/ui/HandTrackingFallback.tsx
+82
View File
@@ -0,0 +1,82 @@
import { useHandTrackingSnapshot } from "@/hooks/handTracking/useHandTrackingSnapshot";
import {
useHandTrackingGloveStatus,
type HandTrackingGloveHandedness,
} from "@/hooks/handTracking/useHandTrackingGloveStatus";
// Simple schematic silhouettes used as a last-resort fallback when the
// rigged glove model has failed to load. Both icons share the same
// 48x48 viewBox and the same stroke/fill rules from the .css.
const OpenHandShape = (): React.JSX.Element => (
<>
<ellipse cx="9" cy="30" rx="3" ry="6" transform="rotate(-25 9 30)" />
<rect x="14" y="8" width="4" height="22" rx="2" />
<rect x="20" y="4" width="4" height="26" rx="2" />
<rect x="26" y="6" width="4" height="24" rx="2" />
<rect x="32" y="10" width="4" height="20" rx="2" />
<rect x="10" y="26" width="28" height="18" rx="6" />
</>
);
const FistShape = (): React.JSX.Element => (
<>
<ellipse cx="8" cy="26" rx="3" ry="5" />
<rect x="10" y="14" width="28" height="30" rx="10" />
<circle cx="15" cy="14" r="3" />
<circle cx="21" cy="13" r="3" />
<circle cx="27" cy="13" r="3" />
<circle cx="33" cy="14" r="3" />
</>
);
function getHandedness(raw: string): HandTrackingGloveHandedness | null {
const lower = raw.toLowerCase();
if (lower === "left" || lower === "right") return lower;
return null;
}
export function HandTrackingFallback(): React.JSX.Element | null {
const { hands } = useHandTrackingSnapshot();
const gloveStatus = useHandTrackingGloveStatus((state) => state.gloves);
const visibleHands = hands.flatMap((hand, index) => {
const handedness = getHandedness(hand.handedness);
if (!handedness) return [];
if (gloveStatus[handedness] !== "error") return [];
const wrist = hand.landmarks[0];
if (!wrist) return [];
return [{ hand, handedness, wrist, index }];
});
if (visibleHands.length === 0) return null;
return (
<div className="hand-tracking-fallback" aria-hidden="true">
{visibleHands.map(({ hand, handedness, wrist, index }) => {
// MediaPipe coords are mirrored (selfie cam), keep the same
// mapping the SVG visualizer uses.
const leftPercent = (1 - wrist.x) * 100;
const topPercent = wrist.y * 100;
const flipX = handedness === "right" ? -1 : 1;
return (
<svg
key={`${handedness}-${index}`}
className="hand-tracking-fallback__icon"
viewBox="0 0 48 48"
style={{
left: `${leftPercent}%`,
top: `${topPercent}%`,
transform: `translate(-50%, -50%) scaleX(${flipX})`,
}}
>
{hand.isFist ? <FistShape /> : <OpenHandShape />}
</svg>
);
})}
</div>
);
}
src/components/ui/HandTrackingVisualizer.tsx
+15 -5
View File
@@ -26,6 +26,12 @@ const HAND_CONNECTIONS: Array<[number, number]> = [
[0, 17],
];
const LANDMARK_FILL = "#67e8f9"; // cyan-300, opaque interior
const LANDMARK_STROKE = "#0c4a6e"; // sky-900, dark blue outline
const LANDMARK_STROKE_FIST = "#1e3a8a"; // blue-900, thicker accent when fist
const CONNECTION_STROKE = "#ffffff"; // white bones
const INDEX_TIP_LANDMARK = 8;
export function HandTrackingVisualizer(): React.JSX.Element | null {
const { hands, status } = useHandTrackingSnapshot();
const showHandTrackingSvg = useDebugStore((debug) =>
@@ -50,7 +56,9 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
const landmarks = hand.landmarks;
if (landmarks.length === 0) return null;
const color = hand.isFist ? "#facc15" : "#38bdf8";
const landmarkStroke = hand.isFist
? LANDMARK_STROKE_FIST
: LANDMARK_STROKE;
return (
<g key={`${hand.handedness}-${handIndex}`}>
@@ -66,8 +74,8 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
y1={`${fromPoint.y * 100}%`}
x2={`${(1 - toPoint.x) * 100}%`}
y2={`${toPoint.y * 100}%`}
stroke={color}
strokeWidth="2"
stroke={CONNECTION_STROKE}
strokeWidth="2.5"
strokeLinecap="round"
/>
);
@@ -78,8 +86,10 @@ export function HandTrackingVisualizer(): React.JSX.Element | null {
key={landmarkIndex}
cx={`${(1 - landmark.x) * 100}%`}
cy={`${landmark.y * 100}%`}
r={landmarkIndex === 8 ? 5 : 3}
fill={landmarkIndex === 8 ? "#ffffff" : color}
r={landmarkIndex === INDEX_TIP_LANDMARK ? 6 : 4}
fill={LANDMARK_FILL}
stroke={landmarkStroke}
strokeWidth={hand.isFist ? 2.5 : 2}
/>
))}
</g>