mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-08 16:39:42 +02:00
feat: vendor gesture-control source into packages/gesture-control
Bring the Ark0N/codeman-gesture-control repo in-tree as the codeman-gesture-control workspace package so the hand-tracking overlay can be developed in the Codeman repo. New npm run build:gesture bundles src/codeman/entry.ts into the served gesture-codeman.js; scripts/build.mjs reruns it on every production build. Source formatted to Codeman's prettier style. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
arkon
@@ -0,0 +1,386 @@
|
||||
// GestureController — core input layer.
|
||||
//
|
||||
// Phase 0/1 scope (this file currently implements):
|
||||
// - Open the webcam (getUserMedia) and attach it to a <video>.
|
||||
// - Load MediaPipe GestureRecognizer in VIDEO mode (wasm + .task from CDN).
|
||||
// - Run a requestAnimationFrame loop calling recognizeForVideo().
|
||||
// - Emit `status` (fps / handPresent / gesture) and `results` (raw, debug).
|
||||
//
|
||||
// Later phases add the cursor (One-Euro filtered), pinch hysteresis, the
|
||||
// hover/grab/drag/drop state machine, and the discrete gesture command bus.
|
||||
// The event surface in types.ts already declares those so the API is stable.
|
||||
|
||||
import { FilesetResolver, GestureRecognizer, type GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
import type {
|
||||
GestureControllerOptions,
|
||||
GestureEventHandler,
|
||||
GestureEventMap,
|
||||
GestureEventName,
|
||||
HandState,
|
||||
} from './types.ts';
|
||||
import { LANDMARK, midpoint } from './landmarks.ts';
|
||||
import { OneEuroFilter } from './OneEuroFilter.ts';
|
||||
import { PinchDetector, pinchDistance } from './pinch.ts';
|
||||
import { CommandDetector } from './commands.ts';
|
||||
|
||||
const DEFAULTS = {
|
||||
numHands: 1,
|
||||
deviceId: '',
|
||||
pinchOn: 0.35,
|
||||
pinchOff: 0.5,
|
||||
minCutoff: 1.0,
|
||||
beta: 0.01,
|
||||
palmHoldMs: 1000,
|
||||
minDetectionConfidence: 0.6,
|
||||
minTrackingConfidence: 0.6,
|
||||
// Pinned to the @mediapipe/tasks-vision version in package.json.
|
||||
wasmBase: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.21/wasm',
|
||||
modelUrl:
|
||||
'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
|
||||
} as const;
|
||||
|
||||
interface PerHandState {
|
||||
cursorX: OneEuroFilter;
|
||||
cursorY: OneEuroFilter;
|
||||
pinch: PinchDetector;
|
||||
/** Original handedness label (the map key may be disambiguated). */
|
||||
label: string;
|
||||
/** Last emitted surface-pixel position, so we can drop on a vanished hand. */
|
||||
lastX: number;
|
||||
lastY: number;
|
||||
}
|
||||
|
||||
export class GestureController {
|
||||
private readonly opts: Required<GestureControllerOptions>;
|
||||
private recognizer: GestureRecognizer | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private rafId: number | null = null;
|
||||
private running = false;
|
||||
|
||||
// Timestamps must be strictly increasing for recognizeForVideo.
|
||||
private lastVideoTime = -1;
|
||||
private lastTimestamp = -1;
|
||||
|
||||
// FPS tracking (rolling over a short window).
|
||||
private frameTimes: number[] = [];
|
||||
|
||||
// Per-hand smoothing + pinch state, keyed by handedness ("Left"/"Right") so a
|
||||
// hand keeps its own filters even when MediaPipe reorders the hands array.
|
||||
private readonly handStates = new Map<string, PerHandState>();
|
||||
|
||||
// Discrete gesture → command bus (debounced; Open_Palm held = halt-all).
|
||||
private readonly commands: CommandDetector;
|
||||
|
||||
// Internal storage is intentionally loose; the public on/off/emit signatures
|
||||
// below keep callers fully type-safe per event name.
|
||||
private listeners: Partial<Record<GestureEventName, Set<(payload: unknown) => void>>> = {};
|
||||
|
||||
constructor(options: GestureControllerOptions) {
|
||||
this.opts = {
|
||||
surface: options.surface ?? options.video,
|
||||
numHands: options.numHands ?? DEFAULTS.numHands,
|
||||
deviceId: options.deviceId ?? DEFAULTS.deviceId,
|
||||
pinchOn: options.pinchOn ?? DEFAULTS.pinchOn,
|
||||
pinchOff: options.pinchOff ?? DEFAULTS.pinchOff,
|
||||
minCutoff: options.minCutoff ?? DEFAULTS.minCutoff,
|
||||
beta: options.beta ?? DEFAULTS.beta,
|
||||
palmHoldMs: options.palmHoldMs ?? DEFAULTS.palmHoldMs,
|
||||
minDetectionConfidence: options.minDetectionConfidence ?? DEFAULTS.minDetectionConfidence,
|
||||
minTrackingConfidence: options.minTrackingConfidence ?? DEFAULTS.minTrackingConfidence,
|
||||
wasmBase: options.wasmBase ?? DEFAULTS.wasmBase,
|
||||
modelUrl: options.modelUrl ?? DEFAULTS.modelUrl,
|
||||
video: options.video,
|
||||
};
|
||||
|
||||
this.commands = new CommandDetector(this.opts.palmHoldMs);
|
||||
}
|
||||
|
||||
/** Get (or lazily create) the smoothing + pinch state for one hand. */
|
||||
private handState(key: string): PerHandState {
|
||||
let state = this.handStates.get(key);
|
||||
if (!state) {
|
||||
state = {
|
||||
cursorX: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
cursorY: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
pinch: new PinchDetector(this.opts.pinchOn, this.opts.pinchOff),
|
||||
label: key,
|
||||
lastX: 0,
|
||||
lastY: 0,
|
||||
};
|
||||
this.handStates.set(key, state);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ---- Event emitter ---------------------------------------------------
|
||||
|
||||
on<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
(this.listeners[event] ??= new Set()).add(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
off<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
this.listeners[event]?.delete(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
private emit<K extends GestureEventName>(event: K, payload: GestureEventMap[K]): void {
|
||||
this.listeners[event]?.forEach((h) => h(payload));
|
||||
}
|
||||
|
||||
// ---- Lifecycle -------------------------------------------------------
|
||||
|
||||
/** Request the camera, load the model, and start the recognition loop. */
|
||||
async start(): Promise<void> {
|
||||
if (this.running) return;
|
||||
|
||||
await this.openCamera();
|
||||
await this.loadRecognizer();
|
||||
|
||||
this.running = true;
|
||||
this.lastVideoTime = -1;
|
||||
this.lastTimestamp = -1;
|
||||
this.frameTimes = [];
|
||||
this.handStates.clear();
|
||||
this.commands.reset();
|
||||
this.loop();
|
||||
}
|
||||
|
||||
/** Stop the loop, release the camera, and close the recognizer. */
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
if (this.rafId !== null) {
|
||||
cancelAnimationFrame(this.rafId);
|
||||
this.rafId = null;
|
||||
}
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
this.opts.video.srcObject = null;
|
||||
this.recognizer?.close();
|
||||
this.recognizer = null;
|
||||
}
|
||||
|
||||
/** The camera currently in use (resolved deviceId), or "" if not started. */
|
||||
get deviceId(): string {
|
||||
return this.opts.deviceId;
|
||||
}
|
||||
|
||||
/** List available video input devices. Labels are only populated once the
|
||||
* user has granted camera permission (i.e. after the first start()). */
|
||||
async listCameras(): Promise<MediaDeviceInfo[]> {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return devices.filter((d) => d.kind === 'videoinput');
|
||||
}
|
||||
|
||||
/** Switch to a different camera. Reopens the stream live if already running. */
|
||||
async useCamera(deviceId: string): Promise<void> {
|
||||
this.opts.deviceId = deviceId;
|
||||
if (!this.running) return;
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
await this.openCamera();
|
||||
// Fresh camera = fresh geometry; drop stale filter state to avoid a snap.
|
||||
this.handStates.clear();
|
||||
this.lastVideoTime = -1;
|
||||
}
|
||||
|
||||
// ---- Setup -----------------------------------------------------------
|
||||
|
||||
private async openCamera(): Promise<void> {
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('getUserMedia is not available (needs a secure context).');
|
||||
}
|
||||
const videoConstraints: MediaTrackConstraints = {
|
||||
width: { ideal: 1280 },
|
||||
height: { ideal: 720 },
|
||||
frameRate: { ideal: 60, max: 60 },
|
||||
};
|
||||
// A specific device wins; otherwise ask for the user-facing camera.
|
||||
if (this.opts.deviceId) videoConstraints.deviceId = { exact: this.opts.deviceId };
|
||||
else videoConstraints.facingMode = 'user';
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: videoConstraints,
|
||||
audio: false,
|
||||
});
|
||||
this.stream = stream;
|
||||
// Record the resolved device so callers can pre-select it in a UI.
|
||||
this.opts.deviceId = stream.getVideoTracks()[0]?.getSettings().deviceId ?? this.opts.deviceId;
|
||||
const video = this.opts.video;
|
||||
video.srcObject = stream;
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
await video.play();
|
||||
|
||||
// Wait until dimensions are known so the overlay can size itself.
|
||||
if (!video.videoWidth) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const onMeta = () => {
|
||||
video.removeEventListener('loadedmetadata', onMeta);
|
||||
resolve();
|
||||
};
|
||||
video.addEventListener('loadedmetadata', onMeta);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async loadRecognizer(): Promise<void> {
|
||||
const fileset = await FilesetResolver.forVisionTasks(this.opts.wasmBase);
|
||||
const build = (delegate: 'GPU' | 'CPU') =>
|
||||
GestureRecognizer.createFromOptions(fileset, {
|
||||
baseOptions: {
|
||||
modelAssetPath: this.opts.modelUrl,
|
||||
delegate,
|
||||
},
|
||||
runningMode: 'VIDEO',
|
||||
numHands: this.opts.numHands,
|
||||
minHandDetectionConfidence: this.opts.minDetectionConfidence,
|
||||
minHandPresenceConfidence: this.opts.minDetectionConfidence,
|
||||
minTrackingConfidence: this.opts.minTrackingConfidence,
|
||||
});
|
||||
// GPU is the fast path (60fps on the MacBook). But some environments fail to
|
||||
// init the GPU delegate — and MediaPipe/Emscripten often throws a *non-Error*
|
||||
// value there (a raw number/string), which surfaces upstream as a useless
|
||||
// "failed: undefined". Fall back to CPU so the recognizer still starts.
|
||||
try {
|
||||
this.recognizer = await build('GPU');
|
||||
} catch (err) {
|
||||
console.warn('[gesture] GPU delegate failed; falling back to CPU.', err);
|
||||
this.recognizer = await build('CPU');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Loop ------------------------------------------------------------
|
||||
|
||||
private loop = (): void => {
|
||||
if (!this.running || !this.recognizer) return;
|
||||
this.rafId = requestAnimationFrame(this.loop);
|
||||
|
||||
const video = this.opts.video;
|
||||
if (video.readyState < 2 /* HAVE_CURRENT_DATA */) return;
|
||||
|
||||
// Strictly increasing timestamp in ms.
|
||||
let ts = performance.now();
|
||||
if (ts <= this.lastTimestamp) ts = this.lastTimestamp + 1;
|
||||
this.lastTimestamp = ts;
|
||||
|
||||
// Only re-run inference when the video frame actually advanced.
|
||||
if (video.currentTime === this.lastVideoTime) return;
|
||||
this.lastVideoTime = video.currentTime;
|
||||
|
||||
let result: GestureRecognizerResult;
|
||||
try {
|
||||
result = this.recognizer.recognizeForVideo(video, ts);
|
||||
} catch (err) {
|
||||
console.error('recognizeForVideo failed', err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.trackFps(ts);
|
||||
this.publish(result, ts);
|
||||
};
|
||||
|
||||
private trackFps(nowMs: number): void {
|
||||
this.frameTimes.push(nowMs);
|
||||
const windowStart = nowMs - 1000;
|
||||
while (this.frameTimes.length && this.frameTimes[0] < windowStart) {
|
||||
this.frameTimes.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private get fps(): number {
|
||||
if (this.frameTimes.length < 2) return 0;
|
||||
const span = this.frameTimes[this.frameTimes.length - 1] - this.frameTimes[0];
|
||||
if (span <= 0) return 0;
|
||||
return Math.round(((this.frameTimes.length - 1) / span) * 1000);
|
||||
}
|
||||
|
||||
private publish(result: GestureRecognizerResult, ts: number): void {
|
||||
const allLandmarks = result.landmarks ?? [];
|
||||
const handedness = result.handedness ?? [];
|
||||
const gestures = result.gestures ?? [];
|
||||
const tSec = ts / 1000;
|
||||
|
||||
// Surface rect → maps normalized [0,1] coords to surface pixels (X mirrored).
|
||||
const rect = this.opts.surface.getBoundingClientRect();
|
||||
|
||||
const hands: HandState[] = [];
|
||||
const activeKeys = new Set<string>();
|
||||
|
||||
for (let i = 0; i < allLandmarks.length; i++) {
|
||||
const lm = allLandmarks[i];
|
||||
if (!lm || lm.length === 0) continue;
|
||||
|
||||
// Key by handedness so each hand keeps its own filters across frames.
|
||||
// Fall back to index, and disambiguate if both hands share a label.
|
||||
const label = handedness[i]?.[0]?.categoryName ?? `hand${i}`;
|
||||
let key = label;
|
||||
if (activeKeys.has(key)) key = `${label}#${i}`;
|
||||
activeKeys.add(key);
|
||||
|
||||
const state = this.handState(key);
|
||||
const mid = midpoint(lm[LANDMARK.THUMB_TIP], lm[LANDMARK.INDEX_TIP]);
|
||||
const cursor = {
|
||||
x: state.cursorX.filter(mid.x, tSec),
|
||||
y: state.cursorY.filter(mid.y, tSec),
|
||||
};
|
||||
const pinchDist = pinchDistance(lm);
|
||||
const changed = state.pinch.update(pinchDist);
|
||||
const gesture = gestures[i]?.[0]?.categoryName ?? null;
|
||||
|
||||
// Pinch state machine → discrete drag events in surface pixels.
|
||||
const sx = (1 - cursor.x) * rect.width;
|
||||
const sy = cursor.y * rect.height;
|
||||
state.lastX = sx;
|
||||
state.lastY = sy;
|
||||
const pointer = { hand: label, x: sx, y: sy };
|
||||
if (state.pinch.isPinching) {
|
||||
this.emit(changed ? 'grab' : 'drag', pointer);
|
||||
} else if (changed) {
|
||||
this.emit('drop', pointer);
|
||||
}
|
||||
|
||||
hands.push({
|
||||
handedness: label,
|
||||
cursor,
|
||||
pinchDist,
|
||||
pinching: state.pinch.isPinching,
|
||||
gesture: gesture && gesture !== 'None' ? gesture : null,
|
||||
});
|
||||
}
|
||||
|
||||
// Drop state for hands that vanished, so a returning hand starts fresh
|
||||
// (no snap from a stale filter position) and the map can't grow unbounded.
|
||||
// If a vanished hand was mid-pinch, emit a drop so nothing stays grabbed.
|
||||
for (const key of [...this.handStates.keys()]) {
|
||||
if (activeKeys.has(key)) continue;
|
||||
const stale = this.handStates.get(key)!;
|
||||
if (stale.pinch.isPinching) {
|
||||
this.emit('drop', { hand: stale.label, x: stale.lastX, y: stale.lastY });
|
||||
}
|
||||
this.handStates.delete(key);
|
||||
}
|
||||
|
||||
// Discrete commands come from open-hand gestures; ignore a hand that's
|
||||
// pinching (mid-drag) so a drag can't be misread as a command.
|
||||
const commandGestures = new Set<string>();
|
||||
for (const h of hands) {
|
||||
if (!h.pinching && h.gesture) commandGestures.add(h.gesture);
|
||||
}
|
||||
for (const name of this.commands.update(commandGestures, ts)) {
|
||||
this.emit('command', { name });
|
||||
}
|
||||
|
||||
// Status first so the overlay can read this frame's cursors in `results`.
|
||||
this.emit('status', {
|
||||
fps: this.fps,
|
||||
hands,
|
||||
haltProgress: this.commands.haltProgress(ts),
|
||||
});
|
||||
this.emit('results', { result, timestampMs: ts });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// One-Euro filter — low-latency smoothing for noisy interactive signals.
|
||||
// Heavy smoothing when the value is still (kills jitter), low lag when it moves
|
||||
// fast. The correct tool for raw landmark streams, which jitter several pixels
|
||||
// even when the hand is held still.
|
||||
//
|
||||
// Reference: Casiez, Roussel, Vogel — "1€ Filter" (CHI 2012),
|
||||
// http://cristal.univ-lille.fr/~casiez/1euro/
|
||||
//
|
||||
// One filter handles a single scalar; use one instance per axis (x, y).
|
||||
|
||||
/** Smoothing factor for a low-pass step given a cutoff (Hz) and timestep (s). */
|
||||
function smoothingAlpha(cutoffHz: number, dtSec: number): number {
|
||||
const tau = 1 / (2 * Math.PI * cutoffHz);
|
||||
return 1 / (1 + tau / dtSec);
|
||||
}
|
||||
|
||||
/** Exponential low-pass that remembers its last output. */
|
||||
class LowPass {
|
||||
private value: number | null = null;
|
||||
|
||||
filter(x: number, alpha: number): number {
|
||||
this.value = this.value === null ? x : alpha * x + (1 - alpha) * this.value;
|
||||
return this.value;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.value = null;
|
||||
}
|
||||
|
||||
get initialized(): boolean {
|
||||
return this.value !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export class OneEuroFilter {
|
||||
private readonly signal = new LowPass();
|
||||
private readonly derivative = new LowPass();
|
||||
private lastTimeSec: number | null = null;
|
||||
private lastRaw = 0;
|
||||
|
||||
/**
|
||||
* @param minCutoff Baseline cutoff (Hz). Lower → smoother but laggier when still.
|
||||
* @param beta Speed coefficient. Higher → less lag during fast moves.
|
||||
* @param dCutoff Cutoff for the derivative low-pass (Hz). 1.0 is fine.
|
||||
*/
|
||||
constructor(
|
||||
private minCutoff = 1.0,
|
||||
private beta = 0.01,
|
||||
private dCutoff = 1.0
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.signal.reset();
|
||||
this.derivative.reset();
|
||||
this.lastTimeSec = null;
|
||||
this.lastRaw = 0;
|
||||
}
|
||||
|
||||
/** @param timeSec strictly-increasing timestamp in seconds. */
|
||||
filter(x: number, timeSec: number): number {
|
||||
let dt = this.lastTimeSec === null ? 1 / 60 : timeSec - this.lastTimeSec;
|
||||
if (dt <= 0) dt = 1 / 60;
|
||||
this.lastTimeSec = timeSec;
|
||||
|
||||
// Rate of change, itself low-passed, drives the adaptive cutoff.
|
||||
const dRaw = this.signal.initialized ? (x - this.lastRaw) / dt : 0;
|
||||
this.lastRaw = x;
|
||||
const edRaw = this.derivative.filter(dRaw, smoothingAlpha(this.dCutoff, dt));
|
||||
|
||||
const cutoff = this.minCutoff + this.beta * Math.abs(edRaw);
|
||||
return this.signal.filter(x, smoothingAlpha(cutoff, dt));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Discrete gesture → command detection.
|
||||
//
|
||||
// Maps the recognizer's canned gesture categories onto Codeman commands, with
|
||||
// debouncing so each command fires once per gesture *entry* (not every frame
|
||||
// while it's held). Open_Palm is special: it must be held continuously for
|
||||
// `palmHoldMs` before firing halt-all — a dead-man's-switch that's hard to
|
||||
// trigger by accident, since pausing every session is a big hammer.
|
||||
|
||||
import type { CommandName } from './types.ts';
|
||||
|
||||
interface CommandSpec {
|
||||
name: CommandName;
|
||||
/** If set, the gesture must be held this long (ms) before it fires. */
|
||||
holdMs?: number;
|
||||
}
|
||||
|
||||
/** MediaPipe canned gesture category → command. */
|
||||
const GESTURE_COMMANDS: Record<string, CommandSpec> = {
|
||||
Open_Palm: { name: 'halt-all', holdMs: -1 }, // holdMs filled from palmHoldMs
|
||||
Thumb_Up: { name: 'approve' },
|
||||
Victory: { name: 'new-session' },
|
||||
};
|
||||
|
||||
export class CommandDetector {
|
||||
/** gesture category → timestamp (ms) it was first seen in the current hold. */
|
||||
private heldSince = new Map<string, number>();
|
||||
/** gestures that already fired during the current hold (cleared on release). */
|
||||
private fired = new Set<string>();
|
||||
|
||||
constructor(private palmHoldMs = 1000) {}
|
||||
|
||||
reset(): void {
|
||||
this.heldSince.clear();
|
||||
this.fired.clear();
|
||||
}
|
||||
|
||||
private holdMsFor(spec: CommandSpec): number {
|
||||
return spec.holdMs === -1 ? this.palmHoldMs : (spec.holdMs ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed the set of command-gestures currently shown (across all hands).
|
||||
* @returns the commands that fired on this frame (usually empty).
|
||||
*/
|
||||
update(gestures: Set<string>, nowMs: number): CommandName[] {
|
||||
// Forget gestures no longer held, so they can re-fire on the next entry.
|
||||
for (const g of [...this.heldSince.keys()]) {
|
||||
if (!gestures.has(g)) {
|
||||
this.heldSince.delete(g);
|
||||
this.fired.delete(g);
|
||||
}
|
||||
}
|
||||
|
||||
const fired: CommandName[] = [];
|
||||
for (const g of gestures) {
|
||||
const spec = GESTURE_COMMANDS[g];
|
||||
if (!spec) continue;
|
||||
|
||||
const since = this.heldSince.get(g) ?? nowMs;
|
||||
if (!this.heldSince.has(g)) this.heldSince.set(g, since);
|
||||
if (this.fired.has(g)) continue;
|
||||
|
||||
if (nowMs - since >= this.holdMsFor(spec)) {
|
||||
fired.push(spec.name);
|
||||
this.fired.add(g);
|
||||
}
|
||||
}
|
||||
return fired;
|
||||
}
|
||||
|
||||
/** 0–1 charge of the held halt-all gesture (1 once fired, 0 when released). */
|
||||
haltProgress(nowMs: number): number {
|
||||
const since = this.heldSince.get('Open_Palm');
|
||||
if (since === undefined) return 0;
|
||||
if (this.fired.has('Open_Palm')) return 1;
|
||||
return Math.min(1, (nowMs - since) / this.palmHoldMs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// MediaPipe hand landmark indices and the bone connections between them.
|
||||
// See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
|
||||
//
|
||||
// 21 landmarks per hand, each normalized to [0,1] in image space.
|
||||
|
||||
export const LANDMARK = {
|
||||
WRIST: 0,
|
||||
THUMB_CMC: 1,
|
||||
THUMB_MCP: 2,
|
||||
THUMB_IP: 3,
|
||||
THUMB_TIP: 4,
|
||||
INDEX_MCP: 5,
|
||||
INDEX_PIP: 6,
|
||||
INDEX_DIP: 7,
|
||||
INDEX_TIP: 8,
|
||||
MIDDLE_MCP: 9,
|
||||
MIDDLE_PIP: 10,
|
||||
MIDDLE_DIP: 11,
|
||||
MIDDLE_TIP: 12,
|
||||
RING_MCP: 13,
|
||||
RING_PIP: 14,
|
||||
RING_DIP: 15,
|
||||
RING_TIP: 16,
|
||||
PINKY_MCP: 17,
|
||||
PINKY_PIP: 18,
|
||||
PINKY_DIP: 19,
|
||||
PINKY_TIP: 20,
|
||||
} as const;
|
||||
|
||||
/** Pairs of landmark indices that form the hand skeleton, for overlay drawing. */
|
||||
export const HAND_CONNECTIONS: ReadonlyArray<readonly [number, number]> = [
|
||||
// Thumb
|
||||
[0, 1],
|
||||
[1, 2],
|
||||
[2, 3],
|
||||
[3, 4],
|
||||
// Index
|
||||
[0, 5],
|
||||
[5, 6],
|
||||
[6, 7],
|
||||
[7, 8],
|
||||
// Middle
|
||||
[5, 9],
|
||||
[9, 10],
|
||||
[10, 11],
|
||||
[11, 12],
|
||||
// Ring
|
||||
[9, 13],
|
||||
[13, 14],
|
||||
[14, 15],
|
||||
[15, 16],
|
||||
// Pinky
|
||||
[13, 17],
|
||||
[17, 18],
|
||||
[18, 19],
|
||||
[19, 20],
|
||||
// Palm base
|
||||
[0, 17],
|
||||
];
|
||||
|
||||
export interface NormalizedLandmark {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
visibility?: number;
|
||||
}
|
||||
|
||||
/** Euclidean distance between two normalized landmarks (x/y plane). */
|
||||
export function dist2d(a: NormalizedLandmark, b: NormalizedLandmark): number {
|
||||
const dx = a.x - b.x;
|
||||
const dy = a.y - b.y;
|
||||
return Math.hypot(dx, dy);
|
||||
}
|
||||
|
||||
/** Midpoint of two normalized landmarks (x/y plane). */
|
||||
export function midpoint(a: NormalizedLandmark, b: NormalizedLandmark): { x: number; y: number } {
|
||||
return { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 };
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Pinch detection from hand landmarks.
|
||||
//
|
||||
// Distance between thumb tip (4) and index tip (8), normalized by hand size
|
||||
// (wrist 0 → middle-finger MCP 9) so the threshold is robust to how close the
|
||||
// hand is to the camera. Hysteresis + N-frame persistence keep the grab/release
|
||||
// edge from flickering — critical for not "dropping" a tab mid-drag.
|
||||
|
||||
import { LANDMARK, dist2d, type NormalizedLandmark } from './landmarks.ts';
|
||||
|
||||
/**
|
||||
* Thumb-tip→index-tip distance as a fraction of hand size. Smaller = more
|
||||
* pinched. Roughly in [0, ~1.5]; ~0.35 is a firm pinch, ~0.5+ is open.
|
||||
*/
|
||||
export function pinchDistance(landmarks: NormalizedLandmark[]): number {
|
||||
const thumb = landmarks[LANDMARK.THUMB_TIP];
|
||||
const index = landmarks[LANDMARK.INDEX_TIP];
|
||||
const wrist = landmarks[LANDMARK.WRIST];
|
||||
const middleMcp = landmarks[LANDMARK.MIDDLE_MCP];
|
||||
const handSize = dist2d(wrist, middleMcp) || 1e-6;
|
||||
return dist2d(thumb, index) / handSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks pinch state with two thresholds (hysteresis) and a persistence count.
|
||||
* Enter a pinch below `onThreshold`; leave it only above `offThreshold`
|
||||
* (offThreshold > onThreshold). A flip must hold for `persistFrames` frames
|
||||
* before it commits, rejecting single-frame noise.
|
||||
*/
|
||||
export class PinchDetector {
|
||||
private pinching = false;
|
||||
private pendingFrames = 0;
|
||||
|
||||
constructor(
|
||||
private onThreshold = 0.35,
|
||||
private offThreshold = 0.5,
|
||||
private persistFrames = 2
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.pinching = false;
|
||||
this.pendingFrames = 0;
|
||||
}
|
||||
|
||||
get isPinching(): boolean {
|
||||
return this.pinching;
|
||||
}
|
||||
|
||||
/** Feed one frame's distance. Returns true if the committed state CHANGED. */
|
||||
update(distance: number): boolean {
|
||||
const target = this.pinching
|
||||
? distance < this.offThreshold // stay pinched until the hand opens wide
|
||||
: distance < this.onThreshold; // start pinching once fingers close
|
||||
|
||||
if (target === this.pinching) {
|
||||
this.pendingFrames = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
this.pendingFrames += 1;
|
||||
if (this.pendingFrames >= this.persistFrames) {
|
||||
this.pinching = target;
|
||||
this.pendingFrames = 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Shared types for the gesture input layer.
|
||||
//
|
||||
// Phase 0/1 scope: only `status` and `results` (debug) events are emitted yet.
|
||||
// The semantic events (hover/grab/drag/drop/command) are declared here so the
|
||||
// public API shape is stable, but they are wired in later phases.
|
||||
|
||||
import type { GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
|
||||
export interface GestureControllerOptions {
|
||||
/** The <video> element the camera stream is attached to. */
|
||||
video: HTMLVideoElement;
|
||||
/** Element whose bounding rect normalized coords are mapped against. Defaults to video. (Used from Phase 2.) */
|
||||
surface?: HTMLElement;
|
||||
/** Number of hands to track. v1 = 1. */
|
||||
numHands?: number;
|
||||
/** Pinch hysteresis thresholds (fractions of hand-size). Used from Phase 2. */
|
||||
pinchOn?: number;
|
||||
pinchOff?: number;
|
||||
/** One-Euro cursor smoothing. Lower minCutoff = smoother/laggier when still;
|
||||
* higher beta = less lag during fast moves. Used from Phase 2. */
|
||||
minCutoff?: number;
|
||||
beta?: number;
|
||||
/** How long Open_Palm must be held to fire halt-all. Used from Phase 4. */
|
||||
palmHoldMs?: number;
|
||||
/** Specific camera to open (from enumerateDevices). Empty = default facingMode "user". */
|
||||
deviceId?: string;
|
||||
/** MediaPipe detection/tracking confidence. */
|
||||
minDetectionConfidence?: number;
|
||||
minTrackingConfidence?: number;
|
||||
/** CDN base used to load the wasm fileset + .task model. */
|
||||
wasmBase?: string;
|
||||
modelUrl?: string;
|
||||
}
|
||||
|
||||
export type CommandName = 'halt-all' | 'approve' | 'new-session';
|
||||
|
||||
/** Per-hand state for the current frame. */
|
||||
export interface HandState {
|
||||
/** "Left" / "Right" as reported by MediaPipe (image-space). Used to key filters. */
|
||||
handedness: string;
|
||||
/** Smoothed cursor in raw (unmirrored) normalized [0,1] coords; consumers mirror X. */
|
||||
cursor: { x: number; y: number };
|
||||
/** Thumb/index distance as a fraction of hand size. */
|
||||
pinchDist: number;
|
||||
/** Whether this hand is currently pinching (after hysteresis). */
|
||||
pinching: boolean;
|
||||
/** Top gesture category for this hand, if any. */
|
||||
gesture: string | null;
|
||||
}
|
||||
|
||||
/** A pointer sample for one hand, in surface pixel coords (origin = surface
|
||||
* top-left, X already mirrored to match the displayed video). */
|
||||
export interface HandPointer {
|
||||
hand: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/** Payloads emitted per event name. */
|
||||
export interface GestureEventMap {
|
||||
/** Pinch just closed — start of a drag. */
|
||||
grab: HandPointer;
|
||||
/** Cursor moved while pinched — emitted every frame during a drag. */
|
||||
drag: HandPointer;
|
||||
/** Pinch released (or the hand vanished mid-pinch) — end of a drag. */
|
||||
drop: HandPointer;
|
||||
command: { name: CommandName };
|
||||
status: {
|
||||
fps: number;
|
||||
/** One entry per detected hand (0–`numHands`). */
|
||||
hands: HandState[];
|
||||
/** 0–1 charge of the held Open_Palm halt-all gesture (Phase 4). */
|
||||
haltProgress: number;
|
||||
};
|
||||
/** Debug-only: the raw recognizer result for the current frame (drives the overlay). */
|
||||
results: { result: GestureRecognizerResult; timestampMs: number };
|
||||
}
|
||||
|
||||
export type GestureEventName = keyof GestureEventMap;
|
||||
export type GestureEventHandler<K extends GestureEventName> = (payload: GestureEventMap[K]) => void;
|
||||
Reference in New Issue
Block a user