mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-07 07:59:42 +02:00
feat: vendor gesture-control source into packages/gesture-control
Bring the Ark0N/codeman-gesture-control repo in-tree as the codeman-gesture-control workspace package so the hand-tracking overlay can be developed in the Codeman repo. New npm run build:gesture bundles src/codeman/entry.ts into the served gesture-codeman.js; scripts/build.mjs reruns it on every production build. Source formatted to Codeman's prettier style. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
arkon
@@ -0,0 +1,518 @@
|
||||
// Phase 5 — Codeman integration entry point.
|
||||
//
|
||||
// This is the *consumer* layer that replaces `demo/tabs.ts` for the real
|
||||
// Codeman dashboard. It is bundled (esbuild, MediaPipe included) into a single
|
||||
// ESM file and served by Codeman from `/gesture/gesture-codeman.js`, loaded into
|
||||
// the dashboard page when Codeman is started with `CODEMAN_GESTURE=1`.
|
||||
//
|
||||
// The gesture *core* (`../gesture/*`) is unchanged and transport-agnostic — it
|
||||
// emits coordinate-only `grab`/`drag`/`drop`. Here we map those onto Codeman's
|
||||
// real session tabs (`.session-tab[data-id]`) and toolbar buttons.
|
||||
//
|
||||
// Three interactions, all off the same pinch:
|
||||
// • Tab "grab-to-float" — pinch a session tab, *pull it out* of the strip (a
|
||||
// ghost follows your hand), release past a threshold → the session opens as
|
||||
// an in-page floating panel at the drop point; a small twitch-and-release
|
||||
// cancels (snaps back).
|
||||
// • Panel "re-grab" — pinch an existing floating panel and move it anywhere;
|
||||
// release over the tab strip to re-dock it (panel goes away, the tab stays).
|
||||
// This is the capability the old OS-window detach lost.
|
||||
// • Button "tap" — pinch over a toolbar button (Run / Run Shell) and release
|
||||
// in place → fires the button's real click handler. Drift too far first and
|
||||
// it's treated as a stray move, not a tap.
|
||||
//
|
||||
// Why in-page floats, not OS-window detach (decided 2026-06-08, see
|
||||
// docs/MULTIMONITOR_DESIGN.md): the hand only exists in the one page that owns
|
||||
// the camera. `window.open`/`detachSession` puts the session in a sealed OS
|
||||
// window the page can't hand-track — a detached session can never be moved
|
||||
// again, a one-way trip. A floating panel (an iframe of the session's solo
|
||||
// route `/session/:id`) stays in this page's DOM, so the hand keeps control:
|
||||
// re-grab, move across the (spanned) viewport, drop, re-dock. The OS-window
|
||||
// detach is kept for later as a *separate, deliberate* verb — no longer the pinch.
|
||||
|
||||
import { GestureController } from '../gesture/GestureController.ts';
|
||||
import type { HandState } from '../gesture/types.ts';
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
__codemanGesture?: GestureBridge;
|
||||
}
|
||||
}
|
||||
|
||||
const TAB_SELECTOR = '.session-tab';
|
||||
/** An in-page floating session panel this layer spawned — re-grabbable to move. */
|
||||
const PANEL_SELECTOR = '.cg-float';
|
||||
/** The session-tab strip; dropping a moved panel over it re-docks the session. */
|
||||
const DOCK_SELECTOR = '.session-tabs';
|
||||
/** Toolbar buttons a pinch can "tap": Run (#runBtn → app.run()) and Run Shell
|
||||
* (.btn-shell → app.runShell()). Pinch over one and release in place to fire
|
||||
* it. Extend this list to expose more buttons to the gesture layer. */
|
||||
const CLICK_SELECTOR = '#runBtn, .btn-shell';
|
||||
const Z = 2147483000; // above the dashboard, below nothing that matters at the desk
|
||||
/** Floating-panel size (px). Fixed for the MVP; resize is a later affordance. */
|
||||
const FLOAT_W = 640;
|
||||
const FLOAT_H = 420;
|
||||
/** Minimum pull distance (px) from the grab point before a release floats the
|
||||
* tab out. Below this it's an accidental pinch and the tab snaps back. */
|
||||
const DETACH_PULL_PX = 70;
|
||||
/** If a button-pinch drifts more than this, it's a stray move, not a tap. */
|
||||
const TAP_CANCEL_PX = 45;
|
||||
|
||||
/** First letter colours: cyan left, violet right; green while pinching. */
|
||||
const handColor = (handedness: string, pinching: boolean): string =>
|
||||
pinching ? '#4ade80' : handedness === 'Right' ? '#a78bfa' : '#38bdf8';
|
||||
|
||||
/** A floating in-page session panel (an iframe of `/session/:id`) the hand can
|
||||
* place and re-grab. Stays in this page's DOM, so it never leaves hand reach. */
|
||||
interface FloatingPanel {
|
||||
id: string;
|
||||
/** The `.cg-float` container element. */
|
||||
el: HTMLElement;
|
||||
}
|
||||
|
||||
/** Live state for one hand's in-progress grab — either a session *tab* being
|
||||
* pulled out into a new float, or an existing *panel* being moved/re-docked. */
|
||||
type Grab =
|
||||
| {
|
||||
kind: 'tab';
|
||||
id: string;
|
||||
tab: HTMLElement;
|
||||
ghost: HTMLElement;
|
||||
/** Grab origin in viewport px, to measure pull distance. */
|
||||
ox: number;
|
||||
oy: number;
|
||||
/** Pulled past the float-out threshold at least once. */
|
||||
armed: boolean;
|
||||
}
|
||||
| {
|
||||
kind: 'panel';
|
||||
id: string;
|
||||
panel: FloatingPanel;
|
||||
/** Cursor→panel-top-left offset at grab, so it doesn't snap when re-grabbed. */
|
||||
dx: number;
|
||||
dy: number;
|
||||
/** Cursor currently over the tab strip → releasing re-docks. */
|
||||
overDock: boolean;
|
||||
};
|
||||
|
||||
/** Live state for one hand pinching a toolbar button (Run / Run Shell). */
|
||||
interface Tap {
|
||||
el: HTMLElement;
|
||||
label: string;
|
||||
ox: number;
|
||||
oy: number;
|
||||
}
|
||||
|
||||
class GestureBridge {
|
||||
private readonly surface: HTMLDivElement;
|
||||
private readonly canvas: HTMLCanvasElement;
|
||||
private readonly ctx: CanvasRenderingContext2D;
|
||||
private readonly video: HTMLVideoElement;
|
||||
private readonly button: HTMLButtonElement;
|
||||
private readonly camBtn: HTMLButtonElement;
|
||||
private readonly status: HTMLSpanElement;
|
||||
private readonly gc: GestureController;
|
||||
private running = false;
|
||||
/** Camera view: full-viewport dimmed background, or small corner preview. */
|
||||
private camMode: 'full' | 'pip' = 'full';
|
||||
|
||||
/** Per-hand in-progress grab (a tab being pulled out, or a panel being moved). */
|
||||
private grabs = new Map<string, Grab>();
|
||||
/** Per-hand in-progress button pinch (fires on release if it didn't drift). */
|
||||
private taps = new Map<string, Tap>();
|
||||
/** Live floating panels, keyed by session id (idempotent per id). */
|
||||
private floats = new Map<string, FloatingPanel>();
|
||||
|
||||
constructor() {
|
||||
injectStyles();
|
||||
|
||||
// Full-viewport, click-through surface so coords map straight to viewport
|
||||
// pixels and elementFromPoint() sees the tabs beneath, not our overlay.
|
||||
this.surface = el('div', 'cg-surface') as HTMLDivElement;
|
||||
this.canvas = el('canvas', 'cg-canvas') as HTMLCanvasElement;
|
||||
this.video = el('video', 'cg-preview') as HTMLVideoElement;
|
||||
this.video.muted = true;
|
||||
this.video.playsInline = true;
|
||||
this.applyCamMode();
|
||||
|
||||
const dock = el('div', 'cg-dock');
|
||||
this.button = el('button', 'cg-btn') as HTMLButtonElement;
|
||||
this.button.textContent = '🖐 Gesture';
|
||||
this.camBtn = el('button', 'cg-btn cg-btn-icon') as HTMLButtonElement;
|
||||
this.camBtn.textContent = '⛶';
|
||||
this.camBtn.title = 'Toggle camera size (fullscreen / corner)';
|
||||
this.status = el('span', 'cg-status') as HTMLSpanElement;
|
||||
this.status.textContent = 'off';
|
||||
dock.append(this.button, this.camBtn, this.status);
|
||||
|
||||
document.body.append(this.surface, this.video, this.canvas, dock);
|
||||
this.ctx = this.canvas.getContext('2d')!;
|
||||
this.sizeCanvas();
|
||||
window.addEventListener('resize', () => this.sizeCanvas());
|
||||
|
||||
this.gc = new GestureController({
|
||||
video: this.video,
|
||||
surface: this.surface,
|
||||
numHands: 1,
|
||||
// Self-host the MediaPipe runtime + model from Codeman (same-origin) instead
|
||||
// of the CDN, so an ad/content blocker, offline desk, or strict browser
|
||||
// can't break startup (the CDN failure surfaced as `failed: {isTrusted}` —
|
||||
// a resource load-error Event). Served from public/gesture/.
|
||||
wasmBase: '/gesture/wasm',
|
||||
modelUrl: '/gesture/gesture_recognizer.task',
|
||||
});
|
||||
|
||||
this.gc.on('grab', (p) => this.onGrab(p.hand, p.x, p.y));
|
||||
this.gc.on('drag', (p) => this.onDrag(p.hand, p.x, p.y));
|
||||
this.gc.on('drop', (p) => this.onDrop(p.hand, p.x, p.y));
|
||||
this.gc.on('status', ({ fps, hands }) => this.onStatus(fps, hands));
|
||||
|
||||
this.button.addEventListener('click', () => void this.toggle());
|
||||
this.camBtn.addEventListener('click', () => this.toggleCamMode());
|
||||
}
|
||||
|
||||
private async toggle(): Promise<void> {
|
||||
if (this.running) {
|
||||
this.gc.stop();
|
||||
this.running = false;
|
||||
this.cancelAllGrabs();
|
||||
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.button.classList.remove('on');
|
||||
this.status.textContent = 'off';
|
||||
return;
|
||||
}
|
||||
this.button.disabled = true;
|
||||
this.status.textContent = 'starting…';
|
||||
try {
|
||||
await this.gc.start();
|
||||
this.running = true;
|
||||
this.button.classList.add('on');
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
} catch (err) {
|
||||
// Surface the *real* cause: MediaPipe/Emscripten can throw a non-Error
|
||||
// (number/string), so `(err as Error).message` was logging "undefined".
|
||||
const msg = describeError(err);
|
||||
this.status.textContent = `failed: ${msg}`;
|
||||
this.status.title = msg;
|
||||
console.error('[gesture] start failed', err);
|
||||
} finally {
|
||||
this.button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
private toggleCamMode(): void {
|
||||
this.camMode = this.camMode === 'full' ? 'pip' : 'full';
|
||||
this.applyCamMode();
|
||||
}
|
||||
|
||||
private applyCamMode(): void {
|
||||
this.video.classList.toggle('cg-full', this.camMode === 'full');
|
||||
this.video.classList.toggle('cg-pip', this.camMode === 'pip');
|
||||
}
|
||||
|
||||
/** Top-most element matching `sel` at a viewport point (overlays are
|
||||
* click-through, so elementFromPoint sees the dashboard beneath). */
|
||||
private hitClosest(x: number, y: number, sel: string): HTMLElement | null {
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
return (hit?.closest(sel) as HTMLElement | null) ?? null;
|
||||
}
|
||||
|
||||
private onGrab(hand: string, x: number, y: number): void {
|
||||
// An existing floating panel → re-grab to move it (priority over tabs).
|
||||
// Make it click-through while held so elementFromPoint sees the dock zone
|
||||
// (and other content) beneath it, and it can't re-grab itself.
|
||||
const panelEl = this.hitClosest(x, y, PANEL_SELECTOR);
|
||||
const panelId = panelEl?.dataset.id;
|
||||
if (panelEl && panelId) {
|
||||
const float = this.floats.get(panelId);
|
||||
if (float) {
|
||||
const rect = panelEl.getBoundingClientRect();
|
||||
panelEl.style.pointerEvents = 'none';
|
||||
panelEl.classList.add('cg-float-grabbed');
|
||||
this.grabs.set(hand, {
|
||||
kind: 'panel',
|
||||
id: panelId,
|
||||
panel: float,
|
||||
dx: x - rect.left,
|
||||
dy: y - rect.top,
|
||||
overDock: false,
|
||||
});
|
||||
this.status.textContent = 'moving — drop over tabs to re-dock';
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// A session tab → grab-and-pull-out into a floating panel (ghost follows).
|
||||
const tab = this.hitClosest(x, y, TAB_SELECTOR);
|
||||
const id = tab?.dataset.id;
|
||||
if (tab && id) {
|
||||
const rect = tab.getBoundingClientRect();
|
||||
const ghost = tab.cloneNode(true) as HTMLElement;
|
||||
ghost.classList.add('cg-ghost');
|
||||
ghost.removeAttribute('id');
|
||||
ghost.style.width = `${rect.width}px`;
|
||||
ghost.style.height = `${rect.height}px`;
|
||||
document.body.append(ghost);
|
||||
|
||||
tab.classList.add('cg-grabbed');
|
||||
this.grabs.set(hand, { kind: 'tab', id, tab, ghost, ox: x, oy: y, armed: false });
|
||||
this.positionGhost(ghost, x, y);
|
||||
return;
|
||||
}
|
||||
|
||||
// A toolbar button (Run / Run Shell) → tap-to-fire on release.
|
||||
const btn = this.hitClosest(x, y, CLICK_SELECTOR);
|
||||
if (btn) {
|
||||
const label = (btn.textContent || btn.getAttribute('title') || 'button').trim();
|
||||
btn.classList.add('cg-tap-armed');
|
||||
this.taps.set(hand, { el: btn, label, ox: x, oy: y });
|
||||
this.status.textContent = `release to ${label.toLowerCase()}`;
|
||||
}
|
||||
}
|
||||
|
||||
private onDrag(hand: string, x: number, y: number): void {
|
||||
const grab = this.grabs.get(hand);
|
||||
if (grab?.kind === 'tab') {
|
||||
this.positionGhost(grab.ghost, x, y);
|
||||
const pulled = Math.hypot(x - grab.ox, y - grab.oy) >= DETACH_PULL_PX;
|
||||
if (pulled !== grab.armed) {
|
||||
grab.armed = pulled;
|
||||
grab.ghost.classList.toggle('cg-armed', pulled);
|
||||
this.status.textContent = pulled ? 'release to float out' : 'on — pinch a tab';
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'panel') {
|
||||
this.moveFloat(grab.panel, x - grab.dx, y - grab.dy);
|
||||
const overDock = !!this.hitClosest(x, y, DOCK_SELECTOR);
|
||||
if (overDock !== grab.overDock) {
|
||||
grab.overDock = overDock;
|
||||
grab.panel.el.classList.toggle('cg-redock', overDock);
|
||||
this.status.textContent = overDock ? 'release to re-dock' : 'moving panel';
|
||||
}
|
||||
return;
|
||||
}
|
||||
// A button pinch that drifts too far is a stray move, not a tap — cancel it.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap && Math.hypot(x - tap.ox, y - tap.oy) > TAP_CANCEL_PX) {
|
||||
tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.delete(hand);
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
}
|
||||
}
|
||||
|
||||
private onDrop(hand: string, x: number, y: number): void {
|
||||
const grab = this.grabs.get(hand);
|
||||
if (grab?.kind === 'tab') {
|
||||
this.grabs.delete(hand);
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove('cg-grabbed');
|
||||
if (grab.armed) this.floatPanel(grab.id, x, y);
|
||||
else this.flash('cancelled');
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'panel') {
|
||||
this.grabs.delete(hand);
|
||||
grab.panel.el.style.pointerEvents = ''; // interactive again (type into it)
|
||||
grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock');
|
||||
if (grab.overDock) this.redock(grab.id);
|
||||
else this.flash('placed');
|
||||
return;
|
||||
}
|
||||
// Release over the same button → fire its real click handler.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap) {
|
||||
this.taps.delete(hand);
|
||||
tap.el.classList.remove('cg-tap-armed');
|
||||
tap.el.click(); // runs the button's onclick (app.run() / app.runShell())
|
||||
this.flash(tap.label.toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pop a session into an in-page floating panel (an iframe of its solo route)
|
||||
* centered on the drop point. Unlike OS-window detach, the panel lives in
|
||||
* this page's DOM, so the hand can re-grab and move it. Idempotent per id:
|
||||
* re-floating an existing id just repositions the panel it already has. */
|
||||
private floatPanel(id: string, x: number, y: number): void {
|
||||
let float = this.floats.get(id);
|
||||
if (!float) {
|
||||
const container = el('div', 'cg-float');
|
||||
container.dataset.id = id;
|
||||
const bar = el('div', 'cg-float-bar');
|
||||
bar.textContent = `session ${id}`;
|
||||
const frame = el('iframe', 'cg-float-frame') as HTMLIFrameElement;
|
||||
frame.src = `/session/${encodeURIComponent(id)}`;
|
||||
frame.title = `Session ${id}`;
|
||||
container.append(bar, frame);
|
||||
document.body.append(container);
|
||||
float = { id, el: container };
|
||||
this.floats.set(id, float);
|
||||
this.flash('floated out');
|
||||
} else {
|
||||
this.flash('re-floated');
|
||||
}
|
||||
this.moveFloat(float, x - FLOAT_W / 2, y - FLOAT_H / 2);
|
||||
}
|
||||
|
||||
/** Re-dock a floated session: drop its in-page panel. The original
|
||||
* `.session-tab` was never removed, so the session is simply back to plain
|
||||
* tab form. Mirrors Codeman's own `.detached` → attach toggle. */
|
||||
private redock(id: string): void {
|
||||
const float = this.floats.get(id);
|
||||
if (!float) return;
|
||||
float.el.remove();
|
||||
this.floats.delete(id);
|
||||
this.flash('re-docked');
|
||||
}
|
||||
|
||||
/** Position a float by its top-left corner, clamped to stay on-screen. */
|
||||
private moveFloat(float: FloatingPanel, left: number, top: number): void {
|
||||
const l = Math.min(Math.max(0, left), Math.max(0, window.innerWidth - FLOAT_W));
|
||||
const t = Math.min(Math.max(0, top), Math.max(0, window.innerHeight - FLOAT_H));
|
||||
float.el.style.left = `${l}px`;
|
||||
float.el.style.top = `${t}px`;
|
||||
}
|
||||
|
||||
private positionGhost(ghost: HTMLElement, x: number, y: number): void {
|
||||
ghost.style.left = `${x}px`;
|
||||
ghost.style.top = `${y}px`;
|
||||
}
|
||||
|
||||
private cancelAllGrabs(): void {
|
||||
// Floats themselves persist (they're placed windows) — only release any
|
||||
// in-progress grab cleanly, restoring a moved panel's interactivity.
|
||||
for (const grab of this.grabs.values()) {
|
||||
if (grab.kind === 'tab') {
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove('cg-grabbed');
|
||||
} else {
|
||||
grab.panel.el.style.pointerEvents = '';
|
||||
grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock');
|
||||
}
|
||||
}
|
||||
this.grabs.clear();
|
||||
for (const tap of this.taps.values()) tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.clear();
|
||||
document
|
||||
.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`)
|
||||
.forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed'));
|
||||
}
|
||||
|
||||
private onStatus(fps: number, hands: HandState[]): void {
|
||||
// Draw per-hand cursor dots (green while pinching) so the user can aim.
|
||||
const { width, height } = this.canvas;
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
this.ctx.clearRect(0, 0, width, height);
|
||||
const rect = this.surface.getBoundingClientRect();
|
||||
for (const h of hands) {
|
||||
const x = (1 - h.cursor.x) * rect.width;
|
||||
const y = h.cursor.y * rect.height;
|
||||
this.ctx.beginPath();
|
||||
this.ctx.arc(x * dpr, y * dpr, (h.pinching ? 14 : 9) * dpr, 0, Math.PI * 2);
|
||||
this.ctx.fillStyle = handColor(h.handedness, h.pinching);
|
||||
this.ctx.globalAlpha = 0.85;
|
||||
this.ctx.fill();
|
||||
this.ctx.globalAlpha = 1;
|
||||
}
|
||||
if (this.running && this.grabs.size === 0 && this.taps.size === 0) {
|
||||
this.status.textContent = `on · ${fps}fps`;
|
||||
}
|
||||
}
|
||||
|
||||
private flash(msg: string): void {
|
||||
this.status.textContent = msg;
|
||||
}
|
||||
|
||||
private sizeCanvas(): void {
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
this.canvas.width = Math.floor(window.innerWidth * dpr);
|
||||
this.canvas.height = Math.floor(window.innerHeight * dpr);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- tiny helpers ------------------------------------------------------
|
||||
|
||||
function el(tag: string, className: string): HTMLElement {
|
||||
const node = document.createElement(tag);
|
||||
node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
/** Best-effort human-readable message for any thrown value (Error or not). */
|
||||
function describeError(err: unknown): string {
|
||||
if (err instanceof Error) return err.message || err.name || 'Error';
|
||||
if (typeof err === 'string') return err;
|
||||
if (typeof err === 'number') return `code ${err}`;
|
||||
if (err && typeof err === 'object') {
|
||||
const m = (err as { message?: unknown }).message;
|
||||
if (typeof m === 'string' && m) return m;
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
return Object.prototype.toString.call(err);
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
function injectStyles(): void {
|
||||
if (document.getElementById('cg-styles')) return;
|
||||
const css = `
|
||||
.cg-surface, .cg-canvas { position: fixed; inset: 0; pointer-events: none; }
|
||||
.cg-surface { z-index: ${Z}; }
|
||||
.cg-canvas { z-index: ${Z + 1}; width: 100vw; height: 100vh; }
|
||||
.cg-preview { transform: scaleX(-1); pointer-events: none; background: #000; }
|
||||
.cg-preview.cg-pip {
|
||||
position: fixed; right: 12px; bottom: 12px; width: 240px; height: 135px;
|
||||
object-fit: cover; border-radius: 8px; z-index: ${Z + 2};
|
||||
box-shadow: 0 4px 16px rgba(0,0,0,.5); opacity: 1;
|
||||
}
|
||||
.cg-preview.cg-full {
|
||||
position: fixed; inset: 0; width: 100vw; height: 100vh;
|
||||
object-fit: cover; opacity: .28; z-index: ${Z};
|
||||
}
|
||||
.cg-ghost {
|
||||
position: fixed; left: 0; top: 0; transform: translate(-50%, -50%) scale(1.06);
|
||||
z-index: ${Z + 2}; pointer-events: none; opacity: .92;
|
||||
box-shadow: 0 8px 28px rgba(0,0,0,.55); border-radius: 8px;
|
||||
outline: 2px solid #38bdf8; outline-offset: -2px;
|
||||
}
|
||||
.cg-ghost.cg-armed { outline-color: #4ade80; box-shadow: 0 8px 28px rgba(74,222,128,.5); }
|
||||
.cg-dock {
|
||||
position: fixed; right: 12px; bottom: 156px; z-index: ${Z + 3};
|
||||
display: flex; align-items: center; gap: 8px; font: 12px/1 system-ui, sans-serif;
|
||||
}
|
||||
.cg-btn {
|
||||
padding: 6px 12px; border-radius: 6px; border: 1px solid #3a3a40;
|
||||
background: #1b1b1f; color: #e5e5e7; cursor: pointer;
|
||||
}
|
||||
.cg-btn-icon { padding: 6px 9px; }
|
||||
.cg-btn.on { background: #16331f; border-color: #2f6b41; color: #4ade80; }
|
||||
.cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; }
|
||||
.cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; }
|
||||
.cg-float {
|
||||
position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px;
|
||||
z-index: ${Z}; display: flex; flex-direction: column; overflow: hidden;
|
||||
background: #0c0c0f; border-radius: 10px; outline: 2px solid #38bdf8;
|
||||
outline-offset: -2px; box-shadow: 0 10px 40px rgba(0,0,0,.6);
|
||||
}
|
||||
.cg-float.cg-float-grabbed { outline-color: #4ade80; box-shadow: 0 12px 48px rgba(74,222,128,.45); }
|
||||
.cg-float.cg-redock { outline-color: #fbbf24; box-shadow: 0 12px 48px rgba(251,191,36,.5); }
|
||||
.cg-float-bar {
|
||||
flex: 0 0 auto; padding: 4px 10px; font: 11px/1.6 system-ui, sans-serif;
|
||||
color: #cbd5e1; background: #15151a; border-bottom: 1px solid #2a2a30;
|
||||
user-select: none; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.cg-float-frame { flex: 1 1 auto; width: 100%; border: 0; background: #000; }
|
||||
`;
|
||||
const style = document.createElement('style');
|
||||
style.id = 'cg-styles';
|
||||
style.textContent = css;
|
||||
document.head.append(style);
|
||||
}
|
||||
|
||||
// Idempotent bootstrap — re-importing must not stack overlays.
|
||||
if (!window.__codemanGesture) {
|
||||
window.__codemanGesture = new GestureBridge();
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Debug overlay: draws the 21-point hand skeleton over the (mirrored) video.
|
||||
//
|
||||
// The <video> is mirrored via CSS (transform: scaleX(-1)). To make the drawn
|
||||
// skeleton line up with what you see, we mirror the X coordinate here
|
||||
// (x_draw = (1 - x) * width) rather than CSS-mirroring the canvas — that keeps
|
||||
// any future text we draw readable.
|
||||
|
||||
import type { GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
import { HAND_CONNECTIONS } from '../gesture/landmarks.ts';
|
||||
|
||||
/** A cursor to draw: raw normalized position + base color + pinch state. */
|
||||
export interface CursorMark {
|
||||
x: number;
|
||||
y: number;
|
||||
color: string;
|
||||
pinching: boolean;
|
||||
}
|
||||
|
||||
export class Overlay {
|
||||
private ctx: CanvasRenderingContext2D;
|
||||
|
||||
constructor(
|
||||
private canvas: HTMLCanvasElement,
|
||||
private video: HTMLVideoElement
|
||||
) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error('Could not get 2D context for overlay canvas');
|
||||
this.ctx = ctx;
|
||||
}
|
||||
|
||||
/** Match the canvas backing-store resolution to the displayed video size. */
|
||||
private syncSize(): void {
|
||||
const w = this.video.videoWidth || this.video.clientWidth;
|
||||
const h = this.video.videoHeight || this.video.clientHeight;
|
||||
if (w && h && (this.canvas.width !== w || this.canvas.height !== h)) {
|
||||
this.canvas.width = w;
|
||||
this.canvas.height = h;
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw the detected hands' skeletons plus one smoothed cursor per hand.
|
||||
* Cursor coords are raw normalized [0,1] (mirrored here, like the skeleton).
|
||||
*/
|
||||
draw(result: GestureRecognizerResult, cursors: CursorMark[] = []): void {
|
||||
this.syncSize();
|
||||
const { width: w, height: h } = this.canvas;
|
||||
const ctx = this.ctx;
|
||||
ctx.clearRect(0, 0, w, h);
|
||||
|
||||
const hands = result.landmarks ?? [];
|
||||
for (const landmarks of hands) {
|
||||
// Connections (bones)
|
||||
ctx.strokeStyle = 'rgba(80, 220, 255, 0.9)';
|
||||
ctx.lineWidth = 3;
|
||||
for (const [a, b] of HAND_CONNECTIONS) {
|
||||
const pa = landmarks[a];
|
||||
const pb = landmarks[b];
|
||||
if (!pa || !pb) continue;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo((1 - pa.x) * w, pa.y * h);
|
||||
ctx.lineTo((1 - pb.x) * w, pb.y * h);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
// Joints (points)
|
||||
for (let i = 0; i < landmarks.length; i++) {
|
||||
const p = landmarks[i];
|
||||
const x = (1 - p.x) * w;
|
||||
const y = p.y * h;
|
||||
// Highlight thumb tip (4) + index tip (8) — these drive the cursor later.
|
||||
const isPinchPoint = i === 4 || i === 8;
|
||||
ctx.fillStyle = isPinchPoint ? '#ffd166' : '#ff5d8f';
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, isPinchPoint ? 7 : 4, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
for (const c of cursors) this.drawCursor(c, w, h);
|
||||
}
|
||||
|
||||
/** A ring + crosshair at the cursor; green + filled while pinching. */
|
||||
private drawCursor(cursor: CursorMark, w: number, h: number): void {
|
||||
const ctx = this.ctx;
|
||||
const cx = (1 - cursor.x) * w; // mirror X to match the displayed video
|
||||
const cy = cursor.y * h;
|
||||
const pinching = cursor.pinching;
|
||||
const r = pinching ? 16 : 12;
|
||||
const color = pinching ? '#4ade80' : cursor.color;
|
||||
|
||||
if (pinching) {
|
||||
ctx.fillStyle = 'rgba(74, 222, 128, 0.25)';
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, r, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 3;
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, r, 0, Math.PI * 2);
|
||||
ctx.stroke();
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(cx - r - 5, cy);
|
||||
ctx.lineTo(cx - r + 4, cy);
|
||||
ctx.moveTo(cx + r - 4, cy);
|
||||
ctx.lineTo(cx + r + 5, cy);
|
||||
ctx.moveTo(cx, cy - r - 5);
|
||||
ctx.lineTo(cx, cy - r + 4);
|
||||
ctx.moveTo(cx, cy + r - 4);
|
||||
ctx.lineTo(cx, cy + r + 5);
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
// demo/tabs.ts — Phase 3 fake-tab board.
|
||||
//
|
||||
// 3 columns of draggable "session" tabs, driven entirely by gesture events
|
||||
// (no mouse). The board overlays the camera stage, so surface-pixel coords
|
||||
// from GestureController map straight onto board-local coords.
|
||||
//
|
||||
// Source of truth for which tab is in which column is the DOM itself. The
|
||||
// state machine here is tiny: grab → (drag)* → drop, tracked per hand so two
|
||||
// hands can drag two tabs at once.
|
||||
|
||||
interface Tab {
|
||||
id: string;
|
||||
el: HTMLElement;
|
||||
}
|
||||
|
||||
interface Column {
|
||||
id: string;
|
||||
title: string;
|
||||
el: HTMLElement;
|
||||
list: HTMLElement;
|
||||
}
|
||||
|
||||
interface Grab {
|
||||
tab: Tab;
|
||||
originList: HTMLElement;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
/** A hovering cursor for highlight purposes. */
|
||||
export interface HoverPoint {
|
||||
x: number;
|
||||
y: number;
|
||||
pinching: boolean;
|
||||
}
|
||||
|
||||
const INITIAL: Array<{ id: string; title: string; tabs: string[] }> = [
|
||||
{ id: 'screen-1', title: 'Screen 1', tabs: ['auth-refactor', 'api-tests'] },
|
||||
{ id: 'screen-2', title: 'Screen 2', tabs: ['db-migrate', 'ui-polish', 'docs'] },
|
||||
{ id: 'screen-3', title: 'Screen 3', tabs: ['ci-fix'] },
|
||||
];
|
||||
|
||||
export class TabsBoard {
|
||||
private columns: Column[] = [];
|
||||
private tabs = new Map<string, Tab>();
|
||||
/** hand id → the tab it is currently dragging. */
|
||||
private grabs = new Map<string, Grab>();
|
||||
|
||||
/** @param onDrop notified after a settled drop: (tab, column or null if cancelled). */
|
||||
constructor(
|
||||
private root: HTMLElement,
|
||||
private onDrop?: (tabId: string, columnId: string | null) => void
|
||||
) {
|
||||
this.build();
|
||||
}
|
||||
|
||||
private build(): void {
|
||||
this.root.classList.add('board');
|
||||
for (const col of INITIAL) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'column';
|
||||
el.dataset.col = col.id;
|
||||
|
||||
const header = document.createElement('header');
|
||||
header.textContent = col.title;
|
||||
|
||||
const list = document.createElement('div');
|
||||
list.className = 'tablist';
|
||||
|
||||
el.append(header, list);
|
||||
this.root.append(el);
|
||||
this.columns.push({ id: col.id, title: col.title, el, list });
|
||||
|
||||
for (const id of col.tabs) {
|
||||
const tab = this.makeTab(id);
|
||||
list.append(tab.el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private makeTab(id: string): Tab {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'tab';
|
||||
el.dataset.tab = id;
|
||||
el.textContent = id;
|
||||
const tab: Tab = { id, el };
|
||||
this.tabs.set(id, tab);
|
||||
return tab;
|
||||
}
|
||||
|
||||
// ---- Geometry --------------------------------------------------------
|
||||
|
||||
/** Element rect in board-local coords (origin = board top-left). */
|
||||
private localRect(el: HTMLElement): DOMRect {
|
||||
const r = el.getBoundingClientRect();
|
||||
const base = this.root.getBoundingClientRect();
|
||||
return new DOMRect(r.left - base.left, r.top - base.top, r.width, r.height);
|
||||
}
|
||||
|
||||
/** Hit-test slop (px). Cursor jitter + a small target shouldn't fight you. */
|
||||
private static readonly GRAB_PAD = 18;
|
||||
|
||||
private static contains(r: DOMRect, x: number, y: number, pad = 0): boolean {
|
||||
return x >= r.left - pad && x <= r.right + pad && y >= r.top - pad && y <= r.bottom + pad;
|
||||
}
|
||||
|
||||
private columnAt(x: number, y: number): Column | null {
|
||||
for (const c of this.columns) {
|
||||
if (TabsBoard.contains(this.localRect(c.el), x, y)) return c;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Topmost ungrabbed tab under the point. Nearest-center wins ties so the
|
||||
* padded hit-areas of adjacent tabs resolve to the most likely target. */
|
||||
private tabAt(x: number, y: number): Tab | null {
|
||||
let best: Tab | null = null;
|
||||
let bestDist = Infinity;
|
||||
for (const tab of this.tabs.values()) {
|
||||
if (this.isGrabbed(tab.id)) continue;
|
||||
const r = this.localRect(tab.el);
|
||||
if (!TabsBoard.contains(r, x, y, TabsBoard.GRAB_PAD)) continue;
|
||||
const cx = r.left + r.width / 2;
|
||||
const cy = r.top + r.height / 2;
|
||||
const d = (x - cx) ** 2 + (y - cy) ** 2;
|
||||
if (d < bestDist) {
|
||||
bestDist = d;
|
||||
best = tab;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
private isGrabbed(tabId: string): boolean {
|
||||
for (const g of this.grabs.values()) if (g.tab.id === tabId) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Highlights (driven each frame from the status snapshot) ---------
|
||||
|
||||
/** Recompute hover highlights from the full set of present cursors. */
|
||||
hover(points: HoverPoint[]): void {
|
||||
this.root.querySelectorAll('.col-hot, .tab-hot').forEach((el) => el.classList.remove('col-hot', 'tab-hot'));
|
||||
|
||||
for (const p of points) {
|
||||
this.columnAt(p.x, p.y)?.el.classList.add('col-hot');
|
||||
// Only show a grab affordance when the hand is open (not pinching).
|
||||
if (!p.pinching) this.tabAt(p.x, p.y)?.el.classList.add('tab-hot');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Drag state machine ---------------------------------------------
|
||||
|
||||
grab(hand: string, x: number, y: number): void {
|
||||
if (this.grabs.has(hand)) return;
|
||||
const tab = this.tabAt(x, y);
|
||||
if (!tab) return;
|
||||
|
||||
const rect = this.localRect(tab.el);
|
||||
this.grabs.set(hand, {
|
||||
tab,
|
||||
originList: tab.el.parentElement as HTMLElement,
|
||||
w: rect.width,
|
||||
h: rect.height,
|
||||
});
|
||||
tab.el.classList.add('dragging');
|
||||
tab.el.style.width = `${rect.width}px`;
|
||||
// Reparent the floating tab onto the board root before positioning it.
|
||||
// A `.column` can't be the containing block: its `backdrop-filter` makes it
|
||||
// the containing block for absolutely-positioned children, so our
|
||||
// board-local left/top would be offset by the column's own position (tabs
|
||||
// in the middle/right columns flew off to the right). #board has no
|
||||
// filter/transform, so it's a stable origin that matches moveTo's coords.
|
||||
this.root.append(tab.el);
|
||||
this.moveTo(tab.el, x, y, rect.width, rect.height);
|
||||
}
|
||||
|
||||
drag(hand: string, x: number, y: number): void {
|
||||
const g = this.grabs.get(hand);
|
||||
if (!g) return;
|
||||
this.moveTo(g.tab.el, x, y, g.w, g.h);
|
||||
}
|
||||
|
||||
drop(hand: string, x: number, y: number): void {
|
||||
const g = this.grabs.get(hand);
|
||||
if (!g) return;
|
||||
this.grabs.delete(hand);
|
||||
|
||||
const target = this.columnAt(x, y);
|
||||
const dest = target ? target.list : g.originList;
|
||||
dest.append(g.tab.el);
|
||||
|
||||
g.tab.el.classList.remove('dragging');
|
||||
g.tab.el.style.removeProperty('width');
|
||||
g.tab.el.style.removeProperty('left');
|
||||
g.tab.el.style.removeProperty('top');
|
||||
|
||||
this.onDrop?.(g.tab.id, target ? target.id : null);
|
||||
}
|
||||
|
||||
/** Center the floating tab on the cursor (clamped to the board). */
|
||||
private moveTo(el: HTMLElement, x: number, y: number, w: number, h: number): void {
|
||||
const base = this.root.getBoundingClientRect();
|
||||
const left = Math.max(0, Math.min(x - w / 2, base.width - w));
|
||||
const top = Math.max(0, Math.min(y - h / 2, base.height - h));
|
||||
el.style.left = `${left}px`;
|
||||
el.style.top = `${top}px`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
// GestureController — core input layer.
|
||||
//
|
||||
// Phase 0/1 scope (this file currently implements):
|
||||
// - Open the webcam (getUserMedia) and attach it to a <video>.
|
||||
// - Load MediaPipe GestureRecognizer in VIDEO mode (wasm + .task from CDN).
|
||||
// - Run a requestAnimationFrame loop calling recognizeForVideo().
|
||||
// - Emit `status` (fps / handPresent / gesture) and `results` (raw, debug).
|
||||
//
|
||||
// Later phases add the cursor (One-Euro filtered), pinch hysteresis, the
|
||||
// hover/grab/drag/drop state machine, and the discrete gesture command bus.
|
||||
// The event surface in types.ts already declares those so the API is stable.
|
||||
|
||||
import { FilesetResolver, GestureRecognizer, type GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
import type {
|
||||
GestureControllerOptions,
|
||||
GestureEventHandler,
|
||||
GestureEventMap,
|
||||
GestureEventName,
|
||||
HandState,
|
||||
} from './types.ts';
|
||||
import { LANDMARK, midpoint } from './landmarks.ts';
|
||||
import { OneEuroFilter } from './OneEuroFilter.ts';
|
||||
import { PinchDetector, pinchDistance } from './pinch.ts';
|
||||
import { CommandDetector } from './commands.ts';
|
||||
|
||||
const DEFAULTS = {
|
||||
numHands: 1,
|
||||
deviceId: '',
|
||||
pinchOn: 0.35,
|
||||
pinchOff: 0.5,
|
||||
minCutoff: 1.0,
|
||||
beta: 0.01,
|
||||
palmHoldMs: 1000,
|
||||
minDetectionConfidence: 0.6,
|
||||
minTrackingConfidence: 0.6,
|
||||
// Pinned to the @mediapipe/tasks-vision version in package.json.
|
||||
wasmBase: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.21/wasm',
|
||||
modelUrl:
|
||||
'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
|
||||
} as const;
|
||||
|
||||
interface PerHandState {
|
||||
cursorX: OneEuroFilter;
|
||||
cursorY: OneEuroFilter;
|
||||
pinch: PinchDetector;
|
||||
/** Original handedness label (the map key may be disambiguated). */
|
||||
label: string;
|
||||
/** Last emitted surface-pixel position, so we can drop on a vanished hand. */
|
||||
lastX: number;
|
||||
lastY: number;
|
||||
}
|
||||
|
||||
export class GestureController {
|
||||
private readonly opts: Required<GestureControllerOptions>;
|
||||
private recognizer: GestureRecognizer | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private rafId: number | null = null;
|
||||
private running = false;
|
||||
|
||||
// Timestamps must be strictly increasing for recognizeForVideo.
|
||||
private lastVideoTime = -1;
|
||||
private lastTimestamp = -1;
|
||||
|
||||
// FPS tracking (rolling over a short window).
|
||||
private frameTimes: number[] = [];
|
||||
|
||||
// Per-hand smoothing + pinch state, keyed by handedness ("Left"/"Right") so a
|
||||
// hand keeps its own filters even when MediaPipe reorders the hands array.
|
||||
private readonly handStates = new Map<string, PerHandState>();
|
||||
|
||||
// Discrete gesture → command bus (debounced; Open_Palm held = halt-all).
|
||||
private readonly commands: CommandDetector;
|
||||
|
||||
// Internal storage is intentionally loose; the public on/off/emit signatures
|
||||
// below keep callers fully type-safe per event name.
|
||||
private listeners: Partial<Record<GestureEventName, Set<(payload: unknown) => void>>> = {};
|
||||
|
||||
constructor(options: GestureControllerOptions) {
|
||||
this.opts = {
|
||||
surface: options.surface ?? options.video,
|
||||
numHands: options.numHands ?? DEFAULTS.numHands,
|
||||
deviceId: options.deviceId ?? DEFAULTS.deviceId,
|
||||
pinchOn: options.pinchOn ?? DEFAULTS.pinchOn,
|
||||
pinchOff: options.pinchOff ?? DEFAULTS.pinchOff,
|
||||
minCutoff: options.minCutoff ?? DEFAULTS.minCutoff,
|
||||
beta: options.beta ?? DEFAULTS.beta,
|
||||
palmHoldMs: options.palmHoldMs ?? DEFAULTS.palmHoldMs,
|
||||
minDetectionConfidence: options.minDetectionConfidence ?? DEFAULTS.minDetectionConfidence,
|
||||
minTrackingConfidence: options.minTrackingConfidence ?? DEFAULTS.minTrackingConfidence,
|
||||
wasmBase: options.wasmBase ?? DEFAULTS.wasmBase,
|
||||
modelUrl: options.modelUrl ?? DEFAULTS.modelUrl,
|
||||
video: options.video,
|
||||
};
|
||||
|
||||
this.commands = new CommandDetector(this.opts.palmHoldMs);
|
||||
}
|
||||
|
||||
/** Get (or lazily create) the smoothing + pinch state for one hand. */
|
||||
private handState(key: string): PerHandState {
|
||||
let state = this.handStates.get(key);
|
||||
if (!state) {
|
||||
state = {
|
||||
cursorX: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
cursorY: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
pinch: new PinchDetector(this.opts.pinchOn, this.opts.pinchOff),
|
||||
label: key,
|
||||
lastX: 0,
|
||||
lastY: 0,
|
||||
};
|
||||
this.handStates.set(key, state);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ---- Event emitter ---------------------------------------------------
|
||||
|
||||
on<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
(this.listeners[event] ??= new Set()).add(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
off<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
this.listeners[event]?.delete(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
private emit<K extends GestureEventName>(event: K, payload: GestureEventMap[K]): void {
|
||||
this.listeners[event]?.forEach((h) => h(payload));
|
||||
}
|
||||
|
||||
// ---- Lifecycle -------------------------------------------------------
|
||||
|
||||
/** Request the camera, load the model, and start the recognition loop. */
|
||||
async start(): Promise<void> {
|
||||
if (this.running) return;
|
||||
|
||||
await this.openCamera();
|
||||
await this.loadRecognizer();
|
||||
|
||||
this.running = true;
|
||||
this.lastVideoTime = -1;
|
||||
this.lastTimestamp = -1;
|
||||
this.frameTimes = [];
|
||||
this.handStates.clear();
|
||||
this.commands.reset();
|
||||
this.loop();
|
||||
}
|
||||
|
||||
/** Stop the loop, release the camera, and close the recognizer. */
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
if (this.rafId !== null) {
|
||||
cancelAnimationFrame(this.rafId);
|
||||
this.rafId = null;
|
||||
}
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
this.opts.video.srcObject = null;
|
||||
this.recognizer?.close();
|
||||
this.recognizer = null;
|
||||
}
|
||||
|
||||
/** The camera currently in use (resolved deviceId), or "" if not started. */
|
||||
get deviceId(): string {
|
||||
return this.opts.deviceId;
|
||||
}
|
||||
|
||||
/** List available video input devices. Labels are only populated once the
|
||||
* user has granted camera permission (i.e. after the first start()). */
|
||||
async listCameras(): Promise<MediaDeviceInfo[]> {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return devices.filter((d) => d.kind === 'videoinput');
|
||||
}
|
||||
|
||||
/** Switch to a different camera. Reopens the stream live if already running. */
|
||||
async useCamera(deviceId: string): Promise<void> {
|
||||
this.opts.deviceId = deviceId;
|
||||
if (!this.running) return;
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
await this.openCamera();
|
||||
// Fresh camera = fresh geometry; drop stale filter state to avoid a snap.
|
||||
this.handStates.clear();
|
||||
this.lastVideoTime = -1;
|
||||
}
|
||||
|
||||
// ---- Setup -----------------------------------------------------------
|
||||
|
||||
private async openCamera(): Promise<void> {
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('getUserMedia is not available (needs a secure context).');
|
||||
}
|
||||
const videoConstraints: MediaTrackConstraints = {
|
||||
width: { ideal: 1280 },
|
||||
height: { ideal: 720 },
|
||||
frameRate: { ideal: 60, max: 60 },
|
||||
};
|
||||
// A specific device wins; otherwise ask for the user-facing camera.
|
||||
if (this.opts.deviceId) videoConstraints.deviceId = { exact: this.opts.deviceId };
|
||||
else videoConstraints.facingMode = 'user';
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: videoConstraints,
|
||||
audio: false,
|
||||
});
|
||||
this.stream = stream;
|
||||
// Record the resolved device so callers can pre-select it in a UI.
|
||||
this.opts.deviceId = stream.getVideoTracks()[0]?.getSettings().deviceId ?? this.opts.deviceId;
|
||||
const video = this.opts.video;
|
||||
video.srcObject = stream;
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
await video.play();
|
||||
|
||||
// Wait until dimensions are known so the overlay can size itself.
|
||||
if (!video.videoWidth) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const onMeta = () => {
|
||||
video.removeEventListener('loadedmetadata', onMeta);
|
||||
resolve();
|
||||
};
|
||||
video.addEventListener('loadedmetadata', onMeta);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async loadRecognizer(): Promise<void> {
|
||||
const fileset = await FilesetResolver.forVisionTasks(this.opts.wasmBase);
|
||||
const build = (delegate: 'GPU' | 'CPU') =>
|
||||
GestureRecognizer.createFromOptions(fileset, {
|
||||
baseOptions: {
|
||||
modelAssetPath: this.opts.modelUrl,
|
||||
delegate,
|
||||
},
|
||||
runningMode: 'VIDEO',
|
||||
numHands: this.opts.numHands,
|
||||
minHandDetectionConfidence: this.opts.minDetectionConfidence,
|
||||
minHandPresenceConfidence: this.opts.minDetectionConfidence,
|
||||
minTrackingConfidence: this.opts.minTrackingConfidence,
|
||||
});
|
||||
// GPU is the fast path (60fps on the MacBook). But some environments fail to
|
||||
// init the GPU delegate — and MediaPipe/Emscripten often throws a *non-Error*
|
||||
// value there (a raw number/string), which surfaces upstream as a useless
|
||||
// "failed: undefined". Fall back to CPU so the recognizer still starts.
|
||||
try {
|
||||
this.recognizer = await build('GPU');
|
||||
} catch (err) {
|
||||
console.warn('[gesture] GPU delegate failed; falling back to CPU.', err);
|
||||
this.recognizer = await build('CPU');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Loop ------------------------------------------------------------
|
||||
|
||||
private loop = (): void => {
|
||||
if (!this.running || !this.recognizer) return;
|
||||
this.rafId = requestAnimationFrame(this.loop);
|
||||
|
||||
const video = this.opts.video;
|
||||
if (video.readyState < 2 /* HAVE_CURRENT_DATA */) return;
|
||||
|
||||
// Strictly increasing timestamp in ms.
|
||||
let ts = performance.now();
|
||||
if (ts <= this.lastTimestamp) ts = this.lastTimestamp + 1;
|
||||
this.lastTimestamp = ts;
|
||||
|
||||
// Only re-run inference when the video frame actually advanced.
|
||||
if (video.currentTime === this.lastVideoTime) return;
|
||||
this.lastVideoTime = video.currentTime;
|
||||
|
||||
let result: GestureRecognizerResult;
|
||||
try {
|
||||
result = this.recognizer.recognizeForVideo(video, ts);
|
||||
} catch (err) {
|
||||
console.error('recognizeForVideo failed', err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.trackFps(ts);
|
||||
this.publish(result, ts);
|
||||
};
|
||||
|
||||
private trackFps(nowMs: number): void {
|
||||
this.frameTimes.push(nowMs);
|
||||
const windowStart = nowMs - 1000;
|
||||
while (this.frameTimes.length && this.frameTimes[0] < windowStart) {
|
||||
this.frameTimes.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private get fps(): number {
|
||||
if (this.frameTimes.length < 2) return 0;
|
||||
const span = this.frameTimes[this.frameTimes.length - 1] - this.frameTimes[0];
|
||||
if (span <= 0) return 0;
|
||||
return Math.round(((this.frameTimes.length - 1) / span) * 1000);
|
||||
}
|
||||
|
||||
private publish(result: GestureRecognizerResult, ts: number): void {
|
||||
const allLandmarks = result.landmarks ?? [];
|
||||
const handedness = result.handedness ?? [];
|
||||
const gestures = result.gestures ?? [];
|
||||
const tSec = ts / 1000;
|
||||
|
||||
// Surface rect → maps normalized [0,1] coords to surface pixels (X mirrored).
|
||||
const rect = this.opts.surface.getBoundingClientRect();
|
||||
|
||||
const hands: HandState[] = [];
|
||||
const activeKeys = new Set<string>();
|
||||
|
||||
for (let i = 0; i < allLandmarks.length; i++) {
|
||||
const lm = allLandmarks[i];
|
||||
if (!lm || lm.length === 0) continue;
|
||||
|
||||
// Key by handedness so each hand keeps its own filters across frames.
|
||||
// Fall back to index, and disambiguate if both hands share a label.
|
||||
const label = handedness[i]?.[0]?.categoryName ?? `hand${i}`;
|
||||
let key = label;
|
||||
if (activeKeys.has(key)) key = `${label}#${i}`;
|
||||
activeKeys.add(key);
|
||||
|
||||
const state = this.handState(key);
|
||||
const mid = midpoint(lm[LANDMARK.THUMB_TIP], lm[LANDMARK.INDEX_TIP]);
|
||||
const cursor = {
|
||||
x: state.cursorX.filter(mid.x, tSec),
|
||||
y: state.cursorY.filter(mid.y, tSec),
|
||||
};
|
||||
const pinchDist = pinchDistance(lm);
|
||||
const changed = state.pinch.update(pinchDist);
|
||||
const gesture = gestures[i]?.[0]?.categoryName ?? null;
|
||||
|
||||
// Pinch state machine → discrete drag events in surface pixels.
|
||||
const sx = (1 - cursor.x) * rect.width;
|
||||
const sy = cursor.y * rect.height;
|
||||
state.lastX = sx;
|
||||
state.lastY = sy;
|
||||
const pointer = { hand: label, x: sx, y: sy };
|
||||
if (state.pinch.isPinching) {
|
||||
this.emit(changed ? 'grab' : 'drag', pointer);
|
||||
} else if (changed) {
|
||||
this.emit('drop', pointer);
|
||||
}
|
||||
|
||||
hands.push({
|
||||
handedness: label,
|
||||
cursor,
|
||||
pinchDist,
|
||||
pinching: state.pinch.isPinching,
|
||||
gesture: gesture && gesture !== 'None' ? gesture : null,
|
||||
});
|
||||
}
|
||||
|
||||
// Drop state for hands that vanished, so a returning hand starts fresh
|
||||
// (no snap from a stale filter position) and the map can't grow unbounded.
|
||||
// If a vanished hand was mid-pinch, emit a drop so nothing stays grabbed.
|
||||
for (const key of [...this.handStates.keys()]) {
|
||||
if (activeKeys.has(key)) continue;
|
||||
const stale = this.handStates.get(key)!;
|
||||
if (stale.pinch.isPinching) {
|
||||
this.emit('drop', { hand: stale.label, x: stale.lastX, y: stale.lastY });
|
||||
}
|
||||
this.handStates.delete(key);
|
||||
}
|
||||
|
||||
// Discrete commands come from open-hand gestures; ignore a hand that's
|
||||
// pinching (mid-drag) so a drag can't be misread as a command.
|
||||
const commandGestures = new Set<string>();
|
||||
for (const h of hands) {
|
||||
if (!h.pinching && h.gesture) commandGestures.add(h.gesture);
|
||||
}
|
||||
for (const name of this.commands.update(commandGestures, ts)) {
|
||||
this.emit('command', { name });
|
||||
}
|
||||
|
||||
// Status first so the overlay can read this frame's cursors in `results`.
|
||||
this.emit('status', {
|
||||
fps: this.fps,
|
||||
hands,
|
||||
haltProgress: this.commands.haltProgress(ts),
|
||||
});
|
||||
this.emit('results', { result, timestampMs: ts });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// One-Euro filter — low-latency smoothing for noisy interactive signals.
|
||||
// Heavy smoothing when the value is still (kills jitter), low lag when it moves
|
||||
// fast. The correct tool for raw landmark streams, which jitter several pixels
|
||||
// even when the hand is held still.
|
||||
//
|
||||
// Reference: Casiez, Roussel, Vogel — "1€ Filter" (CHI 2012),
|
||||
// http://cristal.univ-lille.fr/~casiez/1euro/
|
||||
//
|
||||
// One filter handles a single scalar; use one instance per axis (x, y).
|
||||
|
||||
/** Smoothing factor for a low-pass step given a cutoff (Hz) and timestep (s). */
|
||||
function smoothingAlpha(cutoffHz: number, dtSec: number): number {
|
||||
const tau = 1 / (2 * Math.PI * cutoffHz);
|
||||
return 1 / (1 + tau / dtSec);
|
||||
}
|
||||
|
||||
/** Exponential low-pass that remembers its last output. */
|
||||
class LowPass {
|
||||
private value: number | null = null;
|
||||
|
||||
filter(x: number, alpha: number): number {
|
||||
this.value = this.value === null ? x : alpha * x + (1 - alpha) * this.value;
|
||||
return this.value;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.value = null;
|
||||
}
|
||||
|
||||
get initialized(): boolean {
|
||||
return this.value !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export class OneEuroFilter {
|
||||
private readonly signal = new LowPass();
|
||||
private readonly derivative = new LowPass();
|
||||
private lastTimeSec: number | null = null;
|
||||
private lastRaw = 0;
|
||||
|
||||
/**
|
||||
* @param minCutoff Baseline cutoff (Hz). Lower → smoother but laggier when still.
|
||||
* @param beta Speed coefficient. Higher → less lag during fast moves.
|
||||
* @param dCutoff Cutoff for the derivative low-pass (Hz). 1.0 is fine.
|
||||
*/
|
||||
constructor(
|
||||
private minCutoff = 1.0,
|
||||
private beta = 0.01,
|
||||
private dCutoff = 1.0
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.signal.reset();
|
||||
this.derivative.reset();
|
||||
this.lastTimeSec = null;
|
||||
this.lastRaw = 0;
|
||||
}
|
||||
|
||||
/** @param timeSec strictly-increasing timestamp in seconds. */
|
||||
filter(x: number, timeSec: number): number {
|
||||
let dt = this.lastTimeSec === null ? 1 / 60 : timeSec - this.lastTimeSec;
|
||||
if (dt <= 0) dt = 1 / 60;
|
||||
this.lastTimeSec = timeSec;
|
||||
|
||||
// Rate of change, itself low-passed, drives the adaptive cutoff.
|
||||
const dRaw = this.signal.initialized ? (x - this.lastRaw) / dt : 0;
|
||||
this.lastRaw = x;
|
||||
const edRaw = this.derivative.filter(dRaw, smoothingAlpha(this.dCutoff, dt));
|
||||
|
||||
const cutoff = this.minCutoff + this.beta * Math.abs(edRaw);
|
||||
return this.signal.filter(x, smoothingAlpha(cutoff, dt));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Discrete gesture → command detection.
|
||||
//
|
||||
// Maps the recognizer's canned gesture categories onto Codeman commands, with
|
||||
// debouncing so each command fires once per gesture *entry* (not every frame
|
||||
// while it's held). Open_Palm is special: it must be held continuously for
|
||||
// `palmHoldMs` before firing halt-all — a dead-man's-switch that's hard to
|
||||
// trigger by accident, since pausing every session is a big hammer.
|
||||
|
||||
import type { CommandName } from './types.ts';
|
||||
|
||||
interface CommandSpec {
|
||||
name: CommandName;
|
||||
/** If set, the gesture must be held this long (ms) before it fires. */
|
||||
holdMs?: number;
|
||||
}
|
||||
|
||||
/** MediaPipe canned gesture category → command. */
|
||||
const GESTURE_COMMANDS: Record<string, CommandSpec> = {
|
||||
Open_Palm: { name: 'halt-all', holdMs: -1 }, // holdMs filled from palmHoldMs
|
||||
Thumb_Up: { name: 'approve' },
|
||||
Victory: { name: 'new-session' },
|
||||
};
|
||||
|
||||
export class CommandDetector {
|
||||
/** gesture category → timestamp (ms) it was first seen in the current hold. */
|
||||
private heldSince = new Map<string, number>();
|
||||
/** gestures that already fired during the current hold (cleared on release). */
|
||||
private fired = new Set<string>();
|
||||
|
||||
constructor(private palmHoldMs = 1000) {}
|
||||
|
||||
reset(): void {
|
||||
this.heldSince.clear();
|
||||
this.fired.clear();
|
||||
}
|
||||
|
||||
private holdMsFor(spec: CommandSpec): number {
|
||||
return spec.holdMs === -1 ? this.palmHoldMs : (spec.holdMs ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed the set of command-gestures currently shown (across all hands).
|
||||
* @returns the commands that fired on this frame (usually empty).
|
||||
*/
|
||||
update(gestures: Set<string>, nowMs: number): CommandName[] {
|
||||
// Forget gestures no longer held, so they can re-fire on the next entry.
|
||||
for (const g of [...this.heldSince.keys()]) {
|
||||
if (!gestures.has(g)) {
|
||||
this.heldSince.delete(g);
|
||||
this.fired.delete(g);
|
||||
}
|
||||
}
|
||||
|
||||
const fired: CommandName[] = [];
|
||||
for (const g of gestures) {
|
||||
const spec = GESTURE_COMMANDS[g];
|
||||
if (!spec) continue;
|
||||
|
||||
const since = this.heldSince.get(g) ?? nowMs;
|
||||
if (!this.heldSince.has(g)) this.heldSince.set(g, since);
|
||||
if (this.fired.has(g)) continue;
|
||||
|
||||
if (nowMs - since >= this.holdMsFor(spec)) {
|
||||
fired.push(spec.name);
|
||||
this.fired.add(g);
|
||||
}
|
||||
}
|
||||
return fired;
|
||||
}
|
||||
|
||||
/** 0–1 charge of the held halt-all gesture (1 once fired, 0 when released). */
|
||||
haltProgress(nowMs: number): number {
|
||||
const since = this.heldSince.get('Open_Palm');
|
||||
if (since === undefined) return 0;
|
||||
if (this.fired.has('Open_Palm')) return 1;
|
||||
return Math.min(1, (nowMs - since) / this.palmHoldMs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// MediaPipe hand landmark indices and the bone connections between them.
|
||||
// See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
|
||||
//
|
||||
// 21 landmarks per hand, each normalized to [0,1] in image space.
|
||||
|
||||
export const LANDMARK = {
|
||||
WRIST: 0,
|
||||
THUMB_CMC: 1,
|
||||
THUMB_MCP: 2,
|
||||
THUMB_IP: 3,
|
||||
THUMB_TIP: 4,
|
||||
INDEX_MCP: 5,
|
||||
INDEX_PIP: 6,
|
||||
INDEX_DIP: 7,
|
||||
INDEX_TIP: 8,
|
||||
MIDDLE_MCP: 9,
|
||||
MIDDLE_PIP: 10,
|
||||
MIDDLE_DIP: 11,
|
||||
MIDDLE_TIP: 12,
|
||||
RING_MCP: 13,
|
||||
RING_PIP: 14,
|
||||
RING_DIP: 15,
|
||||
RING_TIP: 16,
|
||||
PINKY_MCP: 17,
|
||||
PINKY_PIP: 18,
|
||||
PINKY_DIP: 19,
|
||||
PINKY_TIP: 20,
|
||||
} as const;
|
||||
|
||||
/** Pairs of landmark indices that form the hand skeleton, for overlay drawing. */
|
||||
export const HAND_CONNECTIONS: ReadonlyArray<readonly [number, number]> = [
|
||||
// Thumb
|
||||
[0, 1],
|
||||
[1, 2],
|
||||
[2, 3],
|
||||
[3, 4],
|
||||
// Index
|
||||
[0, 5],
|
||||
[5, 6],
|
||||
[6, 7],
|
||||
[7, 8],
|
||||
// Middle
|
||||
[5, 9],
|
||||
[9, 10],
|
||||
[10, 11],
|
||||
[11, 12],
|
||||
// Ring
|
||||
[9, 13],
|
||||
[13, 14],
|
||||
[14, 15],
|
||||
[15, 16],
|
||||
// Pinky
|
||||
[13, 17],
|
||||
[17, 18],
|
||||
[18, 19],
|
||||
[19, 20],
|
||||
// Palm base
|
||||
[0, 17],
|
||||
];
|
||||
|
||||
export interface NormalizedLandmark {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
visibility?: number;
|
||||
}
|
||||
|
||||
/** Euclidean distance between two normalized landmarks (x/y plane). */
|
||||
export function dist2d(a: NormalizedLandmark, b: NormalizedLandmark): number {
|
||||
const dx = a.x - b.x;
|
||||
const dy = a.y - b.y;
|
||||
return Math.hypot(dx, dy);
|
||||
}
|
||||
|
||||
/** Midpoint of two normalized landmarks (x/y plane). */
|
||||
export function midpoint(a: NormalizedLandmark, b: NormalizedLandmark): { x: number; y: number } {
|
||||
return { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 };
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Pinch detection from hand landmarks.
|
||||
//
|
||||
// Distance between thumb tip (4) and index tip (8), normalized by hand size
|
||||
// (wrist 0 → middle-finger MCP 9) so the threshold is robust to how close the
|
||||
// hand is to the camera. Hysteresis + N-frame persistence keep the grab/release
|
||||
// edge from flickering — critical for not "dropping" a tab mid-drag.
|
||||
|
||||
import { LANDMARK, dist2d, type NormalizedLandmark } from './landmarks.ts';
|
||||
|
||||
/**
|
||||
* Thumb-tip→index-tip distance as a fraction of hand size. Smaller = more
|
||||
* pinched. Roughly in [0, ~1.5]; ~0.35 is a firm pinch, ~0.5+ is open.
|
||||
*/
|
||||
export function pinchDistance(landmarks: NormalizedLandmark[]): number {
|
||||
const thumb = landmarks[LANDMARK.THUMB_TIP];
|
||||
const index = landmarks[LANDMARK.INDEX_TIP];
|
||||
const wrist = landmarks[LANDMARK.WRIST];
|
||||
const middleMcp = landmarks[LANDMARK.MIDDLE_MCP];
|
||||
const handSize = dist2d(wrist, middleMcp) || 1e-6;
|
||||
return dist2d(thumb, index) / handSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks pinch state with two thresholds (hysteresis) and a persistence count.
|
||||
* Enter a pinch below `onThreshold`; leave it only above `offThreshold`
|
||||
* (offThreshold > onThreshold). A flip must hold for `persistFrames` frames
|
||||
* before it commits, rejecting single-frame noise.
|
||||
*/
|
||||
export class PinchDetector {
|
||||
private pinching = false;
|
||||
private pendingFrames = 0;
|
||||
|
||||
constructor(
|
||||
private onThreshold = 0.35,
|
||||
private offThreshold = 0.5,
|
||||
private persistFrames = 2
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.pinching = false;
|
||||
this.pendingFrames = 0;
|
||||
}
|
||||
|
||||
get isPinching(): boolean {
|
||||
return this.pinching;
|
||||
}
|
||||
|
||||
/** Feed one frame's distance. Returns true if the committed state CHANGED. */
|
||||
update(distance: number): boolean {
|
||||
const target = this.pinching
|
||||
? distance < this.offThreshold // stay pinched until the hand opens wide
|
||||
: distance < this.onThreshold; // start pinching once fingers close
|
||||
|
||||
if (target === this.pinching) {
|
||||
this.pendingFrames = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
this.pendingFrames += 1;
|
||||
if (this.pendingFrames >= this.persistFrames) {
|
||||
this.pinching = target;
|
||||
this.pendingFrames = 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Shared types for the gesture input layer.
|
||||
//
|
||||
// Phase 0/1 scope: only `status` and `results` (debug) events are emitted yet.
|
||||
// The semantic events (hover/grab/drag/drop/command) are declared here so the
|
||||
// public API shape is stable, but they are wired in later phases.
|
||||
|
||||
import type { GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
|
||||
export interface GestureControllerOptions {
|
||||
/** The <video> element the camera stream is attached to. */
|
||||
video: HTMLVideoElement;
|
||||
/** Element whose bounding rect normalized coords are mapped against. Defaults to video. (Used from Phase 2.) */
|
||||
surface?: HTMLElement;
|
||||
/** Number of hands to track. v1 = 1. */
|
||||
numHands?: number;
|
||||
/** Pinch hysteresis thresholds (fractions of hand-size). Used from Phase 2. */
|
||||
pinchOn?: number;
|
||||
pinchOff?: number;
|
||||
/** One-Euro cursor smoothing. Lower minCutoff = smoother/laggier when still;
|
||||
* higher beta = less lag during fast moves. Used from Phase 2. */
|
||||
minCutoff?: number;
|
||||
beta?: number;
|
||||
/** How long Open_Palm must be held to fire halt-all. Used from Phase 4. */
|
||||
palmHoldMs?: number;
|
||||
/** Specific camera to open (from enumerateDevices). Empty = default facingMode "user". */
|
||||
deviceId?: string;
|
||||
/** MediaPipe detection/tracking confidence. */
|
||||
minDetectionConfidence?: number;
|
||||
minTrackingConfidence?: number;
|
||||
/** CDN base used to load the wasm fileset + .task model. */
|
||||
wasmBase?: string;
|
||||
modelUrl?: string;
|
||||
}
|
||||
|
||||
export type CommandName = 'halt-all' | 'approve' | 'new-session';
|
||||
|
||||
/** Per-hand state for the current frame. */
|
||||
export interface HandState {
|
||||
/** "Left" / "Right" as reported by MediaPipe (image-space). Used to key filters. */
|
||||
handedness: string;
|
||||
/** Smoothed cursor in raw (unmirrored) normalized [0,1] coords; consumers mirror X. */
|
||||
cursor: { x: number; y: number };
|
||||
/** Thumb/index distance as a fraction of hand size. */
|
||||
pinchDist: number;
|
||||
/** Whether this hand is currently pinching (after hysteresis). */
|
||||
pinching: boolean;
|
||||
/** Top gesture category for this hand, if any. */
|
||||
gesture: string | null;
|
||||
}
|
||||
|
||||
/** A pointer sample for one hand, in surface pixel coords (origin = surface
|
||||
* top-left, X already mirrored to match the displayed video). */
|
||||
export interface HandPointer {
|
||||
hand: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/** Payloads emitted per event name. */
|
||||
export interface GestureEventMap {
|
||||
/** Pinch just closed — start of a drag. */
|
||||
grab: HandPointer;
|
||||
/** Cursor moved while pinched — emitted every frame during a drag. */
|
||||
drag: HandPointer;
|
||||
/** Pinch released (or the hand vanished mid-pinch) — end of a drag. */
|
||||
drop: HandPointer;
|
||||
command: { name: CommandName };
|
||||
status: {
|
||||
fps: number;
|
||||
/** One entry per detected hand (0–`numHands`). */
|
||||
hands: HandState[];
|
||||
/** 0–1 charge of the held Open_Palm halt-all gesture (Phase 4). */
|
||||
haltProgress: number;
|
||||
};
|
||||
/** Debug-only: the raw recognizer result for the current frame (drives the overlay). */
|
||||
results: { result: GestureRecognizerResult; timestampMs: number };
|
||||
}
|
||||
|
||||
export type GestureEventName = keyof GestureEventMap;
|
||||
export type GestureEventHandler<K extends GestureEventName> = (payload: GestureEventMap[K]) => void;
|
||||
@@ -0,0 +1,188 @@
|
||||
// Phase 3 demo wiring.
|
||||
//
|
||||
// - Start button (camera requires a user gesture) + live camera picker.
|
||||
// - Mirrored video preview + debug overlay (skeleton + per-hand cursors).
|
||||
// - Live HUD: fps / hands / pinch distance / gesture.
|
||||
// - Tab board: pinch-drag session tabs between Screen columns.
|
||||
|
||||
import { GestureController } from './gesture/GestureController.ts';
|
||||
import { Overlay, type CursorMark } from './demo/overlay.ts';
|
||||
import { TabsBoard } from './demo/tabs.ts';
|
||||
import type { HandState } from './gesture/types.ts';
|
||||
|
||||
// Cyan for the left hand, violet for the right; green takes over while pinching.
|
||||
const handColor = (handedness: string): string => (handedness === 'Right' ? '#a78bfa' : '#38bdf8');
|
||||
|
||||
const video = document.getElementById('cam') as HTMLVideoElement;
|
||||
const canvas = document.getElementById('overlay') as HTMLCanvasElement;
|
||||
const stage = document.getElementById('stage') as HTMLDivElement;
|
||||
const boardEl = document.getElementById('board') as HTMLDivElement;
|
||||
const startBtn = document.getElementById('start') as HTMLButtonElement;
|
||||
const stopBtn = document.getElementById('stop') as HTMLButtonElement;
|
||||
const fullscreenBtn = document.getElementById('fullscreen') as HTMLButtonElement;
|
||||
const cameraSel = document.getElementById('camera') as HTMLSelectElement;
|
||||
const fpsEl = document.getElementById('fps') as HTMLSpanElement;
|
||||
const handEl = document.getElementById('hand') as HTMLSpanElement;
|
||||
const pinchEl = document.getElementById('pinch') as HTMLSpanElement;
|
||||
const gestureEl = document.getElementById('gesture') as HTMLSpanElement;
|
||||
const statusEl = document.getElementById('status-msg') as HTMLParagraphElement;
|
||||
|
||||
const overlay = new Overlay(canvas, video);
|
||||
// Coords map against the stage rect (the board overlays it exactly).
|
||||
const gc = new GestureController({ video, surface: stage, numHands: 2 });
|
||||
|
||||
const board = new TabsBoard(boardEl, (tabId, columnId) => {
|
||||
statusEl.textContent = columnId
|
||||
? `Moved “${tabId}” → ${columnId}`
|
||||
: `“${tabId}” dropped outside a column — returned.`;
|
||||
});
|
||||
|
||||
// The board drives off discrete grab/drag/drop; hover highlight off `status`.
|
||||
gc.on('grab', ({ hand, x, y }) => board.grab(hand, x, y));
|
||||
gc.on('drag', ({ hand, x, y }) => board.drag(hand, x, y));
|
||||
gc.on('drop', ({ hand, x, y }) => board.drop(hand, x, y));
|
||||
|
||||
// Discrete gesture commands (halt-all / approve / new-session) are intentionally
|
||||
// not wired up here — this build is pinch-drag only. The controller still emits
|
||||
// `command`/`haltProgress`, but nothing consumes them, so an open palm, thumb,
|
||||
// or victory sign does nothing.
|
||||
|
||||
// Cached from `status` so the `results` handler draws the same frame's cursors.
|
||||
let cursors: CursorMark[] = [];
|
||||
let started = false;
|
||||
|
||||
gc.on('results', ({ result }) => {
|
||||
overlay.draw(result, cursors);
|
||||
});
|
||||
|
||||
gc.on('status', ({ fps, hands }) => {
|
||||
cursors = hands.map((h) => ({
|
||||
x: h.cursor.x,
|
||||
y: h.cursor.y,
|
||||
color: handColor(h.handedness),
|
||||
pinching: h.pinching,
|
||||
}));
|
||||
|
||||
// Hover highlights from the full per-frame snapshot (surface px, X mirrored).
|
||||
const rect = stage.getBoundingClientRect();
|
||||
board.hover(
|
||||
hands.map((h) => ({
|
||||
x: (1 - h.cursor.x) * rect.width,
|
||||
y: h.cursor.y * rect.height,
|
||||
pinching: h.pinching,
|
||||
}))
|
||||
);
|
||||
|
||||
fpsEl.textContent = String(fps);
|
||||
fpsEl.style.color = fps >= 25 ? '#4ade80' : fps > 0 ? '#fbbf24' : '#f87171';
|
||||
|
||||
const anyPinching = hands.some((h) => h.pinching);
|
||||
handEl.textContent = hands.length ? hands.map((h) => h.handedness).join(' + ') : 'no';
|
||||
handEl.style.color = hands.length ? '#4ade80' : '#94a3b8';
|
||||
pinchEl.textContent = hands.length ? hands.map(pinchLabel).join(' ') : '—';
|
||||
pinchEl.style.color = anyPinching ? '#4ade80' : '#94a3b8';
|
||||
gestureEl.textContent =
|
||||
hands
|
||||
.filter((h) => h.gesture)
|
||||
.map((h) => h.gesture)
|
||||
.join(', ') || '—';
|
||||
});
|
||||
|
||||
/** e.g. "L 0.34●" — first letter of handedness, distance, dot when pinching. */
|
||||
function pinchLabel(h: HandState): string {
|
||||
return `${h.handedness[0]} ${h.pinchDist.toFixed(2)}${h.pinching ? '●' : ''}`;
|
||||
}
|
||||
|
||||
startBtn.addEventListener('click', async () => {
|
||||
startBtn.disabled = true;
|
||||
statusEl.textContent = 'Requesting camera + loading model…';
|
||||
try {
|
||||
await gc.start();
|
||||
started = true;
|
||||
statusEl.textContent = 'Running. Pinch-drag tabs between Screens.';
|
||||
stopBtn.disabled = false;
|
||||
await populateCameras();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
statusEl.textContent = `Failed to start: ${(err as Error).message}`;
|
||||
startBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
stopBtn.addEventListener('click', () => {
|
||||
gc.stop();
|
||||
started = false;
|
||||
overlay.clear();
|
||||
board.hover([]);
|
||||
cursors = [];
|
||||
statusEl.textContent = 'Stopped.';
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
cameraSel.disabled = true;
|
||||
fpsEl.textContent = '0';
|
||||
handEl.textContent = 'no';
|
||||
pinchEl.textContent = '—';
|
||||
gestureEl.textContent = '—';
|
||||
});
|
||||
|
||||
// Switch cameras live. On macOS the iPhone (Continuity Camera) shows up as
|
||||
// "iPhone Camera" plus a separate "Desk View Camera" (the ultra-wide lens).
|
||||
cameraSel.addEventListener('change', async () => {
|
||||
statusEl.textContent = `Switching to ${cameraSel.selectedOptions[0]?.text}…`;
|
||||
try {
|
||||
await gc.useCamera(cameraSel.value);
|
||||
statusEl.textContent = 'Running.';
|
||||
// Activating the iPhone can surface its Desk View device — re-check.
|
||||
await populateCameras();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
statusEl.textContent = `Couldn't switch camera: ${(err as Error).message}`;
|
||||
}
|
||||
});
|
||||
|
||||
// Cameras come and go (iPhone mounted/unmounted, Desk View appearing). Keep
|
||||
// the dropdown in sync whenever the device set changes.
|
||||
navigator.mediaDevices?.addEventListener('devicechange', () => {
|
||||
if (started) void populateCameras();
|
||||
});
|
||||
|
||||
// Fullscreen the stage so the board (and grab targets) fill the display.
|
||||
// Standard API with a webkit fallback for Safari.
|
||||
type WebkitEl = HTMLElement & { webkitRequestFullscreen?: () => Promise<void> };
|
||||
type WebkitDoc = Document & {
|
||||
webkitFullscreenElement?: Element | null;
|
||||
webkitExitFullscreen?: () => Promise<void>;
|
||||
};
|
||||
|
||||
function fullscreenElement(): Element | null {
|
||||
return document.fullscreenElement ?? (document as WebkitDoc).webkitFullscreenElement ?? null;
|
||||
}
|
||||
|
||||
fullscreenBtn.addEventListener('click', () => {
|
||||
if (fullscreenElement()) {
|
||||
(document.exitFullscreen ?? (document as WebkitDoc).webkitExitFullscreen)?.call(document);
|
||||
} else {
|
||||
const el = stage as WebkitEl;
|
||||
(el.requestFullscreen ?? el.webkitRequestFullscreen)?.call(el);
|
||||
}
|
||||
});
|
||||
|
||||
function syncFullscreenLabel(): void {
|
||||
fullscreenBtn.textContent = fullscreenElement() ? '⤢ Exit fullscreen' : '⛶ Fullscreen';
|
||||
}
|
||||
document.addEventListener('fullscreenchange', syncFullscreenLabel);
|
||||
document.addEventListener('webkitfullscreenchange', syncFullscreenLabel);
|
||||
|
||||
/** Fill the camera dropdown; labels appear only after camera permission. */
|
||||
async function populateCameras(): Promise<void> {
|
||||
const cams = await gc.listCameras();
|
||||
cameraSel.innerHTML = '';
|
||||
cams.forEach((cam, i) => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = cam.deviceId;
|
||||
opt.text = cam.label || `Camera ${i + 1}`;
|
||||
if (cam.deviceId === gc.deviceId) opt.selected = true;
|
||||
cameraSel.append(opt);
|
||||
});
|
||||
cameraSel.disabled = cams.length < 2;
|
||||
}
|
||||
Reference in New Issue
Block a user