// Phase 5 — Codeman integration entry point. // // This is the *consumer* layer that replaces `demo/tabs.ts` for the real // Codeman dashboard. It is bundled (esbuild, MediaPipe included) into a single // ESM file and served by Codeman from `/gesture/gesture-codeman.js`, loaded into // the dashboard page when Codeman is started with `CODEMAN_GESTURE=1`. // // The gesture *core* (`../gesture/*`) is unchanged and transport-agnostic — it // emits coordinate-only `grab`/`drag`/`drop`. Here we map those onto Codeman's // real session tabs (`.session-tab[data-id]`) and toolbar buttons. // // Three interactions, all off the same pinch: // • Tab "grab-to-float" — pinch a session tab, *pull it out* of the strip (a // ghost follows your hand), release past a threshold → the session opens as // an in-page floating panel at the drop point; a small twitch-and-release // cancels (snaps back). // • Panel "re-grab" — pinch an existing floating panel and move it anywhere; // release over the tab strip to re-dock it (panel goes away, the tab stays). // This is the capability the old OS-window detach lost. // • Button "tap" — pinch over a toolbar button (Run / Run Shell) and release // in place → fires the button's real click handler. Drift too far first and // it's treated as a stray move, not a tap. // // Why in-page floats, not OS-window detach (decided 2026-06-08, see // docs/MULTIMONITOR_DESIGN.md): the hand only exists in the one page that owns // the camera. `window.open`/`detachSession` puts the session in a sealed OS // window the page can't hand-track — a detached session can never be moved // again, a one-way trip. A floating panel (an iframe of the session's solo // route `/session/:id`) stays in this page's DOM, so the hand keeps control: // re-grab, move across the (spanned) viewport, drop, re-dock. The OS-window // detach is kept for later as a *separate, deliberate* verb — no longer the pinch. import { GestureController } from '../gesture/GestureController.ts'; import type { HandState } from '../gesture/types.ts'; declare global { interface Window { __codemanGesture?: GestureBridge; } } const TAB_SELECTOR = '.session-tab'; /** An in-page floating session panel this layer spawned — re-grabbable to move. */ const PANEL_SELECTOR = '.cg-float'; /** The session-tab strip; dropping a moved panel over it re-docks the session. */ const DOCK_SELECTOR = '.session-tabs'; /** Toolbar buttons a pinch can "tap": Run (#runBtn → app.run()) and Run Shell * (.btn-shell → app.runShell()). Pinch over one and release in place to fire * it. Extend this list to expose more buttons to the gesture layer. */ const CLICK_SELECTOR = '#runBtn, .btn-shell'; const Z = 2147483000; // above the dashboard, below nothing that matters at the desk /** Floating-panel size (px). Fixed for the MVP; resize is a later affordance. */ const FLOAT_W = 640; const FLOAT_H = 420; /** Minimum pull distance (px) from the grab point before a release floats the * tab out. Below this it's an accidental pinch and the tab snaps back. */ const DETACH_PULL_PX = 70; /** If a button-pinch drifts more than this, it's a stray move, not a tap. */ const TAP_CANCEL_PX = 45; /** First letter colours: cyan left, violet right; green while pinching. */ const handColor = (handedness: string, pinching: boolean): string => pinching ? '#4ade80' : handedness === 'Right' ? '#a78bfa' : '#38bdf8'; /** A floating in-page session panel (an iframe of `/session/:id`) the hand can * place and re-grab. Stays in this page's DOM, so it never leaves hand reach. */ interface FloatingPanel { id: string; /** The `.cg-float` container element. */ el: HTMLElement; } /** Live state for one hand's in-progress grab — either a session *tab* being * pulled out into a new float, or an existing *panel* being moved/re-docked. */ type Grab = | { kind: 'tab'; id: string; tab: HTMLElement; ghost: HTMLElement; /** Grab origin in viewport px, to measure pull distance. */ ox: number; oy: number; /** Pulled past the float-out threshold at least once. */ armed: boolean; } | { kind: 'panel'; id: string; panel: FloatingPanel; /** Cursor→panel-top-left offset at grab, so it doesn't snap when re-grabbed. */ dx: number; dy: number; /** Cursor currently over the tab strip → releasing re-docks. */ overDock: boolean; }; /** Live state for one hand pinching a toolbar button (Run / Run Shell). */ interface Tap { el: HTMLElement; label: string; ox: number; oy: number; } class GestureBridge { private readonly surface: HTMLDivElement; private readonly canvas: HTMLCanvasElement; private readonly ctx: CanvasRenderingContext2D; private readonly video: HTMLVideoElement; private readonly button: HTMLButtonElement; private readonly camBtn: HTMLButtonElement; private readonly status: HTMLSpanElement; private readonly gc: GestureController; private running = false; /** Camera view: full-viewport dimmed background, or small corner preview. */ private camMode: 'full' | 'pip' = 'full'; /** Per-hand in-progress grab (a tab being pulled out, or a panel being moved). */ private grabs = new Map(); /** Per-hand in-progress button pinch (fires on release if it didn't drift). */ private taps = new Map(); /** Live floating panels, keyed by session id (idempotent per id). */ private floats = new Map(); constructor() { injectStyles(); // Full-viewport, click-through surface so coords map straight to viewport // pixels and elementFromPoint() sees the tabs beneath, not our overlay. this.surface = el('div', 'cg-surface') as HTMLDivElement; this.canvas = el('canvas', 'cg-canvas') as HTMLCanvasElement; this.video = el('video', 'cg-preview') as HTMLVideoElement; this.video.muted = true; this.video.playsInline = true; this.applyCamMode(); const dock = el('div', 'cg-dock'); this.button = el('button', 'cg-btn') as HTMLButtonElement; this.button.textContent = '🖐 Gesture'; this.camBtn = el('button', 'cg-btn cg-btn-icon') as HTMLButtonElement; this.camBtn.textContent = '⛶'; this.camBtn.title = 'Toggle camera size (fullscreen / corner)'; this.status = el('span', 'cg-status') as HTMLSpanElement; this.status.textContent = 'off'; dock.append(this.button, this.camBtn, this.status); document.body.append(this.surface, this.video, this.canvas, dock); this.ctx = this.canvas.getContext('2d')!; this.sizeCanvas(); window.addEventListener('resize', () => this.sizeCanvas()); this.gc = new GestureController({ video: this.video, surface: this.surface, numHands: 1, // Self-host the MediaPipe runtime + model from Codeman (same-origin) instead // of the CDN, so an ad/content blocker, offline desk, or strict browser // can't break startup (the CDN failure surfaced as `failed: {isTrusted}` — // a resource load-error Event). Served from public/gesture/. wasmBase: '/gesture/wasm', modelUrl: '/gesture/gesture_recognizer.task', }); this.gc.on('grab', (p) => this.onGrab(p.hand, p.x, p.y)); this.gc.on('drag', (p) => this.onDrag(p.hand, p.x, p.y)); this.gc.on('drop', (p) => this.onDrop(p.hand, p.x, p.y)); this.gc.on('status', ({ fps, hands }) => this.onStatus(fps, hands)); this.button.addEventListener('click', () => void this.toggle()); this.camBtn.addEventListener('click', () => this.toggleCamMode()); } private async toggle(): Promise { if (this.running) { this.gc.stop(); this.running = false; this.cancelAllGrabs(); this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); this.button.classList.remove('on'); this.status.textContent = 'off'; return; } this.button.disabled = true; this.status.textContent = 'starting…'; try { await this.gc.start(); this.running = true; this.button.classList.add('on'); this.status.textContent = 'on — pinch a tab or button'; } catch (err) { // Surface the *real* cause: MediaPipe/Emscripten can throw a non-Error // (number/string), so `(err as Error).message` was logging "undefined". const msg = describeError(err); this.status.textContent = `failed: ${msg}`; this.status.title = msg; console.error('[gesture] start failed', err); } finally { this.button.disabled = false; } } private toggleCamMode(): void { this.camMode = this.camMode === 'full' ? 'pip' : 'full'; this.applyCamMode(); } private applyCamMode(): void { this.video.classList.toggle('cg-full', this.camMode === 'full'); this.video.classList.toggle('cg-pip', this.camMode === 'pip'); } /** Top-most element matching `sel` at a viewport point (overlays are * click-through, so elementFromPoint sees the dashboard beneath). */ private hitClosest(x: number, y: number, sel: string): HTMLElement | null { const hit = document.elementFromPoint(x, y); return (hit?.closest(sel) as HTMLElement | null) ?? null; } private onGrab(hand: string, x: number, y: number): void { // An existing floating panel → re-grab to move it (priority over tabs). // Make it click-through while held so elementFromPoint sees the dock zone // (and other content) beneath it, and it can't re-grab itself. const panelEl = this.hitClosest(x, y, PANEL_SELECTOR); const panelId = panelEl?.dataset.id; if (panelEl && panelId) { const float = this.floats.get(panelId); if (float) { const rect = panelEl.getBoundingClientRect(); panelEl.style.pointerEvents = 'none'; panelEl.classList.add('cg-float-grabbed'); this.grabs.set(hand, { kind: 'panel', id: panelId, panel: float, dx: x - rect.left, dy: y - rect.top, overDock: false, }); this.status.textContent = 'moving — drop over tabs to re-dock'; return; } } // A session tab → grab-and-pull-out into a floating panel (ghost follows). const tab = this.hitClosest(x, y, TAB_SELECTOR); const id = tab?.dataset.id; if (tab && id) { const rect = tab.getBoundingClientRect(); const ghost = tab.cloneNode(true) as HTMLElement; ghost.classList.add('cg-ghost'); ghost.removeAttribute('id'); ghost.style.width = `${rect.width}px`; ghost.style.height = `${rect.height}px`; document.body.append(ghost); tab.classList.add('cg-grabbed'); this.grabs.set(hand, { kind: 'tab', id, tab, ghost, ox: x, oy: y, armed: false }); this.positionGhost(ghost, x, y); return; } // A toolbar button (Run / Run Shell) → tap-to-fire on release. const btn = this.hitClosest(x, y, CLICK_SELECTOR); if (btn) { const label = (btn.textContent || btn.getAttribute('title') || 'button').trim(); btn.classList.add('cg-tap-armed'); this.taps.set(hand, { el: btn, label, ox: x, oy: y }); this.status.textContent = `release to ${label.toLowerCase()}`; } } private onDrag(hand: string, x: number, y: number): void { const grab = this.grabs.get(hand); if (grab?.kind === 'tab') { this.positionGhost(grab.ghost, x, y); const pulled = Math.hypot(x - grab.ox, y - grab.oy) >= DETACH_PULL_PX; if (pulled !== grab.armed) { grab.armed = pulled; grab.ghost.classList.toggle('cg-armed', pulled); this.status.textContent = pulled ? 'release to float out' : 'on — pinch a tab'; } return; } if (grab?.kind === 'panel') { this.moveFloat(grab.panel, x - grab.dx, y - grab.dy); const overDock = !!this.hitClosest(x, y, DOCK_SELECTOR); if (overDock !== grab.overDock) { grab.overDock = overDock; grab.panel.el.classList.toggle('cg-redock', overDock); this.status.textContent = overDock ? 'release to re-dock' : 'moving panel'; } return; } // A button pinch that drifts too far is a stray move, not a tap — cancel it. const tap = this.taps.get(hand); if (tap && Math.hypot(x - tap.ox, y - tap.oy) > TAP_CANCEL_PX) { tap.el.classList.remove('cg-tap-armed'); this.taps.delete(hand); this.status.textContent = 'on — pinch a tab or button'; } } private onDrop(hand: string, x: number, y: number): void { const grab = this.grabs.get(hand); if (grab?.kind === 'tab') { this.grabs.delete(hand); grab.ghost.remove(); grab.tab.classList.remove('cg-grabbed'); if (grab.armed) this.floatPanel(grab.id, x, y); else this.flash('cancelled'); return; } if (grab?.kind === 'panel') { this.grabs.delete(hand); grab.panel.el.style.pointerEvents = ''; // interactive again (type into it) grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock'); if (grab.overDock) this.redock(grab.id); else this.flash('placed'); return; } // Release over the same button → fire its real click handler. const tap = this.taps.get(hand); if (tap) { this.taps.delete(hand); tap.el.classList.remove('cg-tap-armed'); tap.el.click(); // runs the button's onclick (app.run() / app.runShell()) this.flash(tap.label.toLowerCase()); } } /** Pop a session into an in-page floating panel (an iframe of its solo route) * centered on the drop point. Unlike OS-window detach, the panel lives in * this page's DOM, so the hand can re-grab and move it. Idempotent per id: * re-floating an existing id just repositions the panel it already has. */ private floatPanel(id: string, x: number, y: number): void { let float = this.floats.get(id); if (!float) { const container = el('div', 'cg-float'); container.dataset.id = id; const bar = el('div', 'cg-float-bar'); bar.textContent = `session ${id}`; const frame = el('iframe', 'cg-float-frame') as HTMLIFrameElement; frame.src = `/session/${encodeURIComponent(id)}`; frame.title = `Session ${id}`; container.append(bar, frame); document.body.append(container); float = { id, el: container }; this.floats.set(id, float); this.flash('floated out'); } else { this.flash('re-floated'); } this.moveFloat(float, x - FLOAT_W / 2, y - FLOAT_H / 2); } /** Re-dock a floated session: drop its in-page panel. The original * `.session-tab` was never removed, so the session is simply back to plain * tab form. Mirrors Codeman's own `.detached` → attach toggle. */ private redock(id: string): void { const float = this.floats.get(id); if (!float) return; float.el.remove(); this.floats.delete(id); this.flash('re-docked'); } /** Position a float by its top-left corner, clamped to stay on-screen. */ private moveFloat(float: FloatingPanel, left: number, top: number): void { const l = Math.min(Math.max(0, left), Math.max(0, window.innerWidth - FLOAT_W)); const t = Math.min(Math.max(0, top), Math.max(0, window.innerHeight - FLOAT_H)); float.el.style.left = `${l}px`; float.el.style.top = `${t}px`; } private positionGhost(ghost: HTMLElement, x: number, y: number): void { ghost.style.left = `${x}px`; ghost.style.top = `${y}px`; } private cancelAllGrabs(): void { // Floats themselves persist (they're placed windows) — only release any // in-progress grab cleanly, restoring a moved panel's interactivity. for (const grab of this.grabs.values()) { if (grab.kind === 'tab') { grab.ghost.remove(); grab.tab.classList.remove('cg-grabbed'); } else { grab.panel.el.style.pointerEvents = ''; grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock'); } } this.grabs.clear(); for (const tap of this.taps.values()) tap.el.classList.remove('cg-tap-armed'); this.taps.clear(); document .querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`) .forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed')); } private onStatus(fps: number, hands: HandState[]): void { // Draw per-hand cursor dots (green while pinching) so the user can aim. const { width, height } = this.canvas; const dpr = window.devicePixelRatio || 1; this.ctx.clearRect(0, 0, width, height); const rect = this.surface.getBoundingClientRect(); for (const h of hands) { const x = (1 - h.cursor.x) * rect.width; const y = h.cursor.y * rect.height; this.ctx.beginPath(); this.ctx.arc(x * dpr, y * dpr, (h.pinching ? 14 : 9) * dpr, 0, Math.PI * 2); this.ctx.fillStyle = handColor(h.handedness, h.pinching); this.ctx.globalAlpha = 0.85; this.ctx.fill(); this.ctx.globalAlpha = 1; } if (this.running && this.grabs.size === 0 && this.taps.size === 0) { this.status.textContent = `on · ${fps}fps`; } } private flash(msg: string): void { this.status.textContent = msg; } private sizeCanvas(): void { const dpr = window.devicePixelRatio || 1; this.canvas.width = Math.floor(window.innerWidth * dpr); this.canvas.height = Math.floor(window.innerHeight * dpr); } } // ---- tiny helpers ------------------------------------------------------ function el(tag: string, className: string): HTMLElement { const node = document.createElement(tag); node.className = className; return node; } /** Best-effort human-readable message for any thrown value (Error or not). */ function describeError(err: unknown): string { if (err instanceof Error) return err.message || err.name || 'Error'; if (typeof err === 'string') return err; if (typeof err === 'number') return `code ${err}`; if (err && typeof err === 'object') { const m = (err as { message?: unknown }).message; if (typeof m === 'string' && m) return m; try { return JSON.stringify(err); } catch { return Object.prototype.toString.call(err); } } return String(err); } function injectStyles(): void { if (document.getElementById('cg-styles')) return; const css = ` .cg-surface, .cg-canvas { position: fixed; inset: 0; pointer-events: none; } .cg-surface { z-index: ${Z}; } .cg-canvas { z-index: ${Z + 1}; width: 100vw; height: 100vh; } .cg-preview { transform: scaleX(-1); pointer-events: none; background: #000; } .cg-preview.cg-pip { position: fixed; right: 12px; bottom: 12px; width: 240px; height: 135px; object-fit: cover; border-radius: 8px; z-index: ${Z + 2}; box-shadow: 0 4px 16px rgba(0,0,0,.5); opacity: 1; } .cg-preview.cg-full { position: fixed; inset: 0; width: 100vw; height: 100vh; object-fit: cover; opacity: .28; z-index: ${Z}; } .cg-ghost { position: fixed; left: 0; top: 0; transform: translate(-50%, -50%) scale(1.06); z-index: ${Z + 2}; pointer-events: none; opacity: .92; box-shadow: 0 8px 28px rgba(0,0,0,.55); border-radius: 8px; outline: 2px solid #38bdf8; outline-offset: -2px; } .cg-ghost.cg-armed { outline-color: #4ade80; box-shadow: 0 8px 28px rgba(74,222,128,.5); } .cg-dock { position: fixed; right: 12px; bottom: 156px; z-index: ${Z + 3}; display: flex; align-items: center; gap: 8px; font: 12px/1 system-ui, sans-serif; } .cg-btn { padding: 6px 12px; border-radius: 6px; border: 1px solid #3a3a40; background: #1b1b1f; color: #e5e5e7; cursor: pointer; } .cg-btn-icon { padding: 6px 9px; } .cg-btn.on { background: #16331f; border-color: #2f6b41; color: #4ade80; } .cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; } .cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; } .cg-float { position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px; z-index: ${Z}; display: flex; flex-direction: column; overflow: hidden; background: #0c0c0f; border-radius: 10px; outline: 2px solid #38bdf8; outline-offset: -2px; box-shadow: 0 10px 40px rgba(0,0,0,.6); } .cg-float.cg-float-grabbed { outline-color: #4ade80; box-shadow: 0 12px 48px rgba(74,222,128,.45); } .cg-float.cg-redock { outline-color: #fbbf24; box-shadow: 0 12px 48px rgba(251,191,36,.5); } .cg-float-bar { flex: 0 0 auto; padding: 4px 10px; font: 11px/1.6 system-ui, sans-serif; color: #cbd5e1; background: #15151a; border-bottom: 1px solid #2a2a30; user-select: none; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .cg-float-frame { flex: 1 1 auto; width: 100%; border: 0; background: #000; } `; const style = document.createElement('style'); style.id = 'cg-styles'; style.textContent = css; document.head.append(style); } // Idempotent bootstrap — re-importing must not stack overlays. if (!window.__codemanGesture) { window.__codemanGesture = new GestureBridge(); }