From 09a142d14b45cef47cc5c5d5e355739cb395d522 Mon Sep 17 00:00:00 2001 From: arkon Date: Mon, 8 Jun 2026 20:48:20 +0200 Subject: [PATCH] feat: vendor gesture-control source into packages/gesture-control Bring the Ark0N/codeman-gesture-control repo in-tree as the codeman-gesture-control workspace package so the hand-tracking overlay can be developed in the Codeman repo. New npm run build:gesture bundles src/codeman/entry.ts into the served gesture-codeman.js; scripts/build.mjs reruns it on every production build. Source formatted to Codeman's prettier style. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 9 +- CHANGELOG.md | 12 + CLAUDE.md | 8 +- package-lock.json | 132 ++- package.json | 3 +- packages/gesture-control/.gitignore | 6 + packages/gesture-control/README.md | 258 +++++ packages/gesture-control/docs/BUILD_PLAN.md | 224 ++++ .../docs/CODEMAN_DETACH_BRIEF.md | 80 ++ .../docs/MULTIMONITOR_DESIGN.md | 260 +++++ packages/gesture-control/index.html | 208 ++++ packages/gesture-control/package.json | 28 + packages/gesture-control/src/codeman/entry.ts | 518 ++++++++++ packages/gesture-control/src/demo/overlay.ts | 120 +++ packages/gesture-control/src/demo/tabs.ts | 209 ++++ .../src/gesture/GestureController.ts | 386 +++++++ .../src/gesture/OneEuroFilter.ts | 73 ++ .../gesture-control/src/gesture/commands.ts | 78 ++ .../gesture-control/src/gesture/landmarks.ts | 78 ++ packages/gesture-control/src/gesture/pinch.ts | 67 ++ packages/gesture-control/src/gesture/types.ts | 80 ++ packages/gesture-control/src/main.ts | 188 ++++ packages/gesture-control/tsconfig.json | 22 + packages/gesture-control/vite.config.ts | 12 + scripts/build-gesture-bundle.mjs | 52 + scripts/build.mjs | 3 + src/web/public/gesture/gesture-codeman.js | 957 ++++++++++-------- 27 files changed, 3630 insertions(+), 441 deletions(-) create mode 100644 packages/gesture-control/.gitignore create mode 100644 packages/gesture-control/README.md create mode 100644 packages/gesture-control/docs/BUILD_PLAN.md create mode 100644 packages/gesture-control/docs/CODEMAN_DETACH_BRIEF.md create mode 100644 packages/gesture-control/docs/MULTIMONITOR_DESIGN.md create mode 100644 packages/gesture-control/index.html create mode 100644 packages/gesture-control/package.json create mode 100644 packages/gesture-control/src/codeman/entry.ts create mode 100644 packages/gesture-control/src/demo/overlay.ts create mode 100644 packages/gesture-control/src/demo/tabs.ts create mode 100644 packages/gesture-control/src/gesture/GestureController.ts create mode 100644 packages/gesture-control/src/gesture/OneEuroFilter.ts create mode 100644 packages/gesture-control/src/gesture/commands.ts create mode 100644 packages/gesture-control/src/gesture/landmarks.ts create mode 100644 packages/gesture-control/src/gesture/pinch.ts create mode 100644 packages/gesture-control/src/gesture/types.ts create mode 100644 packages/gesture-control/src/main.ts create mode 100644 packages/gesture-control/tsconfig.json create mode 100644 packages/gesture-control/vite.config.ts create mode 100644 scripts/build-gesture-bundle.mjs diff --git a/.gitignore b/.gitignore index 1950c2b8..2aec0d13 100644 --- a/.gitignore +++ b/.gitignore @@ -61,10 +61,17 @@ tmp/ # Opt-in gesture overlay runtime assets: large MediaPipe wasm + model (~27 MB) # fetched at build/install by scripts/fetch-gesture-assets.mjs, kept out of git. -# (The gesture bundle itself, gesture-codeman.js, IS tracked.) +# (The gesture bundle itself, gesture-codeman.js, IS tracked — built from +# packages/gesture-control source by `npm run build:gesture`.) src/web/public/gesture/wasm/ src/web/public/gesture/*.task +# Gesture-control workspace package build outputs (source is tracked; the +# Codeman bundle is emitted to src/web/public/gesture/gesture-codeman.js instead). +packages/gesture-control/dist/ +packages/gesture-control/dist-codeman/ +packages/gesture-control/.vite/ + # Claude Code plan tracking plan.json diff --git a/CHANGELOG.md b/CHANGELOG.md index e55c97c8..74fd7f59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # aicodeman +## 0.9.2 + +### Patch Changes + +- Vendor the gesture-control source into the repo for in-tree development. + + The hand-tracking overlay's source (previously the standalone `Ark0N/codeman-gesture-control` repo) now lives at `packages/gesture-control/` as the `codeman-gesture-control` workspace package: the transport-agnostic gesture core (`src/gesture/*` — MediaPipe GestureRecognizer → One-Euro-filtered cursor → pinch state machine), the Codeman consumer entry (`src/codeman/entry.ts`, maps grab/drag/drop onto real session tabs + toolbar buttons), and a standalone vite playground for iterating on gesture feel. + - New `npm run build:gesture` (`scripts/build-gesture-bundle.mjs`) esbuild-bundles `entry.ts` into the served `src/web/public/gesture/gesture-codeman.js`; `scripts/build.mjs` now reruns it on every production build so the served bundle always reflects current source. The MediaPipe wasm + model stay runtime-loaded from same-origin `/gesture/` (unchanged). + - Added `@mediapipe/tasks-vision@0.10.21` as the package dependency (kept in sync with `fetch-gesture-assets.mjs`). The playground uses vite 7 (no known advisories). + + No change to the shipped app behavior — gesture control remains opt-in (`CODEMAN_GESTURE=1` + the App Settings → Input toggle). This release just makes the overlay developable inside the Codeman repo. + ## 0.9.1 ### Patch Changes diff --git a/CLAUDE.md b/CLAUDE.md index 018c9c13..b5058705 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,7 +56,7 @@ When user says "COM": CI runs `npm run check:lockfile` on every push/PR, so lockfile drift fails the build even if the `version-packages` script is bypassed. -**Version**: 0.9.1 (must match `package.json`) +**Version**: 0.9.2 (must match `package.json`) ## Project Overview @@ -82,6 +82,8 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph | Continuous typecheck | `tsc --noEmit --watch` | | Test coverage | `npm run test:coverage` | | Dead-code sweep | `npm run knip` (config in `knip.json`) | +| Rebuild gesture overlay | `npm run build:gesture` (esbuild `packages/gesture-control/src/codeman/entry.ts` → `src/web/public/gesture/gesture-codeman.js`; commit the result) | +| Gesture playground | `npm run dev` **in** `packages/gesture-control/` (standalone vite demo, fake tabs) | | Check public-asset formatting | `npm run check:public-assets` (prettier-checks `src/web/public/**` text assets; `scripts/check-public-assets.mjs`) | | Production start | `npm run start` | | Production logs | `journalctl --user -u codeman-web -f` | @@ -130,7 +132,7 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph ★ = Large file (>50KB). All files have `@fileoverview` JSDoc — read that before diving in. Discovery aid: `grep -l '@fileoverview' src/web/routes/*.ts` lists all route modules; same grep works for `src/types/`, `src/web/public/*.js`. -**Local package**: `packages/xterm-zerolag-input/` — local echo overlay for xterm.js; copy embedded in `app.js`. +**Local packages**: `packages/xterm-zerolag-input/` — local echo overlay for xterm.js; copy embedded in `app.js`. `packages/gesture-control/` (`codeman-gesture-control`) — hand-tracking overlay source; built to `src/web/public/gesture/gesture-codeman.js` via `npm run build:gesture` (see Frontend → Gesture control). **Config**: `src/config/` — 10 files. Import from specific files, not barrel. @@ -169,6 +171,8 @@ Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. L **Gesture control** (the camera hand-tracking overlay) is **opt-in, default OFF**, under App Settings → Display → **Input** (`gestureControlEnabled`). `CODEMAN_GESTURE=1` makes the feature *available* on the instance (CSP widening + `/gesture/` assets) and sets `window.__codemanGestureAvailable` (the Input section only shows when set); the overlay bundle is injected by `renderIndexHtml` **only when the setting is enabled**, so that method is `async` and reads `settings.json` via `readSettings(true)` — the `true` forces a **fresh** read (bypassing the 2s `_settingsCache`), because a post-save reload happens within that TTL and the cached value would otherwise render the pre-toggle state. Toggling the setting reloads the page (the bundle is render-injected). +**Gesture-control source lives in-repo** at `packages/gesture-control/` (workspace package `codeman-gesture-control`, was the standalone `Ark0N/codeman-gesture-control` repo). The transport-agnostic core is `src/gesture/*` (MediaPipe GestureRecognizer → One-Euro-filtered cursor → pinch state machine); `src/codeman/entry.ts` is the Codeman *consumer* that maps grab/drag/drop onto real `.session-tab`/toolbar buttons and is the bundle entry. **Edit there, then run `npm run build:gesture`** (`scripts/build-gesture-bundle.mjs` → esbuild bundles `entry.ts`, MediaPipe JS included, into `src/web/public/gesture/gesture-codeman.js`) and **commit the regenerated bundle** — the committed bundle is what dev/`tsx` serves (no bundler at runtime), and `scripts/build.mjs` reruns the same step so prod always reflects current source. The MediaPipe **wasm + model** are NOT bundled — loaded at runtime from same-origin `/gesture/wasm` + `/gesture/gesture_recognizer.task`, fetched by `scripts/fetch-gesture-assets.mjs` (gitignored, see Gotchas). `entry.ts` mounts `window.__codemanGesture = new GestureBridge()` idempotently at module-eval. A standalone vite playground (`npm run dev` in the package — fake tabs, no Codeman) lets you iterate on gesture *feel* in isolation. ⚠️ Keep `MP_VERSION` in `fetch-gesture-assets.mjs` in sync with `@mediapipe/tasks-vision` in `packages/gesture-control/package.json`. + **Respawn presets**: `solo-work` (3s/60min), `subagent-workflow` (45s/240min), `team-lead` (90s/480min), `ralph-todo` (8s/480min), `overnight-autonomous` (10s/480min). **Keyboard shortcuts**: Escape (close), Ctrl+? (help), Ctrl+W (kill), Ctrl+Tab (next), Alt+1-9 (switch tab), Ctrl+Shift+{/} (move tab left/right), Shift+Enter (newline), Ctrl+L (clear), Ctrl+Shift+R (restore size), Ctrl+Shift+V (voice input), Ctrl/Cmd +/- (font). diff --git a/package-lock.json b/package-lock.json index 057b5c3b..8371ef88 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "aicodeman", - "version": "0.9.1", + "version": "0.9.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aicodeman", - "version": "0.9.1", + "version": "0.9.2", "hasInstallScript": true, "license": "MIT", "workspaces": [ @@ -1753,6 +1753,12 @@ "mediabunny": "^1.0.0" } }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.21", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.21.tgz", + "integrity": "sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==", + "license": "Apache-2.0" + }, "node_modules/@module-federation/error-codes": { "version": "0.22.0", "resolved": "https://registry.npmjs.org/@module-federation/error-codes/-/error-codes-0.22.0.tgz", @@ -5334,6 +5340,10 @@ "node": ">=12" } }, + "node_modules/codeman-gesture-control": { + "resolved": "packages/gesture-control", + "link": true + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -12166,6 +12176,124 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "packages/gesture-control": { + "name": "codeman-gesture-control", + "version": "0.1.0", + "dependencies": { + "@mediapipe/tasks-vision": "0.10.21" + }, + "devDependencies": { + "esbuild": "^0.27.3", + "typescript": "^5.5.4", + "vite": "^7.3.5" + } + }, + "packages/gesture-control/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "packages/gesture-control/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "packages/gesture-control/node_modules/vite": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.5.tgz", + "integrity": "sha512-KuOaNhcnGFN2zIPGA7wRmzF+lJA1sea7rHq17aiJ++9lzY1WWG6Jpwqwe1KNbRVPIqHmr8GLYx7jbrQcN/7/ww==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, "packages/xterm-zerolag-input": { "version": "0.1.4", "license": "MIT", diff --git a/package.json b/package.json index ce895b35..99052425 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "aicodeman", - "version": "0.9.1", + "version": "0.9.2", "description": "The missing control plane for AI coding agents - run 20 autonomous agents with real-time monitoring and session persistence", "type": "module", "main": "dist/index.js", @@ -11,6 +11,7 @@ "scripts": { "postinstall": "node scripts/postinstall.js", "build": "node scripts/build.mjs", + "build:gesture": "node scripts/build-gesture-bundle.mjs", "start": "NODE_COMPILE_CACHE=${HOME}/.codeman/compile-cache node dist/index.js", "dev": "tsx src/index.ts web", "web": "node dist/index.js web", diff --git a/packages/gesture-control/.gitignore b/packages/gesture-control/.gitignore new file mode 100644 index 00000000..58fb44a4 --- /dev/null +++ b/packages/gesture-control/.gitignore @@ -0,0 +1,6 @@ +node_modules +dist +dist-codeman/ +*.local +.vite +.DS_Store diff --git a/packages/gesture-control/README.md b/packages/gesture-control/README.md new file mode 100644 index 00000000..42f6071e --- /dev/null +++ b/packages/gesture-control/README.md @@ -0,0 +1,258 @@ +# gesture-proto + +A Jarvis-style hand-tracking input layer for the Codeman dashboard. Your webcam +sees your hands; you pinch-drag session tabs between columns. Runs entirely +in-browser — **the camera feed never leaves the machine.** + +> **This build is pinch-drag only.** The discrete gesture commands (halt-all / +> approve / new-session) were built in Phase 4 but are **intentionally not wired +> up** — an open palm while reaching to pinch kept charging the hold-to-halt. The +> `GestureController` core still emits `command`/`haltProgress` for any future +> consumer; the demo just no longer listens. + +This is a standalone prototype (own folder, fake tabs) so the input *feel* can +be validated on real hardware before integrating with Codeman. The canonical +spec is **[`docs/BUILD_PLAN.md`](./docs/BUILD_PLAN.md)** — read it before +continuing the build. + +--- + +## Status: Phase 0–4 complete + +| Phase | What | State | +|-------|------|-------| +| 0 | Vite+TS scaffold, mirrored webcam preview, start button | ✅ done | +| 1 | MediaPipe `GestureRecognizer` (VIDEO mode, CDN model+wasm), rAF loop, debug skeleton overlay, HUD | ✅ done | +| — | **Checkpoint: ≥25fps on real camera/lighting** | ✅ 60fps (iPhone 17 Pro / Continuity Camera) | +| 2 | One-Euro–filtered cursor + pinch detection (hysteresis) | ✅ done | +| 3 | Drag fake tabs across 3 columns (state machine) — go/no-go | ✅ done (two-handed) | +| 4 | Discrete gesture→command bus (👍 approve · ✌️ new-session · ✋-hold halt-all) + toast | ✅ built, ⛔ **unwired in the demo** (pinch-only) | +| 5 | Integrate `gesture/` into real Codeman (`src/codeman/entry.ts`) | ✅ **working at the desk** — see [Codeman integration](#codeman-integration) | + +Also done beyond the original plan: **two-hand tracking** (drag two tabs at +once) and a **live camera picker** (front-facing iPhone 17 Pro by default; the +superwide / Desk View camera now works too — see the camera note below). + +Also: **fullscreen mode** (toggle button — the board fills the display, so grab +targets get big). + +**Phase 5 is live** in the real Codeman dashboard. The Codeman-side detach + +instance isolation + base gesture overlay are committed on `Ark0N/Codeman` +branch `beta/session-detach` (open as **PR #103**) — including this session's +gesture *improvements* (direct-detach, Run/Run Shell taps, self-hosted MediaPipe, +cache-bust), ported onto the PR in commit `eea84db` (CI green). See +[Codeman integration](#codeman-integration) below and the hand-off brief +[`../docs/CODEMAN_DETACH_BRIEF.md`](../docs/CODEMAN_DETACH_BRIEF.md). + +**Multi-monitor mode — built & validated at the desk (2026-06-08).** Approach +**A+C** (see [`../docs/MULTIMONITOR_DESIGN.md`](../docs/MULTIMONITOR_DESIGN.md)): +(A) gesture "detach" now pops a session into a re-grabbable **in-page floating +panel** (an ``. + The iframe is a complete live session view (its own terminal + stream client); + the gesture layer moves the **div** and the iframe rides along. Lowest new + code; reuses proven fan-out; no terminal wiring. Trade-off: a full app shell + per float (heavier — fine for a few, watch memory at many). +- **Richer alt — native teammate-style panel.** Mount a `new Terminal()` in the + float body, seed via the session buffer endpoint, feed via the teammate + stream. Native (no iframe), lighter per-float, same-document. Use if the + iframe feels heavy or you want tighter integration. + +Either way the hand only **places** the float; typing into it uses a keyboard +(focus the iframe / native terminal) — consistent with "gesture places, +keyboard types." + +### Gesture grammar (replaces the current detach path in `entry.ts`) + +The grab/drag/drop plumbing already exists; only the **drop action** changes. + +- **Grab** a `.session-tab[data-id]` (unchanged: `onGrab`, ghost-follow). +- **Pull** past `DETACH_PULL_PX` to arm (unchanged: `grab.armed`). +- **Drop while armed** → **no longer** `window.app.detachSession`. Instead spawn + an **in-page floating panel** for that session id at the drop point. New + method `floatPanel(id, x, y)` replacing `detach(id)`. +- **Re-grab** a floating panel (new `PANEL_SELECTOR`, e.g. `.cg-float[data-id]`) + → move it; drop anywhere → it stays. This is the capability detach lost. +- **Drop a panel back over the tab strip** (or a dock zone) → **re-dock** + (remove the float; session returns to a plain tab). Mirrors the `.detached` → + attach toggle Codeman already has. +- **Keep `window.open` detach as a separate, deliberate verb** — e.g. a button, + or a distinct "throw up and off-screen" gesture — for intentionally parking a + session in its own OS window. It is *not* the default pinch action anymore. + +### `entry.ts` change surface + +- New state: `floats: Map` (id → element + position), + parallel to the existing `grabs`/`taps` maps. +- `onGrab`: extend hit-testing to also match `PANEL_SELECTOR`, so an existing + float can be re-grabbed (priority: panel over tab when overlapping). +- `onDrop`: replace `if (grab.armed) this.detach(grab.id)` with + `this.floatPanel(...)`; add the panel re-dock branch. +- `floatPanel(id, x, y)`: create/show the in-page panel (Part A option 1/2), + position it absolutely at the drop point. Idempotent per id (re-grab moves the + existing one, never duplicates). +- Coordinate mapping is **already viewport-pixel based** (the click-through + surface maps cursor → viewport px), so it needs **no change** for spanning — + see Part C. + +--- + +## Part C — One window spanning both monitors + +Once the Codeman window physically covers both displays, Part A's panels drag +across the seam for free, because the gesture cursor is already in viewport +pixels and the viewport now spans both monitors. + +### macOS setup (operational, near-zero code) + +1. **System Settings → Desktop & Dock → uncheck "Displays have separate + Spaces."** (Requires a logout/login.) This is what lets a single window + straddle two physical displays. +2. Run Codeman **maximized, not fullscreen.** Browser fullscreen is *per + display* and will **not** span — use a maximized/borderless window dragged to + cover both monitors. (A kiosk/`--app` Chrome window sized to the union rect is + the cleanest.) **Automated by `scripts/span-codeman.sh`** — it reads the + display-union rect (Finder desktop bounds) and launches a **Brave-first** + Chromium-family `--app` window sized to it (fresh per-browser profile so the + geometry flags are honored; `BROWSER=` overrides — plain Chrome bounced on the + desk machine). One-click from the **Codeman header "multi-monitor" button** + (`POST /api/system/span-displays`, which spawns this script), or run it + directly. Step 1 + logout is still manual; the script warns if spanning isn't + active. +3. Arrange the two monitors as a contiguous rectangle in Display settings so the + union has no vertical offset gap. + +### Coordinate model & the bezel seam + +- Enumerate displays with **`window.getScreenDetails()`** (Chrome, secure + context, `window-management` permission). Gives each screen's + `left/top/width/height/availLeft/...` in a **virtual-desktop coordinate + space** spanning all monitors. +- Use it for **screen-edge snapping zones**: e.g. dropping a panel within the + right screen's bounds snaps it to fill that screen; the seam between the two + screens' rects is a "halt / boundary" zone the cursor crosses. +- **Account for the bezel gap.** The two monitors are physically separated, but + the spanned window's pixels are contiguous — a panel dragged across the seam + visually jumps the bezel. Optional: add a dead-band at the seam x-coordinate + so a panel snaps to one side rather than straddling. +- `getScreenDetails` is **only needed for snapping/zone logic**, not for basic + dragging — dragging works the moment the window spans. So Part C can ship in + two steps: (1) just span + free drag, (2) add `getScreenDetails` snapping. + +### Constraints to surface to the user + +- macOS-specific; the "separate Spaces" toggle is global and affects all apps. +- Real fullscreen is unavailable (must run maximized). +- A bezel-width discontinuity sits in the middle of the coordinate space. +- `window-management` permission prompts once. + +--- + +## Build phases + +1. ✅ **A-MVP — in-page live float, single monitor (DONE, `581fcf9`/`3e0447a`).** + `entry.ts` `floatSession(id, x, y)` spawns an **iframe of `/session/:id`** in a + `.cg-float` div (640×420) at the drop point; re-grab + move work (re-dock zone + still TBD). Movability restored — the live session rides inside the page. (The + method is named `floatSession`, not the design-sketch `floatPanel`.) +2. ✅ **C-span — span the window (DONE, `063fd8f`/`59946b8`; validated at desk).** + macOS "separate Spaces" off + `scripts/span-codeman.sh` launches a maximized + `--app` window across the display union (Brave-first; `BROWSER=` override). + **Confirmed:** panels drag across the seam — viewport mapping held with zero + `entry.ts` change. Launchable one-click from the **Codeman header button** → + `POST /api/system/span-displays` (PR #103 `95b0035`), or by running the script. +3. ⏳ **C-snap — screen-aware snapping (pending).** Add `getScreenDetails`; snap a + panel to the screen it's dropped on; add the seam dead-band. +4. **A-alt (optional).** Swap the iframe float for a native teammate-style + `new Terminal()` panel if the iframe shell feels heavy or many floats strain + memory. + +## Open questions / verification before coding + +- [x] **Concurrent rendering — RESOLVED (2026-06-08).** A live session view + *can* be mounted into an arbitrary in-page container concurrently with the + main view. The PR #103 branch already ships it three ways (teammate terminals, + log-viewer SSE windows, the iframe-able `/session/:id` solo route); fan-out + confirmed. Primary path: iframe the solo route. See the resolved section above. +- [ ] **Native-panel live feed (only if taking A-alt, not the iframe):** trace + the exact transport that pushes *continuous* teammate-pane output after the + REST buffer seed (`pendingData` flush source). The iframe path sidesteps this. +- [ ] **`window.app.detachSession` location:** when wiring the *kept* OS-window + detach verb, note this method was **not found in PR #103 source** (only + server-side `detachSessionListeners`); it works at runtime, so confirm where + it's actually defined before extending it. +- [ ] **Spanning feasibility on the actual desk setup** (monitor arrangement, + whether "separate Spaces" off is acceptable to the user globally). +- [ ] **Re-dock target:** decide the re-dock gesture/zone (drop over tab strip + vs a dedicated dock region). + +## Touch-points summary + +| Layer | File | Change | Status | +|-------|------|--------|--------| +| Gesture consumer | `gesture-proto/src/codeman/entry.ts` | `detach()` → `floatSession()`; `floats` map; panel re-grab; (later) re-dock branch + `getScreenDetails` snapping | ✅ float+re-grab done (`581fcf9`/`3e0447a`); re-dock/snap pending | +| Gesture core | `gesture-proto/src/gesture/*` | **none** — stays transport-agnostic | ✅ unchanged | +| Codeman — launch | `src/web/routes/system-routes.ts`, `src/web/public/{index.html,panels-ui.js}`, `scripts/span-codeman.sh` | header button → `POST /api/system/span-displays` → spawn span script (bundled into repo) | ✅ PR #103 `95b0035` | +| Codeman — caching | `src/web/server.ts` | `cacheBustAssets()` — `?v=` on all same-origin `.js`/`.css` so frontend edits show on normal reload | ✅ PR #103 `b5ea711` | +| Ops | macOS display settings + `scripts/span-codeman.sh` | "separate Spaces" off + re-login; Brave-first maximized spanning window | ✅ validated at desk | diff --git a/packages/gesture-control/index.html b/packages/gesture-control/index.html new file mode 100644 index 00000000..3a23b70c --- /dev/null +++ b/packages/gesture-control/index.html @@ -0,0 +1,208 @@ + + + + + + Codeman Gesture Control — Phase 4 + + + +

Codeman Gesture Control

+

Phase 4 — pinch-drag tabs + discrete gesture commands. Camera feed never leaves this machine.

+ +
+ +
+ +
+ +
+ + + + +
+ +
+ FPS0 + Handsno + Pinch— + Gesture— +
+ +

Click “Start camera” to begin (camera access requires a click).

+ + + + diff --git a/packages/gesture-control/package.json b/packages/gesture-control/package.json new file mode 100644 index 00000000..b2160245 --- /dev/null +++ b/packages/gesture-control/package.json @@ -0,0 +1,28 @@ +{ + "name": "codeman-gesture-control", + "private": true, + "version": "0.1.0", + "type": "module", + "description": "Jarvis-style hand-tracking input layer for the Codeman dashboard. Vendored into the Codeman repo; the Codeman bundle (src/codeman/entry.ts) is built into src/web/public/gesture/gesture-codeman.js by `npm run build:gesture` at the repo root.", + "scripts": { + "dev": "vite", + "preview": "vite preview", + "typecheck": "tsc --noEmit", + "build:demo": "tsc && vite build", + "build:codeman": "esbuild src/codeman/entry.ts --bundle --format=esm --target=es2020 --outfile=dist-codeman/gesture-codeman.js" + }, + "dependencies": { + "@mediapipe/tasks-vision": "0.10.21" + }, + "devDependencies": { + "esbuild": "^0.27.3", + "typescript": "^5.5.4", + "vite": "^7.3.5" + }, + "homepage": "https://github.com/Ark0N/Codeman/tree/master/packages/gesture-control#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/Ark0N/Codeman.git", + "directory": "packages/gesture-control" + } +} diff --git a/packages/gesture-control/src/codeman/entry.ts b/packages/gesture-control/src/codeman/entry.ts new file mode 100644 index 00000000..1335f331 --- /dev/null +++ b/packages/gesture-control/src/codeman/entry.ts @@ -0,0 +1,518 @@ +// Phase 5 — Codeman integration entry point. +// +// This is the *consumer* layer that replaces `demo/tabs.ts` for the real +// Codeman dashboard. It is bundled (esbuild, MediaPipe included) into a single +// ESM file and served by Codeman from `/gesture/gesture-codeman.js`, loaded into +// the dashboard page when Codeman is started with `CODEMAN_GESTURE=1`. +// +// The gesture *core* (`../gesture/*`) is unchanged and transport-agnostic — it +// emits coordinate-only `grab`/`drag`/`drop`. Here we map those onto Codeman's +// real session tabs (`.session-tab[data-id]`) and toolbar buttons. +// +// Three interactions, all off the same pinch: +// • Tab "grab-to-float" — pinch a session tab, *pull it out* of the strip (a +// ghost follows your hand), release past a threshold → the session opens as +// an in-page floating panel at the drop point; a small twitch-and-release +// cancels (snaps back). +// • Panel "re-grab" — pinch an existing floating panel and move it anywhere; +// release over the tab strip to re-dock it (panel goes away, the tab stays). +// This is the capability the old OS-window detach lost. +// • Button "tap" — pinch over a toolbar button (Run / Run Shell) and release +// in place → fires the button's real click handler. Drift too far first and +// it's treated as a stray move, not a tap. +// +// Why in-page floats, not OS-window detach (decided 2026-06-08, see +// docs/MULTIMONITOR_DESIGN.md): the hand only exists in the one page that owns +// the camera. `window.open`/`detachSession` puts the session in a sealed OS +// window the page can't hand-track — a detached session can never be moved +// again, a one-way trip. A floating panel (an iframe of the session's solo +// route `/session/:id`) stays in this page's DOM, so the hand keeps control: +// re-grab, move across the (spanned) viewport, drop, re-dock. The OS-window +// detach is kept for later as a *separate, deliberate* verb — no longer the pinch. + +import { GestureController } from '../gesture/GestureController.ts'; +import type { HandState } from '../gesture/types.ts'; + +declare global { + interface Window { + __codemanGesture?: GestureBridge; + } +} + +const TAB_SELECTOR = '.session-tab'; +/** An in-page floating session panel this layer spawned — re-grabbable to move. */ +const PANEL_SELECTOR = '.cg-float'; +/** The session-tab strip; dropping a moved panel over it re-docks the session. */ +const DOCK_SELECTOR = '.session-tabs'; +/** Toolbar buttons a pinch can "tap": Run (#runBtn → app.run()) and Run Shell + * (.btn-shell → app.runShell()). Pinch over one and release in place to fire + * it. Extend this list to expose more buttons to the gesture layer. */ +const CLICK_SELECTOR = '#runBtn, .btn-shell'; +const Z = 2147483000; // above the dashboard, below nothing that matters at the desk +/** Floating-panel size (px). Fixed for the MVP; resize is a later affordance. */ +const FLOAT_W = 640; +const FLOAT_H = 420; +/** Minimum pull distance (px) from the grab point before a release floats the + * tab out. Below this it's an accidental pinch and the tab snaps back. */ +const DETACH_PULL_PX = 70; +/** If a button-pinch drifts more than this, it's a stray move, not a tap. */ +const TAP_CANCEL_PX = 45; + +/** First letter colours: cyan left, violet right; green while pinching. */ +const handColor = (handedness: string, pinching: boolean): string => + pinching ? '#4ade80' : handedness === 'Right' ? '#a78bfa' : '#38bdf8'; + +/** A floating in-page session panel (an iframe of `/session/:id`) the hand can + * place and re-grab. Stays in this page's DOM, so it never leaves hand reach. */ +interface FloatingPanel { + id: string; + /** The `.cg-float` container element. */ + el: HTMLElement; +} + +/** Live state for one hand's in-progress grab — either a session *tab* being + * pulled out into a new float, or an existing *panel* being moved/re-docked. */ +type Grab = + | { + kind: 'tab'; + id: string; + tab: HTMLElement; + ghost: HTMLElement; + /** Grab origin in viewport px, to measure pull distance. */ + ox: number; + oy: number; + /** Pulled past the float-out threshold at least once. */ + armed: boolean; + } + | { + kind: 'panel'; + id: string; + panel: FloatingPanel; + /** Cursor→panel-top-left offset at grab, so it doesn't snap when re-grabbed. */ + dx: number; + dy: number; + /** Cursor currently over the tab strip → releasing re-docks. */ + overDock: boolean; + }; + +/** Live state for one hand pinching a toolbar button (Run / Run Shell). */ +interface Tap { + el: HTMLElement; + label: string; + ox: number; + oy: number; +} + +class GestureBridge { + private readonly surface: HTMLDivElement; + private readonly canvas: HTMLCanvasElement; + private readonly ctx: CanvasRenderingContext2D; + private readonly video: HTMLVideoElement; + private readonly button: HTMLButtonElement; + private readonly camBtn: HTMLButtonElement; + private readonly status: HTMLSpanElement; + private readonly gc: GestureController; + private running = false; + /** Camera view: full-viewport dimmed background, or small corner preview. */ + private camMode: 'full' | 'pip' = 'full'; + + /** Per-hand in-progress grab (a tab being pulled out, or a panel being moved). */ + private grabs = new Map(); + /** Per-hand in-progress button pinch (fires on release if it didn't drift). */ + private taps = new Map(); + /** Live floating panels, keyed by session id (idempotent per id). */ + private floats = new Map(); + + constructor() { + injectStyles(); + + // Full-viewport, click-through surface so coords map straight to viewport + // pixels and elementFromPoint() sees the tabs beneath, not our overlay. + this.surface = el('div', 'cg-surface') as HTMLDivElement; + this.canvas = el('canvas', 'cg-canvas') as HTMLCanvasElement; + this.video = el('video', 'cg-preview') as HTMLVideoElement; + this.video.muted = true; + this.video.playsInline = true; + this.applyCamMode(); + + const dock = el('div', 'cg-dock'); + this.button = el('button', 'cg-btn') as HTMLButtonElement; + this.button.textContent = '🖐 Gesture'; + this.camBtn = el('button', 'cg-btn cg-btn-icon') as HTMLButtonElement; + this.camBtn.textContent = '⛶'; + this.camBtn.title = 'Toggle camera size (fullscreen / corner)'; + this.status = el('span', 'cg-status') as HTMLSpanElement; + this.status.textContent = 'off'; + dock.append(this.button, this.camBtn, this.status); + + document.body.append(this.surface, this.video, this.canvas, dock); + this.ctx = this.canvas.getContext('2d')!; + this.sizeCanvas(); + window.addEventListener('resize', () => this.sizeCanvas()); + + this.gc = new GestureController({ + video: this.video, + surface: this.surface, + numHands: 1, + // Self-host the MediaPipe runtime + model from Codeman (same-origin) instead + // of the CDN, so an ad/content blocker, offline desk, or strict browser + // can't break startup (the CDN failure surfaced as `failed: {isTrusted}` — + // a resource load-error Event). Served from public/gesture/. + wasmBase: '/gesture/wasm', + modelUrl: '/gesture/gesture_recognizer.task', + }); + + this.gc.on('grab', (p) => this.onGrab(p.hand, p.x, p.y)); + this.gc.on('drag', (p) => this.onDrag(p.hand, p.x, p.y)); + this.gc.on('drop', (p) => this.onDrop(p.hand, p.x, p.y)); + this.gc.on('status', ({ fps, hands }) => this.onStatus(fps, hands)); + + this.button.addEventListener('click', () => void this.toggle()); + this.camBtn.addEventListener('click', () => this.toggleCamMode()); + } + + private async toggle(): Promise { + if (this.running) { + this.gc.stop(); + this.running = false; + this.cancelAllGrabs(); + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + this.button.classList.remove('on'); + this.status.textContent = 'off'; + return; + } + this.button.disabled = true; + this.status.textContent = 'starting…'; + try { + await this.gc.start(); + this.running = true; + this.button.classList.add('on'); + this.status.textContent = 'on — pinch a tab or button'; + } catch (err) { + // Surface the *real* cause: MediaPipe/Emscripten can throw a non-Error + // (number/string), so `(err as Error).message` was logging "undefined". + const msg = describeError(err); + this.status.textContent = `failed: ${msg}`; + this.status.title = msg; + console.error('[gesture] start failed', err); + } finally { + this.button.disabled = false; + } + } + + private toggleCamMode(): void { + this.camMode = this.camMode === 'full' ? 'pip' : 'full'; + this.applyCamMode(); + } + + private applyCamMode(): void { + this.video.classList.toggle('cg-full', this.camMode === 'full'); + this.video.classList.toggle('cg-pip', this.camMode === 'pip'); + } + + /** Top-most element matching `sel` at a viewport point (overlays are + * click-through, so elementFromPoint sees the dashboard beneath). */ + private hitClosest(x: number, y: number, sel: string): HTMLElement | null { + const hit = document.elementFromPoint(x, y); + return (hit?.closest(sel) as HTMLElement | null) ?? null; + } + + private onGrab(hand: string, x: number, y: number): void { + // An existing floating panel → re-grab to move it (priority over tabs). + // Make it click-through while held so elementFromPoint sees the dock zone + // (and other content) beneath it, and it can't re-grab itself. + const panelEl = this.hitClosest(x, y, PANEL_SELECTOR); + const panelId = panelEl?.dataset.id; + if (panelEl && panelId) { + const float = this.floats.get(panelId); + if (float) { + const rect = panelEl.getBoundingClientRect(); + panelEl.style.pointerEvents = 'none'; + panelEl.classList.add('cg-float-grabbed'); + this.grabs.set(hand, { + kind: 'panel', + id: panelId, + panel: float, + dx: x - rect.left, + dy: y - rect.top, + overDock: false, + }); + this.status.textContent = 'moving — drop over tabs to re-dock'; + return; + } + } + + // A session tab → grab-and-pull-out into a floating panel (ghost follows). + const tab = this.hitClosest(x, y, TAB_SELECTOR); + const id = tab?.dataset.id; + if (tab && id) { + const rect = tab.getBoundingClientRect(); + const ghost = tab.cloneNode(true) as HTMLElement; + ghost.classList.add('cg-ghost'); + ghost.removeAttribute('id'); + ghost.style.width = `${rect.width}px`; + ghost.style.height = `${rect.height}px`; + document.body.append(ghost); + + tab.classList.add('cg-grabbed'); + this.grabs.set(hand, { kind: 'tab', id, tab, ghost, ox: x, oy: y, armed: false }); + this.positionGhost(ghost, x, y); + return; + } + + // A toolbar button (Run / Run Shell) → tap-to-fire on release. + const btn = this.hitClosest(x, y, CLICK_SELECTOR); + if (btn) { + const label = (btn.textContent || btn.getAttribute('title') || 'button').trim(); + btn.classList.add('cg-tap-armed'); + this.taps.set(hand, { el: btn, label, ox: x, oy: y }); + this.status.textContent = `release to ${label.toLowerCase()}`; + } + } + + private onDrag(hand: string, x: number, y: number): void { + const grab = this.grabs.get(hand); + if (grab?.kind === 'tab') { + this.positionGhost(grab.ghost, x, y); + const pulled = Math.hypot(x - grab.ox, y - grab.oy) >= DETACH_PULL_PX; + if (pulled !== grab.armed) { + grab.armed = pulled; + grab.ghost.classList.toggle('cg-armed', pulled); + this.status.textContent = pulled ? 'release to float out' : 'on — pinch a tab'; + } + return; + } + if (grab?.kind === 'panel') { + this.moveFloat(grab.panel, x - grab.dx, y - grab.dy); + const overDock = !!this.hitClosest(x, y, DOCK_SELECTOR); + if (overDock !== grab.overDock) { + grab.overDock = overDock; + grab.panel.el.classList.toggle('cg-redock', overDock); + this.status.textContent = overDock ? 'release to re-dock' : 'moving panel'; + } + return; + } + // A button pinch that drifts too far is a stray move, not a tap — cancel it. + const tap = this.taps.get(hand); + if (tap && Math.hypot(x - tap.ox, y - tap.oy) > TAP_CANCEL_PX) { + tap.el.classList.remove('cg-tap-armed'); + this.taps.delete(hand); + this.status.textContent = 'on — pinch a tab or button'; + } + } + + private onDrop(hand: string, x: number, y: number): void { + const grab = this.grabs.get(hand); + if (grab?.kind === 'tab') { + this.grabs.delete(hand); + grab.ghost.remove(); + grab.tab.classList.remove('cg-grabbed'); + if (grab.armed) this.floatPanel(grab.id, x, y); + else this.flash('cancelled'); + return; + } + if (grab?.kind === 'panel') { + this.grabs.delete(hand); + grab.panel.el.style.pointerEvents = ''; // interactive again (type into it) + grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock'); + if (grab.overDock) this.redock(grab.id); + else this.flash('placed'); + return; + } + // Release over the same button → fire its real click handler. + const tap = this.taps.get(hand); + if (tap) { + this.taps.delete(hand); + tap.el.classList.remove('cg-tap-armed'); + tap.el.click(); // runs the button's onclick (app.run() / app.runShell()) + this.flash(tap.label.toLowerCase()); + } + } + + /** Pop a session into an in-page floating panel (an iframe of its solo route) + * centered on the drop point. Unlike OS-window detach, the panel lives in + * this page's DOM, so the hand can re-grab and move it. Idempotent per id: + * re-floating an existing id just repositions the panel it already has. */ + private floatPanel(id: string, x: number, y: number): void { + let float = this.floats.get(id); + if (!float) { + const container = el('div', 'cg-float'); + container.dataset.id = id; + const bar = el('div', 'cg-float-bar'); + bar.textContent = `session ${id}`; + const frame = el('iframe', 'cg-float-frame') as HTMLIFrameElement; + frame.src = `/session/${encodeURIComponent(id)}`; + frame.title = `Session ${id}`; + container.append(bar, frame); + document.body.append(container); + float = { id, el: container }; + this.floats.set(id, float); + this.flash('floated out'); + } else { + this.flash('re-floated'); + } + this.moveFloat(float, x - FLOAT_W / 2, y - FLOAT_H / 2); + } + + /** Re-dock a floated session: drop its in-page panel. The original + * `.session-tab` was never removed, so the session is simply back to plain + * tab form. Mirrors Codeman's own `.detached` → attach toggle. */ + private redock(id: string): void { + const float = this.floats.get(id); + if (!float) return; + float.el.remove(); + this.floats.delete(id); + this.flash('re-docked'); + } + + /** Position a float by its top-left corner, clamped to stay on-screen. */ + private moveFloat(float: FloatingPanel, left: number, top: number): void { + const l = Math.min(Math.max(0, left), Math.max(0, window.innerWidth - FLOAT_W)); + const t = Math.min(Math.max(0, top), Math.max(0, window.innerHeight - FLOAT_H)); + float.el.style.left = `${l}px`; + float.el.style.top = `${t}px`; + } + + private positionGhost(ghost: HTMLElement, x: number, y: number): void { + ghost.style.left = `${x}px`; + ghost.style.top = `${y}px`; + } + + private cancelAllGrabs(): void { + // Floats themselves persist (they're placed windows) — only release any + // in-progress grab cleanly, restoring a moved panel's interactivity. + for (const grab of this.grabs.values()) { + if (grab.kind === 'tab') { + grab.ghost.remove(); + grab.tab.classList.remove('cg-grabbed'); + } else { + grab.panel.el.style.pointerEvents = ''; + grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock'); + } + } + this.grabs.clear(); + for (const tap of this.taps.values()) tap.el.classList.remove('cg-tap-armed'); + this.taps.clear(); + document + .querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`) + .forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed')); + } + + private onStatus(fps: number, hands: HandState[]): void { + // Draw per-hand cursor dots (green while pinching) so the user can aim. + const { width, height } = this.canvas; + const dpr = window.devicePixelRatio || 1; + this.ctx.clearRect(0, 0, width, height); + const rect = this.surface.getBoundingClientRect(); + for (const h of hands) { + const x = (1 - h.cursor.x) * rect.width; + const y = h.cursor.y * rect.height; + this.ctx.beginPath(); + this.ctx.arc(x * dpr, y * dpr, (h.pinching ? 14 : 9) * dpr, 0, Math.PI * 2); + this.ctx.fillStyle = handColor(h.handedness, h.pinching); + this.ctx.globalAlpha = 0.85; + this.ctx.fill(); + this.ctx.globalAlpha = 1; + } + if (this.running && this.grabs.size === 0 && this.taps.size === 0) { + this.status.textContent = `on · ${fps}fps`; + } + } + + private flash(msg: string): void { + this.status.textContent = msg; + } + + private sizeCanvas(): void { + const dpr = window.devicePixelRatio || 1; + this.canvas.width = Math.floor(window.innerWidth * dpr); + this.canvas.height = Math.floor(window.innerHeight * dpr); + } +} + +// ---- tiny helpers ------------------------------------------------------ + +function el(tag: string, className: string): HTMLElement { + const node = document.createElement(tag); + node.className = className; + return node; +} + +/** Best-effort human-readable message for any thrown value (Error or not). */ +function describeError(err: unknown): string { + if (err instanceof Error) return err.message || err.name || 'Error'; + if (typeof err === 'string') return err; + if (typeof err === 'number') return `code ${err}`; + if (err && typeof err === 'object') { + const m = (err as { message?: unknown }).message; + if (typeof m === 'string' && m) return m; + try { + return JSON.stringify(err); + } catch { + return Object.prototype.toString.call(err); + } + } + return String(err); +} + +function injectStyles(): void { + if (document.getElementById('cg-styles')) return; + const css = ` + .cg-surface, .cg-canvas { position: fixed; inset: 0; pointer-events: none; } + .cg-surface { z-index: ${Z}; } + .cg-canvas { z-index: ${Z + 1}; width: 100vw; height: 100vh; } + .cg-preview { transform: scaleX(-1); pointer-events: none; background: #000; } + .cg-preview.cg-pip { + position: fixed; right: 12px; bottom: 12px; width: 240px; height: 135px; + object-fit: cover; border-radius: 8px; z-index: ${Z + 2}; + box-shadow: 0 4px 16px rgba(0,0,0,.5); opacity: 1; + } + .cg-preview.cg-full { + position: fixed; inset: 0; width: 100vw; height: 100vh; + object-fit: cover; opacity: .28; z-index: ${Z}; + } + .cg-ghost { + position: fixed; left: 0; top: 0; transform: translate(-50%, -50%) scale(1.06); + z-index: ${Z + 2}; pointer-events: none; opacity: .92; + box-shadow: 0 8px 28px rgba(0,0,0,.55); border-radius: 8px; + outline: 2px solid #38bdf8; outline-offset: -2px; + } + .cg-ghost.cg-armed { outline-color: #4ade80; box-shadow: 0 8px 28px rgba(74,222,128,.5); } + .cg-dock { + position: fixed; right: 12px; bottom: 156px; z-index: ${Z + 3}; + display: flex; align-items: center; gap: 8px; font: 12px/1 system-ui, sans-serif; + } + .cg-btn { + padding: 6px 12px; border-radius: 6px; border: 1px solid #3a3a40; + background: #1b1b1f; color: #e5e5e7; cursor: pointer; + } + .cg-btn-icon { padding: 6px 9px; } + .cg-btn.on { background: #16331f; border-color: #2f6b41; color: #4ade80; } + .cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; } + .cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; } + .cg-float { + position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px; + z-index: ${Z}; display: flex; flex-direction: column; overflow: hidden; + background: #0c0c0f; border-radius: 10px; outline: 2px solid #38bdf8; + outline-offset: -2px; box-shadow: 0 10px 40px rgba(0,0,0,.6); + } + .cg-float.cg-float-grabbed { outline-color: #4ade80; box-shadow: 0 12px 48px rgba(74,222,128,.45); } + .cg-float.cg-redock { outline-color: #fbbf24; box-shadow: 0 12px 48px rgba(251,191,36,.5); } + .cg-float-bar { + flex: 0 0 auto; padding: 4px 10px; font: 11px/1.6 system-ui, sans-serif; + color: #cbd5e1; background: #15151a; border-bottom: 1px solid #2a2a30; + user-select: none; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; + } + .cg-float-frame { flex: 1 1 auto; width: 100%; border: 0; background: #000; } + `; + const style = document.createElement('style'); + style.id = 'cg-styles'; + style.textContent = css; + document.head.append(style); +} + +// Idempotent bootstrap — re-importing must not stack overlays. +if (!window.__codemanGesture) { + window.__codemanGesture = new GestureBridge(); +} diff --git a/packages/gesture-control/src/demo/overlay.ts b/packages/gesture-control/src/demo/overlay.ts new file mode 100644 index 00000000..5a00e5d3 --- /dev/null +++ b/packages/gesture-control/src/demo/overlay.ts @@ -0,0 +1,120 @@ +// Debug overlay: draws the 21-point hand skeleton over the (mirrored) video. +// +// The