Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
09a142d14b | ||
|
|
695e4047a1 | ||
|
|
f475caab87 | ||
|
|
8453e953fd | ||
|
|
a8e0e2a343 | ||
|
|
4d0586a2aa | ||
|
|
67a15b5949 | ||
|
|
6bf69a82c8 | ||
|
|
d2efaa255b | ||
|
|
a721af4552 | ||
|
|
e6b18fd126 | ||
|
|
6ee88be549 | ||
|
|
1316725fdc | ||
|
|
187ce653ae | ||
|
|
da00fa6038 | ||
|
|
a36543c1b9 | ||
|
|
dea015dc91 | ||
|
|
333dc047c3 | ||
|
|
a51c17170e | ||
|
|
6ea73a9251 | ||
|
|
20c01d5b11 | ||
|
|
ce65d5f2ad | ||
|
|
cc45191c62 | ||
|
|
d897c9a1cf | ||
|
|
880b63d2a0 | ||
|
|
eb874339dd | ||
|
|
29d3fd48c1 | ||
|
|
cf6fabc070 | ||
|
|
62b7c4903b | ||
|
|
94b26f7606 | ||
|
|
ef01fb35b3 | ||
|
|
b5ea7112a9 | ||
|
|
95b00357b6 | ||
|
|
59145c48fc | ||
|
|
8dc850f845 | ||
|
|
eea84db05e | ||
|
|
ceca85365c | ||
|
|
44439c951b | ||
|
|
b2f8b03b3c | ||
|
|
afea6d6a1c | ||
|
|
2e341e3897 | ||
|
|
5459da5f9d | ||
|
|
b00a680d42 | ||
|
|
e3c496e1a4 | ||
|
|
eb831487a0 | ||
|
|
68594ac395 | ||
|
|
ec38fd11bf | ||
|
|
06f9ff6d9c | ||
|
|
257695ff8e | ||
|
|
2cfccc745f | ||
|
|
016c23934f | ||
|
|
896dc5b177 | ||
|
|
196646a7ff | ||
|
|
1b652ceb87 | ||
|
|
8abf349cfc | ||
|
|
ae5bcf9330 | ||
|
|
78d5fcf70c | ||
|
|
1ff315a1e6 | ||
|
|
08de6667ab | ||
|
|
d27f8e77f7 | ||
|
|
e248cd8bcf | ||
|
|
73d81afd4d | ||
|
|
7884a37c55 | ||
|
|
ad89a97106 | ||
|
|
0600b7843e | ||
|
|
6d896c781e | ||
|
|
930492058b | ||
|
|
101cee0cec | ||
|
|
7752325c90 | ||
|
|
6b284598cf | ||
|
|
94bcf524a2 | ||
|
|
98966def03 | ||
|
|
e87b03b6c2 | ||
|
|
edd494ec5f | ||
|
|
00721069e1 | ||
|
|
453a5383d2 | ||
|
|
e7b95ae579 | ||
|
|
56c2c29009 | ||
|
|
7beec7194a | ||
|
|
dcc814f40c | ||
|
|
b7e94e7068 | ||
|
|
eade261763 | ||
|
|
41a82fcf02 | ||
|
|
eecf74c001 | ||
|
|
23b4dfcd82 | ||
|
|
e017b275fe | ||
|
|
e8a809ea80 | ||
|
|
8006cc5db3 | ||
|
|
0ded279b55 | ||
|
|
aa5724c390 | ||
|
|
f21df2a9fb | ||
|
|
e549e15cb8 | ||
|
|
d07b59db4e | ||
|
|
a5a7e0c94c | ||
|
|
79d7117e6d | ||
|
|
3cf486730b | ||
|
|
996b096849 | ||
|
|
ffa7fcf839 | ||
|
|
a1c69f7405 | ||
|
|
534899bc2b | ||
|
|
03d91ffddd | ||
|
|
6280998bd8 | ||
|
|
02e2f3e8b5 | ||
|
|
41300f0a34 | ||
|
|
adbc083426 | ||
|
|
3754bcd1aa | ||
|
|
f2f909ca9c | ||
|
|
1c3f2f6571 | ||
|
|
8da1bdf690 | ||
|
|
a93325b312 | ||
|
|
2d03e4efc9 | ||
|
|
ab7c502c2a | ||
|
|
546bbcbe7c | ||
|
|
774d5ff321 | ||
|
|
98ceb5da1d | ||
|
|
34fb5e49f8 | ||
|
|
002cf81b1e | ||
|
|
9b4aab2502 | ||
|
|
829c797726 | ||
|
|
85da3bb898 | ||
|
|
29b2653801 | ||
|
|
7b8b175133 | ||
|
|
c3027b21e1 | ||
|
|
0b231edd43 | ||
|
|
ea1c2ee4ec | ||
|
|
b4a808adcf | ||
|
|
f3cbe9bca6 | ||
|
|
a11bcb0029 | ||
|
|
47fd9a922f | ||
|
|
14f7d8298d | ||
|
|
d32f4debb2 | ||
|
|
3cb7b510f8 | ||
|
|
6a12a72c9c | ||
|
|
f1a126efeb | ||
|
|
12fd780af8 | ||
|
|
fd74a42933 | ||
|
|
7101e64800 | ||
|
|
1b10d9b733 | ||
|
|
5078f5251d | ||
|
|
196af8fba7 | ||
|
|
a9b22b86a4 | ||
|
|
28cace5858 | ||
|
|
0a594b61bd | ||
|
|
89d787a949 | ||
|
|
bd9797b68c | ||
|
|
0e6cd94312 | ||
|
|
24a6f1cac8 | ||
|
|
8e679a280b | ||
|
|
c642689bbd | ||
|
|
28a6247c27 | ||
|
|
0ceb455c4b | ||
|
|
e51117dfa9 | ||
|
|
13d41cf7c7 | ||
|
|
2c7557d002 | ||
|
|
53b473708f | ||
|
|
2cba393ae5 | ||
|
|
64b8ea30b2 | ||
|
|
5743af3339 | ||
|
|
2011bd8d89 | ||
|
|
b76724690d | ||
|
|
f277f9664c | ||
|
|
cd49171bbc | ||
|
|
0f57342b10 | ||
|
|
e1f0ac993a | ||
|
|
a84ef52992 | ||
|
|
692c894760 | ||
|
|
ad0acb6d58 | ||
|
|
d866c8f30e | ||
|
|
28537de39d | ||
|
|
ba09184efa | ||
|
|
93719b41cd | ||
|
|
a448983be3 | ||
|
|
3145eac6d9 | ||
|
|
e3c609f5f0 | ||
|
|
52e774f83c | ||
|
|
82d08df53f | ||
|
|
2709b2fe49 | ||
|
|
b1d3b27e5b | ||
|
|
47963b54fa | ||
|
|
b7c3c30c8c | ||
|
|
0d80524f10 | ||
|
|
a9d83ec4e3 | ||
|
|
84137cdba4 | ||
|
|
6eb3969816 | ||
|
|
de49437a6f | ||
|
|
6a27639083 | ||
|
|
eb1b38c718 | ||
|
|
ea7b103b47 | ||
|
|
bec8e2f9ee | ||
|
|
2203f3a347 | ||
|
|
867a10d78a | ||
|
|
e54d7badc4 | ||
|
|
40dfac3534 | ||
|
|
e899a43a18 | ||
|
|
0cab8a7ece | ||
|
|
7b7cf958c0 | ||
|
|
6e64ddd853 | ||
|
|
afea91b92b | ||
|
|
9449a8f157 | ||
|
|
d322f17f73 | ||
|
|
61b5ec095c | ||
|
|
497ca4891a | ||
|
|
580b7a3f90 | ||
|
|
34c3d8f5ff | ||
|
|
2491471ba5 | ||
|
|
0c4aac8029 | ||
|
|
d436c6375f | ||
|
|
e96baf9f66 | ||
|
|
7b8aa529f2 | ||
|
|
0ad4e0ea24 | ||
|
|
6bc403d88d | ||
|
|
1ad05a5a42 | ||
|
|
192690911f | ||
|
|
551461cb31 | ||
|
|
c4bae75c59 | ||
|
|
88c415fc37 | ||
|
|
7175e4b350 | ||
|
|
08a417997f | ||
|
|
e6cb89b0cd | ||
|
|
d072e773d8 | ||
|
|
a649c91b68 | ||
|
|
3383c23099 | ||
|
|
c3e1e731ef | ||
|
|
405b711c3a | ||
|
|
4295faefc9 | ||
|
|
93e1ba5110 | ||
|
|
abbbf9e90a | ||
|
|
3a41de7b57 | ||
|
|
8267edc6fe | ||
|
|
78c568e5f7 | ||
|
|
cc624d2575 | ||
|
|
5844720525 | ||
|
|
393a2d9c28 | ||
|
|
809bf6a614 | ||
|
|
da71d8d01c | ||
|
|
e5aca6aa4c | ||
|
|
ceaf4624a1 | ||
|
|
a6597e4a9a | ||
|
|
f869e823af | ||
|
|
8d0b179f94 | ||
|
|
98fa55b7b2 | ||
|
|
c46ac30631 | ||
|
|
dfcc14bfd2 | ||
|
|
a068008409 | ||
|
|
0aa31f100e | ||
|
|
314a160458 | ||
|
|
e7ee5595c5 | ||
|
|
625d4976d3 | ||
|
|
d02cddece6 | ||
|
|
abbc4b13fd | ||
|
|
a14e47e19c | ||
|
|
754a966b53 | ||
|
|
28dfc279d4 | ||
|
|
da85e9738b | ||
|
|
8b8907c4ec | ||
|
|
2329dab240 | ||
|
|
31ce7405a6 | ||
|
|
06f7d40c42 | ||
|
|
05eba70598 | ||
|
|
d27974ff6e | ||
|
|
3cca5380ba | ||
|
|
6d7efc13e6 | ||
|
|
63f86807ad | ||
|
|
2e4e646c06 | ||
|
|
507423b776 | ||
|
|
67d0b0b538 | ||
|
|
26cfd8b7ef | ||
|
|
208e6bc175 | ||
|
|
6d52b16edc | ||
|
|
4988e85901 | ||
|
|
e799c83b39 | ||
|
|
0717cfbfec | ||
|
|
b7b2555dc0 | ||
|
|
a609c435fa | ||
|
|
3268a12e5e | ||
|
|
1f30ed445c | ||
|
|
415f02e680 | ||
|
|
d5814947d6 | ||
|
|
b620511d0e | ||
|
|
525f02f502 | ||
|
|
e07c59477d | ||
|
|
2b8a522cbd | ||
|
|
4abe055182 | ||
|
|
15a3b3996b | ||
|
|
1b76e6e2e2 | ||
|
|
f8b81b8478 | ||
|
|
d79e25d5d8 | ||
|
|
b1df5319d5 | ||
|
|
fc92d8a8a4 | ||
|
|
c7cd4f9e17 | ||
|
|
9433de75b8 |
@@ -11,10 +11,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
@@ -22,6 +22,9 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Check package-lock.json version sync
|
||||
run: npm run check:lockfile
|
||||
|
||||
- name: Type check
|
||||
run: npm run typecheck
|
||||
|
||||
@@ -31,6 +34,32 @@ jobs:
|
||||
- name: Format check
|
||||
run: npm run format:check
|
||||
|
||||
- name: Server boot smoke test
|
||||
run: |
|
||||
set -u
|
||||
if ! command -v tmux >/dev/null; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y tmux
|
||||
fi
|
||||
npx tsx src/index.ts web --port 3151 > /tmp/boot.log 2>&1 &
|
||||
SERVER_PID=$!
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
for i in $(seq 1 30); do
|
||||
if curl -fsS http://localhost:3151/api/status -o /dev/null; then
|
||||
echo "Server booted in ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
if ! kill -0 $SERVER_PID 2>/dev/null; then
|
||||
echo "Server exited before becoming ready. Logs:"
|
||||
cat /tmp/boot.log
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "Server did not respond on /api/status within 30s. Logs:"
|
||||
cat /tmp/boot.log
|
||||
exit 1
|
||||
|
||||
# Note: The test suite is intentionally excluded from CI.
|
||||
# Tests spawn real tmux sessions and require a full system environment.
|
||||
# Run tests locally with: npx vitest run test/<file>.test.ts
|
||||
|
||||
@@ -16,12 +16,12 @@ jobs:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
cache: npm
|
||||
registry-url: https://registry.npmjs.org
|
||||
|
||||
|
||||
@@ -48,7 +48,29 @@ Thumbs.db
|
||||
# Generated output
|
||||
out/
|
||||
screenshots-echo-diag/
|
||||
tools/remotion/out/
|
||||
scripts/remotion/out/
|
||||
|
||||
# Artifacts that should not be tracked
|
||||
test-results/
|
||||
tmp/
|
||||
# Root `public` (a symlink to scripts/remotion/public — local artifact). ANCHORED
|
||||
# with a leading slash so it does NOT also match src/web/public (a bare `public`
|
||||
# would swallow the whole web UI source dir and silently un-stage any new asset
|
||||
# added there). No trailing slash so it still matches the symlink, not just dirs.
|
||||
/public
|
||||
|
||||
# Opt-in gesture overlay runtime assets: large MediaPipe wasm + model (~27 MB)
|
||||
# fetched at build/install by scripts/fetch-gesture-assets.mjs, kept out of git.
|
||||
# (The gesture bundle itself, gesture-codeman.js, IS tracked — built from
|
||||
# packages/gesture-control source by `npm run build:gesture`.)
|
||||
src/web/public/gesture/wasm/
|
||||
src/web/public/gesture/*.task
|
||||
|
||||
# Gesture-control workspace package build outputs (source is tracked; the
|
||||
# Codeman bundle is emitted to src/web/public/gesture/gesture-codeman.js instead).
|
||||
packages/gesture-control/dist/
|
||||
packages/gesture-control/dist-codeman/
|
||||
packages/gesture-control/.vite/
|
||||
|
||||
# Claude Code plan tracking
|
||||
plan.json
|
||||
@@ -61,3 +83,6 @@ commands
|
||||
todo.md
|
||||
@fix_plan.md
|
||||
readme-preview.mjs
|
||||
|
||||
# Uploaded images land here under each session working dir (runtime artifact)
|
||||
.claude-images/
|
||||
|
||||
@@ -2,8 +2,27 @@ dist/
|
||||
coverage/
|
||||
node_modules/
|
||||
src/web/public/vendor/
|
||||
src/web/public/gesture/
|
||||
src/web/public/app.js
|
||||
src/web/public/styles.css
|
||||
src/web/public/mobile.css
|
||||
src/web/public/index.html
|
||||
tools/
|
||||
# Hand-formatted public JS modules (never prettier-enforced; the new
|
||||
# check-public-assets.mjs still validates NUL bytes + JS syntax on these).
|
||||
src/web/public/constants.js
|
||||
src/web/public/image-input.js
|
||||
src/web/public/input-cjk.js
|
||||
src/web/public/keyboard-accessory.js
|
||||
src/web/public/notification-manager.js
|
||||
src/web/public/orchestrator-panel.js
|
||||
src/web/public/panels-ui.js
|
||||
src/web/public/ralph-panel.js
|
||||
src/web/public/ralph-wizard.js
|
||||
src/web/public/respawn-ui.js
|
||||
src/web/public/session-ui.js
|
||||
src/web/public/settings-ui.js
|
||||
src/web/public/sw.js
|
||||
src/web/public/terminal-ui.js
|
||||
src/web/public/voice-input.js
|
||||
src/web/public/upload.html
|
||||
scripts/remotion/
|
||||
|
||||
@@ -1,5 +1,612 @@
|
||||
# aicodeman
|
||||
|
||||
## 0.9.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Vendor the gesture-control source into the repo for in-tree development.
|
||||
|
||||
The hand-tracking overlay's source (previously the standalone `Ark0N/codeman-gesture-control` repo) now lives at `packages/gesture-control/` as the `codeman-gesture-control` workspace package: the transport-agnostic gesture core (`src/gesture/*` — MediaPipe GestureRecognizer → One-Euro-filtered cursor → pinch state machine), the Codeman consumer entry (`src/codeman/entry.ts`, maps grab/drag/drop onto real session tabs + toolbar buttons), and a standalone vite playground for iterating on gesture feel.
|
||||
- New `npm run build:gesture` (`scripts/build-gesture-bundle.mjs`) esbuild-bundles `entry.ts` into the served `src/web/public/gesture/gesture-codeman.js`; `scripts/build.mjs` now reruns it on every production build so the served bundle always reflects current source. The MediaPipe wasm + model stay runtime-loaded from same-origin `/gesture/` (unchanged).
|
||||
- Added `@mediapipe/tasks-vision@0.10.21` as the package dependency (kept in sync with `fetch-gesture-assets.mjs`). The playground uses vite 7 (no known advisories).
|
||||
|
||||
No change to the shipped app behavior — gesture control remains opt-in (`CODEMAN_GESTURE=1` + the App Settings → Input toggle). This release just makes the overlay developable inside the Codeman repo.
|
||||
|
||||
## 0.9.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Multi-monitor & settings UX fixes.
|
||||
- **Multi-monitor button (remote servers):** the "span displays" button spawns `scripts/span-codeman.sh` server-side, so on a non-macOS Codeman server it can't open a window on your machine. The non-macOS API error now explains this and points to running the script locally on your Mac with the remote server URL; the script header documents the same remote-client workflow.
|
||||
- **App Settings modal:** stop the modal overflowing horizontally on narrow viewports.
|
||||
- **systemd:** sync the `codeman-web.service` template with the deployed unit.
|
||||
|
||||
## 0.9.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Security hardening release: network-bind policy, auth lockout recovery, download/SVG hardening, dependency & supply-chain fixes, tmux launch reliability, and a full security-architecture doc.
|
||||
|
||||
**Network binding (COD-29, #107):**
|
||||
- The web server now defaults to binding `127.0.0.1` (loopback) instead of `0.0.0.0`, so a fresh install is reachable only from the same machine and needs no password. New `--host` / `-H` / `CODEMAN_HOST` flag to choose the bind host.
|
||||
- Binding a non-loopback host **without** `CODEMAN_PASSWORD` no longer refuses to start — it **starts and prints a loud warning** with the three ways to secure it (set `CODEMAN_PASSWORD`, bind loopback + an authenticated tunnel / `tailscale serve`, or acknowledge with `--allow-unauthenticated-network` / `CODEMAN_ALLOW_UNAUTHENTICATED_NETWORK=1`). This keeps Codeman "just working" for new users while making remote exposure a guided, explicit choice. Host classification lives in the new `src/web/network-auth-policy.ts` (handles `127.0.0.0/8`, `::1`, `::ffff:127.*`, bracketed IPv6).
|
||||
- A post-install security note now explains the loopback default and how to expose safely.
|
||||
|
||||
**Authentication (COD-29, #107):**
|
||||
- Auth lockout now recovers gracefully: the per-IP rate-limit (`429`) check runs **after** the cookie/credential checks, so a valid session cookie or correct password is never locked out by a prior attacker's failures from the same IP (important behind a shared-IP tunnel). Wrong credentials are still counted and still hit the limit, and a `Retry-After` header is returned.
|
||||
|
||||
**Downloads & content-type hardening (COD-29, #107):**
|
||||
- New session-scoped `POST /api/download` route: realpath-bounded to the session working dir, a sensitive-path blocklist (`/etc/shadow`, `~/.ssh/`, `.env`, `*credentials*`, …), `isFile()` + 50 MB cap, forced `attachment`.
|
||||
- Workspace `.svg` files are served as `application/octet-stream` + `attachment` + `nosniff` (closes a stored-XSS-via-SVG vector); `nosniff` now applies to all `file-raw` responses.
|
||||
|
||||
**Dependencies & supply chain (COD-28, #106):**
|
||||
- Bumped security-sensitive deps to patched versions (`@fastify/static` 9, `fastify` 5.8, `uuid` 14, `vitest` 4.1, …) and added `overrides` for patched transitives (`picomatch`, `basic-ftp`, `fast-uri`, `flatted`); `npm audit` goes from 7 advisories to 0.
|
||||
- New `npm run check:public-assets` (`scripts/check-public-assets.mjs`): scans `src/web/public/**` for literal NUL bytes and runs `node --check` on every `.js` file, plus a Prettier pass on maintained files. Removed literal NUL placeholders from `app.js`. Added `test/dependency-security.test.ts` and `test/frontend-public-tooling.test.ts`.
|
||||
|
||||
**tmux launch reliability (COD-31, #110):**
|
||||
- New tmux sessions and respawns launch from a stable `/tmp` and `cd` into the workspace inside the pane, avoiding `new-session` crashes when a FUSE/rclone-mounted workspace has a transient mount blip at launch. The `cd "<dir>" && <cmd>` form is fail-safe (the CLI never runs in `/tmp`) and the path is validated + double-quoted.
|
||||
|
||||
**Test stability (COD-30, #108):**
|
||||
- Cleared leaked auth env in the Vitest setup, corrected stale route status-code / SSE-lifecycle expectations to match shipped behavior, updated the mobile keyboard accessory expectations, and measured DOMContentLoaded via browser navigation timing. Also fixed the `WebServer` title tests for the new `host` constructor arg + async `renderIndexHtml`.
|
||||
|
||||
**Docs:**
|
||||
- New `docs/security-architecture.md` documenting the full model (network binding, auth pipeline, the tunnel `req.ip` caveat, file-serving hardening, supply-chain, multi-instance isolation, security headers, and recommended secure setups). CLAUDE.md updated accordingly.
|
||||
|
||||
## 0.8.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Session detach/undock, opt-in gesture-control overlay, multi-monitor spanning, new App-Settings toggles, and asset cache-busting.
|
||||
- **Session detach/undock + instance isolation (#103):** Detach a session into its own solo (popup) window from the tab strip. Adds multi-instance isolation primitives in `src/config/instance.ts` (`getDataDir()`/`dataPath()`/`DEFAULT_TMUX_SOCKET`) keyed off `CODEMAN_INSTANCE`, so a beta can run side-by-side with prod without discovering/attaching to prod's live tmux sessions or clobbering its `state.json`. `CODEMAN_INSTANCE` defaults to the production layout (`~/.codeman`, `-L codeman`, port 3000), so master installs are unaffected. Adds `scripts/run-beta.sh` (`CODEMAN_INSTANCE=beta` + `CODEMAN_PORT=5000`). The legacy `~/.claudeman` migration is now scoped to the default instance only. Hardened detach edge cases. Tests: `test/config/instance.test.ts`.
|
||||
- **Gesture-control overlay (Phase 5, opt-in via `CODEMAN_GESTURE=1`):** Camera hand-tracking overlay (self-hosted MediaPipe — wasm + model fetched at install/build via `scripts/fetch-gesture-assets.mjs` rather than committed). `CODEMAN_GESTURE=1` makes the feature _available_ (CSP widening + `/gesture/` assets + `window.__codemanGestureAvailable`); the per-user **Gesture Control (beta)** toggle (App Settings → Display → Input, default OFF) is the actual on/off and reloads the page to inject/remove the bundle. Dashboard-only (not solo popups). Labeled "(beta)" (#109).
|
||||
- **Multi-monitor button:** Header button (opt-in via App Settings → Display → Header Displays) that POSTs `/api/system/span-displays` to spawn `scripts/span-codeman.sh` — a maximized browser `--app` window sized to the union of all displays, so the gesture layer's floating panels can drag across the physical monitor seam. Tests: `test/routes/system-span-displays.test.ts`.
|
||||
- **New App-Settings toggles (#105):** Gesture control and the multi-monitor button are both opt-in (default OFF), with live show/hide on save.
|
||||
- **Asset cache-busting:** `renderIndexHtml` appends `?v=<mtime>` to every same-origin `.js`/`.css` reference; `index.html` is served `no-cache`, so a normal reload picks up edited modules/styles without a hard refresh. Tests: `test/render-index-html.test.ts`.
|
||||
- **Gesture Control toggle placement:** the toggle now lives inside the existing **Input** settings section (alongside Local Echo / CJK Input / Extended Keyboard Bar) instead of a duplicate "Input" section; only the toggle itself is hidden when `CODEMAN_GESTURE=1` is unset, leaving the rest of the section intact.
|
||||
- **Service env:** `scripts/codeman-web.service` now sets `CODEMAN_GESTURE=1` so the gesture feature is available on the local install (still gated behind the default-OFF per-user toggle).
|
||||
- **Docs:** CLAUDE.md updated for the orchestrator loop, multi-monitor/span-displays, cache-busting, gesture/multi-monitor toggles, and structural-count fixes.
|
||||
|
||||
## 0.8.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Thinking Effort now flows as a soft default the user can override in-session (PR #104, by @TeigenZhang).
|
||||
|
||||
Previously Codeman carried the effort setting as the `CLAUDE_CODE_EFFORT_LEVEL` env var, which Claude Code treats as a hard override — it locked effort for the whole session and rejected in-session `/effort` switching (including switching to `ultracode`). Effort is now injected at spawn time as a CLI soft default that `/effort` can still change freely in either direction:
|
||||
- Regular levels (`low`/`medium`/`high`/`xhigh`/`max`) are passed via `claude --effort <level>` (the settings `effortLevel` key silently drops `max`, so the flag is used instead).
|
||||
- `ultracode` (xhigh effort + standing dynamic-workflow orchestration) is passed via `claude --settings '{"ultracode":true}'`, since the `--effort` flag rejects it.
|
||||
|
||||
Details:
|
||||
- New `effort` field on the create-session, quick-start, and Ralph-loop request schemas; threaded through `Session._effort` to both spawn paths (tmux `buildSpawnCommand` and direct-PTY `buildInteractiveArgs`), persisted in `SessionState.effort`, and restored on reboot recovery.
|
||||
- `buildEffortCliArgs()` is the single, allowlist-validated source for both carriers (injection-safe).
|
||||
- Settings UI adds an "Ultracode (multi-agent workflows)" option to the Thinking Effort dropdown; the frontend no longer emits `CLAUDE_CODE_EFFORT_LEVEL`.
|
||||
- Legacy migration: sessions persisted with the old env var are auto-migrated into the new `effort` field, and the stale tmux env var is unset so respawned panes are no longer locked.
|
||||
- Adds `test/effort-injection.test.ts` (13 cases) covering carrier mapping, injection guards, args building, and constructor migration.
|
||||
|
||||
## 0.8.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Event-loop responsiveness fix, mobile image upload, response-viewer polish, and a mobile-UI trim.
|
||||
- **fix: avoid event-loop stalls from synchronous tmux/ps calls (#100):** The session manager ran `execSync` for tmux mouse-mode toggles, `list-panes`, and `ps`/`pgrep` resource-stat queries on the main thread. Under multi-session / many-pane load these blocking spawns froze Node's single event loop, stalling SSE broadcasts and PTY I/O (the ":3000 briefly unreachable, process never restarts" class of incident). Converted those calls to async `execAsync` and updated all callers to `await`. Added a lightweight `utils/event-loop-monitor.ts` that samples loop-delay and logs when a stall threshold is exceeded, started on web-server boot and stopped on shutdown — so future regressions leave a timestamped, quantified log line instead of vanishing silently.
|
||||
- **feat(web): mobile image upload to active session via paste dialog (#101):** The mobile keyboard-accessory paste dialog now attaches images, not just text — via a native picker (`accept=image/*` → camera / photo library / files) plus best-effort capture of images pasted into the textarea. Both paths reuse the existing `_uploadAndInsertImages()` → `POST /api/sessions/:id/paste-image` pipeline. Images are re-encoded client-side before upload (PNG→PNG to preserve transparency, everything else→JPEG, animated GIFs passed through untouched) so the bytes always match their declared extension — fixing the Android/MIUI case where a WebP/HEIF mislabeled as `image/jpeg` passed the extension allowlist but failed the server's magic-byte check. The server logs a precise diagnostic on any remaining magic-byte mismatch.
|
||||
- **feat(web): response-viewer transcript fallback + code-block rendering (#102):** A substantial response-viewer styling overhaul — proportional prose font (monospace kept for code), refined heading/code/blockquote/list styling, readable max content width, and a smoother slide-in animation; the `.rv-text` rules now also apply to `.response-viewer-body` so transcript-missing fallback content gets the same typography. Plus a `_renderMarkdown` null-safety fix (`text` → `src = text || ''`).
|
||||
- **feat(web): remove /compact button from the mobile keyboard accessory bar:** Dropped `/compact` from both the simple and extended accessory-bar layouts and the associated action handling. `/clear` retains its double-tap confirmation. Verified on a touch-emulated viewport that neither layout renders a compact action.
|
||||
|
||||
## 0.7.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- **fix(respawn): auto-accept now fires on plan approvals after `Worked for X` line, and on AskUserQuestion menus**
|
||||
|
||||
Two related blockers in the respawn controller's auto-accept path:
|
||||
- Modern Claude Code emits `✻ Worked for Xm Ys` immediately before a plan-approval menu. `_detectCompletionMessage()` cancelled the auto-accept timer and `canAutoAccept()` then rejected on `completionMessageTime !== null`, so plan approvals **never** auto-accepted — the 10 s completion-confirm timer instead started a respawn cycle while the menu sat unanswered.
|
||||
- The same logic in `signalElicitation()` set a hard flag that blocked auto-accept whenever Claude Code fired the `elicitation_dialog` hook, contradicting the in-UI hint ("Auto-accept presses Enter for plan approvals **and default question options**"). AskUserQuestion menus were therefore never auto-accepted either.
|
||||
|
||||
Fix:
|
||||
- `_detectCompletionMessage()` no longer cancels the auto-accept timer; the auto-accept pre-filter is now the authoritative "is there a numbered selection menu?" gate.
|
||||
- `canAutoAccept()` and the AI-plan-check callback both accept `'watching'` AND `'confirming_idle'` states (covers the single-PTY-burst case where `Worked for` and the menu arrive together — `_detectCompletionMessage` returns early before the substantial-output check can demote state back to watching). `sendAutoAcceptEnter()` self-transitions back to `'watching'` before sending Enter.
|
||||
- `signalElicitation()` is now an affirmative hint that primes the auto-accept timer instead of blocking. Still gated on `config.autoAcceptPrompts` AND state ∈ {`watching`, `confirming_idle`} — never fires Enter when respawn is off or auto-accept is disabled.
|
||||
- AI plan-check prompt broadened to recognize AskUserQuestion / elicitation menus as valid for auto-accept (the verdict name `PLAN_MODE` is preserved for compatibility but now means "auto-accept this selection menu").
|
||||
- Removed the now-unused `elicitationDetected` field and its assignments.
|
||||
|
||||
Two new regression tests cover both the separate-PTY-chunk and single-PTY-chunk cases; the previously misleading "should NOT send Enter when completion message was detected" test was renamed and re-scoped to clarify it tests the **no-menu** path (which still correctly rejects via the pre-filter).
|
||||
|
||||
**docs(web): correct `sendPendingCtrlL` comment** — removed the stale "called by foo/bar" note from the dead-call-graph helper after #99.
|
||||
|
||||
## 0.7.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Response viewer & terminal-stability improvements, plus test/error-handling hardening.
|
||||
- **Copy button on code blocks (#98):** Every fenced code block in the response viewer now has a one-click copy button pinned to its top-right, outside the `<pre>` scroll container so it stays put during horizontal scroll. ASCII diagrams keep their line-wrap toggle alongside it. Copy prefers the async Clipboard API and falls back to a hidden-textarea + `execCommand` path, so it works over plain HTTP (tunnel) too, with a brief ✓/✕ feedback state.
|
||||
- **Fix: stop auto-sending Ctrl+L from session-selection paths (#99):** A fast page refresh or SSE reconnect could fire two programmatic Ctrl+L (`\x0c`) sends within Claude Code 2.x's "clear conversation" confirmation window, silently wiping the active conversation. Removed the automatic Ctrl+L sends from `selectSession()`, `restoreTerminalSize()`, and the dead `sendPendingCtrlL()` path; redraws now rely on resize/SIGWINCH. User-initiated Ctrl+L still works. Trade-off: an occasional transient stale Ink frame right after refresh that self-heals on the next keypress — far preferable to silent data loss.
|
||||
- **Test & error-handling hardening (#97):** Repaired route-test harness error rendering via a dedicated `route-error-handler.ts`, and stopped the AI idle/plan checkers from spawning real processes during tests.
|
||||
|
||||
## 0.6.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix new-session crash after a tmux upgrade and isolate Codeman sessions on a dedicated tmux socket.
|
||||
- **Pane file-descriptor limit**: raise `ulimit -Sn` before launching the CLI (in both the spawn and respawn paths) so the newer tmux + macOS launchd combination — which hands panes a low soft `nofile` limit (256) that recent Claude Code refuses to start under — no longer kills every freshly spawned session on startup.
|
||||
- **Single-socket isolation**: all Codeman-owned tmux sessions now live on a dedicated socket (`tmux -L codeman`, overridable via `CODEMAN_TMUX_SOCKET`), fully separated from the user's default tmux server. The socket name is validated and shell-escaped at every call site.
|
||||
- **Drop the drift-prone per-session `tmuxSocket` field**: session reconciliation collapses to a single `list-panes` query against the one socket, eliminating live sessions being wrongly marked dead ("session not found") and duplicate "Restored:" tabs. Stale per-session socket tags and duplicate records are cleaned from disk on load (dedup by `muxName`, keeping the real entry over `restored-` placeholders).
|
||||
- **Route remaining bare-`tmux` call sites through the socket**: the window-size query on re-attach (previously fell back to 120×40 and lost scrollback) and the send-key route (Shift+Enter / Ctrl+Enter newline).
|
||||
- **SSH chooser scripts** (`tmux-manager.sh`, `tmux-chooser.sh`) route every tmux call through the dedicated socket.
|
||||
|
||||
## 0.6.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Resume Conversation: fixes and folder drill-down.
|
||||
- **fix(history)**: `decodeProjectKey()` now uses longest-join-first backtracking with on-disk validation, so sibling directories sharing a prefix (e.g. `diary/` vs `diary-app/`) resolve to the correct path. Previously the greedy shortest-match decoder picked the shorter name and bailed, surfacing `$HOME` in the Resume Conversation list and resuming into the wrong folder. Greedy decode is kept as a fallback so history for deleted projects still resolves. (#92)
|
||||
- **fix(tabs)**: Drop the client-side resurrection of ended-session tabs. The old code cached open tabs in `localStorage` and rebuilt them as grayed-out stubs whenever the server no longer knew them, which left phantom tabs after closing a session on another device. The server is now the single source of truth; legacy `localStorage` keys are purged on init. Net -44 / +6 lines. (#93)
|
||||
- **feat(history)**: New "View all in this folder" drill-down on Resume Conversation. `GET /api/history/sessions` accepts `projectKey` (validated against `^[A-Za-z0-9_-]+$` before any filesystem access), `offset`, and `limit`; single-folder mode bypasses the 50-cap and returns `{ sessions, total }`. Frontend adds a modal listing 20 sessions per page with a "Show more" pagination button. Modal items omit their own "View all" button to prevent recursive entry points. (#94)
|
||||
|
||||
## 0.6.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- ## Security: paste-image endpoint hardening (#90)
|
||||
|
||||
Addresses seven findings from the dismissed review of #84. Most exposed in tunneled deployments where `CODEMAN_PASSWORD` is set but the server is reachable beyond localhost.
|
||||
- **CSRF protection** on `POST /api/sessions/:id/paste-image`. Requires `Origin`/`Referer` to match `req.host`; non-browser clients (no `Origin` and no `Referer`) must send `X-Codeman-CSRF`. Defeats cross-origin `<form enctype="multipart/form-data">` submits that would otherwise plant arbitrary bytes into the victim's `.claude-images/` while their session cookie is live.
|
||||
- **Magic-byte validation** on uploaded images. Sniffs the first 12 bytes against PNG/JPEG/GIF/WebP/BMP signatures and rejects 415 on mismatch. Polyglot HTML-or-SVG-with-image-MIME no longer round-trips through the endpoint.
|
||||
- **Symlink-safe writes** on `.claude-images/`. `lstat` before the write, non-recursive `mkdir`, `O_EXCL|O_NOFOLLOW` on file open. A `node_modules` postinstall (or the agent itself) planting `.claude-images -> ~/.ssh/` no longer redirects pastes outside `workingDir`.
|
||||
- **Multipart parser swap** to `@fastify/multipart` with `limits: { fileSize: 10MB, files: 1, fields: 4 }`. Replaces a hand-rolled boundary scanner that matched the literal boundary anywhere in the body, hard-coded `\r\n` (silently corrupting LF-only clients), and had no part-count cap.
|
||||
- **Rate limit + GC**: token-bucket (30/min per IP+session) and hourly GC of `paste-*` files older than 7 days from each live session's `.claude-images/`. New `paste-image-gc.ts` started/stopped from `WebServer.start/stop`.
|
||||
- **Collision-free filenames**: `paste-${Date.now()}-${randomBytes(4)}${ext}`. Two tabs pasting in the same millisecond no longer silently last-write-wins.
|
||||
- **Bracketed-paste preservation**: text-only paste in `image-input.js` now goes through `terminal.paste(text)` instead of `sendInput(text)`, so xterm preserves `CSI 200~ ... CSI 201~` markers — Claude Code uses them as part of its prompt-injection defenses.
|
||||
|
||||
## Fix: duplicate multipart parser conflict
|
||||
|
||||
Removed a duplicate multipart content-type parser left behind after the swap above. The duplicate registration conflicted with `@fastify/multipart`'s own parser; uploads now flow through the plugin exclusively.
|
||||
|
||||
## WebGL renderer auto-fallback hardening (#91)
|
||||
|
||||
Follow-ups on the longtask auto-fallback shipped in #83.
|
||||
- `PerformanceObserver` is now disconnected on `onContextLoss` as well as on the trip path. Previously the observer outlived its disposed addon after a context loss, holding a closure reference over every longtask the page emitted.
|
||||
- Thresholds (`200ms / 3 longtasks / 30s window / 5s grace / 7d sticky-disable`) are hoisted to `WEBGL_FALLBACK` in `constants.js`. No more inline literals.
|
||||
- New `evaluateWebGLLongTaskTrip()` pure helper splits the rolling-window arithmetic from the `PerformanceObserver` callback so the trip math is unit-testable. New `test/webgl-fallback.test.ts` (9 tests, port 3166): trip inside window, no-trip when spread, sub-threshold filtering, stale-entry pruning, cumulative counting across batches, observer-dispose idempotency.
|
||||
|
||||
## CI: server boot smoke test
|
||||
|
||||
GitHub Actions now boots the server as a final step after typecheck/lint/format. Catches production-only ESM/CJS regressions that `tsx` masks in dev.
|
||||
|
||||
## Docs
|
||||
|
||||
`CLAUDE.md` frontend-module table updated to include `image-input.js` (overlooked when #84 landed).
|
||||
|
||||
## 0.6.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Terminal renderer hardening, SSE bandwidth cut, image paste, and a security tightening on the new live filter:
|
||||
- **Multi-primitive yield for write pacing** (#85): replaces six raw `requestAnimationFrame` callsites in the xterm.js write pipeline with a yielding helper that races `requestAnimationFrame`, `setTimeout(50)`, and a tick Worker. Keeps the terminal responsive when the tab is backgrounded or occluded — Chrome's intensive-throttling no longer stalls long writes.
|
||||
- **WebGL longtask auto-fallback** (#83): a `PerformanceObserver` watches for ≥200ms WebGL frames; three within a 30s window disposes the WebGL addon and falls back to the canvas renderer. Decision is persisted in localStorage for 7 days, and `?webgl=force` clears it.
|
||||
- **Per-client live SSE subscription filter** (#86): each connected client gets a stable UUID and can narrow its terminal stream to one session via `POST /api/events/subscribe` — no EventSource reconnect on tab switches. Cuts SSE bandwidth roughly N× when N sessions are open. Lifecycle/metadata events (`session:*`, `case:*`, `ralph:*`, `hook:*`) now broadcast to every client so sidebars stay in sync.
|
||||
- **Image paste and drag-and-drop into the terminal** (#84): `Ctrl+V` and dropped images upload to `POST /api/sessions/:id/paste-image`, save under `${workingDir}/.claude-images/paste-${ts}.${ext}` and type the path into the terminal. Hard 10MB cap, server-generated filename (no traversal), `.svg` deliberately excluded from the allowlist to avoid a same-origin XSS path through `file-raw`.
|
||||
- **SSE clientId validation**: the per-client identifier introduced in #86 is now constrained to `[A-Za-z0-9_-]{8,64}` at both ingress points. Without this, an authenticated attacker could send another tab's clientId to silently evict it from broadcasts, mutate any clientId's session filter to blackhole the victim's terminal stream, or grow `sseClientsById` unboundedly via long IDs. The subscribe payload is also capped at 64 session entries of ≤128 chars each.
|
||||
|
||||
## 0.6.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Finish the hostname-aware notification plumbing started in 0.6.7 and lock down the recent UI/runtime fixes with regression tests.
|
||||
- Browser Notification API (OS-level desktop pop-ups, layer 3 of the 5-layer notification system) now uses `${originalTitle}: ${title}` instead of the hardcoded `Codeman:` literal — so multi-host users running Codeman on laptop / dev box / NAS see `codeman:<host>: <event>` consistently across tab title, tab-flash, Web Push, and OS notifications.
|
||||
- Inline session rename hardened against three corner cases: IME composition commits (Chinese pinyin Enter no longer ships half-composed text as the session name), mid-rename SSE deletion (orphaned `<input>` no longer 404s on blur), and double-fire on stuck settle-once flag (closure-local `settled` boolean replaces the boolean instance flag).
|
||||
- Test coverage backfilled for two prior shipped fixes:
|
||||
- `<title>codeman:<host></title>` server-side templating (#82): 8 tests covering default `os.hostname()`, `--title-hostname` override, HTML-escape against `<script>`-style breakout, ampersand non-double-encoding, and template-tail byte-identical invariance.
|
||||
- tmux size-query helper (#80): 15 tests covering the browser-resize-between-attaches happy path, the query-then-die race, zero/negative/empty/non-numeric output fallbacks, and argv-form/timeout assertions that lock down the no-shell-interpolation guarantee. Inline 14-line query block extracted into a named `queryTmuxWindowSize()` export in `session.ts` so the test surface is a pure function.
|
||||
- Regression coverage added for `stripInkRedrawBloat` route helper.
|
||||
- CLAUDE.md and README.md updated to document dual-CLI env-prefix discipline (`CLAUDE_CODE_*` vs `OPENCODE_*`), the `xterm-zerolag-input` published-package side-effect of overlay edits, and the unified hostname prefix across tab title / tab-flash / OS notifications.
|
||||
|
||||
## 0.6.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- - **fix(client): preserve inline rename input across tab re-renders** (#81) — Right-click → rename on a session tab no longer loses keystrokes when SSE traffic from sibling sessions triggers a tab re-render. Adds an `_inlineRenameActive` guard at the top of `renderSessionTabs()` and `_fullRenameSessionTabs()` so the in-progress input isn't destroyed mid-typing. Also fixes a latent double-fire of `finishRename` (blur + Enter could both invoke it). Drive-by: safer DOM child clearing in place of `innerHTML = ''`.
|
||||
- **feat: hostname-aware window title** (#82) — The browser tab title is now `codeman:<hostname>` instead of the bare `Codeman` literal, so users running Codeman on multiple hosts (laptop, dev box, NAS) can tell at a glance which tab points at which backend. New `--title-hostname <name>` CLI flag overrides the detected `os.hostname()` when it's noisy or you want a cosmetic name. The title is templated into the served HTML on first byte (with narrow HTML escaping), so it's correct from the first paint and works without JavaScript. Title-flash logic now respects the per-host title.
|
||||
- **perf: larger terminal tail on tab switch** — `TERMINAL_TAIL_SIZE` raised from 128KB to 1MB. When switching back to a busy session tab you now get ~8× more scrollback restored immediately.
|
||||
- **fix: preserve response text in Ink redraw stripping** — `stripInkRedrawBloat()` rewritten from a first-VPA approach to cluster-based detection. The previous algorithm assumed all VPA escapes after the first one belonged to a single redraw region and discarded everything in between, which silently lost 100KB+ of legitimate Claude response text once a render had occurred. The new approach groups VPAs into clusters separated by ≥8KB gaps and only collapses clusters spanning ≥32KB, so streamed response content between redraw bursts is preserved.
|
||||
- **docs**: `CLAUDE.md` Additional Commands gains the `--title-hostname` row; `README.md` gets a "Hostname-Aware Window Title" subsection under Multi-Session Dashboard.
|
||||
|
||||
## 0.6.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- **Terminal scrollback significantly increased** — both the xterm.js viewport and the tmux backing buffer were bottlenecking how far back you could scroll. Three changes:
|
||||
- `DEFAULT_SCROLLBACK` raised from 20000 → 50000 lines (xterm.js, main terminal). The previous bump from 5000 only helped users with empty localStorage; existing users were stuck on whatever value they first picked up. The loader now treats `DEFAULT_SCROLLBACK` as a floor — if your stored value is below the new minimum, you're raised to it automatically.
|
||||
- Subagent / teammate terminals (`panels-ui.js`) were stuck at 5000; now use the same `DEFAULT_SCROLLBACK` constant (50000).
|
||||
- New tmux sessions now run with `history-limit 50000` (tmux defaults to 2000). This matters for hard-reload / re-attach — without it, only the last ~2000 lines survive the round-trip back into a fresh xterm.
|
||||
|
||||
**Tmux flicker on session re-attach fixed (PR #80 by @aakhter)**: the PTY now queries the existing tmux window size via `tmux display -p` before spawning, instead of hardcoding 120x40. Previously, every re-attach forced tmux to resize down to 120x40, causing a visible flicker and one frame of scrollback loss. The `-x 120 -y 40` flag was also dropped from `tmux new-session` so the initial size matches the first attaching client. Uses `execFileSync` (not shell) for safety and falls back to 120x40 on any error.
|
||||
|
||||
**Docs**: CLAUDE.md now documents two recurring foot-guns — the `xterm-zerolag-input` overlay code is duplicated between `packages/xterm-zerolag-input/src/` and inline inside `src/web/public/app.js`, so any overlay change must touch both; and the COM workflow explicitly includes a post-push `gh run watch` step to confirm CI before considering the release done.
|
||||
|
||||
## 0.6.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- **Mobile fix**
|
||||
- Android virtual keyboard: space character was silently dropped on touch devices using GBoard / SwiftKey / similar IMEs. Root cause: the input-event handler in `terminal-ui.js` treated any whitespace-only textarea value as proof that xterm had already processed the input. A lone space (`' '.trim() === ''`) tripped this guard, so the space was consumed but never forwarded. Now skips only when the textarea is truly empty (or whitespace from a non-space key). Reported and diagnosed by @coolk8 in #79.
|
||||
|
||||
**Docs**
|
||||
- `CLAUDE.md`: added Zod `.optional()`-vs-`null` gotcha (recurring trap from 0.6.3 / 0.6.4 incidents) and a more visible warning against running bare `npm test` (kills the host tmux session).
|
||||
- `docs/local-echo-overlay-plan.md`: marked SHIPPED, corrected xterm version reference (v5.3.0 → `@xterm/xterm` ^6.0.0).
|
||||
|
||||
## 0.6.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix "Failed to enable respawn: Invalid request body" error when selecting infinity duration (∞) in the respawn modal. Frontend was sending `durationMinutes: null`, which Zod's `.optional()` schema rejected (it accepts `undefined` only). The body now omits the field when no duration is selected.
|
||||
|
||||
## 0.6.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- **Fix**
|
||||
- Allowlist `opusContext1mEnabled` in `SettingsUpdateSchema`. Without this entry, the strict schema rejected `PUT /api/settings {"opusContext1mEnabled":...}` with `INVALID_INPUT`, so the toggle's value never persisted across reloads. The frontend was already reading and writing this key (`settings-ui.js:336/1137`, `session-ui.js:340`), so saves were silently failing — users never noticed because the load path falls back to `false` on missing keys, hiding the bug. (#78)
|
||||
|
||||
## 0.6.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- **Mobile UX**
|
||||
- Resume Conversation list (welcome page) reworked for narrow screens: 2-line title clamp so more of the first prompt is visible; case-aware subtitle that renders `#caseName` (or `#caseName/sub`) when `workingDir` matches a known case, otherwise falls back to the directory basename; inline `⋯` toggle that expands a detail panel with full prompt, full path, timestamp, size, and short session id; `/Users/<user>/` now collapses to `~/` alongside `/home/<user>/`. (#77)
|
||||
- Response viewer: ASCII diagram wrap toggle, dedicated mobile code-block layout, and chrome-stripping fallback when the model wraps its reply in extra markup. (#75)
|
||||
- Mobile keyboard accessory bar no longer triggers vertical scroll. (#72)
|
||||
|
||||
**Sessions & settings**
|
||||
- New `thinkingEffort` setting on session creation, with `xhigh` option and `/effort max` mobile shortcut. (#73)
|
||||
- `thinkingEffort` is now allowlisted in `SettingsUpdateSchema` so it round-trips through PATCH /api/settings.
|
||||
- `envOverrides` (`CLAUDE_CODE_*` / `OPENCODE_*`) are now passed to Claude via tmux env exports at spawn time instead of being written to `<case>/.claude/settings.local.json`. Eliminates UI/disk drift; the value lives on `Session._envOverrides`, is exported by `tmux-manager.buildEnvExports()`, and is persisted in `SessionState.envOverrides`. (#74)
|
||||
|
||||
**Fixes**
|
||||
- Eye icon (active-session indicator) now follows `/clear` to the new Claude conversation instead of getting stuck on the previous transcript. (#76)
|
||||
- `tmux-manager.reconcileSessions` now uses `|` as the field separator, fixing parsing when session names contain other delimiters. (#71)
|
||||
|
||||
**Docs**
|
||||
- CLAUDE.md: added `npm run knip` to the dead-code sweep table and a `Common Gotchas` entry documenting the `envOverrides` → tmux export flow.
|
||||
|
||||
## 0.6.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Internal cleanup and release hygiene:
|
||||
- **Dead-code sweep via knip**: added `knip.json` for dead-code detection and ran a full sweep — removed unused test files, unused scripts, and narrowed internal module exports to the minimum surface area actually consumed.
|
||||
- **Lockfile drift prevention**: `version-packages` now runs `npm install --package-lock-only` and verifies the lockfile is in sync via `scripts/check-lockfile-sync.mjs`; CI runs the same check on every push/PR so version drift fails the build instead of reaching production. Resolves the `package-lock.json` / `package.json` version mismatch that shipped in 0.6.0.
|
||||
- **Docs tightening**: archived 22 completed plan docs from `docs/`, corrected file/handler counts in `CLAUDE.md`, documented the lockfile step in the COM workflow, and removed footer redundancy.
|
||||
|
||||
## 0.6.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Community contributions from @aakhter:
|
||||
- **feat (#66): Tab reorder shortcuts** — `Ctrl+Shift+{` and `Ctrl+Shift+}` move the active session tab left/right, matching WezTerm convention. Order persists across reloads via `saveSessionOrder()`.
|
||||
- **feat (#67): Active tab visibility + Alt+N badges** — active tab now has a bright green border with color-matched glow, and the first 9 tabs display number badges hinting at the `Alt+N` switch shortcut. Badges update on reorder/rerender.
|
||||
- **feat (#68): Clipboard API** — new `POST /api/clipboard` accepting `{text}` broadcasts a `clipboard:write` SSE event; connected browsers attempt `navigator.clipboard.writeText()` with a manual-copy modal fallback when the page isn't focused. Auth-protected via the standard middleware. Useful for pushing snippets from remote sessions to the user's local clipboard.
|
||||
- **fix (#65): Android Shift+key double character** — pressing `Shift+A` on attached Android keyboards no longer produces "AA". Tracks xterm-handled keydown timestamps and skips the orphaned-input listener for 50ms after a real keydown, while still catching Gboard symbol-keyboard inputs (keyCode 229).
|
||||
|
||||
## 0.5.13
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix "Case path not found" error in Quick Start when `~/codeman-cases/` does not exist (issue #64). Two bugs in `session-ui.js`:
|
||||
- `runClaude()` auto-create read `createCaseData.case`, but `POST /api/cases` returns `{ success, data: { case } }` — corrected to `createCaseData.data.case`.
|
||||
- `runShell()` had no auto-create logic and would immediately throw on a missing case directory — now mirrors `runClaude()`'s create-on-demand flow.
|
||||
|
||||
## 0.5.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix quick-start to resolve linked cases before codeman-cases fallback. `/api/quick-start` was always resolving `caseName` against `CASES_DIR`, ignoring entries in `~/.codeman/linked-cases.json`. Sessions started via quick-start now correctly honour linked external project directories, consistent with regular case routes.
|
||||
|
||||
## 0.5.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Community contributions and security hardening:
|
||||
- Mobile response viewer: native-scroll panel for reading full Claude responses with markdown rendering via marked.js (PR #62)
|
||||
- PWA support: service worker caching, web app manifest, and Android home screen install (PR #59)
|
||||
- Named Cloudflare tunnel support (PR #58)
|
||||
- Markdown rendering for response viewer with HTML sanitization (XSS prevention) — strips dangerous elements, event handlers, and javascript: URIs
|
||||
- Service worker switched from stale-while-revalidate to network-first caching so deploys take effect immediately
|
||||
- Content-Disposition filename sanitization to prevent header injection in file downloads
|
||||
- Expose session.muxName public getter, replace unsafe `as any` cast in session-routes
|
||||
- Static import for execFile in session-routes
|
||||
- Keyboard shortcut updates: Alt+1-9 tab switching, Shift+Enter newline
|
||||
- Repo restructure for cleaner GitHub landing page
|
||||
- Mobile logo, expandable history, session resume fixes
|
||||
|
||||
## 0.5.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fix: allow bracket characters in model validation regex so models like opus[1m] (1M context window) are accepted instead of silently dropped. Quote the model flag value in tmux spawn commands to prevent bash glob expansion of bracket patterns.
|
||||
|
||||
docs: update macOS launchd instructions to use `launchctl bootstrap` instead of deprecated `load`. Clean up README install and service sections.
|
||||
|
||||
## 0.5.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Mobile keyboard accessory bar: add configurable "Extended Keyboard Bar" setting (Settings > Display > Input) that toggles between simple mode (up/down arrows, /init, /clear, /compact, paste, dismiss) and extended mode (adds left/right arrows, Tab, Shift+Tab, Ctrl+O, Alt+Enter, Esc). Default is simple mode. Setting is device-specific (not synced to server).
|
||||
|
||||
Restyle dismiss button: muted steel-blue tone, fills remaining bar space via flex, larger tap target. Arrow buttons now blue.
|
||||
|
||||
Fix paste overlay visibility on mobile: dialog repositioned to top of screen (15vh from top) so the virtual keyboard doesn't cover it. Textarea enlarged for better usability.
|
||||
|
||||
(Also includes all v0.5.8 changes: case reorder/delete, XSS sanitization, auto-attach PTY on restart, mobile keyboard buttons, macOS installer fixes, terminal flicker fix, state store collision fix.)
|
||||
|
||||
## 0.5.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Case management: add Manage tab with reorder (up/down arrows) and delete for cases; linked cases are unlinked (folder preserved), CASES_DIR cases are permanently deleted. New endpoints: DELETE /api/cases/:name, PUT /api/cases/order. SSE events: case:deleted, case:order-changed.
|
||||
|
||||
Security: sanitize case names from filesystem with /^[a-zA-Z0-9_-]+$/ regex before returning from GET /api/cases to prevent XSS via maliciously-named directories reaching frontend inline onclick handlers.
|
||||
|
||||
Auto-attach PTY: server now calls startInteractive() for recovered tmux sessions during startup so all sessions resume capturing output immediately after deploy, instead of waiting for client selection. Frontend auto-attach condition relaxed from (pid===null && status==='idle') to (pid===null && !\_ended).
|
||||
|
||||
Mobile keyboard accessory: add Shift+Tab, Tab, Esc, Alt+Enter, Left/Right arrow, and Ctrl+O buttons.
|
||||
|
||||
Terminal: fix flicker regression by moving viewport clear inside dimension guard.
|
||||
|
||||
State store: fix temp file collisions on concurrent writes.
|
||||
|
||||
macOS: fix installer failures when piped via curl | bash, add HTML cache support, launchd service template, and trust dialog handling.
|
||||
|
||||
Housekeeping: remove accidentally committed dist/state-store.js build artifact.
|
||||
|
||||
## 0.5.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- feat: support "Default (CLI default)" option for model selection. Adds a new empty-value option to the model dropdown that defers to the CLI's own default model instead of forcing a specific model. Ensures empty defaultModel values are treated as undefined when passed to session creation and Ralph loop start, preventing empty strings from being sent as model flags.
|
||||
|
||||
## 0.5.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fix: default new sessions to opus[1m] (1M context window) instead of plain opus (200k context)
|
||||
|
||||
## 0.5.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Add 1M Opus context quick setting — per-case and global toggle that writes `model: "opus[1m]"` to `.claude/settings.local.json` when creating new sessions. Fix mobile layout: banners (respawn, timer, orchestrator) between header and main content now visible by switching from margin-top on `.main` to padding-top on `.app`. Add tablet-optimized respawn banner styles and mobile phone banner refinements.
|
||||
|
||||
## 0.5.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix terminal flicker regression — re-add server-side DEC 2026 synchronized output wrapping around batched terminal data. Ink spinner frames (cursor-up + redraw cycles) do not emit their own DEC 2026 markers, so without the server wrapper each partial cursor update rendered individually causing visible flicker. Also: extract SSE stream management, session listener wiring, and respawn event wiring from server.ts into dedicated modules; deduplicate error message extraction across 7 files with shared getErrorMessage() helper; update SSE event count in CLAUDE.md (106 → 117).
|
||||
|
||||
## 0.5.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Readability refactor across 12 core files, extracting ~35 helper methods to reduce duplication:
|
||||
- state-store: extract serializeState(), split assembleStateJson() into focused sub-methods
|
||||
- session: extract \_resetBuffers() (3x dedup), \_clearAllTimers() (10 timer cleanups), \_handleJsonMessage()
|
||||
- ralph-tracker: extract completeAllTodos() (4x dedup), emitValidationWarning(), named similarity constants
|
||||
- subagent-watcher: extract markSubagentAsCompleted(), extractFirstTextContent(), emitToolResult(), findOldestInactiveAgent()
|
||||
- respawn-controller: extract recoveryResetToWatching(), canAutoAccept(), formatRemainingSeconds(), validatePositiveTimeout()
|
||||
- tmux-manager: replace 15 path.includes() with UNSAFE_PATH_CHARS regex, extract buildEnvExports/buildPathExport/\_configureOpenCode helpers
|
||||
- session-auto-ops: extract executeWhenIdle() shared retry helper, convert to options object, add validateThreshold()
|
||||
- app.js: add \_clearTimer() (11 call sites), \_isStaleSelect(), keyboard shortcut lookup table, \_cleanupPreviousSession(), \_resetAllAppState()
|
||||
- route-helpers: add readJsonConfig() (5 inline patterns replaced), validateSessionFilePath() (2 duplicated blocks replaced)
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Make buffer size limits configurable via CODEMAN\_\* environment variables (MAX_TERMINAL_BUFFER, TRIM_TERMINAL_TO, MAX_TEXT_OUTPUT, TRIM_TEXT_TO, MAX_MESSAGES), falling back to existing defaults. Allows users with fewer sessions or more RAM to tune buffer sizes without patching source.
|
||||
|
||||
Fix duplicate terminal output on tab switch to busy sessions by clearing the terminal before writing the new buffer.
|
||||
|
||||
Fix stale Ink CUP frames after tab switch by sending Ctrl+L to force a clean redraw.
|
||||
|
||||
Fix mobile CJK input handling: resolve textarea positioning, terminal flicker during composition, and layout overflow on small screens. Improve CJK composition lifecycle with better event handling and fallback flush timers.
|
||||
|
||||
## 0.5.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- refactor: codebase cleanup — extract route helpers, eliminate boilerplate, optimize hot paths
|
||||
- Add `parseBody()` helper to route-helpers.ts: validates request body against Zod schema with structured 400 error on failure, replacing 37 identical safeParse + error-check blocks across 10 route files
|
||||
- Add `persistAndBroadcastSession()` helper: combines persist + SessionUpdated broadcast into one call, replacing 5 repeated 2-line pairs
|
||||
- Migrate session-routes.ts to use `findSessionOrFail()` consistently (17 inline session lookups replaced) and `parseBody()` (12 patterns)
|
||||
- Migrate ralph-routes.ts to use `findSessionOrFail()` (9 lookups) and `parseBody()` (4 patterns)
|
||||
- Migrate 8 remaining route files to use `parseBody()` (21 patterns total)
|
||||
- Fix O(n log n) eviction in bash-tool-parser.ts: replace `Array.from().sort()[0]` with O(n) min-scan for oldest active tool
|
||||
- Extract `_debouncedCall()` utility in frontend: replaces 4 manual debounce patterns (7 lines each → 1 line) in app.js, panels-ui.js, ralph-panel.js
|
||||
- Net reduction: 208 lines removed across 16 files
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Visual redesign with glass morphism, refined colors, and polished UI. Optimize history endpoint with buffer reuse and line iterator. Fix Ink frame search window (4KB→64KB) to prevent partial frames. Fix stale terminal data on tab switch via chunkedTerminalWrite cancellation. Improve history prompt extraction with expanded command filtering and tail scan fallback. Align case select group height to match dropdown. Fix no-control-regex lint error for ANSI strip pattern. Add browser-testing-guide to CLAUDE.md references.
|
||||
|
||||
## 0.4.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- feat: improve session navigability in history and monitor panel (closes #45)
|
||||
- History items now show the first user prompt as the title with the project path as a subtitle, making it much easier to distinguish sessions from the same project
|
||||
- The `/api/history/sessions` endpoint extracts the first user message from each transcript JSONL, stripping system-injected XML tags and command artifacts, truncating to 120 chars
|
||||
- Monitor panel session rows are now clickable — clicking navigates directly to that session's tab via `selectSession()`; Kill button retains independent behavior via `stopPropagation()`
|
||||
- Updated CLAUDE.md architecture tables to reflect Orchestrator Loop additions (14 route modules, 15 type files, orchestrator domain files, orchestrator-panel.js frontend module)
|
||||
- fix: stop subagent monitor windows from auto-opening on discovery
|
||||
- feat: add Orchestrator Loop with phased plan execution, live progress during plan generation, and toolbar button (hidden until fully tested)
|
||||
- fix: patch 3 production bugs found during deep audit
|
||||
- fix: restore mobile terminal scrollback using JS scrollLines() instead of broken native scroll
|
||||
|
||||
## 0.4.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix mobile keyboard scroll and layout issues:
|
||||
- Prevent iOS Safari from scrolling the page when typing with the keyboard open (position:fixed on .app + window.scroll reset)
|
||||
- Eliminate dead space between terminal and keyboard accessory bar by removing redundant CSS padding, tightening JS padding constant, and adding row quantization gap compensation
|
||||
- Fix toolbar overlapping terminal content when keyboard is hidden by adding proper padding-bottom to .main, including iOS Safari bottom bar offset
|
||||
- Strip Ink spinner bloat from terminal buffer before tailing
|
||||
- Fix resolveCasePath priority order and suppress JSON parse warnings
|
||||
|
||||
## 0.4.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix mobile keyboard toolbar positioning on iOS Safari: toolbar (Run/Stop/Run Shell) was hidden behind the accessory bar when virtual keyboard was active due to overlapping CSS positions. Remove the aggressive safety check in `updateLayoutForKeyboard()` that incorrectly dismissed keyboard state when iOS scrolled the visual viewport during typing. Add Safari-bar CSS offset to accessory bar so it properly stacks above the toolbar. Remove the double-counted Safari-bar offset when keyboard is visible since the JS transform already covers the full distance.
|
||||
|
||||
## 0.4.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fix: mobile keyboard hides terminal content on iPhone
|
||||
|
||||
Fixed a bug where opening the virtual keyboard on iPhone left zero visible terminal space. Two independent mechanisms were both accounting for the keyboard height: `MobileDetection.updateAppHeight()` shrunk `--app-height` to the visual viewport height, while `KeyboardHandler.updateLayoutForKeyboard()` added a large `paddingBottom`. These double-counted, leaving negative space for the terminal (user saw accessory bar + toolbar but no terminal content).
|
||||
|
||||
Fix: `updateAppHeight()` now skips when the keyboard is visible, and `handleViewportResize()` restores `--app-height` to the pre-keyboard value on first detection (since MobileDetection's listener fires before KeyboardHandler's). On keyboard close, `--app-height` is re-synced to the current visual viewport.
|
||||
|
||||
## 0.4.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Refactor case routes: extract readLinkedCases() and resolveCasePath() helpers to eliminate 6x duplicated linked-cases.json path construction and 5x duplicated file read/parse logic. Replace O(n) .some() duplicate check with O(1) Set.has() in case listing. Un-export unused isError() type guard. Standardize reply.status() to reply.code() in system routes. Update CLAUDE.md frontend module listing and SSE event count.
|
||||
|
||||
## 0.4.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Extract monolithic app.js (~12.5K lines) into 6 focused domain modules that extend CodemanApp.prototype via Object.assign: terminal-ui.js (terminal setup, rendering pipeline, controls), respawn-ui.js (respawn banner, countdown, presets, run summary), ralph-panel.js (Ralph state panel, fix_plan, plan versioning), settings-ui.js (app settings, visibility, web push, tunnel/QR, help), panels-ui.js (subagent panel, teams, insights, file browser, log viewer), session-ui.js (quick start, session options, case settings). Fix critical deferred script init ordering bug: wrap CodemanApp instantiation in DOMContentLoaded so all defer'd mixin modules execute their Object.assign before the constructor runs. Guard missing cleanupWizardDragging() call in subagent-windows.js. Update build.mjs to minify/hash all new modules.
|
||||
|
||||
## 0.4.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Performance optimizations: V8 compile cache for 10-20% faster cold starts, lazy-load WebGL addon (244KB saved on mobile), preload hints for critical scripts, batch tmux reconciliation (N subprocess calls → 1). Also: WebSocket session lifecycle fixes, CJK IME input support, CI upgrade to Node 24/actions v6, install.sh fork support, and CLAUDE.md/README documentation refresh.
|
||||
|
||||
## 0.4.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- Add CJK IME input textarea for xterm.js terminal (env toggle INPUT_CJK_FORM=ON). Always-visible textarea below terminal handles native browser IME composition, forwarding completed text to PTY on Enter. Supports arrow keys, Ctrl combos, backspace passthrough, and Escape to clear.
|
||||
|
||||
Add fork installation support to install.sh with CODEMAN_REPO_URL and CODEMAN_BRANCH env vars, allowing custom repository and branch for git clone/update operations. README updated with fork installation instructions.
|
||||
|
||||
Fix WebSocket session lifecycle: close WS connections when session exits (prevents orphaned listeners and stale writes to dead PTY), add readyState guard in onTerminal to stop buffering after socket closes, simplify heartbeat by removing redundant alive flag.
|
||||
|
||||
Add WebSocket reconnection with exponential backoff (1s-10s) on unexpected close, skipping server rejection codes (4004/4008/4009). Falls back gracefully to SSE+POST during reconnection.
|
||||
|
||||
Clear CJK textarea on session switch to prevent sending stale text to wrong session.
|
||||
|
||||
## 0.3.12
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Add WebSocket terminal I/O with server-side DEC 2026 synchronized update markers. Replaces per-keystroke HTTP POST + SSE terminal output with a single bidirectional WebSocket connection for dramatically lower input latency. Server-side 8ms micro-batching with 16KB flush threshold groups rapid PTY events into single WS frames wrapped in DEC 2026 markers for flicker-free atomic rendering. Includes 30s ping/pong heartbeat with 10s timeout for stale connection detection through tunnels. Existing SSE + HTTP POST paths remain fully functional as transparent fallback. Resize messages validated to match HTTP route bounds (cols 1-500, rows 1-200, integers only). 16 automated route tests added for WS endpoint. Also patches 5 dependency vulnerabilities (basic-ftp, fastify, minimatch, serialize-javascript).
|
||||
|
||||
## 0.3.11
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- ### Session Resume & History
|
||||
- Add `resumeSessionId` support for conversation resume after reboot
|
||||
- Add history session resume UI and API with route shell sessions routing fix
|
||||
- Improve session resume reliability and persist user settings across refresh
|
||||
- Correct `claudeSessionId` for resumed sessions
|
||||
|
||||
### Terminal & Frontend
|
||||
- Upgrade xterm.js 5.3 → 6.0 with native DEC 2026 synchronized output
|
||||
- Increase terminal scrollback from 5,000 to 20,000 lines
|
||||
- Reduce default font size and persist tab state across refresh
|
||||
- Resolve terminal resize scrollback ghost renders
|
||||
- Hide subagent monitor panel by default
|
||||
|
||||
### Installer
|
||||
- Auto-detect existing install and run update instead of fresh install
|
||||
- Auto-restart codeman-web service after update if running
|
||||
- Show restart command when codeman-web is not a systemd service
|
||||
- Fix one-liner restart command for background processes
|
||||
|
||||
### Codebase Quality
|
||||
- Remove dead code, consolidate imports, extract constants
|
||||
- Repair 15 pre-existing subagent-watcher test failures
|
||||
- Clean up DEC sync dead code
|
||||
|
||||
## 0.3.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- - feat: upgrade xterm.js from 5.3 to 6.0 with native DEC 2026 synchronized output support
|
||||
- feat: add history session resume UI and API — resume Claude conversations after reboot
|
||||
- feat: add resumeSessionId support for conversation resume across session restarts
|
||||
- feat: persist active tabs across page refresh
|
||||
- feat: improve session resume reliability and persist user settings
|
||||
- perf: increase terminal scrollback from 5,000 to 20,000 lines
|
||||
- fix: resolve terminal resize scrollback ghost renders
|
||||
- fix: route shell sessions to correct endpoint on tab click
|
||||
- fix: correct claudeSessionId for resumed sessions (use original Claude conversation ID)
|
||||
- fix: increase default desktop font size from 12 to 14
|
||||
- refactor: extract shared \_fetchHistorySessions() method to eliminate duplication
|
||||
- refactor: remove dead DEC 2026 sync code (extractSyncSegments, DEC_SYNC_START/END constants)
|
||||
|
||||
## 0.3.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Add content-hash cache busting for static assets — build step now renames JS/CSS files with MD5 content hashes (e.g. app.js → app.94b71235.js) and rewrites index.html references. HTML served with Cache-Control: no-cache so browsers always revalidate and pick up new hashed filenames after deploys. Hashed assets keep immutable 1-year cache. Eliminates the need for manual hard refresh (Ctrl+Shift+R) after deployments.
|
||||
|
||||
Refactor path traversal validation into shared validatePathWithinBase() helper in route-helpers.ts, replacing 6 duplicate inline checks across case-routes, plan-routes, and session-routes.
|
||||
|
||||
Deduplicate stripAnsi in bash-tool-parser.ts — use shared utility from utils/index.ts instead of private method.
|
||||
|
||||
## 0.3.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Add tunnel status indicator with control panel — green pulsing dot in header when Cloudflare tunnel is active, dropdown with URL, remote clients, auth sessions, and start/stop/QR/revoke controls
|
||||
|
||||
## 0.3.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Operation Lightspeed: 5 parallel performance optimizations — multi-layer backpressure to prevent terminal write freezes, TERMINAL_TAIL_SIZE constant with client-drop recovery, tab switching SSE gating, and local echo improvements
|
||||
- Codebase cleanup: remove dead code (unused token validation exports, PlanPhase alias), add execPattern() regex helper to eliminate repetitive .lastIndex resets, centralize 11 magic number constants into config files, fix CLAUDE.md inaccuracies, and add 316 new tests for utilities, respawn helpers, and system-routes
|
||||
|
||||
## 0.3.6
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Re-enable WebGL renderer with 48KB/frame flush cap protection against GPU stalls
|
||||
|
||||
## 0.3.5
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix Chrome "page unresponsive" crashes caused by xterm.js WebGL renderer GPU stalls during heavy terminal output. Disable WebGL by default (canvas renderer used instead), gate SSE terminal writes during tab switches, and add crash diagnostics with server-side breadcrumb collection.
|
||||
|
||||
## 0.3.4
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Fix Chrome tab freeze from flicker filter buffer accumulation during active sessions, and fix shell mode feedback delay by excluding shell sessions from cursor-up filter
|
||||
|
||||
## 0.3.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fix: eliminate WebGL re-render flicker during tab switch by keeping renderer active instead of toggling it off/on around large buffer writes
|
||||
|
||||
## 0.3.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Make file browser panel draggable by its header
|
||||
|
||||
## 0.3.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -6,11 +6,12 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
|
||||
| Task | Command |
|
||||
|------|---------|
|
||||
| Dev server | `npx tsx src/index.ts web` |
|
||||
| Dev server | `npm run dev` (or `npx tsx src/index.ts web`) |
|
||||
| Type check | `tsc --noEmit` |
|
||||
| Lint | `npm run lint` (fix: `npm run lint:fix`) |
|
||||
| Format | `npm run format` (check: `npm run format:check`) |
|
||||
| Single test | `npx vitest run test/<file>.test.ts` |
|
||||
| Single test | `npm test -- test/<file>.test.ts` (or `npx vitest run --config config/vitest.config.ts test/<file>.test.ts`) — ⚠ **never** run bare `npm test`, see Testing section |
|
||||
| Build | `npm run build` (esbuild via `scripts/build.mjs`, NOT tsc — `tsc --noEmit` is type-check only) |
|
||||
| Production | `npm run build && systemctl --user restart codeman-web` |
|
||||
|
||||
## CRITICAL: Session Safety
|
||||
@@ -29,7 +30,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
||||
2. **Frontend changes**: Use Playwright to load the page and assert the UI renders correctly. Use `waitUntil: 'domcontentloaded'` (not `networkidle` — SSE keeps the connection open). Wait 3-4s for polling/async data to populate, then check element visibility, text content, and CSS values
|
||||
3. **Only after verification passes**, proceed with COM
|
||||
|
||||
The production server caches static files for 1 year (`maxAge: '1y'` in `server.ts`). After deploying frontend changes, users may need a hard refresh (Ctrl+Shift+R) to see updates.
|
||||
The production server caches static files for 1 year, `immutable` (`maxAge: '1y'` in `server.ts`). To avoid stale frontend after a deploy, `renderIndexHtml` runs `cacheBustAssets(html)` — it appends `?v=<mtime>` to **every same-origin `.js`/`.css`** reference (mtime memoized ~1s so a burst of renders is cheap; external/already-versioned/missing refs untouched). Because `index.html` is served `no-cache`, a **normal reload now picks up edited modules/styles — no hard refresh needed** (the gesture bundle is injected separately with its own `?v=`). If you add an asset referenced by an *absolute* URL or from JS rather than a `<script>/<link>` tag, it won't be auto-busted.
|
||||
|
||||
## COM Shorthand (Deployment)
|
||||
|
||||
@@ -44,15 +45,18 @@ When user says "COM":
|
||||
"aicodeman": patch
|
||||
---
|
||||
|
||||
Description of changes
|
||||
Detailed description of ALL changes since last release (not just the most recent commit — review full git log since last version tag)
|
||||
CHANGESET
|
||||
```
|
||||
Replace `patch` with `minor` or `major` as needed. Include `"xterm-zerolag-input": patch` on a separate line if that package changed too.
|
||||
3. **Consume the changeset**: `npm run version-packages` (bumps versions in `package.json` files and updates `CHANGELOG.md`)
|
||||
3. **Consume the changeset**: `npm run version-packages` (auto-bumps `package.json` files, updates `CHANGELOG.md`, runs `npm install --package-lock-only`, and verifies lockfile sync via `scripts/check-lockfile-sync.mjs` — all in one command; never hand-edit `CHANGELOG.md` or `package-lock.json` versions)
|
||||
4. **Sync CLAUDE.md version**: Update the `**Version**` line below to match the new version from `package.json`
|
||||
5. **Commit and deploy**: `git add -A && git commit -m "chore: version packages" && git push && npm run build && systemctl --user restart codeman-web`
|
||||
6. **Wait for CI**: after `git push`, find the run with `gh run list -L 1 --json databaseId,headBranch -q '.[0].databaseId'` and watch it with `gh run watch <id> --exit-status`. Confirm all checks pass before considering the release done.
|
||||
|
||||
**Version**: 0.3.1 (must match `package.json`)
|
||||
CI runs `npm run check:lockfile` on every push/PR, so lockfile drift fails the build even if the `version-packages` script is bypassed.
|
||||
|
||||
**Version**: 0.9.2 (must match `package.json`)
|
||||
|
||||
## Project Overview
|
||||
|
||||
@@ -68,26 +72,39 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
|
||||
|
||||
## Additional Commands
|
||||
|
||||
`npm run dev` = dev server. Default port: `3000`. Commands not in Quick Reference:
|
||||
`npm run dev` = dev server. Default port: `3000` (override with `--port` or the `CODEMAN_PORT` env var). To run this beta isolated alongside a prod Codeman, use `scripts/run-beta.sh` (sets `CODEMAN_INSTANCE=beta` + `CODEMAN_PORT=5000`). Commands not in Quick Reference:
|
||||
|
||||
| Task | Command |
|
||||
|------|---------|
|
||||
| Dev with TLS | `npx tsx src/index.ts web --https` |
|
||||
| Override window title hostname | `npx tsx src/index.ts web --title-hostname <name>` (default: `os.hostname()` — `codeman:<name>` is used for tab title, title-flash, and OS desktop notification prefix) |
|
||||
| Bind a non-loopback host | `npx tsx src/index.ts web --host 0.0.0.0` (or `-H`; env `CODEMAN_HOST`; default `127.0.0.1`). Without `CODEMAN_PASSWORD` it **starts but warns loudly** — see Common Gotchas + `docs/security-architecture.md` |
|
||||
| Continuous typecheck | `tsc --noEmit --watch` |
|
||||
| Test coverage | `npm run test:coverage` |
|
||||
| Dead-code sweep | `npm run knip` (config in `knip.json`) |
|
||||
| Rebuild gesture overlay | `npm run build:gesture` (esbuild `packages/gesture-control/src/codeman/entry.ts` → `src/web/public/gesture/gesture-codeman.js`; commit the result) |
|
||||
| Gesture playground | `npm run dev` **in** `packages/gesture-control/` (standalone vite demo, fake tabs) |
|
||||
| Check public-asset formatting | `npm run check:public-assets` (prettier-checks `src/web/public/**` text assets; `scripts/check-public-assets.mjs`) |
|
||||
| Production start | `npm run start` |
|
||||
| Production logs | `journalctl --user -u codeman-web -f` |
|
||||
|
||||
**CI**: `.github/workflows/ci.yml` runs `typecheck`, `lint`, `format:check` on push to master (Node 22). Tests excluded (they spawn tmux).
|
||||
**CI**: `.github/workflows/ci.yml` runs `check:lockfile`, `typecheck`, `lint`, `format:check`, then a **server boot smoke test** (`tsx src/index.ts web --port 3151` must answer `/api/status` within 30s) on push to master/main and on PRs (Node 22). The unit test suite is excluded (it spawns tmux).
|
||||
|
||||
**Code style**: Prettier (`singleQuote: true`, `printWidth: 120`, `trailingComma: "es5"`). ESLint allows `no-console`, warns on `@typescript-eslint/no-explicit-any`. Does not lint `app.js` or `scripts/**/*.mjs`.
|
||||
**Code style**: Prettier (`singleQuote: true`, `printWidth: 120`, `trailingComma: "es5"`). ESLint flat config (`config/eslint.config.js`) allows `no-console`, warns on `@typescript-eslint/no-explicit-any`. Ignores: `app.js`, `scripts/**/*.mjs`, `src/web/public/vendor/**`, `scripts/remotion/**`.
|
||||
|
||||
## Common Gotchas
|
||||
|
||||
- **Single-line prompts only** — `writeViaMux()` sends text+Enter separately; multi-line breaks Ink
|
||||
- **ESM only** — Never `require()`, use `await import()`. `tsx` masks CJS/ESM issues in dev but production breaks
|
||||
- **Package ≠ product name** — npm: `aicodeman`, product: **Codeman**. Release renames tags accordingly
|
||||
- **Global regex `lastIndex`** — Use `createAnsiPatternFull/Simple()` factories, not shared `g`-flag patterns in loops
|
||||
- **Global regex `lastIndex`** — Shared `g`-flag patterns in loops must reset `lastIndex = 0` first, or use the `execPattern()` helper in `utils/regex-patterns.ts` (resets automatically)
|
||||
- **`envOverrides` flow `CLAUDE_CODE_*` / `OPENCODE_*` env vars** — Set via `POST /api/sessions { envOverrides }`, stored on `Session._envOverrides`, exported by `tmux-manager.buildEnvExports()` at spawn time, persisted in `SessionState.envOverrides`. **Do NOT** write these to `<case>/.claude/settings.local.json` — that's the old path and creates UI/disk drift
|
||||
- **Effort is NOT an env var** — never carry effort as `CLAUDE_CODE_EFFORT_LEVEL`: the env var hard-locks effort and blocks in-session `/effort` switching (incl. ultracode). It flows as the dedicated `effort` payload field → `Session._effort` → `claude --effort <level>` for regular levels incl. `max` (the settings `effortLevel` key is `enum(["low","medium","high","xhigh"]).catch(undefined)` — `max` gets SILENTLY dropped there), or `claude --settings '{"ultracode":true}'` for ultracode (rejected by `--effort`). Both are soft defaults the user can override anytime. Legacy env-var entries are auto-migrated by the Session constructor and unset from tmux sessions in `applyEnvOverrides()`. See `buildEffortCliArgs()` in `session-cli-builder.ts`, tests in `test/effort-injection.test.ts`
|
||||
- **Dual-CLI prefix discipline** — Codeman supports both Claude Code and OpenCode (`claude-cli-resolver.ts` / `opencode-cli-resolver.ts`); env-var prefix is CLI-specific (`CLAUDE_CODE_*` vs `OPENCODE_*`) and the allowlist in `schemas.ts` enforces this. When adding settings, decide which CLI(s) it applies to and gate the env export accordingly — don't blindly forward both prefixes. See `docs/opencode-integration.md` for the OpenCode resolver design
|
||||
- **Zod `.optional()` rejects `null`** — accepts `undefined` only. When the frontend builds a request body with `JSON.stringify`, an explicit `null` field is preserved on the wire and fails validation with `INVALID_INPUT`. Convert `null` → `undefined` before stringifying (e.g. `field: value ?? undefined`), or declare the schema `.nullish()`. Real bugs caused: 0.6.4 (`durationMinutes` for ∞ respawn), and the same shape pattern hit `opusContext1mEnabled` in 0.6.3
|
||||
- **`xterm-zerolag-input` is duplicated** — the local-echo overlay lives in BOTH `packages/xterm-zerolag-input/src/` (published to npm as a standalone library for external consumers — see README "Published Packages") AND inline inside `src/web/public/app.js` (runtime copy the web UI actually loads, since the page ships as plain JS without a bundler). Any change to overlay behavior MUST be applied to both, or dev and prod diverge — and a public API break in the package warrants a separate version bump for `xterm-zerolag-input` in the changeset. Always test on mobile after touching it. See `docs/local-echo-overlay-plan.md`.
|
||||
- **Default bind is loopback-only; non-loopback without a password starts but warns** — since COD-29 (PR #107) the web server defaults to `--host 127.0.0.1` (was `0.0.0.0`). As of **0.9.0** binding a non-loopback host (`--host`/`-H`/`CODEMAN_HOST`) without `CODEMAN_PASSWORD` **no longer refuses to start — it starts and prints a loud warning** listing the fixes (set `CODEMAN_PASSWORD`, bind loopback + tunnel/`tailscale serve`, or `--allow-unauthenticated-network` / `CODEMAN_ALLOW_UNAUTHENTICATED_NETWORK=1` to acknowledge → terser note). Host classification is `isLoopbackBindHost()` in `network-auth-policy.ts`; the warn-vs-start logic is in `server.ts` `start()`; flags wired in `cli.ts`. ⚠️ Operational note: the production systemd unit runs `node dist/index.js web --https` with no `--host`, so it binds **localhost only** — reach it remotely via `tailscale serve`/tunnel to `127.0.0.1`, or add `Environment=CODEMAN_HOST=0.0.0.0` + `Environment=CODEMAN_PASSWORD=…` to `~/.config/systemd/user/codeman-web.service`. A loopback bind is reachable through a same-host tunnel (cloudflared/tailscale → `127.0.0.1`) but NOT by a browser hitting the box's LAN IP. Auth user defaults to `admin`. **Full model: `docs/security-architecture.md`.**
|
||||
- **Instance isolation / multi-instance attach danger** — data dir (`~/.codeman`) and tmux socket (`tmux -L codeman`) are PROCESS-WIDE and shared by every Codeman on the machine, derived from `CODEMAN_INSTANCE` via `src/config/instance.ts` (`getDataDir()`/`dataPath()`/`DEFAULT_TMUX_SOCKET`). ⚠️ A 2nd instance on the SAME socket **discovers and attaches PTYs to the first instance's live sessions** (`tmux -L codeman attach-session …`), resizing/mutating them — `$HOME` isolation is NOT enough (tmux is system-global). To run two instances, give each a distinct `CODEMAN_INSTANCE` (scopes BOTH dir+socket: `~/.codeman-<name>` + `-L codeman-<name>`), or set `CODEMAN_TMUX_SOCKET` + `CODEMAN_DATA_DIR` individually. **`CODEMAN_INSTANCE` defaults to empty = the production layout (`~/.codeman`, `-L codeman`, port 3000)**, so this branch is safe to ship to master without disturbing existing installs. To run THIS beta alongside prod, launch with `scripts/run-beta.sh` (`CODEMAN_INSTANCE=beta` + `CODEMAN_PORT=5000`) — it never collides with prod's data dir/socket/port. Any new `~/.codeman/...` path MUST go through `dataPath()`, never `join(homedir(), '.codeman', …)`.
|
||||
|
||||
**Import conventions**: Utils from `./utils`, types from `./types` (barrel), config from specific `./config/*` files.
|
||||
|
||||
@@ -98,27 +115,28 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
|
||||
| Domain | Key files | Notes |
|
||||
|--------|-----------|-------|
|
||||
| **Entry** | `src/index.ts`, `src/cli.ts` | |
|
||||
| **Session** | `src/session.ts` ★, `src/session-manager.ts`, `src/session-auto-ops.ts`, `src/session-cli-builder.ts` | |
|
||||
| **Mux** | `src/mux-interface.ts`, `src/mux-factory.ts`, `src/tmux-manager.ts` | |
|
||||
| **Session** | `src/session.ts` ★, `src/session-manager.ts`, `src/session-auto-ops.ts`, `src/session-cli-builder.ts`, `src/session-lifecycle-log.ts`, `src/session-task-cache.ts` | |
|
||||
| **Mux** | `src/mux-interface.ts`, `src/mux-factory.ts`, `src/tmux-manager.ts` ★ | |
|
||||
| **Respawn** | `src/respawn-controller.ts` ★ + 4 helpers (`-adaptive-timing`, `-health`, `-metrics`, `-patterns`) | Read `docs/respawn-state-machine.md` first |
|
||||
| **Ralph** | `src/ralph-tracker.ts` ★, `src/ralph-loop.ts` + 5 helpers (`-config`, `-fix-plan-watcher`, `-plan-tracker`, `-stall-detector`, `-status-parser`) | Read `docs/ralph-wiggum-guide.md` first |
|
||||
| **Orchestrator** | `src/orchestrator-loop.ts`, `src/orchestrator-planner.ts`, `src/orchestrator-verifier.ts` | Read `docs/orchestrator-loop-architecture.md` first |
|
||||
| **Agents** | `src/subagent-watcher.ts` ★, `src/team-watcher.ts`, `src/bash-tool-parser.ts`, `src/transcript-watcher.ts` | |
|
||||
| **AI** | `src/ai-checker-base.ts`, `src/ai-idle-checker.ts`, `src/ai-plan-checker.ts` | |
|
||||
| **Tasks** | `src/task.ts`, `src/task-queue.ts`, `src/task-tracker.ts` | |
|
||||
| **State** | `src/state-store.ts`, `src/run-summary.ts`, `src/session-lifecycle-log.ts` | |
|
||||
| **Infra** | `src/hooks-config.ts`, `src/push-store.ts`, `src/tunnel-manager.ts`, `src/image-watcher.ts`, `src/file-stream-manager.ts` | |
|
||||
| **Plan** | `src/plan-orchestrator.ts`, `src/prompts/*.ts`, `src/templates/claude-md.ts` | |
|
||||
| **Web** | `src/web/server.ts`, `src/web/sse-events.ts`, `src/web/routes/*.ts` (13 modules), `src/web/ports/*.ts`, `src/web/middleware/auth.ts`, `src/web/schemas.ts` | |
|
||||
| **Frontend** | `src/web/public/app.js` ★ (~11.7K lines) + 9 JS modules | |
|
||||
| **Types** | `src/types/index.ts` → 14 domain files | See `@fileoverview` in index.ts |
|
||||
| **Web** | `src/web/server.ts`, `src/web/sse-events.ts`, `src/web/routes/*.ts` (15 route modules + barrel), `src/web/route-helpers.ts`, `src/web/ports/*.ts`, `src/web/middleware/auth.ts`, `src/web/schemas.ts` | |
|
||||
| **Frontend** | `src/web/public/app.js` (~3.4K lines, core) + 5 infra modules (`constants.js`, `mobile-handlers.js`, `voice-input.js`, `notification-manager.js`, `keyboard-accessory.js`) + 7 domain modules (`terminal-ui.js`, `respawn-ui.js`, `ralph-panel.js`, `orchestrator-panel.js`, `settings-ui.js`, `panels-ui.js`, `session-ui.js`) + 5 feature modules (`ralph-wizard.js`, `api-client.js`, `subagent-windows.js`, `input-cjk.js`, `image-input.js`) + `sw.js` | |
|
||||
| **Types** | `src/types/index.ts` (barrel) → 14 domain files; also `src/types.ts` root re-export | See `@fileoverview` in index.ts |
|
||||
|
||||
★ = Large file (>50KB). All files have `@fileoverview` JSDoc — read that before diving in.
|
||||
★ = Large file (>50KB). All files have `@fileoverview` JSDoc — read that before diving in. Discovery aid: `grep -l '@fileoverview' src/web/routes/*.ts` lists all route modules; same grep works for `src/types/`, `src/web/public/*.js`.
|
||||
|
||||
**Local package**: `packages/xterm-zerolag-input/` — local echo overlay for xterm.js; copy embedded in `app.js`.
|
||||
**Local packages**: `packages/xterm-zerolag-input/` — local echo overlay for xterm.js; copy embedded in `app.js`. `packages/gesture-control/` (`codeman-gesture-control`) — hand-tracking overlay source; built to `src/web/public/gesture/gesture-codeman.js` via `npm run build:gesture` (see Frontend → Gesture control).
|
||||
|
||||
**Config**: `src/config/` — 9 files. Import from specific files, not barrel.
|
||||
**Config**: `src/config/` — 10 files. Import from specific files, not barrel.
|
||||
|
||||
**Utilities**: `src/utils/` — re-exported via index. Key: `CleanupManager`, `LRUMap`, `StaleExpirationMap`, `BufferAccumulator`, `stripAnsi`, `Debouncer`.
|
||||
**Utilities**: `src/utils/` — re-exported via index. Key: `CleanupManager`, `LRUMap`, `StaleExpirationMap`, `BufferAccumulator`, `stripAnsi`, `Debouncer`, `KeyedDebouncer`. Also: `claude-cli-resolver`/`opencode-cli-resolver` (CLI path resolution), `string-similarity` (fuzzy matching), `regex-patterns` (ANSI/token/spinner patterns), `assertNever` (exhaustive checks), `token-validation` (auth tokens), `nice-wrapper` (process priority).
|
||||
|
||||
### Data Flow
|
||||
|
||||
@@ -133,9 +151,11 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
|
||||
|
||||
**Idle detection**: Multi-layer (completion message → AI check → output silence → token stability). See `docs/respawn-state-machine.md`.
|
||||
|
||||
**Orchestrator**: State machine that turns a user goal into a phased plan and drives it to completion: `idle → planning → approval → executing → verifying → (replanning) → completed/failed`. `OrchestratorLoop` (engine) delegates plan generation to `orchestrator-planner` and per-phase verification gates to `orchestrator-verifier`, executing phases via team agents/`task-queue`. State persists under the `orchestrator` key in `state.json`. Distinct from Ralph (single-session autonomous loop) — orchestrator coordinates multi-phase, multi-agent execution. See `docs/orchestrator-loop-architecture.md`.
|
||||
|
||||
**Hook events**: Claude Code hooks trigger via `/api/hook-event`. Key events: `permission_prompt`, `elicitation_dialog`, `idle_prompt`, `stop`, `teammate_idle`, `task_completed`. See `src/hooks-config.ts`.
|
||||
|
||||
**Agent Teams**: `TeamWatcher` polls `~/.claude/teams/`, matches to sessions via `leadSessionId`. Teammates are in-process threads appearing as subagents. Enable: `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. See `agent-teams/`.
|
||||
**Agent Teams**: `TeamWatcher` polls `~/.claude/teams/`, matches to sessions via `leadSessionId`. Teammates are in-process threads appearing as subagents. Enable: `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. See `docs/agent-teams/`.
|
||||
|
||||
**Circuit breaker**: Prevents respawn thrashing. States: `CLOSED` → `HALF_OPEN` → `OPEN`. Reset: `/api/sessions/:id/ralph-circuit-breaker/reset`.
|
||||
|
||||
@@ -143,19 +163,28 @@ Codeman is a Claude Code session manager with web interface and autonomous Ralph
|
||||
|
||||
### Frontend
|
||||
|
||||
Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. Load order: `constants.js`(1) → `mobile-handlers.js`(2) → `voice-input.js`(3) → `notification-manager.js`(4) → `keyboard-accessory.js`(5) → `app.js`(6) → `ralph-wizard.js`(7) → `api-client.js`(8) → `subagent-windows.js`(9).
|
||||
Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. Load order: `constants.js`(1) → `mobile-handlers.js`(2) → `voice-input.js`(3) → `notification-manager.js`(4) → `keyboard-accessory.js`(5) → `input-cjk.js`(5.5) → `app.js`(6) → `terminal-ui.js`(7) → `respawn-ui.js`(8) → `ralph-panel.js`(9) → `orchestrator-panel.js`(9.5) → `settings-ui.js`(10) → `panels-ui.js`(11) → `session-ui.js`(12) → `ralph-wizard.js`(13) → `api-client.js`(14) → `subagent-windows.js`(15). `input-cjk.js` handles CJK IME composition via an always-visible textarea below the terminal (`window.cjkActive` blocks xterm's onData).
|
||||
|
||||
**Z-index layers**: subagent windows (1000), plan agents (1100), log viewers (2000), image popups (3000), local echo overlay (7).
|
||||
|
||||
**Multi-monitor button** (header, top-right; the notification bell it sits beside stays hidden — notifications live in Settings → Notifications). `app.launchMultiMonitor()` (in `panels-ui.js`) POSTs `/api/system/span-displays`, which spawns `scripts/span-codeman.sh` — a fresh, maximized browser `--app` window sized to the union of all displays (macOS; needs "Displays have separate Spaces" OFF). Supports the gesture layer's in-page floating session panels dragging across the physical monitor seam. **Opt-in:** hidden by default; enable under App Settings → Display → **Header Displays** ("Multi-monitor Button", `showMultiMonitorButton`). The button carries a `btn-multimonitor--hidden` class in the template; `renderIndexHtml` strips that class at render when the setting is on (a unique class token, not a brittle match on the aria-label/style copy), and `applyHeaderVisibilitySettings()` toggles the same class live on save. Solo (detached) windows hide it via `body.solo-mode`.
|
||||
|
||||
**Gesture control** (the camera hand-tracking overlay) is **opt-in, default OFF**, under App Settings → Display → **Input** (`gestureControlEnabled`). `CODEMAN_GESTURE=1` makes the feature *available* on the instance (CSP widening + `/gesture/` assets) and sets `window.__codemanGestureAvailable` (the Input section only shows when set); the overlay bundle is injected by `renderIndexHtml` **only when the setting is enabled**, so that method is `async` and reads `settings.json` via `readSettings(true)` — the `true` forces a **fresh** read (bypassing the 2s `_settingsCache`), because a post-save reload happens within that TTL and the cached value would otherwise render the pre-toggle state. Toggling the setting reloads the page (the bundle is render-injected).
|
||||
|
||||
**Gesture-control source lives in-repo** at `packages/gesture-control/` (workspace package `codeman-gesture-control`, was the standalone `Ark0N/codeman-gesture-control` repo). The transport-agnostic core is `src/gesture/*` (MediaPipe GestureRecognizer → One-Euro-filtered cursor → pinch state machine); `src/codeman/entry.ts` is the Codeman *consumer* that maps grab/drag/drop onto real `.session-tab`/toolbar buttons and is the bundle entry. **Edit there, then run `npm run build:gesture`** (`scripts/build-gesture-bundle.mjs` → esbuild bundles `entry.ts`, MediaPipe JS included, into `src/web/public/gesture/gesture-codeman.js`) and **commit the regenerated bundle** — the committed bundle is what dev/`tsx` serves (no bundler at runtime), and `scripts/build.mjs` reruns the same step so prod always reflects current source. The MediaPipe **wasm + model** are NOT bundled — loaded at runtime from same-origin `/gesture/wasm` + `/gesture/gesture_recognizer.task`, fetched by `scripts/fetch-gesture-assets.mjs` (gitignored, see Gotchas). `entry.ts` mounts `window.__codemanGesture = new GestureBridge()` idempotently at module-eval. A standalone vite playground (`npm run dev` in the package — fake tabs, no Codeman) lets you iterate on gesture *feel* in isolation. ⚠️ Keep `MP_VERSION` in `fetch-gesture-assets.mjs` in sync with `@mediapipe/tasks-vision` in `packages/gesture-control/package.json`.
|
||||
|
||||
**Respawn presets**: `solo-work` (3s/60min), `subagent-workflow` (45s/240min), `team-lead` (90s/480min), `ralph-todo` (8s/480min), `overnight-autonomous` (10s/480min).
|
||||
|
||||
**Keyboard shortcuts**: Escape (close), Ctrl+? (help), Ctrl+Enter (quick start), Ctrl+W (kill), Ctrl+Tab (next), Ctrl+K (kill all), Ctrl+L (clear), Ctrl+Shift+R (restore size), Ctrl/Cmd +/- (font).
|
||||
**Keyboard shortcuts**: Escape (close), Ctrl+? (help), Ctrl+W (kill), Ctrl+Tab (next), Alt+1-9 (switch tab), Ctrl+Shift+{/} (move tab left/right), Shift+Enter (newline), Ctrl+L (clear), Ctrl+Shift+R (restore size), Ctrl+Shift+V (voice input), Ctrl/Cmd +/- (font).
|
||||
|
||||
### Security
|
||||
|
||||
**Full model: [`docs/security-architecture.md`](docs/security-architecture.md)** — network binding, auth pipeline, the tunnel caveat, file-serving hardening, supply-chain, instance isolation, and recommended secure setups.
|
||||
|
||||
| Layer | Details |
|
||||
|-------|---------|
|
||||
| **Auth** | Optional HTTP Basic via `CODEMAN_USERNAME`/`CODEMAN_PASSWORD` env vars |
|
||||
| **Auth** | Optional HTTP Basic via `CODEMAN_USERNAME` (defaults to `admin`) / `CODEMAN_PASSWORD` env vars. Active only when `CODEMAN_PASSWORD` is set (`middleware/auth.ts`) |
|
||||
| **Network bind** | Defaults to `127.0.0.1` (loopback). A non-loopback bind (`--host`/`CODEMAN_HOST`) without `CODEMAN_PASSWORD` **starts but warns loudly** (0.9.0; was fail-closed in COD-29/#107). `--allow-unauthenticated-network` / `CODEMAN_ALLOW_UNAUTHENTICATED_NETWORK=1` acknowledges the warning. Classifier: `network-auth-policy.ts` |
|
||||
| **QR Auth** | Single-use 6-char tokens (60s TTL) for tunnel login. See `docs/qr-auth-plan.md` |
|
||||
| **Sessions** | 24h cookie (`codeman_session`), auto-extend, device context audit |
|
||||
| **Rate limit** | 10 failed auth/IP → 429 (15min decay). QR has separate limiter |
|
||||
@@ -166,11 +195,11 @@ Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. L
|
||||
|
||||
### SSE Event Registry
|
||||
|
||||
~100 event types in `src/web/sse-events.ts` (backend) and `SSE_EVENTS` in `constants.js` (frontend). Both must be kept in sync.
|
||||
~120 event types in `src/web/sse-events.ts` (backend) and `SSE_EVENTS` in `constants.js` (frontend). Both must be kept in sync.
|
||||
|
||||
### API Routes
|
||||
|
||||
~111 handlers across 13 route files in `src/web/routes/`: system (35), sessions (24), ralph (9), plan (8), respawn (7), cases (7), files (5), mux (5), scheduled (4), push (4), teams (2), hooks (1). Each file has `@fileoverview` with endpoint details.
|
||||
~130 handlers across 15 route files in `src/web/routes/`: system (37, incl. `POST /api/system/span-displays` → spawns `scripts/span-codeman.sh`), sessions (28), orchestrator (10), cases (9), ralph (9), plan (8), respawn (7), files (5), mux (5), push (4), scheduled (4), teams (2), hooks (1), clipboard (1), ws (1 WebSocket). Each file has `@fileoverview` with endpoint details.
|
||||
|
||||
## Adding Features
|
||||
|
||||
@@ -179,7 +208,7 @@ Frontend JS modules have `@fileoverview` with `@dependency`/`@loadorder` tags. L
|
||||
- **Session setting**: Add to `SessionState`, include in `session.toState()`, call `persistSessionState()`
|
||||
- **Hook event**: Add to `HookEventType`, add hook in `hooks-config.ts:generateHooksConfig()`, update `HookEventSchema`
|
||||
- **Mobile feature**: Add to relevant singleton, guard with `MobileDetection.isMobile()`
|
||||
- **New test**: Pick unique port (search `const PORT =`). Integration: ports 3099-3211. Route tests: `app.inject()` — see `test/routes/_route-test-utils.ts`.
|
||||
- **New test**: Pick unique port (search `const PORT =`). Route tests use `app.inject()` (no port needed) — see `test/routes/_route-test-utils.ts`.
|
||||
|
||||
**Validation**: Zod v4 (different API from v3). Define schemas in `schemas.ts`, use `.parse()`/`.safeParse()`.
|
||||
|
||||
@@ -192,22 +221,20 @@ All in `~/.codeman/`: `state.json` (sessions, settings, respawn), `mux-sessions.
|
||||
**CRITICAL: You are running inside a Codeman-managed tmux session.** Never run `npx vitest run` (full suite) — it spawns/kills tmux sessions and will crash your own session. Only run individual files:
|
||||
|
||||
```bash
|
||||
npx vitest run test/<specific-file>.test.ts # Single file (SAFE)
|
||||
npx vitest run -t "pattern" # By name (SAFE)
|
||||
# npx vitest run # DANGEROUS — DON'T DO THIS
|
||||
npm test -- test/<specific-file>.test.ts # Single file (SAFE, uses config/vitest.config.ts)
|
||||
npm test -- -t "pattern" # By name (SAFE)
|
||||
# npm test # DANGEROUS — runs full suite, DON'T DO THIS
|
||||
```
|
||||
|
||||
Raw `npx vitest` skips `config/vitest.config.ts`; always use `npm test --` or pass `--config config/vitest.config.ts`.
|
||||
|
||||
**Config**: Vitest with `globals: true`, `fileParallelism: false`. Timeout 30s, teardown 60s.
|
||||
|
||||
**Safety**: `test/setup.ts` snapshots pre-existing tmux sessions and never kills them. Only `registerTestTmuxSession()` sessions get cleaned up.
|
||||
|
||||
**Ports**: Pick unique ports manually. Search `const PORT =` before adding new tests.
|
||||
|
||||
**Respawn tests**: Use `MockSession` from `test/respawn-test-utils.ts`. **Route tests**: `app.inject()` in `test/routes/`. **Mobile tests**: Playwright suite in `mobile-test/` (135 device profiles).
|
||||
|
||||
## Screenshots
|
||||
|
||||
Mobile screenshots in `~/.codeman/screenshots/`. API: `GET /api/screenshots`, `POST /api/screenshots`.
|
||||
**Respawn tests**: Use `MockSession` from `test/respawn-test-utils.ts`. **Route tests**: `app.inject({ method, url, payload })` in `test/routes/` — no live port needed. **Mobile tests**: Playwright suite in `test/mobile/` (135 device profiles).
|
||||
|
||||
## Debugging
|
||||
|
||||
@@ -219,28 +246,14 @@ curl localhost:3000/api/subagents | jq # Background agents
|
||||
cat ~/.codeman/state.json | jq # Persisted state
|
||||
```
|
||||
|
||||
Mobile screenshots: `~/.codeman/screenshots/`, accessed via `GET/POST /api/screenshots`.
|
||||
|
||||
## Performance & Limits
|
||||
|
||||
Target: 20 sessions, 50 agent windows at 60fps. Limits in `src/config/`: terminal 2MB, text 1MB, messages 1000, max agents 500, max sessions 50, max SSE clients 100. Use `LRUMap` for bounded caches, `StaleExpirationMap` for TTL cleanup. Anti-flicker pipeline: `docs/terminal-anti-flicker.md`.
|
||||
|
||||
## References
|
||||
**Memory leaks (24+ hour sessions)**: use `CleanupManager`, clear Maps in `stop()`, guard async with `if (this.cleanup.isStopped) return`. Frontend: store handler refs, clean in `close*()`. Verify: `npm test -- test/memory-leak-prevention.test.ts`.
|
||||
|
||||
Deep-dive docs in `docs/`: `respawn-state-machine.md`, `ralph-wiggum-guide.md`, `claude-code-hooks-reference.md`, `terminal-anti-flicker.md`, `opencode-integration.md`, `qr-auth-plan.md`. Agent Teams: `agent-teams/README.md`. SSE events: `src/web/sse-events.ts` + `constants.js`.
|
||||
## Scripts & Tunnel
|
||||
|
||||
## Scripts
|
||||
|
||||
Key: `scripts/tmux-manager.sh` (safe tmux mgmt), `scripts/tunnel.sh` (tunnel start/stop/url). Production: `scripts/codeman-web.service`, `scripts/codeman-tunnel.service`.
|
||||
|
||||
## Memory Leak Prevention
|
||||
|
||||
24+ hour sessions: use `CleanupManager`, clear Maps in `stop()`, guard async with `if (this.cleanup.isStopped) return`. Frontend: store handler refs, clean in `close*()`. Verify: `npx vitest run test/memory-leak-prevention.test.ts`.
|
||||
|
||||
## Common Workflows
|
||||
|
||||
**Bug investigation**: Dev server → reproduce in browser → check terminal + `~/.codeman/state.json`.
|
||||
**API endpoint**: Types in `src/types/*.ts` → route in `src/web/routes/*-routes.ts` → SSE event if needed → handle in `app.js`.
|
||||
**Respawn changes**: Read `docs/respawn-state-machine.md` first. Use `MockSession` from `test/respawn-test-utils.ts`.
|
||||
|
||||
## Tunnel
|
||||
|
||||
`./scripts/tunnel.sh start|stop|url`. **Always set `CODEMAN_PASSWORD`** before exposing via tunnel.
|
||||
Key scripts: `scripts/tmux-manager.sh` (safe tmux mgmt), `scripts/tunnel.sh start|stop|url` (tunnel). Production services: `scripts/codeman-web.service`, `scripts/codeman-tunnel.service`. **Always set `CODEMAN_PASSWORD`** before exposing via tunnel.
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
<p align="center">
|
||||
<a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-1e3a5f?style=flat-square" alt="License: MIT"></a>
|
||||
<a href="https://nodejs.org/"><img src="https://img.shields.io/badge/Node.js-18%2B-22c55e?style=flat-square&logo=node.js&logoColor=white" alt="Node.js 18+"></a>
|
||||
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/TypeScript-5.5-3b82f6?style=flat-square&logo=typescript&logoColor=white" alt="TypeScript 5.5"></a>
|
||||
<a href="https://www.typescriptlang.org/"><img src="https://img.shields.io/badge/TypeScript-5.9-3b82f6?style=flat-square&logo=typescript&logoColor=white" alt="TypeScript 5.9"></a>
|
||||
<a href="https://fastify.dev/"><img src="https://img.shields.io/badge/Fastify-5.x-1e3a5f?style=flat-square&logo=fastify&logoColor=white" alt="Fastify"></a>
|
||||
<img src="https://img.shields.io/badge/Tests-1435%20total-22c55e?style=flat-square" alt="Tests">
|
||||
</p>
|
||||
@@ -28,29 +28,67 @@
|
||||
curl -fsSL https://raw.githubusercontent.com/Ark0N/Codeman/master/install.sh | bash
|
||||
```
|
||||
|
||||
This installs Node.js and tmux if missing, clones Codeman to `~/.codeman/app`, and builds it. You'll need at least one AI coding CLI installed — [Claude Code](https://docs.anthropic.com/en/docs/claude-code) or [OpenCode](https://opencode.ai) (or both). After install:
|
||||
This installs Node.js and tmux if missing, clones Codeman to `~/.codeman/app`, and builds it.
|
||||
|
||||
You'll need at least one AI coding CLI installed — [Claude Code](https://docs.anthropic.com/en/docs/claude-code) or [OpenCode](https://opencode.ai) (or both). After install:
|
||||
|
||||
```bash
|
||||
codeman web
|
||||
# Open http://localhost:3000 — press Ctrl+Enter to start your first session
|
||||
```
|
||||
|
||||
**Update to latest version:**
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/Ark0N/Codeman/master/install.sh | bash -s update
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><strong>Run as a background service</strong></summary>
|
||||
|
||||
**Linux (systemd):**
|
||||
```bash
|
||||
mkdir -p ~/.config/systemd/user && printf '[Unit]\nDescription=Codeman Web Server\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=%s %s/dist/index.js web\nRestart=always\nRestartSec=10\n\n[Install]\nWantedBy=default.target\n' "$(which node)" "$HOME/.codeman/app" > ~/.config/systemd/user/codeman-web.service && systemctl --user daemon-reload && systemctl --user enable --now codeman-web && loginctl enable-linger $USER
|
||||
mkdir -p ~/.config/systemd/user
|
||||
cat > ~/.config/systemd/user/codeman-web.service << EOF
|
||||
[Unit]
|
||||
Description=Codeman Web Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=$(which node) $HOME/.codeman/app/dist/index.js web
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
EOF
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now codeman-web
|
||||
loginctl enable-linger $USER
|
||||
```
|
||||
|
||||
**macOS (launchd):**
|
||||
```bash
|
||||
mkdir -p ~/Library/LaunchAgents && printf '<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict><key>Label</key><string>com.codeman.web</string><key>ProgramArguments</key><array><string>%s</string><string>%s/dist/index.js</string><string>web</string></array><key>RunAtLoad</key><true/><key>KeepAlive</key><true/><key>StandardOutPath</key><string>/tmp/codeman.log</string><key>StandardErrorPath</key><string>/tmp/codeman.log</string></dict></plist>\n' "$(which node)" "$HOME/.codeman/app" > ~/Library/LaunchAgents/com.codeman.web.plist && launchctl load ~/Library/LaunchAgents/com.codeman.web.plist
|
||||
mkdir -p ~/Library/LaunchAgents
|
||||
cat > ~/Library/LaunchAgents/com.codeman.web.plist << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>com.codeman.web</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$(which node)</string>
|
||||
<string>$HOME/.codeman/app/dist/index.js</string>
|
||||
<string>web</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/codeman.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/codeman.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.codeman.web.plist
|
||||
```
|
||||
</details>
|
||||
|
||||
@@ -143,6 +181,10 @@ Watch background agents work in real-time. Codeman monitors agent activity and d
|
||||
|
||||
## Zero-Lag Input Overlay
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/images/zerolag-demo.gif" alt="Zerolag Demo — local echo vs server echo side-by-side" width="900">
|
||||
</p>
|
||||
|
||||
When accessing your coding agent remotely (VPN, Tailscale, SSH tunnel), every keystroke normally takes 200-300ms to round-trip. Codeman implements a **Mosh-inspired local echo system** that makes typing feel instant regardless of latency.
|
||||
|
||||
A pixel-perfect DOM overlay inside xterm.js renders keystrokes at 0ms. Background forwarding silently sends every character to the PTY in 50ms debounced batches, so Tab completion, `Ctrl+R` history search, and all shell features work normally. When the server echo arrives 200-300ms later, the overlay seamlessly disappears and the real terminal text takes over — the transition is invisible.
|
||||
@@ -184,6 +226,17 @@ Run **20 parallel sessions** with full visibility — real-time xterm.js termina
|
||||
|
||||
Every session runs inside **tmux** — sessions survive server restarts, network drops, and machine sleep. Auto-recovery on startup with dual redundancy. Ghost session discovery finds orphaned tmux sessions. Managed sessions are environment-tagged so the agent won't kill its own session.
|
||||
|
||||
### Hostname-Aware Window Title
|
||||
|
||||
Running Codeman on multiple hosts (laptop, dev box, NAS)? The browser tab title is `codeman:<hostname>` so you can tell which backend each tab points at without clicking in:
|
||||
|
||||
```bash
|
||||
codeman web # codeman:<os.hostname()>
|
||||
codeman web --title-hostname dev-box # codeman:dev-box (manual override for noisy hostnames)
|
||||
```
|
||||
|
||||
The title is templated into the served HTML on first byte, so it's correct from the very first paint and works without JavaScript. The same hostname prefix is applied to the tab-flash format (`⚠️ (N) codeman:<host>`) and to OS-level desktop notifications (`codeman:<host>: <event>`), so cross-host alerts in the system notification center are also unambiguous.
|
||||
|
||||
### Smart Token Management
|
||||
|
||||
| Threshold | Action | Result |
|
||||
@@ -359,9 +412,12 @@ Single-digit selection (1-9), color-coded status, token counts, auto-refresh. De
|
||||
| `Ctrl+Enter` | Quick-start session |
|
||||
| `Ctrl+W` | Close session |
|
||||
| `Ctrl+Tab` | Next session |
|
||||
| `Alt+1`–`Alt+9` | Switch to tab N |
|
||||
| `Ctrl+Shift+{` / `Ctrl+Shift+}` | Move active tab left / right |
|
||||
| `Ctrl+K` | Kill all sessions |
|
||||
| `Ctrl+L` | Clear terminal |
|
||||
| `Ctrl+Shift+R` | Restore terminal size |
|
||||
| `Ctrl+Shift+V` | Toggle voice input |
|
||||
| `Ctrl/Cmd +/-` | Font size |
|
||||
| `Escape` | Close panels |
|
||||
|
||||
@@ -404,6 +460,7 @@ Single-digit selection (1-9), color-coded status, token counts, auto-refresh. De
|
||||
| `GET` | `/api/events` | SSE stream |
|
||||
| `GET` | `/api/status` | Full app state |
|
||||
| `POST` | `/api/hook-event` | Hook callbacks |
|
||||
| `POST` | `/api/clipboard` | Push text to all connected browsers (`{text}`) |
|
||||
| `GET` | `/api/sessions/:id/run-summary` | Timeline + stats |
|
||||
|
||||
---
|
||||
@@ -480,9 +537,9 @@ The codebase went through a comprehensive 7-phase refactoring that eliminated go
|
||||
| Phase | What changed | Impact |
|
||||
|-------|-------------|--------|
|
||||
| **Performance** | Cached endpoints, SSE adaptive batching, buffer chunking | Sub-16ms terminal latency |
|
||||
| **Route extraction** | `server.ts` split into 12 domain route modules + auth middleware + port interfaces | **−60%** server.ts LOC (6,736 → 2,697) |
|
||||
| **Route extraction** | `server.ts` split into 13 domain route modules + auth middleware + port interfaces | **−60%** server.ts LOC (6,736 → 2,697) |
|
||||
| **Domain splitting** | `types.ts` → 14 domain files, `ralph-tracker` → 7 files, `respawn-controller` → 5 files, `session` → 6 files | No more god files |
|
||||
| **Frontend modules** | `app.js` → 8 extracted modules (constants, mobile, voice, notifications, keyboard, API, subagent windows) | **−24%** app.js LOC (15.2K → 11.5K) |
|
||||
| **Frontend modules** | `app.js` → 9 extracted modules (constants, mobile, voice, notifications, keyboard, CJK input, API, Ralph wizard, subagent windows) | **−24%** app.js LOC (15.2K → 11.5K) |
|
||||
| **Config consolidation** | ~70 scattered magic numbers → 9 domain-focused config files | Zero cross-file duplicates |
|
||||
| **Test infrastructure** | Shared mock library, 12 route test files, consolidated MockSession | Testable route handlers via `app.inject()` |
|
||||
|
||||
|
||||
@@ -22,8 +22,7 @@ export default tseslint.config(
|
||||
'src/web/public/vendor/**',
|
||||
'src/web/public/app.js',
|
||||
'scripts/**/*.mjs',
|
||||
'tools/**',
|
||||
'remotion/**',
|
||||
'scripts/remotion/**',
|
||||
],
|
||||
}
|
||||
);
|
||||
@@ -1,7 +1,11 @@
|
||||
import { resolve } from 'node:path';
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
const root = resolve(import.meta.dirname, '..');
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
root,
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['test/**/*.test.ts'],
|
||||
@@ -1,3 +1,9 @@
|
||||
> **⚠️ ARCHIVED 2026-05-21 — superseded, kept for history.**
|
||||
> The headline items here were verified resolved: the P0 `{WORKING_DIR}` placeholder
|
||||
> is now replaced (`plan-orchestrator.ts:431`), and the "~66 dead functions in app.js"
|
||||
> are gone (app.js was modularized 15K→3K LOC). A fresh `npm run knip` sweep on
|
||||
> 2026-05-21 found only a handful of unused test helpers. Do not treat this as a live TODO.
|
||||
|
||||
# Codebase Cleanup Findings
|
||||
|
||||
Compiled from parallel analysis of the entire Codeman codebase by 3 research agents (2026-02-19).
|
||||
@@ -1,3 +1,9 @@
|
||||
> **⚠️ ARCHIVED 2026-05-21 — superseded, kept for history.**
|
||||
> The "Critical" structural items here are done: `server.ts` 6,736→2,065 LOC,
|
||||
> `app.js` 15,196→3,083 LOC, `types.ts` 1,443→12 LOC (now a barrel → `src/types/`).
|
||||
> The phase plans that executed this work are in `docs/archive/phase*-plan.md`.
|
||||
> Do not treat this as a live TODO; see CLAUDE.md for current architecture.
|
||||
|
||||
# Code Structure & Quality Findings
|
||||
|
||||
**Date**: 2026-02-28
|
||||
@@ -0,0 +1,423 @@
|
||||
# Performance Analysis & Optimization Opportunities
|
||||
|
||||
**Date**: 2026-03-07
|
||||
**Scope**: Full-stack performance analysis — backend PTY handling, SSE broadcasting, frontend terminal rendering, local echo overlay, DOM updates, config/scaling limits.
|
||||
**Constraint**: All recommendations preserve existing functionality including local echo, backpressure, anti-flicker pipeline, and mobile support.
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The codebase is already well-optimized in critical paths. The multi-layer backpressure system, adaptive terminal batching, DEC 2026 sync markers, and incremental state serialization are strong. The main opportunities are in **reducing unnecessary work** (SSE filtering, DOM rebuilds, lazy terminal init) rather than algorithmic changes.
|
||||
|
||||
**Top 5 high-impact opportunities:**
|
||||
|
||||
| # | Optimization | Impact | Risk | Effort |
|
||||
|---|-------------|--------|------|--------|
|
||||
| 1 | Session-scoped SSE subscriptions | Bandwidth -60-80%, CPU -40% | Medium | Medium |
|
||||
| 2 | Lazy xterm.js for minimized subagent windows | Memory -3.5MB at 50 agents | Low | Low |
|
||||
| 3 | Targeted badge update (skip full tab rebuild) | Eliminates O(n) reflow on badge change | Low | Low |
|
||||
| 4 | Conditional SSE padding (tunnel-only, terminal-only) | Bandwidth -70% when tunneled | Low | Low |
|
||||
| 5 | Canvas renderer on mobile | GPU pressure reduction, battery savings | Low | Low |
|
||||
|
||||
---
|
||||
|
||||
## 1. SSE Broadcasting
|
||||
|
||||
### Current State
|
||||
- **92 event types** broadcast to all connected clients (max 100)
|
||||
- Single `JSON.stringify()` per event, shared across all clients (efficient)
|
||||
- **No per-client filtering** — every client receives every event regardless of which session they're viewing
|
||||
- 8KB padding appended to **every** event when tunnel is active (forces Cloudflare proxy flush)
|
||||
- Backpressure: clients marked as backpressured if `reply.raw.write()` returns false; recovery via `session:needsRefresh`
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B1: No session-scoped SSE subscriptions** (`server.ts:1986`)
|
||||
- Client viewing session A still receives all events for sessions B through T
|
||||
- With 20 active sessions, ~95% of terminal events are irrelevant to any given client
|
||||
- Cost: wasted bandwidth, CPU for JSON parsing, and event handler dispatch on client
|
||||
|
||||
**B2: Unconditional 8KB padding** (`server.ts:1977`)
|
||||
- Every event gets 8KB comment padding when tunnel is active
|
||||
- A `task:updated` event (~200 bytes payload) becomes ~8.2KB
|
||||
- High-frequency events like `session:terminal` need the padding; low-frequency events like `session:created` don't
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R1: Session-scoped SSE subscriptions** (High impact)
|
||||
- Add `?sessions=id1,id2` query param to `/api/events` SSE endpoint
|
||||
- Server filters events by session ID before broadcasting
|
||||
- Client subscribes to active session + "global" events (session lifecycle, system)
|
||||
- Re-subscribes on tab switch (or subscribe to all with client-side filter as fallback)
|
||||
- **Savings**: ~80% bandwidth reduction for single-session viewers; ~60% for multi-session dashboards
|
||||
|
||||
**R2: Tiered SSE padding** (Medium impact)
|
||||
- Only pad `session:terminal` events and SSE heartbeats (the two that need proxy flush)
|
||||
- Skip padding for low-frequency structural events (`session:created`, `task:updated`, etc.)
|
||||
- **Savings**: ~70% padding overhead reduction; terminal events already large enough to flush
|
||||
|
||||
---
|
||||
|
||||
## 2. Terminal Rendering
|
||||
|
||||
### Current State (Well-Optimized)
|
||||
- **6-layer anti-flicker pipeline**: Server batching (adaptive 16-50ms) → DEC 2026 sync wrap → single JSON serialize → client rAF batching → sync segment parser → chunked buffer loading (32KB/frame)
|
||||
- **64KB/frame write budget** with DEC 2026 sync-segment awareness (prevents 141KB single-frame freezes)
|
||||
- **3-layer backpressure**: SSE cap (128KB queued → drop + refresh), frame budget (64KB/frame), chunked restore (32KB/frame)
|
||||
- WebGL renderer enabled by default with canvas fallback on context loss
|
||||
- Typical latency: 16-32ms; worst case: ~115ms (50ms server batch + 50ms sync wait + 16ms rAF)
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B3: WebGL on mobile** (`app.js:627-637`)
|
||||
- Mobile GPUs are weaker; WebGL context loss more likely on low-end devices
|
||||
- Canvas renderer is sufficient for mobile (typically 1 session, smaller viewport)
|
||||
|
||||
**B4: Static scrollback for all sessions** (`app.js:572`)
|
||||
- Default 5000 lines scrollback for all sessions regardless of activity level
|
||||
- Heavy output sessions (build logs, test runners) accumulate large scroll buffers
|
||||
|
||||
**B5: No addon lazy loading**
|
||||
- FitAddon, Unicode11Addon, and WebGLAddon all loaded at terminal init
|
||||
- Unicode11Addon only needed for CJK content; WebGLAddon is large
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R3: Force canvas renderer on mobile** (Low risk)
|
||||
- Detect `MobileDetection.isMobile()` and skip WebGL addon loading
|
||||
- Reduces GPU memory pressure, prevents context loss crashes
|
||||
- Mobile typically has 1-2 sessions — canvas performance is more than adequate
|
||||
|
||||
**R4: Dynamic scrollback based on session activity** (Low risk)
|
||||
- Active sessions (working state): 5000 lines (current default)
|
||||
- Inactive/idle sessions: reduce to 2000 lines
|
||||
- Restore on session select (fetch from server buffer)
|
||||
- **Savings**: ~60% scrollback memory for idle sessions
|
||||
|
||||
**R5: Lazy-load Unicode11Addon** (Low risk)
|
||||
- Only load when CJK content is detected in terminal output
|
||||
- Detection: check for characters in CJK Unicode ranges during ANSI stripping (already iterating)
|
||||
- Most sessions never need it
|
||||
|
||||
---
|
||||
|
||||
## 3. DOM & Session Tab Rendering
|
||||
|
||||
### Current State
|
||||
- Session tabs use **intelligent incremental updates** with debounced 100ms rendering
|
||||
- Incremental path: only updates changed properties (classes, textContent, badges) when session list is stable
|
||||
- Full rebuild path: triggered when sessions added/removed **or badge count changes**
|
||||
- Subagent windows: per-window xterm.js instances, even when minimized
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B6: Badge count change triggers full tab rebuild** (`app.js:3207-3209`)
|
||||
- A single subagent badge increment on one tab triggers `_fullRenderSessionTabs()` — rebuilds entire sidebar HTML via `innerHTML =`
|
||||
- With 20 sessions, this is an O(n) reflow for a single badge number change
|
||||
- Badge changes are frequent during active subagent work
|
||||
|
||||
**B7: Minimized subagent windows retain xterm.js instances** (`subagent-windows.js`)
|
||||
- 50 subagent windows × ~75KB per xterm.js instance = ~3.75MB DOM memory
|
||||
- Minimized windows are invisible but their terminals remain in DOM
|
||||
- xterm.js instances continue processing resize events even when hidden
|
||||
|
||||
**B8: `backdrop-filter: blur()` on overlays** (`styles.css:2246-2247, 3098`)
|
||||
- Forces new stacking context, disables browser compositing optimizations
|
||||
- 50-100ms layout thrashing on modal open/close
|
||||
- Only 2 uses, but they're on frequently toggled overlays
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R6: Targeted badge update without full rebuild** (Low risk)
|
||||
- When badge count changes but session list is stable, update only the badge `<span>` textContent
|
||||
- Keep incremental path for badge changes; only use full rebuild for structural changes (add/remove sessions)
|
||||
- **Savings**: Eliminates O(n) reflow per badge change; reduces to O(1) targeted update
|
||||
|
||||
**R7: Lazy xterm.js initialization for subagent windows** (Medium impact)
|
||||
- Only create xterm.js Terminal instance when window is restored/maximized
|
||||
- On minimize: serialize terminal buffer, dispose Terminal instance, keep buffer in memory
|
||||
- On restore: create new Terminal, write buffer back
|
||||
- **Savings**: ~3.5MB DOM reduction at 50 minimized agents; eliminates hidden resize processing
|
||||
- **Trade-off**: ~200-500ms restore delay (buffer write), mitigated by chunked loading
|
||||
|
||||
**R8: Replace `backdrop-filter: blur()` with `background: rgba()`** (Low risk)
|
||||
- Use semi-transparent background instead of blur effect
|
||||
- Or use `will-change: transform` hint if blur is kept
|
||||
- **Savings**: Eliminates forced recomposition layer; 50-100ms faster overlay open
|
||||
|
||||
---
|
||||
|
||||
## 4. Backend PTY & State Management
|
||||
|
||||
### Current State (Excellent)
|
||||
- **BufferAccumulator**: Array-based chunking with lazy join on read — avoids O(n) string concatenation
|
||||
- **ANSI stripping**: Throttled at 150ms intervals with lazy evaluation (not per-chunk)
|
||||
- **State persistence**: 500ms debounce + incremental JSON caching per session (only dirty sessions re-serialized)
|
||||
- **Expensive parsers**: Throttled to 150ms window, accumulated data capped at 64KB
|
||||
- **Memory**: All buffers have hard limits (2MB terminal, 1MB text, 1000 messages, 64KB line buffer)
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B9: Pending clean data cap at 64KB** (`session.ts:1097-1133`)
|
||||
- Between 150ms processing windows, raw PTY data accumulates in `_pendingCleanData`
|
||||
- Capped at 64KB — excess data rolls off (old data discarded)
|
||||
- During heavy output (large build logs), this means parsers may miss content
|
||||
- Acceptable trade-off for performance, but worth documenting
|
||||
|
||||
**B10: `LRUMap.delete()` is O(n) worst case** (`utils/lru-map.ts:137-138`)
|
||||
- When deleting the newest entry, iterates all keys to find new newest
|
||||
- Rare in practice (delete is uncommon; set/get are hot paths)
|
||||
- Could matter during mass cleanup of 500 agents
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R9: Consider adaptive pending data cap** (Low priority)
|
||||
- During idle detection (critical to get right), increase cap to 128KB
|
||||
- During active working state, keep at 64KB (parsers less critical)
|
||||
- **Benefit**: More accurate idle detection during heavy output
|
||||
|
||||
**R10: Track second-newest in LRUMap** (Low priority)
|
||||
- Maintain a `_secondNewestKey` alongside `_newestKey`
|
||||
- On delete of newest, promote second-newest without iteration
|
||||
- Only matters at scale (500+ agents with frequent eviction)
|
||||
|
||||
---
|
||||
|
||||
## 5. Local Echo & Input Path
|
||||
|
||||
### Current State (Well-Designed)
|
||||
- **DOM overlay approach** — `<span>` elements in `.xterm-screen` at z-index 7, completely independent of `terminal.write()`
|
||||
- **Render caching**: `_lastRenderKey` includes text, position, column offsets — skips redundant re-renders
|
||||
- **Input flow**: Char accumulation → Enter triggers flush → 80ms delay before `\r` (ensures text reaches PTY first)
|
||||
- **Tab completion**: Baseline snapshot → detect buffer change → 300ms fallback timer
|
||||
- **CJK support**: Per-character width detection with `terminal.unicode.getStringCellWidth()` preferred, manual fallback
|
||||
- **Prompt detection**: Bottom-up line scan, O(rows) — cached position, column-lock prevents jitter
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B11: tmux send-keys latency** (~50-100ms per input)
|
||||
- Each `writeViaMux()` spawns a child process (`tmux send-keys`)
|
||||
- Text and Enter sent separately with 50ms delay between
|
||||
- For rapid typing: characters batch before Enter, so overhead is per-command not per-keystroke
|
||||
- **Acceptable trade-off** for session persistence (tmux survives server restarts)
|
||||
|
||||
**B12: 80ms delay between text flush and Enter** (`app.js:872-875`)
|
||||
- Intentional: ensures text reaches PTY before Enter, preventing Ink from processing empty input
|
||||
- Adds 80ms to perceived Enter-to-response latency
|
||||
- Could potentially be reduced with acknowledgment-based approach
|
||||
|
||||
**B13: Scroll listener on terminal viewport** (`zerolag-input-addon.ts:139`)
|
||||
- 50ms debounced re-render on scroll — acceptable but fires frequently during heavy output
|
||||
- Overlay hidden when scrolled up (correct behavior), shown when at bottom
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R11: Reduce Enter delay from 80ms to 50ms** (Low risk, test carefully)
|
||||
- The tmux `send-keys` already has 50ms internal delay
|
||||
- Combined with network latency, 80ms client-side may be excessive
|
||||
- Test with Ink-heavy sessions (Claude Code's status bar) — if text arrives before Enter at 50ms, reduce
|
||||
- **Savings**: 30ms perceived latency reduction per command
|
||||
|
||||
**R12: Batch tmux send-keys via stdin pipe** (Medium effort, high impact for rapid input)
|
||||
- Instead of spawning `tmux send-keys` per input, maintain a persistent connection
|
||||
- Use `tmux -C` (control mode) for programmatic interaction without child process spawning
|
||||
- **Savings**: Eliminate ~50-100ms process spawn overhead per input
|
||||
- **Risk**: Control mode has different semantics; needs careful testing with session persistence
|
||||
|
||||
**R13: Skip overlay re-render during heavy output scroll** (Low risk)
|
||||
- When terminal is receiving >10KB/s output, hide overlay entirely (user isn't typing during heavy output)
|
||||
- Re-show overlay after 500ms of output silence
|
||||
- **Savings**: Eliminates unnecessary DOM overlay re-renders during build logs / test output
|
||||
|
||||
---
|
||||
|
||||
## 6. Polling & File Watchers
|
||||
|
||||
### Current State
|
||||
- **SubagentWatcher**: 1s base poll, full scan throttled to every 5s, fs.watch() on known directories
|
||||
- **TranscriptWatcher**: 1 per session, fs.watch() primary with 1s poll fallback
|
||||
- **ImageWatcher**: chokidar per session with 100ms stability poll, burst limit 20/10s
|
||||
- **TeamWatcher**: chokidar primary with 30s poll fallback, LRU caches (50 teams, 200 tasks)
|
||||
- **RalphTracker**: Todo cleanup every 5 minutes
|
||||
|
||||
### Scaling Profile (20 sessions)
|
||||
| Component | Instances | Frequency | Total ops/sec |
|
||||
|-----------|-----------|-----------|---------------|
|
||||
| SubagentWatcher | 1 (global) | Full scan every 5s | 0.2/s |
|
||||
| TranscriptWatcher | 20 | 1s poll (fallback) | 20/s max |
|
||||
| ImageWatcher | 20 | 100ms poll (during writes only) | 200/s burst |
|
||||
| TeamWatcher | 1 (global) | 30s poll (fallback) | 0.03/s |
|
||||
| SSE heartbeat | 1 (global) | 15s | 0.07/s |
|
||||
| SSE dead client check | 1 (global) | 30s | 0.03/s |
|
||||
| Mux stats collection | 1 (global) | 2s | 0.5/s |
|
||||
| **Total steady-state** | | | **~21/s** |
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R14: Increase TranscriptWatcher poll interval to 2s** (Low risk)
|
||||
- Transcript changes are infrequent (new messages every few seconds at most)
|
||||
- fs.watch() is the primary mechanism; polling is fallback
|
||||
- **Savings**: Halves fallback filesystem checks (20/s → 10/s for 20 sessions)
|
||||
|
||||
**R15: Share chokidar instances for co-located session directories** (Medium effort)
|
||||
- Sessions in the same parent directory could share a single chokidar watcher with depth:3
|
||||
- Common case: multiple sessions in `~/projects/foo/` — one watcher covers all
|
||||
- **Savings**: Reduce chokidar instances from 20 to ~5-10 for typical workloads
|
||||
|
||||
---
|
||||
|
||||
## 7. Frontend Asset Delivery
|
||||
|
||||
### Current State
|
||||
- **app.js**: 12,027 lines (source) → esbuild minified → gzip/brotli compressed (~30-40KB gzipped)
|
||||
- **Static caching**: `maxAge: '1y'` via `@fastify/static`
|
||||
- **Service worker**: Push notification handler only — no asset caching
|
||||
- **No code splitting**: Single monolithic app.js bundle
|
||||
|
||||
### Bottlenecks
|
||||
|
||||
**B14: No cache-busting mechanism**
|
||||
- `maxAge: '1y'` means browsers cache aggressively
|
||||
- After deployment, users need `Ctrl+Shift+R` to see updates
|
||||
- No content hash in filenames or ETags for automatic invalidation
|
||||
|
||||
**B15: Monolithic app.js**
|
||||
- All 12K lines loaded on initial page load regardless of which features are used
|
||||
- Ralph wizard, plan orchestrator UI, team management — all loaded upfront
|
||||
- Mobile loads the same bundle as desktop
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R16: Add content hash to asset filenames** (Medium impact)
|
||||
- Build step: rename `app.js` → `app.[hash].js`
|
||||
- Generate a manifest or inject hash into HTML template
|
||||
- Keep `maxAge: '1y'` — cache invalidation happens via filename change
|
||||
- **Savings**: Eliminates stale cache issues after deployment; removes need for manual hard refresh
|
||||
|
||||
**R17: Code-split app.js into core + feature modules** (High effort, medium impact)
|
||||
- Core (~4K lines): terminal, SSE, session management, tabs, input handling
|
||||
- Deferred (~8K lines): Ralph wizard, plan UI, team management, subagent windows, image viewer
|
||||
- Load deferred modules on first use via dynamic `import()` or lazy `<script>` injection
|
||||
- **Savings**: ~60% reduction in initial load size; faster time-to-interactive
|
||||
- **Risk**: Complexity increase; need to handle loading states for deferred features
|
||||
- **Note**: May not be worth the effort given the app is already gzipped to ~30-40KB
|
||||
|
||||
---
|
||||
|
||||
## 8. CSS Performance
|
||||
|
||||
### Current State
|
||||
- **styles.css**: 7,153 lines with ~45 box-shadow uses, 2 backdrop-filter uses
|
||||
- Animations: GPU-accelerated keyframes for pulsing alerts, loading spinners
|
||||
- Z-index layering: well-organized (subagent 1000, plan 1100, log 2000, image 3000, overlay 7)
|
||||
|
||||
### Recommendations
|
||||
|
||||
**R18: Replace backdrop-filter with opaque overlay** (Low risk, covered in R8)
|
||||
|
||||
**R19: Use `contain: content` on subagent windows** (Low risk)
|
||||
- Add CSS containment to subagent window containers
|
||||
- Prevents layout changes inside windows from triggering reflow on parent
|
||||
- Especially valuable with 50 windows: changes in one window won't invalidate others
|
||||
- ```css
|
||||
.subagent-window { contain: content; }
|
||||
```
|
||||
- **Savings**: Reduces layout recalculation scope from global to per-window
|
||||
|
||||
**R20: Use `content-visibility: auto` on off-screen subagent windows** (Low risk)
|
||||
- Browser skips rendering of off-screen windows entirely
|
||||
- Combined with `contain-intrinsic-size` to prevent layout shift
|
||||
- ```css
|
||||
.subagent-window.minimized { content-visibility: hidden; }
|
||||
```
|
||||
- **Savings**: Browser skips paint/layout for minimized windows; complements R7
|
||||
|
||||
---
|
||||
|
||||
## 9. Memory & Scaling Limits
|
||||
|
||||
### Current Budget (20 sessions)
|
||||
| Component | Per Session | Total | Status |
|
||||
|-----------|-----------|-------|--------|
|
||||
| Terminal buffer | 2MB | 40MB | Hard-limited, auto-trim |
|
||||
| Text output | 1MB | 20MB | Hard-limited, auto-trim |
|
||||
| Messages | ~1MB | 20MB | Capped at 1000, trims to 800 |
|
||||
| Respawn buffer | 1MB | 20MB | Hard-limited |
|
||||
| **Buffers total** | | **100MB** | Acceptable |
|
||||
| TranscriptWatcher | ~100KB | 2MB | |
|
||||
| ImageWatcher | ~50KB | 1MB | |
|
||||
| SubagentWatcher | ~500KB | 500KB | Global |
|
||||
| Frontend terminal cache | ~256KB | 5MB | LRU, max 20 entries |
|
||||
| **Total estimated** | | **~110MB** | Comfortable |
|
||||
|
||||
### At Max Scale (50 sessions)
|
||||
- Buffers: ~250MB
|
||||
- Watchers: ~5MB
|
||||
- **Total: ~255MB** + Node.js overhead — acceptable on modern hardware
|
||||
|
||||
### Potential Leak Vectors (All Mitigated)
|
||||
- `_shortIdCache` in server — unbounded Map, but entries are tiny (string→string); grows at O(sessions created), not O(events)
|
||||
- All CleanupManager-registered resources tracked and disposed on session stop
|
||||
- `isStopped` guard prevents new timers after session cleanup
|
||||
|
||||
---
|
||||
|
||||
## 10. Implementation Priority Matrix
|
||||
|
||||
### Phase 1 — Quick Wins (1-2 hours each, low risk)
|
||||
| # | Optimization | Files to Change |
|
||||
|---|-------------|-----------------|
|
||||
| R6 | Targeted badge update | `app.js` (3207-3209) |
|
||||
| R3 | Canvas renderer on mobile | `app.js` (627-637) |
|
||||
| R8 | Replace backdrop-filter blur | `styles.css` (2246, 3098) |
|
||||
| R19 | CSS containment on subagent windows | `styles.css` |
|
||||
| R20 | `content-visibility: hidden` on minimized windows | `styles.css` |
|
||||
|
||||
### Phase 2 — Medium Effort (half-day each)
|
||||
| # | Optimization | Files to Change |
|
||||
|---|-------------|-----------------|
|
||||
| R2 | Tiered SSE padding | `server.ts` (broadcast function) |
|
||||
| R7 | Lazy xterm.js for minimized subagents | `subagent-windows.js` |
|
||||
| R11 | Reduce Enter delay to 50ms | `app.js` (872-875), test with Ink |
|
||||
| R14 | TranscriptWatcher 2s poll | `transcript-watcher.ts` |
|
||||
| R16 | Content-hash asset filenames | `build.mjs`, `server.ts` |
|
||||
|
||||
### Phase 3 — Larger Initiatives (1-2 days each)
|
||||
| # | Optimization | Files to Change |
|
||||
|---|-------------|-----------------|
|
||||
| R1 | Session-scoped SSE subscriptions | `server.ts`, `app.js` (SSE connect) |
|
||||
| R5 | Lazy Unicode11Addon loading | `app.js`, build pipeline |
|
||||
| R12 | Persistent tmux control mode | `tmux-manager.ts` |
|
||||
| R17 | Code-split app.js | `app.js`, `build.mjs`, HTML template |
|
||||
|
||||
### Not Recommended (Low ROI or High Risk)
|
||||
| # | Why Not |
|
||||
|---|---------|
|
||||
| R4 | Dynamic scrollback adds complexity; memory savings marginal vs total budget |
|
||||
| R9 | Adaptive pending data cap adds state; current 64KB cap rarely matters |
|
||||
| R10 | LRUMap.delete() O(n) is theoretical; never triggered at current scale |
|
||||
| R15 | Shared chokidar instances add directory-matching complexity for minimal gain |
|
||||
|
||||
---
|
||||
|
||||
## Appendix: Key File Locations
|
||||
|
||||
| Area | File | Key Lines |
|
||||
|------|------|-----------|
|
||||
| SSE broadcast | `src/web/server.ts` | 1961-1989 (broadcast), 1934-1959 (backpressure) |
|
||||
| Terminal batching | `src/web/server.ts` | 1994-2048 (per-session adaptive batching) |
|
||||
| Frame budget | `src/web/public/app.js` | 1370-1478 (flushPendingWrites, 64KB cap) |
|
||||
| Flicker filter | `src/web/public/app.js` | 1176-1255 (50ms sync wait, 256KB safety) |
|
||||
| Tab rendering | `src/web/public/app.js` | 3108-3357 (incremental + full rebuild) |
|
||||
| Tab switching | `src/web/public/app.js` | 3560-3760 (cache + chunked load + deferred UI) |
|
||||
| Local echo | `packages/xterm-zerolag-input/src/` | All files (overlay, prompt, CJK) |
|
||||
| Local echo integration | `src/web/public/app.js` | 640, 815-988 (input flow) |
|
||||
| Subagent windows | `src/web/public/subagent-windows.js` | Full file (window mgmt, drag, minimize) |
|
||||
| State persistence | `src/state-store.ts` | 161-250 (debounced save, incremental JSON) |
|
||||
| Buffer accumulator | `src/utils/buffer-accumulator.ts` | Full file (array chunks, lazy join) |
|
||||
| PTY handling | `src/session.ts` | 1046-1133 (data flow), 1173-1230 (parsing) |
|
||||
| Config limits | `src/config/` | 9 files (buffer, map, timing, auth, etc.) |
|
||||
| Anti-flicker docs | `docs/terminal-anti-flicker.md` | Architecture reference |
|
||||
| CSS | `src/web/public/styles.css` | 2246 (backdrop-filter), full file |
|
||||
| Build pipeline | `scripts/build.mjs` | 59-68 (minify + compress) |
|
||||
@@ -0,0 +1,74 @@
|
||||
# Codeman Performance Optimization Plan
|
||||
|
||||
## Current State
|
||||
|
||||
The backend is **already production-grade** — SSE broadcasting, state persistence, terminal batching, buffer management, and memory patterns are all well-optimized. The biggest gains are on the **frontend delivery** side.
|
||||
|
||||
## Implemented Optimizations
|
||||
|
||||
### 1. V8 Compile Cache (10-20% faster cold start)
|
||||
|
||||
**Files:** `scripts/codeman-web.service`, `package.json`
|
||||
|
||||
Node.js re-parses and compiles all JS on every cold start. `NODE_COMPILE_CACHE` caches V8 compiled bytecode to disk, reusing it on subsequent starts.
|
||||
|
||||
- Added `Environment=NODE_COMPILE_CACHE=/home/arkon/.codeman/compile-cache` to systemd service
|
||||
- Added to `npm start` script for non-systemd usage
|
||||
- Zero code changes, immediate win on every restart
|
||||
|
||||
### 2. WebGL Addon Lazy-Loading (244KB saved on mobile, non-blocking on desktop)
|
||||
|
||||
**Files:** `src/web/public/index.html`, `src/web/public/app.js`
|
||||
|
||||
`xterm-addon-webgl.min.js` (244KB) was loaded eagerly for all users via `<script defer>`, but only used on desktop with WebGL2 support.
|
||||
|
||||
- Removed `<script defer>` from `index.html`
|
||||
- Added dynamic script loading in `app.js` — only downloads on desktop when WebGL is needed
|
||||
- Mobile users never download the file at all (244KB saved)
|
||||
- Desktop: loads in parallel with page rendering, addon initializes when ready
|
||||
- Graceful fallback: canvas renderer used if WebGL unavailable or script fails
|
||||
|
||||
### 3. Preload Hints (~50-100ms faster perceived load)
|
||||
|
||||
**Files:** `src/web/public/index.html`
|
||||
|
||||
Browser discovers `<script defer>` tags only when the parser reaches them at the bottom of `<body>`. By then, the HTML parse has blocked for hundreds of lines.
|
||||
|
||||
- Added `<link rel="preload" as="script">` in `<head>` for `vendor/xterm.min.js`, `constants.js`, `app.js`
|
||||
- Browser starts fetching critical scripts immediately during HTML parse (before reaching `<body>`)
|
||||
- Zero runtime overhead — just hints for the browser's preload scanner
|
||||
|
||||
### 4. Batch Tmux Reconciliation (N subprocess calls → 1)
|
||||
|
||||
**Files:** `src/tmux-manager.ts`
|
||||
|
||||
`reconcileSessions()` previously called `tmux has-session` + `tmux display-message` per known session, plus `tmux list-sessions` for discovery, plus `tmux display-message` per discovered session. With 20 sessions: 41+ subprocess calls.
|
||||
|
||||
- Replaced with single `tmux list-panes -a -F '#{session_name}\t#{pane_pid}'` call
|
||||
- Builds a Map from the result, then does O(1) lookups for both known and discovered sessions
|
||||
- Also replaced inner O(n) `isKnown` scan with a Set lookup
|
||||
- 20 sessions: 41 subprocess calls → 1, with faster lookups
|
||||
|
||||
### 5. Asset Hashing / Cache Busting (already implemented)
|
||||
|
||||
**Files:** `scripts/build.mjs` (pre-existing)
|
||||
|
||||
Content-hash cache busting was already implemented in the build script:
|
||||
- All app JS/CSS files get content hashes (`app.abc123.js`)
|
||||
- `index.html` rewritten to reference hashed filenames
|
||||
- Pre-compressed with gzip + Brotli
|
||||
- 1-year immutable cache works correctly — new deploys get new filenames
|
||||
|
||||
## Already Optimized (No Action Needed)
|
||||
|
||||
| Area | Why It's Fine |
|
||||
|------|---------------|
|
||||
| **SSE Broadcasting** | Single serialization per broadcast, preformatted frames, backpressure handling, session subscription filtering |
|
||||
| **State Persistence** | 500ms debounce, incremental per-session JSON caching, async atomic writes, circuit breaker on failures |
|
||||
| **Terminal Batching** | Adaptive intervals (16-50ms), per-session queues, immediate flush at 32KB, array-based accumulation |
|
||||
| **Buffer Management** | BufferAccumulator (array-push, lazy join), auto-trim at 2MB/1MB, no string concatenation in hot paths |
|
||||
| **ANSI Stripping** | Pre-compiled regex via factory functions, single-pass processing |
|
||||
| **Static File Serving** | @fastify/static with 1-year cache, pre-compressed Brotli/gzip, no-cache for HTML |
|
||||
| **Memory Management** | CleanupManager, LRUMap, StaleExpirationMap, bounded buffers, explicit listener cleanup |
|
||||
| **Import Patterns** | Pure ESM, lazy web server import, no circular deps, no dynamic imports in hot paths |
|
||||
| **Config Loading** | Small constant files, no I/O at import time, specific imports (no barrel) |
|
||||
|
After Width: | Height: | Size: 806 KiB |
@@ -1,5 +1,7 @@
|
||||
# Local Echo Overlay — Implementation Plan
|
||||
|
||||
> **Status: SHIPPED.** Implementation lives in `packages/xterm-zerolag-input/src/` (overlay-renderer.ts, prompt-finder.ts, cell-dimensions.ts, zerolag-input-addon.ts) with the embedded copy in `src/web/public/app.js`. This document is retained as historical design context.
|
||||
|
||||
## Context
|
||||
|
||||
User accesses Codeman remotely from Thailand to Switzerland over Tailscale (~200-300ms RTT).
|
||||
@@ -18,9 +20,9 @@ redraws. A DOM overlay sits in a separate rendering layer (z-index 7) and doesn'
|
||||
with Ink's cursor management or screen redraws at all. When Ink redraws (server output arrives),
|
||||
we simply hide the overlay.
|
||||
|
||||
**Why it will look indistinguishable:** We use the DOM renderer (not canvas/WebGL) in our
|
||||
xterm.js v5.3.0, so both terminal text and overlay text are rendered by the same browser
|
||||
font engine with identical sub-pixel rendering.
|
||||
**Why it will look indistinguishable:** We use the DOM renderer (not canvas/WebGL), so both
|
||||
terminal text and overlay text are rendered by the same browser font engine with identical
|
||||
sub-pixel rendering. (Originally designed against xterm.js v5.3.0; project now on `@xterm/xterm` ^6.0.0 — the internal `_core._renderService.dimensions` access path still works in v6.)
|
||||
|
||||
## Key Technical Details (from research)
|
||||
|
||||
@@ -36,7 +38,7 @@ const top = cursorY * dims.css.cell.height; // CSS pixels, relative to .xterm-
|
||||
- `cursorY` = `terminal.buffer.active.cursorY` (0 to terminal.rows-1, ALREADY viewport-relative)
|
||||
- No scroll offset math needed
|
||||
|
||||
### Cell Dimensions (v5.3.0 — no public API, use internal)
|
||||
### Cell Dimensions (no public API in v5/v6 — use internal; public in v7+)
|
||||
```js
|
||||
const dims = terminal._core._renderService.dimensions;
|
||||
dims.css.cell.width // e.g., 8.4px
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
# Orchestrator Loop — Architecture & Data Flow
|
||||
|
||||
> Technical architecture document. Not for GitHub.
|
||||
|
||||
## System Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ CODEMAN WEB UI │
|
||||
│ ┌──────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Orchestrator Dashboard │ │
|
||||
│ │ [Goal Input] [Plan View] [Phase Progress] [Agent Activity] │ │
|
||||
│ └───────────────────────────┬──────────────────────────────────┘ │
|
||||
│ │ SSE Events │
|
||||
│ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Orchestrator API Routes (/api/orchestrator/*) │ │
|
||||
│ └───────────────────────────┬──────────────────────────────────┘ │
|
||||
└───────────────────────────────┼─────────────────────────────────────┘
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ ORCHESTRATOR LOOP │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
|
||||
│ │ Orchestrator │ │ Orchestrator │ │ Orchestrator │ │
|
||||
│ │ Planner │ │ Loop (state │ │ Verifier │ │
|
||||
│ │ │ │ machine) │ │ │ │
|
||||
│ │ • Research │◄──►│ • Phase mgmt │◄──►│ • Test runner │ │
|
||||
│ │ • Plan gen │ │ • Task queue │ │ • AI review │ │
|
||||
│ │ • Phasing │ │ • Event loop │ │ • Output checks │ │
|
||||
│ └──────┬───────┘ └──────┬───────┘ └──────────┬───────────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌──────────────────────────────────────────────────────────────┐ │
|
||||
│ │ EXISTING CODEMAN INFRASTRUCTURE │ │
|
||||
│ │ │ │
|
||||
│ │ SessionManager ←→ Sessions ←→ PTY (Claude CLI) │ │
|
||||
│ │ ↑ ↑ ↑ │ │
|
||||
│ │ │ │ │ │ │
|
||||
│ │ TaskQueue RalphTracker RespawnController │ │
|
||||
│ │ StateStore HooksConfig TeamWatcher │ │
|
||||
│ │ Auto-Ops SubagentWatcher SSE Broadcast │ │
|
||||
│ └──────────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Data Flow: Complete Lifecycle
|
||||
|
||||
### 1. User Submits Goal
|
||||
|
||||
```
|
||||
User → POST /api/orchestrator/start { goal: "Build a REST API...", config: {...} }
|
||||
→ OrchestratorLoop.start(goal)
|
||||
→ state = PLANNING
|
||||
→ emit('stateChanged', 'planning')
|
||||
→ SSE: orchestrator:stateChanged
|
||||
```
|
||||
|
||||
### 2. Planning Phase
|
||||
|
||||
```
|
||||
OrchestratorPlanner.generatePlan(goal)
|
||||
→ PlanOrchestrator.generateDetailedPlan(goal)
|
||||
→ [Research Agent] → enriched task description
|
||||
→ [Planner Agent] → PlanItem[]
|
||||
→ groupIntoPhases(planItems)
|
||||
→ topological sort by dependencies
|
||||
→ group into layers
|
||||
→ assign team strategies
|
||||
→ OrchestratorPlan { phases: [...] }
|
||||
→ state = APPROVAL
|
||||
→ emit('planReady', plan)
|
||||
→ SSE: orchestrator:planReady
|
||||
```
|
||||
|
||||
### 3. User Approves Plan
|
||||
|
||||
```
|
||||
User → POST /api/orchestrator/approve
|
||||
→ OrchestratorLoop.approvePlan()
|
||||
→ state = EXECUTING
|
||||
→ executePhase(phases[0])
|
||||
```
|
||||
|
||||
### 4. Phase Execution
|
||||
|
||||
```
|
||||
executePhase(phase)
|
||||
→ For each task in phase:
|
||||
→ Convert to CreateTaskOptions
|
||||
→ Add to TaskQueue with completion phrase "PHASE_{N}_TASK_{M}_DONE"
|
||||
→ If phase.teamStrategy.type === 'team':
|
||||
→ Start session with AGENT_TEAMS enabled
|
||||
→ Send team orchestration prompt to lead
|
||||
→ Else:
|
||||
→ Assign tasks to available sessions (same as RalphLoop)
|
||||
|
||||
→ Listen for task completion events:
|
||||
→ TaskQueue emits taskCompleted
|
||||
→ Check: all phase tasks done?
|
||||
→ Yes → state = VERIFYING → verifyPhase(phase)
|
||||
→ No → wait for more completions
|
||||
```
|
||||
|
||||
### 5. Verification
|
||||
|
||||
```
|
||||
verifyPhase(phase)
|
||||
→ OrchestratorVerifier.verify(phase, session)
|
||||
→ Run test commands via session
|
||||
→ Check file existence
|
||||
→ AI review (optional)
|
||||
→ If passed:
|
||||
→ phase.status = 'passed'
|
||||
→ emit('phaseCompleted', phase)
|
||||
→ If more phases: executePhase(nextPhase)
|
||||
→ If last phase: state = COMPLETED
|
||||
→ If failed:
|
||||
→ phase.attempts++
|
||||
→ If attempts < maxAttempts:
|
||||
→ state = REPLANNING
|
||||
→ Generate recovery tasks
|
||||
→ state = EXECUTING (retry)
|
||||
→ Else:
|
||||
→ state = FAILED
|
||||
→ emit('phaseFailed', phase, reason)
|
||||
```
|
||||
|
||||
### 6. Context Management Between Phases
|
||||
|
||||
```
|
||||
After phase completion:
|
||||
→ If config.compactBetweenPhases:
|
||||
→ session.sendInput('/compact')
|
||||
→ Wait for compact to complete
|
||||
→ If config.respawnBetweenMilestones && phase is a milestone:
|
||||
→ Save orchestrator state to StateStore
|
||||
→ Respawn session (kill + recreate)
|
||||
→ Send resume prompt with phase context
|
||||
```
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
src/
|
||||
├── orchestrator-loop.ts # Main state machine (~400 lines)
|
||||
├── orchestrator-planner.ts # Plan generation + phase grouping (~300 lines)
|
||||
├── orchestrator-verifier.ts # Phase verification (~200 lines)
|
||||
├── types/
|
||||
│ └── orchestrator.ts # All orchestrator types (~150 lines)
|
||||
├── prompts/
|
||||
│ └── orchestrator.ts # Prompt templates (~200 lines)
|
||||
├── web/
|
||||
│ ├── routes/
|
||||
│ │ └── orchestrator-routes.ts # API endpoints (~250 lines)
|
||||
│ └── public/
|
||||
│ └── orchestrator-ui.js # Frontend panel (~500 lines)
|
||||
```
|
||||
|
||||
## Integration Points with Existing Code
|
||||
|
||||
### StateStore (`src/state-store.ts`)
|
||||
```typescript
|
||||
// Add to AppState interface
|
||||
orchestrator?: OrchestratorPersistState;
|
||||
|
||||
// Add methods
|
||||
getOrchestratorState(): OrchestratorPersistState;
|
||||
setOrchestratorState(state: Partial<OrchestratorPersistState>): void;
|
||||
```
|
||||
|
||||
### SSE Events (`src/web/sse-events.ts`)
|
||||
```typescript
|
||||
// Add ~8 new events
|
||||
export const SseEvent = {
|
||||
// ... existing
|
||||
ORCHESTRATOR_STATE_CHANGED: 'orchestrator:stateChanged',
|
||||
ORCHESTRATOR_PLAN_READY: 'orchestrator:planReady',
|
||||
ORCHESTRATOR_PHASE_STARTED: 'orchestrator:phaseStarted',
|
||||
ORCHESTRATOR_PHASE_COMPLETED: 'orchestrator:phaseCompleted',
|
||||
ORCHESTRATOR_PHASE_FAILED: 'orchestrator:phaseFailed',
|
||||
ORCHESTRATOR_VERIFICATION: 'orchestrator:verificationResult',
|
||||
ORCHESTRATOR_COMPLETED: 'orchestrator:completed',
|
||||
ORCHESTRATOR_ERROR: 'orchestrator:error',
|
||||
} as const;
|
||||
```
|
||||
|
||||
### Frontend Constants (`src/web/public/constants.js`)
|
||||
```javascript
|
||||
// Mirror SSE events
|
||||
SSE_EVENTS.ORCHESTRATOR_STATE_CHANGED = 'orchestrator:stateChanged';
|
||||
// ... etc
|
||||
```
|
||||
|
||||
### Route Registration (`src/web/routes/index.ts`)
|
||||
```typescript
|
||||
import { registerOrchestratorRoutes } from './orchestrator-routes.js';
|
||||
// Add to barrel export
|
||||
```
|
||||
|
||||
### Server (`src/web/server.ts`)
|
||||
```typescript
|
||||
// Initialize OrchestratorLoop alongside RalphLoop
|
||||
const orchestratorLoop = new OrchestratorLoop(config);
|
||||
|
||||
// Register routes
|
||||
registerOrchestratorRoutes(app, { ...ctx, orchestrator: orchestratorLoop });
|
||||
```
|
||||
|
||||
### Port Interface (`src/web/ports/`)
|
||||
```typescript
|
||||
// New port
|
||||
export interface OrchestratorPort {
|
||||
orchestrator: OrchestratorLoop;
|
||||
}
|
||||
```
|
||||
|
||||
## Prompt Flow Through System
|
||||
|
||||
The key insight is how prompts flow from Orchestrator → Session → Claude:
|
||||
|
||||
```
|
||||
OrchestratorLoop decides to execute Phase 3, Task 2
|
||||
│
|
||||
▼
|
||||
Converts OrchestratorTask to CreateTaskOptions:
|
||||
{
|
||||
prompt: "Implement the rate limiter middleware. Read src/middleware/auth.ts
|
||||
for the pattern. Add to src/middleware/rate-limiter.ts. Must export
|
||||
a Fastify plugin. When done: <promise>PHASE_3_TASK_2_DONE</promise>",
|
||||
priority: 100,
|
||||
dependencies: ["phase-3-task-1"], // Must finish auth middleware first
|
||||
completionPhrase: "PHASE_3_TASK_2_DONE",
|
||||
timeoutMs: 600000 // 10 minutes
|
||||
}
|
||||
│
|
||||
▼
|
||||
TaskQueue.addTask(options)
|
||||
│
|
||||
▼
|
||||
RalphLoop.tick() → assignTasks() // OR OrchestratorLoop does its own assignment
|
||||
│
|
||||
▼
|
||||
session.sendInput(task.prompt)
|
||||
│
|
||||
▼
|
||||
writeViaMux() → tmux send-keys -l "prompt..." + Enter
|
||||
│
|
||||
▼
|
||||
Claude CLI receives prompt, executes, outputs results
|
||||
│
|
||||
▼
|
||||
RalphTracker.processData() → detects "PHASE_3_TASK_2_DONE"
|
||||
│
|
||||
▼
|
||||
emit('completionDetected') → OrchestratorLoop.handleTaskCompleted()
|
||||
│
|
||||
▼
|
||||
Check: all tasks in Phase 3 done? → If yes → verifyPhase(phase3)
|
||||
```
|
||||
|
||||
## Team Agent Flow (When Enabled)
|
||||
|
||||
```
|
||||
Phase has teamStrategy.type === 'team'
|
||||
│
|
||||
▼
|
||||
OrchestratorLoop creates/reuses a session with:
|
||||
env: { CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS: '1' }
|
||||
│
|
||||
▼
|
||||
Sends team orchestration prompt:
|
||||
"You're the team lead for Phase 3: Core Implementation.
|
||||
|
||||
Your team should work on these tasks in parallel:
|
||||
1. Rate limiter middleware (teammate 1)
|
||||
2. Error handling middleware (teammate 2)
|
||||
3. Validation layer (teammate 3)
|
||||
|
||||
Context files to read first: [...]
|
||||
Each teammate should output their task's completion phrase when done.
|
||||
When ALL tasks are complete, output: <promise>PHASE_3_COMPLETE</promise>"
|
||||
│
|
||||
▼
|
||||
Claude Code team-lead spawns teammates
|
||||
│
|
||||
▼
|
||||
TeamWatcher detects new team in ~/.claude/teams/
|
||||
→ Matches to session via leadSessionId
|
||||
→ Tracks teammate activity
|
||||
│
|
||||
▼
|
||||
Teammates work in parallel (in-process threads)
|
||||
│
|
||||
▼
|
||||
hook: teammate_idle → POST /api/hook-event
|
||||
→ OrchestratorLoop notes teammate finished
|
||||
│
|
||||
▼
|
||||
hook: task_completed → POST /api/hook-event
|
||||
→ Or: RalphTracker detects PHASE_3_COMPLETE
|
||||
→ OrchestratorLoop → phase complete → verify
|
||||
```
|
||||
|
||||
## Error Recovery Strategy
|
||||
|
||||
```
|
||||
Task fails (timeout, error, session crash)
|
||||
│
|
||||
├─ Task-level retry (up to 2 retries per task)
|
||||
│ → Reset task to pending
|
||||
│ → Re-queue with modified prompt: "Previous attempt failed: {error}. Try again..."
|
||||
│
|
||||
├─ Phase-level retry (up to 3 retries per phase)
|
||||
│ → Respawn session (fresh context)
|
||||
│ → Re-execute entire phase with learnings from failure
|
||||
│ → Modified prompt includes what went wrong
|
||||
│
|
||||
└─ Orchestration-level failure
|
||||
→ All retries exhausted
|
||||
→ state = FAILED
|
||||
→ Notify user with detailed failure report
|
||||
→ User can: modify plan → retry, skip phase → continue, or stop
|
||||
```
|
||||
|
||||
## Interaction with Ralph Loop
|
||||
|
||||
Ralph Loop and Orchestrator Loop are **mutually exclusive** on the same sessions:
|
||||
|
||||
```
|
||||
if (orchestratorLoop.isRunning()) {
|
||||
// Orchestrator controls task assignment
|
||||
// Ralph Loop should not interfere
|
||||
// Respawn Controller uses 'orchestrator' preset
|
||||
}
|
||||
|
||||
if (ralphLoop.isRunning()) {
|
||||
// Ralph controls task assignment
|
||||
// Orchestrator should not start
|
||||
}
|
||||
```
|
||||
|
||||
The Orchestrator can optionally USE the Ralph Loop internally for phase execution (delegate phase tasks to Ralph's queue), or manage task assignment directly. Decision: **manage directly** — gives more control over phase boundaries and verification timing.
|
||||
|
||||
## Summary of What Touches What
|
||||
|
||||
| Existing File | Change |
|
||||
|---|---|
|
||||
| `src/types/index.ts` | Export orchestrator types |
|
||||
| `src/state-store.ts` | Add orchestrator state persistence |
|
||||
| `src/web/sse-events.ts` | Add ~8 orchestrator events |
|
||||
| `src/web/routes/index.ts` | Register orchestrator routes |
|
||||
| `src/web/server.ts` | Initialize OrchestratorLoop |
|
||||
| `src/web/public/constants.js` | Mirror SSE events |
|
||||
| `src/web/public/app.js` | Add orchestrator event listeners, panel toggle |
|
||||
| `src/web/route-helpers.ts` | Add 'orchestrator' respawn preset |
|
||||
|
||||
| New File | Purpose |
|
||||
|---|---|
|
||||
| `src/orchestrator-loop.ts` | Core state machine |
|
||||
| `src/orchestrator-planner.ts` | Plan generation + phasing |
|
||||
| `src/orchestrator-verifier.ts` | Phase verification |
|
||||
| `src/types/orchestrator.ts` | Type definitions |
|
||||
| `src/prompts/orchestrator.ts` | Prompt templates |
|
||||
| `src/web/routes/orchestrator-routes.ts` | API endpoints |
|
||||
| `src/web/public/orchestrator-ui.js` | Frontend panel |
|
||||
| `src/web/ports/orchestrator-port.ts` | Port interface |
|
||||
@@ -0,0 +1,633 @@
|
||||
# Orchestrator Loop — Detailed Implementation Plan (v2)
|
||||
|
||||
> Internal research/planning document. Not for GitHub.
|
||||
|
||||
## Vision
|
||||
|
||||
The **Orchestrator Loop** is a new autonomous execution mode that transforms high-level user goals into phased, verified, team-coordinated implementations. Unlike Ralph Loop (flat task queue → idle sessions), the Orchestrator manages the full lifecycle: **plan → approve → execute → verify → adapt → complete**.
|
||||
|
||||
```
|
||||
USER: "Add OAuth2 login with Google/GitHub, role-based access control, and API key management"
|
||||
|
||||
ORCHESTRATOR:
|
||||
Phase 1: Research & Setup ✅ (3m) — scaffold, deps, config
|
||||
Phase 2: Auth Core ✅ (8m) — OAuth2 flow, session mgmt
|
||||
Phase 3: Provider Integration 🔄 (12m) — Google + GitHub (parallel via team agents)
|
||||
Phase 4: RBAC ⏳ — roles, permissions, middleware
|
||||
Phase 5: API Keys ⏳ — generation, validation, rate limits
|
||||
Phase 6: Testing & Review ⏳ — integration tests, security review
|
||||
|
||||
Progress: ━━━━━━━━━━━━━━━━━━━━ 40% | Agents: 3 active | Time: 23m
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ OrchestratorLoop │
|
||||
│ │
|
||||
│ ┌────────────────┐ ┌────────────────┐ ┌──────────────────┐ │
|
||||
│ │ Orchestrator │ │ Orchestrator │ │ Orchestrator │ │
|
||||
│ │ Planner │ │ Executor │ │ Verifier │ │
|
||||
│ │ │ │ │ │ │ │
|
||||
│ │ PlanOrchestrator│ │ TaskQueue │ │ AI review │ │
|
||||
│ │ + phase grouper│ │ SessionManager │ │ Test commands │ │
|
||||
│ │ + team strategy│ │ Team prompts │ │ File checks │ │
|
||||
│ └───────┬────────┘ └───────┬────────┘ └─────────┬────────┘ │
|
||||
│ │ │ │ │
|
||||
│ └───────────────────┼──────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌─────────▼─────────┐ │
|
||||
│ │ Existing Codeman │ │
|
||||
│ │ Infrastructure │ │
|
||||
│ │ │ │
|
||||
│ │ SessionManager │ │
|
||||
│ │ TaskQueue │ │
|
||||
│ │ RespawnController │ │
|
||||
│ │ TeamWatcher │ │
|
||||
│ │ PlanOrchestrator │ │
|
||||
│ │ StateStore │ │
|
||||
│ │ Hooks + SSE │ │
|
||||
│ └────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## State Machine
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ IDLE │
|
||||
└────┬────┘
|
||||
│ start(goal)
|
||||
▼
|
||||
┌─────────┐
|
||||
┌────────│PLANNING │────────┐
|
||||
│ fail └────┬────┘ │
|
||||
▼ │ plan ready │ user cancels
|
||||
┌────────┐ ▼ ▼
|
||||
│ FAILED │ ┌─────────┐ ┌────────┐
|
||||
└────────┘ │APPROVAL │ │ IDLE │
|
||||
▲ └────┬────┘ └────────┘
|
||||
│ │ approve
|
||||
│ ▼
|
||||
│ ┌──────────┐
|
||||
│ ┌───►│EXECUTING │◄────────────────────┐
|
||||
│ │ └────┬─────┘ │
|
||||
│ │ │ all tasks in phase done │
|
||||
│ │ ▼ │
|
||||
│ │ ┌──────────┐ │
|
||||
│ │ │VERIFYING │ │
|
||||
│ │ └────┬─────┘ │
|
||||
│ │ pass │ │ fail │
|
||||
│ │ ▼ ▼ │
|
||||
│ │ more ┌──────────┐ │
|
||||
│ │ phases?│REPLANNING│── retry ────────┘
|
||||
│ │ │ └────┬─────┘
|
||||
│ │ │ │ max retries
|
||||
│ │ │ ▼
|
||||
│ │ │ ┌────────┐
|
||||
│ └────┘ │ FAILED │
|
||||
│ next └────────┘
|
||||
│ phase
|
||||
│ │
|
||||
│ ▼
|
||||
│ ┌───────────┐
|
||||
└─│ COMPLETED │
|
||||
└───────────┘
|
||||
```
|
||||
|
||||
**States:** `idle` | `planning` | `approval` | `executing` | `verifying` | `replanning` | `completed` | `failed` | `paused`
|
||||
|
||||
Transitions are event-driven. The state machine is the single source of truth — all methods check `this.state` before acting.
|
||||
|
||||
## Type Definitions
|
||||
|
||||
### `src/types/orchestrator.ts`
|
||||
|
||||
```typescript
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// State Machine
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export type OrchestratorState =
|
||||
| 'idle'
|
||||
| 'planning'
|
||||
| 'approval'
|
||||
| 'executing'
|
||||
| 'verifying'
|
||||
| 'replanning'
|
||||
| 'completed'
|
||||
| 'failed'
|
||||
| 'paused';
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Plan Structure
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export interface OrchestratorPlan {
|
||||
id: string;
|
||||
goal: string;
|
||||
createdAt: number;
|
||||
phases: OrchestratorPhase[];
|
||||
metadata: {
|
||||
totalTasks: number;
|
||||
estimatedComplexity: 'low' | 'medium' | 'high';
|
||||
modelUsed: string;
|
||||
planDurationMs: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface OrchestratorPhase {
|
||||
id: string; // "phase-1", "phase-2"
|
||||
name: string; // Human-readable name
|
||||
description: string;
|
||||
order: number;
|
||||
status: PhaseStatus;
|
||||
tasks: OrchestratorTask[];
|
||||
verificationCriteria: string[];
|
||||
testCommands: string[];
|
||||
maxAttempts: number; // Default: 3
|
||||
attempts: number; // Current attempt count
|
||||
startedAt: number | null;
|
||||
completedAt: number | null;
|
||||
durationMs: number | null;
|
||||
teamStrategy: TeamStrategy;
|
||||
}
|
||||
|
||||
export type PhaseStatus =
|
||||
| 'pending'
|
||||
| 'executing'
|
||||
| 'verifying'
|
||||
| 'passed'
|
||||
| 'failed'
|
||||
| 'skipped';
|
||||
|
||||
export interface OrchestratorTask {
|
||||
id: string; // "phase-1-task-1"
|
||||
phaseId: string;
|
||||
prompt: string; // Single-line prompt for Claude
|
||||
status: 'pending' | 'running' | 'completed' | 'failed';
|
||||
assignedSessionId: string | null;
|
||||
queueTaskId: string | null; // Links to TaskQueue task
|
||||
parallel: boolean; // Can run in parallel with sibling tasks
|
||||
completionPhrase: string; // Unique phrase for completion detection
|
||||
timeoutMs: number;
|
||||
startedAt: number | null;
|
||||
completedAt: number | null;
|
||||
error: string | null;
|
||||
retries: number;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Team Strategy
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export type TeamStrategy =
|
||||
| { type: 'single' } // One session handles all
|
||||
| { type: 'parallel'; maxSessions: number } // Multiple sessions
|
||||
| { type: 'team'; config: TeamSetup } // Agent teams
|
||||
|
||||
export interface TeamSetup {
|
||||
leadPrompt: string;
|
||||
suggestedTeammates: string[]; // Role descriptions
|
||||
maxTeammates: number;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Verification
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export interface VerificationResult {
|
||||
passed: boolean;
|
||||
checks: VerificationCheck[];
|
||||
summary: string;
|
||||
suggestions: string[]; // Recovery hints for replanning
|
||||
}
|
||||
|
||||
export interface VerificationCheck {
|
||||
type: 'test_command' | 'ai_review' | 'file_check';
|
||||
description: string;
|
||||
passed: boolean;
|
||||
output?: string;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Configuration
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export interface OrchestratorConfig {
|
||||
plannerModel: string; // Default: 'opus'
|
||||
researchEnabled: boolean; // Default: true
|
||||
autoApprove: boolean; // Default: false
|
||||
maxPhaseRetries: number; // Default: 3
|
||||
phaseTimeoutMs: number; // Default: 1800000 (30min)
|
||||
enableTeamAgents: boolean; // Default: true
|
||||
maxParallelSessions: number; // Default: 3
|
||||
verificationMode: 'strict' | 'moderate' | 'lenient';
|
||||
compactBetweenPhases: boolean; // Default: true
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
// Persistence (saved to ~/.codeman/state.json)
|
||||
// ═══════════════════════════════════════════════════════════════
|
||||
|
||||
export interface OrchestratorPersistState {
|
||||
state: OrchestratorState;
|
||||
plan: OrchestratorPlan | null;
|
||||
currentPhaseIndex: number;
|
||||
startedAt: number | null;
|
||||
completedAt: number | null;
|
||||
config: OrchestratorConfig;
|
||||
stats: OrchestratorStats;
|
||||
}
|
||||
|
||||
export interface OrchestratorStats {
|
||||
phasesCompleted: number;
|
||||
phasesFailed: number;
|
||||
totalTasksCompleted: number;
|
||||
totalTasksFailed: number;
|
||||
totalDurationMs: number;
|
||||
replanCount: number;
|
||||
}
|
||||
```
|
||||
|
||||
## New Files (Implementation Order)
|
||||
|
||||
### Step 1: `src/types/orchestrator.ts` — Type definitions
|
||||
All interfaces above. No dependencies. ~120 lines.
|
||||
|
||||
### Step 2: `src/orchestrator-planner.ts` — Plan generation + phase grouping
|
||||
~300 lines. Wraps existing PlanOrchestrator.
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* @fileoverview Orchestrator plan generation — converts goals into phased plans.
|
||||
*
|
||||
* Uses PlanOrchestrator for AI plan generation, then groups PlanItems into
|
||||
* sequential phases with team strategies and verification criteria.
|
||||
*
|
||||
* @module orchestrator-planner
|
||||
*/
|
||||
|
||||
export class OrchestratorPlanner {
|
||||
constructor(mux: TerminalMultiplexer, workingDir: string, config: OrchestratorConfig);
|
||||
|
||||
/** Generate plan from goal. Uses PlanOrchestrator internally. */
|
||||
async generatePlan(goal: string, onProgress?: ProgressCallback): Promise<OrchestratorPlan>;
|
||||
|
||||
/** Cancel in-progress plan generation. */
|
||||
async cancel(): Promise<void>;
|
||||
|
||||
// Internal
|
||||
private groupIntoPhases(items: PlanItem[], goal: string): OrchestratorPhase[];
|
||||
private assignTeamStrategies(phases: OrchestratorPhase[]): void;
|
||||
private generateCompletionPhrases(plan: OrchestratorPlan): void;
|
||||
}
|
||||
```
|
||||
|
||||
**Phase grouping algorithm:**
|
||||
1. Topological sort by `PlanItem.dependencies`
|
||||
2. Group into dependency layers (Kahn's algorithm)
|
||||
3. Within each layer, sub-group by `tddPhase` (setup → test → impl → verify → review)
|
||||
4. Merge adjacent small phases (< 2 tasks) if they share the same tddPhase
|
||||
5. Assign team strategies:
|
||||
- 1-2 tasks → `{ type: 'single' }`
|
||||
- 3+ independent tasks → `{ type: 'parallel', maxSessions: Math.min(taskCount, config.maxParallelSessions) }`
|
||||
- 4+ tasks with high complexity → `{ type: 'team', config: { ... } }`
|
||||
6. Generate unique completion phrases per task: `ORCH_P{phaseOrder}_T{taskIndex}`
|
||||
|
||||
### Step 3: `src/orchestrator-verifier.ts` — Phase verification
|
||||
~200 lines.
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* @fileoverview Orchestrator phase verification.
|
||||
*
|
||||
* Runs verification checks after each phase completes:
|
||||
* test commands, AI review, and file existence checks.
|
||||
*
|
||||
* @module orchestrator-verifier
|
||||
*/
|
||||
|
||||
export class OrchestratorVerifier {
|
||||
constructor(config: OrchestratorConfig);
|
||||
|
||||
/** Run all verification checks for a completed phase. */
|
||||
async verifyPhase(
|
||||
phase: OrchestratorPhase,
|
||||
session: Session,
|
||||
mode: 'strict' | 'moderate' | 'lenient'
|
||||
): Promise<VerificationResult>;
|
||||
|
||||
// Verification strategies
|
||||
private async runTestCommands(commands: string[], session: Session): Promise<VerificationCheck[]>;
|
||||
private async aiReview(phase: OrchestratorPhase, session: Session): Promise<VerificationCheck>;
|
||||
}
|
||||
```
|
||||
|
||||
**Verification modes:**
|
||||
- `strict`: ALL test commands must pass AND AI review must approve
|
||||
- `moderate`: Test commands must pass, AI review is advisory
|
||||
- `lenient`: At least one test command passes, AI review skipped
|
||||
|
||||
**AI review prompt (sent as a task to the session):**
|
||||
```
|
||||
Review Phase "{phase.name}" completion. Check:
|
||||
1. Expected functionality works
|
||||
2. No obvious regressions
|
||||
3. Code quality is acceptable
|
||||
|
||||
Criteria: {phase.verificationCriteria.join('\n')}
|
||||
|
||||
If ALL criteria are met, respond: ORCH_VERIFY_PASS
|
||||
If ANY criteria fail, respond: ORCH_VERIFY_FAIL and explain what failed.
|
||||
```
|
||||
|
||||
### Step 4: `src/orchestrator-loop.ts` — Core state machine
|
||||
~500 lines. Main orchestrator engine.
|
||||
|
||||
```typescript
|
||||
/**
|
||||
* @fileoverview Orchestrator Loop — phased plan execution with team agents.
|
||||
*
|
||||
* State machine that generates plans from user goals, executes them
|
||||
* phase-by-phase with verification gates, and adapts on failure.
|
||||
*
|
||||
* @module orchestrator-loop
|
||||
*/
|
||||
|
||||
export interface OrchestratorLoopEvents {
|
||||
stateChanged: (state: OrchestratorState, prevState: OrchestratorState) => void;
|
||||
planReady: (plan: OrchestratorPlan) => void;
|
||||
phaseStarted: (phase: OrchestratorPhase) => void;
|
||||
phaseCompleted: (phase: OrchestratorPhase) => void;
|
||||
phaseFailed: (phase: OrchestratorPhase, reason: string) => void;
|
||||
taskAssigned: (task: OrchestratorTask, sessionId: string) => void;
|
||||
taskCompleted: (task: OrchestratorTask) => void;
|
||||
taskFailed: (task: OrchestratorTask, error: string) => void;
|
||||
verificationResult: (phase: OrchestratorPhase, result: VerificationResult) => void;
|
||||
completed: (stats: OrchestratorStats) => void;
|
||||
error: (error: Error) => void;
|
||||
}
|
||||
|
||||
export class OrchestratorLoop extends EventEmitter {
|
||||
private state: OrchestratorState = 'idle';
|
||||
private plan: OrchestratorPlan | null = null;
|
||||
private currentPhaseIndex = 0;
|
||||
private config: OrchestratorConfig;
|
||||
private planner: OrchestratorPlanner;
|
||||
private verifier: OrchestratorVerifier;
|
||||
private sessionManager: SessionManager;
|
||||
private taskQueue: TaskQueue;
|
||||
private store: StateStore;
|
||||
private stats: OrchestratorStats;
|
||||
private cleanup: CleanupManager;
|
||||
private pausedState: OrchestratorState | null = null; // State before pause
|
||||
|
||||
// ── Lifecycle ──────────────────────────────────────────────
|
||||
|
||||
constructor(mux: TerminalMultiplexer, workingDir: string, config?: Partial<OrchestratorConfig>);
|
||||
|
||||
/** Start orchestration with a goal. Transitions: idle → planning */
|
||||
async start(goal: string): Promise<void>;
|
||||
|
||||
/** Approve the generated plan. Transitions: approval → executing */
|
||||
async approve(): Promise<void>;
|
||||
|
||||
/** Reject plan with feedback. Transitions: approval → planning (regenerate) */
|
||||
async reject(feedback: string): Promise<void>;
|
||||
|
||||
/** Pause execution. Saves current state. */
|
||||
pause(): void;
|
||||
|
||||
/** Resume from pause. */
|
||||
resume(): void;
|
||||
|
||||
/** Stop everything and clean up. → idle */
|
||||
async stop(): Promise<void>;
|
||||
|
||||
/** Skip current phase. → executing (next phase) or completed */
|
||||
async skipPhase(phaseId: string): Promise<void>;
|
||||
|
||||
/** Retry a failed phase. → executing */
|
||||
async retryPhase(phaseId: string): Promise<void>;
|
||||
|
||||
// ── Getters ────────────────────────────────────────────────
|
||||
|
||||
getState(): OrchestratorState;
|
||||
getPlan(): OrchestratorPlan | null;
|
||||
getCurrentPhase(): OrchestratorPhase | null;
|
||||
getStats(): OrchestratorStats;
|
||||
getStatus(): OrchestratorPersistState;
|
||||
|
||||
// ── Internal: Phase Execution ──────────────────────────────
|
||||
|
||||
private async executeCurrentPhase(): Promise<void>;
|
||||
private async executePhase(phase: OrchestratorPhase): Promise<void>;
|
||||
private async assignPhaseTasks(phase: OrchestratorPhase): Promise<void>;
|
||||
private handleTaskCompleted(taskId: string): void;
|
||||
private handleTaskFailed(taskId: string, error: string): void;
|
||||
private async onPhaseTasksComplete(phase: OrchestratorPhase): Promise<void>;
|
||||
|
||||
// ── Internal: Verification ─────────────────────────────────
|
||||
|
||||
private async verifyCurrentPhase(): Promise<void>;
|
||||
private async handleVerificationResult(phase: OrchestratorPhase, result: VerificationResult): Promise<void>;
|
||||
|
||||
// ── Internal: Replanning ───────────────────────────────────
|
||||
|
||||
private async replanPhase(phase: OrchestratorPhase, failures: string[]): Promise<void>;
|
||||
|
||||
// ── Internal: State Machine ────────────────────────────────
|
||||
|
||||
private setState(newState: OrchestratorState): void;
|
||||
private advanceToNextPhase(): Promise<void>;
|
||||
private persist(): void;
|
||||
private restore(): void;
|
||||
}
|
||||
```
|
||||
|
||||
**Key execution flow in `executePhase()`:**
|
||||
1. Mark phase as `executing`, emit `phaseStarted`
|
||||
2. For each task in phase:
|
||||
- Create a `CreateTaskOptions` from `OrchestratorTask`
|
||||
- Add to `TaskQueue` with proper dependencies + completion phrase
|
||||
- Store the TaskQueue task ID in `OrchestratorTask.queueTaskId`
|
||||
3. Poll task completion (listen to TaskQueue events)
|
||||
4. When all tasks complete → call `onPhaseTasksComplete()`
|
||||
5. `onPhaseTasksComplete()` triggers verification
|
||||
|
||||
**How tasks get assigned to sessions:**
|
||||
The OrchestratorLoop does NOT manage session assignment directly. It adds tasks to the existing TaskQueue and starts a mini poll loop that assigns pending tasks to idle sessions — the same pattern as RalphLoop's `assignTasks()`. This reuses existing session management.
|
||||
|
||||
**Team agent flow:**
|
||||
For phases with `teamStrategy.type === 'team'`:
|
||||
- Start a single session with `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`
|
||||
- Instead of adding individual tasks to TaskQueue, send ONE comprehensive prompt to the lead
|
||||
- The prompt instructs the lead to create teammates and delegate
|
||||
- Monitor via TeamWatcher for team task completion + hook events
|
||||
- Phase completion is detected via the lead's completion phrase
|
||||
|
||||
### Step 5: `src/web/routes/orchestrator-routes.ts` — API endpoints
|
||||
~300 lines.
|
||||
|
||||
```
|
||||
POST /api/orchestrator/start — { goal, config? } → start planning
|
||||
POST /api/orchestrator/approve — approve generated plan
|
||||
POST /api/orchestrator/reject — { feedback } → reject + replan
|
||||
POST /api/orchestrator/pause — pause execution
|
||||
POST /api/orchestrator/resume — resume execution
|
||||
POST /api/orchestrator/stop — stop orchestration
|
||||
GET /api/orchestrator/status — full state + plan + stats
|
||||
GET /api/orchestrator/plan — plan details only
|
||||
POST /api/orchestrator/phase/:id/skip — skip a phase
|
||||
POST /api/orchestrator/phase/:id/retry — retry a failed phase
|
||||
```
|
||||
|
||||
Port dependency: `SessionPort & EventPort & RespawnPort & ConfigPort & InfraPort`
|
||||
|
||||
The route module receives the OrchestratorLoop instance via the InfraPort (added to `createRouteContext()`).
|
||||
|
||||
### Step 6: SSE Events — `src/web/sse-events.ts` additions
|
||||
|
||||
```typescript
|
||||
// ─── Orchestrator ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Orchestrator state machine transitioned. */
|
||||
export const OrchestratorStateChanged = 'orchestrator:stateChanged' as const;
|
||||
/** Orchestrator plan generated and ready for approval. */
|
||||
export const OrchestratorPlanReady = 'orchestrator:planReady' as const;
|
||||
/** Orchestrator phase started executing. */
|
||||
export const OrchestratorPhaseStarted = 'orchestrator:phaseStarted' as const;
|
||||
/** Orchestrator phase completed successfully. */
|
||||
export const OrchestratorPhaseCompleted = 'orchestrator:phaseCompleted' as const;
|
||||
/** Orchestrator phase failed. */
|
||||
export const OrchestratorPhaseFailed = 'orchestrator:phaseFailed' as const;
|
||||
/** Orchestrator verification result for a phase. */
|
||||
export const OrchestratorVerification = 'orchestrator:verification' as const;
|
||||
/** Orchestrator task assigned to session. */
|
||||
export const OrchestratorTaskAssigned = 'orchestrator:taskAssigned' as const;
|
||||
/** Orchestrator task completed. */
|
||||
export const OrchestratorTaskCompleted = 'orchestrator:taskCompleted' as const;
|
||||
/** Orchestrator task failed. */
|
||||
export const OrchestratorTaskFailed = 'orchestrator:taskFailed' as const;
|
||||
/** All phases completed successfully. */
|
||||
export const OrchestratorCompleted = 'orchestrator:completed' as const;
|
||||
/** Orchestrator error. */
|
||||
export const OrchestratorError = 'orchestrator:error' as const;
|
||||
```
|
||||
|
||||
11 new events. Add to `SseEvent` namespace object + mirror in `constants.js`.
|
||||
|
||||
### Step 7: State persistence — `src/state-store.ts` additions
|
||||
|
||||
Add to `AppState`:
|
||||
```typescript
|
||||
orchestrator?: OrchestratorPersistState;
|
||||
```
|
||||
|
||||
Add methods:
|
||||
```typescript
|
||||
getOrchestratorState(): OrchestratorPersistState | null;
|
||||
setOrchestratorState(state: Partial<OrchestratorPersistState>): void;
|
||||
clearOrchestratorState(): void;
|
||||
```
|
||||
|
||||
### Step 8: Server integration — `src/web/server.ts` modifications
|
||||
|
||||
1. Import `OrchestratorLoop` and `registerOrchestratorRoutes`
|
||||
2. Add `private orchestratorLoop: OrchestratorLoop` field
|
||||
3. Initialize in constructor (lazy — created on first start, not at boot)
|
||||
4. Add to `createRouteContext()` InfraPort: `orchestratorLoop: this.orchestratorLoop`
|
||||
5. Wire up OrchestratorLoop events → SSE broadcasts
|
||||
6. Register routes: `registerOrchestratorRoutes(this.app, ctx)`
|
||||
7. Clean up in `stop()`
|
||||
|
||||
### Step 9: `src/web/public/orchestrator-ui.js` — Frontend panel
|
||||
~500 lines. New frontend module.
|
||||
|
||||
**Load order**: After `panels-ui.js` (11), before `ralph-wizard.js` (13). So load order = 11.5.
|
||||
|
||||
**UI elements:**
|
||||
- Goal input form (text area + config toggles)
|
||||
- Plan approval view (phase list, task details, approve/reject buttons)
|
||||
- Execution dashboard (progress bar, phase cards, task status indicators)
|
||||
- Agent activity panel (session count, team status)
|
||||
- Controls (pause, resume, stop, skip phase, retry phase)
|
||||
|
||||
**SSE listeners:**
|
||||
- All 11 orchestrator events → update UI state
|
||||
- Reuses existing session/respawn/team event handlers for agent monitoring
|
||||
|
||||
### Step 10: `src/prompts/orchestrator.ts` — Prompt templates
|
||||
~200 lines.
|
||||
|
||||
Templates for:
|
||||
- Phase execution prompt (tells Claude what to do in this phase)
|
||||
- Team lead delegation prompt (instructs lead to create and coordinate teammates)
|
||||
- Verification prompt (asks Claude to verify phase output)
|
||||
- Replan prompt (gives failure context, asks for recovery steps)
|
||||
|
||||
### Step 11: Constants, schemas, route barrel updates
|
||||
|
||||
- `src/web/public/constants.js` — Add 11 SSE event mirrors
|
||||
- `src/web/schemas.ts` — Add Zod schemas for orchestrator API input validation
|
||||
- `src/web/routes/index.ts` — Export `registerOrchestratorRoutes`
|
||||
- `src/web/ports/infra-port.ts` — Add `orchestratorLoop` to InfraPort
|
||||
- `src/types/index.ts` — Export orchestrator types
|
||||
|
||||
## Existing File Modifications Summary
|
||||
|
||||
| File | Change | Lines |
|
||||
|------|--------|-------|
|
||||
| `src/types/index.ts` | Add orchestrator barrel export | +1 |
|
||||
| `src/web/sse-events.ts` | Add 11 orchestrator events + SseEvent entries | +30 |
|
||||
| `src/web/public/constants.js` | Mirror 11 SSE events | +15 |
|
||||
| `src/web/routes/index.ts` | Export registerOrchestratorRoutes | +1 |
|
||||
| `src/web/ports/infra-port.ts` | Add orchestratorLoop to InfraPort | +3 |
|
||||
| `src/web/server.ts` | Initialize OrchestratorLoop, wire events, register routes | +40 |
|
||||
| `src/web/schemas.ts` | Add orchestrator Zod schemas | +20 |
|
||||
| `src/state-store.ts` | Add orchestrator state persistence | +20 |
|
||||
| `src/web/public/app.js` | Add orchestrator SSE listeners + panel toggle | +30 |
|
||||
| `src/web/public/index.html` | Add orchestrator-ui.js script tag | +1 |
|
||||
|
||||
**Total new code**: ~2,300 lines across 6 new files
|
||||
**Total modifications**: ~160 lines across 10 existing files
|
||||
|
||||
## Implementation Execution Order
|
||||
|
||||
This is the actual build order — each step is a commit checkpoint:
|
||||
|
||||
1. **Types** — `src/types/orchestrator.ts` + barrel export. Zero risk, pure types.
|
||||
2. **SSE events** — Add all 11 events to both `sse-events.ts` and `constants.js`. Wire in SseEvent namespace.
|
||||
3. **State persistence** — Add orchestrator state to StateStore. Small, isolated change.
|
||||
4. **Schemas** — Add Zod validation schemas for API input.
|
||||
5. **Planner** — `src/orchestrator-planner.ts`. Can test in isolation.
|
||||
6. **Verifier** — `src/orchestrator-verifier.ts`. Can test in isolation.
|
||||
7. **Core loop** — `src/orchestrator-loop.ts`. The big one. Depends on planner + verifier.
|
||||
8. **Prompts** — `src/prompts/orchestrator.ts`. Templates used by core loop.
|
||||
9. **Port + routes** — `src/web/ports/infra-port.ts` update + `src/web/routes/orchestrator-routes.ts`.
|
||||
10. **Server integration** — Wire OrchestratorLoop into WebServer. Routes become live.
|
||||
11. **Frontend** — `src/web/public/orchestrator-ui.js` + app.js listeners + index.html script tag.
|
||||
12. **Tests** — `test/orchestrator-*.test.ts`.
|
||||
13. **Typecheck + lint** — Fix all issues, ensure CI passes.
|
||||
|
||||
## Edge Cases & Error Handling
|
||||
|
||||
- **Session limit reached**: Queue tasks and wait for sessions to free up (existing SessionManager handles this)
|
||||
- **All sessions crash during phase**: Mark phase as failed, attempt replan
|
||||
- **Verification flaky**: `moderate` mode allows test retries; `lenient` skips AI review
|
||||
- **Plan too large**: Cap at 10 phases, 50 total tasks. Warn user.
|
||||
- **Context overflow**: Auto-compact between phases. Respawn if needed (orchestrator state is external).
|
||||
- **User pauses mid-phase**: Pause task assignment, don't cancel running tasks. Resume picks up where it left off.
|
||||
- **Network/API errors during planning**: Retry plan generation up to 2 times, then fail with clear message.
|
||||
- **Orchestrator vs Ralph conflict**: Mutually exclusive. Starting orchestrator stops Ralph if running. Starting Ralph stops orchestrator.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
- **Unit tests**: `test/orchestrator-planner.test.ts` — phase grouping algorithm, team strategy assignment
|
||||
- **Unit tests**: `test/orchestrator-verifier.test.ts` — verification logic with mocked sessions
|
||||
- **Integration tests**: `test/orchestrator-loop.test.ts` — state machine transitions, task lifecycle
|
||||
- **Route tests**: `test/routes/orchestrator-routes.test.ts` — API validation, status responses
|
||||
|
||||
All tests use `MockSession` pattern from existing test infrastructure. No real tmux needed.
|
||||
@@ -0,0 +1,157 @@
|
||||
# Orchestrator Loop — Research Findings
|
||||
|
||||
> Research doc for the new "Orchestrator Loop" feature. Not for GitHub.
|
||||
|
||||
## What We're Building
|
||||
|
||||
A new autonomous loop variant — **Orchestrator Loop** — that takes high-level user tasks, decomposes them into a detailed plan using team agents, and executes the plan step-by-step with quality gates. Unlike Ralph Loop (which executes a flat task queue), the Orchestrator coordinates **planning, delegation, and verification** as a continuous cycle.
|
||||
|
||||
**Core idea**: User inputs a goal → Orchestrator creates a detailed plan → spins up team agents for parallel execution → validates each step → adapts the plan based on results → delivers polished output.
|
||||
|
||||
## Existing Infrastructure Analysis
|
||||
|
||||
### What We Can Reuse
|
||||
|
||||
#### 1. Ralph Loop (`src/ralph-loop.ts`)
|
||||
- **Pattern**: Poll loop with `start() → tick() → stop()` lifecycle
|
||||
- **Reusable**: Event-driven task assignment, session completion handling, timeout management
|
||||
- **Limitation**: Flat task queue — no concept of phases, dependencies between task groups, or adaptive replanning
|
||||
- **Key insight**: `assignTaskToSession()` uses `session.sendInput(task.prompt)` — simple prompt injection into PTY
|
||||
|
||||
#### 2. Task Queue (`src/task-queue.ts`) + Task (`src/task.ts`)
|
||||
- **Already has**: Priority ordering, dependency tracking between tasks, completion phrase detection
|
||||
- **Limitation**: No task *groups* or *phases*. Dependencies are task-to-task, not phase-to-phase
|
||||
- **Key insight**: Tasks support `completionPhrase` — a string the task watches for in output. This is how Ralph knows a task is done
|
||||
|
||||
#### 3. Plan Orchestrator (`src/plan-orchestrator.ts`)
|
||||
- **Already has**: 2-agent plan generation (Research Agent → Planner Agent), TDD-aware plan items with P0/P1/P2 priorities
|
||||
- **Output**: `PlanItem[]` with dependencies, verification criteria, TDD phases, complexity ratings
|
||||
- **Limitation**: Plan generation only — no execution. Plans are generated then sit in state/UI for human review
|
||||
- **Key insight**: Uses `Session` directly to run Claude subagent instances for research and planning. Returns structured JSON
|
||||
|
||||
#### 4. Team Agents (`src/team-watcher.ts`, `~/.claude/teams/`)
|
||||
- **Already has**: Team creation, member tracking, filesystem inbox messaging, task management via `~/.claude/tasks/{team-name}/`
|
||||
- **Limitation**: Codeman can only *observe* teams (TeamWatcher is read-only polling), not *create* or *orchestrate* them
|
||||
- **Key insight**: Teams are a Claude Code feature. Codeman monitors them but doesn't control them. We can't programmatically create teammates — Claude Code does that when you use `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`
|
||||
|
||||
#### 5. Respawn Controller (`src/respawn-controller.ts`)
|
||||
- **Already has**: Preset-based automation (ralph-todo, overnight-autonomous), circuit breaker, health scoring
|
||||
- **Key insight**: The `ralph-todo` preset (8s idle, 480min max) is designed for autonomous task execution. We'd need a new preset or make Orchestrator Loop set its own timing
|
||||
|
||||
#### 6. Session Auto-Ops (`src/session-auto-ops.ts`)
|
||||
- **Already has**: Auto-compact at token thresholds, auto-clear for context management
|
||||
- **Key insight**: Critical for long Orchestrator runs — prevents context overflow during multi-step execution
|
||||
|
||||
#### 7. Hooks (`src/hooks-config.ts`)
|
||||
- **Already has**: `idle_prompt`, `stop`, `teammate_idle`, `task_completed` hook events
|
||||
- **Key insight**: Hooks fire POST to `/api/hook-event` — this is how Codeman knows when Claude is idle, stopped, or completed a task. The Orchestrator Loop can listen to these same events
|
||||
|
||||
### What We Need to Build New
|
||||
|
||||
1. **Plan → Task decomposition**: Convert PlanOrchestrator output (PlanItem[]) into executable task groups with phase ordering
|
||||
2. **Multi-phase execution engine**: Execute plan phases sequentially, tasks within phases in parallel
|
||||
3. **Verification gates**: After each phase, run verification (test commands, AI review) before proceeding
|
||||
4. **Adaptive replanning**: When a task fails or verification fails, generate a recovery plan
|
||||
5. **Team agent orchestration**: Leverage Claude Code's agent teams for parallel execution within phases
|
||||
6. **Progress tracking & UI**: Real-time dashboard showing plan progress, phase status, agent activity
|
||||
|
||||
## How Teams Actually Work (Important Constraint)
|
||||
|
||||
After deep research, here's the reality of agent teams:
|
||||
|
||||
```
|
||||
User starts session with CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1
|
||||
→ Claude Code creates a team-lead
|
||||
→ Team-lead spawns teammates (in-process threads)
|
||||
→ Teammates appear as subagents (detected by SubagentWatcher)
|
||||
→ Communication via ~/.claude/teams/{name}/inboxes/{member}.json
|
||||
→ Tasks tracked in ~/.claude/tasks/{team-name}/{N}.json
|
||||
```
|
||||
|
||||
**Codeman cannot programmatically create team members.** This is a Claude Code internal feature. However, Codeman CAN:
|
||||
- Start a session that has teams enabled
|
||||
- Send a prompt to the lead that instructs it to use agent teams
|
||||
- Monitor team activity via TeamWatcher
|
||||
- React to teammate_idle and task_completed hook events
|
||||
- Read team task status from the filesystem
|
||||
|
||||
**This means**: The Orchestrator Loop orchestrates at the *session prompt* level, not the *team member* level. We tell the lead what to do, and the lead decides how to use its team.
|
||||
|
||||
## Architecture Decision: Prompt-Level Orchestration
|
||||
|
||||
Given the team constraint, the Orchestrator Loop works by:
|
||||
|
||||
1. **Planning phase**: Use PlanOrchestrator to generate a detailed plan from user input
|
||||
2. **Execution phase**: Feed plan steps as prompts to sessions, one phase at a time
|
||||
3. **Verification phase**: After each phase, run verification prompts and check results
|
||||
4. **Adaptation phase**: If verification fails, generate recovery prompts
|
||||
|
||||
The "team agents" aspect works by:
|
||||
- Starting sessions with `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`
|
||||
- Crafting prompts that *instruct the lead to delegate* to teammates
|
||||
- Monitoring team activity to track parallel progress
|
||||
- The lead agent is smart enough to decompose work across its team
|
||||
|
||||
## Key Technical Findings
|
||||
|
||||
### Session Input Mechanics
|
||||
```typescript
|
||||
// From session.ts - how we send prompts
|
||||
await session.sendInput(task.prompt); // Uses writeViaMux() internally
|
||||
// writeViaMux() does: tmux send-keys -l "prompt text" + tmux send-keys Enter
|
||||
// CRITICAL: Single-line only! Multi-line breaks Ink rendering
|
||||
```
|
||||
|
||||
### Completion Detection Chain
|
||||
```
|
||||
PTY output → RalphTracker.processData() → completion phrase fuzzy match
|
||||
→ CompletionConfidence scoring (multi-signal: promise tag + todos + exit signal)
|
||||
→ If confident → emit 'completionDetected'
|
||||
→ RalphLoop listens → marks task complete → assigns next
|
||||
```
|
||||
|
||||
### How Plan Items Map to Tasks
|
||||
```typescript
|
||||
// PlanItem has:
|
||||
interface PlanItem {
|
||||
id: string; // "P0-001"
|
||||
content: string; // "Implement error handling for API endpoints"
|
||||
priority: 'P0' | 'P1' | 'P2';
|
||||
dependencies: string[]; // ["P0-000"] — other PlanItem IDs
|
||||
verificationCriteria: string;
|
||||
testCommand: string;
|
||||
tddPhase: 'setup' | 'test' | 'impl' | 'verify' | 'review';
|
||||
complexity: 'low' | 'medium' | 'high';
|
||||
}
|
||||
|
||||
// Task has:
|
||||
interface CreateTaskOptions {
|
||||
prompt: string;
|
||||
priority: number;
|
||||
dependencies: string[]; // Task IDs
|
||||
completionPhrase: string;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
// Natural mapping: PlanItem.content → Task.prompt
|
||||
// PlanItem.dependencies → Task.dependencies
|
||||
// PlanItem.priority → Task.priority (P0=100, P1=50, P2=10)
|
||||
// PlanItem.verificationCriteria → verification task prompt
|
||||
```
|
||||
|
||||
### Context Management for Long Runs
|
||||
- Auto-compact at ~110k tokens (configurable)
|
||||
- Auto-clear at ~140k tokens (configurable)
|
||||
- Respawn cycling: kill + restart session to reset context entirely
|
||||
- For Orchestrator: we want compact between phases, respawn between major milestones
|
||||
|
||||
## Risk Assessment
|
||||
|
||||
| Risk | Severity | Mitigation |
|
||||
|------|----------|------------|
|
||||
| Context overflow during complex phases | High | Auto-compact between tasks, respawn between phases |
|
||||
| Team agents not predictable | Medium | Orchestrate at session level, let Claude decide team delegation |
|
||||
| Plan too ambitious → infinite loop | High | Phase budgets (max attempts per phase), circuit breaker |
|
||||
| Verification too strict → blocks progress | Medium | Configurable strictness, human override via UI |
|
||||
| Single-line prompt limit | Medium | Use CLAUDE.md file for complex instructions, prompt references file |
|
||||
| Long planning phase delays execution | Low | Show plan for approval before execution |
|
||||
@@ -0,0 +1,319 @@
|
||||
# Security Architecture
|
||||
|
||||
This document describes Codeman's security model: how it decides who may reach
|
||||
the web UI, how requests are authenticated, how the file-serving and tmux layers
|
||||
are hardened, and the recommended ways to expose an instance safely.
|
||||
|
||||
Codeman spawns and drives Claude/OpenCode CLIs with
|
||||
`--dangerously-skip-permissions`. **Anyone who can reach an unauthenticated
|
||||
instance can run arbitrary commands as your user.** The defaults below are chosen
|
||||
so that a fresh install is safe on the machine it runs on, while remote access is
|
||||
an explicit, guided opt‑in.
|
||||
|
||||
> TL;DR — Codeman binds **loopback only (`127.0.0.1`) by default**, so out of the
|
||||
> box it is reachable only from the same machine and needs no password. To reach
|
||||
> it from elsewhere, either put it behind an **authenticated tunnel**
|
||||
> (`tailscale serve` / `cloudflared`) **or** bind a wider host **and set
|
||||
> `CODEMAN_PASSWORD`**. If you bind a non‑loopback host with no password, Codeman
|
||||
> still starts but prints a **loud warning** telling you how to secure it.
|
||||
|
||||
---
|
||||
|
||||
## 1. Network binding model
|
||||
|
||||
| Setting | Default | Source |
|
||||
|---------|---------|--------|
|
||||
| Bind host | `127.0.0.1` (loopback) | `--host` / `CODEMAN_HOST` → `WebServer` ctor |
|
||||
| Port | `3000` | `--port` / `CODEMAN_PORT` |
|
||||
| TLS | off (`--https` to enable) | `--https` |
|
||||
|
||||
### Bind host classification
|
||||
|
||||
`isLoopbackBindHost()` (`src/web/network-auth-policy.ts`) decides whether a bind
|
||||
host is loopback-only. It returns `true` for:
|
||||
|
||||
- `localhost`
|
||||
- any IPv4 in `127.0.0.0/8` (e.g. `127.0.0.1`, `127.42.0.9`)
|
||||
- IPv6 loopback `::1` (bracketed `[::1]` and the long form `0:0:0:0:0:0:0:1`)
|
||||
- IPv4‑mapped loopback `::ffff:127.*`
|
||||
|
||||
It returns `false` for `0.0.0.0`, `::` (all interfaces), LAN IPs, and hostnames.
|
||||
The classification is **fail‑safe in the dangerous direction**: any host that is
|
||||
not provably loopback is treated as non‑loopback (it never mistakes `0.0.0.0`
|
||||
for loopback). Shorthand forms like `127.1` or integer/octal IPs classify as
|
||||
non‑loopback (you'll get a warning, not a silent wide‑open bind) — use
|
||||
`127.0.0.1` for an unambiguous loopback bind.
|
||||
|
||||
### Startup policy (the "warn, don't block" rule)
|
||||
|
||||
At `WebServer.start()`:
|
||||
|
||||
| Bind host | `CODEMAN_PASSWORD` | Behavior |
|
||||
|-----------|--------------------|----------|
|
||||
| loopback (default) | unset | **Start.** Safe — reachable only from this machine. |
|
||||
| loopback | set | **Start.** Auth required even locally. |
|
||||
| non‑loopback | set | **Start.** Auth protects the open bind. |
|
||||
| non‑loopback | unset | **Start + LOUD warning** listing how to secure it. |
|
||||
| non‑loopback | unset, `--allow-unauthenticated-network` | **Start + terse acknowledged note.** |
|
||||
|
||||
> History: an earlier iteration (unreleased COD‑29) *refused to start* on a
|
||||
> non‑loopback bind without a password. That surprised setups that "just worked"
|
||||
> before, so **0.9.0 changed it to start‑and‑warn**. Loopback is still the safe
|
||||
> default; the warning (with three concrete fixes) replaces the hard failure.
|
||||
|
||||
The warning points at three ways to secure the instance:
|
||||
|
||||
1. `CODEMAN_PASSWORD=<password>` — turns on HTTP Basic auth (see §2).
|
||||
2. `--host 127.0.0.1` + an authenticated tunnel (`cloudflared` / `tailscale serve`).
|
||||
3. `--allow-unauthenticated-network` / `CODEMAN_ALLOW_UNAUTHENTICATED_NETWORK=1`
|
||||
— explicitly accept the risk (downgrades the warning to a one‑line note). This
|
||||
flag is **only** an acknowledgement; it does not change reachability.
|
||||
|
||||
`CODEMAN_API_URL` (used by hooks/child processes) is always derived as a loopback
|
||||
address (`0.0.0.0`/`localhost`/`::1` → `127.0.0.1`) so in‑process hooks reach the
|
||||
server over loopback regardless of the public bind.
|
||||
|
||||
---
|
||||
|
||||
## 2. Authentication
|
||||
|
||||
Auth is **optional** and controlled by env vars captured at startup:
|
||||
|
||||
- `CODEMAN_USERNAME` (default `admin` when only a password is set)
|
||||
- `CODEMAN_PASSWORD`
|
||||
|
||||
When `CODEMAN_PASSWORD` is unset, no auth is enforced — which is why the default
|
||||
loopback bind matters. The auth pipeline (`src/web/middleware/auth.ts`,
|
||||
`onRequest` hook) runs in this order:
|
||||
|
||||
1. **Localhost‑only exemptions** (always first): `POST /api/hook-event` and the QR
|
||||
`/q/` short‑code path are exempt when `req.ip` is loopback (see §3).
|
||||
2. **Session cookie** check — a valid `codeman_session` cookie short‑circuits to
|
||||
allow.
|
||||
3. **HTTP Basic** check — correct credentials short‑circuit to allow and clear
|
||||
that IP's failure counter.
|
||||
4. **Rate‑limit gate** — if neither cookie nor credentials passed and the IP is
|
||||
locked out, return `429` with a `Retry-After` header.
|
||||
5. Otherwise return `401`, incrementing the IP's failure counter.
|
||||
|
||||
### Session cookies
|
||||
|
||||
On successful Basic auth the server issues `codeman_session`, an opaque
|
||||
server‑side token (`randomBytes(32)`), valid 24h with auto‑extend and device
|
||||
context for the audit log. Tokens are **not** client‑signed — they're validated
|
||||
by presence in a server‑side map, so they cannot be forged offline.
|
||||
|
||||
### Rate limiting / lockout recovery
|
||||
|
||||
Failed auth is tracked **per IP**: 10 failures → `429`, with a 15‑minute decay.
|
||||
The QR path has its own separate limiter.
|
||||
|
||||
The lockout check sits **after** the cookie/credential checks (step 4, not first).
|
||||
This is deliberate: a user with a **valid cookie or correct password recovers
|
||||
immediately** even while an attacker is hammering the same IP — important because
|
||||
all traffic through a tunnel shares one source IP (loopback). Wrong credentials
|
||||
are still counted and still hit the `429` at the threshold, so brute‑force
|
||||
protection is unchanged.
|
||||
|
||||
---
|
||||
|
||||
## 3. Request‑origin trust & the tunnel caveat
|
||||
|
||||
`req.ip` is derived from the **TCP socket only** — Fastify runs with
|
||||
`trustProxy: false`, so `X-Forwarded-For` / `X-Real-IP` / `Forwarded` are
|
||||
**ignored**. A remote client cannot forge `req.ip` to `127.0.0.1`.
|
||||
|
||||
**However**, a reverse tunnel that connects to the server over loopback (e.g.
|
||||
`cloudflared --url http://localhost:3000`) makes **every tunneled request arrive
|
||||
with `req.ip = 127.0.0.1`**. The localhost‑only exemptions then treat those
|
||||
requests as local:
|
||||
|
||||
- `POST /api/hook-event` — auth‑exempt for loopback. Bounded impact: it is
|
||||
`HookEventSchema`‑validated and requires a valid in‑memory `sessionId`; it can
|
||||
drive respawn signals, SSE broadcasts, push notifications, and transcript
|
||||
watching — **not** arbitrary terminal input or file reads. It is a
|
||||
session‑disruption / notification‑spoofing surface, not RCE.
|
||||
- QR `/q/` — still protected by its own short‑code brute‑force limiter
|
||||
(10 failures / 60s against a 62⁶ space).
|
||||
|
||||
**Mitigation:** set `CODEMAN_PASSWORD` whenever a loopback‑connecting tunnel is
|
||||
up (it does not gate the hook‑event exemption, but it gates everything else and
|
||||
is the documented practice). Prefer `tailscale serve` (below), which authenticates
|
||||
at the tailnet layer so untrusted clients never reach the loopback port at all.
|
||||
A future hardening could gate the hook‑event exemption on a shared secret while a
|
||||
tunnel is active.
|
||||
|
||||
---
|
||||
|
||||
## 4. Recommended remote‑access setups
|
||||
|
||||
Ordered most‑to‑least recommended:
|
||||
|
||||
### A. Tailscale serve (recommended)
|
||||
|
||||
Bind loopback, let Tailscale front it on your tailnet with a real cert:
|
||||
|
||||
```bash
|
||||
codeman web --https # binds 127.0.0.1:3000
|
||||
tailscale serve --bg https / http://127.0.0.1:3000
|
||||
```
|
||||
|
||||
Only devices on your tailnet can reach it; Tailscale handles identity. No app
|
||||
password and no `0.0.0.0` bind required. (This is the maintainer's production
|
||||
setup.)
|
||||
|
||||
### B. Authenticated cloudflared tunnel + password
|
||||
|
||||
```bash
|
||||
export CODEMAN_PASSWORD=<password>
|
||||
codeman web --https
|
||||
cloudflared tunnel --url https://localhost:3000
|
||||
```
|
||||
|
||||
Always set `CODEMAN_PASSWORD` here — the tunnel connects over loopback, so the
|
||||
hook‑event exemption (§3) would otherwise be reachable from the public URL.
|
||||
|
||||
### C. Direct LAN bind + password
|
||||
|
||||
```bash
|
||||
export CODEMAN_PASSWORD=<password>
|
||||
codeman web --https --host 0.0.0.0
|
||||
```
|
||||
|
||||
Exposes the port on all interfaces; the password is the only thing protecting it.
|
||||
|
||||
### Avoid
|
||||
|
||||
`--host 0.0.0.0` **without** a password. Codeman will start (and warn), but
|
||||
anyone on the network can control your Claude sessions. Never re‑expose `0.0.0.0`
|
||||
without a password.
|
||||
|
||||
---
|
||||
|
||||
## 5. File‑serving hardening
|
||||
|
||||
Three routes serve workspace files; all require a valid `sessionId` and run the
|
||||
shared path validator `validateSessionFilePath()` (`src/web/route-helpers.ts`):
|
||||
it `realpath`s the target **before** the boundary check and rejects anything that
|
||||
escapes the session working directory (`..`, absolute paths, and symlinks that
|
||||
resolve outside). The realpath‑before‑check ordering closes the validation‑time
|
||||
TOCTOU window.
|
||||
|
||||
| Route | Cap | Notes |
|
||||
|-------|-----|-------|
|
||||
| `file-content` | 10 MB | text preview |
|
||||
| `file-raw` | 50 MB | inline MIME map; **`X-Content-Type-Options: nosniff` on all responses** |
|
||||
| `POST /api/download` | 50 MB | forced `attachment`; sensitive‑path blocklist |
|
||||
|
||||
### SVG / content‑type XSS
|
||||
|
||||
A workspace `.svg` served inline as `image/svg+xml` is a stored‑XSS vector (SVG
|
||||
can carry `<script>`, same‑origin = full session control). `file-raw` therefore
|
||||
serves `.svg` as `application/octet-stream` + `Content-Disposition: attachment` +
|
||||
`nosniff`. With global `nosniff` + CSP `default-src 'self'`, other text types
|
||||
(`.html`, `.xml`, …) that fall through to `octet-stream` are not rendered as HTML
|
||||
either. Trusted QR/welcome SVGs are injected from API JSON (`innerHTML`), not via
|
||||
`file-raw`, so they are unaffected.
|
||||
|
||||
### Download sensitive‑path blocklist
|
||||
|
||||
`/api/download` additionally refuses a blocklist of sensitive paths
|
||||
(`/etc/shadow`, `~/.ssh/`, `.env`, `*credentials*`, `.aws/credentials`, …). This
|
||||
is **defense‑in‑depth, not the primary boundary** — the realpath containment is
|
||||
the control.
|
||||
|
||||
### Known limitation — `workingDir` scope
|
||||
|
||||
The file‑route boundary is the session's `workingDir`, and `POST /api/sessions`
|
||||
currently accepts an arbitrary absolute `workingDir` (validated as "exists + is a
|
||||
directory"). A session created with `workingDir=/` can therefore read files
|
||||
across the filesystem within that boundary. This is **pre‑existing** across all
|
||||
file routes and not widened by the recent changes. Recommended follow‑up:
|
||||
constrain `workingDir` to an allowlist (e.g. under the cases dir / `$HOME`).
|
||||
|
||||
---
|
||||
|
||||
## 6. tmux launch hardening (COD‑31)
|
||||
|
||||
New sessions and respawns launch the tmux server/pane from a stable `/tmp`
|
||||
(`TMUX_LAUNCH_CWD`) and then `cd` into the real workspace **inside** the pane,
|
||||
against the live mount table:
|
||||
|
||||
```
|
||||
respawn-pane -k -c /tmp -t <session> bash -c "cd <workingDir> && <cmd>"
|
||||
```
|
||||
|
||||
This avoids a class of failures on FUSE/rclone‑mounted workspaces where a
|
||||
transient mount blip at launch poisons tmux's long‑lived cwd and crashes
|
||||
`new-session`. Safety properties:
|
||||
|
||||
- **Fail‑safe cwd:** the command is `cd "<dir>" && <cmd>` — if `cd` fails the CLI
|
||||
does **not** run in `/tmp`; the pane dies with a visible error instead.
|
||||
- **No injection:** `workingDir` passes `isValidWorkingDir` (absolute, rejects
|
||||
`;&|$\`(){}<>'"` and newlines and `..`) and `isValidPath`, and is double‑quoted
|
||||
in the pane command. Paths with spaces work; metacharacters are rejected before
|
||||
reaching the shell.
|
||||
- It does not change which tmux socket is targeted, so instance isolation (§8) is
|
||||
preserved.
|
||||
|
||||
---
|
||||
|
||||
## 7. Supply‑chain & build‑asset hardening (COD‑28)
|
||||
|
||||
- **Dependency advisories:** security‑sensitive ranges are bumped to patched
|
||||
versions, and `overrides` force patched transitive deps (`picomatch`,
|
||||
`basic-ftp`, `fast-uri`, `flatted`). `test/dependency-security.test.ts` asserts
|
||||
these stay patched in the lockfile.
|
||||
- **Lockfile integrity:** `npm run check:lockfile` (CI on every push/PR) fails on
|
||||
drift between `package.json` and `package-lock.json`. All lockfile entries
|
||||
resolve to `registry.npmjs.org` with `sha512` integrity hashes.
|
||||
- **Public‑asset checker:** `npm run check:public-assets`
|
||||
(`scripts/check-public-assets.mjs`) scans `src/web/public/**` for literal NUL
|
||||
bytes and runs `node --check` on every `.js` file (syntax validation), plus a
|
||||
Prettier pass on maintained files. It uses `execFileSync` with argv arrays (no
|
||||
shell), so filenames/content cannot inject commands; `node --check` only parses,
|
||||
never executes. Large hand‑formatted/generated assets (`app.js`, the gesture
|
||||
bundle, vendored libs) are `.prettierignore`d for the style pass, but the NUL +
|
||||
syntax checks still cover them.
|
||||
|
||||
---
|
||||
|
||||
## 8. Multi‑instance isolation
|
||||
|
||||
The tmux socket (`tmux -L codeman[-<instance>]`) and data dir
|
||||
(`~/.codeman[-<instance>]`) are **process‑wide and shared by every Codeman on the
|
||||
machine**, derived from `CODEMAN_INSTANCE` (`src/config/instance.ts`). A second
|
||||
instance on the **same** socket discovers and attaches PTYs to the first
|
||||
instance's live sessions. To run instances side by side, give each a distinct
|
||||
`CODEMAN_INSTANCE` (scopes both dir + socket), or set `CODEMAN_TMUX_SOCKET` +
|
||||
`CODEMAN_DATA_DIR` individually. `CODEMAN_INSTANCE` defaults to empty = the
|
||||
production layout (`~/.codeman`, `-L codeman`, port 3000).
|
||||
|
||||
---
|
||||
|
||||
## 9. Transport security headers
|
||||
|
||||
`registerSecurityHeaders` applies on every response:
|
||||
|
||||
- `Content-Security-Policy: default-src 'self'` (widened only for `/gesture/`
|
||||
assets when `CODEMAN_GESTURE=1`, to load self‑hosted MediaPipe)
|
||||
- `X-Content-Type-Options: nosniff`
|
||||
- `X-Frame-Options`
|
||||
- `Strict-Transport-Security` when served over HTTPS
|
||||
- CORS restricted to localhost origins
|
||||
|
||||
---
|
||||
|
||||
## 10. Quick reference
|
||||
|
||||
| Env / flag | Effect |
|
||||
|------------|--------|
|
||||
| `CODEMAN_PASSWORD` (+ `CODEMAN_USERNAME`) | Enable HTTP Basic auth |
|
||||
| `--host` / `CODEMAN_HOST` | Bind host (default `127.0.0.1`) |
|
||||
| `--allow-unauthenticated-network` / `CODEMAN_ALLOW_UNAUTHENTICATED_NETWORK` | Acknowledge an unauthenticated non‑loopback bind (downgrades the warning) |
|
||||
| `--https` | Enable TLS (adds HSTS) |
|
||||
| `CODEMAN_INSTANCE` | Scope tmux socket + data dir for isolation |
|
||||
| `CODEMAN_GESTURE=1` | Make the gesture overlay available (widens CSP) |
|
||||
|
||||
**Audit log:** session lifecycle and server start are recorded in
|
||||
`~/.codeman/session-lifecycle.jsonl`.
|
||||
@@ -7,8 +7,10 @@
|
||||
# Environment variables:
|
||||
# CODEMAN_NONINTERACTIVE=1 - Skip all prompts (for CI/automation)
|
||||
# CODEMAN_INSTALL_DIR - Custom install directory (default: ~/.codeman/app)
|
||||
# CODEMAN_SKIP_SYSTEMD=1 - Skip systemd service setup prompt
|
||||
# CODEMAN_SKIP_SYSTEMD=1 - Skip systemd/launchd service setup prompt
|
||||
# CODEMAN_NODE_VERSION - Node.js major version to install (default: 22)
|
||||
# CODEMAN_REPO_URL - Custom git repository URL (default: upstream Codeman)
|
||||
# CODEMAN_BRANCH - Git branch to install (default: master)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -17,7 +19,8 @@ set -euo pipefail
|
||||
# ============================================================================
|
||||
|
||||
INSTALL_DIR="${CODEMAN_INSTALL_DIR:-$HOME/.codeman/app}"
|
||||
REPO_URL="https://github.com/Ark0N/Codeman.git"
|
||||
REPO_URL="${CODEMAN_REPO_URL:-https://github.com/Ark0N/Codeman.git}"
|
||||
BRANCH="${CODEMAN_BRANCH:-master}"
|
||||
MIN_NODE_VERSION=18
|
||||
TARGET_NODE_VERSION="${CODEMAN_NODE_VERSION:-22}"
|
||||
NONINTERACTIVE="${CODEMAN_NONINTERACTIVE:-0}"
|
||||
@@ -350,8 +353,15 @@ ensure_sudo() {
|
||||
die "sudo is required but not installed. Please install packages manually or run as root."
|
||||
fi
|
||||
# Validate sudo access
|
||||
if ! sudo -v 2>/dev/null; then
|
||||
die "Failed to obtain sudo privileges."
|
||||
# When piped (curl | bash), stdin is the pipe — redirect from /dev/tty so sudo can prompt
|
||||
if [[ -e /dev/tty ]]; then
|
||||
if ! sudo -v 2>/dev/null < /dev/tty; then
|
||||
die "Failed to obtain sudo privileges."
|
||||
fi
|
||||
else
|
||||
if ! sudo -v 2>/dev/null; then
|
||||
die "Failed to obtain sudo privileges. Try running the script directly instead of piping."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -369,7 +379,12 @@ ensure_homebrew() {
|
||||
fi
|
||||
|
||||
info "Installing Homebrew first..."
|
||||
/bin/bash -c "$(download_to_stdout https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
||||
# When piped (curl | bash), stdin is the pipe — Homebrew needs TTY for sudo password prompt
|
||||
if [[ -e /dev/tty ]]; then
|
||||
/bin/bash -c "$(download_to_stdout https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" < /dev/tty
|
||||
else
|
||||
NONINTERACTIVE=1 /bin/bash -c "$(download_to_stdout https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
||||
fi
|
||||
|
||||
# Add Homebrew to PATH for Apple Silicon
|
||||
if [[ -f /opt/homebrew/bin/brew ]]; then
|
||||
@@ -784,9 +799,84 @@ setup_sc_alias() {
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Systemd Service Setup (Linux only)
|
||||
# Service Setup (Linux systemd / macOS launchd)
|
||||
# ============================================================================
|
||||
|
||||
setup_launchd_service() {
|
||||
local plist_label="com.codeman.web"
|
||||
local agent_dir="$HOME/Library/LaunchAgents"
|
||||
local agent_plist="$agent_dir/$plist_label.plist"
|
||||
local daemon_plist="/Library/LaunchDaemons/$plist_label.plist"
|
||||
|
||||
info "Setting up macOS LaunchAgent..."
|
||||
|
||||
# Remove any existing LaunchDaemon (system-level) to prevent duplicates.
|
||||
# We standardize on LaunchAgent (user-level) — it doesn't require sudo,
|
||||
# inherits the user's environment, and is the correct choice for user apps.
|
||||
if [[ -f "$daemon_plist" ]]; then
|
||||
warn "Found system-level LaunchDaemon at $daemon_plist — removing to prevent duplicate"
|
||||
sudo launchctl unload "$daemon_plist" 2>/dev/null || true
|
||||
sudo rm -f "$daemon_plist"
|
||||
success "Removed duplicate LaunchDaemon"
|
||||
fi
|
||||
|
||||
# Unload existing agent before overwriting
|
||||
if [[ -f "$agent_plist" ]]; then
|
||||
launchctl unload "$agent_plist" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
mkdir -p "$agent_dir"
|
||||
|
||||
# Build PATH: ensure /opt/homebrew/bin (Apple Silicon) and ~/.local/bin are included
|
||||
local svc_path="/opt/homebrew/bin:/usr/local/bin:$HOME/.local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
|
||||
|
||||
# Find node binary path
|
||||
local node_path
|
||||
node_path=$(command -v node)
|
||||
|
||||
cat > "$agent_plist" << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>$plist_label</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$node_path</string>
|
||||
<string>$INSTALL_DIR/dist/index.js</string>
|
||||
<string>web</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>PATH</key>
|
||||
<string>$svc_path</string>
|
||||
<key>HOME</key>
|
||||
<string>$HOME</string>
|
||||
<key>LANG</key>
|
||||
<string>en_US.UTF-8</string>
|
||||
</dict>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>$HOME</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>ThrottleInterval</key>
|
||||
<integer>10</integer>
|
||||
<key>StandardOutPath</key>
|
||||
<string>/tmp/codeman.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>/tmp/codeman.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
launchctl load "$agent_plist" 2>/dev/null || true
|
||||
|
||||
success "LaunchAgent installed and started"
|
||||
}
|
||||
|
||||
setup_systemd_service() {
|
||||
local service_dir="$HOME/.config/systemd/user"
|
||||
local service_file="$service_dir/codeman-web.service"
|
||||
@@ -1062,26 +1152,27 @@ main() {
|
||||
if [[ -d "$INSTALL_DIR/.git" ]]; then
|
||||
info "Existing installation found, updating..."
|
||||
cd "$INSTALL_DIR"
|
||||
git remote set-url origin "$REPO_URL" 2>/dev/null || true
|
||||
|
||||
# Check for local changes
|
||||
if ! git diff --quiet 2>/dev/null || ! git diff --staged --quiet 2>/dev/null; then
|
||||
warn "Local changes detected in $INSTALL_DIR"
|
||||
if prompt_yes_no "Discard local changes and update?" "n"; then
|
||||
git fetch --quiet origin
|
||||
git reset --hard origin/master --quiet
|
||||
git reset --hard "origin/$BRANCH" --quiet
|
||||
else
|
||||
info "Keeping existing installation, skipping update"
|
||||
fi
|
||||
else
|
||||
git fetch --quiet origin
|
||||
git reset --hard origin/master --quiet
|
||||
git reset --hard "origin/$BRANCH" --quiet
|
||||
fi
|
||||
else
|
||||
# Create parent directory
|
||||
mkdir -p "$(dirname "$INSTALL_DIR")"
|
||||
|
||||
# Clone repository (shallow for speed)
|
||||
git clone --quiet --depth 1 "$REPO_URL" "$INSTALL_DIR"
|
||||
git clone --quiet --depth 1 --branch "$BRANCH" "$REPO_URL" "$INSTALL_DIR"
|
||||
cd "$INSTALL_DIR"
|
||||
fi
|
||||
|
||||
@@ -1135,17 +1226,25 @@ main() {
|
||||
echo ""
|
||||
|
||||
local launch_choice=""
|
||||
local has_systemd=false
|
||||
local has_service=false
|
||||
local service_type=""
|
||||
|
||||
if [[ "$os" == "linux" ]] && [[ "$SKIP_SYSTEMD" != "1" ]] && command -v systemctl &>/dev/null; then
|
||||
has_systemd=true
|
||||
has_service=true
|
||||
service_type="systemd"
|
||||
elif [[ "$os" == "macos" ]] && [[ "$SKIP_SYSTEMD" != "1" ]]; then
|
||||
has_service=true
|
||||
service_type="launchd"
|
||||
fi
|
||||
|
||||
if [[ "$has_systemd" == "true" ]]; then
|
||||
if [[ "$has_service" == "true" ]]; then
|
||||
local service_label="systemd service"
|
||||
[[ "$service_type" == "launchd" ]] && service_label="LaunchAgent"
|
||||
|
||||
echo -e " ${BOLD}How would you like to run Codeman?${NC}"
|
||||
echo ""
|
||||
echo -e " ${CYAN}1)${NC} Run now in this terminal"
|
||||
echo -e " ${CYAN}2)${NC} Install as systemd service (auto-start on boot)"
|
||||
echo -e " ${CYAN}2)${NC} Install as $service_label (auto-start on boot)"
|
||||
echo -e " ${CYAN}3)${NC} Don't start — I'll run it later"
|
||||
echo ""
|
||||
|
||||
@@ -1162,7 +1261,7 @@ main() {
|
||||
done
|
||||
fi
|
||||
else
|
||||
# macOS or no systemd — only offer run now or skip
|
||||
# No service manager available — only offer run now or skip
|
||||
echo -e " ${BOLD}Would you like to start Codeman now?${NC}"
|
||||
echo ""
|
||||
echo -e " ${CYAN}1)${NC} Run now in this terminal"
|
||||
@@ -1188,12 +1287,16 @@ main() {
|
||||
|
||||
echo ""
|
||||
|
||||
# Handle systemd setup
|
||||
# Handle service setup
|
||||
if [[ "$launch_choice" == "2" ]]; then
|
||||
setup_systemd_service
|
||||
if [[ "$service_type" == "launchd" ]]; then
|
||||
setup_launchd_service
|
||||
else
|
||||
setup_systemd_service
|
||||
fi
|
||||
|
||||
# Offer tunnel service if cloudflared is available
|
||||
if check_cloudflared && [[ -f "$INSTALL_DIR/scripts/codeman-tunnel.service" ]]; then
|
||||
# Offer tunnel service if cloudflared is available (Linux only — systemd tunnel service)
|
||||
if [[ "$service_type" == "systemd" ]] && check_cloudflared && [[ -f "$INSTALL_DIR/scripts/codeman-tunnel.service" ]]; then
|
||||
echo ""
|
||||
if prompt_yes_no "Also set up Cloudflare tunnel service? (requires CODEMAN_PASSWORD)" "n"; then
|
||||
setup_tunnel_service
|
||||
@@ -1208,10 +1311,16 @@ main() {
|
||||
echo ""
|
||||
echo -e " ${BOLD}Manage the service:${NC}"
|
||||
echo ""
|
||||
echo -e " ${CYAN}systemctl --user stop codeman-web${NC} # Stop"
|
||||
echo -e " ${CYAN}systemctl --user restart codeman-web${NC} # Restart"
|
||||
echo -e " ${CYAN}systemctl --user status codeman-web${NC} # Check status"
|
||||
echo -e " ${CYAN}journalctl --user -u codeman-web -f${NC} # View logs"
|
||||
if [[ "$service_type" == "launchd" ]]; then
|
||||
echo -e " ${CYAN}launchctl unload ~/Library/LaunchAgents/com.codeman.web.plist${NC} # Stop"
|
||||
echo -e " ${CYAN}launchctl load ~/Library/LaunchAgents/com.codeman.web.plist${NC} # Start"
|
||||
echo -e " ${CYAN}tail -f /tmp/codeman.log${NC} # View logs"
|
||||
else
|
||||
echo -e " ${CYAN}systemctl --user stop codeman-web${NC} # Stop"
|
||||
echo -e " ${CYAN}systemctl --user restart codeman-web${NC} # Restart"
|
||||
echo -e " ${CYAN}systemctl --user status codeman-web${NC} # Check status"
|
||||
echo -e " ${CYAN}journalctl --user -u codeman-web -f${NC} # View logs"
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
@@ -1277,13 +1386,29 @@ update() {
|
||||
|
||||
info "Updating Codeman..."
|
||||
cd "$INSTALL_DIR"
|
||||
git remote set-url origin "$REPO_URL" 2>/dev/null || true
|
||||
git fetch --quiet origin
|
||||
git reset --hard origin/master --quiet
|
||||
git reset --hard "origin/$BRANCH" --quiet
|
||||
npm install --quiet --no-fund --no-audit 2>/dev/null || npm install --no-fund --no-audit
|
||||
npm run build --quiet 2>/dev/null || npm run build
|
||||
success "Updated to $(node -e "console.log(require('./package.json').version)")"
|
||||
echo ""
|
||||
echo -e " ${DIM}Restart codeman web to use the new version.${NC}"
|
||||
|
||||
# Auto-restart service if running, otherwise tell the user
|
||||
local agent_plist="$HOME/Library/LaunchAgents/com.codeman.web.plist"
|
||||
if systemctl --user is-active codeman-web.service &>/dev/null 2>&1; then
|
||||
info "Restarting codeman-web service..."
|
||||
systemctl --user restart codeman-web.service
|
||||
success "codeman-web service restarted"
|
||||
elif [[ -f "$agent_plist" ]]; then
|
||||
info "Restarting LaunchAgent..."
|
||||
launchctl unload "$agent_plist" 2>/dev/null || true
|
||||
launchctl load "$agent_plist" 2>/dev/null || true
|
||||
success "LaunchAgent restarted"
|
||||
else
|
||||
echo -e " ${DIM}Restart codeman web to use the new version:${NC}"
|
||||
echo -e " ${CYAN}pkill -f 'codeman.*web'; codeman web &${NC}"
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
@@ -1292,9 +1417,9 @@ uninstall() {
|
||||
info "Uninstalling Codeman..."
|
||||
echo ""
|
||||
|
||||
# Stop and remove systemd services
|
||||
# Stop and remove systemd services (Linux)
|
||||
for svc in codeman-web codeman-tunnel; do
|
||||
if systemctl --user is-active "${svc}.service" &>/dev/null; then
|
||||
if systemctl --user is-active "${svc}.service" &>/dev/null 2>&1; then
|
||||
info "Stopping ${svc} service..."
|
||||
systemctl --user stop "${svc}.service"
|
||||
fi
|
||||
@@ -1310,6 +1435,20 @@ uninstall() {
|
||||
done
|
||||
systemctl --user daemon-reload 2>/dev/null || true
|
||||
|
||||
# Stop and remove launchd services (macOS)
|
||||
local agent_plist="$HOME/Library/LaunchAgents/com.codeman.web.plist"
|
||||
local daemon_plist="/Library/LaunchDaemons/com.codeman.web.plist"
|
||||
if [[ -f "$agent_plist" ]]; then
|
||||
launchctl unload "$agent_plist" 2>/dev/null || true
|
||||
rm -f "$agent_plist"
|
||||
success "Removed LaunchAgent"
|
||||
fi
|
||||
if [[ -f "$daemon_plist" ]]; then
|
||||
sudo launchctl unload "$daemon_plist" 2>/dev/null || true
|
||||
sudo rm -f "$daemon_plist"
|
||||
success "Removed LaunchDaemon"
|
||||
fi
|
||||
|
||||
# Remove symlinks
|
||||
local symlink_dir="$HOME/.local/bin"
|
||||
if [[ -L "$symlink_dir/codeman" ]]; then
|
||||
@@ -1355,5 +1494,12 @@ uninstall() {
|
||||
case "${1:-}" in
|
||||
update) update ;;
|
||||
uninstall) uninstall ;;
|
||||
*) main "$@" ;;
|
||||
*)
|
||||
if [[ -z "${1:-}" && -d "$INSTALL_DIR/.git" ]]; then
|
||||
print_banner
|
||||
update
|
||||
else
|
||||
main "$@"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"$schema": "https://unpkg.com/knip@5/schema.json",
|
||||
"entry": [
|
||||
"scripts/*.mjs",
|
||||
"scripts/*.js",
|
||||
"scripts/watch-subagents.ts",
|
||||
"scripts/remotion/Root.tsx",
|
||||
"scripts/remotion/index.ts",
|
||||
"test/**/*.test.ts",
|
||||
"test/mobile/vitest.config.ts",
|
||||
"test/**/*.mjs"
|
||||
],
|
||||
"project": ["src/**/*.{ts,tsx}", "scripts/**/*.{ts,tsx,mjs,js}", "test/**/*.{ts,mjs}"],
|
||||
"ignoreExportsUsedInFile": true,
|
||||
"ignoreDependencies": ["@remotion/cli", "@remotion/transitions", "esbuild", "agent-browser"]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "aicodeman",
|
||||
"version": "0.3.1",
|
||||
"version": "0.9.2",
|
||||
"description": "The missing control plane for AI coding agents - run 20 autonomous agents with real-time monitoring and session persistence",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
@@ -11,21 +11,25 @@
|
||||
"scripts": {
|
||||
"postinstall": "node scripts/postinstall.js",
|
||||
"build": "node scripts/build.mjs",
|
||||
"start": "node dist/index.js",
|
||||
"build:gesture": "node scripts/build-gesture-bundle.mjs",
|
||||
"start": "NODE_COMPILE_CACHE=${HOME}/.codeman/compile-cache node dist/index.js",
|
||||
"dev": "tsx src/index.ts web",
|
||||
"web": "node dist/index.js web",
|
||||
"clean": "rm -rf dist",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test": "vitest run --config config/vitest.config.ts",
|
||||
"test:watch": "vitest --config config/vitest.config.ts",
|
||||
"test:coverage": "vitest run --config config/vitest.config.ts --coverage",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"lint": "eslint 'src/**/*.ts'",
|
||||
"lint:fix": "eslint 'src/**/*.ts' --fix",
|
||||
"format": "prettier --write 'src/**/*.ts'",
|
||||
"format:check": "prettier --check 'src/**/*.ts'",
|
||||
"lint": "eslint --config config/eslint.config.js 'src/**/*.ts'",
|
||||
"lint:fix": "eslint --config config/eslint.config.js 'src/**/*.ts' --fix",
|
||||
"format": "prettier --write 'src/**/*.ts' 'src/web/public/**/*.{js,css,html,json}'",
|
||||
"format:check": "prettier --check 'src/**/*.ts' 'src/web/public/**/*.{js,css,html,json}'",
|
||||
"check:public-assets": "node scripts/check-public-assets.mjs",
|
||||
"capture:subagents": "node scripts/capture-subagent-screenshots.mjs",
|
||||
"changeset": "changeset",
|
||||
"version-packages": "changeset version",
|
||||
"version-packages": "changeset version && npm install --package-lock-only && node scripts/check-lockfile-sync.mjs",
|
||||
"check:lockfile": "node scripts/check-lockfile-sync.mjs",
|
||||
"knip": "npx --yes knip@latest",
|
||||
"release": "changeset publish"
|
||||
},
|
||||
"workspaces": [
|
||||
@@ -50,33 +54,36 @@
|
||||
"dependencies": {
|
||||
"@fastify/compress": "^8.3.1",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/static": "^8.0.0",
|
||||
"@fastify/multipart": "^10.0.0",
|
||||
"@fastify/static": "^9.1.3",
|
||||
"@fastify/websocket": "^11.2.0",
|
||||
"@xterm/addon-fit": "^0.11.0",
|
||||
"@xterm/addon-unicode11": "^0.9.0",
|
||||
"@xterm/addon-webgl": "^0.19.0",
|
||||
"@xterm/xterm": "^6.0.0",
|
||||
"chalk": "^5.3.0",
|
||||
"chokidar": "^3.6.0",
|
||||
"commander": "^12.1.0",
|
||||
"fastify": "^5.1.0",
|
||||
"fastify": "^5.8.5",
|
||||
"node-pty": "^1.1.0",
|
||||
"qrcode": "^1.5.4",
|
||||
"uuid": "^10.0.0",
|
||||
"uuid": "^14.0.0",
|
||||
"web-push": "^3.6.7",
|
||||
"xterm": "^5.3.0",
|
||||
"xterm-addon-fit": "^0.8.0",
|
||||
"xterm-addon-unicode11": "^0.6.0",
|
||||
"xterm-addon-webgl": "^0.16.0",
|
||||
"zod": "^4.3.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@changesets/cli": "^2.29.8",
|
||||
"@eslint/js": "^9.0.0",
|
||||
"@remotion/cli": "4.0.429",
|
||||
"@remotion/transitions": "4.0.429",
|
||||
"@remotion/cli": "4.0.473",
|
||||
"@remotion/transitions": "4.0.473",
|
||||
"@types/node": "^20.19.33",
|
||||
"@types/pngjs": "^6.0.5",
|
||||
"@types/qrcode": "^1.5.6",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@types/web-push": "^3.6.4",
|
||||
"@vitest/coverage-v8": "^4.0.18",
|
||||
"@types/ws": "^8.18.1",
|
||||
"@vitest/coverage-v8": "^4.1.8",
|
||||
"agent-browser": "^0.6.0",
|
||||
"esbuild": "^0.27.3",
|
||||
"eslint": "^9.0.0",
|
||||
@@ -85,11 +92,29 @@
|
||||
"pngjs": "^7.0.0",
|
||||
"prettier": "^3.4.0",
|
||||
"puppeteer": "^24.36.0",
|
||||
"remotion": "4.0.429",
|
||||
"remotion": "4.0.473",
|
||||
"tsx": "^4.15.0",
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.0.0",
|
||||
"vitest": "^4.0.18"
|
||||
"vitest": "^4.1.8"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@remotion/compositor-linux-x64-gnu": "^4.0.432",
|
||||
"@rspack/binding-linux-x64-gnu": "^1.7.7"
|
||||
},
|
||||
"overrides": {
|
||||
"basic-ftp": "^5.3.1",
|
||||
"fast-uri": "^3.1.2",
|
||||
"flatted": "^3.4.2",
|
||||
"anymatch": {
|
||||
"picomatch": "^2.3.2"
|
||||
},
|
||||
"micromatch": {
|
||||
"picomatch": "^2.3.2"
|
||||
},
|
||||
"readdirp": {
|
||||
"picomatch": "^2.3.2"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
node_modules
|
||||
dist
|
||||
dist-codeman/
|
||||
*.local
|
||||
.vite
|
||||
.DS_Store
|
||||
@@ -0,0 +1,258 @@
|
||||
# gesture-proto
|
||||
|
||||
A Jarvis-style hand-tracking input layer for the Codeman dashboard. Your webcam
|
||||
sees your hands; you pinch-drag session tabs between columns. Runs entirely
|
||||
in-browser — **the camera feed never leaves the machine.**
|
||||
|
||||
> **This build is pinch-drag only.** The discrete gesture commands (halt-all /
|
||||
> approve / new-session) were built in Phase 4 but are **intentionally not wired
|
||||
> up** — an open palm while reaching to pinch kept charging the hold-to-halt. The
|
||||
> `GestureController` core still emits `command`/`haltProgress` for any future
|
||||
> consumer; the demo just no longer listens.
|
||||
|
||||
This is a standalone prototype (own folder, fake tabs) so the input *feel* can
|
||||
be validated on real hardware before integrating with Codeman. The canonical
|
||||
spec is **[`docs/BUILD_PLAN.md`](./docs/BUILD_PLAN.md)** — read it before
|
||||
continuing the build.
|
||||
|
||||
---
|
||||
|
||||
## Status: Phase 0–4 complete
|
||||
|
||||
| Phase | What | State |
|
||||
|-------|------|-------|
|
||||
| 0 | Vite+TS scaffold, mirrored webcam preview, start button | ✅ done |
|
||||
| 1 | MediaPipe `GestureRecognizer` (VIDEO mode, CDN model+wasm), rAF loop, debug skeleton overlay, HUD | ✅ done |
|
||||
| — | **Checkpoint: ≥25fps on real camera/lighting** | ✅ 60fps (iPhone 17 Pro / Continuity Camera) |
|
||||
| 2 | One-Euro–filtered cursor + pinch detection (hysteresis) | ✅ done |
|
||||
| 3 | Drag fake tabs across 3 columns (state machine) — go/no-go | ✅ done (two-handed) |
|
||||
| 4 | Discrete gesture→command bus (👍 approve · ✌️ new-session · ✋-hold halt-all) + toast | ✅ built, ⛔ **unwired in the demo** (pinch-only) |
|
||||
| 5 | Integrate `gesture/` into real Codeman (`src/codeman/entry.ts`) | ✅ **working at the desk** — see [Codeman integration](#codeman-integration) |
|
||||
|
||||
Also done beyond the original plan: **two-hand tracking** (drag two tabs at
|
||||
once) and a **live camera picker** (front-facing iPhone 17 Pro by default; the
|
||||
superwide / Desk View camera now works too — see the camera note below).
|
||||
|
||||
Also: **fullscreen mode** (toggle button — the board fills the display, so grab
|
||||
targets get big).
|
||||
|
||||
**Phase 5 is live** in the real Codeman dashboard. The Codeman-side detach +
|
||||
instance isolation + base gesture overlay are committed on `Ark0N/Codeman`
|
||||
branch `beta/session-detach` (open as **PR #103**) — including this session's
|
||||
gesture *improvements* (direct-detach, Run/Run Shell taps, self-hosted MediaPipe,
|
||||
cache-bust), ported onto the PR in commit `eea84db` (CI green). See
|
||||
[Codeman integration](#codeman-integration) below and the hand-off brief
|
||||
[`../docs/CODEMAN_DETACH_BRIEF.md`](../docs/CODEMAN_DETACH_BRIEF.md).
|
||||
|
||||
**Multi-monitor mode — built & validated at the desk (2026-06-08).** Approach
|
||||
**A+C** (see [`../docs/MULTIMONITOR_DESIGN.md`](../docs/MULTIMONITOR_DESIGN.md)):
|
||||
(A) gesture "detach" now pops a session into a re-grabbable **in-page floating
|
||||
panel** (an `<iframe src="/session/:id">`, not an OS window — so the hand keeps
|
||||
control); (C) `scripts/span-codeman.sh` runs Codeman in a **single window spanned
|
||||
across both monitors** (Brave-first; needs macOS "Displays have separate Spaces"
|
||||
OFF + re-login), launchable one-click from a new **multi-monitor button** in
|
||||
Codeman's header. Confirmed: a panel drags across the monitor seam. Still pending:
|
||||
`getScreenDetails` screen-snapping (C-snap) and the re-dock gesture.
|
||||
|
||||
---
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
cd gesture-proto
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Open the URL Vite prints (http://localhost:5173). `getUserMedia` requires a
|
||||
secure context — `localhost` qualifies, so the dev server is fine. Click
|
||||
**Start camera**, allow the camera, hold a hand up. You should see each hand's
|
||||
21-point skeleton, a cursor ring per hand (cyan = left, violet = right, green
|
||||
while pinching), and a live HUD.
|
||||
|
||||
**Choosing a camera:** after Start, the dropdown lists every video device. On a
|
||||
Mac with an iPhone nearby (Continuity Camera) you'll typically see the built-in
|
||||
FaceTime cam, the **iPhone Camera** (main/wide lens), and a **Desk View Camera**
|
||||
— the latter is driven by the iPhone's ultra-wide lens aimed down at the desk
|
||||
(the overhead angle). Switching is live; no restart needed. The browser can't
|
||||
select the ultra-wide lens directly, so Desk View is how you reach it. **The
|
||||
Desk View / superwide camera now works with no issues and tracking is confirmed
|
||||
on it** — earlier it was Safari-only and rendered stretched; that's resolved.
|
||||
|
||||
**Using it:** point at a tab (it highlights), **pinch** thumb+index to grab,
|
||||
move to another Screen column, release to drop. Both hands work at once. **⛶
|
||||
Fullscreen** makes the board fill the display (Esc exits). No open-hand gesture
|
||||
commands in this build — it's pinch-drag only (see the note up top).
|
||||
|
||||
Other scripts: `npm run build` (tsc + vite build), `npm run preview`.
|
||||
|
||||
### Developing on the Mac mini, running on the MacBook
|
||||
|
||||
The prototype is *run/tested* on the MacBook (better for sitting at the desk
|
||||
with the camera). To sync:
|
||||
|
||||
```bash
|
||||
git pull # on the MacBook
|
||||
cd gesture-proto && npm install && npm run dev
|
||||
```
|
||||
|
||||
No machine-specific state is committed (`node_modules/`, `dist/`, and
|
||||
`.claude/settings.local.json` are gitignored).
|
||||
|
||||
### Framerate note
|
||||
|
||||
The HUD fps turns **green at ≥ 25**, amber below. Confirmed **60fps** on the
|
||||
iPhone 17 Pro (Continuity Camera). If it's ever low, improve **even, frontal
|
||||
lighting on the hand zone** first — that matters more than the sensor.
|
||||
|
||||
---
|
||||
|
||||
## Codeman integration
|
||||
|
||||
Phase 5 ships a **separate consumer**, `src/codeman/entry.ts` (the demo +
|
||||
`main.ts` are untouched — they stay as a desk-testing harness). It imports the
|
||||
same unchanged `src/gesture/` core and binds its events to the **real** Codeman
|
||||
dashboard. Build it standalone (MediaPipe inlined) with:
|
||||
|
||||
```bash
|
||||
npm run build:codeman # esbuild → dist-codeman/gesture-codeman.js
|
||||
```
|
||||
|
||||
Codeman serves that bundle at `/gesture/gesture-codeman.js` and injects it into
|
||||
the dashboard **only when started with `CODEMAN_GESTURE=1`** (which also widens
|
||||
its CSP for WebAssembly). Deploy = copy the bundle into Codeman's
|
||||
`src/web/public/gesture/` and reload (static is served from disk).
|
||||
|
||||
**Gestures (all off one pinch, routed by what's under your fingertips):**
|
||||
- **Fullscreen camera** by default — mirrored, dimmed, full-viewport so you see
|
||||
your hands over the real tabs; the **⛶** button toggles a corner preview.
|
||||
- **Grab → in-page floating panel** *(pivoted 2026-06-08 — replaced the old
|
||||
OS-window detach)* — pinch a session tab, a ghost of it follows your hand, pull
|
||||
it out of the strip (>70px) and release → the session pops into a **re-grabbable
|
||||
in-page `.cg-float` panel** (an `<iframe src="/session/:id">`, 640×420) at the
|
||||
drop point. Pinch the panel again to move it anywhere. It stays inside the
|
||||
camera-owning page, so the hand keeps control (the old `window.app.detachSession`
|
||||
`window.open` was a one-way trip). A small twitch-and-release cancels.
|
||||
- **Run / Run Shell** — pinch over the **Run** (`#runBtn`) or **Run Shell**
|
||||
(`.btn-shell`) toolbar button and release in place to fire it; drifting too
|
||||
far first cancels the tap. The button list is `CLICK_SELECTOR` in `entry.ts`.
|
||||
|
||||
**Self-hosted MediaPipe.** The Codeman consumer loads the wasm runtime + the
|
||||
`gesture_recognizer.task` model **same-origin** from `/gesture/` (via
|
||||
`wasmBase`/`modelUrl` options), not the CDN — a browser content/ad blocker can
|
||||
otherwise block the CDN and startup fails with `failed: {"isTrusted":true}`.
|
||||
|
||||
**Multi-monitor button.** Codeman's header has a **multi-monitor button** (it
|
||||
replaced the notification bell) → `POST /api/system/span-displays` → spawns
|
||||
`scripts/span-codeman.sh`, opening a fresh browser `--app` window spanned across
|
||||
all displays so floating panels can cross the monitor seam (PR #103 `95b0035`).
|
||||
|
||||
**Caveats:** Codeman serves static with a 1-year **`immutable`** cache, so
|
||||
`server.ts` `cacheBustAssets()` appends `?v=<mtime>` to **every** same-origin
|
||||
`.js`/`.css` (and the gesture bundle), re-stat'd per render — without it an edited
|
||||
module stays cached until a hard refresh (PR #103 `b5ea711`). The old OS-window
|
||||
detach verb (kept only as a deliberate, non-default action) does `window.open`,
|
||||
which a pinch can get popup-blocked — allow popups once if you ever wire it back.
|
||||
|
||||
---
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
gesture-proto/
|
||||
├── index.html # demo page: video, tab board, HUD, controls
|
||||
├── docs/
|
||||
│ └── BUILD_PLAN.md # canonical spec: goals, algorithms, phases, tuning
|
||||
├── src/
|
||||
│ ├── main.ts # wires GestureController -> demo UI (board, HUD, camera, fullscreen)
|
||||
│ ├── gesture/
|
||||
│ │ ├── GestureController.ts # camera + recognizer + loop + per-hand state + event bus
|
||||
│ │ ├── OneEuroFilter.ts # per-axis cursor smoothing
|
||||
│ │ ├── pinch.ts # pinch distance + hysteresis detector
|
||||
│ │ ├── commands.ts # debounced gesture→command bus (Phase 4)
|
||||
│ │ ├── landmarks.ts # landmark indices, connections, helpers
|
||||
│ │ └── types.ts # event payload types + config (full API surface)
|
||||
│ ├── demo/
|
||||
│ │ ├── overlay.ts # draws the hand skeleton + per-hand cursors
|
||||
│ │ └── tabs.ts # the 3-column board: hit-testing + drag mechanics
|
||||
│ └── codeman/
|
||||
│ └── entry.ts # Phase 5 Codeman consumer (real tabs + Run/Run Shell);
|
||||
│ # esbuild-bundled by `npm run build:codeman`
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## `GestureController` API
|
||||
|
||||
Transport-agnostic: it owns the camera, recognizer, per-hand smoothing, pinch
|
||||
state machine, and gesture→command bus, and emits **coordinate-only** events. It
|
||||
knows nothing about tabs or the DOM — hit-testing lives in the consumer (the
|
||||
demo's `tabs.ts`, later Codeman). Integration (Phase 5) is just subscribing and
|
||||
calling Codeman's existing tab-move / command functions.
|
||||
|
||||
```ts
|
||||
const gc = new GestureController({
|
||||
video: videoEl,
|
||||
surface: stageEl, // normalized coords map against this element's rect
|
||||
numHands: 2,
|
||||
pinchOn: 0.35, pinchOff: 0.5, // pinch hysteresis (fractions of hand size)
|
||||
minCutoff: 1.0, beta: 0.01, // One-Euro cursor smoothing
|
||||
palmHoldMs: 1000, // Open_Palm hold before halt-all fires
|
||||
deviceId: "", // specific camera; "" = default user-facing
|
||||
});
|
||||
|
||||
// Drag events — surface pixels (X already mirrored), with a per-hand id.
|
||||
gc.on("grab", ({ hand, x, y }) => {}); // pinch closed → start drag
|
||||
gc.on("drag", ({ hand, x, y }) => {}); // moving while pinched (per frame)
|
||||
gc.on("drop", ({ hand, x, y }) => {}); // released (or hand vanished mid-pinch)
|
||||
|
||||
// Discrete commands: "halt-all" | "approve" | "new-session".
|
||||
// Still emitted by the controller, but THIS build's demo does not subscribe
|
||||
// (pinch-only). Wire these up in Codeman (Phase 5) or re-enable in the demo.
|
||||
gc.on("command", ({ name }) => {});
|
||||
|
||||
// Per-frame snapshot for HUD / hover highlighting / debug overlay.
|
||||
// `haltProgress` (0–1 Open_Palm charge) is also still emitted but unused here.
|
||||
gc.on("status", ({ fps, hands, haltProgress }) => {});
|
||||
// hands: { handedness, cursor:{x,y}/*normalized*/, pinchDist, pinching, gesture }[]
|
||||
gc.on("results", ({ result, timestampMs }) => {}); // raw recognizer result
|
||||
|
||||
await gc.start(); // requests camera, loads model
|
||||
await gc.useCamera(deviceId); // switch camera live
|
||||
await gc.listCameras(); // enumerate video inputs
|
||||
gc.stop();
|
||||
```
|
||||
|
||||
All events are live. Hover highlighting is derived from the `status` snapshot
|
||||
(not a dedicated event), since only the consumer can hit-test against its tabs.
|
||||
Full types in [`src/gesture/types.ts`](./src/gesture/types.ts).
|
||||
|
||||
---
|
||||
|
||||
## Tuning defaults (start here, then adjust by feel)
|
||||
|
||||
- **One-Euro filter:** `minCutoff ≈ 1.0`, `beta ≈ 0.01`. Raise `beta` if drag
|
||||
lags during fast moves; lower `minCutoff` if it jitters when still.
|
||||
- **Pinch:** `PINCH_ON 0.35`, `PINCH_OFF 0.5` (fractions of hand-size reference
|
||||
distance, wrist→middle-MCP). Two thresholds = hysteresis = no flicker.
|
||||
- **Confidence:** detection/tracking ≈ 0.6. Lower if quick gestures get missed,
|
||||
raise if you get false hands.
|
||||
- **Camera:** target 30–60fps, even frontal lighting on the hand zone.
|
||||
|
||||
---
|
||||
|
||||
## Design notes
|
||||
|
||||
- **Standalone first.** Fake tabs let us validate feel before touching Codeman.
|
||||
- **Main thread first.** Inference runs on the main thread (60fps, no stutter);
|
||||
a Web Worker stays an optional optimization only if the UI ever stutters.
|
||||
- **Two hands** (`numHands: 2`). Each hand keeps its own cursor + pinch state,
|
||||
keyed by handedness so filters don't swap when MediaPipe reorders the hands.
|
||||
- **Camera angle: front-facing default; superwide now usable too** — iPhone 17
|
||||
Pro main lens via Continuity Camera, pointing / pinch-to-grab grammar. The
|
||||
superwide / Desk View (overhead, ultra-wide) camera now works with no issues
|
||||
and tracking is confirmed on it; the earlier Safari-only / stretched blocker is
|
||||
resolved. The in-app picker switches cameras live.
|
||||
- **Transport-agnostic controller.** It emits coordinate-only `grab`/`drag`/
|
||||
`drop` + `command`; the consumer hit-tests. So Phase 5 only swaps the demo's
|
||||
`tabs.ts` for Codeman wiring — the controller is untouched.
|
||||
@@ -0,0 +1,224 @@
|
||||
# Codeman Gesture Control — Prototype Build Plan
|
||||
|
||||
> Canonical spec for this project. A Jarvis-style hand-tracking input layer for
|
||||
> the Codeman dashboard. Webcam sees your hands; you pinch-drag session tabs
|
||||
> between Screen columns and fire discrete gesture commands. Runs entirely
|
||||
> in-browser, camera feed never leaves the machine.
|
||||
|
||||
Build it as a **standalone prototype first** (its own folder, fake tabs) so the
|
||||
input feel can be validated on real hardware before any integration with
|
||||
Codeman's existing drag/command code.
|
||||
|
||||
---
|
||||
|
||||
## Goal & success criteria
|
||||
|
||||
Build a `GestureController` module + a self-contained demo page that:
|
||||
|
||||
1. Opens the webcam and runs MediaPipe Gesture Recognizer at ~30fps.
|
||||
2. Emits a smoothed cursor position and a `pinch` state (grab/release) from hand landmarks.
|
||||
3. Lets you **drag fake tabs between columns by pinching**, dropping on release.
|
||||
4. Fires **discrete gesture commands** (open palm, thumbs up, victory) onto an event bus.
|
||||
5. Feels responsive — drag lag is not perceptible, jitter is filtered out.
|
||||
|
||||
**Done when:** you can sit at your desk, pinch a tab, move it to another column,
|
||||
release, and it lands — reliably, without visible jitter, with the camera
|
||||
mounted at your chosen angle.
|
||||
|
||||
---
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **MediaPipe Tasks Vision** (`@mediapipe/tasks-vision`) — `GestureRecognizer` in `VIDEO` running mode, `numHands: 1` for v1 (add 2 later). Loads the prebuilt `gesture_recognizer.task` model + WASM from CDN.
|
||||
- **Vanilla TS + Vite** for the prototype (no framework needed; keep it portable so the module drops into Codeman regardless of its stack). If Codeman is React, the module stays framework-agnostic and you wrap it in a hook at integration time.
|
||||
- **One-Euro filter** for cursor smoothing — implement it directly, it's ~40 lines and is the correct tool for noisy interactive landmark streams (low lag at speed, heavy smoothing when still).
|
||||
- **Web Worker** for inference is a **Phase 4** optimization — do NOT start there. Get it working on the main thread first; only move to a worker if the dashboard UI stutters.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
```
|
||||
gesture-proto/
|
||||
├── index.html # demo page: video preview + columns of fake tabs
|
||||
├── package.json
|
||||
├── vite.config.ts
|
||||
├── src/
|
||||
│ ├── main.ts # wires GestureController -> demo UI
|
||||
│ ├── gesture/
|
||||
│ │ ├── GestureController.ts # core: camera + recognizer + state machine + events
|
||||
│ │ ├── OneEuroFilter.ts # cursor smoothing
|
||||
│ │ ├── pinch.ts # pinch detection w/ hysteresis
|
||||
│ │ ├── landmarks.ts # landmark index constants + helpers
|
||||
│ │ └── types.ts # event payload types, config
|
||||
│ └── demo/
|
||||
│ ├── tabs.ts # fake tab/column model + render
|
||||
│ └── overlay.ts # draws hand skeleton + cursor dot over video (debug)
|
||||
└── README.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Core algorithms (the parts that decide whether it feels good)
|
||||
|
||||
### 1. Cursor from landmarks
|
||||
The drag cursor is the **midpoint of thumb tip (landmark 4) and index tip (landmark 8)**, in normalized [0,1] coords from MediaPipe.
|
||||
|
||||
- **Mirror X** (`x = 1 - x`) — the webcam image is flipped relative to the user.
|
||||
- Map normalized → screen pixels against the dashboard's bounding rect.
|
||||
- Run the resulting (x, y) through **two independent One-Euro filters** (one per axis) before using it. Raw landmarks jitter by several pixels even when the hand is still; this is the single most important quality step.
|
||||
|
||||
### 2. Pinch detection with hysteresis
|
||||
Compute euclidean distance between landmark 4 and landmark 8. **Normalize by hand size** (e.g. distance wrist→middle-finger-MCP, landmarks 0→9) so the threshold is robust to how close the hand is to the camera.
|
||||
|
||||
- Use **two thresholds, not one** (hysteresis): enter pinch below `PINCH_ON` (e.g. 0.35 of hand size), exit only above `PINCH_OFF` (e.g. 0.5). This stops flickering between grab/release at the boundary — critical for not "dropping" a tab mid-drag.
|
||||
- Require the pinch state to persist N frames (e.g. 2–3) before firing, to reject single-frame noise.
|
||||
|
||||
### 3. State machine
|
||||
```
|
||||
IDLE ──hand detected──> HOVER ──pinch on──> GRABBED ──pinch off──> (drop) ──> HOVER
|
||||
^ | |
|
||||
└────hand lost───────────┴────────────────hand lost────────────────────────┘
|
||||
```
|
||||
- `HOVER`: cursor moves, highlights the tab/column under it (hit-test).
|
||||
- `GRABBED`: the grabbed tab follows the cursor; emit `drag` events.
|
||||
- On `pinch off` in GRABBED: hit-test cursor against drop columns, emit `drop {tabId, targetColumnId}` or `dropCancelled` if outside any column.
|
||||
|
||||
### 4. Discrete gestures → command bus
|
||||
From `result.gestures[0].categoryName`, debounced (fire once per gesture entry, not every frame while held):
|
||||
- `Open_Palm` held ~1s → `command: "halt-all"` (dead-man's-switch — pauses every session; genuinely useful for autonomous loops).
|
||||
- `Thumb_Up` → `command: "approve"`.
|
||||
- `Victory` → `command: "new-session"`.
|
||||
- Map these to the SAME command names your voice layer already dispatches, so both input sources converge on one dispatcher.
|
||||
|
||||
---
|
||||
|
||||
## GestureController public API (target shape)
|
||||
|
||||
```ts
|
||||
const gc = new GestureController({
|
||||
video: videoEl,
|
||||
surface: dashboardEl, // coords mapped against this element's rect
|
||||
numHands: 1,
|
||||
pinchOn: 0.35, pinchOff: 0.5,
|
||||
palmHoldMs: 1000,
|
||||
});
|
||||
|
||||
gc.on("hover", ({ x, y, targetId }) => {...});
|
||||
gc.on("grab", ({ x, y }) => {...});
|
||||
gc.on("drag", ({ x, y }) => {...}); // throttled to frame rate
|
||||
gc.on("drop", ({ targetColumnId }) => {...});
|
||||
gc.on("command",({ name }) => {...}); // halt-all | approve | new-session
|
||||
gc.on("status", ({ fps, handPresent, pinchDist }) => {...}); // debug HUD
|
||||
|
||||
await gc.start(); // requests camera, loads model
|
||||
gc.stop();
|
||||
```
|
||||
|
||||
Keep it **transport-agnostic**: it emits semantic events, it does NOT know about
|
||||
Codeman's DOM. Integration is just subscribing to these events and calling
|
||||
Codeman's existing tab-move / command functions.
|
||||
|
||||
---
|
||||
|
||||
## Phased build (each phase is independently testable — stop and feel it before moving on)
|
||||
|
||||
**Phase 0 — Scaffold & camera (½ day)**
|
||||
Vite + TS project. `index.html` with a mirrored `<video>` and a "start" button (camera must be a user gesture). Confirm `getUserMedia` works and you see yourself. Must be served over http(s), not `file://`.
|
||||
|
||||
**Phase 1 — Recognizer + debug overlay (½ day)**
|
||||
Load `GestureRecognizer` (`VIDEO` mode, CDN model+wasm). Run `recognizeForVideo(video, performance.now())` in a `requestAnimationFrame` loop. Draw the 21-point skeleton + an FPS counter on a canvas over the video. **Checkpoint: confirm you're getting ≥25fps on your actual camera/lighting setup.** Tune lighting here.
|
||||
|
||||
**Phase 2 — Cursor + pinch (1 day)**
|
||||
Implement `OneEuroFilter` and `pinch.ts`. Render a cursor dot driven by the filtered thumb/index midpoint. Show live pinch distance in the HUD and a color change on grab. **Checkpoint: the dot is steady when your hand is still, and pinch grab/release is crisp with no flicker.** Tune filter constants (`minCutoff`, `beta`) and pinch thresholds here — this is where the "feel" is won or lost.
|
||||
|
||||
**Phase 3 — Drag the fake tabs (1 day)**
|
||||
Build `tabs.ts`: 3 columns of draggable fake "sessions." Wire the state machine: hover-highlight, grab, drag-follow, drop-with-hit-test. **Checkpoint: you can move a tab across columns reliably 10/10 times.** This is the core demo and the real go/no-go for the whole idea.
|
||||
|
||||
**Phase 4 — Discrete commands + polish (1 day)**
|
||||
Add gesture→command bus with debouncing and the 1s open-palm halt. Add an on-screen toast when a command fires. Optional: move inference to a Web Worker if the UI stutters; add second-hand support.
|
||||
|
||||
**Phase 5 — Codeman integration (✅ working, 2026-06-07)**
|
||||
Drop `gesture/` into Codeman via a new consumer `src/codeman/entry.ts` (the core is unchanged; the demo's `main.ts` is *not* the integration point). It binds `grab`/`drag`/`drop` to real `.session-tab`s (grab-to-detach → `app.detachSession`) and pinch-taps the Run / Run Shell toolbar buttons. Runs in Codeman behind `CODEMAN_GESTURE=1`. See the detailed status under "Implementation status" below.
|
||||
|
||||
> **Prerequisite (decided 2026-06-06, ✅ done 2026-06-07): Codeman tab-detach first.**
|
||||
> Codeman needed a **tab-detach / undock** feature — a session pops out into its
|
||||
> own browser window — *before* gesture wiring, because gestures can only drag DOM
|
||||
> *within* the one page that owns the camera (you can't drag a node across isolated
|
||||
> tabs/OS windows). So undock is a Codeman session-placement op the gesture `drop`
|
||||
> *triggers*. **Shipped** as `app.detachSession(id)` → `/session/:id` solo window +
|
||||
> BroadcastChannel sync + re-dock on close (the gesture layer calls it directly).
|
||||
> Per-monitor placement via `getScreenDetails` stays in the multi-monitor backlog.
|
||||
|
||||
---
|
||||
|
||||
## Tuning defaults to start from (then adjust by feel)
|
||||
- One-Euro: `minCutoff ≈ 1.0`, `beta ≈ 0.01` (raise `beta` if drag lags during fast moves; lower `minCutoff` if it's jittery when still).
|
||||
- Pinch: `PINCH_ON 0.35`, `PINCH_OFF 0.5` (fractions of hand-size reference distance).
|
||||
- `min_detection_confidence` / `min_tracking_confidence` ≈ 0.6; lower if quick gestures get missed, raise if you get false hands.
|
||||
- Camera: target 30–60fps, even frontal lighting on the hand zone (matters more than the sensor).
|
||||
|
||||
---
|
||||
|
||||
## Hardware note
|
||||
|
||||
Prototype on the **MacBook M1 Max built-in cam** for zero-friction Phase 0–3.
|
||||
For the real setup, switch to **iPhone via Continuity Camera** mounted at desk
|
||||
level aimed at your hand-gesture zone — best sensor + best angle. Decide
|
||||
front-facing (pointing/pinch-to-grab) vs overhead (swipe/drag-on-a-plane) mount
|
||||
before Phase 3, since it slightly changes the gesture grammar.
|
||||
|
||||
---
|
||||
|
||||
## Implementation status (kept current)
|
||||
|
||||
- ✅ Phase 0, ✅ Phase 1 — see top-level `CLAUDE.md` and `gesture-proto/README.md`.
|
||||
- ✅ Phase 1 fps checkpoint — 60fps on MacBook + iPhone 17 Pro (Continuity Camera).
|
||||
- ✅ Phase 2 — `OneEuroFilter.ts` + `pinch.ts`: filtered cursor dot + pinch hysteresis. Cursor + `pinchDist` + `pinching` on the `status` event; HUD shows pinch distance, cursor ring turns green on grab.
|
||||
- ✅ Phase 3 — `demo/tabs.ts`: 3 Screen columns of draggable session tabs. Controller owns the per-hand pinch state machine and emits `grab`/`drag`/`drop` in surface pixels (with a `hand` id); the demo hit-tests and moves tabs. Two-handed (drag two tabs at once); drop-on-vanish releases a tab if a pinched hand leaves frame.
|
||||
- ✅ Beyond plan — two-hand tracking (`numHands: 2`, filters keyed by handedness) and a live camera picker.
|
||||
- ✅ Camera (2026-06-06) — front-facing iPhone 17 Pro main lens (Chrome) is the default. **The superwide / Desk View (ultra-wide, overhead) camera now works with no issues and tracking is confirmed on it** — the earlier "Safari-only / stretched / unusable" finding is superseded. Pick either via the in-app camera picker.
|
||||
- ✅ Phase 4 (built) — `commands.ts`: debounced gesture→command bus. `Thumb_Up`→approve, `Victory`→new-session (edge-triggered, fire once per entry), `Open_Palm` held `palmHoldMs`→halt-all (dead-man's-switch) with a 0–1 `haltProgress` charge surfaced on `status`. Commands ignore a pinching (mid-drag) hand.
|
||||
- ⛔ **Phase 4 unwired in the demo (2026-06-06).** User wants pinch-drag only — an open palm while reaching to pinch kept charging the hold-to-halt. `main.ts` no longer subscribes to `command`/`haltProgress` and the command/charge toasts are gone. The `GestureController` core is untouched and still emits both events, so Phase 5 (or a re-enabled demo) can pick them up unchanged.
|
||||
- 🐛 **Drag-position fix (2026-06-06).** A `.dragging` tab is `position: absolute`; the `.column`s establish a containing block via `backdrop-filter`, so board-local left/top were offset by the column's own position — tabs in the middle/right columns flew to the right on grab. Fix: `tabs.ts` reparents the floating tab onto `#board` (no filter/transform) for the drag, so the coordinates `moveTo` computes match the containing block.
|
||||
- ✅ **Phase 5 — WORKING at the desk (2026-06-07).** Prerequisite cleared: Codeman tab-detach/undock works in the runtime (`app.detachSession(id)` is the idempotent hook). The gesture overlay runs live in the real Codeman dashboard on `:5000` and was confirmed by the user (normal tab): fullscreen cam + hand/cursor tracking, undock-by-pinch, and Run/Run Shell taps.
|
||||
- **Integration shape: in-page overlay, built into Codeman beta.** The gesture `core` (`src/gesture/`) ships **unchanged**; the consumer is `src/codeman/entry.ts`, esbuild-bundled (`npm run build:codeman`) and served by Codeman at `/gesture/gesture-codeman.js`. A full-viewport, click-through overlay maps coords straight to `elementFromPoint`.
|
||||
- **Gestures (routed by what the pinch lands on):** (a) **grab → in-page floating panel** *(⚠️ pivoted 2026-06-08 — was grab-to-detach)*: pinch a `.session-tab`, a ghost clone follows the hand, pull >`DETACH_PULL_PX` (70) and release → `floatSession(id, x, y)` spawns a re-grabbable `.cg-float` iframe of `/session/:id` (640×420). This **replaced** `window.app.detachSession(id)` (an OS window is a sealed box the hand can't move again — a one-way trip); the float stays in-page so the hand keeps control. See `../../docs/MULTIMONITOR_DESIGN.md`. (b) **Run / Run Shell taps** — pinch over `#runBtn`→`app.run()` / `.btn-shell`→`app.runShell()` and release in place; drift >`TAP_CANCEL_PX` (45) cancels. `CLICK_SELECTOR` is the extensible list. (c) **Fullscreen dimmed cam** by default, **⛶** toggles corner PiP.
|
||||
- **Self-hosted MediaPipe (no CDN).** `entry.ts` passes `wasmBase: "/gesture/wasm"` + `modelUrl: "/gesture/gesture_recognizer.task"`; Codeman serves them same-origin. The CDN path failed in the normal browser tab (content/ad blocker blocking `jsdelivr`/`googleapis`) → surfaced as `failed: {"isTrusted":true}` once `entry.ts` learned to report non-Error throws. Core also gained a GPU→CPU delegate fallback.
|
||||
- **Gated by `CODEMAN_GESTURE=1` (OFF by default).** Under the flag Codeman injects the module script (dashboard only, not `/session/:id` solo popups), cache-busts it with `?v=<mtime>` (static is `max-age=1y`), and widens CSP (`'wasm-unsafe-eval'` + `worker-src 'self' blob:`; same-origin assets now covered by `'self'`). Flag off ⇒ Codeman HTML/CSP unchanged.
|
||||
- **Codeman-side / version control:** the detach + instance isolation + base gesture overlay are committed on `Ark0N/Codeman` branch `beta/session-detach`, open as **PR #103** (tip `afea6d6`; `ceca853` after I fixed its `auth.ts` format:check → CI green). Gotcha: the local prod clone `~/.codeman/app` tracks only `master`, so the branch is hidden until `git fetch origin beta/session-detach` (this briefly misled me into a bogus local reconstruction `03b31b8`, since deleted). The session improvements — direct-detach via `window.app.detachSession`, Run/Run Shell pinch-taps, self-hosted MediaPipe (`/gesture/wasm` + `.task`), and the `server.ts` mtime cache-bust — were **ported onto PR #103** in commit `eea84db` (CI green); their source is `Ark0N/codeman-gesture-control` (`src/codeman/entry.ts`).
|
||||
- **Commits (gesture-proto):** `ddf9cda` (consumer: detach/cam/errors) → `2dd97da` (detach direct) → `21ef793` (Run/Run Shell taps) → `e055b79` (self-host MediaPipe).
|
||||
- **Next:** tune feel; optional in-strip reorder (deferred — user chose detach-only) and more buttons (Stop). Discrete `command` events remain available but unwired (pinch-only). Hand-off brief: `../docs/CODEMAN_DETACH_BRIEF.md`.
|
||||
|
||||
## Backlog (requested, for later)
|
||||
|
||||
- ✅ **Fullscreen mode** — done. Toggle button fullscreens the `#stage`;
|
||||
`:fullscreen` CSS fills the viewport and the coord mapping adapts since it
|
||||
reads the stage rect every frame.
|
||||
- ✅ **Multi-monitor mode — A+C BUILT & validated at the desk (2026-06-08).** The
|
||||
eventual real goal (fling a Codeman session onto an external display by gesture)
|
||||
is reached. **Design → [`../../docs/MULTIMONITOR_DESIGN.md`](../../docs/MULTIMONITOR_DESIGN.md)**,
|
||||
approach **A+C**:
|
||||
- **A — in-page floating panels** (`581fcf9`, `3e0447a`): `entry.ts`
|
||||
`floatSession(id, x, y)` pops a tab into a re-grabbable `.cg-float` iframe of
|
||||
`/session/:id` (640×420) instead of `window.app.detachSession`. The session
|
||||
stays in the camera-owning page's DOM, so the hand keeps control — fixing the
|
||||
one-way-trip flaw of OS-window detach.
|
||||
- **C-span — spanned window** (`063fd8f`, `59946b8`): `scripts/span-codeman.sh`
|
||||
launches a Brave-first (`BROWSER=` override) `--app` window sized to the
|
||||
display union; prereq macOS "Displays have separate Spaces" OFF + re-login.
|
||||
One-click via the **Codeman header button** → `POST /api/system/span-displays`
|
||||
(PR #103 `95b0035`). **Validated:** one window spans both monitors and a panel
|
||||
drags across the seam.
|
||||
- **C-snap** (`getScreenDetails` snapping + seam dead-band) and the **re-dock**
|
||||
gesture/zone are **still pending** — not needed for basic cross-seam dragging.
|
||||
- **Concurrent-rendering question** was RESOLVED first: Codeman already mounts
|
||||
live terminals into floating panels (teammate terminals, log-viewer SSE
|
||||
windows, the iframe-able `/session/:id` solo route), so the live float needed
|
||||
no new Codeman rendering.
|
||||
|
||||
Note: the public event surface evolved from the original API sketch. The
|
||||
controller stays transport-agnostic but emits coordinate-only `grab`/`drag`/
|
||||
`drop` (hit-testing lives in the consumer, since only it knows the DOM/columns).
|
||||
`hover`/`dropCancelled`/`targetId` were dropped; hover highlighting is derived
|
||||
from the `status` snapshot instead.
|
||||
@@ -0,0 +1,80 @@
|
||||
# Feature brief: Session tab detach / undock (for Codeman)
|
||||
|
||||
> ✅ **SHIPPED — on GitHub as PR #103 (open).** Branch `beta/session-detach` on
|
||||
> `Ark0N/Codeman` (base `master`): "feat(web): session detach/undock + beta
|
||||
> instance isolation (port 5000)", containing detach/undock + instance isolation
|
||||
> + the base gesture overlay (commit `afea6d6`). `app.detachSession(id)` in
|
||||
> `app.js` opens `/session/:id` as a solo window (another live client of the same
|
||||
> session), tracks it (badge + `BroadcastChannel` sync + re-dock on close), and is
|
||||
> the single idempotent entry point both the on-tab ⧉ icon and the gesture layer
|
||||
> call. PTY fan-out (the open question below) resolved **yes**, so no streaming
|
||||
> work was needed. CI green after I fixed a prettier format:check on `auth.ts`
|
||||
> (commit `ceca853`).
|
||||
>
|
||||
> ⚠️ **Note:** the local **prod** clone `~/.codeman/app` only tracks `master`, so
|
||||
> the PR branch is invisible there until `git fetch origin beta/session-detach`.
|
||||
> (Earlier today I briefly mis-concluded the PR didn't exist and made a bogus
|
||||
> local reconstruction — deleted. The PR was real all along.) The gesture-side
|
||||
> *improvements* from this session — direct-detach, Run/Run Shell pinch-taps,
|
||||
> self-hosted MediaPipe, `server.ts` cache-bust — were **ported onto PR #103**
|
||||
> (commit `eea84db`, CI green); their source is `Ark0N/codeman-gesture-control`.
|
||||
> The rest of this doc is the original hand-off brief, kept for history.
|
||||
|
||||
> Hand-off brief for **Codeman** to refine and implement **on a beta branch**.
|
||||
> Authored from the gesture-control project, which needs this as a prerequisite.
|
||||
> Codeman is "aicodeman": a Fastify + WebSocket server streaming xterm.js
|
||||
> terminal (tmux) sessions to a web dashboard.
|
||||
|
||||
## Goal
|
||||
|
||||
Let a session "tab" pop out of the main dashboard into its **own browser
|
||||
window** (and back). Each detached window shows just that one session's
|
||||
terminal, fully live. This is a standalone UX win *and* a prerequisite for
|
||||
gesture control later (a hand-gesture "drop" will eventually trigger
|
||||
detach/relocate — but that's a separate project; **this feature is plain UI
|
||||
buttons only**).
|
||||
|
||||
## Core approach (refine as needed)
|
||||
|
||||
- Add a **"Detach" control** on each tab. It opens a new browser window
|
||||
(`window.open`) pointing at a **single-session view** — ideally a real route
|
||||
like `/session/:id` so the popup just loads a URL and attaches like a normal
|
||||
client.
|
||||
- The detached window runs its **own xterm.js instance connected to the same
|
||||
session's WebSocket**, so it's live, not a screenshot.
|
||||
- Keep the dashboard and detached windows **in sync** (session list, titles,
|
||||
alive/dead state, focus) — via the existing events channel, or a
|
||||
`BroadcastChannel` if simpler.
|
||||
- Support **re-dock** (close popup → tab returns to the dashboard) and handle the
|
||||
popup being closed/refreshed gracefully.
|
||||
|
||||
## The one critical question to resolve first (in Codeman's own code)
|
||||
|
||||
Can the server currently **fan out one session's PTY/tmux output to multiple
|
||||
concurrent WebSocket clients**, or is it single-consumer? A detached window is a
|
||||
*second* viewer of the same session. If it's single-consumer today, that's the
|
||||
main change: make the pty→socket stream **broadcast to N subscribers** (and merge
|
||||
input) so dashboard + popup can both watch/type. This likely matters more than
|
||||
the UI work.
|
||||
|
||||
## Other decisions for Codeman
|
||||
|
||||
- Per-session route (`/session/:id`) vs. a single-page popup that's told which id
|
||||
to show.
|
||||
- Multi-monitor placement later via the Window Management API
|
||||
(`getScreenDetails`) — **out of scope now**, just don't design against it.
|
||||
- Auth/cookie sharing so a popup window authenticates the same as the dashboard.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Implement on a **beta branch**, not `main`.
|
||||
- The gesture-control side keeps a **read-only** copy of Codeman (its `.git`
|
||||
removed); the live `Ark0N/Codeman` repo is **not** touched from here. Codeman
|
||||
implements this itself.
|
||||
|
||||
## Why this is sequenced before gesture wiring
|
||||
|
||||
Gestures can only drag DOM **within the single page that owns the camera**; you
|
||||
cannot drag a node across isolated browser tabs / OS windows. So undock must be a
|
||||
**Codeman session-placement operation** that a gesture `drop` later *triggers* —
|
||||
not something the gesture layer does. Detach first; wire gestures to it after.
|
||||
@@ -0,0 +1,260 @@
|
||||
# Multi-monitor gesture design — in-page panels + spanned window (A + C)
|
||||
|
||||
**Status:** **A + C-span BUILT & validated at the desk (2026-06-08).** C-snap
|
||||
(`getScreenDetails` snapping) still pending. Decided + built 2026-06-08.
|
||||
**Supersedes** the OS-window detach as the *gesture* verb (see "Why detach
|
||||
broke movability") — *now actually replaced in `entry.ts`, not just planned.*
|
||||
**Companion docs:** `CODEMAN_DETACH_BRIEF.md` (the original window.open detach),
|
||||
`../gesture-proto/docs/BUILD_PLAN.md` (canonical build spec, Phase 5).
|
||||
|
||||
> ## Implementation status (2026-06-08)
|
||||
> - ✅ **A — in-page floating panels.** `entry.ts` `floatSession(id, x, y)` spawns
|
||||
> a `.cg-float` div with an `<iframe src="/session/:id">` (640×420) at the drop
|
||||
> point; panels are re-grabbable. Replaces `window.app.detachSession`. Commits
|
||||
> `581fcf9`, `3e0447a`.
|
||||
> - ✅ **C-span — spanned window.** `scripts/span-codeman.sh` (Brave-first;
|
||||
> `BROWSER=` override) launches a `--app` window sized to the display union.
|
||||
> Commits `063fd8f`, `59946b8`. **Validated at the desk:** one window spans
|
||||
> both monitors (3432×1080) and a panel drags across the seam.
|
||||
> - ✅ **Launch entry point in Codeman.** A header "multi-monitor" button (replaces
|
||||
> the notification bell) → `POST /api/system/span-displays` → spawns the span
|
||||
> script. Bundled `span-codeman.sh` into Codeman's repo. **PR #103** `95b0035`.
|
||||
> - ✅ **Cache-bust** all same-origin module scripts/CSS (`renderIndexHtml` →
|
||||
> `cacheBustAssets`), so frontend edits show on a normal reload. PR #103 `b5ea711`.
|
||||
> - ⏳ **C-snap** (`getScreenDetails` snapping + seam dead-band) — not built; not
|
||||
> required for basic cross-seam dragging. Also pending: the re-dock gesture/zone.
|
||||
> - **Known caveats from the desk run:** dead band on a taller/offset external
|
||||
> display (inherent to one spanning rect); superwide lens not selectable in the
|
||||
> fresh span-window browser profile; 3-monitor works unchanged but dead-space +
|
||||
> cursor-sensitivity caveats grow.
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
Pinch a Codeman session, drag it anywhere — including **across a second
|
||||
physical monitor** — drop it, and have it stay where you put it and stay
|
||||
grabbable again. The "fling a session onto the external display by gesture"
|
||||
end-goal from the BUILD_PLAN backlog, made real **without losing the ability to
|
||||
move a session after you've placed it.**
|
||||
|
||||
## The root constraint (why this design, not the others)
|
||||
|
||||
The hand only exists in **the one page that owns the camera**. Hand tracking,
|
||||
the cursor, and `document.elementFromPoint` hit-testing all live in that single
|
||||
document. **An OS window created by `window.open` is a sealed box that page
|
||||
cannot reach into** — no shared DOM, no shared cursor.
|
||||
|
||||
> **Why detach broke movability.** Today `entry.ts` → `detach(id)` calls
|
||||
> `window.app.detachSession(id)`, which `window.open`s the session into its own
|
||||
> OS window. The instant it leaves the camera-owning page, the hand can never
|
||||
> touch it again. Detach-by-pinch *works*, but it's a one-way trip.
|
||||
|
||||
**Rule:** anything you want to keep gesture-movable must stay inside **one
|
||||
page's DOM.** This design honors that with two composed pieces:
|
||||
|
||||
- **A — In-page floating panels.** "Detach" pops a session into a free-floating,
|
||||
absolutely-positioned element *in the same page* (not an OS window), so the
|
||||
hand keeps control forever.
|
||||
- **C — One window spanning both monitors.** Run Codeman in a single window
|
||||
stretched across both physical displays, so "drag across monitors" is just
|
||||
"drag across the page," and the second monitor's pixels are actually used.
|
||||
|
||||
Option **B** (one OS window per monitor + a distributed BroadcastChannel cursor
|
||||
protocol + cross-window session hand-off) was considered and deferred: it's the
|
||||
only path to *independent* per-monitor OS windows, but it's a much larger build
|
||||
and reintroduces the cross-window wall this design exists to avoid.
|
||||
|
||||
---
|
||||
|
||||
## Part A — In-page floating panels
|
||||
|
||||
### Codeman already has the primitive
|
||||
|
||||
`panels-ui.js` has a `.detached` "floating window": an absolutely-positioned
|
||||
`<div>` with `panel.style.top/left/width/height` and a drag handler
|
||||
(`setupMonitorDrag`) — **all in-page, no `window.open`.** The session-tab detach
|
||||
simply picked the wrong primitive (the OS-window one). Part A reuses the
|
||||
in-page one.
|
||||
|
||||
### Concurrent session rendering — RESOLVED (2026-06-08): live floats are feasible now
|
||||
|
||||
The main *dashboard view* renders **one active session at a time** (a single
|
||||
shared `this.terminal` opened into `#terminalContainer` in `terminal-ui.js:26`,
|
||||
swapped by `selectSession()`). But the page is **not** limited to one terminal —
|
||||
the PR #103 branch already ships **three independent in-page concurrent
|
||||
floating-content subsystems** we can reuse, so a live floating panel per session
|
||||
needs **no new Codeman rendering architecture**:
|
||||
|
||||
1. **Teammate terminals** (`panels-ui.js` `teammateTerminals` Map +
|
||||
`subagent-windows.js`). A `Map` of **concurrent live `new Terminal()`
|
||||
instances**, each `terminal.open(body)`'d into a floating panel, bound to a
|
||||
`sessionId` + tmux `paneTarget`, **seeded via REST** buffer fetch
|
||||
(`/api/sessions/:id/teammate-pane-buffer/:pane`), **input via REST**
|
||||
(`/api/sessions/:id/teammate-pane-input`), with lazy-mount (`_lazyPaneTarget`)
|
||||
and `dispose()` cleanup. *This is option 2 already built and shipping.*
|
||||
2. **Log-viewer windows** (`panels-ui.js:2632`). Concurrent draggable floating
|
||||
windows, each with its own `EventSource` **SSE stream** and lifecycle map —
|
||||
proof of the generic "floating window + independent per-window stream"
|
||||
pattern.
|
||||
3. **`/session/:id` solo route** (`server.ts:573`, `renderIndexHtml(soloId)`,
|
||||
`text/event-stream` at `:633`). A full **standalone live session page** —
|
||||
**iframe-able** — reusing the exact multi-client fan-out detach already
|
||||
depends on. Solo mode deliberately **omits** the gesture overlay
|
||||
(`server.ts:1010-1014`), so there's no nested-overlay problem.
|
||||
|
||||
**The PTY multi-viewer fan-out is confirmed** (the brief's open "yes"): a session
|
||||
is addressable by multiple concurrent clients — the solo route, the teammate
|
||||
REST endpoints, and the on-tab pop-out all view the same live session.
|
||||
|
||||
**Resolution:** skip the static-preview fallback. Build live floats directly,
|
||||
ranked by new-code cost:
|
||||
|
||||
- **Primary — iframe the solo route.** `floatPanel(id)` =
|
||||
`<div class="cg-float" data-id><iframe src="/session/:id"></iframe></div>`.
|
||||
The iframe is a complete live session view (its own terminal + stream client);
|
||||
the gesture layer moves the **div** and the iframe rides along. Lowest new
|
||||
code; reuses proven fan-out; no terminal wiring. Trade-off: a full app shell
|
||||
per float (heavier — fine for a few, watch memory at many).
|
||||
- **Richer alt — native teammate-style panel.** Mount a `new Terminal()` in the
|
||||
float body, seed via the session buffer endpoint, feed via the teammate
|
||||
stream. Native (no iframe), lighter per-float, same-document. Use if the
|
||||
iframe feels heavy or you want tighter integration.
|
||||
|
||||
Either way the hand only **places** the float; typing into it uses a keyboard
|
||||
(focus the iframe / native terminal) — consistent with "gesture places,
|
||||
keyboard types."
|
||||
|
||||
### Gesture grammar (replaces the current detach path in `entry.ts`)
|
||||
|
||||
The grab/drag/drop plumbing already exists; only the **drop action** changes.
|
||||
|
||||
- **Grab** a `.session-tab[data-id]` (unchanged: `onGrab`, ghost-follow).
|
||||
- **Pull** past `DETACH_PULL_PX` to arm (unchanged: `grab.armed`).
|
||||
- **Drop while armed** → **no longer** `window.app.detachSession`. Instead spawn
|
||||
an **in-page floating panel** for that session id at the drop point. New
|
||||
method `floatPanel(id, x, y)` replacing `detach(id)`.
|
||||
- **Re-grab** a floating panel (new `PANEL_SELECTOR`, e.g. `.cg-float[data-id]`)
|
||||
→ move it; drop anywhere → it stays. This is the capability detach lost.
|
||||
- **Drop a panel back over the tab strip** (or a dock zone) → **re-dock**
|
||||
(remove the float; session returns to a plain tab). Mirrors the `.detached` →
|
||||
attach toggle Codeman already has.
|
||||
- **Keep `window.open` detach as a separate, deliberate verb** — e.g. a button,
|
||||
or a distinct "throw up and off-screen" gesture — for intentionally parking a
|
||||
session in its own OS window. It is *not* the default pinch action anymore.
|
||||
|
||||
### `entry.ts` change surface
|
||||
|
||||
- New state: `floats: Map<string, FloatingPanel>` (id → element + position),
|
||||
parallel to the existing `grabs`/`taps` maps.
|
||||
- `onGrab`: extend hit-testing to also match `PANEL_SELECTOR`, so an existing
|
||||
float can be re-grabbed (priority: panel over tab when overlapping).
|
||||
- `onDrop`: replace `if (grab.armed) this.detach(grab.id)` with
|
||||
`this.floatPanel(...)`; add the panel re-dock branch.
|
||||
- `floatPanel(id, x, y)`: create/show the in-page panel (Part A option 1/2),
|
||||
position it absolutely at the drop point. Idempotent per id (re-grab moves the
|
||||
existing one, never duplicates).
|
||||
- Coordinate mapping is **already viewport-pixel based** (the click-through
|
||||
surface maps cursor → viewport px), so it needs **no change** for spanning —
|
||||
see Part C.
|
||||
|
||||
---
|
||||
|
||||
## Part C — One window spanning both monitors
|
||||
|
||||
Once the Codeman window physically covers both displays, Part A's panels drag
|
||||
across the seam for free, because the gesture cursor is already in viewport
|
||||
pixels and the viewport now spans both monitors.
|
||||
|
||||
### macOS setup (operational, near-zero code)
|
||||
|
||||
1. **System Settings → Desktop & Dock → uncheck "Displays have separate
|
||||
Spaces."** (Requires a logout/login.) This is what lets a single window
|
||||
straddle two physical displays.
|
||||
2. Run Codeman **maximized, not fullscreen.** Browser fullscreen is *per
|
||||
display* and will **not** span — use a maximized/borderless window dragged to
|
||||
cover both monitors. (A kiosk/`--app` Chrome window sized to the union rect is
|
||||
the cleanest.) **Automated by `scripts/span-codeman.sh`** — it reads the
|
||||
display-union rect (Finder desktop bounds) and launches a **Brave-first**
|
||||
Chromium-family `--app` window sized to it (fresh per-browser profile so the
|
||||
geometry flags are honored; `BROWSER=` overrides — plain Chrome bounced on the
|
||||
desk machine). One-click from the **Codeman header "multi-monitor" button**
|
||||
(`POST /api/system/span-displays`, which spawns this script), or run it
|
||||
directly. Step 1 + logout is still manual; the script warns if spanning isn't
|
||||
active.
|
||||
3. Arrange the two monitors as a contiguous rectangle in Display settings so the
|
||||
union has no vertical offset gap.
|
||||
|
||||
### Coordinate model & the bezel seam
|
||||
|
||||
- Enumerate displays with **`window.getScreenDetails()`** (Chrome, secure
|
||||
context, `window-management` permission). Gives each screen's
|
||||
`left/top/width/height/availLeft/...` in a **virtual-desktop coordinate
|
||||
space** spanning all monitors.
|
||||
- Use it for **screen-edge snapping zones**: e.g. dropping a panel within the
|
||||
right screen's bounds snaps it to fill that screen; the seam between the two
|
||||
screens' rects is a "halt / boundary" zone the cursor crosses.
|
||||
- **Account for the bezel gap.** The two monitors are physically separated, but
|
||||
the spanned window's pixels are contiguous — a panel dragged across the seam
|
||||
visually jumps the bezel. Optional: add a dead-band at the seam x-coordinate
|
||||
so a panel snaps to one side rather than straddling.
|
||||
- `getScreenDetails` is **only needed for snapping/zone logic**, not for basic
|
||||
dragging — dragging works the moment the window spans. So Part C can ship in
|
||||
two steps: (1) just span + free drag, (2) add `getScreenDetails` snapping.
|
||||
|
||||
### Constraints to surface to the user
|
||||
|
||||
- macOS-specific; the "separate Spaces" toggle is global and affects all apps.
|
||||
- Real fullscreen is unavailable (must run maximized).
|
||||
- A bezel-width discontinuity sits in the middle of the coordinate space.
|
||||
- `window-management` permission prompts once.
|
||||
|
||||
---
|
||||
|
||||
## Build phases
|
||||
|
||||
1. ✅ **A-MVP — in-page live float, single monitor (DONE, `581fcf9`/`3e0447a`).**
|
||||
`entry.ts` `floatSession(id, x, y)` spawns an **iframe of `/session/:id`** in a
|
||||
`.cg-float` div (640×420) at the drop point; re-grab + move work (re-dock zone
|
||||
still TBD). Movability restored — the live session rides inside the page. (The
|
||||
method is named `floatSession`, not the design-sketch `floatPanel`.)
|
||||
2. ✅ **C-span — span the window (DONE, `063fd8f`/`59946b8`; validated at desk).**
|
||||
macOS "separate Spaces" off + `scripts/span-codeman.sh` launches a maximized
|
||||
`--app` window across the display union (Brave-first; `BROWSER=` override).
|
||||
**Confirmed:** panels drag across the seam — viewport mapping held with zero
|
||||
`entry.ts` change. Launchable one-click from the **Codeman header button** →
|
||||
`POST /api/system/span-displays` (PR #103 `95b0035`), or by running the script.
|
||||
3. ⏳ **C-snap — screen-aware snapping (pending).** Add `getScreenDetails`; snap a
|
||||
panel to the screen it's dropped on; add the seam dead-band.
|
||||
4. **A-alt (optional).** Swap the iframe float for a native teammate-style
|
||||
`new Terminal()` panel if the iframe shell feels heavy or many floats strain
|
||||
memory.
|
||||
|
||||
## Open questions / verification before coding
|
||||
|
||||
- [x] **Concurrent rendering — RESOLVED (2026-06-08).** A live session view
|
||||
*can* be mounted into an arbitrary in-page container concurrently with the
|
||||
main view. The PR #103 branch already ships it three ways (teammate terminals,
|
||||
log-viewer SSE windows, the iframe-able `/session/:id` solo route); fan-out
|
||||
confirmed. Primary path: iframe the solo route. See the resolved section above.
|
||||
- [ ] **Native-panel live feed (only if taking A-alt, not the iframe):** trace
|
||||
the exact transport that pushes *continuous* teammate-pane output after the
|
||||
REST buffer seed (`pendingData` flush source). The iframe path sidesteps this.
|
||||
- [ ] **`window.app.detachSession` location:** when wiring the *kept* OS-window
|
||||
detach verb, note this method was **not found in PR #103 source** (only
|
||||
server-side `detachSessionListeners`); it works at runtime, so confirm where
|
||||
it's actually defined before extending it.
|
||||
- [ ] **Spanning feasibility on the actual desk setup** (monitor arrangement,
|
||||
whether "separate Spaces" off is acceptable to the user globally).
|
||||
- [ ] **Re-dock target:** decide the re-dock gesture/zone (drop over tab strip
|
||||
vs a dedicated dock region).
|
||||
|
||||
## Touch-points summary
|
||||
|
||||
| Layer | File | Change | Status |
|
||||
|-------|------|--------|--------|
|
||||
| Gesture consumer | `gesture-proto/src/codeman/entry.ts` | `detach()` → `floatSession()`; `floats` map; panel re-grab; (later) re-dock branch + `getScreenDetails` snapping | ✅ float+re-grab done (`581fcf9`/`3e0447a`); re-dock/snap pending |
|
||||
| Gesture core | `gesture-proto/src/gesture/*` | **none** — stays transport-agnostic | ✅ unchanged |
|
||||
| Codeman — launch | `src/web/routes/system-routes.ts`, `src/web/public/{index.html,panels-ui.js}`, `scripts/span-codeman.sh` | header button → `POST /api/system/span-displays` → spawn span script (bundled into repo) | ✅ PR #103 `95b0035` |
|
||||
| Codeman — caching | `src/web/server.ts` | `cacheBustAssets()` — `?v=<mtime>` on all same-origin `.js`/`.css` so frontend edits show on normal reload | ✅ PR #103 `b5ea711` |
|
||||
| Ops | macOS display settings + `scripts/span-codeman.sh` | "separate Spaces" off + re-login; Brave-first maximized spanning window | ✅ validated at desk |
|
||||
@@ -0,0 +1,208 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Codeman Gesture Control — Phase 4</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #0b0f17;
|
||||
--panel: #131a26;
|
||||
--ink: #e6edf3;
|
||||
--muted: #94a3b8;
|
||||
--accent: #38bdf8;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--ink);
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 1.5rem;
|
||||
}
|
||||
h1 { font-size: 1.25rem; margin: 0; font-weight: 650; }
|
||||
.sub { color: var(--muted); font-size: 0.85rem; margin: 0; }
|
||||
|
||||
.stage {
|
||||
position: relative;
|
||||
width: min(90vw, 960px);
|
||||
aspect-ratio: 16 / 9;
|
||||
background: #000;
|
||||
border-radius: 12px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
/* Fullscreen: fill the display so grab targets get big. Coord mapping
|
||||
reads the stage rect each frame, so it adapts automatically. */
|
||||
#stage:fullscreen,
|
||||
#stage:-webkit-full-screen {
|
||||
width: 100vw;
|
||||
height: 100vh;
|
||||
max-width: none;
|
||||
aspect-ratio: auto;
|
||||
border-radius: 0;
|
||||
}
|
||||
/* Mirror the camera so it reads like a, er, mirror. */
|
||||
#cam {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
transform: scaleX(-1);
|
||||
}
|
||||
/* Overlay is NOT CSS-mirrored; overlay.ts mirrors coords itself.
|
||||
z-index 2 keeps the cursor + skeleton drawn over the tab board. */
|
||||
#overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
/* Phase 3 tab board — overlays the video; gesture-driven, no mouse. */
|
||||
#board {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
padding: 12px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
}
|
||||
#board .column {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: rgba(11, 15, 23, 0.55);
|
||||
border: 1px solid #25324a;
|
||||
border-radius: 10px;
|
||||
backdrop-filter: blur(3px);
|
||||
transition: border-color 0.1s, background 0.1s;
|
||||
}
|
||||
#board .column.col-hot {
|
||||
border-color: var(--accent);
|
||||
background: rgba(56, 189, 248, 0.18);
|
||||
}
|
||||
#board .column > header {
|
||||
font-size: 0.78rem;
|
||||
font-weight: 650;
|
||||
color: var(--muted);
|
||||
padding: 0.5rem 0.6rem;
|
||||
border-bottom: 1px solid #25324a;
|
||||
}
|
||||
#board .tablist {
|
||||
flex: 1;
|
||||
padding: 0.5rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.4rem;
|
||||
overflow: hidden;
|
||||
}
|
||||
#board .tab {
|
||||
background: #1b2740;
|
||||
border: 1px solid #2c3c5c;
|
||||
border-radius: 8px;
|
||||
padding: 0.85rem 0.7rem;
|
||||
min-height: 3rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
font-size: 0.95rem;
|
||||
font-weight: 550;
|
||||
font-variant-numeric: tabular-nums;
|
||||
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.3);
|
||||
transition: border-color 0.1s, transform 0.05s;
|
||||
}
|
||||
#board .tab.tab-hot {
|
||||
border-color: var(--accent);
|
||||
background: #22304d;
|
||||
}
|
||||
#board .tab.dragging {
|
||||
position: absolute;
|
||||
z-index: 5;
|
||||
width: auto;
|
||||
box-shadow: 0 12px 30px rgba(0, 0, 0, 0.55);
|
||||
transform: scale(1.05);
|
||||
opacity: 0.97;
|
||||
border-color: #4ade80;
|
||||
}
|
||||
|
||||
.controls { display: flex; gap: 0.75rem; align-items: center; }
|
||||
button {
|
||||
background: var(--accent);
|
||||
color: #04222e;
|
||||
border: 0;
|
||||
border-radius: 8px;
|
||||
padding: 0.55rem 1.1rem;
|
||||
font-size: 0.95rem;
|
||||
font-weight: 650;
|
||||
cursor: pointer;
|
||||
}
|
||||
button:disabled { opacity: 0.4; cursor: not-allowed; }
|
||||
button.ghost { background: var(--panel); color: var(--ink); }
|
||||
select {
|
||||
background: var(--panel);
|
||||
color: var(--ink);
|
||||
border: 1px solid #25324a;
|
||||
border-radius: 8px;
|
||||
padding: 0.5rem 0.75rem;
|
||||
font-size: 0.9rem;
|
||||
max-width: 16rem;
|
||||
}
|
||||
select:disabled { opacity: 0.4; cursor: not-allowed; }
|
||||
|
||||
.hud {
|
||||
display: flex;
|
||||
gap: 1.5rem;
|
||||
background: var(--panel);
|
||||
border-radius: 10px;
|
||||
padding: 0.75rem 1.25rem;
|
||||
font-variant-numeric: tabular-nums;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.hud .label { color: var(--muted); margin-right: 0.4rem; }
|
||||
.hud b { font-weight: 650; }
|
||||
|
||||
#status-msg { color: var(--muted); font-size: 0.85rem; min-height: 1.2em; margin: 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Codeman Gesture Control</h1>
|
||||
<p class="sub">Phase 4 — pinch-drag tabs + discrete gesture commands. Camera feed never leaves this machine.</p>
|
||||
|
||||
<div class="stage" id="stage">
|
||||
<video id="cam" autoplay muted playsinline></video>
|
||||
<div id="board"></div>
|
||||
<canvas id="overlay"></canvas>
|
||||
</div>
|
||||
|
||||
<div class="controls">
|
||||
<button id="start">Start camera</button>
|
||||
<button id="stop" class="ghost" disabled>Stop</button>
|
||||
<button id="fullscreen" class="ghost" title="Toggle fullscreen (Esc to exit)">⛶ Fullscreen</button>
|
||||
<select id="camera" disabled title="Choose camera (populated after start)">
|
||||
<option>Default camera</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="hud">
|
||||
<span><span class="label">FPS</span><b id="fps">0</b></span>
|
||||
<span><span class="label">Hands</span><b id="hand">no</b></span>
|
||||
<span><span class="label">Pinch</span><b id="pinch">—</b></span>
|
||||
<span><span class="label">Gesture</span><b id="gesture">—</b></span>
|
||||
</div>
|
||||
|
||||
<p id="status-msg">Click “Start camera” to begin (camera access requires a click).</p>
|
||||
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "codeman-gesture-control",
|
||||
"private": true,
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"description": "Jarvis-style hand-tracking input layer for the Codeman dashboard. Vendored into the Codeman repo; the Codeman bundle (src/codeman/entry.ts) is built into src/web/public/gesture/gesture-codeman.js by `npm run build:gesture` at the repo root.",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"preview": "vite preview",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"build:demo": "tsc && vite build",
|
||||
"build:codeman": "esbuild src/codeman/entry.ts --bundle --format=esm --target=es2020 --outfile=dist-codeman/gesture-codeman.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mediapipe/tasks-vision": "0.10.21"
|
||||
},
|
||||
"devDependencies": {
|
||||
"esbuild": "^0.27.3",
|
||||
"typescript": "^5.5.4",
|
||||
"vite": "^7.3.5"
|
||||
},
|
||||
"homepage": "https://github.com/Ark0N/Codeman/tree/master/packages/gesture-control#readme",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Ark0N/Codeman.git",
|
||||
"directory": "packages/gesture-control"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
// Phase 5 — Codeman integration entry point.
|
||||
//
|
||||
// This is the *consumer* layer that replaces `demo/tabs.ts` for the real
|
||||
// Codeman dashboard. It is bundled (esbuild, MediaPipe included) into a single
|
||||
// ESM file and served by Codeman from `/gesture/gesture-codeman.js`, loaded into
|
||||
// the dashboard page when Codeman is started with `CODEMAN_GESTURE=1`.
|
||||
//
|
||||
// The gesture *core* (`../gesture/*`) is unchanged and transport-agnostic — it
|
||||
// emits coordinate-only `grab`/`drag`/`drop`. Here we map those onto Codeman's
|
||||
// real session tabs (`.session-tab[data-id]`) and toolbar buttons.
|
||||
//
|
||||
// Three interactions, all off the same pinch:
|
||||
// • Tab "grab-to-float" — pinch a session tab, *pull it out* of the strip (a
|
||||
// ghost follows your hand), release past a threshold → the session opens as
|
||||
// an in-page floating panel at the drop point; a small twitch-and-release
|
||||
// cancels (snaps back).
|
||||
// • Panel "re-grab" — pinch an existing floating panel and move it anywhere;
|
||||
// release over the tab strip to re-dock it (panel goes away, the tab stays).
|
||||
// This is the capability the old OS-window detach lost.
|
||||
// • Button "tap" — pinch over a toolbar button (Run / Run Shell) and release
|
||||
// in place → fires the button's real click handler. Drift too far first and
|
||||
// it's treated as a stray move, not a tap.
|
||||
//
|
||||
// Why in-page floats, not OS-window detach (decided 2026-06-08, see
|
||||
// docs/MULTIMONITOR_DESIGN.md): the hand only exists in the one page that owns
|
||||
// the camera. `window.open`/`detachSession` puts the session in a sealed OS
|
||||
// window the page can't hand-track — a detached session can never be moved
|
||||
// again, a one-way trip. A floating panel (an iframe of the session's solo
|
||||
// route `/session/:id`) stays in this page's DOM, so the hand keeps control:
|
||||
// re-grab, move across the (spanned) viewport, drop, re-dock. The OS-window
|
||||
// detach is kept for later as a *separate, deliberate* verb — no longer the pinch.
|
||||
|
||||
import { GestureController } from '../gesture/GestureController.ts';
|
||||
import type { HandState } from '../gesture/types.ts';
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
__codemanGesture?: GestureBridge;
|
||||
}
|
||||
}
|
||||
|
||||
const TAB_SELECTOR = '.session-tab';
|
||||
/** An in-page floating session panel this layer spawned — re-grabbable to move. */
|
||||
const PANEL_SELECTOR = '.cg-float';
|
||||
/** The session-tab strip; dropping a moved panel over it re-docks the session. */
|
||||
const DOCK_SELECTOR = '.session-tabs';
|
||||
/** Toolbar buttons a pinch can "tap": Run (#runBtn → app.run()) and Run Shell
|
||||
* (.btn-shell → app.runShell()). Pinch over one and release in place to fire
|
||||
* it. Extend this list to expose more buttons to the gesture layer. */
|
||||
const CLICK_SELECTOR = '#runBtn, .btn-shell';
|
||||
const Z = 2147483000; // above the dashboard, below nothing that matters at the desk
|
||||
/** Floating-panel size (px). Fixed for the MVP; resize is a later affordance. */
|
||||
const FLOAT_W = 640;
|
||||
const FLOAT_H = 420;
|
||||
/** Minimum pull distance (px) from the grab point before a release floats the
|
||||
* tab out. Below this it's an accidental pinch and the tab snaps back. */
|
||||
const DETACH_PULL_PX = 70;
|
||||
/** If a button-pinch drifts more than this, it's a stray move, not a tap. */
|
||||
const TAP_CANCEL_PX = 45;
|
||||
|
||||
/** First letter colours: cyan left, violet right; green while pinching. */
|
||||
const handColor = (handedness: string, pinching: boolean): string =>
|
||||
pinching ? '#4ade80' : handedness === 'Right' ? '#a78bfa' : '#38bdf8';
|
||||
|
||||
/** A floating in-page session panel (an iframe of `/session/:id`) the hand can
|
||||
* place and re-grab. Stays in this page's DOM, so it never leaves hand reach. */
|
||||
interface FloatingPanel {
|
||||
id: string;
|
||||
/** The `.cg-float` container element. */
|
||||
el: HTMLElement;
|
||||
}
|
||||
|
||||
/** Live state for one hand's in-progress grab — either a session *tab* being
|
||||
* pulled out into a new float, or an existing *panel* being moved/re-docked. */
|
||||
type Grab =
|
||||
| {
|
||||
kind: 'tab';
|
||||
id: string;
|
||||
tab: HTMLElement;
|
||||
ghost: HTMLElement;
|
||||
/** Grab origin in viewport px, to measure pull distance. */
|
||||
ox: number;
|
||||
oy: number;
|
||||
/** Pulled past the float-out threshold at least once. */
|
||||
armed: boolean;
|
||||
}
|
||||
| {
|
||||
kind: 'panel';
|
||||
id: string;
|
||||
panel: FloatingPanel;
|
||||
/** Cursor→panel-top-left offset at grab, so it doesn't snap when re-grabbed. */
|
||||
dx: number;
|
||||
dy: number;
|
||||
/** Cursor currently over the tab strip → releasing re-docks. */
|
||||
overDock: boolean;
|
||||
};
|
||||
|
||||
/** Live state for one hand pinching a toolbar button (Run / Run Shell). */
|
||||
interface Tap {
|
||||
el: HTMLElement;
|
||||
label: string;
|
||||
ox: number;
|
||||
oy: number;
|
||||
}
|
||||
|
||||
class GestureBridge {
|
||||
private readonly surface: HTMLDivElement;
|
||||
private readonly canvas: HTMLCanvasElement;
|
||||
private readonly ctx: CanvasRenderingContext2D;
|
||||
private readonly video: HTMLVideoElement;
|
||||
private readonly button: HTMLButtonElement;
|
||||
private readonly camBtn: HTMLButtonElement;
|
||||
private readonly status: HTMLSpanElement;
|
||||
private readonly gc: GestureController;
|
||||
private running = false;
|
||||
/** Camera view: full-viewport dimmed background, or small corner preview. */
|
||||
private camMode: 'full' | 'pip' = 'full';
|
||||
|
||||
/** Per-hand in-progress grab (a tab being pulled out, or a panel being moved). */
|
||||
private grabs = new Map<string, Grab>();
|
||||
/** Per-hand in-progress button pinch (fires on release if it didn't drift). */
|
||||
private taps = new Map<string, Tap>();
|
||||
/** Live floating panels, keyed by session id (idempotent per id). */
|
||||
private floats = new Map<string, FloatingPanel>();
|
||||
|
||||
constructor() {
|
||||
injectStyles();
|
||||
|
||||
// Full-viewport, click-through surface so coords map straight to viewport
|
||||
// pixels and elementFromPoint() sees the tabs beneath, not our overlay.
|
||||
this.surface = el('div', 'cg-surface') as HTMLDivElement;
|
||||
this.canvas = el('canvas', 'cg-canvas') as HTMLCanvasElement;
|
||||
this.video = el('video', 'cg-preview') as HTMLVideoElement;
|
||||
this.video.muted = true;
|
||||
this.video.playsInline = true;
|
||||
this.applyCamMode();
|
||||
|
||||
const dock = el('div', 'cg-dock');
|
||||
this.button = el('button', 'cg-btn') as HTMLButtonElement;
|
||||
this.button.textContent = '🖐 Gesture';
|
||||
this.camBtn = el('button', 'cg-btn cg-btn-icon') as HTMLButtonElement;
|
||||
this.camBtn.textContent = '⛶';
|
||||
this.camBtn.title = 'Toggle camera size (fullscreen / corner)';
|
||||
this.status = el('span', 'cg-status') as HTMLSpanElement;
|
||||
this.status.textContent = 'off';
|
||||
dock.append(this.button, this.camBtn, this.status);
|
||||
|
||||
document.body.append(this.surface, this.video, this.canvas, dock);
|
||||
this.ctx = this.canvas.getContext('2d')!;
|
||||
this.sizeCanvas();
|
||||
window.addEventListener('resize', () => this.sizeCanvas());
|
||||
|
||||
this.gc = new GestureController({
|
||||
video: this.video,
|
||||
surface: this.surface,
|
||||
numHands: 1,
|
||||
// Self-host the MediaPipe runtime + model from Codeman (same-origin) instead
|
||||
// of the CDN, so an ad/content blocker, offline desk, or strict browser
|
||||
// can't break startup (the CDN failure surfaced as `failed: {isTrusted}` —
|
||||
// a resource load-error Event). Served from public/gesture/.
|
||||
wasmBase: '/gesture/wasm',
|
||||
modelUrl: '/gesture/gesture_recognizer.task',
|
||||
});
|
||||
|
||||
this.gc.on('grab', (p) => this.onGrab(p.hand, p.x, p.y));
|
||||
this.gc.on('drag', (p) => this.onDrag(p.hand, p.x, p.y));
|
||||
this.gc.on('drop', (p) => this.onDrop(p.hand, p.x, p.y));
|
||||
this.gc.on('status', ({ fps, hands }) => this.onStatus(fps, hands));
|
||||
|
||||
this.button.addEventListener('click', () => void this.toggle());
|
||||
this.camBtn.addEventListener('click', () => this.toggleCamMode());
|
||||
}
|
||||
|
||||
private async toggle(): Promise<void> {
|
||||
if (this.running) {
|
||||
this.gc.stop();
|
||||
this.running = false;
|
||||
this.cancelAllGrabs();
|
||||
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
||||
this.button.classList.remove('on');
|
||||
this.status.textContent = 'off';
|
||||
return;
|
||||
}
|
||||
this.button.disabled = true;
|
||||
this.status.textContent = 'starting…';
|
||||
try {
|
||||
await this.gc.start();
|
||||
this.running = true;
|
||||
this.button.classList.add('on');
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
} catch (err) {
|
||||
// Surface the *real* cause: MediaPipe/Emscripten can throw a non-Error
|
||||
// (number/string), so `(err as Error).message` was logging "undefined".
|
||||
const msg = describeError(err);
|
||||
this.status.textContent = `failed: ${msg}`;
|
||||
this.status.title = msg;
|
||||
console.error('[gesture] start failed', err);
|
||||
} finally {
|
||||
this.button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
private toggleCamMode(): void {
|
||||
this.camMode = this.camMode === 'full' ? 'pip' : 'full';
|
||||
this.applyCamMode();
|
||||
}
|
||||
|
||||
private applyCamMode(): void {
|
||||
this.video.classList.toggle('cg-full', this.camMode === 'full');
|
||||
this.video.classList.toggle('cg-pip', this.camMode === 'pip');
|
||||
}
|
||||
|
||||
/** Top-most element matching `sel` at a viewport point (overlays are
|
||||
* click-through, so elementFromPoint sees the dashboard beneath). */
|
||||
private hitClosest(x: number, y: number, sel: string): HTMLElement | null {
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
return (hit?.closest(sel) as HTMLElement | null) ?? null;
|
||||
}
|
||||
|
||||
private onGrab(hand: string, x: number, y: number): void {
|
||||
// An existing floating panel → re-grab to move it (priority over tabs).
|
||||
// Make it click-through while held so elementFromPoint sees the dock zone
|
||||
// (and other content) beneath it, and it can't re-grab itself.
|
||||
const panelEl = this.hitClosest(x, y, PANEL_SELECTOR);
|
||||
const panelId = panelEl?.dataset.id;
|
||||
if (panelEl && panelId) {
|
||||
const float = this.floats.get(panelId);
|
||||
if (float) {
|
||||
const rect = panelEl.getBoundingClientRect();
|
||||
panelEl.style.pointerEvents = 'none';
|
||||
panelEl.classList.add('cg-float-grabbed');
|
||||
this.grabs.set(hand, {
|
||||
kind: 'panel',
|
||||
id: panelId,
|
||||
panel: float,
|
||||
dx: x - rect.left,
|
||||
dy: y - rect.top,
|
||||
overDock: false,
|
||||
});
|
||||
this.status.textContent = 'moving — drop over tabs to re-dock';
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// A session tab → grab-and-pull-out into a floating panel (ghost follows).
|
||||
const tab = this.hitClosest(x, y, TAB_SELECTOR);
|
||||
const id = tab?.dataset.id;
|
||||
if (tab && id) {
|
||||
const rect = tab.getBoundingClientRect();
|
||||
const ghost = tab.cloneNode(true) as HTMLElement;
|
||||
ghost.classList.add('cg-ghost');
|
||||
ghost.removeAttribute('id');
|
||||
ghost.style.width = `${rect.width}px`;
|
||||
ghost.style.height = `${rect.height}px`;
|
||||
document.body.append(ghost);
|
||||
|
||||
tab.classList.add('cg-grabbed');
|
||||
this.grabs.set(hand, { kind: 'tab', id, tab, ghost, ox: x, oy: y, armed: false });
|
||||
this.positionGhost(ghost, x, y);
|
||||
return;
|
||||
}
|
||||
|
||||
// A toolbar button (Run / Run Shell) → tap-to-fire on release.
|
||||
const btn = this.hitClosest(x, y, CLICK_SELECTOR);
|
||||
if (btn) {
|
||||
const label = (btn.textContent || btn.getAttribute('title') || 'button').trim();
|
||||
btn.classList.add('cg-tap-armed');
|
||||
this.taps.set(hand, { el: btn, label, ox: x, oy: y });
|
||||
this.status.textContent = `release to ${label.toLowerCase()}`;
|
||||
}
|
||||
}
|
||||
|
||||
private onDrag(hand: string, x: number, y: number): void {
|
||||
const grab = this.grabs.get(hand);
|
||||
if (grab?.kind === 'tab') {
|
||||
this.positionGhost(grab.ghost, x, y);
|
||||
const pulled = Math.hypot(x - grab.ox, y - grab.oy) >= DETACH_PULL_PX;
|
||||
if (pulled !== grab.armed) {
|
||||
grab.armed = pulled;
|
||||
grab.ghost.classList.toggle('cg-armed', pulled);
|
||||
this.status.textContent = pulled ? 'release to float out' : 'on — pinch a tab';
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'panel') {
|
||||
this.moveFloat(grab.panel, x - grab.dx, y - grab.dy);
|
||||
const overDock = !!this.hitClosest(x, y, DOCK_SELECTOR);
|
||||
if (overDock !== grab.overDock) {
|
||||
grab.overDock = overDock;
|
||||
grab.panel.el.classList.toggle('cg-redock', overDock);
|
||||
this.status.textContent = overDock ? 'release to re-dock' : 'moving panel';
|
||||
}
|
||||
return;
|
||||
}
|
||||
// A button pinch that drifts too far is a stray move, not a tap — cancel it.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap && Math.hypot(x - tap.ox, y - tap.oy) > TAP_CANCEL_PX) {
|
||||
tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.delete(hand);
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
}
|
||||
}
|
||||
|
||||
private onDrop(hand: string, x: number, y: number): void {
|
||||
const grab = this.grabs.get(hand);
|
||||
if (grab?.kind === 'tab') {
|
||||
this.grabs.delete(hand);
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove('cg-grabbed');
|
||||
if (grab.armed) this.floatPanel(grab.id, x, y);
|
||||
else this.flash('cancelled');
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'panel') {
|
||||
this.grabs.delete(hand);
|
||||
grab.panel.el.style.pointerEvents = ''; // interactive again (type into it)
|
||||
grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock');
|
||||
if (grab.overDock) this.redock(grab.id);
|
||||
else this.flash('placed');
|
||||
return;
|
||||
}
|
||||
// Release over the same button → fire its real click handler.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap) {
|
||||
this.taps.delete(hand);
|
||||
tap.el.classList.remove('cg-tap-armed');
|
||||
tap.el.click(); // runs the button's onclick (app.run() / app.runShell())
|
||||
this.flash(tap.label.toLowerCase());
|
||||
}
|
||||
}
|
||||
|
||||
/** Pop a session into an in-page floating panel (an iframe of its solo route)
|
||||
* centered on the drop point. Unlike OS-window detach, the panel lives in
|
||||
* this page's DOM, so the hand can re-grab and move it. Idempotent per id:
|
||||
* re-floating an existing id just repositions the panel it already has. */
|
||||
private floatPanel(id: string, x: number, y: number): void {
|
||||
let float = this.floats.get(id);
|
||||
if (!float) {
|
||||
const container = el('div', 'cg-float');
|
||||
container.dataset.id = id;
|
||||
const bar = el('div', 'cg-float-bar');
|
||||
bar.textContent = `session ${id}`;
|
||||
const frame = el('iframe', 'cg-float-frame') as HTMLIFrameElement;
|
||||
frame.src = `/session/${encodeURIComponent(id)}`;
|
||||
frame.title = `Session ${id}`;
|
||||
container.append(bar, frame);
|
||||
document.body.append(container);
|
||||
float = { id, el: container };
|
||||
this.floats.set(id, float);
|
||||
this.flash('floated out');
|
||||
} else {
|
||||
this.flash('re-floated');
|
||||
}
|
||||
this.moveFloat(float, x - FLOAT_W / 2, y - FLOAT_H / 2);
|
||||
}
|
||||
|
||||
/** Re-dock a floated session: drop its in-page panel. The original
|
||||
* `.session-tab` was never removed, so the session is simply back to plain
|
||||
* tab form. Mirrors Codeman's own `.detached` → attach toggle. */
|
||||
private redock(id: string): void {
|
||||
const float = this.floats.get(id);
|
||||
if (!float) return;
|
||||
float.el.remove();
|
||||
this.floats.delete(id);
|
||||
this.flash('re-docked');
|
||||
}
|
||||
|
||||
/** Position a float by its top-left corner, clamped to stay on-screen. */
|
||||
private moveFloat(float: FloatingPanel, left: number, top: number): void {
|
||||
const l = Math.min(Math.max(0, left), Math.max(0, window.innerWidth - FLOAT_W));
|
||||
const t = Math.min(Math.max(0, top), Math.max(0, window.innerHeight - FLOAT_H));
|
||||
float.el.style.left = `${l}px`;
|
||||
float.el.style.top = `${t}px`;
|
||||
}
|
||||
|
||||
private positionGhost(ghost: HTMLElement, x: number, y: number): void {
|
||||
ghost.style.left = `${x}px`;
|
||||
ghost.style.top = `${y}px`;
|
||||
}
|
||||
|
||||
private cancelAllGrabs(): void {
|
||||
// Floats themselves persist (they're placed windows) — only release any
|
||||
// in-progress grab cleanly, restoring a moved panel's interactivity.
|
||||
for (const grab of this.grabs.values()) {
|
||||
if (grab.kind === 'tab') {
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove('cg-grabbed');
|
||||
} else {
|
||||
grab.panel.el.style.pointerEvents = '';
|
||||
grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock');
|
||||
}
|
||||
}
|
||||
this.grabs.clear();
|
||||
for (const tap of this.taps.values()) tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.clear();
|
||||
document
|
||||
.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`)
|
||||
.forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed'));
|
||||
}
|
||||
|
||||
private onStatus(fps: number, hands: HandState[]): void {
|
||||
// Draw per-hand cursor dots (green while pinching) so the user can aim.
|
||||
const { width, height } = this.canvas;
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
this.ctx.clearRect(0, 0, width, height);
|
||||
const rect = this.surface.getBoundingClientRect();
|
||||
for (const h of hands) {
|
||||
const x = (1 - h.cursor.x) * rect.width;
|
||||
const y = h.cursor.y * rect.height;
|
||||
this.ctx.beginPath();
|
||||
this.ctx.arc(x * dpr, y * dpr, (h.pinching ? 14 : 9) * dpr, 0, Math.PI * 2);
|
||||
this.ctx.fillStyle = handColor(h.handedness, h.pinching);
|
||||
this.ctx.globalAlpha = 0.85;
|
||||
this.ctx.fill();
|
||||
this.ctx.globalAlpha = 1;
|
||||
}
|
||||
if (this.running && this.grabs.size === 0 && this.taps.size === 0) {
|
||||
this.status.textContent = `on · ${fps}fps`;
|
||||
}
|
||||
}
|
||||
|
||||
private flash(msg: string): void {
|
||||
this.status.textContent = msg;
|
||||
}
|
||||
|
||||
private sizeCanvas(): void {
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
this.canvas.width = Math.floor(window.innerWidth * dpr);
|
||||
this.canvas.height = Math.floor(window.innerHeight * dpr);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- tiny helpers ------------------------------------------------------
|
||||
|
||||
function el(tag: string, className: string): HTMLElement {
|
||||
const node = document.createElement(tag);
|
||||
node.className = className;
|
||||
return node;
|
||||
}
|
||||
|
||||
/** Best-effort human-readable message for any thrown value (Error or not). */
|
||||
function describeError(err: unknown): string {
|
||||
if (err instanceof Error) return err.message || err.name || 'Error';
|
||||
if (typeof err === 'string') return err;
|
||||
if (typeof err === 'number') return `code ${err}`;
|
||||
if (err && typeof err === 'object') {
|
||||
const m = (err as { message?: unknown }).message;
|
||||
if (typeof m === 'string' && m) return m;
|
||||
try {
|
||||
return JSON.stringify(err);
|
||||
} catch {
|
||||
return Object.prototype.toString.call(err);
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
function injectStyles(): void {
|
||||
if (document.getElementById('cg-styles')) return;
|
||||
const css = `
|
||||
.cg-surface, .cg-canvas { position: fixed; inset: 0; pointer-events: none; }
|
||||
.cg-surface { z-index: ${Z}; }
|
||||
.cg-canvas { z-index: ${Z + 1}; width: 100vw; height: 100vh; }
|
||||
.cg-preview { transform: scaleX(-1); pointer-events: none; background: #000; }
|
||||
.cg-preview.cg-pip {
|
||||
position: fixed; right: 12px; bottom: 12px; width: 240px; height: 135px;
|
||||
object-fit: cover; border-radius: 8px; z-index: ${Z + 2};
|
||||
box-shadow: 0 4px 16px rgba(0,0,0,.5); opacity: 1;
|
||||
}
|
||||
.cg-preview.cg-full {
|
||||
position: fixed; inset: 0; width: 100vw; height: 100vh;
|
||||
object-fit: cover; opacity: .28; z-index: ${Z};
|
||||
}
|
||||
.cg-ghost {
|
||||
position: fixed; left: 0; top: 0; transform: translate(-50%, -50%) scale(1.06);
|
||||
z-index: ${Z + 2}; pointer-events: none; opacity: .92;
|
||||
box-shadow: 0 8px 28px rgba(0,0,0,.55); border-radius: 8px;
|
||||
outline: 2px solid #38bdf8; outline-offset: -2px;
|
||||
}
|
||||
.cg-ghost.cg-armed { outline-color: #4ade80; box-shadow: 0 8px 28px rgba(74,222,128,.5); }
|
||||
.cg-dock {
|
||||
position: fixed; right: 12px; bottom: 156px; z-index: ${Z + 3};
|
||||
display: flex; align-items: center; gap: 8px; font: 12px/1 system-ui, sans-serif;
|
||||
}
|
||||
.cg-btn {
|
||||
padding: 6px 12px; border-radius: 6px; border: 1px solid #3a3a40;
|
||||
background: #1b1b1f; color: #e5e5e7; cursor: pointer;
|
||||
}
|
||||
.cg-btn-icon { padding: 6px 9px; }
|
||||
.cg-btn.on { background: #16331f; border-color: #2f6b41; color: #4ade80; }
|
||||
.cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; }
|
||||
.cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; }
|
||||
.cg-float {
|
||||
position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px;
|
||||
z-index: ${Z}; display: flex; flex-direction: column; overflow: hidden;
|
||||
background: #0c0c0f; border-radius: 10px; outline: 2px solid #38bdf8;
|
||||
outline-offset: -2px; box-shadow: 0 10px 40px rgba(0,0,0,.6);
|
||||
}
|
||||
.cg-float.cg-float-grabbed { outline-color: #4ade80; box-shadow: 0 12px 48px rgba(74,222,128,.45); }
|
||||
.cg-float.cg-redock { outline-color: #fbbf24; box-shadow: 0 12px 48px rgba(251,191,36,.5); }
|
||||
.cg-float-bar {
|
||||
flex: 0 0 auto; padding: 4px 10px; font: 11px/1.6 system-ui, sans-serif;
|
||||
color: #cbd5e1; background: #15151a; border-bottom: 1px solid #2a2a30;
|
||||
user-select: none; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
.cg-float-frame { flex: 1 1 auto; width: 100%; border: 0; background: #000; }
|
||||
`;
|
||||
const style = document.createElement('style');
|
||||
style.id = 'cg-styles';
|
||||
style.textContent = css;
|
||||
document.head.append(style);
|
||||
}
|
||||
|
||||
// Idempotent bootstrap — re-importing must not stack overlays.
|
||||
if (!window.__codemanGesture) {
|
||||
window.__codemanGesture = new GestureBridge();
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Debug overlay: draws the 21-point hand skeleton over the (mirrored) video.
|
||||
//
|
||||
// The <video> is mirrored via CSS (transform: scaleX(-1)). To make the drawn
|
||||
// skeleton line up with what you see, we mirror the X coordinate here
|
||||
// (x_draw = (1 - x) * width) rather than CSS-mirroring the canvas — that keeps
|
||||
// any future text we draw readable.
|
||||
|
||||
import type { GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
import { HAND_CONNECTIONS } from '../gesture/landmarks.ts';
|
||||
|
||||
/** A cursor to draw: raw normalized position + base color + pinch state. */
|
||||
export interface CursorMark {
|
||||
x: number;
|
||||
y: number;
|
||||
color: string;
|
||||
pinching: boolean;
|
||||
}
|
||||
|
||||
export class Overlay {
|
||||
private ctx: CanvasRenderingContext2D;
|
||||
|
||||
constructor(
|
||||
private canvas: HTMLCanvasElement,
|
||||
private video: HTMLVideoElement
|
||||
) {
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error('Could not get 2D context for overlay canvas');
|
||||
this.ctx = ctx;
|
||||
}
|
||||
|
||||
/** Match the canvas backing-store resolution to the displayed video size. */
|
||||
private syncSize(): void {
|
||||
const w = this.video.videoWidth || this.video.clientWidth;
|
||||
const h = this.video.videoHeight || this.video.clientHeight;
|
||||
if (w && h && (this.canvas.width !== w || this.canvas.height !== h)) {
|
||||
this.canvas.width = w;
|
||||
this.canvas.height = h;
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw the detected hands' skeletons plus one smoothed cursor per hand.
|
||||
* Cursor coords are raw normalized [0,1] (mirrored here, like the skeleton).
|
||||
*/
|
||||
draw(result: GestureRecognizerResult, cursors: CursorMark[] = []): void {
|
||||
this.syncSize();
|
||||
const { width: w, height: h } = this.canvas;
|
||||
const ctx = this.ctx;
|
||||
ctx.clearRect(0, 0, w, h);
|
||||
|
||||
const hands = result.landmarks ?? [];
|
||||
for (const landmarks of hands) {
|
||||
// Connections (bones)
|
||||
ctx.strokeStyle = 'rgba(80, 220, 255, 0.9)';
|
||||
ctx.lineWidth = 3;
|
||||
for (const [a, b] of HAND_CONNECTIONS) {
|
||||
const pa = landmarks[a];
|
||||
const pb = landmarks[b];
|
||||
if (!pa || !pb) continue;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo((1 - pa.x) * w, pa.y * h);
|
||||
ctx.lineTo((1 - pb.x) * w, pb.y * h);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
// Joints (points)
|
||||
for (let i = 0; i < landmarks.length; i++) {
|
||||
const p = landmarks[i];
|
||||
const x = (1 - p.x) * w;
|
||||
const y = p.y * h;
|
||||
// Highlight thumb tip (4) + index tip (8) — these drive the cursor later.
|
||||
const isPinchPoint = i === 4 || i === 8;
|
||||
ctx.fillStyle = isPinchPoint ? '#ffd166' : '#ff5d8f';
|
||||
ctx.beginPath();
|
||||
ctx.arc(x, y, isPinchPoint ? 7 : 4, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
}
|
||||
|
||||
for (const c of cursors) this.drawCursor(c, w, h);
|
||||
}
|
||||
|
||||
/** A ring + crosshair at the cursor; green + filled while pinching. */
|
||||
private drawCursor(cursor: CursorMark, w: number, h: number): void {
|
||||
const ctx = this.ctx;
|
||||
const cx = (1 - cursor.x) * w; // mirror X to match the displayed video
|
||||
const cy = cursor.y * h;
|
||||
const pinching = cursor.pinching;
|
||||
const r = pinching ? 16 : 12;
|
||||
const color = pinching ? '#4ade80' : cursor.color;
|
||||
|
||||
if (pinching) {
|
||||
ctx.fillStyle = 'rgba(74, 222, 128, 0.25)';
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, r, 0, Math.PI * 2);
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
ctx.strokeStyle = color;
|
||||
ctx.lineWidth = 3;
|
||||
ctx.beginPath();
|
||||
ctx.arc(cx, cy, r, 0, Math.PI * 2);
|
||||
ctx.stroke();
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(cx - r - 5, cy);
|
||||
ctx.lineTo(cx - r + 4, cy);
|
||||
ctx.moveTo(cx + r - 4, cy);
|
||||
ctx.lineTo(cx + r + 5, cy);
|
||||
ctx.moveTo(cx, cy - r - 5);
|
||||
ctx.lineTo(cx, cy - r + 4);
|
||||
ctx.moveTo(cx, cy + r - 4);
|
||||
ctx.lineTo(cx, cy + r + 5);
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
// demo/tabs.ts — Phase 3 fake-tab board.
|
||||
//
|
||||
// 3 columns of draggable "session" tabs, driven entirely by gesture events
|
||||
// (no mouse). The board overlays the camera stage, so surface-pixel coords
|
||||
// from GestureController map straight onto board-local coords.
|
||||
//
|
||||
// Source of truth for which tab is in which column is the DOM itself. The
|
||||
// state machine here is tiny: grab → (drag)* → drop, tracked per hand so two
|
||||
// hands can drag two tabs at once.
|
||||
|
||||
interface Tab {
|
||||
id: string;
|
||||
el: HTMLElement;
|
||||
}
|
||||
|
||||
interface Column {
|
||||
id: string;
|
||||
title: string;
|
||||
el: HTMLElement;
|
||||
list: HTMLElement;
|
||||
}
|
||||
|
||||
interface Grab {
|
||||
tab: Tab;
|
||||
originList: HTMLElement;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
/** A hovering cursor for highlight purposes. */
|
||||
export interface HoverPoint {
|
||||
x: number;
|
||||
y: number;
|
||||
pinching: boolean;
|
||||
}
|
||||
|
||||
const INITIAL: Array<{ id: string; title: string; tabs: string[] }> = [
|
||||
{ id: 'screen-1', title: 'Screen 1', tabs: ['auth-refactor', 'api-tests'] },
|
||||
{ id: 'screen-2', title: 'Screen 2', tabs: ['db-migrate', 'ui-polish', 'docs'] },
|
||||
{ id: 'screen-3', title: 'Screen 3', tabs: ['ci-fix'] },
|
||||
];
|
||||
|
||||
export class TabsBoard {
|
||||
private columns: Column[] = [];
|
||||
private tabs = new Map<string, Tab>();
|
||||
/** hand id → the tab it is currently dragging. */
|
||||
private grabs = new Map<string, Grab>();
|
||||
|
||||
/** @param onDrop notified after a settled drop: (tab, column or null if cancelled). */
|
||||
constructor(
|
||||
private root: HTMLElement,
|
||||
private onDrop?: (tabId: string, columnId: string | null) => void
|
||||
) {
|
||||
this.build();
|
||||
}
|
||||
|
||||
private build(): void {
|
||||
this.root.classList.add('board');
|
||||
for (const col of INITIAL) {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'column';
|
||||
el.dataset.col = col.id;
|
||||
|
||||
const header = document.createElement('header');
|
||||
header.textContent = col.title;
|
||||
|
||||
const list = document.createElement('div');
|
||||
list.className = 'tablist';
|
||||
|
||||
el.append(header, list);
|
||||
this.root.append(el);
|
||||
this.columns.push({ id: col.id, title: col.title, el, list });
|
||||
|
||||
for (const id of col.tabs) {
|
||||
const tab = this.makeTab(id);
|
||||
list.append(tab.el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private makeTab(id: string): Tab {
|
||||
const el = document.createElement('div');
|
||||
el.className = 'tab';
|
||||
el.dataset.tab = id;
|
||||
el.textContent = id;
|
||||
const tab: Tab = { id, el };
|
||||
this.tabs.set(id, tab);
|
||||
return tab;
|
||||
}
|
||||
|
||||
// ---- Geometry --------------------------------------------------------
|
||||
|
||||
/** Element rect in board-local coords (origin = board top-left). */
|
||||
private localRect(el: HTMLElement): DOMRect {
|
||||
const r = el.getBoundingClientRect();
|
||||
const base = this.root.getBoundingClientRect();
|
||||
return new DOMRect(r.left - base.left, r.top - base.top, r.width, r.height);
|
||||
}
|
||||
|
||||
/** Hit-test slop (px). Cursor jitter + a small target shouldn't fight you. */
|
||||
private static readonly GRAB_PAD = 18;
|
||||
|
||||
private static contains(r: DOMRect, x: number, y: number, pad = 0): boolean {
|
||||
return x >= r.left - pad && x <= r.right + pad && y >= r.top - pad && y <= r.bottom + pad;
|
||||
}
|
||||
|
||||
private columnAt(x: number, y: number): Column | null {
|
||||
for (const c of this.columns) {
|
||||
if (TabsBoard.contains(this.localRect(c.el), x, y)) return c;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Topmost ungrabbed tab under the point. Nearest-center wins ties so the
|
||||
* padded hit-areas of adjacent tabs resolve to the most likely target. */
|
||||
private tabAt(x: number, y: number): Tab | null {
|
||||
let best: Tab | null = null;
|
||||
let bestDist = Infinity;
|
||||
for (const tab of this.tabs.values()) {
|
||||
if (this.isGrabbed(tab.id)) continue;
|
||||
const r = this.localRect(tab.el);
|
||||
if (!TabsBoard.contains(r, x, y, TabsBoard.GRAB_PAD)) continue;
|
||||
const cx = r.left + r.width / 2;
|
||||
const cy = r.top + r.height / 2;
|
||||
const d = (x - cx) ** 2 + (y - cy) ** 2;
|
||||
if (d < bestDist) {
|
||||
bestDist = d;
|
||||
best = tab;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
private isGrabbed(tabId: string): boolean {
|
||||
for (const g of this.grabs.values()) if (g.tab.id === tabId) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Highlights (driven each frame from the status snapshot) ---------
|
||||
|
||||
/** Recompute hover highlights from the full set of present cursors. */
|
||||
hover(points: HoverPoint[]): void {
|
||||
this.root.querySelectorAll('.col-hot, .tab-hot').forEach((el) => el.classList.remove('col-hot', 'tab-hot'));
|
||||
|
||||
for (const p of points) {
|
||||
this.columnAt(p.x, p.y)?.el.classList.add('col-hot');
|
||||
// Only show a grab affordance when the hand is open (not pinching).
|
||||
if (!p.pinching) this.tabAt(p.x, p.y)?.el.classList.add('tab-hot');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Drag state machine ---------------------------------------------
|
||||
|
||||
grab(hand: string, x: number, y: number): void {
|
||||
if (this.grabs.has(hand)) return;
|
||||
const tab = this.tabAt(x, y);
|
||||
if (!tab) return;
|
||||
|
||||
const rect = this.localRect(tab.el);
|
||||
this.grabs.set(hand, {
|
||||
tab,
|
||||
originList: tab.el.parentElement as HTMLElement,
|
||||
w: rect.width,
|
||||
h: rect.height,
|
||||
});
|
||||
tab.el.classList.add('dragging');
|
||||
tab.el.style.width = `${rect.width}px`;
|
||||
// Reparent the floating tab onto the board root before positioning it.
|
||||
// A `.column` can't be the containing block: its `backdrop-filter` makes it
|
||||
// the containing block for absolutely-positioned children, so our
|
||||
// board-local left/top would be offset by the column's own position (tabs
|
||||
// in the middle/right columns flew off to the right). #board has no
|
||||
// filter/transform, so it's a stable origin that matches moveTo's coords.
|
||||
this.root.append(tab.el);
|
||||
this.moveTo(tab.el, x, y, rect.width, rect.height);
|
||||
}
|
||||
|
||||
drag(hand: string, x: number, y: number): void {
|
||||
const g = this.grabs.get(hand);
|
||||
if (!g) return;
|
||||
this.moveTo(g.tab.el, x, y, g.w, g.h);
|
||||
}
|
||||
|
||||
drop(hand: string, x: number, y: number): void {
|
||||
const g = this.grabs.get(hand);
|
||||
if (!g) return;
|
||||
this.grabs.delete(hand);
|
||||
|
||||
const target = this.columnAt(x, y);
|
||||
const dest = target ? target.list : g.originList;
|
||||
dest.append(g.tab.el);
|
||||
|
||||
g.tab.el.classList.remove('dragging');
|
||||
g.tab.el.style.removeProperty('width');
|
||||
g.tab.el.style.removeProperty('left');
|
||||
g.tab.el.style.removeProperty('top');
|
||||
|
||||
this.onDrop?.(g.tab.id, target ? target.id : null);
|
||||
}
|
||||
|
||||
/** Center the floating tab on the cursor (clamped to the board). */
|
||||
private moveTo(el: HTMLElement, x: number, y: number, w: number, h: number): void {
|
||||
const base = this.root.getBoundingClientRect();
|
||||
const left = Math.max(0, Math.min(x - w / 2, base.width - w));
|
||||
const top = Math.max(0, Math.min(y - h / 2, base.height - h));
|
||||
el.style.left = `${left}px`;
|
||||
el.style.top = `${top}px`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
// GestureController — core input layer.
|
||||
//
|
||||
// Phase 0/1 scope (this file currently implements):
|
||||
// - Open the webcam (getUserMedia) and attach it to a <video>.
|
||||
// - Load MediaPipe GestureRecognizer in VIDEO mode (wasm + .task from CDN).
|
||||
// - Run a requestAnimationFrame loop calling recognizeForVideo().
|
||||
// - Emit `status` (fps / handPresent / gesture) and `results` (raw, debug).
|
||||
//
|
||||
// Later phases add the cursor (One-Euro filtered), pinch hysteresis, the
|
||||
// hover/grab/drag/drop state machine, and the discrete gesture command bus.
|
||||
// The event surface in types.ts already declares those so the API is stable.
|
||||
|
||||
import { FilesetResolver, GestureRecognizer, type GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
import type {
|
||||
GestureControllerOptions,
|
||||
GestureEventHandler,
|
||||
GestureEventMap,
|
||||
GestureEventName,
|
||||
HandState,
|
||||
} from './types.ts';
|
||||
import { LANDMARK, midpoint } from './landmarks.ts';
|
||||
import { OneEuroFilter } from './OneEuroFilter.ts';
|
||||
import { PinchDetector, pinchDistance } from './pinch.ts';
|
||||
import { CommandDetector } from './commands.ts';
|
||||
|
||||
const DEFAULTS = {
|
||||
numHands: 1,
|
||||
deviceId: '',
|
||||
pinchOn: 0.35,
|
||||
pinchOff: 0.5,
|
||||
minCutoff: 1.0,
|
||||
beta: 0.01,
|
||||
palmHoldMs: 1000,
|
||||
minDetectionConfidence: 0.6,
|
||||
minTrackingConfidence: 0.6,
|
||||
// Pinned to the @mediapipe/tasks-vision version in package.json.
|
||||
wasmBase: 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.21/wasm',
|
||||
modelUrl:
|
||||
'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
|
||||
} as const;
|
||||
|
||||
interface PerHandState {
|
||||
cursorX: OneEuroFilter;
|
||||
cursorY: OneEuroFilter;
|
||||
pinch: PinchDetector;
|
||||
/** Original handedness label (the map key may be disambiguated). */
|
||||
label: string;
|
||||
/** Last emitted surface-pixel position, so we can drop on a vanished hand. */
|
||||
lastX: number;
|
||||
lastY: number;
|
||||
}
|
||||
|
||||
export class GestureController {
|
||||
private readonly opts: Required<GestureControllerOptions>;
|
||||
private recognizer: GestureRecognizer | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private rafId: number | null = null;
|
||||
private running = false;
|
||||
|
||||
// Timestamps must be strictly increasing for recognizeForVideo.
|
||||
private lastVideoTime = -1;
|
||||
private lastTimestamp = -1;
|
||||
|
||||
// FPS tracking (rolling over a short window).
|
||||
private frameTimes: number[] = [];
|
||||
|
||||
// Per-hand smoothing + pinch state, keyed by handedness ("Left"/"Right") so a
|
||||
// hand keeps its own filters even when MediaPipe reorders the hands array.
|
||||
private readonly handStates = new Map<string, PerHandState>();
|
||||
|
||||
// Discrete gesture → command bus (debounced; Open_Palm held = halt-all).
|
||||
private readonly commands: CommandDetector;
|
||||
|
||||
// Internal storage is intentionally loose; the public on/off/emit signatures
|
||||
// below keep callers fully type-safe per event name.
|
||||
private listeners: Partial<Record<GestureEventName, Set<(payload: unknown) => void>>> = {};
|
||||
|
||||
constructor(options: GestureControllerOptions) {
|
||||
this.opts = {
|
||||
surface: options.surface ?? options.video,
|
||||
numHands: options.numHands ?? DEFAULTS.numHands,
|
||||
deviceId: options.deviceId ?? DEFAULTS.deviceId,
|
||||
pinchOn: options.pinchOn ?? DEFAULTS.pinchOn,
|
||||
pinchOff: options.pinchOff ?? DEFAULTS.pinchOff,
|
||||
minCutoff: options.minCutoff ?? DEFAULTS.minCutoff,
|
||||
beta: options.beta ?? DEFAULTS.beta,
|
||||
palmHoldMs: options.palmHoldMs ?? DEFAULTS.palmHoldMs,
|
||||
minDetectionConfidence: options.minDetectionConfidence ?? DEFAULTS.minDetectionConfidence,
|
||||
minTrackingConfidence: options.minTrackingConfidence ?? DEFAULTS.minTrackingConfidence,
|
||||
wasmBase: options.wasmBase ?? DEFAULTS.wasmBase,
|
||||
modelUrl: options.modelUrl ?? DEFAULTS.modelUrl,
|
||||
video: options.video,
|
||||
};
|
||||
|
||||
this.commands = new CommandDetector(this.opts.palmHoldMs);
|
||||
}
|
||||
|
||||
/** Get (or lazily create) the smoothing + pinch state for one hand. */
|
||||
private handState(key: string): PerHandState {
|
||||
let state = this.handStates.get(key);
|
||||
if (!state) {
|
||||
state = {
|
||||
cursorX: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
cursorY: new OneEuroFilter(this.opts.minCutoff, this.opts.beta),
|
||||
pinch: new PinchDetector(this.opts.pinchOn, this.opts.pinchOff),
|
||||
label: key,
|
||||
lastX: 0,
|
||||
lastY: 0,
|
||||
};
|
||||
this.handStates.set(key, state);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ---- Event emitter ---------------------------------------------------
|
||||
|
||||
on<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
(this.listeners[event] ??= new Set()).add(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
off<K extends GestureEventName>(event: K, handler: GestureEventHandler<K>): this {
|
||||
this.listeners[event]?.delete(handler as (payload: unknown) => void);
|
||||
return this;
|
||||
}
|
||||
|
||||
private emit<K extends GestureEventName>(event: K, payload: GestureEventMap[K]): void {
|
||||
this.listeners[event]?.forEach((h) => h(payload));
|
||||
}
|
||||
|
||||
// ---- Lifecycle -------------------------------------------------------
|
||||
|
||||
/** Request the camera, load the model, and start the recognition loop. */
|
||||
async start(): Promise<void> {
|
||||
if (this.running) return;
|
||||
|
||||
await this.openCamera();
|
||||
await this.loadRecognizer();
|
||||
|
||||
this.running = true;
|
||||
this.lastVideoTime = -1;
|
||||
this.lastTimestamp = -1;
|
||||
this.frameTimes = [];
|
||||
this.handStates.clear();
|
||||
this.commands.reset();
|
||||
this.loop();
|
||||
}
|
||||
|
||||
/** Stop the loop, release the camera, and close the recognizer. */
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
if (this.rafId !== null) {
|
||||
cancelAnimationFrame(this.rafId);
|
||||
this.rafId = null;
|
||||
}
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
this.opts.video.srcObject = null;
|
||||
this.recognizer?.close();
|
||||
this.recognizer = null;
|
||||
}
|
||||
|
||||
/** The camera currently in use (resolved deviceId), or "" if not started. */
|
||||
get deviceId(): string {
|
||||
return this.opts.deviceId;
|
||||
}
|
||||
|
||||
/** List available video input devices. Labels are only populated once the
|
||||
* user has granted camera permission (i.e. after the first start()). */
|
||||
async listCameras(): Promise<MediaDeviceInfo[]> {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return devices.filter((d) => d.kind === 'videoinput');
|
||||
}
|
||||
|
||||
/** Switch to a different camera. Reopens the stream live if already running. */
|
||||
async useCamera(deviceId: string): Promise<void> {
|
||||
this.opts.deviceId = deviceId;
|
||||
if (!this.running) return;
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
await this.openCamera();
|
||||
// Fresh camera = fresh geometry; drop stale filter state to avoid a snap.
|
||||
this.handStates.clear();
|
||||
this.lastVideoTime = -1;
|
||||
}
|
||||
|
||||
// ---- Setup -----------------------------------------------------------
|
||||
|
||||
private async openCamera(): Promise<void> {
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('getUserMedia is not available (needs a secure context).');
|
||||
}
|
||||
const videoConstraints: MediaTrackConstraints = {
|
||||
width: { ideal: 1280 },
|
||||
height: { ideal: 720 },
|
||||
frameRate: { ideal: 60, max: 60 },
|
||||
};
|
||||
// A specific device wins; otherwise ask for the user-facing camera.
|
||||
if (this.opts.deviceId) videoConstraints.deviceId = { exact: this.opts.deviceId };
|
||||
else videoConstraints.facingMode = 'user';
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: videoConstraints,
|
||||
audio: false,
|
||||
});
|
||||
this.stream = stream;
|
||||
// Record the resolved device so callers can pre-select it in a UI.
|
||||
this.opts.deviceId = stream.getVideoTracks()[0]?.getSettings().deviceId ?? this.opts.deviceId;
|
||||
const video = this.opts.video;
|
||||
video.srcObject = stream;
|
||||
video.muted = true;
|
||||
video.playsInline = true;
|
||||
await video.play();
|
||||
|
||||
// Wait until dimensions are known so the overlay can size itself.
|
||||
if (!video.videoWidth) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const onMeta = () => {
|
||||
video.removeEventListener('loadedmetadata', onMeta);
|
||||
resolve();
|
||||
};
|
||||
video.addEventListener('loadedmetadata', onMeta);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async loadRecognizer(): Promise<void> {
|
||||
const fileset = await FilesetResolver.forVisionTasks(this.opts.wasmBase);
|
||||
const build = (delegate: 'GPU' | 'CPU') =>
|
||||
GestureRecognizer.createFromOptions(fileset, {
|
||||
baseOptions: {
|
||||
modelAssetPath: this.opts.modelUrl,
|
||||
delegate,
|
||||
},
|
||||
runningMode: 'VIDEO',
|
||||
numHands: this.opts.numHands,
|
||||
minHandDetectionConfidence: this.opts.minDetectionConfidence,
|
||||
minHandPresenceConfidence: this.opts.minDetectionConfidence,
|
||||
minTrackingConfidence: this.opts.minTrackingConfidence,
|
||||
});
|
||||
// GPU is the fast path (60fps on the MacBook). But some environments fail to
|
||||
// init the GPU delegate — and MediaPipe/Emscripten often throws a *non-Error*
|
||||
// value there (a raw number/string), which surfaces upstream as a useless
|
||||
// "failed: undefined". Fall back to CPU so the recognizer still starts.
|
||||
try {
|
||||
this.recognizer = await build('GPU');
|
||||
} catch (err) {
|
||||
console.warn('[gesture] GPU delegate failed; falling back to CPU.', err);
|
||||
this.recognizer = await build('CPU');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Loop ------------------------------------------------------------
|
||||
|
||||
private loop = (): void => {
|
||||
if (!this.running || !this.recognizer) return;
|
||||
this.rafId = requestAnimationFrame(this.loop);
|
||||
|
||||
const video = this.opts.video;
|
||||
if (video.readyState < 2 /* HAVE_CURRENT_DATA */) return;
|
||||
|
||||
// Strictly increasing timestamp in ms.
|
||||
let ts = performance.now();
|
||||
if (ts <= this.lastTimestamp) ts = this.lastTimestamp + 1;
|
||||
this.lastTimestamp = ts;
|
||||
|
||||
// Only re-run inference when the video frame actually advanced.
|
||||
if (video.currentTime === this.lastVideoTime) return;
|
||||
this.lastVideoTime = video.currentTime;
|
||||
|
||||
let result: GestureRecognizerResult;
|
||||
try {
|
||||
result = this.recognizer.recognizeForVideo(video, ts);
|
||||
} catch (err) {
|
||||
console.error('recognizeForVideo failed', err);
|
||||
return;
|
||||
}
|
||||
|
||||
this.trackFps(ts);
|
||||
this.publish(result, ts);
|
||||
};
|
||||
|
||||
private trackFps(nowMs: number): void {
|
||||
this.frameTimes.push(nowMs);
|
||||
const windowStart = nowMs - 1000;
|
||||
while (this.frameTimes.length && this.frameTimes[0] < windowStart) {
|
||||
this.frameTimes.shift();
|
||||
}
|
||||
}
|
||||
|
||||
private get fps(): number {
|
||||
if (this.frameTimes.length < 2) return 0;
|
||||
const span = this.frameTimes[this.frameTimes.length - 1] - this.frameTimes[0];
|
||||
if (span <= 0) return 0;
|
||||
return Math.round(((this.frameTimes.length - 1) / span) * 1000);
|
||||
}
|
||||
|
||||
private publish(result: GestureRecognizerResult, ts: number): void {
|
||||
const allLandmarks = result.landmarks ?? [];
|
||||
const handedness = result.handedness ?? [];
|
||||
const gestures = result.gestures ?? [];
|
||||
const tSec = ts / 1000;
|
||||
|
||||
// Surface rect → maps normalized [0,1] coords to surface pixels (X mirrored).
|
||||
const rect = this.opts.surface.getBoundingClientRect();
|
||||
|
||||
const hands: HandState[] = [];
|
||||
const activeKeys = new Set<string>();
|
||||
|
||||
for (let i = 0; i < allLandmarks.length; i++) {
|
||||
const lm = allLandmarks[i];
|
||||
if (!lm || lm.length === 0) continue;
|
||||
|
||||
// Key by handedness so each hand keeps its own filters across frames.
|
||||
// Fall back to index, and disambiguate if both hands share a label.
|
||||
const label = handedness[i]?.[0]?.categoryName ?? `hand${i}`;
|
||||
let key = label;
|
||||
if (activeKeys.has(key)) key = `${label}#${i}`;
|
||||
activeKeys.add(key);
|
||||
|
||||
const state = this.handState(key);
|
||||
const mid = midpoint(lm[LANDMARK.THUMB_TIP], lm[LANDMARK.INDEX_TIP]);
|
||||
const cursor = {
|
||||
x: state.cursorX.filter(mid.x, tSec),
|
||||
y: state.cursorY.filter(mid.y, tSec),
|
||||
};
|
||||
const pinchDist = pinchDistance(lm);
|
||||
const changed = state.pinch.update(pinchDist);
|
||||
const gesture = gestures[i]?.[0]?.categoryName ?? null;
|
||||
|
||||
// Pinch state machine → discrete drag events in surface pixels.
|
||||
const sx = (1 - cursor.x) * rect.width;
|
||||
const sy = cursor.y * rect.height;
|
||||
state.lastX = sx;
|
||||
state.lastY = sy;
|
||||
const pointer = { hand: label, x: sx, y: sy };
|
||||
if (state.pinch.isPinching) {
|
||||
this.emit(changed ? 'grab' : 'drag', pointer);
|
||||
} else if (changed) {
|
||||
this.emit('drop', pointer);
|
||||
}
|
||||
|
||||
hands.push({
|
||||
handedness: label,
|
||||
cursor,
|
||||
pinchDist,
|
||||
pinching: state.pinch.isPinching,
|
||||
gesture: gesture && gesture !== 'None' ? gesture : null,
|
||||
});
|
||||
}
|
||||
|
||||
// Drop state for hands that vanished, so a returning hand starts fresh
|
||||
// (no snap from a stale filter position) and the map can't grow unbounded.
|
||||
// If a vanished hand was mid-pinch, emit a drop so nothing stays grabbed.
|
||||
for (const key of [...this.handStates.keys()]) {
|
||||
if (activeKeys.has(key)) continue;
|
||||
const stale = this.handStates.get(key)!;
|
||||
if (stale.pinch.isPinching) {
|
||||
this.emit('drop', { hand: stale.label, x: stale.lastX, y: stale.lastY });
|
||||
}
|
||||
this.handStates.delete(key);
|
||||
}
|
||||
|
||||
// Discrete commands come from open-hand gestures; ignore a hand that's
|
||||
// pinching (mid-drag) so a drag can't be misread as a command.
|
||||
const commandGestures = new Set<string>();
|
||||
for (const h of hands) {
|
||||
if (!h.pinching && h.gesture) commandGestures.add(h.gesture);
|
||||
}
|
||||
for (const name of this.commands.update(commandGestures, ts)) {
|
||||
this.emit('command', { name });
|
||||
}
|
||||
|
||||
// Status first so the overlay can read this frame's cursors in `results`.
|
||||
this.emit('status', {
|
||||
fps: this.fps,
|
||||
hands,
|
||||
haltProgress: this.commands.haltProgress(ts),
|
||||
});
|
||||
this.emit('results', { result, timestampMs: ts });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// One-Euro filter — low-latency smoothing for noisy interactive signals.
|
||||
// Heavy smoothing when the value is still (kills jitter), low lag when it moves
|
||||
// fast. The correct tool for raw landmark streams, which jitter several pixels
|
||||
// even when the hand is held still.
|
||||
//
|
||||
// Reference: Casiez, Roussel, Vogel — "1€ Filter" (CHI 2012),
|
||||
// http://cristal.univ-lille.fr/~casiez/1euro/
|
||||
//
|
||||
// One filter handles a single scalar; use one instance per axis (x, y).
|
||||
|
||||
/** Smoothing factor for a low-pass step given a cutoff (Hz) and timestep (s). */
|
||||
function smoothingAlpha(cutoffHz: number, dtSec: number): number {
|
||||
const tau = 1 / (2 * Math.PI * cutoffHz);
|
||||
return 1 / (1 + tau / dtSec);
|
||||
}
|
||||
|
||||
/** Exponential low-pass that remembers its last output. */
|
||||
class LowPass {
|
||||
private value: number | null = null;
|
||||
|
||||
filter(x: number, alpha: number): number {
|
||||
this.value = this.value === null ? x : alpha * x + (1 - alpha) * this.value;
|
||||
return this.value;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.value = null;
|
||||
}
|
||||
|
||||
get initialized(): boolean {
|
||||
return this.value !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export class OneEuroFilter {
|
||||
private readonly signal = new LowPass();
|
||||
private readonly derivative = new LowPass();
|
||||
private lastTimeSec: number | null = null;
|
||||
private lastRaw = 0;
|
||||
|
||||
/**
|
||||
* @param minCutoff Baseline cutoff (Hz). Lower → smoother but laggier when still.
|
||||
* @param beta Speed coefficient. Higher → less lag during fast moves.
|
||||
* @param dCutoff Cutoff for the derivative low-pass (Hz). 1.0 is fine.
|
||||
*/
|
||||
constructor(
|
||||
private minCutoff = 1.0,
|
||||
private beta = 0.01,
|
||||
private dCutoff = 1.0
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.signal.reset();
|
||||
this.derivative.reset();
|
||||
this.lastTimeSec = null;
|
||||
this.lastRaw = 0;
|
||||
}
|
||||
|
||||
/** @param timeSec strictly-increasing timestamp in seconds. */
|
||||
filter(x: number, timeSec: number): number {
|
||||
let dt = this.lastTimeSec === null ? 1 / 60 : timeSec - this.lastTimeSec;
|
||||
if (dt <= 0) dt = 1 / 60;
|
||||
this.lastTimeSec = timeSec;
|
||||
|
||||
// Rate of change, itself low-passed, drives the adaptive cutoff.
|
||||
const dRaw = this.signal.initialized ? (x - this.lastRaw) / dt : 0;
|
||||
this.lastRaw = x;
|
||||
const edRaw = this.derivative.filter(dRaw, smoothingAlpha(this.dCutoff, dt));
|
||||
|
||||
const cutoff = this.minCutoff + this.beta * Math.abs(edRaw);
|
||||
return this.signal.filter(x, smoothingAlpha(cutoff, dt));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Discrete gesture → command detection.
|
||||
//
|
||||
// Maps the recognizer's canned gesture categories onto Codeman commands, with
|
||||
// debouncing so each command fires once per gesture *entry* (not every frame
|
||||
// while it's held). Open_Palm is special: it must be held continuously for
|
||||
// `palmHoldMs` before firing halt-all — a dead-man's-switch that's hard to
|
||||
// trigger by accident, since pausing every session is a big hammer.
|
||||
|
||||
import type { CommandName } from './types.ts';
|
||||
|
||||
interface CommandSpec {
|
||||
name: CommandName;
|
||||
/** If set, the gesture must be held this long (ms) before it fires. */
|
||||
holdMs?: number;
|
||||
}
|
||||
|
||||
/** MediaPipe canned gesture category → command. */
|
||||
const GESTURE_COMMANDS: Record<string, CommandSpec> = {
|
||||
Open_Palm: { name: 'halt-all', holdMs: -1 }, // holdMs filled from palmHoldMs
|
||||
Thumb_Up: { name: 'approve' },
|
||||
Victory: { name: 'new-session' },
|
||||
};
|
||||
|
||||
export class CommandDetector {
|
||||
/** gesture category → timestamp (ms) it was first seen in the current hold. */
|
||||
private heldSince = new Map<string, number>();
|
||||
/** gestures that already fired during the current hold (cleared on release). */
|
||||
private fired = new Set<string>();
|
||||
|
||||
constructor(private palmHoldMs = 1000) {}
|
||||
|
||||
reset(): void {
|
||||
this.heldSince.clear();
|
||||
this.fired.clear();
|
||||
}
|
||||
|
||||
private holdMsFor(spec: CommandSpec): number {
|
||||
return spec.holdMs === -1 ? this.palmHoldMs : (spec.holdMs ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed the set of command-gestures currently shown (across all hands).
|
||||
* @returns the commands that fired on this frame (usually empty).
|
||||
*/
|
||||
update(gestures: Set<string>, nowMs: number): CommandName[] {
|
||||
// Forget gestures no longer held, so they can re-fire on the next entry.
|
||||
for (const g of [...this.heldSince.keys()]) {
|
||||
if (!gestures.has(g)) {
|
||||
this.heldSince.delete(g);
|
||||
this.fired.delete(g);
|
||||
}
|
||||
}
|
||||
|
||||
const fired: CommandName[] = [];
|
||||
for (const g of gestures) {
|
||||
const spec = GESTURE_COMMANDS[g];
|
||||
if (!spec) continue;
|
||||
|
||||
const since = this.heldSince.get(g) ?? nowMs;
|
||||
if (!this.heldSince.has(g)) this.heldSince.set(g, since);
|
||||
if (this.fired.has(g)) continue;
|
||||
|
||||
if (nowMs - since >= this.holdMsFor(spec)) {
|
||||
fired.push(spec.name);
|
||||
this.fired.add(g);
|
||||
}
|
||||
}
|
||||
return fired;
|
||||
}
|
||||
|
||||
/** 0–1 charge of the held halt-all gesture (1 once fired, 0 when released). */
|
||||
haltProgress(nowMs: number): number {
|
||||
const since = this.heldSince.get('Open_Palm');
|
||||
if (since === undefined) return 0;
|
||||
if (this.fired.has('Open_Palm')) return 1;
|
||||
return Math.min(1, (nowMs - since) / this.palmHoldMs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// MediaPipe hand landmark indices and the bone connections between them.
|
||||
// See: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker
|
||||
//
|
||||
// 21 landmarks per hand, each normalized to [0,1] in image space.
|
||||
|
||||
export const LANDMARK = {
|
||||
WRIST: 0,
|
||||
THUMB_CMC: 1,
|
||||
THUMB_MCP: 2,
|
||||
THUMB_IP: 3,
|
||||
THUMB_TIP: 4,
|
||||
INDEX_MCP: 5,
|
||||
INDEX_PIP: 6,
|
||||
INDEX_DIP: 7,
|
||||
INDEX_TIP: 8,
|
||||
MIDDLE_MCP: 9,
|
||||
MIDDLE_PIP: 10,
|
||||
MIDDLE_DIP: 11,
|
||||
MIDDLE_TIP: 12,
|
||||
RING_MCP: 13,
|
||||
RING_PIP: 14,
|
||||
RING_DIP: 15,
|
||||
RING_TIP: 16,
|
||||
PINKY_MCP: 17,
|
||||
PINKY_PIP: 18,
|
||||
PINKY_DIP: 19,
|
||||
PINKY_TIP: 20,
|
||||
} as const;
|
||||
|
||||
/** Pairs of landmark indices that form the hand skeleton, for overlay drawing. */
|
||||
export const HAND_CONNECTIONS: ReadonlyArray<readonly [number, number]> = [
|
||||
// Thumb
|
||||
[0, 1],
|
||||
[1, 2],
|
||||
[2, 3],
|
||||
[3, 4],
|
||||
// Index
|
||||
[0, 5],
|
||||
[5, 6],
|
||||
[6, 7],
|
||||
[7, 8],
|
||||
// Middle
|
||||
[5, 9],
|
||||
[9, 10],
|
||||
[10, 11],
|
||||
[11, 12],
|
||||
// Ring
|
||||
[9, 13],
|
||||
[13, 14],
|
||||
[14, 15],
|
||||
[15, 16],
|
||||
// Pinky
|
||||
[13, 17],
|
||||
[17, 18],
|
||||
[18, 19],
|
||||
[19, 20],
|
||||
// Palm base
|
||||
[0, 17],
|
||||
];
|
||||
|
||||
export interface NormalizedLandmark {
|
||||
x: number;
|
||||
y: number;
|
||||
z: number;
|
||||
visibility?: number;
|
||||
}
|
||||
|
||||
/** Euclidean distance between two normalized landmarks (x/y plane). */
|
||||
export function dist2d(a: NormalizedLandmark, b: NormalizedLandmark): number {
|
||||
const dx = a.x - b.x;
|
||||
const dy = a.y - b.y;
|
||||
return Math.hypot(dx, dy);
|
||||
}
|
||||
|
||||
/** Midpoint of two normalized landmarks (x/y plane). */
|
||||
export function midpoint(a: NormalizedLandmark, b: NormalizedLandmark): { x: number; y: number } {
|
||||
return { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 };
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Pinch detection from hand landmarks.
|
||||
//
|
||||
// Distance between thumb tip (4) and index tip (8), normalized by hand size
|
||||
// (wrist 0 → middle-finger MCP 9) so the threshold is robust to how close the
|
||||
// hand is to the camera. Hysteresis + N-frame persistence keep the grab/release
|
||||
// edge from flickering — critical for not "dropping" a tab mid-drag.
|
||||
|
||||
import { LANDMARK, dist2d, type NormalizedLandmark } from './landmarks.ts';
|
||||
|
||||
/**
|
||||
* Thumb-tip→index-tip distance as a fraction of hand size. Smaller = more
|
||||
* pinched. Roughly in [0, ~1.5]; ~0.35 is a firm pinch, ~0.5+ is open.
|
||||
*/
|
||||
export function pinchDistance(landmarks: NormalizedLandmark[]): number {
|
||||
const thumb = landmarks[LANDMARK.THUMB_TIP];
|
||||
const index = landmarks[LANDMARK.INDEX_TIP];
|
||||
const wrist = landmarks[LANDMARK.WRIST];
|
||||
const middleMcp = landmarks[LANDMARK.MIDDLE_MCP];
|
||||
const handSize = dist2d(wrist, middleMcp) || 1e-6;
|
||||
return dist2d(thumb, index) / handSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks pinch state with two thresholds (hysteresis) and a persistence count.
|
||||
* Enter a pinch below `onThreshold`; leave it only above `offThreshold`
|
||||
* (offThreshold > onThreshold). A flip must hold for `persistFrames` frames
|
||||
* before it commits, rejecting single-frame noise.
|
||||
*/
|
||||
export class PinchDetector {
|
||||
private pinching = false;
|
||||
private pendingFrames = 0;
|
||||
|
||||
constructor(
|
||||
private onThreshold = 0.35,
|
||||
private offThreshold = 0.5,
|
||||
private persistFrames = 2
|
||||
) {}
|
||||
|
||||
reset(): void {
|
||||
this.pinching = false;
|
||||
this.pendingFrames = 0;
|
||||
}
|
||||
|
||||
get isPinching(): boolean {
|
||||
return this.pinching;
|
||||
}
|
||||
|
||||
/** Feed one frame's distance. Returns true if the committed state CHANGED. */
|
||||
update(distance: number): boolean {
|
||||
const target = this.pinching
|
||||
? distance < this.offThreshold // stay pinched until the hand opens wide
|
||||
: distance < this.onThreshold; // start pinching once fingers close
|
||||
|
||||
if (target === this.pinching) {
|
||||
this.pendingFrames = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
this.pendingFrames += 1;
|
||||
if (this.pendingFrames >= this.persistFrames) {
|
||||
this.pinching = target;
|
||||
this.pendingFrames = 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Shared types for the gesture input layer.
|
||||
//
|
||||
// Phase 0/1 scope: only `status` and `results` (debug) events are emitted yet.
|
||||
// The semantic events (hover/grab/drag/drop/command) are declared here so the
|
||||
// public API shape is stable, but they are wired in later phases.
|
||||
|
||||
import type { GestureRecognizerResult } from '@mediapipe/tasks-vision';
|
||||
|
||||
export interface GestureControllerOptions {
|
||||
/** The <video> element the camera stream is attached to. */
|
||||
video: HTMLVideoElement;
|
||||
/** Element whose bounding rect normalized coords are mapped against. Defaults to video. (Used from Phase 2.) */
|
||||
surface?: HTMLElement;
|
||||
/** Number of hands to track. v1 = 1. */
|
||||
numHands?: number;
|
||||
/** Pinch hysteresis thresholds (fractions of hand-size). Used from Phase 2. */
|
||||
pinchOn?: number;
|
||||
pinchOff?: number;
|
||||
/** One-Euro cursor smoothing. Lower minCutoff = smoother/laggier when still;
|
||||
* higher beta = less lag during fast moves. Used from Phase 2. */
|
||||
minCutoff?: number;
|
||||
beta?: number;
|
||||
/** How long Open_Palm must be held to fire halt-all. Used from Phase 4. */
|
||||
palmHoldMs?: number;
|
||||
/** Specific camera to open (from enumerateDevices). Empty = default facingMode "user". */
|
||||
deviceId?: string;
|
||||
/** MediaPipe detection/tracking confidence. */
|
||||
minDetectionConfidence?: number;
|
||||
minTrackingConfidence?: number;
|
||||
/** CDN base used to load the wasm fileset + .task model. */
|
||||
wasmBase?: string;
|
||||
modelUrl?: string;
|
||||
}
|
||||
|
||||
export type CommandName = 'halt-all' | 'approve' | 'new-session';
|
||||
|
||||
/** Per-hand state for the current frame. */
|
||||
export interface HandState {
|
||||
/** "Left" / "Right" as reported by MediaPipe (image-space). Used to key filters. */
|
||||
handedness: string;
|
||||
/** Smoothed cursor in raw (unmirrored) normalized [0,1] coords; consumers mirror X. */
|
||||
cursor: { x: number; y: number };
|
||||
/** Thumb/index distance as a fraction of hand size. */
|
||||
pinchDist: number;
|
||||
/** Whether this hand is currently pinching (after hysteresis). */
|
||||
pinching: boolean;
|
||||
/** Top gesture category for this hand, if any. */
|
||||
gesture: string | null;
|
||||
}
|
||||
|
||||
/** A pointer sample for one hand, in surface pixel coords (origin = surface
|
||||
* top-left, X already mirrored to match the displayed video). */
|
||||
export interface HandPointer {
|
||||
hand: string;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
/** Payloads emitted per event name. */
|
||||
export interface GestureEventMap {
|
||||
/** Pinch just closed — start of a drag. */
|
||||
grab: HandPointer;
|
||||
/** Cursor moved while pinched — emitted every frame during a drag. */
|
||||
drag: HandPointer;
|
||||
/** Pinch released (or the hand vanished mid-pinch) — end of a drag. */
|
||||
drop: HandPointer;
|
||||
command: { name: CommandName };
|
||||
status: {
|
||||
fps: number;
|
||||
/** One entry per detected hand (0–`numHands`). */
|
||||
hands: HandState[];
|
||||
/** 0–1 charge of the held Open_Palm halt-all gesture (Phase 4). */
|
||||
haltProgress: number;
|
||||
};
|
||||
/** Debug-only: the raw recognizer result for the current frame (drives the overlay). */
|
||||
results: { result: GestureRecognizerResult; timestampMs: number };
|
||||
}
|
||||
|
||||
export type GestureEventName = keyof GestureEventMap;
|
||||
export type GestureEventHandler<K extends GestureEventName> = (payload: GestureEventMap[K]) => void;
|
||||
@@ -0,0 +1,188 @@
|
||||
// Phase 3 demo wiring.
|
||||
//
|
||||
// - Start button (camera requires a user gesture) + live camera picker.
|
||||
// - Mirrored video preview + debug overlay (skeleton + per-hand cursors).
|
||||
// - Live HUD: fps / hands / pinch distance / gesture.
|
||||
// - Tab board: pinch-drag session tabs between Screen columns.
|
||||
|
||||
import { GestureController } from './gesture/GestureController.ts';
|
||||
import { Overlay, type CursorMark } from './demo/overlay.ts';
|
||||
import { TabsBoard } from './demo/tabs.ts';
|
||||
import type { HandState } from './gesture/types.ts';
|
||||
|
||||
// Cyan for the left hand, violet for the right; green takes over while pinching.
|
||||
const handColor = (handedness: string): string => (handedness === 'Right' ? '#a78bfa' : '#38bdf8');
|
||||
|
||||
const video = document.getElementById('cam') as HTMLVideoElement;
|
||||
const canvas = document.getElementById('overlay') as HTMLCanvasElement;
|
||||
const stage = document.getElementById('stage') as HTMLDivElement;
|
||||
const boardEl = document.getElementById('board') as HTMLDivElement;
|
||||
const startBtn = document.getElementById('start') as HTMLButtonElement;
|
||||
const stopBtn = document.getElementById('stop') as HTMLButtonElement;
|
||||
const fullscreenBtn = document.getElementById('fullscreen') as HTMLButtonElement;
|
||||
const cameraSel = document.getElementById('camera') as HTMLSelectElement;
|
||||
const fpsEl = document.getElementById('fps') as HTMLSpanElement;
|
||||
const handEl = document.getElementById('hand') as HTMLSpanElement;
|
||||
const pinchEl = document.getElementById('pinch') as HTMLSpanElement;
|
||||
const gestureEl = document.getElementById('gesture') as HTMLSpanElement;
|
||||
const statusEl = document.getElementById('status-msg') as HTMLParagraphElement;
|
||||
|
||||
const overlay = new Overlay(canvas, video);
|
||||
// Coords map against the stage rect (the board overlays it exactly).
|
||||
const gc = new GestureController({ video, surface: stage, numHands: 2 });
|
||||
|
||||
const board = new TabsBoard(boardEl, (tabId, columnId) => {
|
||||
statusEl.textContent = columnId
|
||||
? `Moved “${tabId}” → ${columnId}`
|
||||
: `“${tabId}” dropped outside a column — returned.`;
|
||||
});
|
||||
|
||||
// The board drives off discrete grab/drag/drop; hover highlight off `status`.
|
||||
gc.on('grab', ({ hand, x, y }) => board.grab(hand, x, y));
|
||||
gc.on('drag', ({ hand, x, y }) => board.drag(hand, x, y));
|
||||
gc.on('drop', ({ hand, x, y }) => board.drop(hand, x, y));
|
||||
|
||||
// Discrete gesture commands (halt-all / approve / new-session) are intentionally
|
||||
// not wired up here — this build is pinch-drag only. The controller still emits
|
||||
// `command`/`haltProgress`, but nothing consumes them, so an open palm, thumb,
|
||||
// or victory sign does nothing.
|
||||
|
||||
// Cached from `status` so the `results` handler draws the same frame's cursors.
|
||||
let cursors: CursorMark[] = [];
|
||||
let started = false;
|
||||
|
||||
gc.on('results', ({ result }) => {
|
||||
overlay.draw(result, cursors);
|
||||
});
|
||||
|
||||
gc.on('status', ({ fps, hands }) => {
|
||||
cursors = hands.map((h) => ({
|
||||
x: h.cursor.x,
|
||||
y: h.cursor.y,
|
||||
color: handColor(h.handedness),
|
||||
pinching: h.pinching,
|
||||
}));
|
||||
|
||||
// Hover highlights from the full per-frame snapshot (surface px, X mirrored).
|
||||
const rect = stage.getBoundingClientRect();
|
||||
board.hover(
|
||||
hands.map((h) => ({
|
||||
x: (1 - h.cursor.x) * rect.width,
|
||||
y: h.cursor.y * rect.height,
|
||||
pinching: h.pinching,
|
||||
}))
|
||||
);
|
||||
|
||||
fpsEl.textContent = String(fps);
|
||||
fpsEl.style.color = fps >= 25 ? '#4ade80' : fps > 0 ? '#fbbf24' : '#f87171';
|
||||
|
||||
const anyPinching = hands.some((h) => h.pinching);
|
||||
handEl.textContent = hands.length ? hands.map((h) => h.handedness).join(' + ') : 'no';
|
||||
handEl.style.color = hands.length ? '#4ade80' : '#94a3b8';
|
||||
pinchEl.textContent = hands.length ? hands.map(pinchLabel).join(' ') : '—';
|
||||
pinchEl.style.color = anyPinching ? '#4ade80' : '#94a3b8';
|
||||
gestureEl.textContent =
|
||||
hands
|
||||
.filter((h) => h.gesture)
|
||||
.map((h) => h.gesture)
|
||||
.join(', ') || '—';
|
||||
});
|
||||
|
||||
/** e.g. "L 0.34●" — first letter of handedness, distance, dot when pinching. */
|
||||
function pinchLabel(h: HandState): string {
|
||||
return `${h.handedness[0]} ${h.pinchDist.toFixed(2)}${h.pinching ? '●' : ''}`;
|
||||
}
|
||||
|
||||
startBtn.addEventListener('click', async () => {
|
||||
startBtn.disabled = true;
|
||||
statusEl.textContent = 'Requesting camera + loading model…';
|
||||
try {
|
||||
await gc.start();
|
||||
started = true;
|
||||
statusEl.textContent = 'Running. Pinch-drag tabs between Screens.';
|
||||
stopBtn.disabled = false;
|
||||
await populateCameras();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
statusEl.textContent = `Failed to start: ${(err as Error).message}`;
|
||||
startBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
stopBtn.addEventListener('click', () => {
|
||||
gc.stop();
|
||||
started = false;
|
||||
overlay.clear();
|
||||
board.hover([]);
|
||||
cursors = [];
|
||||
statusEl.textContent = 'Stopped.';
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
cameraSel.disabled = true;
|
||||
fpsEl.textContent = '0';
|
||||
handEl.textContent = 'no';
|
||||
pinchEl.textContent = '—';
|
||||
gestureEl.textContent = '—';
|
||||
});
|
||||
|
||||
// Switch cameras live. On macOS the iPhone (Continuity Camera) shows up as
|
||||
// "iPhone Camera" plus a separate "Desk View Camera" (the ultra-wide lens).
|
||||
cameraSel.addEventListener('change', async () => {
|
||||
statusEl.textContent = `Switching to ${cameraSel.selectedOptions[0]?.text}…`;
|
||||
try {
|
||||
await gc.useCamera(cameraSel.value);
|
||||
statusEl.textContent = 'Running.';
|
||||
// Activating the iPhone can surface its Desk View device — re-check.
|
||||
await populateCameras();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
statusEl.textContent = `Couldn't switch camera: ${(err as Error).message}`;
|
||||
}
|
||||
});
|
||||
|
||||
// Cameras come and go (iPhone mounted/unmounted, Desk View appearing). Keep
|
||||
// the dropdown in sync whenever the device set changes.
|
||||
navigator.mediaDevices?.addEventListener('devicechange', () => {
|
||||
if (started) void populateCameras();
|
||||
});
|
||||
|
||||
// Fullscreen the stage so the board (and grab targets) fill the display.
|
||||
// Standard API with a webkit fallback for Safari.
|
||||
type WebkitEl = HTMLElement & { webkitRequestFullscreen?: () => Promise<void> };
|
||||
type WebkitDoc = Document & {
|
||||
webkitFullscreenElement?: Element | null;
|
||||
webkitExitFullscreen?: () => Promise<void>;
|
||||
};
|
||||
|
||||
function fullscreenElement(): Element | null {
|
||||
return document.fullscreenElement ?? (document as WebkitDoc).webkitFullscreenElement ?? null;
|
||||
}
|
||||
|
||||
fullscreenBtn.addEventListener('click', () => {
|
||||
if (fullscreenElement()) {
|
||||
(document.exitFullscreen ?? (document as WebkitDoc).webkitExitFullscreen)?.call(document);
|
||||
} else {
|
||||
const el = stage as WebkitEl;
|
||||
(el.requestFullscreen ?? el.webkitRequestFullscreen)?.call(el);
|
||||
}
|
||||
});
|
||||
|
||||
function syncFullscreenLabel(): void {
|
||||
fullscreenBtn.textContent = fullscreenElement() ? '⤢ Exit fullscreen' : '⛶ Fullscreen';
|
||||
}
|
||||
document.addEventListener('fullscreenchange', syncFullscreenLabel);
|
||||
document.addEventListener('webkitfullscreenchange', syncFullscreenLabel);
|
||||
|
||||
/** Fill the camera dropdown; labels appear only after camera permission. */
|
||||
async function populateCameras(): Promise<void> {
|
||||
const cams = await gc.listCameras();
|
||||
cameraSel.innerHTML = '';
|
||||
cams.forEach((cam, i) => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = cam.deviceId;
|
||||
opt.text = cam.label || `Camera ${i + 1}`;
|
||||
if (cam.deviceId === gc.deviceId) opt.selected = true;
|
||||
cameraSel.append(opt);
|
||||
});
|
||||
cameraSel.disabled = cams.length < 2;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"module": "ESNext",
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"skipLibCheck": true,
|
||||
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { defineConfig } from 'vite';
|
||||
|
||||
// getUserMedia requires a secure context. http://localhost counts as secure,
|
||||
// so the dev server below is fine. If you serve to another device, use https.
|
||||
export default defineConfig({
|
||||
root: '.',
|
||||
server: {
|
||||
host: 'localhost',
|
||||
port: 5173,
|
||||
open: true,
|
||||
},
|
||||
});
|
||||
@@ -45,6 +45,6 @@
|
||||
"jsdom": "^24.1.3",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "^5.5.0",
|
||||
"vitest": "^2.1.9"
|
||||
"vitest": "^4.1.8"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,50 @@
|
||||
import type { RenderParams, FontStyle } from './types.js';
|
||||
import type { RenderParams, FontStyle, XtermTerminal } from './types.js';
|
||||
|
||||
// ─── CJK / fullwidth character width detection ───────────────────────
|
||||
|
||||
/**
|
||||
* Get visual cell width of a single character.
|
||||
* CJK wide characters occupy 2 cells, others occupy 1.
|
||||
* Prefers the terminal's Unicode addon when available.
|
||||
*/
|
||||
export function charCellWidth(terminal: XtermTerminal | null | undefined, ch: string): number {
|
||||
if (terminal?.unicode?.getStringCellWidth) {
|
||||
return terminal.unicode.getStringCellWidth(ch);
|
||||
}
|
||||
// Fallback: detect CJK wide characters by Unicode range
|
||||
const code = ch.codePointAt(0);
|
||||
if (
|
||||
code !== undefined &&
|
||||
code >= 0x1100 &&
|
||||
(code <= 0x115f || // Hangul Jamo
|
||||
(code >= 0x2e80 && code <= 0x303e) || // CJK Radicals, Kangxi, Ideographic
|
||||
(code >= 0x3040 && code <= 0x33bf) || // Hiragana, Katakana, Bopomofo, CJK Compat
|
||||
(code >= 0x3400 && code <= 0x4dbf) || // CJK Unified Ext A
|
||||
(code >= 0x4e00 && code <= 0xa4cf) || // CJK Unified, Yi
|
||||
(code >= 0xa960 && code <= 0xa97c) || // Hangul Jamo Extended-A
|
||||
(code >= 0xac00 && code <= 0xd7a3) || // Hangul Syllables
|
||||
(code >= 0xf900 && code <= 0xfaff) || // CJK Compat Ideographs
|
||||
(code >= 0xfe30 && code <= 0xfe6f) || // CJK Compat Forms
|
||||
(code >= 0xff01 && code <= 0xff60) || // Fullwidth Forms
|
||||
(code >= 0xffe0 && code <= 0xffe6) || // Fullwidth Signs
|
||||
(code >= 0x1f000 && code <= 0x1fbff) || // Mahjong, Domino, Emoji
|
||||
(code >= 0x20000 && code <= 0x2ffff) || // CJK Unified Ext B-F
|
||||
(code >= 0x30000 && code <= 0x3ffff)) // CJK Unified Ext G+
|
||||
)
|
||||
return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get visual cell width of a string (sum of all character widths).
|
||||
*/
|
||||
export function stringCellWidth(terminal: XtermTerminal | null | undefined, str: string): number {
|
||||
let w = 0;
|
||||
for (const ch of str) w += charCellWidth(terminal, ch);
|
||||
return w;
|
||||
}
|
||||
|
||||
// ─── Overlay rendering ────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Render the overlay content into the container element.
|
||||
@@ -6,100 +52,114 @@ import type { RenderParams, FontStyle } from './types.js';
|
||||
* Creates per-character `<span>` elements positioned on an exact grid
|
||||
* matching xterm.js's canvas renderer. This avoids sub-pixel drift that
|
||||
* occurs with normal DOM text flow.
|
||||
*
|
||||
* CJK wide characters are rendered with double-width spans.
|
||||
*/
|
||||
export function renderOverlay(container: HTMLDivElement, params: RenderParams): void {
|
||||
const { lines, startCol, totalCols, cellW, cellH, charTop, charHeight, promptRow, font, showCursor, cursorColor } = params;
|
||||
const {
|
||||
lines,
|
||||
startCol,
|
||||
totalCols,
|
||||
cellW,
|
||||
cellH,
|
||||
charTop,
|
||||
charHeight,
|
||||
promptRow,
|
||||
font,
|
||||
showCursor,
|
||||
cursorColor,
|
||||
terminal,
|
||||
} = params;
|
||||
|
||||
// Position container at prompt row.
|
||||
container.style.left = '0px';
|
||||
container.style.top = (promptRow * cellH) + 'px';
|
||||
// Position container at prompt row.
|
||||
container.style.left = '0px';
|
||||
container.style.top = promptRow * cellH + 'px';
|
||||
|
||||
// Clear and rebuild (typically 1-3 line divs, negligible cost)
|
||||
container.innerHTML = '';
|
||||
const fullWidthPx = totalCols * cellW;
|
||||
// Clear and rebuild (typically 1-3 line divs, negligible cost)
|
||||
container.innerHTML = '';
|
||||
const fullWidthPx = totalCols * cellW;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const leftPx = i === 0 ? startCol * cellW : 0;
|
||||
const widthPx = i === 0 ? (fullWidthPx - leftPx) : fullWidthPx;
|
||||
const topPx = i * cellH;
|
||||
const lineEl = makeLine(lines[i], leftPx, topPx, widthPx, cellH, cellW, charTop, charHeight, font);
|
||||
container.appendChild(lineEl);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const leftPx = i === 0 ? startCol * cellW : 0;
|
||||
const widthPx = i === 0 ? fullWidthPx - leftPx : fullWidthPx;
|
||||
const topPx = i * cellH;
|
||||
const lineEl = makeLine(lines[i], leftPx, topPx, widthPx, cellH, cellW, charTop, charHeight, font, terminal);
|
||||
container.appendChild(lineEl);
|
||||
}
|
||||
|
||||
// Block cursor at end of last line (use visual width for CJK support)
|
||||
if (showCursor) {
|
||||
const lastLine = lines[lines.length - 1];
|
||||
const lastLineLeft = lines.length === 1 ? startCol : 0;
|
||||
const cursorCol = lastLineLeft + stringCellWidth(terminal, lastLine);
|
||||
if (cursorCol < totalCols) {
|
||||
const cursor = document.createElement('span');
|
||||
cursor.style.cssText = 'position:absolute;display:inline-block';
|
||||
cursor.style.left = cursorCol * cellW + 'px';
|
||||
cursor.style.top = (lines.length - 1) * cellH + 'px';
|
||||
cursor.style.width = cellW + 'px';
|
||||
cursor.style.height = cellH + 'px';
|
||||
cursor.style.backgroundColor = cursorColor;
|
||||
container.appendChild(cursor);
|
||||
}
|
||||
}
|
||||
|
||||
// Block cursor at end of last line
|
||||
if (showCursor) {
|
||||
const lastLine = lines[lines.length - 1];
|
||||
const lastLineLeft = lines.length === 1 ? startCol : 0;
|
||||
const cursorCol = lastLineLeft + lastLine.length;
|
||||
if (cursorCol < totalCols) {
|
||||
const cursor = document.createElement('span');
|
||||
cursor.style.cssText = 'position:absolute;display:inline-block';
|
||||
cursor.style.left = (cursorCol * cellW) + 'px';
|
||||
cursor.style.top = ((lines.length - 1) * cellH) + 'px';
|
||||
cursor.style.width = cellW + 'px';
|
||||
cursor.style.height = cellH + 'px';
|
||||
cursor.style.backgroundColor = cursorColor;
|
||||
container.appendChild(cursor);
|
||||
}
|
||||
}
|
||||
|
||||
container.style.display = '';
|
||||
container.style.display = '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a styled line `<div>` with per-character grid positioning.
|
||||
*
|
||||
* Each character gets its own `<span>` placed at `i * cellW` pixels.
|
||||
* This matches xterm's canvas renderer where each glyph occupies exactly
|
||||
* one cell width, regardless of the actual glyph metrics.
|
||||
* Each character gets its own `<span>` positioned by visual column offset.
|
||||
* CJK wide characters occupy 2 cell widths.
|
||||
*/
|
||||
function makeLine(
|
||||
text: string,
|
||||
leftPx: number,
|
||||
topPx: number,
|
||||
widthPx: number,
|
||||
cellH: number,
|
||||
cellW: number,
|
||||
charTop: number,
|
||||
charHeight: number,
|
||||
font: FontStyle,
|
||||
text: string,
|
||||
leftPx: number,
|
||||
topPx: number,
|
||||
widthPx: number,
|
||||
cellH: number,
|
||||
cellW: number,
|
||||
_charTop: number,
|
||||
_charHeight: number,
|
||||
font: FontStyle,
|
||||
terminal?: XtermTerminal | null
|
||||
): HTMLDivElement {
|
||||
const el = document.createElement('div');
|
||||
el.style.cssText = 'position:absolute;pointer-events:none';
|
||||
el.style.backgroundColor = font.backgroundColor;
|
||||
el.style.left = leftPx + 'px';
|
||||
el.style.top = topPx + 'px';
|
||||
el.style.width = widthPx + 'px';
|
||||
// Extend background 1px past cell boundary to cover the compositing
|
||||
// seam between the overlay layer (z-index:7) and the canvas layer below.
|
||||
// The extra 1px lands in the next row's charTop gap (empty area before
|
||||
// text rendering starts), so no canvas content is obscured.
|
||||
el.style.height = (cellH + 1) + 'px';
|
||||
const el = document.createElement('div');
|
||||
el.style.cssText = 'position:absolute;pointer-events:none';
|
||||
el.style.backgroundColor = font.backgroundColor;
|
||||
el.style.left = leftPx + 'px';
|
||||
el.style.top = topPx + 'px';
|
||||
el.style.width = widthPx + 'px';
|
||||
// Extend background 1px past cell boundary to cover the compositing
|
||||
// seam between the overlay layer (z-index:7) and the canvas layer below.
|
||||
// The extra 1px lands in the next row's charTop gap (empty area before
|
||||
// text rendering starts), so no canvas content is obscured.
|
||||
el.style.height = cellH + 1 + 'px';
|
||||
|
||||
// Spans fill the full cell height with matching lineHeight for natural
|
||||
// CSS vertical centering. No transform — any sub-pixel overhang past
|
||||
// the line div causes visible anti-aliasing artifacts at the boundary.
|
||||
// The ≤0.5px difference from canvas ceil() rounding is imperceptible.
|
||||
// CJK wide chars occupy 2 cells — position by visual column offset
|
||||
let colOffset = 0;
|
||||
for (const ch of text) {
|
||||
const cw = charCellWidth(terminal, ch);
|
||||
const span = document.createElement('span');
|
||||
// No ligatures — canvas renders each glyph independently.
|
||||
span.style.cssText =
|
||||
'position:absolute;display:inline-block;text-align:center;pointer-events:none;' +
|
||||
"font-feature-settings:'liga' 0,'calt' 0";
|
||||
span.style.left = colOffset * cellW + 'px';
|
||||
span.style.top = '0px';
|
||||
span.style.width = cw * cellW + 'px';
|
||||
span.style.height = cellH + 'px';
|
||||
span.style.lineHeight = cellH + 'px';
|
||||
span.style.fontFamily = font.fontFamily;
|
||||
span.style.fontSize = font.fontSize;
|
||||
span.style.fontWeight = font.fontWeight;
|
||||
span.style.color = font.color;
|
||||
if (font.letterSpacing) span.style.letterSpacing = font.letterSpacing;
|
||||
span.textContent = ch;
|
||||
el.appendChild(span);
|
||||
colOffset += cw;
|
||||
}
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const span = document.createElement('span');
|
||||
// No ligatures — canvas renders each glyph independently.
|
||||
span.style.cssText =
|
||||
'position:absolute;display:inline-block;text-align:center;pointer-events:none;' +
|
||||
"font-feature-settings:'liga' 0,'calt' 0";
|
||||
span.style.left = (i * cellW) + 'px';
|
||||
span.style.top = '0px';
|
||||
span.style.width = cellW + 'px';
|
||||
span.style.height = cellH + 'px';
|
||||
span.style.lineHeight = cellH + 'px';
|
||||
span.style.fontFamily = font.fontFamily;
|
||||
span.style.fontSize = font.fontSize;
|
||||
span.style.fontWeight = font.fontWeight;
|
||||
span.style.color = font.color;
|
||||
if (font.letterSpacing) span.style.letterSpacing = font.letterSpacing;
|
||||
span.textContent = text[i];
|
||||
el.appendChild(span);
|
||||
}
|
||||
|
||||
return el;
|
||||
return el;
|
||||
}
|
||||
|
||||
@@ -5,28 +5,35 @@
|
||||
* Consumers pass their real Terminal instance — we only use these properties.
|
||||
*/
|
||||
export interface XtermTerminal {
|
||||
readonly element: HTMLElement | undefined;
|
||||
readonly cols: number;
|
||||
readonly rows: number;
|
||||
readonly options: {
|
||||
fontFamily?: string;
|
||||
fontSize?: number;
|
||||
fontWeight?: string | number;
|
||||
theme?: {
|
||||
background?: string;
|
||||
foreground?: string;
|
||||
cursor?: string;
|
||||
};
|
||||
readonly element: HTMLElement | undefined;
|
||||
readonly cols: number;
|
||||
readonly rows: number;
|
||||
readonly options: {
|
||||
fontFamily?: string;
|
||||
fontSize?: number;
|
||||
fontWeight?: string | number;
|
||||
theme?: {
|
||||
background?: string;
|
||||
foreground?: string;
|
||||
cursor?: string;
|
||||
};
|
||||
readonly buffer: {
|
||||
readonly active: {
|
||||
readonly viewportY: number;
|
||||
readonly baseY: number;
|
||||
getLine(y: number): {
|
||||
translateToString(trimRight?: boolean): string;
|
||||
} | undefined;
|
||||
};
|
||||
};
|
||||
readonly buffer: {
|
||||
readonly active: {
|
||||
readonly viewportY: number;
|
||||
readonly baseY: number;
|
||||
getLine(y: number):
|
||||
| {
|
||||
translateToString(trimRight?: boolean): string;
|
||||
}
|
||||
| undefined;
|
||||
};
|
||||
};
|
||||
/** Unicode addon (e.g. Unicode11Addon) for CJK wide character width */
|
||||
readonly unicode?: {
|
||||
getStringCellWidth(str: string): number;
|
||||
activeVersion?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -35,18 +42,18 @@ export interface XtermTerminal {
|
||||
* The consumer calls `terminal.loadAddon(addon)` which invokes `activate()`.
|
||||
*/
|
||||
export interface XtermAddon {
|
||||
activate(terminal: XtermTerminal): void;
|
||||
dispose(): void;
|
||||
activate(terminal: XtermTerminal): void;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Position of the prompt in the terminal viewport.
|
||||
*/
|
||||
export interface PromptPosition {
|
||||
/** Viewport-relative row (0 = top of viewport) */
|
||||
row: number;
|
||||
/** Column of the prompt marker character */
|
||||
col: number;
|
||||
/** Viewport-relative row (0 = top of viewport) */
|
||||
row: number;
|
||||
/** Column of the prompt marker character */
|
||||
col: number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -60,112 +67,114 @@ export interface PromptPosition {
|
||||
* - `custom`: Full escape hatch — provide your own finder function
|
||||
*/
|
||||
export type PromptFinder =
|
||||
| { type: 'character'; char: string; offset?: number }
|
||||
| { type: 'regex'; pattern: RegExp; offset?: number }
|
||||
| { type: 'custom'; find: (terminal: XtermTerminal) => PromptPosition | null; offset?: number };
|
||||
| { type: 'character'; char: string; offset?: number }
|
||||
| { type: 'regex'; pattern: RegExp; offset?: number }
|
||||
| { type: 'custom'; find: (terminal: XtermTerminal) => PromptPosition | null; offset?: number };
|
||||
|
||||
/**
|
||||
* Configuration options for ZerolagInputAddon.
|
||||
*/
|
||||
export interface ZerolagInputOptions {
|
||||
/**
|
||||
* How to find the prompt in the terminal buffer.
|
||||
*
|
||||
* The `offset` controls how many characters after the prompt marker
|
||||
* the user input begins (e.g., `"> "` = offset 2).
|
||||
*
|
||||
* @default { type: 'character', char: '>', offset: 2 }
|
||||
*/
|
||||
prompt?: PromptFinder;
|
||||
/**
|
||||
* How to find the prompt in the terminal buffer.
|
||||
*
|
||||
* The `offset` controls how many characters after the prompt marker
|
||||
* the user input begins (e.g., `"> "` = offset 2).
|
||||
*
|
||||
* @default { type: 'character', char: '>', offset: 2 }
|
||||
*/
|
||||
prompt?: PromptFinder;
|
||||
|
||||
/**
|
||||
* Z-index for the overlay element.
|
||||
* @default 7
|
||||
*/
|
||||
zIndex?: number;
|
||||
/**
|
||||
* Z-index for the overlay element.
|
||||
* @default 7
|
||||
*/
|
||||
zIndex?: number;
|
||||
|
||||
/**
|
||||
* Background color for the overlay.
|
||||
* Set to `'transparent'` to disable the opaque background.
|
||||
* @default Read from terminal.options.theme.background
|
||||
*/
|
||||
backgroundColor?: string;
|
||||
/**
|
||||
* Background color for the overlay.
|
||||
* Set to `'transparent'` to disable the opaque background.
|
||||
* @default Read from terminal.options.theme.background
|
||||
*/
|
||||
backgroundColor?: string;
|
||||
|
||||
/**
|
||||
* Foreground color for overlay text.
|
||||
* @default Read from terminal.options.theme.foreground
|
||||
*/
|
||||
foregroundColor?: string;
|
||||
/**
|
||||
* Foreground color for overlay text.
|
||||
* @default Read from terminal.options.theme.foreground
|
||||
*/
|
||||
foregroundColor?: string;
|
||||
|
||||
/**
|
||||
* Whether to show a block cursor at the end of the overlay text.
|
||||
* @default true
|
||||
*/
|
||||
showCursor?: boolean;
|
||||
/**
|
||||
* Whether to show a block cursor at the end of the overlay text.
|
||||
* @default true
|
||||
*/
|
||||
showCursor?: boolean;
|
||||
|
||||
/**
|
||||
* Cursor color (block cursor at end of text).
|
||||
* @default Read from terminal.options.theme.cursor
|
||||
*/
|
||||
cursorColor?: string;
|
||||
/**
|
||||
* Cursor color (block cursor at end of text).
|
||||
* @default Read from terminal.options.theme.cursor
|
||||
*/
|
||||
cursorColor?: string;
|
||||
|
||||
/**
|
||||
* Scroll debounce time in ms for re-rendering when user scrolls
|
||||
* back to the bottom of the terminal.
|
||||
* @default 50
|
||||
*/
|
||||
scrollDebounceMs?: number;
|
||||
/**
|
||||
* Scroll debounce time in ms for re-rendering when user scrolls
|
||||
* back to the bottom of the terminal.
|
||||
* @default 50
|
||||
*/
|
||||
scrollDebounceMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only state snapshot of the overlay.
|
||||
*/
|
||||
export interface ZerolagInputState {
|
||||
/** Characters typed but not yet acknowledged by the server */
|
||||
pendingText: string;
|
||||
/** Number of characters flushed to PTY but echo not yet received */
|
||||
flushedLength: number;
|
||||
/** Text content of the flushed portion */
|
||||
flushedText: string;
|
||||
/** Whether the overlay is currently visible */
|
||||
visible: boolean;
|
||||
/** Last detected prompt position, if any */
|
||||
promptPosition: PromptPosition | null;
|
||||
/** Characters typed but not yet acknowledged by the server */
|
||||
pendingText: string;
|
||||
/** Number of characters flushed to PTY but echo not yet received */
|
||||
flushedLength: number;
|
||||
/** Text content of the flushed portion */
|
||||
flushedText: string;
|
||||
/** Whether the overlay is currently visible */
|
||||
visible: boolean;
|
||||
/** Last detected prompt position, if any */
|
||||
promptPosition: PromptPosition | null;
|
||||
}
|
||||
|
||||
/** Cell dimensions in CSS pixels. */
|
||||
export interface CellDimensions {
|
||||
width: number;
|
||||
height: number;
|
||||
/** Vertical offset (px) from cell top to where characters render. */
|
||||
charTop: number;
|
||||
/** Height of the character rendering area (px). */
|
||||
charHeight: number;
|
||||
width: number;
|
||||
height: number;
|
||||
/** Vertical offset (px) from cell top to where characters render. */
|
||||
charTop: number;
|
||||
/** Height of the character rendering area (px). */
|
||||
charHeight: number;
|
||||
}
|
||||
|
||||
/** Parameters for the overlay renderer. */
|
||||
export interface RenderParams {
|
||||
lines: string[];
|
||||
startCol: number;
|
||||
totalCols: number;
|
||||
cellW: number;
|
||||
cellH: number;
|
||||
/** Vertical offset (px) from cell top to character rendering area. */
|
||||
charTop: number;
|
||||
/** Height of the character rendering area (px). */
|
||||
charHeight: number;
|
||||
promptRow: number;
|
||||
font: FontStyle;
|
||||
showCursor: boolean;
|
||||
cursorColor: string;
|
||||
lines: string[];
|
||||
startCol: number;
|
||||
totalCols: number;
|
||||
cellW: number;
|
||||
cellH: number;
|
||||
/** Vertical offset (px) from cell top to character rendering area. */
|
||||
charTop: number;
|
||||
/** Height of the character rendering area (px). */
|
||||
charHeight: number;
|
||||
promptRow: number;
|
||||
font: FontStyle;
|
||||
showCursor: boolean;
|
||||
cursorColor: string;
|
||||
/** Terminal instance for CJK wide character width detection */
|
||||
terminal?: XtermTerminal | null;
|
||||
}
|
||||
|
||||
/** Cached font style properties for overlay rendering. */
|
||||
export interface FontStyle {
|
||||
fontFamily: string;
|
||||
fontSize: string;
|
||||
fontWeight: string;
|
||||
color: string;
|
||||
backgroundColor: string;
|
||||
letterSpacing: string;
|
||||
fontFamily: string;
|
||||
fontSize: string;
|
||||
fontWeight: string;
|
||||
color: string;
|
||||
backgroundColor: string;
|
||||
letterSpacing: string;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,843 @@
|
||||
/**
|
||||
* Comprehensive CJK wide character test plan for PR #30.
|
||||
*
|
||||
* Tests all 7 items from the test plan:
|
||||
* 1. Chinese text input — no character overlap
|
||||
* 2. CJK text renders with correct double-width spacing
|
||||
* 3. Japanese (こんにちは) and Korean (안녕하세요) input
|
||||
* 4. Cursor positions correctly after CJK characters
|
||||
* 5. Long CJK input wraps at correct column boundary
|
||||
* 6. Existing ASCII input is unaffected
|
||||
* 7. Teammate terminal panels render CJK correctly (app.js embedded copy)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { renderOverlay, charCellWidth, stringCellWidth } from '../src/overlay-renderer.js';
|
||||
import type { RenderParams, FontStyle } from '../src/types.js';
|
||||
import { createMockTerminal } from './helpers.js';
|
||||
import { ZerolagInputAddon } from '../src/zerolag-input-addon.js';
|
||||
|
||||
// ─── Shared fixtures ─────────────────────────────────────────────────
|
||||
|
||||
const FONT: FontStyle = {
|
||||
fontFamily: 'monospace',
|
||||
fontSize: '14px',
|
||||
fontWeight: 'normal',
|
||||
color: '#eeeeee',
|
||||
backgroundColor: '#0d0d0d',
|
||||
letterSpacing: '',
|
||||
};
|
||||
|
||||
function makeParams(overrides: Partial<RenderParams> = {}): RenderParams {
|
||||
return {
|
||||
lines: ['hello'],
|
||||
startCol: 2,
|
||||
totalCols: 80,
|
||||
cellW: 10,
|
||||
cellH: 17,
|
||||
charTop: 2,
|
||||
charHeight: 14,
|
||||
promptRow: 0,
|
||||
font: FONT,
|
||||
showCursor: true,
|
||||
cursorColor: '#e0e0e0',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/** Extract span data from a rendered line div */
|
||||
function getSpans(lineDiv: HTMLDivElement) {
|
||||
const spans: { text: string; left: number; width: number }[] = [];
|
||||
for (let i = 0; i < lineDiv.children.length; i++) {
|
||||
const span = lineDiv.children[i] as HTMLSpanElement;
|
||||
spans.push({
|
||||
text: span.textContent || '',
|
||||
left: parseFloat(span.style.left),
|
||||
width: parseFloat(span.style.width),
|
||||
});
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
|
||||
// ─── Addon setup helpers ─────────────────────────────────────────────
|
||||
|
||||
let cleanups: (() => void)[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const fn of cleanups) fn();
|
||||
cleanups = [];
|
||||
});
|
||||
|
||||
function tracked(lines: string[] = ['$ '], promptChar = '$') {
|
||||
const mock = createMockTerminal({ buffer: { lines }, cols: 80 });
|
||||
const addon = new ZerolagInputAddon({
|
||||
prompt: { type: 'character', char: promptChar, offset: 2 },
|
||||
});
|
||||
mock.terminal.loadAddon(addon);
|
||||
cleanups.push(() => {
|
||||
addon.dispose();
|
||||
mock.cleanup();
|
||||
});
|
||||
return { addon, mock };
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 1: Chinese text input — no character overlap
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 1: Chinese text — no character overlap', () => {
|
||||
it('consecutive Chinese characters have contiguous non-overlapping spans', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = '你好世界';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
expect(spans.length).toBe(4);
|
||||
|
||||
// Each span: left = previous span's (left + width), width = 20px (2 cells)
|
||||
for (let i = 0; i < spans.length; i++) {
|
||||
expect(spans[i].width).toBe(20); // 2 cells * 10px
|
||||
if (i > 0) {
|
||||
const expectedLeft = spans[i - 1].left + spans[i - 1].width;
|
||||
expect(spans[i].left).toBe(expectedLeft);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('Chinese sentence (simulated pinyin output) renders without gaps or overlaps', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = '我是一个测试';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 8 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
expect(spans.length).toBe(6);
|
||||
|
||||
// Verify contiguous positioning
|
||||
let expectedLeft = 0;
|
||||
for (const span of spans) {
|
||||
expect(span.left).toBe(expectedLeft);
|
||||
expect(span.width).toBe(16); // 2 * 8px
|
||||
expectedLeft += span.width;
|
||||
}
|
||||
|
||||
// Total visual width should be 6 chars * 2 cells * 8px = 96px
|
||||
expect(expectedLeft).toBe(96);
|
||||
});
|
||||
|
||||
it('mixed Chinese + ASCII has no gaps between spans', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = 'hello你好world';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
// h(10) e(10) l(10) l(10) o(10) 你(20) 好(20) w(10) o(10) r(10) l(10) d(10)
|
||||
expect(spans.length).toBe(12);
|
||||
|
||||
// Verify contiguity: no gaps between any adjacent spans
|
||||
for (let i = 1; i < spans.length; i++) {
|
||||
const prevEnd = spans[i - 1].left + spans[i - 1].width;
|
||||
expect(spans[i].left).toBe(prevEnd);
|
||||
}
|
||||
});
|
||||
|
||||
it('addChar with Chinese characters accumulates correctly in addon', () => {
|
||||
const { addon } = tracked();
|
||||
for (const ch of '你好世界') {
|
||||
addon.addChar(ch);
|
||||
}
|
||||
expect(addon.pendingText).toBe('你好世界');
|
||||
expect(addon.hasPending).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 2: CJK text renders with correct double-width spacing
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 2: CJK double-width spacing', () => {
|
||||
it('charCellWidth returns 2 for CJK Unified Ideographs (0x4E00-0x9FFF)', () => {
|
||||
// Common Chinese characters
|
||||
const chars = '中文测试你好世界天地人';
|
||||
for (const ch of chars) {
|
||||
expect(charCellWidth(null, ch)).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for CJK Extension A (0x3400-0x4DBF)', () => {
|
||||
expect(charCellWidth(null, '\u3400')).toBe(2); // First Extension A char
|
||||
expect(charCellWidth(null, '\u4DB5')).toBe(2); // One of the last Extension A chars
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for CJK Radicals (0x2E80-0x2EFF)', () => {
|
||||
expect(charCellWidth(null, '\u2E80')).toBe(2); // CJK Radical Repeat
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for fullwidth ASCII forms (0xFF01-0xFF5E)', () => {
|
||||
expect(charCellWidth(null, '\uFF01')).toBe(2); // !
|
||||
expect(charCellWidth(null, '\uFF21')).toBe(2); // A
|
||||
expect(charCellWidth(null, '\uFF41')).toBe(2); // a
|
||||
expect(charCellWidth(null, '\uFF10')).toBe(2); // 0
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for CJK Compatibility Ideographs (0xF900-0xFAFF)', () => {
|
||||
expect(charCellWidth(null, '\uF900')).toBe(2);
|
||||
});
|
||||
|
||||
it('stringCellWidth calculates correct visual width for CJK strings', () => {
|
||||
expect(stringCellWidth(null, '你好')).toBe(4); // 2 + 2
|
||||
expect(stringCellWidth(null, '你好世界')).toBe(8); // 4 * 2
|
||||
expect(stringCellWidth(null, 'hi你好')).toBe(6); // 1 + 1 + 2 + 2
|
||||
expect(stringCellWidth(null, '你a好b')).toBe(6); // 2 + 1 + 2 + 1
|
||||
expect(stringCellWidth(null, 'abc你好def')).toBe(10); // 3 + 4 + 3
|
||||
});
|
||||
|
||||
it('CJK span widths are exactly 2 * cellW pixels', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['你'], cellW: 8.4 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.width).toBe('16.8px'); // 2 * 8.4
|
||||
});
|
||||
|
||||
it('ASCII span widths remain 1 * cellW pixels', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'], cellW: 8.4 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.width).toBe('8.4px'); // 1 * 8.4
|
||||
});
|
||||
|
||||
it('terminal unicode addon is preferred over fallback when available', () => {
|
||||
const mockTerminal = {
|
||||
unicode: {
|
||||
getStringCellWidth: (s: string) => {
|
||||
// Custom width: treat 'W' as wide
|
||||
return s === 'W' ? 2 : 1;
|
||||
},
|
||||
},
|
||||
} as any;
|
||||
expect(charCellWidth(mockTerminal, 'W')).toBe(2);
|
||||
expect(charCellWidth(mockTerminal, 'a')).toBe(1);
|
||||
// Fallback ignores terminal addon result for actual CJK
|
||||
expect(charCellWidth(null, '你')).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 3: Japanese (こんにちは) and Korean (안녕하세요) input
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 3: Japanese and Korean input', () => {
|
||||
describe('Japanese', () => {
|
||||
it('charCellWidth returns 2 for Hiragana (0x3040-0x309F)', () => {
|
||||
const hiragana = 'あいうえおかきくけこさしすせそ';
|
||||
for (const ch of hiragana) {
|
||||
expect(charCellWidth(null, ch)).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for Katakana (0x30A0-0x30FF)', () => {
|
||||
const katakana = 'アイウエオカキクケコサシスセソ';
|
||||
for (const ch of katakana) {
|
||||
expect(charCellWidth(null, ch)).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('Japanese greeting renders with correct span positions', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = 'こんにちは';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
expect(spans.length).toBe(5);
|
||||
// こ(0,20) ん(20,20) に(40,20) ち(60,20) は(80,20)
|
||||
expect(spans[0]).toEqual({ text: 'こ', left: 0, width: 20 });
|
||||
expect(spans[1]).toEqual({ text: 'ん', left: 20, width: 20 });
|
||||
expect(spans[2]).toEqual({ text: 'に', left: 40, width: 20 });
|
||||
expect(spans[3]).toEqual({ text: 'ち', left: 60, width: 20 });
|
||||
expect(spans[4]).toEqual({ text: 'は', left: 80, width: 20 });
|
||||
});
|
||||
|
||||
it('mixed Japanese + ASCII positions correctly', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = 'hello こんにちは';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
// h(0) e(10) l(20) l(30) o(40) space(50) こ(60) ん(80) に(100) ち(120) は(140)
|
||||
expect(spans.length).toBe(11);
|
||||
expect(spans[5]).toEqual({ text: ' ', left: 50, width: 10 }); // space
|
||||
expect(spans[6]).toEqual({ text: 'こ', left: 60, width: 20 }); // first CJK after ASCII
|
||||
});
|
||||
|
||||
it('addon handles Japanese input via addChar', () => {
|
||||
const { addon } = tracked();
|
||||
for (const ch of 'こんにちは') {
|
||||
addon.addChar(ch);
|
||||
}
|
||||
expect(addon.pendingText).toBe('こんにちは');
|
||||
});
|
||||
|
||||
it('addon handles Japanese input via appendText (paste)', () => {
|
||||
const { addon } = tracked();
|
||||
addon.appendText('こんにちは世界');
|
||||
expect(addon.pendingText).toBe('こんにちは世界');
|
||||
expect(addon.hasPending).toBe(true);
|
||||
});
|
||||
|
||||
it('stringCellWidth correct for Japanese greeting', () => {
|
||||
expect(stringCellWidth(null, 'こんにちは')).toBe(10); // 5 * 2
|
||||
});
|
||||
});
|
||||
|
||||
describe('Korean', () => {
|
||||
it('charCellWidth returns 2 for Hangul Syllables (0xAC00-0xD7A3)', () => {
|
||||
const hangul = '가나다라마바사아자차카타파하';
|
||||
for (const ch of hangul) {
|
||||
expect(charCellWidth(null, ch)).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('charCellWidth returns 2 for Hangul Jamo (0x1100-0x115F)', () => {
|
||||
expect(charCellWidth(null, '\u1100')).toBe(2); // ᄀ
|
||||
expect(charCellWidth(null, '\u1112')).toBe(2); // ᄒ
|
||||
});
|
||||
|
||||
it('Korean greeting renders with correct span positions', () => {
|
||||
const container = document.createElement('div');
|
||||
const text = '안녕하세요';
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
expect(spans.length).toBe(5);
|
||||
expect(spans[0]).toEqual({ text: '안', left: 0, width: 20 });
|
||||
expect(spans[1]).toEqual({ text: '녕', left: 20, width: 20 });
|
||||
expect(spans[2]).toEqual({ text: '하', left: 40, width: 20 });
|
||||
expect(spans[3]).toEqual({ text: '세', left: 60, width: 20 });
|
||||
expect(spans[4]).toEqual({ text: '요', left: 80, width: 20 });
|
||||
});
|
||||
|
||||
it('addon handles Korean input', () => {
|
||||
const { addon } = tracked();
|
||||
addon.appendText('안녕하세요');
|
||||
expect(addon.pendingText).toBe('안녕하세요');
|
||||
expect(addon.hasPending).toBe(true);
|
||||
});
|
||||
|
||||
it('removeChar removes Korean characters one at a time', () => {
|
||||
const { addon } = tracked();
|
||||
addon.appendText('안녕');
|
||||
addon.removeChar();
|
||||
expect(addon.pendingText).toBe('안');
|
||||
addon.removeChar();
|
||||
expect(addon.pendingText).toBe('');
|
||||
});
|
||||
|
||||
it('stringCellWidth correct for Korean greeting', () => {
|
||||
expect(stringCellWidth(null, '안녕하세요')).toBe(10); // 5 * 2
|
||||
});
|
||||
});
|
||||
|
||||
describe('mixed CJK scripts', () => {
|
||||
it('Chinese + Japanese + Korean in one string', () => {
|
||||
const text = '你好こんにちは안녕';
|
||||
expect(stringCellWidth(null, text)).toBe(18); // 9 chars * 2 each
|
||||
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: [text], cellW: 10 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
// All 9 CJK chars: contiguous double-width spans
|
||||
expect(spans.length).toBe(9);
|
||||
let expectedLeft = 0;
|
||||
for (const span of spans) {
|
||||
expect(span.left).toBe(expectedLeft);
|
||||
expect(span.width).toBe(20);
|
||||
expectedLeft += 20;
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 4: Cursor positions correctly after CJK characters
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 4: Cursor positioning after CJK', () => {
|
||||
it('cursor after Chinese text on first line', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世界'],
|
||||
startCol: 2,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// cursorCol = startCol(2) + stringCellWidth('你好世界')(8) = 10
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('100px'); // 10 * 10px
|
||||
});
|
||||
|
||||
it('cursor after Japanese text on first line', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['こんにちは'],
|
||||
startCol: 2,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// cursorCol = 2 + 10 = 12
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('120px');
|
||||
});
|
||||
|
||||
it('cursor after Korean text on first line', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['안녕하세요'],
|
||||
startCol: 2,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// cursorCol = 2 + 10 = 12
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('120px');
|
||||
});
|
||||
|
||||
it('cursor after mixed ASCII + CJK', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['hi你好'],
|
||||
startCol: 3,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// cursorCol = 3 + stringCellWidth('hi你好')(6) = 9
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('90px');
|
||||
});
|
||||
|
||||
it('cursor on wrapped line after CJK text', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世界', '再见'],
|
||||
startCol: 2,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// Last line is '再见', starts at col 0 (wrapped line), width = 4
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('40px'); // 0 + 4 = 4, * 10 = 40
|
||||
expect(cursor.style.top).toBe('20px'); // row 1 * cellH
|
||||
});
|
||||
|
||||
it('cursor hidden when it would exceed totalCols', () => {
|
||||
const container = document.createElement('div');
|
||||
// 4 CJK chars = 8 visual cols, startCol=73, cursorCol = 73 + 8 = 81 > 80
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世界'],
|
||||
startCol: 73,
|
||||
totalCols: 80,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// Should be 1 line div, no cursor span (cursor at col 81 >= totalCols 80)
|
||||
// Actually cursor check is cursorCol < totalCols, so at 81 it's hidden
|
||||
const children = container.children;
|
||||
// If cursor is rendered, last child would be a cursor span
|
||||
// With cursorCol 81 >= 80, cursor should NOT be rendered
|
||||
expect(children.length).toBe(1); // only line div
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 5: Long CJK input wraps at correct column boundary
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 5: CJK line wrapping at column boundaries', () => {
|
||||
it('CJK chars wrap when they would overflow first line', () => {
|
||||
const container = document.createElement('div');
|
||||
// totalCols=10, startCol=2 → firstLineCols=8
|
||||
// Each CJK char = 2 cols → first line fits 4 chars (8 cols)
|
||||
// 5th char wraps to second line
|
||||
const text = '你好世界啊'; // 5 chars = 10 visual cols
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世界', '啊'],
|
||||
startCol: 2,
|
||||
totalCols: 10,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
})
|
||||
);
|
||||
|
||||
expect(container.children.length).toBe(3); // 2 line divs + cursor
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(4); // 你好世界
|
||||
expect(line1.style.left).toBe('20px'); // startCol * cellW
|
||||
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.children.length).toBe(1); // 啊
|
||||
expect(line2.style.left).toBe('0px'); // wrapped line starts at col 0
|
||||
expect(line2.style.top).toBe('20px'); // second row
|
||||
});
|
||||
|
||||
it('CJK char that would partially overflow stays on next line', () => {
|
||||
// totalCols=9, startCol=2 → firstLineCols=7
|
||||
// CJK chars are 2-wide. 3 chars = 6 cols (fits). 4th char = 8 cols > 7. Wraps.
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世', '界'],
|
||||
startCol: 2,
|
||||
totalCols: 9,
|
||||
cellW: 10,
|
||||
})
|
||||
);
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(3); // 3 CJK chars fit (6 cols <= 7)
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.children.length).toBe(1); // 界 wraps
|
||||
});
|
||||
|
||||
it('addon _render splits CJK text into visual lines correctly', () => {
|
||||
// Narrow terminal: 10 cols, startCol=2 → firstLineCols=8 → 4 CJK chars
|
||||
const mock = createMockTerminal({
|
||||
buffer: { lines: ['$ '] },
|
||||
cols: 10,
|
||||
});
|
||||
const addon = new ZerolagInputAddon({
|
||||
prompt: { type: 'character', char: '$', offset: 2 },
|
||||
});
|
||||
mock.terminal.loadAddon(addon);
|
||||
cleanups.push(() => {
|
||||
addon.dispose();
|
||||
mock.cleanup();
|
||||
});
|
||||
|
||||
// Type 6 CJK chars (12 visual cols)
|
||||
for (const ch of '你好世界再见') {
|
||||
addon.addChar(ch);
|
||||
}
|
||||
expect(addon.pendingText).toBe('你好世界再见');
|
||||
expect(addon.hasPending).toBe(true);
|
||||
});
|
||||
|
||||
it('mixed ASCII + CJK wraps correctly', () => {
|
||||
const container = document.createElement('div');
|
||||
// totalCols=10, startCol=2 → firstLineCols=8
|
||||
// 'ab' = 2 cols, '你好' = 4 cols, 'cd' = 2 cols → total 8 cols (fits line 1)
|
||||
// '世' = 2 cols → wraps to line 2
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['ab你好cd', '世'],
|
||||
startCol: 2,
|
||||
totalCols: 10,
|
||||
cellW: 10,
|
||||
})
|
||||
);
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(6); // a,b,你,好,c,d
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.children.length).toBe(1); // 世
|
||||
});
|
||||
|
||||
it('odd column count: CJK char does not split across boundary', () => {
|
||||
// totalCols=11, startCol=2 → firstLineCols=9
|
||||
// 4 CJK chars = 8 cols (fits). 5th CJK = 10 cols > 9 (wraps).
|
||||
// 1 empty column remains on first line (CJK can't fit in 1 col).
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世界', '啊'],
|
||||
startCol: 2,
|
||||
totalCols: 11,
|
||||
cellW: 10,
|
||||
})
|
||||
);
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(4); // 4 chars = 8 cols, 5th would need 10 > 9
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.children.length).toBe(1);
|
||||
});
|
||||
|
||||
it('CJK fills entire wrapped line', () => {
|
||||
const container = document.createElement('div');
|
||||
// totalCols=6, startCol=0 → firstLineCols=6
|
||||
// 3 CJK chars = 6 cols (fills line 1), next 3 fill line 2
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好世', '界再见'],
|
||||
startCol: 0,
|
||||
totalCols: 6,
|
||||
cellW: 10,
|
||||
})
|
||||
);
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(3);
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.children.length).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 6: Existing ASCII input is unaffected
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 6: ASCII input — no regression', () => {
|
||||
it('ASCII characters still get 1-cell-wide spans', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abcdef'], cellW: 10 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
for (const span of spans) {
|
||||
expect(span.width).toBe(10); // 1 * cellW
|
||||
}
|
||||
});
|
||||
|
||||
it('ASCII span positions are sequential at 1-cell intervals', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['xyz'], cellW: 8.4 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
|
||||
expect(spans[0].left).toBe(0);
|
||||
expect(spans[1].left).toBeCloseTo(8.4, 5);
|
||||
expect(spans[2].left).toBeCloseTo(16.8, 5);
|
||||
});
|
||||
|
||||
it('ASCII cursor positions at correct column', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['hello'],
|
||||
startCol: 3,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// cursor at startCol(3) + 5 = 8
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('80px');
|
||||
});
|
||||
|
||||
it('charCellWidth returns 1 for all printable ASCII', () => {
|
||||
for (let code = 32; code < 127; code++) {
|
||||
const ch = String.fromCharCode(code);
|
||||
expect(charCellWidth(null, ch)).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('stringCellWidth equals length for pure ASCII', () => {
|
||||
expect(stringCellWidth(null, 'hello world')).toBe(11);
|
||||
expect(stringCellWidth(null, 'test123!@#')).toBe(10);
|
||||
expect(stringCellWidth(null, '')).toBe(0);
|
||||
});
|
||||
|
||||
it('ASCII multi-line wrapping is unaffected', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['abcde', 'fgh'],
|
||||
startCol: 5,
|
||||
totalCols: 10,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
})
|
||||
);
|
||||
|
||||
expect(container.children.length).toBe(3); // 2 lines + cursor
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line1.children.length).toBe(5);
|
||||
expect(line2.children.length).toBe(3);
|
||||
expect(line1.style.left).toBe('50px'); // startCol * cellW
|
||||
expect(line2.style.left).toBe('0px');
|
||||
});
|
||||
|
||||
it('addon addChar/removeChar/clear work for ASCII', () => {
|
||||
const { addon } = tracked();
|
||||
addon.addChar('h');
|
||||
addon.addChar('e');
|
||||
addon.addChar('l');
|
||||
addon.addChar('l');
|
||||
addon.addChar('o');
|
||||
expect(addon.pendingText).toBe('hello');
|
||||
|
||||
addon.removeChar();
|
||||
expect(addon.pendingText).toBe('hell');
|
||||
|
||||
addon.clear();
|
||||
expect(addon.pendingText).toBe('');
|
||||
expect(addon.hasPending).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// TEST 7: Teammate terminal panels render CJK correctly
|
||||
//
|
||||
// Teammate panels share the global xterm-zerolag-input.js vendor bundle
|
||||
// (loaded via <script> in index.html). This is the IIFE build of the
|
||||
// same package source tested in tests 1-6. The build pipeline is:
|
||||
// src/overlay-renderer.ts → tsup → dist/index.global.js → vendor copy
|
||||
//
|
||||
// Since all terminals (main + teammate) use the same LocalEchoOverlay
|
||||
// class from the global scope, CJK correctness is guaranteed by:
|
||||
// (a) The package source handles CJK correctly (tests 1-6 above)
|
||||
// (b) The IIFE build bundles the exact same charCellWidth / makeLine code
|
||||
//
|
||||
// These tests verify the exported module includes CJK-aware functions
|
||||
// and that teammate-style terminal instances work identically.
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
describe('Test 7: Teammate terminal panels render CJK correctly', () => {
|
||||
it('package exports charCellWidth and stringCellWidth', () => {
|
||||
// These are the CJK-aware functions that the IIFE build exposes
|
||||
expect(typeof charCellWidth).toBe('function');
|
||||
expect(typeof stringCellWidth).toBe('function');
|
||||
});
|
||||
|
||||
it('ZerolagInputAddon (used by LocalEchoOverlay) handles CJK in teammate terminals', () => {
|
||||
// Simulate a teammate terminal panel: separate terminal instance,
|
||||
// same addon class, different prompt character
|
||||
const mock = createMockTerminal({
|
||||
buffer: { lines: ['❯ '] },
|
||||
cols: 40,
|
||||
});
|
||||
const addon = new ZerolagInputAddon({
|
||||
prompt: { type: 'character', char: '❯', offset: 2 },
|
||||
});
|
||||
mock.terminal.loadAddon(addon);
|
||||
cleanups.push(() => {
|
||||
addon.dispose();
|
||||
mock.cleanup();
|
||||
});
|
||||
|
||||
// Type CJK in teammate terminal
|
||||
for (const ch of '你好世界') {
|
||||
addon.addChar(ch);
|
||||
}
|
||||
expect(addon.pendingText).toBe('你好世界');
|
||||
expect(addon.hasPending).toBe(true);
|
||||
});
|
||||
|
||||
it('teammate terminal with narrow width wraps CJK correctly', () => {
|
||||
// Teammate panels are often narrower (sidebar, split view)
|
||||
const mock = createMockTerminal({
|
||||
buffer: { lines: ['❯ '] },
|
||||
cols: 12, // narrow panel
|
||||
});
|
||||
const addon = new ZerolagInputAddon({
|
||||
prompt: { type: 'character', char: '❯', offset: 2 },
|
||||
});
|
||||
mock.terminal.loadAddon(addon);
|
||||
cleanups.push(() => {
|
||||
addon.dispose();
|
||||
mock.cleanup();
|
||||
});
|
||||
|
||||
// Type 6 CJK chars (12 visual cols) with startCol=2 → only 10 available
|
||||
// First line: 5 CJK = 10 cols. 6th wraps.
|
||||
for (const ch of '你好世界再见') {
|
||||
addon.addChar(ch);
|
||||
}
|
||||
expect(addon.pendingText).toBe('你好世界再见');
|
||||
});
|
||||
|
||||
it('mixed CJK scripts render identically in teammate and main terminals', () => {
|
||||
// Verify that the same input produces the same overlay output
|
||||
// regardless of which terminal instance it's on
|
||||
const text = '你好こんにちは안녕';
|
||||
|
||||
// "Main" terminal
|
||||
const container1 = document.createElement('div');
|
||||
renderOverlay(container1, makeParams({ lines: [text], cellW: 10 }));
|
||||
const line1 = container1.children[0] as HTMLDivElement;
|
||||
const spans1 = getSpans(line1);
|
||||
|
||||
// "Teammate" terminal (same params — same bundle)
|
||||
const container2 = document.createElement('div');
|
||||
renderOverlay(container2, makeParams({ lines: [text], cellW: 10 }));
|
||||
const line2 = container2.children[0] as HTMLDivElement;
|
||||
const spans2 = getSpans(line2);
|
||||
|
||||
// Identical rendering
|
||||
expect(spans1.length).toBe(spans2.length);
|
||||
for (let i = 0; i < spans1.length; i++) {
|
||||
expect(spans1[i]).toEqual(spans2[i]);
|
||||
}
|
||||
});
|
||||
|
||||
it('CJK rendering correct in overlay with teammate-typical prompt (❯)', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['こんにちは世界'],
|
||||
startCol: 2, // after ❯ prompt
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const spans = getSpans(lineDiv);
|
||||
expect(spans.length).toBe(7);
|
||||
|
||||
// All CJK, all double-width, contiguous
|
||||
let expectedLeft = 0;
|
||||
for (const span of spans) {
|
||||
expect(span.left).toBe(expectedLeft);
|
||||
expect(span.width).toBe(20);
|
||||
expectedLeft += 20;
|
||||
}
|
||||
|
||||
// Cursor position: startCol(2) + 14 visual cols = 16
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('160px');
|
||||
});
|
||||
});
|
||||
@@ -1,300 +1,420 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { renderOverlay } from '../src/overlay-renderer.js';
|
||||
import { renderOverlay, charCellWidth, stringCellWidth } from '../src/overlay-renderer.js';
|
||||
import type { RenderParams, FontStyle } from '../src/types.js';
|
||||
|
||||
const FONT: FontStyle = {
|
||||
fontFamily: 'monospace',
|
||||
fontSize: '14px',
|
||||
fontWeight: 'normal',
|
||||
color: '#eeeeee',
|
||||
backgroundColor: '#0d0d0d',
|
||||
letterSpacing: '',
|
||||
fontFamily: 'monospace',
|
||||
fontSize: '14px',
|
||||
fontWeight: 'normal',
|
||||
color: '#eeeeee',
|
||||
backgroundColor: '#0d0d0d',
|
||||
letterSpacing: '',
|
||||
};
|
||||
|
||||
function makeParams(overrides: Partial<RenderParams> = {}): RenderParams {
|
||||
return {
|
||||
lines: ['hello'],
|
||||
startCol: 2,
|
||||
totalCols: 80,
|
||||
cellW: 8.4,
|
||||
cellH: 17,
|
||||
charTop: 2,
|
||||
charHeight: 14,
|
||||
promptRow: 10,
|
||||
font: FONT,
|
||||
showCursor: true,
|
||||
cursorColor: '#e0e0e0',
|
||||
...overrides,
|
||||
};
|
||||
return {
|
||||
lines: ['hello'],
|
||||
startCol: 2,
|
||||
totalCols: 80,
|
||||
cellW: 8.4,
|
||||
cellH: 17,
|
||||
charTop: 2,
|
||||
charHeight: 14,
|
||||
promptRow: 10,
|
||||
font: FONT,
|
||||
showCursor: true,
|
||||
cursorColor: '#e0e0e0',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('renderOverlay', () => {
|
||||
it('positions container at prompt row', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ promptRow: 5 }));
|
||||
expect(container.style.top).toBe((5 * 17) + 'px');
|
||||
expect(container.style.left).toBe('0px');
|
||||
});
|
||||
it('positions container at prompt row', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ promptRow: 5 }));
|
||||
expect(container.style.top).toBe(5 * 17 + 'px');
|
||||
expect(container.style.left).toBe('0px');
|
||||
});
|
||||
|
||||
it('creates per-character spans in a line div', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'] }));
|
||||
it('creates per-character spans in a line div', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'] }));
|
||||
|
||||
// Line div + cursor span
|
||||
expect(container.children.length).toBe(2);
|
||||
// Line div + cursor span
|
||||
expect(container.children.length).toBe(2);
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(3); // a, b, c
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(3); // a, b, c
|
||||
|
||||
const spanA = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(spanA.textContent).toBe('a');
|
||||
expect(spanA.style.left).toBe('0px');
|
||||
const spanA = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(spanA.textContent).toBe('a');
|
||||
expect(spanA.style.left).toBe('0px');
|
||||
|
||||
const spanB = lineDiv.children[1] as HTMLSpanElement;
|
||||
expect(spanB.textContent).toBe('b');
|
||||
expect(spanB.style.left).toBe('8.4px');
|
||||
const spanB = lineDiv.children[1] as HTMLSpanElement;
|
||||
expect(spanB.textContent).toBe('b');
|
||||
expect(spanB.style.left).toBe('8.4px');
|
||||
|
||||
const spanC = lineDiv.children[2] as HTMLSpanElement;
|
||||
expect(spanC.textContent).toBe('c');
|
||||
expect(spanC.style.left).toBe('16.8px');
|
||||
});
|
||||
const spanC = lineDiv.children[2] as HTMLSpanElement;
|
||||
expect(spanC.textContent).toBe('c');
|
||||
expect(spanC.style.left).toBe('16.8px');
|
||||
});
|
||||
|
||||
it('sets span width to cellW', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['x'], cellW: 9.5 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.width).toBe('9.5px');
|
||||
});
|
||||
it('sets span width to cellW', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['x'], cellW: 9.5 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.width).toBe('9.5px');
|
||||
});
|
||||
|
||||
it('applies font styles to spans', () => {
|
||||
const font: FontStyle = {
|
||||
fontFamily: 'Fira Code',
|
||||
fontSize: '16px',
|
||||
fontWeight: 'bold',
|
||||
color: '#ff0000',
|
||||
backgroundColor: '#000000',
|
||||
letterSpacing: '0.5px',
|
||||
};
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['A'], font }));
|
||||
it('applies font styles to spans', () => {
|
||||
const font: FontStyle = {
|
||||
fontFamily: 'Fira Code',
|
||||
fontSize: '16px',
|
||||
fontWeight: 'bold',
|
||||
color: '#ff0000',
|
||||
backgroundColor: '#000000',
|
||||
letterSpacing: '0.5px',
|
||||
};
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['A'], font }));
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(lineDiv.style.backgroundColor).toBe('rgb(0, 0, 0)');
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(lineDiv.style.backgroundColor).toBe('rgb(0, 0, 0)');
|
||||
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.fontFamily).toBe('Fira Code');
|
||||
expect(span.style.fontSize).toBe('16px');
|
||||
expect(span.style.fontWeight).toBe('bold');
|
||||
expect(span.style.color).toBe('rgb(255, 0, 0)');
|
||||
expect(span.style.letterSpacing).toBe('0.5px');
|
||||
});
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.fontFamily).toBe('Fira Code');
|
||||
expect(span.style.fontSize).toBe('16px');
|
||||
expect(span.style.fontWeight).toBe('bold');
|
||||
expect(span.style.color).toBe('rgb(255, 0, 0)');
|
||||
expect(span.style.letterSpacing).toBe('0.5px');
|
||||
});
|
||||
|
||||
it('offsets first line by startCol', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['hi'], startCol: 5, cellW: 10 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// First line left = startCol * cellW
|
||||
expect(lineDiv.style.left).toBe('50px');
|
||||
});
|
||||
it('offsets first line by startCol', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['hi'], startCol: 5, cellW: 10 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// First line left = startCol * cellW
|
||||
expect(lineDiv.style.left).toBe('50px');
|
||||
});
|
||||
|
||||
it('renders cursor at end of text', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({
|
||||
lines: ['ab'],
|
||||
startCol: 3,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
showCursor: true,
|
||||
cursorColor: '#ff00ff',
|
||||
}));
|
||||
it('renders cursor at end of text', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['ab'],
|
||||
startCol: 3,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
showCursor: true,
|
||||
cursorColor: '#ff00ff',
|
||||
})
|
||||
);
|
||||
|
||||
// Last child is cursor (after line div)
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
// cursorCol = startCol(3) + text.length(2) = 5
|
||||
expect(cursor.style.left).toBe('50px');
|
||||
expect(cursor.style.width).toBe('10px');
|
||||
expect(cursor.style.height).toBe('20px');
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(cursor.style.backgroundColor).toBe('rgb(255, 0, 255)');
|
||||
});
|
||||
// Last child is cursor (after line div)
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
// cursorCol = startCol(3) + text.length(2) = 5
|
||||
expect(cursor.style.left).toBe('50px');
|
||||
expect(cursor.style.width).toBe('10px');
|
||||
expect(cursor.style.height).toBe('20px');
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(cursor.style.backgroundColor).toBe('rgb(255, 0, 255)');
|
||||
});
|
||||
|
||||
it('does not render cursor when showCursor is false', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['ab'], showCursor: false }));
|
||||
// Only line div, no cursor
|
||||
expect(container.children.length).toBe(1);
|
||||
});
|
||||
it('does not render cursor when showCursor is false', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['ab'], showCursor: false }));
|
||||
// Only line div, no cursor
|
||||
expect(container.children.length).toBe(1);
|
||||
});
|
||||
|
||||
it('renders multi-line text', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({
|
||||
lines: ['first', 'second'],
|
||||
startCol: 5,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
}));
|
||||
it('renders multi-line text', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['first', 'second'],
|
||||
startCol: 5,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
})
|
||||
);
|
||||
|
||||
// 2 line divs + cursor
|
||||
expect(container.children.length).toBe(3);
|
||||
// 2 line divs + cursor
|
||||
expect(container.children.length).toBe(3);
|
||||
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.style.left).toBe('50px'); // startCol * cellW
|
||||
expect(line1.style.top).toBe('0px');
|
||||
expect(line1.children.length).toBe(5); // 'first'
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
expect(line1.style.left).toBe('50px'); // startCol * cellW
|
||||
expect(line1.style.top).toBe('0px');
|
||||
expect(line1.children.length).toBe(5); // 'first'
|
||||
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.style.left).toBe('0px'); // wrapped lines start at col 0
|
||||
expect(line2.style.top).toBe('20px'); // second row
|
||||
expect(line2.children.length).toBe(6); // 'second'
|
||||
});
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line2.style.left).toBe('0px'); // wrapped lines start at col 0
|
||||
expect(line2.style.top).toBe('20px'); // second row
|
||||
expect(line2.children.length).toBe(6); // 'second'
|
||||
});
|
||||
|
||||
it('clears previous content on re-render', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'] }));
|
||||
expect(container.children.length).toBe(2); // line + cursor
|
||||
it('clears previous content on re-render', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'] }));
|
||||
expect(container.children.length).toBe(2); // line + cursor
|
||||
|
||||
renderOverlay(container, makeParams({ lines: ['xy'] }));
|
||||
expect(container.children.length).toBe(2); // line + cursor (rebuilt)
|
||||
renderOverlay(container, makeParams({ lines: ['xy'] }));
|
||||
expect(container.children.length).toBe(2); // line + cursor (rebuilt)
|
||||
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(2); // x, y
|
||||
});
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(2); // x, y
|
||||
});
|
||||
|
||||
it('shows container (display not none)', () => {
|
||||
const container = document.createElement('div');
|
||||
container.style.display = 'none';
|
||||
renderOverlay(container, makeParams());
|
||||
expect(container.style.display).toBe('');
|
||||
});
|
||||
it('shows container (display not none)', () => {
|
||||
const container = document.createElement('div');
|
||||
container.style.display = 'none';
|
||||
renderOverlay(container, makeParams());
|
||||
expect(container.style.display).toBe('');
|
||||
});
|
||||
|
||||
// ─── Anti-flicker / compositing seam tests ────────────────────
|
||||
// ─── Anti-flicker / compositing seam tests ────────────────────
|
||||
|
||||
it('line div height extends 1px past cellH to cover compositing seam', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// cellH + 1 = 20px — the extra 1px covers the compositing seam
|
||||
expect(lineDiv.style.height).toBe('20px');
|
||||
});
|
||||
it('line div height extends 1px past cellH to cover compositing seam', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['abc'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// cellH + 1 = 20px — the extra 1px covers the compositing seam
|
||||
expect(lineDiv.style.height).toBe('20px');
|
||||
});
|
||||
|
||||
it('line div height is cellH+1 for various cell heights', () => {
|
||||
for (const cellH of [15, 17, 19, 22]) {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['x'], cellH }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.style.height).toBe((cellH + 1) + 'px');
|
||||
}
|
||||
});
|
||||
it('line div height is cellH+1 for various cell heights', () => {
|
||||
for (const cellH of [15, 17, 19, 22]) {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['x'], cellH }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.style.height).toBe(cellH + 1 + 'px');
|
||||
}
|
||||
});
|
||||
|
||||
it('multi-line overlay has cellH+1 height on each line div', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({
|
||||
lines: ['first', 'second'],
|
||||
cellH: 19,
|
||||
}));
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line1.style.height).toBe('20px');
|
||||
expect(line2.style.height).toBe('20px');
|
||||
});
|
||||
it('multi-line overlay has cellH+1 height on each line div', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['first', 'second'],
|
||||
cellH: 19,
|
||||
})
|
||||
);
|
||||
const line1 = container.children[0] as HTMLDivElement;
|
||||
const line2 = container.children[1] as HTMLDivElement;
|
||||
expect(line1.style.height).toBe('20px');
|
||||
expect(line2.style.height).toBe('20px');
|
||||
});
|
||||
|
||||
// ─── Span vertical centering tests ────────────────────────────
|
||||
// ─── Span vertical centering tests ────────────────────────────
|
||||
|
||||
it('span uses full cellH for height and lineHeight (CSS centering)', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.height).toBe('19px');
|
||||
expect(span.style.lineHeight).toBe('19px');
|
||||
});
|
||||
it('span uses full cellH for height and lineHeight (CSS centering)', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.height).toBe('19px');
|
||||
expect(span.style.lineHeight).toBe('19px');
|
||||
});
|
||||
|
||||
it('span top is 0px (no vertical offset / no transform)', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.top).toBe('0px');
|
||||
// No translateY transform — sub-pixel overhang causes artifacts
|
||||
expect(span.style.transform).toBe('');
|
||||
});
|
||||
it('span top is 0px (no vertical offset / no transform)', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'], cellH: 19 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.top).toBe('0px');
|
||||
// No translateY transform — sub-pixel overhang causes artifacts
|
||||
expect(span.style.transform).toBe('');
|
||||
});
|
||||
|
||||
// ─── Font rendering tests ─────────────────────────────────────
|
||||
// ─── Font rendering tests ─────────────────────────────────────
|
||||
|
||||
it('span disables ligatures via font-feature-settings', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['fi'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
// Check cssText includes the ligature-disabling settings
|
||||
// jsdom may normalize whitespace; check that both liga and calt are disabled
|
||||
expect(span.style.cssText).toContain("font-feature-settings:");
|
||||
expect(span.style.cssText).toContain("'liga' 0");
|
||||
expect(span.style.cssText).toContain("'calt' 0");
|
||||
});
|
||||
it('span disables ligatures via font-feature-settings', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['fi'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
// Check cssText includes the ligature-disabling settings
|
||||
// jsdom may normalize whitespace; check that both liga and calt are disabled
|
||||
expect(span.style.cssText).toContain('font-feature-settings:');
|
||||
expect(span.style.cssText).toContain("'liga' 0");
|
||||
expect(span.style.cssText).toContain("'calt' 0");
|
||||
});
|
||||
|
||||
it('span has text-align: center for glyph centering', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['m'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.textAlign).toBe('center');
|
||||
});
|
||||
it('span has text-align: center for glyph centering', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['m'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.textAlign).toBe('center');
|
||||
});
|
||||
|
||||
it('span has pointer-events: none', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.pointerEvents).toBe('none');
|
||||
});
|
||||
it('span has pointer-events: none', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a'] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
const span = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(span.style.pointerEvents).toBe('none');
|
||||
});
|
||||
|
||||
// ─── Multi-line cursor positioning ────────────────────────────
|
||||
// ─── Multi-line cursor positioning ────────────────────────────
|
||||
|
||||
it('cursor on wrapped line uses col 0 as base', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({
|
||||
lines: ['first', 'ab'],
|
||||
startCol: 5,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
showCursor: true,
|
||||
}));
|
||||
// Cursor at end of second line: col = 0 + 2 = 2
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('20px'); // 2 * 10
|
||||
expect(cursor.style.top).toBe('20px'); // row 1 * cellH
|
||||
});
|
||||
it('cursor on wrapped line uses col 0 as base', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['first', 'ab'],
|
||||
startCol: 5,
|
||||
cellW: 10,
|
||||
cellH: 20,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// Cursor at end of second line: col = 0 + 2 = 2
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('20px'); // 2 * 10
|
||||
expect(cursor.style.top).toBe('20px'); // row 1 * cellH
|
||||
});
|
||||
|
||||
// ─── charTop/charHeight passed through ────────────────────────
|
||||
// ─── charTop/charHeight passed through ────────────────────────
|
||||
|
||||
it('accepts charTop and charHeight params without error', () => {
|
||||
const container = document.createElement('div');
|
||||
expect(() => renderOverlay(container, makeParams({
|
||||
lines: ['test'],
|
||||
charTop: 2,
|
||||
charHeight: 14,
|
||||
}))).not.toThrow();
|
||||
expect(container.children.length).toBeGreaterThan(0);
|
||||
});
|
||||
it('accepts charTop and charHeight params without error', () => {
|
||||
const container = document.createElement('div');
|
||||
expect(() =>
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['test'],
|
||||
charTop: 2,
|
||||
charHeight: 14,
|
||||
})
|
||||
)
|
||||
).not.toThrow();
|
||||
expect(container.children.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// ─── Line div positioning regression ──────────────────────────
|
||||
// ─── Line div positioning regression ──────────────────────────
|
||||
|
||||
it('line div background color matches font.backgroundColor', () => {
|
||||
const container = document.createElement('div');
|
||||
const font: FontStyle = { ...FONT, backgroundColor: '#1a1a1a' };
|
||||
renderOverlay(container, makeParams({ lines: ['x'], font }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(lineDiv.style.backgroundColor).toBe('rgb(26, 26, 26)');
|
||||
});
|
||||
it('line div background color matches font.backgroundColor', () => {
|
||||
const container = document.createElement('div');
|
||||
const font: FontStyle = { ...FONT, backgroundColor: '#1a1a1a' };
|
||||
renderOverlay(container, makeParams({ lines: ['x'], font }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// jsdom normalizes hex to rgb()
|
||||
expect(lineDiv.style.backgroundColor).toBe('rgb(26, 26, 26)');
|
||||
});
|
||||
|
||||
it('empty line produces line div with no spans', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: [''] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(0);
|
||||
});
|
||||
it('empty line produces line div with no spans', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: [''] }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(0);
|
||||
});
|
||||
|
||||
// ─── CJK wide character support ───────────────────────────────
|
||||
|
||||
it('CJK characters get double-width spans', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['a你b'], cellW: 10 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
expect(lineDiv.children.length).toBe(3);
|
||||
|
||||
const spanA = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(spanA.textContent).toBe('a');
|
||||
expect(spanA.style.left).toBe('0px');
|
||||
expect(spanA.style.width).toBe('10px'); // 1 cell
|
||||
|
||||
const spanCJK = lineDiv.children[1] as HTMLSpanElement;
|
||||
expect(spanCJK.textContent).toBe('你');
|
||||
expect(spanCJK.style.left).toBe('10px'); // col 1
|
||||
expect(spanCJK.style.width).toBe('20px'); // 2 cells
|
||||
|
||||
const spanB = lineDiv.children[2] as HTMLSpanElement;
|
||||
expect(spanB.textContent).toBe('b');
|
||||
expect(spanB.style.left).toBe('30px'); // col 3
|
||||
expect(spanB.style.width).toBe('10px'); // 1 cell
|
||||
});
|
||||
|
||||
it('cursor position accounts for CJK width', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(
|
||||
container,
|
||||
makeParams({
|
||||
lines: ['你好'],
|
||||
startCol: 2,
|
||||
cellW: 10,
|
||||
showCursor: true,
|
||||
})
|
||||
);
|
||||
// 你(2) + 好(2) = 4 visual cols, cursor at startCol(2) + 4 = 6
|
||||
const cursor = container.children[container.children.length - 1] as HTMLSpanElement;
|
||||
expect(cursor.style.left).toBe('60px');
|
||||
});
|
||||
|
||||
it('mixed ASCII and CJK characters position correctly', () => {
|
||||
const container = document.createElement('div');
|
||||
renderOverlay(container, makeParams({ lines: ['hi你'], cellW: 8 }));
|
||||
const lineDiv = container.children[0] as HTMLDivElement;
|
||||
// h(col 0), i(col 1), 你(col 2, width 2)
|
||||
const spanH = lineDiv.children[0] as HTMLSpanElement;
|
||||
expect(spanH.style.left).toBe('0px');
|
||||
const spanI = lineDiv.children[1] as HTMLSpanElement;
|
||||
expect(spanI.style.left).toBe('8px');
|
||||
const spanCJK = lineDiv.children[2] as HTMLSpanElement;
|
||||
expect(spanCJK.style.left).toBe('16px');
|
||||
expect(spanCJK.style.width).toBe('16px');
|
||||
});
|
||||
});
|
||||
|
||||
describe('charCellWidth', () => {
|
||||
it('returns 1 for ASCII characters', () => {
|
||||
expect(charCellWidth(null, 'a')).toBe(1);
|
||||
expect(charCellWidth(null, '!')).toBe(1);
|
||||
expect(charCellWidth(null, ' ')).toBe(1);
|
||||
});
|
||||
|
||||
it('returns 2 for CJK ideographs', () => {
|
||||
expect(charCellWidth(null, '你')).toBe(2);
|
||||
expect(charCellWidth(null, '好')).toBe(2);
|
||||
expect(charCellWidth(null, '中')).toBe(2);
|
||||
});
|
||||
|
||||
it('returns 2 for Japanese hiragana', () => {
|
||||
expect(charCellWidth(null, 'こ')).toBe(2);
|
||||
expect(charCellWidth(null, 'ん')).toBe(2);
|
||||
});
|
||||
|
||||
it('returns 2 for Korean syllables', () => {
|
||||
expect(charCellWidth(null, '안')).toBe(2);
|
||||
expect(charCellWidth(null, '녕')).toBe(2);
|
||||
});
|
||||
|
||||
it('returns 2 for fullwidth forms', () => {
|
||||
expect(charCellWidth(null, '\uff01')).toBe(2); // !
|
||||
expect(charCellWidth(null, '\uff21')).toBe(2); // A
|
||||
});
|
||||
|
||||
it('uses terminal unicode addon when available', () => {
|
||||
const mockTerminal = {
|
||||
unicode: { getStringCellWidth: (s: string) => (s === 'W' ? 2 : 1) },
|
||||
} as any;
|
||||
expect(charCellWidth(mockTerminal, 'W')).toBe(2);
|
||||
expect(charCellWidth(mockTerminal, 'n')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stringCellWidth', () => {
|
||||
it('sums individual character widths', () => {
|
||||
expect(stringCellWidth(null, 'abc')).toBe(3);
|
||||
expect(stringCellWidth(null, '你好')).toBe(4);
|
||||
expect(stringCellWidth(null, 'a你b')).toBe(4);
|
||||
});
|
||||
|
||||
it('returns 0 for empty string', () => {
|
||||
expect(stringCellWidth(null, '')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,6 +19,10 @@ const PORTS = {
|
||||
|
||||
const results = [];
|
||||
|
||||
function isCodemanTitle(title) {
|
||||
return typeof title === 'string' && title.startsWith('codeman:');
|
||||
}
|
||||
|
||||
function logSection(title) {
|
||||
console.log('\n' + '='.repeat(60));
|
||||
console.log(` ${title}`);
|
||||
@@ -88,7 +92,7 @@ async function main() {
|
||||
const page = await playwrightBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.playwright}`);
|
||||
const title = await page.title();
|
||||
if (title !== 'Codeman') throw new Error(`Expected Codeman, got ${title}`);
|
||||
if (!isCodemanTitle(title)) throw new Error(`Expected codeman:<hostname>, got ${title}`);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
@@ -149,7 +153,7 @@ async function main() {
|
||||
const page = await puppeteerBrowser.newPage();
|
||||
await page.goto(`http://localhost:${PORTS.puppeteer}`);
|
||||
const title = await page.title();
|
||||
if (title !== 'Codeman') throw new Error(`Expected Codeman, got ${title}`);
|
||||
if (!isCodemanTitle(title)) throw new Error(`Expected codeman:<hostname>, got ${title}`);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
@@ -202,7 +206,7 @@ async function main() {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
const title = agentBrowserJson('get title');
|
||||
agentBrowserAvailable = title.title === 'Codeman';
|
||||
agentBrowserAvailable = isCodemanTitle(title.title);
|
||||
console.log(' Browser launched');
|
||||
|
||||
// Test 1: Page load
|
||||
@@ -210,7 +214,7 @@ async function main() {
|
||||
agentBrowser(`open http://localhost:${PORTS.agentBrowser}`);
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
const title = agentBrowserJson('get title');
|
||||
if (title.title !== 'Codeman') throw new Error(`Expected Codeman, got ${title.title}`);
|
||||
if (!isCodemanTitle(title.title)) throw new Error(`Expected codeman:<hostname>, got ${title.title}`);
|
||||
});
|
||||
|
||||
// Test 2: Element selection
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @fileoverview Build the opt-in gesture-overlay bundle from its vendored source
|
||||
* in `packages/gesture-control/` into the web bundle Codeman actually serves,
|
||||
* `src/web/public/gesture/gesture-codeman.js`.
|
||||
*
|
||||
* The source lives in this repo (workspace package `codeman-gesture-control`,
|
||||
* was the standalone `Ark0N/codeman-gesture-control` repo before it was vendored
|
||||
* in). `src/codeman/entry.ts` is the Codeman *consumer* entry — it imports the
|
||||
* transport-agnostic gesture core (`src/gesture/*`) and maps grab/drag/drop onto
|
||||
* Codeman's real session tabs + toolbar buttons. esbuild bundles it (MediaPipe
|
||||
* tasks-vision JS included) into a single ESM file; the MediaPipe wasm + model
|
||||
* are loaded at runtime from same-origin `/gesture/wasm` + `/gesture/*.task`
|
||||
* (fetched separately by scripts/fetch-gesture-assets.mjs), NOT bundled here.
|
||||
*
|
||||
* Run it after editing anything under packages/gesture-control/src/ and commit
|
||||
* the regenerated bundle (the committed copy is what `npm run dev` / tsx serves,
|
||||
* since the web UI ships as plain JS with no bundler). `npm run build` also runs
|
||||
* this so a production build always reflects the current source.
|
||||
*
|
||||
* Usage: node scripts/build-gesture-bundle.mjs [--out <path>]
|
||||
* --out output file (default src/web/public/gesture/gesture-codeman.js)
|
||||
*
|
||||
* NOT minified — matches the historical bundle and keeps it debuggable; the
|
||||
* build's compress step gzips/brotlis it for production anyway.
|
||||
*/
|
||||
import { build } from 'esbuild';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { join, dirname, isAbsolute } from 'node:path';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const ENTRY = join(ROOT, 'packages/gesture-control/src/codeman/entry.ts');
|
||||
const DEFAULT_OUT = join(ROOT, 'src/web/public/gesture/gesture-codeman.js');
|
||||
|
||||
const outArg = process.argv[process.argv.indexOf('--out') + 1];
|
||||
const outfile =
|
||||
process.argv.includes('--out') && outArg
|
||||
? isAbsolute(outArg)
|
||||
? outArg
|
||||
: join(process.cwd(), outArg)
|
||||
: DEFAULT_OUT;
|
||||
|
||||
await build({
|
||||
entryPoints: [ENTRY],
|
||||
bundle: true,
|
||||
format: 'esm',
|
||||
target: 'es2020',
|
||||
outfile,
|
||||
logLevel: 'info',
|
||||
});
|
||||
|
||||
console.log(`[gesture] bundle built → ${outfile}`);
|
||||
@@ -8,13 +8,15 @@
|
||||
* 2. Copy static assets (web/public, templates)
|
||||
* 3. Build vendor xterm bundles
|
||||
* 4. Minify frontend assets (app.js, styles.css, mobile.css)
|
||||
* 5. Compress with gzip + brotli
|
||||
* 5. Content-hash cache busting (rename assets, rewrite index.html)
|
||||
* 6. Compress with gzip + brotli
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { appendFileSync } from 'fs';
|
||||
import { appendFileSync, readFileSync, writeFileSync, renameSync } from 'fs';
|
||||
import { createHash } from 'crypto';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { join } from 'path';
|
||||
import { join, extname, basename, dirname } from 'path';
|
||||
|
||||
const ROOT = join(fileURLToPath(import.meta.url), '..', '..');
|
||||
|
||||
@@ -27,17 +29,24 @@ function run(label, cmd) {
|
||||
run('tsc', 'tsc');
|
||||
run('chmod dist/index.js', 'chmod +x dist/index.js');
|
||||
|
||||
// 2. Copy static assets
|
||||
// 2. Copy static assets (clean first to remove stale hashed files from previous builds)
|
||||
run('clean public', 'rm -rf dist/web/public');
|
||||
run('prepare dirs', 'mkdir -p dist/web dist/templates dist/web/public/vendor');
|
||||
// Fetch the opt-in gesture overlay's MediaPipe wasm + model into src/ (idempotent,
|
||||
// non-fatal, kept out of git) so the copy below carries them into dist/.
|
||||
run('gesture assets', 'node scripts/fetch-gesture-assets.mjs');
|
||||
// Rebuild the gesture overlay bundle from its vendored source (packages/gesture-control)
|
||||
// into src/web/public/gesture/gesture-codeman.js, so prod always reflects current source.
|
||||
run('gesture bundle', 'node scripts/build-gesture-bundle.mjs');
|
||||
run('copy web assets', 'cp -r src/web/public dist/web/');
|
||||
run('copy template', 'cp src/templates/case-template.md dist/templates/');
|
||||
|
||||
// 3. Vendor xterm bundles
|
||||
run('xterm css', 'cp node_modules/xterm/css/xterm.css dist/web/public/vendor/');
|
||||
run('xterm js', 'npx esbuild node_modules/xterm/lib/xterm.js --minify --outfile=dist/web/public/vendor/xterm.min.js');
|
||||
run('xterm-addon-fit', 'npx esbuild node_modules/xterm-addon-fit/lib/xterm-addon-fit.js --minify --outfile=dist/web/public/vendor/xterm-addon-fit.min.js');
|
||||
run('xterm-addon-webgl', 'cp node_modules/xterm-addon-webgl/lib/xterm-addon-webgl.js dist/web/public/vendor/xterm-addon-webgl.min.js');
|
||||
run('xterm-addon-unicode11', 'npx esbuild node_modules/xterm-addon-unicode11/lib/xterm-addon-unicode11.js --minify --outfile=dist/web/public/vendor/xterm-addon-unicode11.min.js');
|
||||
// 3. Vendor xterm bundles (xterm.js 6.x — @xterm scoped packages)
|
||||
run('xterm css', 'cp node_modules/@xterm/xterm/css/xterm.css dist/web/public/vendor/');
|
||||
run('xterm js', 'npx esbuild node_modules/@xterm/xterm/lib/xterm.js --minify --outfile=dist/web/public/vendor/xterm.min.js');
|
||||
run('xterm-addon-fit', 'npx esbuild node_modules/@xterm/addon-fit/lib/addon-fit.js --minify --outfile=dist/web/public/vendor/xterm-addon-fit.min.js');
|
||||
run('xterm-addon-webgl', 'cp node_modules/@xterm/addon-webgl/lib/addon-webgl.js dist/web/public/vendor/xterm-addon-webgl.min.js');
|
||||
run('xterm-addon-unicode11', 'npx esbuild node_modules/@xterm/addon-unicode11/lib/addon-unicode11.js --minify --outfile=dist/web/public/vendor/xterm-addon-unicode11.min.js');
|
||||
run('xterm-zerolag-input', 'npx esbuild packages/xterm-zerolag-input/src/zerolag-input-addon.ts --bundle --minify --format=iife --global-name=XtermZerolagInput --outfile=dist/web/public/vendor/xterm-zerolag-input.js');
|
||||
|
||||
// Append global aliases so app.js can use `new LocalEchoOverlay(terminal)`
|
||||
@@ -56,11 +65,68 @@ appendFileSync(
|
||||
);
|
||||
|
||||
// 4. Minify frontend assets
|
||||
run('minify app.js', 'npx esbuild dist/web/public/app.js --minify --drop:console --outfile=dist/web/public/app.js --allow-overwrite');
|
||||
run('minify input-cjk.js', 'npx esbuild dist/web/public/input-cjk.js --minify --outfile=dist/web/public/input-cjk.js --allow-overwrite');
|
||||
run('minify app.js', 'npx esbuild dist/web/public/app.js --minify --outfile=dist/web/public/app.js --allow-overwrite');
|
||||
run('minify terminal-ui.js', 'npx esbuild dist/web/public/terminal-ui.js --minify --outfile=dist/web/public/terminal-ui.js --allow-overwrite');
|
||||
run('minify respawn-ui.js', 'npx esbuild dist/web/public/respawn-ui.js --minify --outfile=dist/web/public/respawn-ui.js --allow-overwrite');
|
||||
run('minify ralph-panel.js', 'npx esbuild dist/web/public/ralph-panel.js --minify --outfile=dist/web/public/ralph-panel.js --allow-overwrite');
|
||||
run('minify settings-ui.js', 'npx esbuild dist/web/public/settings-ui.js --minify --outfile=dist/web/public/settings-ui.js --allow-overwrite');
|
||||
run('minify panels-ui.js', 'npx esbuild dist/web/public/panels-ui.js --minify --outfile=dist/web/public/panels-ui.js --allow-overwrite');
|
||||
run('minify session-ui.js', 'npx esbuild dist/web/public/session-ui.js --minify --outfile=dist/web/public/session-ui.js --allow-overwrite');
|
||||
run('minify styles.css', 'npx esbuild dist/web/public/styles.css --minify --outfile=dist/web/public/styles.css --allow-overwrite');
|
||||
run('minify mobile.css', 'npx esbuild dist/web/public/mobile.css --minify --outfile=dist/web/public/mobile.css --allow-overwrite');
|
||||
|
||||
// 5. Compress with gzip + brotli
|
||||
// 5. Content-hash cache busting
|
||||
console.log('\n[build] content-hash cache busting');
|
||||
{
|
||||
const distPublic = join(ROOT, 'dist/web/public');
|
||||
const HASHABLE = [
|
||||
'styles.css',
|
||||
'mobile.css',
|
||||
'constants.js',
|
||||
'mobile-handlers.js',
|
||||
'voice-input.js',
|
||||
'notification-manager.js',
|
||||
'keyboard-accessory.js',
|
||||
'input-cjk.js',
|
||||
'app.js',
|
||||
'terminal-ui.js',
|
||||
'respawn-ui.js',
|
||||
'ralph-panel.js',
|
||||
'settings-ui.js',
|
||||
'panels-ui.js',
|
||||
'session-ui.js',
|
||||
'ralph-wizard.js',
|
||||
'api-client.js',
|
||||
'subagent-windows.js',
|
||||
'image-input.js',
|
||||
'vendor/xterm-zerolag-input.js',
|
||||
];
|
||||
const manifest = {};
|
||||
for (const file of HASHABLE) {
|
||||
const filePath = join(distPublic, file);
|
||||
const content = readFileSync(filePath);
|
||||
const hash = createHash('md5').update(content).digest('hex').slice(0, 8);
|
||||
const ext = extname(file);
|
||||
const base = basename(file, ext);
|
||||
const dir = dirname(file);
|
||||
const hashed = dir === '.' ? `${base}.${hash}${ext}` : `${dir}/${base}.${hash}${ext}`;
|
||||
renameSync(filePath, join(distPublic, hashed));
|
||||
manifest[file] = hashed;
|
||||
}
|
||||
// Rewrite index.html to reference hashed filenames
|
||||
let html = readFileSync(join(distPublic, 'index.html'), 'utf8');
|
||||
for (const [original, hashed] of Object.entries(manifest)) {
|
||||
html = html.replaceAll(`"${original}"`, `"${hashed}"`);
|
||||
}
|
||||
writeFileSync(join(distPublic, 'index.html'), html);
|
||||
console.log(' Hashed files:');
|
||||
for (const [orig, hashed] of Object.entries(manifest)) {
|
||||
console.log(` ${orig} -> ${hashed}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Compress with gzip + brotli
|
||||
run(
|
||||
'compress',
|
||||
`for f in dist/web/public/*.js dist/web/public/*.css dist/web/public/*.html dist/web/public/vendor/*.js dist/web/public/vendor/*.css; do` +
|
||||
|
||||
@@ -428,7 +428,7 @@ const SUBAGENT_ACTIVITY = {
|
||||
'agent-002': [
|
||||
{ type: 'tool', tool: 'Glob', input: { pattern: 'test/**/*.test.ts' }, timestamp: new Date().toISOString(), agentId: 'agent-002' },
|
||||
{ type: 'tool', tool: 'Read', input: { file_path: '/home/arkon/codeman/test/respawn-test-utils.ts' }, timestamp: new Date().toISOString(), agentId: 'agent-002' },
|
||||
{ type: 'tool', tool: 'Read', input: { file_path: '/home/arkon/codeman/vitest.config.ts' }, timestamp: new Date().toISOString(), agentId: 'agent-002' },
|
||||
{ type: 'tool', tool: 'Read', input: { file_path: '/home/arkon/codeman/config/vitest.config.ts' }, timestamp: new Date().toISOString(), agentId: 'agent-002' },
|
||||
{ type: 'message', role: 'assistant', text: 'Analyzing test patterns: MockSession, unique ports, fileParallelism: false...', timestamp: new Date().toISOString(), agentId: 'agent-002' },
|
||||
],
|
||||
};
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
*
|
||||
* Usage: node scripts/capture-video-screenshots.mjs
|
||||
* Port: 3198 (static file server)
|
||||
* Output: remotion/public/ (6 PNGs)
|
||||
* Output: scripts/scripts/remotion/public/ (6 PNGs)
|
||||
*/
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
@@ -21,7 +21,7 @@ import { fileURLToPath } from 'url';
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const PROJECT_ROOT = join(__dirname, '..');
|
||||
const PUBLIC_DIR = join(PROJECT_ROOT, 'src', 'web', 'public');
|
||||
const OUTPUT_DIR = join(PROJECT_ROOT, 'remotion', 'public');
|
||||
const OUTPUT_DIR = join(PROJECT_ROOT, 'scripts', 'remotion', 'public');
|
||||
const PORT = 3198;
|
||||
|
||||
const DESKTOP_VIEWPORT = { width: 1920, height: 1080 };
|
||||
@@ -516,7 +516,7 @@ async function captureDesktopWelcome(browser) {
|
||||
path: join(OUTPUT_DIR, 'desktop-welcome.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/desktop-welcome.png');
|
||||
console.log(' Saved: scripts/remotion/public/desktop-welcome.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -545,7 +545,7 @@ async function captureDesktopClaude(browser) {
|
||||
path: join(OUTPUT_DIR, 'desktop-claude.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/desktop-claude.png');
|
||||
console.log(' Saved: scripts/remotion/public/desktop-claude.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -575,7 +575,7 @@ async function captureDesktopBothClaude(browser) {
|
||||
path: join(OUTPUT_DIR, 'desktop-both-claude.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/desktop-both-claude.png');
|
||||
console.log(' Saved: scripts/remotion/public/desktop-both-claude.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -605,7 +605,7 @@ async function captureDesktopBothOpencode(browser) {
|
||||
path: join(OUTPUT_DIR, 'desktop-both-opencode.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/desktop-both-opencode.png');
|
||||
console.log(' Saved: scripts/remotion/public/desktop-both-opencode.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -634,7 +634,7 @@ async function captureMobileClaude(browser) {
|
||||
path: join(OUTPUT_DIR, 'mobile-claude.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/mobile-claude.png');
|
||||
console.log(' Saved: scripts/remotion/public/mobile-claude.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -663,7 +663,7 @@ async function captureMobileOpencode(browser) {
|
||||
path: join(OUTPUT_DIR, 'mobile-opencode.png'),
|
||||
fullPage: false,
|
||||
});
|
||||
console.log(' Saved: remotion/public/mobile-opencode.png');
|
||||
console.log(' Saved: scripts/remotion/public/mobile-opencode.png');
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
@@ -706,12 +706,12 @@ async function main() {
|
||||
console.log('All 6 screenshots captured!');
|
||||
console.log('='.repeat(60));
|
||||
console.log('\nOutput files:');
|
||||
console.log(' remotion/public/desktop-welcome.png');
|
||||
console.log(' remotion/public/desktop-claude.png');
|
||||
console.log(' remotion/public/desktop-both-claude.png');
|
||||
console.log(' remotion/public/desktop-both-opencode.png');
|
||||
console.log(' remotion/public/mobile-claude.png');
|
||||
console.log(' remotion/public/mobile-opencode.png');
|
||||
console.log(' scripts/remotion/public/desktop-welcome.png');
|
||||
console.log(' scripts/remotion/public/desktop-claude.png');
|
||||
console.log(' scripts/remotion/public/desktop-both-claude.png');
|
||||
console.log(' scripts/remotion/public/desktop-both-opencode.png');
|
||||
console.log(' scripts/remotion/public/mobile-claude.png');
|
||||
console.log(' scripts/remotion/public/mobile-opencode.png');
|
||||
} catch (err) {
|
||||
console.error('\nFatal error:', err.message);
|
||||
console.error(err.stack);
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env node
|
||||
// Fails if package-lock.json's version fields don't match package.json.
|
||||
// Changesets bumps package.json but NOT the lockfile — this catches that drift
|
||||
// (the top-level `version` in lockfiles is metadata, so `npm ci` won't flag it).
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const pkg = JSON.parse(readFileSync(resolve(repoRoot, 'package.json'), 'utf8'));
|
||||
const lock = JSON.parse(readFileSync(resolve(repoRoot, 'package-lock.json'), 'utf8'));
|
||||
|
||||
const expected = pkg.version;
|
||||
const rootVersion = lock.version;
|
||||
const selfVersion = lock.packages?.['']?.version;
|
||||
|
||||
const mismatches = [];
|
||||
if (rootVersion !== expected) mismatches.push(` package-lock.json#.version = ${rootVersion} (expected ${expected})`);
|
||||
if (selfVersion !== expected) mismatches.push(` package-lock.json#.packages[""].version = ${selfVersion} (expected ${expected})`);
|
||||
|
||||
if (mismatches.length > 0) {
|
||||
console.error(`\nLockfile version drift detected (package.json is ${expected}):`);
|
||||
console.error(mismatches.join('\n'));
|
||||
console.error('\nFix: run `npm install --package-lock-only` and commit the updated package-lock.json.\n');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Lockfile in sync with package.json (${expected}).`);
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readdirSync, readFileSync } from 'node:fs';
|
||||
import { dirname, extname, join, relative, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const publicRoot = resolve(repoRoot, 'src/web/public');
|
||||
const prettierBin = resolve(repoRoot, 'node_modules/.bin/prettier');
|
||||
const checkedExtensions = new Set(['.js', '.css', '.html', '.json']);
|
||||
|
||||
function collectTextAssets(dir) {
|
||||
const files = [];
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const fullPath = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...collectTextAssets(fullPath));
|
||||
continue;
|
||||
}
|
||||
if (checkedExtensions.has(extname(entry.name))) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function findNullByte(buffer) {
|
||||
for (let i = 0; i < buffer.length; i += 1) {
|
||||
if (buffer[i] === 0) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
const files = collectTextAssets(publicRoot);
|
||||
const failures = [];
|
||||
|
||||
for (const file of files) {
|
||||
const rel = relative(repoRoot, file);
|
||||
const data = readFileSync(file);
|
||||
const nullByteIndex = findNullByte(data);
|
||||
if (nullByteIndex !== -1) {
|
||||
failures.push(`${rel}: contains literal NUL byte at offset ${nullByteIndex}`);
|
||||
}
|
||||
|
||||
if (extname(file) === '.js') {
|
||||
try {
|
||||
execFileSync(process.execPath, ['--check', file], { cwd: repoRoot, stdio: 'pipe' });
|
||||
} catch (err) {
|
||||
failures.push(`${rel}: JavaScript syntax check failed\n${String(err.stderr || err.message).trim()}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
execFileSync(prettierBin, ['--check', ...files], { cwd: repoRoot, stdio: 'pipe' });
|
||||
} catch (err) {
|
||||
failures.push(`Prettier public asset check failed\n${String(err.stdout || err.stderr || err.message).trim()}`);
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(failures.join('\n\n'));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Public asset checks passed (${files.length} files).`);
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Codeman Cloudflare Named Tunnel
|
||||
After=network-online.target codeman-web.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/cloudflared tunnel --config %h/.cloudflared/codeman.yml run codeman
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
KillMode=process
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=codeman-tunnel-named
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
@@ -11,6 +11,15 @@ RestartSec=5
|
||||
KillMode=process
|
||||
Environment=NODE_ENV=production
|
||||
Environment=HOME=/home/arkon
|
||||
Environment=NODE_COMPILE_CACHE=/home/arkon/.codeman/compile-cache
|
||||
# Loopback bind (default, no --host) + no password: safe out of the box. Hooks
|
||||
# reach 127.0.0.1, and `tailscale serve` fronts it on the tailnet only (real
|
||||
# cert, no LAN exposure, no app login). To expose on the LAN instead, add
|
||||
# Environment=CODEMAN_HOST=0.0.0.0 + Environment=CODEMAN_PASSWORD=... .
|
||||
# See docs/security-architecture.md.
|
||||
Environment=CODEMAN_GESTURE=1
|
||||
# ^ Makes the gesture-control overlay AVAILABLE (CSP widening + /gesture/ assets);
|
||||
# the actual on/off stays the per-user App Settings toggle (default OFF).
|
||||
|
||||
# Logging
|
||||
StandardOutput=journal
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* @fileoverview Fetch the gesture-overlay runtime assets (MediaPipe wasm + the
|
||||
* gesture-recognizer model) into src/web/public/gesture/ so Codeman can serve
|
||||
* them same-origin (a browser content-blocker otherwise blocks the public CDNs
|
||||
* and the overlay fails to start). These are large binaries (~27 MB) kept OUT of
|
||||
* git (ignored explicitly via `src/web/public/gesture/wasm/` + `*.task` in
|
||||
* .gitignore); they are fetched here at install (postinstall) and build time.
|
||||
*
|
||||
* Idempotent: skips files already present. Non-fatal: the gesture overlay is
|
||||
* opt-in (CODEMAN_GESTURE=1), so a fetch failure only warns — it must not break
|
||||
* `npm install` / `npm run build`. The build then copies src/web/public into
|
||||
* dist/ as usual, so prod gets these too.
|
||||
*
|
||||
* The @mediapipe/tasks-vision version MUST match the one bundled into the gesture
|
||||
* overlay (Ark0N/codeman-gesture-control) so the wasm loader matches its JS API.
|
||||
*/
|
||||
import { mkdirSync, existsSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const GESTURE = join(__dirname, '..', 'src', 'web', 'public', 'gesture');
|
||||
const WASM = join(GESTURE, 'wasm');
|
||||
|
||||
const MP_VERSION = '0.10.21'; // keep in sync with the gesture overlay's @mediapipe/tasks-vision
|
||||
const WASM_BASE = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MP_VERSION}/wasm`;
|
||||
const MODEL_URL =
|
||||
'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task';
|
||||
|
||||
const ASSETS = [
|
||||
{ url: `${WASM_BASE}/vision_wasm_internal.js`, path: join(WASM, 'vision_wasm_internal.js') },
|
||||
{ url: `${WASM_BASE}/vision_wasm_internal.wasm`, path: join(WASM, 'vision_wasm_internal.wasm') },
|
||||
{ url: `${WASM_BASE}/vision_wasm_nosimd_internal.js`, path: join(WASM, 'vision_wasm_nosimd_internal.js') },
|
||||
{ url: `${WASM_BASE}/vision_wasm_nosimd_internal.wasm`, path: join(WASM, 'vision_wasm_nosimd_internal.wasm') },
|
||||
{ url: MODEL_URL, path: join(GESTURE, 'gesture_recognizer.task') },
|
||||
];
|
||||
|
||||
async function main() {
|
||||
mkdirSync(WASM, { recursive: true });
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
for (const a of ASSETS) {
|
||||
if (existsSync(a.path) && statSync(a.path).size > 0) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
const res = await fetch(a.url);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status} for ${a.url}`);
|
||||
writeFileSync(a.path, Buffer.from(await res.arrayBuffer()));
|
||||
fetched++;
|
||||
}
|
||||
console.log(`[gesture] MediaPipe assets ready (${fetched} fetched, ${skipped} cached) → ${GESTURE}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
// Non-fatal: opt-in feature. Warn and exit 0 so install/build still succeed.
|
||||
console.warn(`[gesture] could not fetch MediaPipe assets — overlay disabled until fetched: ${err.message}`);
|
||||
});
|
||||
@@ -250,10 +250,10 @@ if (isGlobalInstall) {
|
||||
} else {
|
||||
try {
|
||||
const require = createRequire(import.meta.url);
|
||||
const xtermDir = join(require.resolve('xterm'), '..', '..');
|
||||
const fitDir = join(require.resolve('xterm-addon-fit'), '..', '..');
|
||||
const webglDir = join(require.resolve('xterm-addon-webgl'), '..', '..');
|
||||
const unicode11Dir = join(require.resolve('xterm-addon-unicode11'), '..', '..');
|
||||
const xtermDir = join(require.resolve('@xterm/xterm'), '..', '..');
|
||||
const fitDir = join(require.resolve('@xterm/addon-fit'), '..', '..');
|
||||
const webglDir = join(require.resolve('@xterm/addon-webgl'), '..', '..');
|
||||
const unicode11Dir = join(require.resolve('@xterm/addon-unicode11'), '..', '..');
|
||||
const vendorDir = join(srcDir, 'web', 'public', 'vendor');
|
||||
|
||||
const { mkdirSync, copyFileSync } = await import('fs');
|
||||
@@ -263,19 +263,19 @@ if (isGlobalInstall) {
|
||||
// Minify xterm JS for dev vendor dir (npm packages don't ship .min.js)
|
||||
try {
|
||||
execSync(`npx esbuild "${join(xtermDir, 'lib', 'xterm.js')}" --minify --outfile="${join(vendorDir, 'xterm.min.js')}"`, { stdio: 'pipe' });
|
||||
execSync(`npx esbuild "${join(fitDir, 'lib', 'xterm-addon-fit.js')}" --minify --outfile="${join(vendorDir, 'xterm-addon-fit.min.js')}"`, { stdio: 'pipe' });
|
||||
execSync(`npx esbuild "${join(unicode11Dir, 'lib', 'xterm-addon-unicode11.js')}" --minify --outfile="${join(vendorDir, 'xterm-addon-unicode11.min.js')}"`, { stdio: 'pipe' });
|
||||
execSync(`npx esbuild "${join(fitDir, 'lib', 'addon-fit.js')}" --minify --outfile="${join(vendorDir, 'xterm-addon-fit.min.js')}"`, { stdio: 'pipe' });
|
||||
execSync(`npx esbuild "${join(unicode11Dir, 'lib', 'addon-unicode11.js')}" --minify --outfile="${join(vendorDir, 'xterm-addon-unicode11.min.js')}"`, { stdio: 'pipe' });
|
||||
console.log(colors.green('✓ xterm vendor files copied to src/web/public/vendor/'));
|
||||
} catch {
|
||||
// Fallback: copy unminified
|
||||
copyFileSync(join(xtermDir, 'lib', 'xterm.js'), join(vendorDir, 'xterm.min.js'));
|
||||
copyFileSync(join(fitDir, 'lib', 'xterm-addon-fit.js'), join(vendorDir, 'xterm-addon-fit.min.js'));
|
||||
copyFileSync(join(unicode11Dir, 'lib', 'xterm-addon-unicode11.js'), join(vendorDir, 'xterm-addon-unicode11.min.js'));
|
||||
copyFileSync(join(fitDir, 'lib', 'addon-fit.js'), join(vendorDir, 'xterm-addon-fit.min.js'));
|
||||
copyFileSync(join(unicode11Dir, 'lib', 'addon-unicode11.js'), join(vendorDir, 'xterm-addon-unicode11.min.js'));
|
||||
console.log(colors.green('✓ xterm vendor files copied') + colors.dim(' (unminified — esbuild not available)'));
|
||||
}
|
||||
|
||||
// WebGL addon: copy unminified (matches build script behavior)
|
||||
copyFileSync(join(webglDir, 'lib', 'xterm-addon-webgl.js'), join(vendorDir, 'xterm-addon-webgl.min.js'));
|
||||
copyFileSync(join(webglDir, 'lib', 'addon-webgl.js'), join(vendorDir, 'xterm-addon-webgl.min.js'));
|
||||
|
||||
// xterm-zerolag-input: bundle local package as IIFE for <script> tag loading
|
||||
try {
|
||||
@@ -312,6 +312,20 @@ if (isGlobalInstall) {
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// 4b. Fetch gesture-overlay runtime assets (MediaPipe wasm + model) for dev mode
|
||||
// (src/web/public/gesture/). Opt-in feature (CODEMAN_GESTURE=1); non-fatal.
|
||||
// Large binaries kept out of git; the build copies them into dist/.
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
if (!isGlobalInstall) {
|
||||
try {
|
||||
execSync(`node "${join(import.meta.dirname, 'fetch-gesture-assets.mjs')}"`, { stdio: 'inherit' });
|
||||
} catch {
|
||||
// Non-fatal — the gesture overlay is opt-in.
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// 5. Install git pre-commit hook (format check)
|
||||
// ----------------------------------------------------------------------------
|
||||
@@ -446,3 +460,16 @@ if (process.env.CI || process.env.CODEMAN_NO_AUTOSTART) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Security note — printed on every install path
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
console.log(colors.bold('Security:'));
|
||||
console.log(colors.dim(' Codeman binds ') + colors.cyan('127.0.0.1') + colors.dim(' (this machine only) — no password needed by default.'));
|
||||
console.log(colors.dim(' To reach it from another device, do ONE of:'));
|
||||
console.log(colors.dim(' • ') + colors.cyan('tailscale serve') + colors.dim(' / ') + colors.cyan('cloudflared tunnel') + colors.dim(' (recommended), or'));
|
||||
console.log(colors.dim(' • ') + colors.cyan('codeman web --host 0.0.0.0') + colors.dim(' AND set ') + colors.cyan('CODEMAN_PASSWORD'));
|
||||
console.log(colors.dim(' A non-loopback bind without a password still starts, but warns loudly.'));
|
||||
console.log(colors.dim(' Details: docs/security-architecture.md'));
|
||||
console.log('');
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import React from 'react';
|
||||
import { Composition } from 'remotion';
|
||||
import { CodemanDemo, TOTAL_FRAMES } from './compositions/CodemanDemo';
|
||||
import { ZerolagDemo, ZEROLAG_TOTAL_FRAMES } from './compositions/ZerolagDemo';
|
||||
|
||||
export const RemotionRoot: React.FC = () => {
|
||||
return (
|
||||
<>
|
||||
<Composition
|
||||
id="CodemanDemo"
|
||||
component={CodemanDemo}
|
||||
durationInFrames={TOTAL_FRAMES}
|
||||
fps={30}
|
||||
width={1920}
|
||||
height={1080}
|
||||
/>
|
||||
<Composition
|
||||
id="ZerolagDemo"
|
||||
component={ZerolagDemo}
|
||||
durationInFrames={ZEROLAG_TOTAL_FRAMES}
|
||||
fps={30}
|
||||
width={1920}
|
||||
height={1080}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,207 @@
|
||||
import React from 'react';
|
||||
import { fonts } from '../lib/theme';
|
||||
|
||||
type IOSKeyboardProps = {
|
||||
activeKey?: string;
|
||||
/** How many frames since the key was pressed (for highlight decay) */
|
||||
pressAge?: number;
|
||||
};
|
||||
|
||||
const ROW_1 = ['q', 'w', 'e', 'r', 't', 'y', 'u', 'i', 'o', 'p'];
|
||||
const ROW_2 = ['a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l'];
|
||||
const ROW_3 = ['z', 'x', 'c', 'v', 'b', 'n', 'm'];
|
||||
|
||||
const KEY_H = 42;
|
||||
const KEY_GAP = 6;
|
||||
const ROW_GAP = 11;
|
||||
const SIDE_PAD = 3;
|
||||
|
||||
const KEYBOARD_BG = '#1c1c1e';
|
||||
const KEY_BG = '#3a3a3c';
|
||||
const KEY_BG_ACTIVE = '#636366';
|
||||
const KEY_TEXT = '#fff';
|
||||
const SPECIAL_BG = '#2c2c2e';
|
||||
|
||||
const Key: React.FC<{
|
||||
label: string;
|
||||
width: number;
|
||||
isActive: boolean;
|
||||
pressAge: number;
|
||||
fontSize?: number;
|
||||
}> = ({ label, width, isActive, pressAge, fontSize = 22 }) => {
|
||||
// Highlight decays over 4 frames
|
||||
const highlightOpacity = isActive && pressAge < 4 ? 1 - pressAge / 4 : 0;
|
||||
const bg = highlightOpacity > 0
|
||||
? lerpColor(KEY_BG, KEY_BG_ACTIVE, highlightOpacity)
|
||||
: KEY_BG;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: bg,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
fontSize,
|
||||
fontFamily: fonts.ui,
|
||||
color: KEY_TEXT,
|
||||
fontWeight: 300,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
function lerpColor(a: string, b: string, t: number): string {
|
||||
const pa = parseInt(a.slice(1), 16);
|
||||
const pb = parseInt(b.slice(1), 16);
|
||||
const r = Math.round(((pa >> 16) & 0xff) * (1 - t) + ((pb >> 16) & 0xff) * t);
|
||||
const g = Math.round(((pa >> 8) & 0xff) * (1 - t) + ((pb >> 8) & 0xff) * t);
|
||||
const bl = Math.round((pa & 0xff) * (1 - t) + (pb & 0xff) * t);
|
||||
return `#${((r << 16) | (g << 8) | bl).toString(16).padStart(6, '0')}`;
|
||||
}
|
||||
|
||||
export const IOSKeyboard: React.FC<IOSKeyboardProps> = ({ activeKey, pressAge = 99 }) => {
|
||||
const isActive = (key: string) =>
|
||||
activeKey !== undefined && key.toLowerCase() === activeKey.toLowerCase();
|
||||
const age = (key: string) => (isActive(key) ? pressAge : 99);
|
||||
|
||||
// Backspace/delete key highlight
|
||||
const deleteHighlight = activeKey === '⌫' && pressAge < 4 ? 1 - pressAge / 4 : 0;
|
||||
const deleteBg = deleteHighlight > 0 ? lerpColor(SPECIAL_BG, KEY_BG_ACTIVE, deleteHighlight) : SPECIAL_BG;
|
||||
|
||||
// Key widths: 10 keys + 9 gaps in ~375px row → each key ~33px
|
||||
const letterKeyW = 33;
|
||||
// Row 2 has 9 keys → same key width but centered with side padding
|
||||
// Row 3 has shift + 7 keys + delete
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: '100%',
|
||||
background: KEYBOARD_BG,
|
||||
padding: `${ROW_GAP}px ${SIDE_PAD}px 20px`,
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: ROW_GAP,
|
||||
}}
|
||||
>
|
||||
{/* Row 1: q-p */}
|
||||
<div style={{ display: 'flex', gap: KEY_GAP, justifyContent: 'center' }}>
|
||||
{ROW_1.map((k) => (
|
||||
<Key key={k} label={k} width={letterKeyW} isActive={isActive(k)} pressAge={age(k)} />
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Row 2: a-l */}
|
||||
<div style={{ display: 'flex', gap: KEY_GAP, justifyContent: 'center' }}>
|
||||
{ROW_2.map((k) => (
|
||||
<Key key={k} label={k} width={letterKeyW} isActive={isActive(k)} pressAge={age(k)} />
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Row 3: shift + z-m + delete */}
|
||||
<div style={{ display: 'flex', gap: KEY_GAP, justifyContent: 'center' }}>
|
||||
<div
|
||||
style={{
|
||||
width: 42,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: SPECIAL_BG,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
>
|
||||
<svg width="20" height="16" viewBox="0 0 20 16" fill="none">
|
||||
<path d="M10 2L17 9H13V14H7V9H3L10 2Z" fill="#fff" />
|
||||
</svg>
|
||||
</div>
|
||||
{ROW_3.map((k) => (
|
||||
<Key key={k} label={k} width={letterKeyW} isActive={isActive(k)} pressAge={age(k)} />
|
||||
))}
|
||||
<div
|
||||
style={{
|
||||
width: 42,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: deleteBg,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
>
|
||||
<svg width="22" height="16" viewBox="0 0 22 16" fill="none">
|
||||
<path d="M7 1L1 8L7 15H21V1H7Z" stroke="#fff" strokeWidth="1.5" fill="none" />
|
||||
<path d="M12 5L17 10M17 5L12 10" stroke="#fff" strokeWidth="1.5" strokeLinecap="round" />
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Row 4: 123 / globe / space / return */}
|
||||
<div style={{ display: 'flex', gap: KEY_GAP, justifyContent: 'center' }}>
|
||||
<div
|
||||
style={{
|
||||
width: 42,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: SPECIAL_BG,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
fontSize: 15,
|
||||
fontFamily: fonts.ui,
|
||||
color: '#fff',
|
||||
}}
|
||||
>
|
||||
123
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
width: 38,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: SPECIAL_BG,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
>
|
||||
<svg width="20" height="20" viewBox="0 0 20 20" fill="none">
|
||||
<circle cx="10" cy="10" r="8" stroke="#fff" strokeWidth="1.2" />
|
||||
<path d="M4 10H16M10 4C7 7 7 13 10 16M10 4C13 7 13 13 10 16" stroke="#fff" strokeWidth="1" />
|
||||
</svg>
|
||||
</div>
|
||||
{/* Space bar */}
|
||||
<Key
|
||||
label="space"
|
||||
width={186}
|
||||
isActive={isActive(' ')}
|
||||
pressAge={age(' ')}
|
||||
fontSize={15}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
width: 88,
|
||||
height: KEY_H,
|
||||
borderRadius: 5,
|
||||
background: SPECIAL_BG,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
fontSize: 15,
|
||||
fontFamily: fonts.ui,
|
||||
color: '#fff',
|
||||
}}
|
||||
>
|
||||
return
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,171 @@
|
||||
import React from 'react';
|
||||
import { spring, useCurrentFrame, useVideoConfig } from 'remotion';
|
||||
|
||||
/**
|
||||
* Pixel-accurate iPhone 17 Pro frame.
|
||||
*
|
||||
* Dimensions based on iPhone 16 Pro (same form factor):
|
||||
* - Screen: 393×852 CSS points (2622×1206 @3x)
|
||||
* - Corner radius: 55px (device), 50px (screen inner)
|
||||
* - Bezel: ~3.5px (thinnest in any iPhone)
|
||||
* - Dynamic Island: 126×37 pill, centered 13px from top
|
||||
* - Safe area: top 59px, bottom 34px (home indicator)
|
||||
* - Frame: natural titanium (#8a8a8e border)
|
||||
*/
|
||||
|
||||
type IPhone17ProFrameProps = {
|
||||
children: React.ReactNode;
|
||||
/** Disable the spring entrance animation */
|
||||
noAnimation?: boolean;
|
||||
};
|
||||
|
||||
// Device dimensions (CSS points)
|
||||
const SCREEN_W = 393;
|
||||
const SCREEN_H = 852;
|
||||
const BEZEL = 4;
|
||||
const DEVICE_W = SCREEN_W + BEZEL * 2; // 401
|
||||
const DEVICE_H = SCREEN_H + BEZEL * 2; // 860
|
||||
const DEVICE_RADIUS = 55;
|
||||
const SCREEN_RADIUS = 50;
|
||||
|
||||
// Dynamic Island
|
||||
const DI_W = 126;
|
||||
const DI_H = 37;
|
||||
const DI_TOP = 13; // from top of screen
|
||||
const DI_RADIUS = DI_H / 2; // pill shape
|
||||
|
||||
export { SCREEN_W, SCREEN_H };
|
||||
|
||||
export const IPhone17ProFrame: React.FC<IPhone17ProFrameProps> = ({ children, noAnimation }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const scale = noAnimation
|
||||
? 1
|
||||
: spring({ frame, fps, config: { damping: 15, stiffness: 80 } });
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: DEVICE_W,
|
||||
height: DEVICE_H,
|
||||
transform: `scale(${scale})`,
|
||||
transformOrigin: 'center center',
|
||||
position: 'relative',
|
||||
}}
|
||||
>
|
||||
{/* Titanium frame (outer body) */}
|
||||
<div
|
||||
style={{
|
||||
width: DEVICE_W,
|
||||
height: DEVICE_H,
|
||||
borderRadius: DEVICE_RADIUS,
|
||||
background: '#2c2c2e', // dark titanium
|
||||
border: '1.5px solid #48484a', // subtle edge highlight
|
||||
boxShadow: [
|
||||
'0 2px 4px rgba(0,0,0,0.3)', // close shadow
|
||||
'0 12px 40px rgba(0,0,0,0.5)', // mid shadow
|
||||
'0 30px 80px rgba(0,0,0,0.4)', // far shadow
|
||||
'inset 0 1px 0 rgba(255,255,255,0.05)', // top edge gleam
|
||||
].join(', '),
|
||||
position: 'relative',
|
||||
overflow: 'hidden',
|
||||
}}
|
||||
>
|
||||
{/* Side button (right) — power */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
right: -2,
|
||||
top: 180,
|
||||
width: 3,
|
||||
height: 65,
|
||||
borderRadius: '0 2px 2px 0',
|
||||
background: '#48484a',
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Side buttons (left) — volume up, down, action */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
left: -2,
|
||||
top: 140,
|
||||
width: 3,
|
||||
height: 28,
|
||||
borderRadius: '2px 0 0 2px',
|
||||
background: '#48484a',
|
||||
}}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
left: -2,
|
||||
top: 185,
|
||||
width: 3,
|
||||
height: 50,
|
||||
borderRadius: '2px 0 0 2px',
|
||||
background: '#48484a',
|
||||
}}
|
||||
/>
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
left: -2,
|
||||
top: 250,
|
||||
width: 3,
|
||||
height: 50,
|
||||
borderRadius: '2px 0 0 2px',
|
||||
background: '#48484a',
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Screen */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: BEZEL,
|
||||
left: BEZEL,
|
||||
width: SCREEN_W,
|
||||
height: SCREEN_H,
|
||||
borderRadius: SCREEN_RADIUS,
|
||||
overflow: 'hidden',
|
||||
background: '#000',
|
||||
}}
|
||||
>
|
||||
{/* App content */}
|
||||
{children}
|
||||
|
||||
{/* Dynamic Island (on top of everything) */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: DI_TOP,
|
||||
left: (SCREEN_W - DI_W) / 2,
|
||||
width: DI_W,
|
||||
height: DI_H,
|
||||
borderRadius: DI_RADIUS,
|
||||
background: '#000',
|
||||
zIndex: 50,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Home indicator */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
bottom: BEZEL + 8,
|
||||
left: '50%',
|
||||
transform: 'translateX(-50%)',
|
||||
width: 134,
|
||||
height: 5,
|
||||
borderRadius: 3,
|
||||
background: 'rgba(255,255,255,0.2)',
|
||||
zIndex: 60,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,569 @@
|
||||
import React from 'react';
|
||||
import {
|
||||
AbsoluteFill,
|
||||
Img,
|
||||
interpolate,
|
||||
Sequence,
|
||||
spring,
|
||||
staticFile,
|
||||
useCurrentFrame,
|
||||
useVideoConfig,
|
||||
} from 'remotion';
|
||||
import { colors, fonts } from '../lib/theme';
|
||||
import { IPhone17ProFrame, SCREEN_W, SCREEN_H } from '../components/IPhone17ProFrame';
|
||||
import { IOSKeyboard } from '../components/IOSKeyboard';
|
||||
|
||||
// ─── Scene timing (frames @ 30fps) ───
|
||||
const TITLE_DUR = 60;
|
||||
const PHONES_DUR = 30;
|
||||
const TYPING_DUR = 610;
|
||||
const HOLD_DUR = 50;
|
||||
const OUTRO_DUR = 45;
|
||||
|
||||
const TITLE_START = 0;
|
||||
const PHONES_START = TITLE_DUR; // 60
|
||||
const TYPING_START = PHONES_START + PHONES_DUR; // 90
|
||||
const HOLD_START = TYPING_START + TYPING_DUR; // 700
|
||||
const OUTRO_START = HOLD_START + HOLD_DUR; // 750
|
||||
|
||||
export const ZEROLAG_TOTAL_FRAMES = OUTRO_START + OUTRO_DUR; // 795
|
||||
|
||||
// ─── iPhone 17 Pro safe area ───
|
||||
const SAFE_AREA_TOP = 59; // Below Dynamic Island
|
||||
const PHONE_SCALE = 1.12; // Scale up to fill more of the frame
|
||||
|
||||
// Claude Code header: tab bar + session info + prompt context from screenshot
|
||||
const HEADER_H = 120;
|
||||
|
||||
// Terminal typing overlay: aligned with the ❯ prompt position in the Claude Code screenshot
|
||||
const TERMINAL_TOP = 185;
|
||||
const TERMINAL_LEFT = 14;
|
||||
const TERMINAL_FONT = 21; // Slightly smaller to fit toolbar below
|
||||
|
||||
// Codeman toolbar from screenshot (bottom section showing /init, /clear, Run, etc.)
|
||||
const TOOLBAR_H = 95;
|
||||
|
||||
// ─── Typing schedule ───
|
||||
const CORRECT_TEXT =
|
||||
'zerolag technology brings in a visual dom overlay to make typing instant, even if your codeman server is on the other side of the world';
|
||||
const FRAME_GAP = 4; // ~133ms per keystroke (~75 WPM)
|
||||
|
||||
// Remote connection lag: 600ms–2.7s per char (18–80 frames @ 30fps).
|
||||
// Periodic spikes simulate packet loss / retransmission bursts.
|
||||
// TCP head-of-line blocking causes a single spike to freeze all subsequent chars.
|
||||
const LAGGY_DELAYS = [
|
||||
24, 30, 26, 32, 72, 28, 22, 34, 26, 30, 20, 28, 36, 24, 30, 22, 26, 34, 28, 20, 68, 30, 24, 32,
|
||||
26, 22, 28, 34, 30, 26, 32, 24, 80, 22, 30, 26, 28, 34, 24, 30,
|
||||
];
|
||||
|
||||
type KeyAction = { frame: number; char: string; lagDelay: number };
|
||||
|
||||
const buildSchedule = (): KeyAction[] => {
|
||||
const actions: KeyAction[] = [];
|
||||
const lag = (i: number) => LAGGY_DELAYS[i % LAGGY_DELAYS.length];
|
||||
|
||||
for (let i = 0; i < CORRECT_TEXT.length; i++) {
|
||||
actions.push({ frame: i * FRAME_GAP, char: CORRECT_TEXT[i], lagDelay: lag(i) });
|
||||
}
|
||||
|
||||
return actions;
|
||||
};
|
||||
|
||||
const TYPING_SCHEDULE = buildSchedule();
|
||||
|
||||
/**
|
||||
* Replay actions in order up to current frame, computing the visible text buffer.
|
||||
* TCP-ordered: stops at first unresolved echo (head-of-line blocking).
|
||||
*/
|
||||
const computeVisibleText = (frame: number, withLag: boolean): string => {
|
||||
let buffer = '';
|
||||
for (const a of TYPING_SCHEDULE) {
|
||||
const threshold = withLag ? a.frame + a.lagDelay : a.frame;
|
||||
if (frame < threshold) break;
|
||||
buffer += a.char;
|
||||
}
|
||||
return buffer;
|
||||
};
|
||||
|
||||
// ─── iOS Status Bar (sits in the safe area, flanking Dynamic Island) ───
|
||||
const IOSStatusBar: React.FC = () => (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 17,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: 20,
|
||||
display: 'flex',
|
||||
justifyContent: 'space-between',
|
||||
padding: '0 30px',
|
||||
fontSize: 15,
|
||||
fontFamily: fonts.ui,
|
||||
fontWeight: 600,
|
||||
color: '#fff',
|
||||
zIndex: 40,
|
||||
}}
|
||||
>
|
||||
<span>9:41</span>
|
||||
<div style={{ display: 'flex', gap: 6, alignItems: 'center' }}>
|
||||
{/* Signal */}
|
||||
<svg width="17" height="12" viewBox="0 0 17 12">
|
||||
<rect x="0" y="8" width="3" height="4" rx="0.5" fill="#fff" />
|
||||
<rect x="4.5" y="5" width="3" height="7" rx="0.5" fill="#fff" />
|
||||
<rect x="9" y="2" width="3" height="10" rx="0.5" fill="#fff" />
|
||||
<rect x="13.5" y="0" width="3" height="12" rx="0.5" fill="#fff" />
|
||||
</svg>
|
||||
{/* WiFi */}
|
||||
<svg width="16" height="12" viewBox="0 0 16 12">
|
||||
<path
|
||||
d="M4.5 8.5C5.5 7.2 6.7 6.5 8 6.5s2.5.7 3.5 2"
|
||||
stroke="#fff"
|
||||
strokeWidth="1.5"
|
||||
fill="none"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<path
|
||||
d="M1.5 5.5C3.5 3 5.7 1.5 8 1.5s4.5 1.5 6.5 4"
|
||||
stroke="#fff"
|
||||
strokeWidth="1.5"
|
||||
fill="none"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<circle cx="8" cy="11" r="1.5" fill="#fff" />
|
||||
</svg>
|
||||
{/* Battery */}
|
||||
<svg width="27" height="12" viewBox="0 0 27 12">
|
||||
<rect x="0" y="0.5" width="23" height="11" rx="2" stroke="#fff" strokeWidth="1" fill="none" />
|
||||
<rect x="24" y="3.5" width="2.5" height="5" rx="1" fill="#fff" opacity="0.4" />
|
||||
<rect x="1.5" y="2" width="20" height="8" rx="1" fill="#32d74b" />
|
||||
</svg>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
// ─── Typing overlay ───
|
||||
const TypingOverlay: React.FC<{
|
||||
typed: string;
|
||||
cursorVisible: boolean;
|
||||
}> = ({ typed, cursorVisible }) => {
|
||||
const frame = useCurrentFrame();
|
||||
const cursorOpacity = cursorVisible ? (Math.floor(frame / 18) % 2 === 0 ? 0.85 : 0.5) : 0;
|
||||
const lineH = Math.round(TERMINAL_FONT * 1.4);
|
||||
const charW = TERMINAL_FONT * 0.62;
|
||||
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: TERMINAL_TOP,
|
||||
left: TERMINAL_LEFT,
|
||||
right: TERMINAL_LEFT,
|
||||
fontFamily: '"Fira Code", "Cascadia Code", "JetBrains Mono", "SF Mono", Monaco, monospace',
|
||||
fontSize: TERMINAL_FONT,
|
||||
lineHeight: `${lineH}px`,
|
||||
zIndex: 10,
|
||||
}}
|
||||
>
|
||||
<span style={{ color: '#339af0', fontWeight: 700 }}>{'❯ '}</span>
|
||||
<span style={{ color: '#e0e0e0' }}>{typed}</span>
|
||||
<span
|
||||
style={{
|
||||
display: 'inline-block',
|
||||
width: charW,
|
||||
height: lineH * 0.82,
|
||||
background: `rgba(224, 224, 224, ${cursorOpacity})`,
|
||||
verticalAlign: 'text-bottom',
|
||||
marginLeft: 1,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Single phone: iPhone 17 Pro + Claude Code header + typing + keyboard ───
|
||||
const MobileCodeman: React.FC<{
|
||||
typed: string;
|
||||
cursorVisible: boolean;
|
||||
activeKey?: string;
|
||||
pressAge?: number;
|
||||
showKeyboard?: boolean;
|
||||
noAnimation?: boolean;
|
||||
showPromo?: boolean;
|
||||
}> = ({ typed, cursorVisible, activeKey, pressAge, showKeyboard = true, noAnimation, showPromo }) => (
|
||||
<IPhone17ProFrame noAnimation={noAnimation}>
|
||||
<div
|
||||
style={{ width: SCREEN_W, height: SCREEN_H, position: 'relative', overflow: 'hidden', background: '#0d0d0d' }}
|
||||
>
|
||||
{/* iOS status bar in the safe area */}
|
||||
<IOSStatusBar />
|
||||
|
||||
{/* Claude Code header from screenshot (tabs + session info) */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: SAFE_AREA_TOP,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: HEADER_H,
|
||||
overflow: 'hidden',
|
||||
zIndex: 5,
|
||||
}}
|
||||
>
|
||||
<Img
|
||||
src={staticFile('mobile-claude.png')}
|
||||
style={{
|
||||
width: SCREEN_W,
|
||||
objectFit: 'cover',
|
||||
objectPosition: 'top left',
|
||||
}}
|
||||
/>
|
||||
{/* Fade to terminal background */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
bottom: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: 30,
|
||||
background: 'linear-gradient(transparent, #0d0d0d)',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Dark mask: hides screenshot text below Claude Code info (e.g. "Try edit...") */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: SAFE_AREA_TOP + 95,
|
||||
left: 0,
|
||||
right: 0,
|
||||
bottom: 0,
|
||||
zIndex: 8,
|
||||
}}
|
||||
>
|
||||
{/* Smooth gradient blend from screenshot into dark terminal */}
|
||||
<div
|
||||
style={{
|
||||
height: 18,
|
||||
background: 'linear-gradient(transparent, #0d0d0d)',
|
||||
}}
|
||||
/>
|
||||
<div style={{ flex: 1, background: '#0d0d0d' }} />
|
||||
</div>
|
||||
|
||||
{/* Typing animation */}
|
||||
<TypingOverlay typed={typed} cursorVisible={cursorVisible} />
|
||||
|
||||
{/* Codeman toolbar from screenshot (/init, /clear, /compact, Run, Run Shell, voice) */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
bottom: 232,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: TOOLBAR_H,
|
||||
overflow: 'hidden',
|
||||
zIndex: 15,
|
||||
}}
|
||||
>
|
||||
{/* Gradient blend at top */}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: 0,
|
||||
left: 0,
|
||||
right: 0,
|
||||
height: 18,
|
||||
background: 'linear-gradient(#0d0d0d, transparent)',
|
||||
zIndex: 1,
|
||||
}}
|
||||
/>
|
||||
<Img
|
||||
src={staticFile('mobile-claude.png')}
|
||||
style={{
|
||||
width: SCREEN_W,
|
||||
position: 'absolute',
|
||||
bottom: 0,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Promo cards between text and toolbar (left phone only) */}
|
||||
{showPromo && <PromoBanners />}
|
||||
|
||||
{/* iOS keyboard at bottom */}
|
||||
{showKeyboard && (
|
||||
<div style={{ position: 'absolute', bottom: 0, left: 0, right: 0, zIndex: 20 }}>
|
||||
<IOSKeyboard activeKey={activeKey} pressAge={pressAge} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</IPhone17ProFrame>
|
||||
);
|
||||
|
||||
// ─── Label above phone (prominent header) ───
|
||||
const PhoneLabel: React.FC<{
|
||||
title: string;
|
||||
detail: string;
|
||||
dotColor: string;
|
||||
detailColor: string;
|
||||
}> = ({ title, detail, dotColor, detailColor }) => (
|
||||
<div style={{ textAlign: 'center', marginBottom: 16 }}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: 10,
|
||||
fontSize: 28,
|
||||
fontWeight: 700,
|
||||
fontFamily: fonts.ui,
|
||||
color: '#fff',
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 12,
|
||||
height: 12,
|
||||
borderRadius: '50%',
|
||||
background: dotColor,
|
||||
boxShadow: `0 0 12px ${dotColor}`,
|
||||
}}
|
||||
/>
|
||||
{title}
|
||||
</div>
|
||||
<div style={{ fontSize: 16, fontFamily: fonts.mono, color: detailColor, marginTop: 6, opacity: 0.9 }}>
|
||||
{detail}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
// ─── Promo banners (inside left phone, between text and keyboard) ───
|
||||
const PromoBanners: React.FC = () => (
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute',
|
||||
bottom: 332,
|
||||
left: 8,
|
||||
right: 8,
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 8,
|
||||
zIndex: 15,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 12,
|
||||
padding: '12px 14px',
|
||||
background: 'rgba(255,255,255,0.06)',
|
||||
borderRadius: 12,
|
||||
border: '1px solid rgba(255,255,255,0.1)',
|
||||
}}
|
||||
>
|
||||
<svg width="30" height="30" viewBox="0 0 16 16" fill="#ccc" style={{ flexShrink: 0 }}>
|
||||
<path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z" />
|
||||
</svg>
|
||||
<div>
|
||||
<div style={{ fontSize: 18, fontWeight: 700, fontFamily: fonts.ui, color: '#e0e0e0' }}>Codeman</div>
|
||||
<div style={{ fontSize: 12, fontFamily: fonts.mono, color: '#888' }}>github.com/Ark0N/Codeman</div>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 12,
|
||||
padding: '12px 14px',
|
||||
background: 'rgba(255,255,255,0.06)',
|
||||
borderRadius: 12,
|
||||
border: '1px solid rgba(255,255,255,0.1)',
|
||||
}}
|
||||
>
|
||||
<svg width="30" height="30" viewBox="0 0 16 16" style={{ flexShrink: 0 }}>
|
||||
<rect width="16" height="16" rx="2" fill="#cb3837" />
|
||||
<path d="M3 3h10v10H8V5H5v8H3z" fill="#fff" />
|
||||
</svg>
|
||||
<div>
|
||||
<div style={{ fontSize: 18, fontWeight: 700, fontFamily: fonts.ui, color: '#e0e0e0' }}>xterm-zerolag-input</div>
|
||||
<div style={{ fontSize: 12, fontFamily: fonts.mono, color: '#888' }}>npmjs.com/package/xterm-zerolag-input</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
// ─── Title scene ───
|
||||
const TitleScene: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const titleScale = spring({ frame, fps, config: { damping: 15, stiffness: 80 } });
|
||||
const titleOpacity = interpolate(frame, [0, 20], [0, 1], { extrapolateRight: 'clamp' });
|
||||
const subtitleOpacity = interpolate(frame, [15, 35], [0, 1], { extrapolateRight: 'clamp' });
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: colors.bg.dark, justifyContent: 'center', alignItems: 'center' }}>
|
||||
<div style={{ textAlign: 'center', transform: `scale(${titleScale})` }}>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 80,
|
||||
fontWeight: 700,
|
||||
fontFamily: fonts.ui,
|
||||
color: '#fff',
|
||||
opacity: titleOpacity,
|
||||
letterSpacing: -1.5,
|
||||
}}
|
||||
>
|
||||
Zerolag Input
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 30,
|
||||
fontFamily: fonts.ui,
|
||||
color: colors.text.dim,
|
||||
opacity: subtitleOpacity,
|
||||
marginTop: 16,
|
||||
}}
|
||||
>
|
||||
Local echo for remote terminal sessions
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Outro ───
|
||||
const OutroScene: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const opacity = interpolate(frame, [0, 20], [0, 1], { extrapolateRight: 'clamp' });
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: colors.bg.dark, justifyContent: 'center', alignItems: 'center', opacity }}>
|
||||
<div style={{ textAlign: 'center' }}>
|
||||
<div style={{ fontSize: 56, fontWeight: 700, fontFamily: fonts.ui, color: '#fff' }}>Codeman</div>
|
||||
<div style={{ fontSize: 26, fontFamily: fonts.ui, color: colors.accent.green, marginTop: 10 }}>
|
||||
Zero-latency mobile input
|
||||
</div>
|
||||
<div style={{ fontSize: 16, fontFamily: fonts.mono, color: colors.text.muted, marginTop: 20 }}>
|
||||
npm i xterm-zerolag-input
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Typing demo scene ───
|
||||
const TypingDemo: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
|
||||
const laggyTyped = computeVisibleText(frame, true);
|
||||
const zerolagTyped = computeVisibleText(frame, false);
|
||||
|
||||
let activeKey: string | undefined;
|
||||
let pressAge = 99;
|
||||
for (let i = TYPING_SCHEDULE.length - 1; i >= 0; i--) {
|
||||
const ev = TYPING_SCHEDULE[i];
|
||||
if (frame >= ev.frame && frame < ev.frame + 5) {
|
||||
activeKey = ev.char;
|
||||
pressAge = frame - ev.frame;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<AbsoluteFill style={{ background: colors.bg.dark, justifyContent: 'center', alignItems: 'center' }}>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: 50,
|
||||
alignItems: 'flex-start',
|
||||
transform: `scale(${PHONE_SCALE})`,
|
||||
transformOrigin: 'center center',
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<PhoneLabel
|
||||
title="With Zerolag"
|
||||
detail="0ms local echo"
|
||||
dotColor={colors.accent.green}
|
||||
detailColor={colors.accent.green}
|
||||
/>
|
||||
<MobileCodeman typed={zerolagTyped} cursorVisible activeKey={activeKey} pressAge={pressAge} noAnimation showPromo />
|
||||
</div>
|
||||
<div>
|
||||
<PhoneLabel
|
||||
title="Without Zerolag"
|
||||
detail="600ms–2.7s server echo"
|
||||
dotColor={colors.accent.red}
|
||||
detailColor={colors.accent.red}
|
||||
/>
|
||||
<MobileCodeman typed={laggyTyped} cursorVisible activeKey={activeKey} pressAge={pressAge} noAnimation />
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Phones entrance ───
|
||||
const PanelsEntrance: React.FC = () => {
|
||||
const frame = useCurrentFrame();
|
||||
const { fps } = useVideoConfig();
|
||||
|
||||
const scale = spring({ frame, fps, config: { damping: 15, stiffness: 80 } });
|
||||
const opacity = interpolate(frame, [0, 10], [0, 1], { extrapolateRight: 'clamp' });
|
||||
|
||||
return (
|
||||
<AbsoluteFill
|
||||
style={{
|
||||
background: colors.bg.dark,
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
opacity,
|
||||
transform: `scale(${scale * PHONE_SCALE})`,
|
||||
}}
|
||||
>
|
||||
<div style={{ display: 'flex', gap: 50, alignItems: 'flex-start' }}>
|
||||
<div>
|
||||
<PhoneLabel
|
||||
title="With Zerolag"
|
||||
detail="0ms local echo"
|
||||
dotColor={colors.accent.green}
|
||||
detailColor={colors.accent.green}
|
||||
/>
|
||||
<MobileCodeman typed="" cursorVisible noAnimation />
|
||||
</div>
|
||||
<div>
|
||||
<PhoneLabel
|
||||
title="Without Zerolag"
|
||||
detail="600ms–2.7s server echo"
|
||||
dotColor={colors.accent.red}
|
||||
detailColor={colors.accent.red}
|
||||
/>
|
||||
<MobileCodeman typed="" cursorVisible noAnimation />
|
||||
</div>
|
||||
</div>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
||||
// ─── Main composition ───
|
||||
export const ZerolagDemo: React.FC = () => {
|
||||
return (
|
||||
<AbsoluteFill style={{ background: colors.bg.dark }}>
|
||||
<Sequence from={TITLE_START} durationInFrames={TITLE_DUR}>
|
||||
<TitleScene />
|
||||
</Sequence>
|
||||
<Sequence from={PHONES_START} durationInFrames={PHONES_DUR} premountFor={5}>
|
||||
<PanelsEntrance />
|
||||
</Sequence>
|
||||
<Sequence from={TYPING_START} durationInFrames={TYPING_DUR + HOLD_DUR} premountFor={5}>
|
||||
<TypingDemo />
|
||||
</Sequence>
|
||||
<Sequence from={OUTRO_START} durationInFrames={OUTRO_DUR} premountFor={5}>
|
||||
<OutroScene />
|
||||
</Sequence>
|
||||
</AbsoluteFill>
|
||||
);
|
||||
};
|
||||
|
Before Width: | Height: | Size: 41 KiB After Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 37 KiB After Width: | Height: | Size: 37 KiB |
|
Before Width: | Height: | Size: 39 KiB After Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 57 KiB After Width: | Height: | Size: 57 KiB |
|
After Width: | Height: | Size: 390 KiB |