mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-09-30 12:39:42 +02:00
feat(gesture): hand-drag subagent & ultracode windows in the gesture beta
Pinch any floating subagent or ultracode run/transcript window with the camera hand-tracking overlay and move it anywhere. Adds a 'window' grab kind to entry.ts, slotted into the pinch priority chain (cg-float panel → agent window → session tab → toolbar button). It moves the window via its own style.left/top (matching app.js's mouse drag, incl. bottom:'auto') and calls window.app.updateConnectionLines() so the glowing connector line to the session tab tracks live — app.js redraws from fresh rects, so no reach into its internals. Hardening: el.isConnected guard (ultracode windows tear down mid-grab on SSE reconnect / auto-close), all window.app calls optional-chained + try/caught so the standalone playground still works, bring-to-front via app.js's own z-counters, rAF-coalesced redraws cleared on drop so the final placement always redraws. Rebuilt the committed gesture-codeman.js bundle. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Codeman maintainer
@@ -17,6 +17,13 @@
|
||||
// • Panel "re-grab" — pinch an existing floating panel and move it anywhere;
|
||||
// release over the tab strip to re-dock it (panel goes away, the tab stays).
|
||||
// This is the capability the old OS-window detach lost.
|
||||
// • Agent-window "grab-to-move" — pinch any floating *subagent* or *ultracode*
|
||||
// run/transcript window (the dashboard's own `.subagent-window` /
|
||||
// `.ultracode-window` floats) and move it anywhere. These windows stay owned
|
||||
// by app.js — we only nudge their `style.left/top` and ask app.js to redraw
|
||||
// the glowing connector line back to their session tab (its redraw reads live
|
||||
// rects, so the line tracks without us touching app.js internals). This is the
|
||||
// multi-monitor verb that lets these windows cross the physical monitor seam.
|
||||
// • Button "tap" — pinch over a toolbar button (Run / Run Shell) and release
|
||||
// in place → fires the button's real click handler. Drift too far first and
|
||||
// it's treated as a stray move, not a tap.
|
||||
@@ -36,12 +43,29 @@ import type { HandState } from '../gesture/types.ts';
|
||||
declare global {
|
||||
interface Window {
|
||||
__codemanGesture?: GestureBridge;
|
||||
/** The Codeman dashboard singleton (app.js, `window.app`). The gesture layer
|
||||
* reaches into it to redraw the floating-window connector lines and bump a
|
||||
* grabbed window's z-order while moving the subagent / ultracode windows.
|
||||
* Loosely typed — only the few members we touch. */
|
||||
app?: {
|
||||
updateConnectionLines?: () => void;
|
||||
saveSubagentWindowStates?: () => void;
|
||||
subagentWindowZIndex?: number;
|
||||
ultracodeWindowZIndex?: number;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const TAB_SELECTOR = '.session-tab';
|
||||
/** An in-page floating session panel this layer spawned — re-grabbable to move. */
|
||||
const PANEL_SELECTOR = '.cg-float';
|
||||
/** The dashboard's own floating agent windows (subagent runs + ultracode run and
|
||||
* transcript windows). All three carry one of these classes, position via
|
||||
* `style.left/top`, and redraw their connector line from
|
||||
* `window.app.updateConnectionLines()` — so the hand can pick one up and move it
|
||||
* without app.js knowing. (`.ultracode-agent-window` also carries
|
||||
* `.ultracode-window`, so this matches it too.) */
|
||||
const WINDOW_SELECTOR = '.subagent-window, .ultracode-window';
|
||||
/** The session-tab strip; dropping a moved panel over it re-docks the session. */
|
||||
const DOCK_SELECTOR = '.session-tabs';
|
||||
/** Toolbar buttons a pinch can "tap": Run (#runBtn → app.run()) and Run Shell
|
||||
@@ -93,6 +117,17 @@ type Grab =
|
||||
dy: number;
|
||||
/** Cursor currently over the tab strip → releasing re-docks. */
|
||||
overDock: boolean;
|
||||
}
|
||||
| {
|
||||
/** A dashboard-owned floating agent window (subagent / ultracode) being
|
||||
* moved. We never remove or re-parent it — just reposition + redraw its
|
||||
* connector. The element ref can go stale mid-grab (SSE reconnect tears
|
||||
* ultracode windows down), so every move guards on `el.isConnected`. */
|
||||
kind: 'window';
|
||||
el: HTMLElement;
|
||||
/** Cursor→window-top-left offset at grab, so it doesn't snap. */
|
||||
dx: number;
|
||||
dy: number;
|
||||
};
|
||||
|
||||
/** Live state for one hand pinching a toolbar button (Run / Run Shell). */
|
||||
@@ -122,6 +157,8 @@ class GestureBridge {
|
||||
private taps = new Map<string, Tap>();
|
||||
/** Live floating panels, keyed by session id (idempotent per id). */
|
||||
private floats = new Map<string, FloatingPanel>();
|
||||
/** rAF coalescing for connector-line redraws while dragging an agent window. */
|
||||
private connectorRedrawScheduled = false;
|
||||
|
||||
constructor() {
|
||||
injectStyles();
|
||||
@@ -187,7 +224,7 @@ class GestureBridge {
|
||||
await this.gc.start();
|
||||
this.running = true;
|
||||
this.button.classList.add('on');
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
this.status.textContent = 'on — pinch a tab, window, or button';
|
||||
} catch (err) {
|
||||
// Surface the *real* cause: MediaPipe/Emscripten can throw a non-Error
|
||||
// (number/string), so `(err as Error).message` was logging "undefined".
|
||||
@@ -242,6 +279,22 @@ class GestureBridge {
|
||||
}
|
||||
}
|
||||
|
||||
// A dashboard-owned floating agent window (subagent / ultracode run or
|
||||
// transcript) → pick it up and move it. Priority below cg-float panels
|
||||
// (which sit far above), above tabs/buttons. We grab anywhere on the window
|
||||
// (not just its titlebar) since the hand is choosing the whole window.
|
||||
const win = this.hitClosest(x, y, WINDOW_SELECTOR);
|
||||
if (win) {
|
||||
const rect = win.getBoundingClientRect();
|
||||
// Match app.js's own drag: drop any bottom-anchor so left/top take effect.
|
||||
win.style.bottom = 'auto';
|
||||
win.classList.add('cg-win-grabbed');
|
||||
this.bringWindowToFront(win);
|
||||
this.grabs.set(hand, { kind: 'window', el: win, dx: x - rect.left, dy: y - rect.top });
|
||||
this.status.textContent = 'moving window';
|
||||
return;
|
||||
}
|
||||
|
||||
// A session tab → grab-and-pull-out into a floating panel (ghost follows).
|
||||
const tab = this.hitClosest(x, y, TAB_SELECTOR);
|
||||
const id = tab?.dataset.id;
|
||||
@@ -292,12 +345,16 @@ class GestureBridge {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'window') {
|
||||
this.moveWindow(grab.el, x - grab.dx, y - grab.dy);
|
||||
return;
|
||||
}
|
||||
// A button pinch that drifts too far is a stray move, not a tap — cancel it.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap && Math.hypot(x - tap.ox, y - tap.oy) > TAP_CANCEL_PX) {
|
||||
tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.delete(hand);
|
||||
this.status.textContent = 'on — pinch a tab or button';
|
||||
this.status.textContent = 'on — pinch a tab, window, or button';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,6 +376,23 @@ class GestureBridge {
|
||||
else this.flash('placed');
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === 'window') {
|
||||
this.grabs.delete(hand);
|
||||
grab.el.classList.remove('cg-win-grabbed');
|
||||
// Clear the coalescer so the final placement always redraws, even if a
|
||||
// mid-drag rAF was throttled (tab briefly backgrounded) and left it latched.
|
||||
this.connectorRedrawScheduled = false;
|
||||
this.redrawWindowConnectors();
|
||||
// Persist subagent-window positions like app.js's own drag end does
|
||||
// (a no-op for ultracode windows, which aren't position-persisted).
|
||||
try {
|
||||
window.app?.saveSubagentWindowStates?.();
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
this.flash('placed window');
|
||||
return;
|
||||
}
|
||||
// Release over the same button → fire its real click handler.
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap) {
|
||||
@@ -373,6 +447,59 @@ class GestureBridge {
|
||||
float.el.style.top = `${t}px`;
|
||||
}
|
||||
|
||||
/** Move a dashboard-owned agent window by its top-left, clamped on-screen, then
|
||||
* redraw its connector line. The window self-positions via `style.left/top` and
|
||||
* app.js's connector redraw reads live rects, so this tracks without touching
|
||||
* app.js internals. Guards on `isConnected`: ultracode windows can be torn down
|
||||
* (SSE reconnect / auto-close) while still held. Clamps to `innerWidth/Height`,
|
||||
* which equals the *spanned* viewport in a multi-monitor window — so the window
|
||||
* can still travel across the physical monitor seam, just not off-screen. */
|
||||
private moveWindow(el: HTMLElement, left: number, top: number): void {
|
||||
if (!el.isConnected) return;
|
||||
const w = el.offsetWidth || 380;
|
||||
const h = el.offsetHeight || 320;
|
||||
const l = Math.min(Math.max(4, left), Math.max(4, window.innerWidth - w - 4));
|
||||
const t = Math.min(Math.max(4, top), Math.max(4, window.innerHeight - h - 4));
|
||||
el.style.left = `${l}px`;
|
||||
el.style.top = `${t}px`;
|
||||
this.redrawWindowConnectors();
|
||||
}
|
||||
|
||||
/** Ask app.js to redraw all connector lines (subagent + ultracode), coalesced to
|
||||
* one per frame so per-frame drags don't thrash. `updateConnectionLines()` is
|
||||
* itself debounced in app.js, but we rAF-gate too in case an older dashboard
|
||||
* build isn't, and to no-op cleanly when app.js isn't present (standalone). */
|
||||
private redrawWindowConnectors(): void {
|
||||
if (this.connectorRedrawScheduled) return;
|
||||
this.connectorRedrawScheduled = true;
|
||||
requestAnimationFrame(() => {
|
||||
this.connectorRedrawScheduled = false;
|
||||
try {
|
||||
window.app?.updateConnectionLines?.();
|
||||
} catch {
|
||||
/* app.js may not expose it (standalone playground) */
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Pop a grabbed window above its siblings using app.js's own z-counter, so a
|
||||
* picked-up window comes to the front like a real focus. Cosmetic + best-effort. */
|
||||
private bringWindowToFront(el: HTMLElement): void {
|
||||
const app = window.app;
|
||||
if (!app) return;
|
||||
try {
|
||||
if (el.classList.contains('ultracode-window')) {
|
||||
app.ultracodeWindowZIndex = (app.ultracodeWindowZIndex ?? 1000) + 1;
|
||||
el.style.zIndex = String(app.ultracodeWindowZIndex);
|
||||
} else {
|
||||
app.subagentWindowZIndex = (app.subagentWindowZIndex ?? 1000) + 1;
|
||||
el.style.zIndex = String(app.subagentWindowZIndex);
|
||||
}
|
||||
} catch {
|
||||
/* cosmetic only */
|
||||
}
|
||||
}
|
||||
|
||||
private positionGhost(ghost: HTMLElement, x: number, y: number): void {
|
||||
ghost.style.left = `${x}px`;
|
||||
ghost.style.top = `${y}px`;
|
||||
@@ -385,17 +512,19 @@ class GestureBridge {
|
||||
if (grab.kind === 'tab') {
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove('cg-grabbed');
|
||||
} else {
|
||||
} else if (grab.kind === 'panel') {
|
||||
grab.panel.el.style.pointerEvents = '';
|
||||
grab.panel.el.classList.remove('cg-float-grabbed', 'cg-redock');
|
||||
} else {
|
||||
grab.el.classList.remove('cg-win-grabbed');
|
||||
}
|
||||
}
|
||||
this.grabs.clear();
|
||||
for (const tap of this.taps.values()) tap.el.classList.remove('cg-tap-armed');
|
||||
this.taps.clear();
|
||||
document
|
||||
.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`)
|
||||
.forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed'));
|
||||
.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed, .cg-win-grabbed`)
|
||||
.forEach((t) => t.classList.remove('cg-grabbed', 'cg-tap-armed', 'cg-win-grabbed'));
|
||||
}
|
||||
|
||||
private onStatus(fps: number, hands: HandState[]): void {
|
||||
@@ -491,6 +620,10 @@ function injectStyles(): void {
|
||||
.cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; }
|
||||
.cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; }
|
||||
.subagent-window.cg-win-grabbed, .ultracode-window.cg-win-grabbed {
|
||||
outline: 2px solid #4ade80 !important; outline-offset: -2px;
|
||||
box-shadow: 0 12px 48px rgba(74,222,128,.5) !important;
|
||||
}
|
||||
.cg-float {
|
||||
position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px;
|
||||
z-index: ${Z}; display: flex; flex-direction: column; overflow: hidden;
|
||||
|
||||
@@ -4449,6 +4449,7 @@ var GestureController = class {
|
||||
// packages/gesture-control/src/codeman/entry.ts
|
||||
var TAB_SELECTOR = ".session-tab";
|
||||
var PANEL_SELECTOR = ".cg-float";
|
||||
var WINDOW_SELECTOR = ".subagent-window, .ultracode-window";
|
||||
var DOCK_SELECTOR = ".session-tabs";
|
||||
var CLICK_SELECTOR = "#runBtn, .btn-shell";
|
||||
var Z2 = 2147483e3;
|
||||
@@ -4476,6 +4477,8 @@ var GestureBridge = class {
|
||||
__publicField(this, "taps", /* @__PURE__ */ new Map());
|
||||
/** Live floating panels, keyed by session id (idempotent per id). */
|
||||
__publicField(this, "floats", /* @__PURE__ */ new Map());
|
||||
/** rAF coalescing for connector-line redraws while dragging an agent window. */
|
||||
__publicField(this, "connectorRedrawScheduled", false);
|
||||
injectStyles();
|
||||
this.surface = el("div", "cg-surface");
|
||||
this.canvas = el("canvas", "cg-canvas");
|
||||
@@ -4530,7 +4533,7 @@ var GestureBridge = class {
|
||||
await this.gc.start();
|
||||
this.running = true;
|
||||
this.button.classList.add("on");
|
||||
this.status.textContent = "on \u2014 pinch a tab or button";
|
||||
this.status.textContent = "on \u2014 pinch a tab, window, or button";
|
||||
} catch (err) {
|
||||
const msg = describeError(err);
|
||||
this.status.textContent = `failed: ${msg}`;
|
||||
@@ -4575,6 +4578,16 @@ var GestureBridge = class {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const win = this.hitClosest(x2, y2, WINDOW_SELECTOR);
|
||||
if (win) {
|
||||
const rect = win.getBoundingClientRect();
|
||||
win.style.bottom = "auto";
|
||||
win.classList.add("cg-win-grabbed");
|
||||
this.bringWindowToFront(win);
|
||||
this.grabs.set(hand, { kind: "window", el: win, dx: x2 - rect.left, dy: y2 - rect.top });
|
||||
this.status.textContent = "moving window";
|
||||
return;
|
||||
}
|
||||
const tab = this.hitClosest(x2, y2, TAB_SELECTOR);
|
||||
const id = tab?.dataset.id;
|
||||
if (tab && id) {
|
||||
@@ -4620,11 +4633,15 @@ var GestureBridge = class {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === "window") {
|
||||
this.moveWindow(grab.el, x2 - grab.dx, y2 - grab.dy);
|
||||
return;
|
||||
}
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap && Math.hypot(x2 - tap.ox, y2 - tap.oy) > TAP_CANCEL_PX) {
|
||||
tap.el.classList.remove("cg-tap-armed");
|
||||
this.taps.delete(hand);
|
||||
this.status.textContent = "on \u2014 pinch a tab or button";
|
||||
this.status.textContent = "on \u2014 pinch a tab, window, or button";
|
||||
}
|
||||
}
|
||||
onDrop(hand, x2, y2) {
|
||||
@@ -4645,6 +4662,18 @@ var GestureBridge = class {
|
||||
else this.flash("placed");
|
||||
return;
|
||||
}
|
||||
if (grab?.kind === "window") {
|
||||
this.grabs.delete(hand);
|
||||
grab.el.classList.remove("cg-win-grabbed");
|
||||
this.connectorRedrawScheduled = false;
|
||||
this.redrawWindowConnectors();
|
||||
try {
|
||||
window.app?.saveSubagentWindowStates?.();
|
||||
} catch {
|
||||
}
|
||||
this.flash("placed window");
|
||||
return;
|
||||
}
|
||||
const tap = this.taps.get(hand);
|
||||
if (tap) {
|
||||
this.taps.delete(hand);
|
||||
@@ -4694,6 +4723,54 @@ var GestureBridge = class {
|
||||
float.el.style.left = `${l}px`;
|
||||
float.el.style.top = `${t2}px`;
|
||||
}
|
||||
/** Move a dashboard-owned agent window by its top-left, clamped on-screen, then
|
||||
* redraw its connector line. The window self-positions via `style.left/top` and
|
||||
* app.js's connector redraw reads live rects, so this tracks without touching
|
||||
* app.js internals. Guards on `isConnected`: ultracode windows can be torn down
|
||||
* (SSE reconnect / auto-close) while still held. Clamps to `innerWidth/Height`,
|
||||
* which equals the *spanned* viewport in a multi-monitor window — so the window
|
||||
* can still travel across the physical monitor seam, just not off-screen. */
|
||||
moveWindow(el2, left, top) {
|
||||
if (!el2.isConnected) return;
|
||||
const w2 = el2.offsetWidth || 380;
|
||||
const h2 = el2.offsetHeight || 320;
|
||||
const l = Math.min(Math.max(4, left), Math.max(4, window.innerWidth - w2 - 4));
|
||||
const t2 = Math.min(Math.max(4, top), Math.max(4, window.innerHeight - h2 - 4));
|
||||
el2.style.left = `${l}px`;
|
||||
el2.style.top = `${t2}px`;
|
||||
this.redrawWindowConnectors();
|
||||
}
|
||||
/** Ask app.js to redraw all connector lines (subagent + ultracode), coalesced to
|
||||
* one per frame so per-frame drags don't thrash. `updateConnectionLines()` is
|
||||
* itself debounced in app.js, but we rAF-gate too in case an older dashboard
|
||||
* build isn't, and to no-op cleanly when app.js isn't present (standalone). */
|
||||
redrawWindowConnectors() {
|
||||
if (this.connectorRedrawScheduled) return;
|
||||
this.connectorRedrawScheduled = true;
|
||||
requestAnimationFrame(() => {
|
||||
this.connectorRedrawScheduled = false;
|
||||
try {
|
||||
window.app?.updateConnectionLines?.();
|
||||
} catch {
|
||||
}
|
||||
});
|
||||
}
|
||||
/** Pop a grabbed window above its siblings using app.js's own z-counter, so a
|
||||
* picked-up window comes to the front like a real focus. Cosmetic + best-effort. */
|
||||
bringWindowToFront(el2) {
|
||||
const app = window.app;
|
||||
if (!app) return;
|
||||
try {
|
||||
if (el2.classList.contains("ultracode-window")) {
|
||||
app.ultracodeWindowZIndex = (app.ultracodeWindowZIndex ?? 1e3) + 1;
|
||||
el2.style.zIndex = String(app.ultracodeWindowZIndex);
|
||||
} else {
|
||||
app.subagentWindowZIndex = (app.subagentWindowZIndex ?? 1e3) + 1;
|
||||
el2.style.zIndex = String(app.subagentWindowZIndex);
|
||||
}
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
positionGhost(ghost, x2, y2) {
|
||||
ghost.style.left = `${x2}px`;
|
||||
ghost.style.top = `${y2}px`;
|
||||
@@ -4703,15 +4780,17 @@ var GestureBridge = class {
|
||||
if (grab.kind === "tab") {
|
||||
grab.ghost.remove();
|
||||
grab.tab.classList.remove("cg-grabbed");
|
||||
} else {
|
||||
} else if (grab.kind === "panel") {
|
||||
grab.panel.el.style.pointerEvents = "";
|
||||
grab.panel.el.classList.remove("cg-float-grabbed", "cg-redock");
|
||||
} else {
|
||||
grab.el.classList.remove("cg-win-grabbed");
|
||||
}
|
||||
}
|
||||
this.grabs.clear();
|
||||
for (const tap of this.taps.values()) tap.el.classList.remove("cg-tap-armed");
|
||||
this.taps.clear();
|
||||
document.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed`).forEach((t2) => t2.classList.remove("cg-grabbed", "cg-tap-armed"));
|
||||
document.querySelectorAll(`${TAB_SELECTOR}.cg-grabbed, .cg-tap-armed, .cg-win-grabbed`).forEach((t2) => t2.classList.remove("cg-grabbed", "cg-tap-armed", "cg-win-grabbed"));
|
||||
}
|
||||
onStatus(fps, hands) {
|
||||
const { width, height } = this.canvas;
|
||||
@@ -4797,6 +4876,10 @@ function injectStyles() {
|
||||
.cg-status { color: #9aa0a6; max-width: 220px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.session-tab.cg-grabbed { opacity: .35; outline: 2px dashed #4ade80; outline-offset: -2px; }
|
||||
.cg-tap-armed { outline: 2px solid #4ade80 !important; outline-offset: 2px; box-shadow: 0 0 0 4px rgba(74,222,128,.25) !important; }
|
||||
.subagent-window.cg-win-grabbed, .ultracode-window.cg-win-grabbed {
|
||||
outline: 2px solid #4ade80 !important; outline-offset: -2px;
|
||||
box-shadow: 0 12px 48px rgba(74,222,128,.5) !important;
|
||||
}
|
||||
.cg-float {
|
||||
position: fixed; left: 0; top: 0; width: ${FLOAT_W}px; height: ${FLOAT_H}px;
|
||||
z-index: ${Z2}; display: flex; flex-direction: column; overflow: hidden;
|
||||
|
||||
Reference in New Issue
Block a user