From 02dc46dcd7d4b4d83df27b926542a3e2f5521349 Mon Sep 17 00:00:00 2001 From: Michael Grundberg Date: Mon, 21 Sep 2026 10:00:54 +0200 Subject: [PATCH 1/5] feat(tmux): report a dead pane's exit from the batched pane list Codeman creates every tmux pane with `remain-on-exit on`. When the agent exits, tmux keeps the pane, the tmux session, and the `tmux attach-session` process Codeman records as the session's pid, so no PTY exit handler fires and nothing writes the exit down. tmux itself knows: it marks the pane dead and reports the exit status. This reads that. `PANE_LIST_FORMAT` gains `#{pane_dead}`, `#{pane_dead_status}` and `#{pane_dead_signal}`, and `startPaneExitWatcher()` refreshes a muxName-to-observation map from ONE batched `tmux list-panes -a` per tick. Boot reconciliation already ran that same call, so it now fills the map too and recovery starts with a reading. The watcher owns its own interval rather than riding `startStatsCollection()`, which the issue suggested. That collector is armed when a browser opens the Monitor panel and DISARMED when it closes it, and boot skips it entirely unless recovery found a live session, so a session created on a freshly booted server would publish nothing and one browser could turn detection off for every other. Measured on an isolated instance: a dead pane with status 0 reported nothing until `POST /api/mux-sessions/stats/start` was called by hand. It is still one batched read per tick; only the timer changed. Three rules keep a positive answer trustworthy. A session answers only when tmux listed exactly one pane for it, because Codeman never splits a pane and a session the user split by hand has none that speaks for the agent. A pane answers only when `#{pane_dead}` said 1 or 0, because an empty field is a tmux that did not answer. An absent status stays absent rather than becoming 0: measured on tmux 3.2a, a SIGKILLed pane reports neither a status nor a signal, and calling that a clean exit would be wrong in the direction that matters. Two guards stop a slow read undoing a fast one. `EXEC_TIMEOUT_MS` is 5000 ms against a 2000 ms interval, so a read can outlive two ticks: one already in flight suppresses the next, and a generation counter that every `clearPaneExit()` bumps discards a read that started before a respawn or a kill. An observation also carries its pane pid, so a second command in the same pane that exits the same way starts a new timestamp rather than inheriting the first death's. A non-empty read of `list-panes -a` is authoritative for the whole socket, so sessions missing from it are pruned, which also bounds the map as tmux sessions come and go outside `killSession()`. A failed or empty read retracts nothing. The manager reports the raw pane reading and applies no session-shape scoping, because the remote-reconnect watcher beside it needs exactly that raw reading. `parsePaneList` becomes `parsePaneRows`, returning one row per pane instead of a name-to-pid map; reconciliation builds its map from the rows. The parser's existing cases carry over unchanged, including the launchd/systemd literal-tab regression from PR #71. Refs Ark0N/Codeman#446. Co-Authored-By: Claude Opus 5 (1M context) --- src/mux-interface.ts | 30 ++++ src/tmux-manager.ts | 312 ++++++++++++++++++++++++++++++++++++-- src/types/session.ts | 49 ++++++ test/tmux-manager.test.ts | 187 ++++++++++++++++++++--- 4 files changed, 546 insertions(+), 32 deletions(-) diff --git a/src/mux-interface.ts b/src/mux-interface.ts index f76ca47a..46f9f16e 100644 --- a/src/mux-interface.ts +++ b/src/mux-interface.ts @@ -24,6 +24,7 @@ import type { OmpConfig, SessionRemote, SessionDocker, + PaneExit, } from './types.js'; /** @@ -56,6 +57,16 @@ export interface MuxSession { respawnConfig?: PersistedRespawnConfig; /** Whether Ralph / Todo tracking is enabled */ ralphEnabled?: boolean; + /** + * This record was rebuilt from the tmux socket rather than from Codeman's own + * bookkeeping, so everything on it but the name and the pid is a guess. Its + * synthetic `restored-` id cannot find the session's `state.json` + * entry either, which means a remote or docker session rediscovered this way + * arrives with no `remote`/`docker` metadata and looks local. Anything that + * would be WRONG about such a session rather than merely vague must fail + * closed on this flag. + */ + discovered?: boolean; } /** @@ -180,6 +191,7 @@ export interface PaneCaptureOptions { * - `sessionKilled` (data: { sessionId: string }) - Session terminated * - `sessionDied` (data: { sessionId: string }) - Session died unexpectedly * - `statsUpdated` (sessions: MuxSessionWithStats[]) - Stats refreshed + * - `paneExitsUpdated` () - A pane read finished; ask `getPaneExit()` per session */ export interface TerminalMultiplexer extends EventEmitter { /** Which backend this instance uses */ @@ -294,6 +306,24 @@ export interface TerminalMultiplexer extends EventEmitter { /** Check if the pane in a session is dead (command exited but remain-on-exit keeps it alive) */ isPaneDead(muxName: string): boolean; + /** + * What the last pane read saw of this session's agent, or `undefined` for + * UNKNOWN (Ark0N/Codeman#446). Unlike `isPaneDead()` this costs nothing: it + * reads a map the batched watcher fills, so it answers no fresher than that + * watcher's interval and the three synchronous `isPaneDead()` callers still + * need their own probe. See {@link PaneExit}. + */ + getPaneExit?(muxName: string): PaneExit | undefined; + + /** Forget a session's exit observation, e.g. once its pane has been respawned. */ + clearPaneExit?(muxName: string): void; + + /** Start polling every pane on the socket for an exited agent. */ + startPaneExitWatcher?(intervalMs?: number): void; + + /** Stop the pane-exit watcher. */ + stopPaneExitWatcher?(): void; + /** Respawn a dead pane with a fresh command. Returns the new PID or null on failure. */ respawnPane(options: RespawnPaneOptions): Promise; diff --git a/src/tmux-manager.ts b/src/tmux-manager.ts index bc02fac9..aad961a6 100644 --- a/src/tmux-manager.ts +++ b/src/tmux-manager.ts @@ -58,6 +58,7 @@ import { type SessionRemote, type SessionDocker, type DockerCommandMode, + type PaneExit, } from './types.js'; import { getCli } from './config/cli-registry/registry.js'; import { missingCliMessage, resolveCliBinDir } from './utils/cli-resolver.js'; @@ -152,6 +153,9 @@ const GRACEFUL_SHUTDOWN_WAIT_MS = 100; /** Default stats collection interval (2 seconds) */ const DEFAULT_STATS_INTERVAL_MS = 2000; +/** How often the pane-exit watcher re-reads every pane on the socket. */ +const DEFAULT_PANE_EXIT_INTERVAL_MS = 2000; + /** Default remote-reconnect watcher poll interval (5 seconds) — COD-108 */ const DEFAULT_REMOTE_RECONNECT_INTERVAL_MS = 5000; @@ -219,8 +223,18 @@ const DEFAULT_CODEMAN_TMUX_SOCKET = DEFAULT_TMUX_SOCKET; */ const PANE_LIST_SEP = '|'; -/** Format string for `tmux list-panes -F`. Keep in sync with {@link parsePaneList}. */ -const PANE_LIST_FORMAT = `#{session_name}${PANE_LIST_SEP}#{pane_pid}`; +/** + * Format string for `tmux list-panes -F`. Keep in sync with {@link parsePaneRows}. + * + * The three `pane_dead*` fields carry the agent-exit signal of Ark0N/Codeman#446. + * Appending them is backward compatible in both directions. A tmux that does not + * know a variable substitutes the empty string rather than failing, which is how + * tmux 3.2a answers `#{pane_dead_signal}` (added in 3.4), and the parser reads a + * short row as "pid known, deadness unknown" rather than discarding it. + */ +const PANE_LIST_FORMAT = + `#{session_name}${PANE_LIST_SEP}#{pane_pid}` + + `${PANE_LIST_SEP}#{pane_dead}${PANE_LIST_SEP}#{pane_dead_status}${PANE_LIST_SEP}#{pane_dead_signal}`; /** * 构建 pane 启动前的 nofile 修复命令。 @@ -235,26 +249,114 @@ export function buildNofileLimitCommand(targetLimit = CLAUDE_CODE_NOFILE_LIMIT): return `ulimit -Sn ${safeLimit} 2>/dev/null || ulimit -n ${safeLimit} 2>/dev/null || true`; } +/** One pane of one tmux session, as {@link parsePaneRows} reads it off the wire. */ +export interface PaneRow { + /** tmux session this pane belongs to. Repeats once per pane of a split session. */ + sessionName: string; + /** `#{pane_pid}` — the process tmux started in the pane. */ + pid: number; + /** `#{pane_dead}` — true for 1, false for 0, undefined when tmux said nothing. */ + dead?: boolean; + /** `#{pane_dead_status}` — the exit code, absent when tmux reported none. */ + exitStatus?: number; + /** `#{pane_dead_signal}` — the killing signal, absent before tmux 3.4 and when unsignalled. */ + exitSignal?: number; +} + /** - * Parse the output of `tmux list-panes -a -F '#{session_name}|#{pane_pid}'` - * into a Map of session-name → pane pid. Exported for unit testing. + * One pane-exit reading, with the pane pid that produced it. + * + * The pid never leaves this module. It is what distinguishes "the same dead + * pane, seen again" from "a second command in the same pane that also exited + * with the same status", so the `at` stamp can hold across the first and must + * not across the second. {@link PaneExit} itself stays free of it: the pid on + * the session record is the attach client's, and a second pid there would + * invite exactly the confusion Ark0N/Codeman#446 is about. + */ +export interface PaneExitObservation { + /** `#{pane_pid}` of the pane this reading came from. */ + panePid: number; + /** What to publish on the session record. */ + exit: PaneExit; +} + +/** Read one optional numeric field; a blank or non-numeric value is "not reported". */ +function paneField(fields: string[], index: number): number | undefined { + const raw = fields[index]; + if (raw === undefined || raw === '') return undefined; + const value = parseInt(raw, 10); + return Number.isNaN(value) ? undefined : value; +} + +/** + * Parse the output of `tmux list-panes -a -F` under {@link PANE_LIST_FORMAT} + * into one row per pane, in tmux's own order. Exported for unit testing. * * - Skips empty lines and lines without the separator. * - Skips entries with a non-numeric pid or empty name. + * - Leaves every field after the pid undefined when it is blank or absent, so a + * row from an older tmux still yields its pid. */ -export function parsePaneList(output: string): Map { - const result = new Map(); +export function parsePaneRows(output: string): PaneRow[] { + const rows: PaneRow[] = []; for (const line of output.split('\n')) { if (!line) continue; - const sep = line.indexOf(PANE_LIST_SEP); - if (sep === -1) continue; - const name = line.slice(0, sep); - const pid = parseInt(line.slice(sep + 1), 10); - if (name && !Number.isNaN(pid)) { - result.set(name, pid); - } + if (!line.includes(PANE_LIST_SEP)) continue; + const fields = line.split(PANE_LIST_SEP); + const sessionName = fields[0]; + const pid = parseInt(fields[1] ?? '', 10); + if (!sessionName || Number.isNaN(pid)) continue; + const deadFlag = fields[2]; + rows.push({ + sessionName, + pid, + dead: deadFlag === '1' ? true : deadFlag === '0' ? false : undefined, + exitStatus: paneField(fields, 3), + exitSignal: paneField(fields, 4), + }); } - return result; + return rows; +} + +/** + * Decide, from every pane tmux listed, which tmux sessions have an exited agent. + * Exported for unit testing. Returns one entry per session with a known answer; + * a session absent from the map is UNKNOWN, which must never render as alive. + * + * Two rules make a positive answer trustworthy: + * + * A session answers only when tmux listed EXACTLY ONE pane for it. Codeman + * creates one pane per session and `isPaneDead()` reads one pane, so a session + * the user has split by hand has no single "the agent" to report on, and + * guessing which of its panes speaks for the session could report a live + * session as exited. + * + * A pane answers only when `#{pane_dead}` said 1 or 0. An empty field is a tmux + * that did not answer, not a live pane. + * + * `status` and `signal` stay absent when tmux did not report them. Measured on + * tmux 3.2a, a SIGKILLed pane reports neither, so folding an absent status into + * 0 would turn an unexplained death into a clean exit. + */ +export function derivePaneExits(rows: PaneRow[], now: number): Map { + const panesPerSession = new Map(); + for (const row of rows) { + panesPerSession.set(row.sessionName, (panesPerSession.get(row.sessionName) ?? 0) + 1); + } + const exits = new Map(); + for (const row of rows) { + if (panesPerSession.get(row.sessionName) !== 1) continue; + if (row.dead !== true) continue; + exits.set(row.sessionName, { + panePid: row.pid, + exit: { + ...(row.exitStatus !== undefined ? { status: row.exitStatus } : {}), + ...(row.exitSignal !== undefined ? { signal: row.exitSignal } : {}), + at: now, + }, + }); + } + return exits; } /** @@ -1527,6 +1629,30 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { private mouseSyncInterval: NodeJS.Timeout | null = null; /** Track last-known pane count per session to avoid unnecessary tmux set-option calls */ private lastPaneCount: Map = new Map(); + /** + * muxName → the exited agent the pane-exit watcher last observed + * (Ark0N/Codeman#446). Absence is the UNKNOWN arm of the tri-state, so an + * entry goes the moment tmux stops reporting the pane dead, and the map is + * empty until the first read runs. The manager reports what tmux says and + * nothing more: the scoping that hides this for remote and docker sessions + * lives on `Session`, because the remote-reconnect watcher above needs the + * raw pane reading. + */ + private paneExits: Map = new Map(); + /** The pane-exit watcher's own interval. Runs whether or not stats are on. */ + private paneExitInterval: NodeJS.Timeout | null = null; + /** + * True while a pane read is in flight. `EXEC_TIMEOUT_MS` is 5000 ms against a + * poll interval of 2000 ms, so without this a slow read overlaps the next two + * and the older one can resolve last and win. + */ + private paneExitReadInFlight = false; + /** + * Bumped by every deliberate {@link clearPaneExit}. A read that started before + * a clear carries the older generation and is discarded rather than writing + * the death back over the pane that has just replaced it. + */ + private paneExitGeneration = 0; // ── COD-108 remote-reconnect watcher state ──────────────────────────────── /** Periodic watcher that re-establishes dropped remote sessions. */ @@ -2297,6 +2423,11 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { ); // Wait for the respawned process to start await new Promise((resolve) => setTimeout(resolve, TMUX_CREATION_WAIT_MS)); + // The pane now runs a fresh command, so whatever the last read observed of + // the old one is history. Clearing it here rather than waiting for the next + // poll also invalidates any read already in flight, which would otherwise + // write the old death back over the pane that just replaced it. + this.clearPaneExit(muxName); const pid = this.getPanePid(muxName); if (pid) session.pid = pid; return pid; @@ -2508,6 +2639,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { } } this.lastPaneCount.delete(session.muxName); + this.clearPaneExit(session.muxName); this.sessions.delete(sessionId); this.clearRemoteReconnectState(sessionId); this.saveSessions(); @@ -2618,6 +2750,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { } this.lastPaneCount.delete(session.muxName); + this.clearPaneExit(session.muxName); this.sessions.delete(sessionId); this.clearRemoteReconnectState(sessionId); this.saveSessions(); @@ -2671,7 +2804,14 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { encoding: 'utf-8', timeout: EXEC_TIMEOUT_MS, }).trim(); - active = parsePaneList(output); + const rows = parsePaneRows(output); + active = new Map(rows.map((row) => [row.sessionName, row.pid])); + // The same read answers both questions, so recovery starts with a pane-exit + // reading rather than waiting for the first stats tick — which may never + // come, since the collector only starts when boot found a live session. + if (rows.length > 0) { + this.applyPaneExits(derivePaneExits(rows, Date.now())); + } } catch (err) { console.error('[TmuxManager] Failed to list tmux panes:', err); active = new Map(); @@ -2686,6 +2826,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { } else { dead.push(sessionId); this.sessions.delete(sessionId); + this.clearPaneExit(session.muxName); this.clearRemoteReconnectState(sessionId); this.emit('sessionDied', { sessionId }); } @@ -2722,6 +2863,13 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { mode: 'claude', attached: false, name: `Restored: ${sessionName}`, + // Every field above except the name and the pid is a guess: this record + // was rebuilt from the socket because Codeman's own bookkeeping did not + // have it. The synthetic id also cannot find the session's state.json + // entry, so a remote or docker session rediscovered this way arrives + // looking local. Consumers that would be wrong about such a session + // read this flag and fail closed — see `Session.paneExitApplies`. + discovered: true, }; this.sessions.set(sessionId, session); knownMuxNames.add(sessionName); @@ -2882,6 +3030,138 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { })); } + /** + * What the last pane read saw of this tmux session's agent. `undefined` is + * the UNKNOWN answer and must never be rendered as "alive": it covers a pane + * that is running, a tmux session that no longer exists, a probe that failed, + * and every poll that has not run yet. See {@link PaneExit}. + */ + getPaneExit(muxName: string): PaneExit | undefined { + return this.paneExits.get(muxName)?.exit; + } + + /** + * Re-read every pane on the socket and refresh {@link paneExits}. ONE batched + * `tmux list-panes -a` answers for every session at once, which is why this + * polls rather than probing per session. + * + * A failed or empty probe leaves the previous answers ALONE rather than + * clearing them. An empty read is "tmux did not answer", and clearing on it + * would turn a transient failure into a silent retraction of a death Codeman + * had already observed. A NON-empty read is different: `list-panes -a` lists + * every pane on the socket, so it is authoritative and {@link applyPaneExits} + * prunes against it. + * + * Two guards keep a slow read from undoing a fast one. A read already in + * flight suppresses the next poll, and a read that started before a + * {@link clearPaneExit} is discarded when it lands. + */ + async refreshPaneExits(now: number = Date.now()): Promise { + if (IS_TEST_MODE) return; + if (this.paneExitReadInFlight) return; + + const generation = this.paneExitGeneration; + this.paneExitReadInFlight = true; + let rows: PaneRow[]; + try { + // execAsync, not execSync: this runs on a 2000 ms timer, and a synchronous + // exec freezes the port while the process stays alive (see the + // event-loop-monitor note in CLAUDE.md). The three `isPaneDead()` callers + // stay synchronous because each is answering one request right then. + const { stdout } = await execAsync(`${this.tmux()} list-panes -a -F '${PANE_LIST_FORMAT}' 2>/dev/null || true`, { + encoding: 'utf-8', + timeout: EXEC_TIMEOUT_MS, + }); + rows = parsePaneRows(stdout.trim()); + } catch (err) { + console.error('[TmuxManager] Failed to read pane exit state:', err); + return; + } finally { + this.paneExitReadInFlight = false; + } + if (rows.length === 0) return; + // A pane was respawned or killed while this read was out, so what it saw is + // already history. Dropping it is what stops a freshly respawned pane from + // being republished as exited. + if (generation !== this.paneExitGeneration) return; + + this.applyPaneExits(derivePaneExits(rows, now)); + } + + /** + * Fold one authoritative observation into {@link paneExits}. Split out from + * the tmux call so the merge rules are unit-testable. + * + * `observed` comes from a read of EVERY pane on the socket, so a session + * missing from it has no exit to report and its entry goes. That is what + * keeps the map from growing without bound as tmux sessions come and go + * outside `killSession()`. Only the caller may decide a read is authoritative: + * a failed or empty one never reaches here. + * + * An entry keeps the `at` of the FIRST read that saw that exit, so the stamp + * says when the agent was found gone rather than when the last poll ran. A + * changed status, a changed signal, or a different pane pid all start a new + * observation — the pid is what catches a second command in the same pane + * that happened to exit the same way. + */ + applyPaneExits(observed: Map): void { + for (const muxName of [...this.paneExits.keys()]) { + if (!observed.has(muxName)) this.paneExits.delete(muxName); + } + for (const [muxName, next] of observed) { + const prev = this.paneExits.get(muxName); + const sameExit = + prev !== undefined && + prev.panePid === next.panePid && + prev.exit.status === next.exit.status && + prev.exit.signal === next.exit.signal; + this.paneExits.set(muxName, sameExit ? prev : next); + } + } + + /** + * Forget a session's exit observation, e.g. once its pane has been respawned. + * Also invalidates any read already in flight, so the answer this retracts + * cannot be written back a moment later. + */ + clearPaneExit(muxName: string): void { + this.paneExits.delete(muxName); + this.paneExitGeneration++; + } + + /** + * Poll for exited agents, on the manager's own interval. + * + * Deliberately NOT part of `startStatsCollection()`. That collector is armed + * when the browser opens the Monitor panel and DISARMED when it closes it + * (`panels-ui.js`), and it is skipped at boot entirely when no session was + * recovered — so riding it would leave a session created on a freshly booted + * server reporting nothing at all, and would let one browser turn exit + * detection off for every other. Started unconditionally, like the mouse-mode + * sync and the remote-reconnect watcher below. + * + * The `paneExitsUpdated` event is internal to the server; nothing here adds an + * SSE event, and the field reaches the browser on `session:updated`. + */ + startPaneExitWatcher(intervalMs: number = DEFAULT_PANE_EXIT_INTERVAL_MS): void { + if (this.paneExitInterval) { + clearInterval(this.paneExitInterval); + } + this.paneExitInterval = setInterval(() => { + if (IS_TEST_MODE) return; + void this.refreshPaneExits() + .then(() => this.emit('paneExitsUpdated')) + .catch((err) => console.error('[TmuxManager] Pane exit watcher error:', err)); + }, intervalMs); + } + + stopPaneExitWatcher(): void { + if (this.paneExitInterval) { + clearInterval(this.paneExitInterval); + this.paneExitInterval = null; + } + } + startStatsCollection(intervalMs: number = DEFAULT_STATS_INTERVAL_MS): void { if (this.statsInterval) { clearInterval(this.statsInterval); @@ -3101,6 +3381,8 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { destroy(): void { this.stopStatsCollection(); + this.stopPaneExitWatcher(); + this.paneExits.clear(); this.stopMouseModeSync(); this.stopRemoteReconnectWatcher(); this.reconnectState.clear(); diff --git a/src/types/session.ts b/src/types/session.ts index 20d788b2..258e5de1 100644 --- a/src/types/session.ts +++ b/src/types/session.ts @@ -625,6 +625,40 @@ export interface CustomModelBookkeeping extends CustomModelSelection { launchModel?: string; } +/** + * The agent inside a LOCAL tmux pane has exited, and the pane survived it. + * + * Codeman creates every pane with `remain-on-exit on`, so `/exit` ends the CLI + * while tmux keeps the pane, the tmux session and the `tmux attach-session` + * process Codeman records as the session's pid. No PTY exit handler runs, so + * without this record the session reads as a live idle one (Ark0N/Codeman#446). + * + * The field is TRI-STATE, and the third state is the absence of the field: + * `undefined` means Codeman does not know, and it must never be rendered as + * "alive". It is absent for a direct-PTY session (no pane exists), for a remote + * SSH session (the local pane holds the ssh client, whose death means transport + * drop OR exit) and for a docker case (the local pane holds a `docker exec` + * into the container's own tmux). + * + * `status` and `signal` are independently optional because tmux may know that + * the pane died without reporting how. Measured on tmux 3.2a: a SIGKILLed pane + * reports `pane_dead=1` with BOTH `#{pane_dead_status}` and `#{pane_dead_signal}` + * empty, and `#{pane_dead_signal}` does not exist at all before tmux 3.4. So an + * absent `status` means "the exit code is unknown", never "the exit code is 0". + */ +export interface PaneExit { + /** tmux `#{pane_dead_status}` — the command's exit code. Absent when tmux reported none. */ + status?: number; + /** tmux `#{pane_dead_signal}` — the signal that killed the command. Absent when unsignalled or unsupported. */ + signal?: number; + /** + * Wall-clock ms when THIS server process first observed the pane dead. It is + * not when the agent exited, which nothing records, and a restart that finds + * the pane still dead respawns it rather than re-timing the old exit. + */ + at: number; +} + export interface SessionState { /** Unique session identifier */ id: string; @@ -780,6 +814,21 @@ export interface SessionState { * (COD-118). Runtime-only: never restored on boot (fresh server = fresh breaker). */ respawnBlocked?: boolean; + /** + * The agent in this session's LOCAL tmux pane has exited (Ark0N/Codeman#446). + * See {@link PaneExit} for the tri-state rule and for which session shapes + * leave it absent. `status` and `pid` are deliberately untouched by it: the + * PTY-exit breaker owns `status: 'error'`, and a null `pid` is what makes the + * browser re-attach and launch a fresh CLI. + * + * Persisted so a reboot restore can tell a session whose agent exited from one + * that was merely idle when the power went. `reboot-restore.ts` reads the + * persisted record and never builds a `Session`, so the record is the only + * place that survives the reboot to carry it. Nothing reads it there YET: + * making the restore refuse such a session is a behavior change, and it + * belongs with the part of Ark0N/Codeman#446 that closes exited sessions. + */ + paneExit?: PaneExit; } /** diff --git a/test/tmux-manager.test.ts b/test/tmux-manager.test.ts index 7f65aea1..2e9d6809 100644 --- a/test/tmux-manager.test.ts +++ b/test/tmux-manager.test.ts @@ -14,7 +14,8 @@ import { buildRemoteKillCommand, buildRemoteLaunchCommand, formatPaneSnapshot, - parsePaneList, + parsePaneRows, + derivePaneExits, resolveActivePaneTarget, } from '../src/tmux-manager.js'; import { execSync, exec } from 'node:child_process'; @@ -910,41 +911,44 @@ describe('TmuxManager (unit)', () => { // exec without TTY). See PR #71. // ============================================================================ -describe('parsePaneList', () => { +describe('parsePaneRows', () => { + /** Pull the name → pid map reconciliation builds, so these cases read as they used to. */ + const pids = (output: string) => new Map(parsePaneRows(output).map((row) => [row.sessionName, row.pid])); + it('parses well-formed output into name → pid', () => { const out = 'codeman-aaaa|1234\ncodeman-bbbb|5678\nclaudeman-cccc|9999'; - const result = parsePaneList(out); + const result = pids(out); expect(result.size).toBe(3); expect(result.get('codeman-aaaa')).toBe(1234); expect(result.get('codeman-bbbb')).toBe(5678); expect(result.get('claudeman-cccc')).toBe(9999); }); - it('returns an empty map for empty output', () => { - expect(parsePaneList('').size).toBe(0); + it('returns no rows for empty output', () => { + expect(parsePaneRows('')).toEqual([]); }); it('skips blank lines', () => { - const result = parsePaneList('\ncodeman-aaaa|100\n\n\ncodeman-bbbb|200\n'); + const result = pids('\ncodeman-aaaa|100\n\n\ncodeman-bbbb|200\n'); expect(result.size).toBe(2); expect(result.get('codeman-aaaa')).toBe(100); expect(result.get('codeman-bbbb')).toBe(200); }); it('skips lines without the separator', () => { - const result = parsePaneList('codeman-aaaa 1234\ncodeman-bbbb|5678'); + const result = pids('codeman-aaaa 1234\ncodeman-bbbb|5678'); expect(result.size).toBe(1); expect(result.get('codeman-bbbb')).toBe(5678); }); it('skips lines with a non-numeric pid', () => { - const result = parsePaneList('codeman-aaaa|notapid\ncodeman-bbbb|5678'); + const result = pids('codeman-aaaa|notapid\ncodeman-bbbb|5678'); expect(result.size).toBe(1); expect(result.get('codeman-bbbb')).toBe(5678); }); it('skips lines with an empty session name', () => { - const result = parsePaneList('|1234\ncodeman-bbbb|5678'); + const result = pids('|1234\ncodeman-bbbb|5678'); expect(result.size).toBe(1); expect(result.get('codeman-bbbb')).toBe(5678); }); @@ -955,15 +959,164 @@ describe('parsePaneList', () => { // tab byte. With the '|' separator, such literals must not be silently // treated as a delimiter — the line is discarded because there is no '|'. const literalBackslashT = 'codeman-aaaa\\t1234'; - const result = parsePaneList(literalBackslashT); - expect(result.size).toBe(0); + expect(parsePaneRows(literalBackslashT)).toEqual([]); }); - it('splits on the first separator only', () => { - // Numeric trailing junk after the pid is tolerated by parseInt — proves - // that splitting on the first '|' leaves the pid extractable even if a - // future tmux ever appended extra fields. - const result = parsePaneList('codeman-aaaa|1234|extra-field'); - expect(result.get('codeman-aaaa')).toBe(1234); + it('keeps a row whose pane_dead fields are missing, and calls its deadness unknown', () => { + // A tmux old enough to have shipped the previous two-field format, or one + // that dropped the trailing fields, must still yield its pid. + const [row] = parsePaneRows('codeman-aaaa|1234'); + expect(row.pid).toBe(1234); + expect(row.dead).toBeUndefined(); + expect(row.exitStatus).toBeUndefined(); + expect(row.exitSignal).toBeUndefined(); + }); + + it('reads a live pane as not dead, with no status or signal', () => { + // Measured against tmux 3.2a: a live pane leaves both numeric fields blank. + const [row] = parsePaneRows('codeman-aaaa|1234|0|||1'); + expect(row.dead).toBe(false); + expect(row.exitStatus).toBeUndefined(); + expect(row.exitSignal).toBeUndefined(); + }); + + it('reads a dead pane with its exit status', () => { + const [row] = parsePaneRows('codeman-aaaa|1234|1|7|'); + expect(row.dead).toBe(true); + expect(row.exitStatus).toBe(7); + expect(row.exitSignal).toBeUndefined(); + }); + + it('reads a dead pane with its killing signal', () => { + const [row] = parsePaneRows('codeman-aaaa|1234|1||9'); + expect(row.dead).toBe(true); + expect(row.exitStatus).toBeUndefined(); + expect(row.exitSignal).toBe(9); + }); + + it('leaves a status of 0 as 0 rather than dropping it', () => { + // The whole point of the field: a clean exit is the case part 2 acts on. + const [row] = parsePaneRows('codeman-aaaa|1234|1|0|'); + expect(row.exitStatus).toBe(0); + }); + + it('calls a non-numeric dead flag unknown rather than false', () => { + const [row] = parsePaneRows('codeman-aaaa|1234|?||'); + expect(row.dead).toBeUndefined(); + }); + + it('returns one row per pane of a split session, in tmux order', () => { + const rows = parsePaneRows('codeman-aaaa|100|0|||\ncodeman-aaaa|200|1|0|'); + expect(rows.map((row) => row.pid)).toEqual([100, 200]); + expect(rows.map((row) => row.sessionName)).toEqual(['codeman-aaaa', 'codeman-aaaa']); + }); +}); + +describe('derivePaneExits', () => { + const NOW = 1_700_000_000_000; + + it('reports a single dead pane with its exit status', () => { + const exits = derivePaneExits(parsePaneRows('codeman-aaaa|1234|1|0|'), NOW); + expect(exits.get('codeman-aaaa')).toEqual({ panePid: 1234, exit: { status: 0, at: NOW } }); + }); + + it('reports a signalled death without inventing a status', () => { + // Folding an absent status into 0 would turn an unexplained death into the + // clean exit part 2 closes on sight. + const exits = derivePaneExits(parsePaneRows('codeman-aaaa|1234|1||9'), NOW); + expect(exits.get('codeman-aaaa')).toEqual({ panePid: 1234, exit: { signal: 9, at: NOW } }); + }); + + it('reports a death tmux could not explain at all', () => { + // Measured on tmux 3.2a: a SIGKILLed pane reports pane_dead=1 and nothing else. + const exits = derivePaneExits(parsePaneRows('codeman-aaaa|1234|1||'), NOW); + expect(exits.get('codeman-aaaa')).toEqual({ panePid: 1234, exit: { at: NOW } }); + }); + + it('says nothing about a live pane', () => { + const exits = derivePaneExits(parsePaneRows('codeman-aaaa|1234|0|||'), NOW); + expect(exits.has('codeman-aaaa')).toBe(false); + }); + + it('says nothing about a pane whose deadness tmux did not report', () => { + expect(derivePaneExits(parsePaneRows('codeman-aaaa|1234'), NOW).size).toBe(0); + }); + + it('says nothing about a session with more than one pane, even when all are dead', () => { + // A session the user split by hand has no single "the agent" to report on, + // and guessing which pane speaks for it could call a live session exited. + const exits = derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|\ncodeman-aaaa|200|1|0|'), NOW); + expect(exits.size).toBe(0); + }); + + it("answers per session, so one session's split does not silence another", () => { + const exits = derivePaneExits( + parsePaneRows('codeman-aaaa|100|1|0|\ncodeman-bbbb|200|1|0|\ncodeman-bbbb|201|0|||'), + NOW + ); + expect([...exits.keys()]).toEqual(['codeman-aaaa']); + }); +}); + +describe('TmuxManager pane-exit bookkeeping', () => { + const NOW = 1_700_000_000_000; + + it('reports nothing before any tick has run', () => { + const manager = new TmuxManager(); + expect(manager.getPaneExit('codeman-aaaa')).toBeUndefined(); + }); + + it('keeps the timestamp of the FIRST tick that saw an unchanged exit', () => { + // The stamp says when the agent was found gone, so a pane that stays dead + // must not have its age reset every two seconds. + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW + 2000)); + expect(manager.getPaneExit('codeman-aaaa')).toEqual({ status: 0, at: NOW }); + }); + + it('starts a new observation when the exit status changes', () => { + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|137|'), NOW + 2000)); + expect(manager.getPaneExit('codeman-aaaa')).toEqual({ status: 137, at: NOW + 2000 }); + }); + + it('forgets the exit once the same session reports a live pane', () => { + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|101|0|||'), NOW + 2000)); + expect(manager.getPaneExit('codeman-aaaa')).toBeUndefined(); + }); + + it('prunes an exit for a session an authoritative read did not mention', () => { + // `list-panes -a` lists every pane on the socket, so a session missing from + // a successful read has no pane at all and no exit to report. Keeping the + // entry would grow the map forever as tmux sessions come and go outside + // killSession(). A FAILED or empty read never reaches here — refreshPaneExits + // returns before calling this, which is the case the next test covers. + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-bbbb|200|1|0|'), NOW)); + expect(manager.getPaneExit('codeman-aaaa')).toBeUndefined(); + expect(manager.getPaneExit('codeman-bbbb')).toEqual({ status: 0, at: NOW }); + }); + + it('starts a new observation when the same status comes from a different pane pid', () => { + // A second command in the same pane that also exited 0 is a NEW death, and + // its `at` must say so. Only reachable when the respawn bypassed + // respawnPane() — a hand-run `tmux respawn-pane` — since every Codeman path + // clears the entry outright. + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|101|1|0|'), NOW + 60_000)); + expect(manager.getPaneExit('codeman-aaaa')).toEqual({ status: 0, at: NOW + 60_000 }); + }); + + it('forgets an exit on request, which is what a respawned pane needs', () => { + const manager = new TmuxManager(); + manager.applyPaneExits(derivePaneExits(parsePaneRows('codeman-aaaa|100|1|0|'), NOW)); + manager.clearPaneExit('codeman-aaaa'); + expect(manager.getPaneExit('codeman-aaaa')).toBeUndefined(); }); }); From 90a95f562b950a125aa4b3d24d725adb490ebea2 Mon Sep 17 00:00:00 2001 From: Michael Grundberg Date: Mon, 21 Sep 2026 10:01:09 +0200 Subject: [PATCH 2/5] feat(session): publish and persist a local pane's agent exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mux layer now knows a pane's agent has exited. This puts it on the session record, where the board and, later, the reboot restore can see it. `SessionState.paneExit` carries `{ status?, signal?, at }` and rides the existing `session:updated` broadcast through `toState()`. No new SSE event. The server pulls each answer from `mux.getPaneExit()` rather than off a broadcast payload, so the raw reading never reaches a browser: for a remote or docker session that reading is the death of an ssh client or a `docker exec`, not of the agent. The field is tri-state, and the third state is its absence: `undefined` means Codeman does not know, and it never reads as alive. `Session.setPaneExit()` forces that unknown for every shape a dead local pane does not describe. A direct-PTY session owns no pane. A remote SSH session's local pane holds the ssh client, whose death means a transport drop OR an exit, which is the ambiguity PR #355 settled by not guessing. A docker case's local pane holds a `docker exec` into the container's own tmux. And a session rebuilt from the socket has no provenance at all: `reconcileSessions()` gives it a synthetic `restored-` id that matches no `state.json` entry, so a remote session rediscovered after `mux-sessions.json` was lost arrives with no `remote` field and looks local — `MuxSession.discovered` marks it, and absent metadata there counts as unproven rather than as proof. The scoping lives on `Session` rather than in `TmuxManager` so there is one copy of the rule. `status` and `pid` are untouched. `status: 'error'` belongs to the PTY-exit circuit breaker and makes the browser offer a restart, and a null `pid` is what makes the browser re-attach and launch a fresh CLI. A reading that repeats the previous answer writes nothing and broadcasts nothing. An unknown answer never reads as alive, but a stale KNOWN one would keep reading as exited, so `clearPaneExitForNewPane()` retracts it on every path that puts a new command in the pane: the start/attach path, the `restartCli()` relaunch behind a custom-model switch, and the remote reattach. Without the second of those, switching an endpoint on an exited session launched a new command and then persisted and broadcast the old exit straight back onto it. `toState()` is also what `state.json` persists, so the record survives a reboot, which is the only thing that does: a reboot takes the tmux server, and with it every live signal and every `mux-sessions.json` entry. Nothing reads it there yet — making the restore refuse such a session is a behavior change that belongs with the part that closes them. Recovery threads the saved value back through the constructor so the first persist after boot cannot blank it. Refs Ark0N/Codeman#446. Co-Authored-By: Claude Opus 5 (1M context) --- src/session.ts | 112 ++++++++++++ src/web/server.ts | 52 ++++++ test/session-pane-exit.test.ts | 312 +++++++++++++++++++++++++++++++++ 3 files changed, 476 insertions(+) create mode 100644 test/session-pane-exit.test.ts diff --git a/src/session.ts b/src/session.ts index fdad22f8..a1a0294c 100644 --- a/src/session.ts +++ b/src/session.ts @@ -60,6 +60,7 @@ import { type SessionDocker, type SessionNameSource, type SessionWriteOptions, + type PaneExit, } from './types.js'; import { resolveAndClaimOmpSessionId } from './utils/omp-session-resolver.js'; import { probeDockerCliVersion } from './docker-hosts.js'; @@ -542,6 +543,20 @@ export class Session extends EventEmitter { private _mux: TerminalMultiplexer | null = null; private _muxSession: MuxSession | null = null; private _useMux: boolean = false; + /** + * The agent in this session's local tmux pane has exited (Ark0N/Codeman#446). + * `null` is the UNKNOWN arm of the tri-state and is what {@link setPaneExit} + * stores for every session shape the field does not apply to. See + * {@link PaneExit} for the shapes and for why an unknown answer must never be + * rendered as "alive". + */ + private _paneExit: PaneExit | null = null; + /** + * This session was rebuilt from the tmux socket rather than from Codeman's + * own records, so its `remote`/`docker` metadata is missing rather than known + * to be absent. See {@link MuxSession.discovered}. + */ + private _discoveredMuxSession = false; // Flag to prevent new timers after session is stopped private _isStopped: boolean = false; @@ -718,6 +733,10 @@ export class Session extends EventEmitter { lastSubmitAt?: number; /** Restored conversation chain, oldest first (see `claudeSessionChain`). */ claudeSessionChain?: string[]; + /** Restored agent-exit observation for this session's pane (see `paneExit`). */ + paneExit?: PaneExit; + /** This session was rebuilt from the tmux socket, so its metadata is a guess. */ + discoveredMuxSession?: boolean; /** Restored wall-clock ms of the pane's last output (recovery only; see `_wireActivityAt`). */ lastActivityAt?: number; /** Remote execution metadata for sessions launched through SSH inside local tmux. */ @@ -870,6 +889,15 @@ export class Session extends EventEmitter { this._remote = config.remote; this._docker = config.docker; this._owner = config.owner; + this._discoveredMuxSession = config.discoveredMuxSession === true; + // Restored so a record that says the agent exited survives a server restart + // rather than being blanked by the first persist after boot. It runs here + // because the scoping reads `_remote`, `_docker` and the mux fields, all of + // which are set by now. It is a claim about a pane this process has not + // looked at yet, so every path that starts or re-attaches a pane drops it + // (see `_setupOrAttachMuxSession`) and the stats tick replaces it with a + // first-hand reading. + this.setPaneExit(config.paneExit); // Never self-parent: a session pointing at itself would draw a zero-length // lineage arc under its own tab. Only reachable via the recovery path, where // both the id and the saved parent come from disk. @@ -1097,6 +1125,71 @@ export class Session extends EventEmitter { return this._muxSession?.muxName ?? null; } + /** + * True when a tmux pane's death would mean THIS session's agent has exited. + * + * Four shapes fail the test, and each would otherwise publish a death that is + * not the agent's. A direct-PTY session owns no pane at all. A remote SSH + * session's local pane holds the ssh client, whose death means a transport + * drop OR an exit, which is the ambiguity PR #355 was about. A docker case's + * local pane holds a `docker exec` into the container's own tmux. + * + * The fourth is a session rebuilt from the socket. Absent `remote`/`docker` + * normally means "this is local", but on a discovered record it only means + * "Codeman never found the metadata": the synthetic `restored-` id + * matches no `state.json` entry, so a remote session rediscovered after + * `mux-sessions.json` was lost arrives looking local, and its next transport + * drop would be published as an agent exit. Unproven locality fails closed. + */ + private get paneExitApplies(): boolean { + if (this._discoveredMuxSession) return false; + return this._useMux && this._muxSession !== null && !this._remote && !this._docker; + } + + /** What Codeman last observed of this pane's agent, or undefined for UNKNOWN. */ + get paneExit(): PaneExit | undefined { + return this._paneExit ?? undefined; + } + + /** + * Forget this pane's exit, on both this record and the mux layer's cache. + * Every path that starts or relaunches a command in the pane calls it, and + * the mux half also invalidates a pane read already in flight. + * + * It does not persist or broadcast by itself. Each caller is already followed + * by the route's or recovery's own persist, and the pane-exit watcher would + * reach the same answer within one interval regardless. + */ + private clearPaneExitForNewPane(): void { + this.setPaneExit(undefined); + if (this._muxSession) this._mux?.clearPaneExit?.(this._muxSession.muxName); + } + + /** + * Record what the mux layer observed of this pane's agent, and say whether + * that changed the answer. The caller persists and broadcasts on a true. + * + * A session the field does not apply to is forced to UNKNOWN here rather than + * at the reporting end, so the rule lives in one place and the mux layer stays + * free to report the raw pane reading its own remote-reconnect watcher needs. + */ + setPaneExit(next: PaneExit | undefined): boolean { + const resolved = this.paneExitApplies ? (next ?? null) : null; + const prev = this._paneExit; + if (prev === resolved) return false; + if ( + prev !== null && + resolved !== null && + prev.status === resolved.status && + prev.signal === resolved.signal && + prev.at === resolved.at + ) { + return false; + } + this._paneExit = resolved; + return true; + } + /** * True when this session's PTY is a tmux client rather than the program itself. * Read by the replay-side alt-screen strip, which must apply the same @@ -1620,6 +1713,10 @@ export class Session extends EventEmitter { // by the constructor: a Codeman restart starts with a fresh breaker so boot // recovery can re-attach. respawnBlocked: this._respawnBlocked || undefined, + // Ark0N/Codeman#446 — the agent in this pane has exited, published here so + // it rides the existing `session:updated` broadcast and lands in state.json + // through the same persist. `status` and `pid` above stay untouched by it. + paneExit: this._paneExit ?? undefined, attachmentHistory: this.attachmentHistory.length > 0 ? this.attachmentHistory : undefined, lastSubmitAt: this._lastSubmitAt || undefined, // Only a chain the CLI's own hooks vouched for is persisted, and only when @@ -1760,6 +1857,14 @@ export class Session extends EventEmitter { } } + // Whatever the last reading said about the OLD command in this pane is now + // history: the branch above either respawned the pane or found it alive, and + // the branch below creates a new one. The paths that reach here are boot + // recovery and an explicit start, NOT a click on an exited tab — the browser + // re-attaches only on a null pid, and the premise of Ark0N/Codeman#446 is + // that an exited pane keeps its pid. `restartCli()` clears separately. + this.clearPaneExitForNewPane(); + // Check if we already have a mux session (restored session) const isRestored = this._muxSession !== null && !needsNewSession; if (isRestored) { @@ -1844,6 +1949,9 @@ export class Session extends EventEmitter { console.error('[Session] reattachRemote: respawnPane failed for', this._muxSession.muxName); return false; } + // No-op for the record (a remote session's field is always UNKNOWN), but the + // mux layer's cache is keyed by muxName and this pane now runs a new client. + this.clearPaneExitForNewPane(); console.log('[Session] reattachRemote: reattached remote session', this._muxSession.muxName, 'pid', newPid); return true; } @@ -1898,6 +2006,10 @@ export class Session extends EventEmitter { return false; } this._pendingEnvUnsets.clear(); + // A relaunch in the same pane, so any exit observed of the previous command + // is history. Without this the caller's persist-and-broadcast writes the old + // exit straight back onto a session that is running again. + this.clearPaneExitForNewPane(); console.log('[Session] restartCli: restarted CLI for', this._muxSession.muxName, 'pid', newPid); return true; } diff --git a/src/web/server.ts b/src/web/server.ts index 8d8f9368..7257eb2a 100644 --- a/src/web/server.ts +++ b/src/web/server.ts @@ -463,6 +463,11 @@ export class WebServer extends EventEmitter { this.mux.on('statsUpdated', (sessions) => { this.broadcast(SseEvent.MuxStatsUpdated, sessions); }); + // Ark0N/Codeman#446 — a pane read finished. Internal only: the field reaches + // the browser on `session:updated`, and no SSE event was added for it. + this.mux.on('paneExitsUpdated', () => { + this.applyPaneExits(); + }); // COD-108 — remote-session auto-reconnect. The TmuxManager watcher detects a // dead remote pane and emits `remoteSessionDropped`; the session owner (here) @@ -2455,6 +2460,36 @@ export class WebServer extends EventEmitter { this.sse.broadcastSessionStateDebounced(sessionId); } + /** + * Fold the latest pane readings into the sessions they belong to + * (Ark0N/Codeman#446). A reading that changes a session's answer persists the + * record and pushes a `session:updated`, which is how the tab learns; a read + * that repeats what the last one said costs nothing. + * + * The answer is pulled per session from the mux rather than taken off a + * broadcast payload. The mux reports the RAW pane reading, which for a remote + * or docker session is the death of an ssh client or a `docker exec` rather + * than of the agent, so it must not travel to a browser at all; + * `Session.setPaneExit()` is where that scoping is applied. + * + * Nothing here touches `status` or `pid`. `status: 'error'` belongs to the + * PTY-exit breaker and makes the browser offer a restart, and a null `pid` is + * what makes the browser re-attach and launch a fresh CLI. + */ + private applyPaneExits(): void { + const getPaneExit = this.mux.getPaneExit?.bind(this.mux); + if (!getPaneExit) return; + for (const session of this.sessions.values()) { + const muxName = session.muxName; + // No pane, so nothing to report — and `setPaneExit()` would force UNKNOWN + // for such a session anyway. + if (!muxName) continue; + if (!session.setPaneExit(getPaneExit(muxName))) continue; + this.persistSessionState(session); + this.broadcastSessionStateDebounced(session.id); + } + } + // ========== Web Push ========== /** Map SSE event names to push notification payloads */ @@ -3336,6 +3371,14 @@ export class WebServer extends EventEmitter { // the conversation the CLI was on when the server stopped, which is // what a re-attach must point the viewer at instead of the launch id. claudeSessionChain: savedState?.claudeSessionChain, + // What the previous run last observed of this pane's agent. Carried + // over so the first persist after boot does not blank a record that + // says the agent exited; the attach below drops it, and the stats + // tick replaces it with a first-hand reading. + paneExit: savedState?.paneExit, + // A record rebuilt from the socket has no provenance, so its + // apparent locality is a guess (see `MuxSession.discovered`). + discoveredMuxSession: muxSession.discovered, // The pane's last output, previous run's value. Without it every // restart restamped all sessions "now" (constructor + the attach // repaint within the same second), flattening the home screens' @@ -3545,6 +3588,15 @@ export class WebServer extends EventEmitter { (this.mux as { startMouseModeSync: (ms?: number) => void }).startMouseModeSync(); } + // Ark0N/Codeman#446 — poll every pane for an exited agent. Always start, + // even with no sessions, for the same reason as the two watchers around + // it: sessions arrive later. Deliberately NOT folded into the stats + // collector above, which the browser arms and disarms with the Monitor + // panel and which boot skips entirely when nothing was recovered. + if ('startPaneExitWatcher' in this.mux) { + (this.mux as { startPaneExitWatcher: (ms?: number) => void }).startPaneExitWatcher(); + } + // COD-108 — start the remote-session auto-reconnect watcher (tmux only). // Always-on (D3) with a `remoteAutoReconnect` kill-switch the watcher reads // each tick. Start even with no sessions — remote sessions may arrive later. diff --git a/test/session-pane-exit.test.ts b/test/session-pane-exit.test.ts new file mode 100644 index 00000000..bed3ebf5 --- /dev/null +++ b/test/session-pane-exit.test.ts @@ -0,0 +1,312 @@ +/** + * @fileoverview `SessionState.paneExit` — Codeman noticing that a pane's agent + * has exited (Ark0N/Codeman#446). + * + * Codeman creates every tmux pane with `remain-on-exit on`, so `/exit` ends the + * CLI while tmux keeps the pane and the `tmux attach-session` process Codeman + * records as the session's pid. No PTY exit handler runs, and the record used to + * keep both its pid and `status: 'idle'`, so the board showed an exited session + * as a live idle one. + * + * Four properties are pinned here, each because getting it wrong costs something + * specific: + * + * 1. **The field is tri-state, and absence means UNKNOWN.** A direct-PTY + * session owns no pane, a remote SSH session's local pane holds the ssh + * client, and a docker case's local pane holds a `docker exec`. In all + * three, a dead local pane is not the agent exiting. + * 2. **`status` and `pid` are never touched.** `status: 'error'` belongs to the + * PTY-exit circuit breaker and makes the browser offer a restart, and a null + * `pid` is what makes the browser re-attach and launch a fresh CLI. + * 3. **It reaches `toState()`**, which is both the `session:updated` payload + * and what `state.json` persists. + * 4. **It round-trips through the store**, because a reboot takes the tmux + * server and the persisted record is the only thing left that can say the + * agent was already gone. + * + * Port: 3187 + */ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { Session } from '../src/session.js'; +import { WebServer } from '../src/web/server.js'; +import { StateStore } from '../src/state-store.js'; +import type { PaneExit, SessionRemote, SessionDocker, SessionState } from '../src/types.js'; +import type { MuxSession, TerminalMultiplexer } from '../src/mux-interface.js'; + +const PORT = 3187; + +const EXIT: PaneExit = { status: 0, at: 1_700_000_000_000 }; + +/** The two mux members `Session` reads when it decides whether the field applies. */ +const stubMux = () => ({ isAvailable: () => true }) as unknown as TerminalMultiplexer; + +const stubMuxSession = (muxName = 'codeman-aaaa') => ({ muxName, sessionId: 'aaaa' }) as unknown as MuxSession; + +const remote: SessionRemote = { hostId: 'h1', label: 'box', host: 'box', user: 'dev' } as SessionRemote; + +const docker: SessionDocker = { hostId: 'd1', label: 'ctr', containerName: 'ctr' } as SessionDocker; + +/** A local, mux-backed session: the one shape the field applies to. */ +function localMuxSession(extra: Record = {}) { + return new Session({ + workingDir: '/tmp', + mode: 'claude', + useMux: true, + mux: stubMux(), + muxSession: stubMuxSession(), + ...extra, + }); +} + +describe('Session.setPaneExit scoping', () => { + it('accepts an exit for a local mux-backed session', () => { + const session = localMuxSession(); + expect(session.setPaneExit(EXIT)).toBe(true); + expect(session.paneExit).toEqual(EXIT); + }); + + it('stays unknown for a direct-PTY session, which owns no pane at all', () => { + const session = new Session({ workingDir: '/tmp', mode: 'claude', useMux: false }); + expect(session.setPaneExit(EXIT)).toBe(false); + expect(session.paneExit).toBeUndefined(); + }); + + it('stays unknown while the session has no mux session yet', () => { + const session = new Session({ workingDir: '/tmp', mode: 'claude', useMux: true, mux: stubMux() }); + expect(session.setPaneExit(EXIT)).toBe(false); + expect(session.paneExit).toBeUndefined(); + }); + + it('stays unknown for a remote SSH session, whose pane holds the ssh client', () => { + // A dead ssh client means a transport drop OR an exit, and telling those two + // apart is the whole of PR #355. Publishing it as an agent exit would assert + // the answer Codeman does not have. + const session = localMuxSession({ remote }); + expect(session.setPaneExit(EXIT)).toBe(false); + expect(session.paneExit).toBeUndefined(); + }); + + it('stays unknown for a docker case, whose pane holds a docker exec', () => { + const session = localMuxSession({ docker }); + expect(session.setPaneExit(EXIT)).toBe(false); + expect(session.paneExit).toBeUndefined(); + }); + + it('clears a stored exit when a later tick reports nothing', () => { + const session = localMuxSession(); + session.setPaneExit(EXIT); + expect(session.setPaneExit(undefined)).toBe(true); + expect(session.paneExit).toBeUndefined(); + }); + + it('reports no change when a tick repeats the same observation', () => { + // The caller persists and broadcasts on a true, and this tick runs every + // 2000 ms for every session. + const session = localMuxSession(); + session.setPaneExit(EXIT); + expect(session.setPaneExit({ ...EXIT })).toBe(false); + }); + + it('reports a change when the exit status changes', () => { + const session = localMuxSession(); + session.setPaneExit(EXIT); + expect(session.setPaneExit({ status: 137, at: EXIT.at })).toBe(true); + expect(session.paneExit).toEqual({ status: 137, at: EXIT.at }); + }); +}); + +describe('Session.toState with an exited agent', () => { + it('publishes the exit and leaves status and pid alone', () => { + const session = localMuxSession(); + const before = session.toState(); + session.setPaneExit(EXIT); + const after = session.toState(); + + expect(before.paneExit).toBeUndefined(); + expect(after.paneExit).toEqual(EXIT); + expect(after.status).toBe(before.status); + expect(after.pid).toBe(before.pid); + // `status: 'error'` is the PTY-exit breaker's value; the browser answers it + // with a "restart it?" confirm. + expect(after.status).not.toBe('error'); + }); + + it('restores a persisted exit so the first persist after boot cannot blank it', () => { + const session = localMuxSession({ paneExit: EXIT }); + expect(session.toState().paneExit).toEqual(EXIT); + }); + + it('ignores a persisted exit for a session shape the field never applies to', () => { + // The scoping is not only about live ticks: a record written before a + // session was reconfigured must not resurrect an answer that cannot hold. + // The constructor alone has to enforce it, before any tick runs. + expect(new Session({ workingDir: '/tmp', mode: 'claude', useMux: false, paneExit: EXIT }).paneExit).toBeUndefined(); + expect(localMuxSession({ remote, paneExit: EXIT }).paneExit).toBeUndefined(); + expect(localMuxSession({ docker, paneExit: EXIT }).paneExit).toBeUndefined(); + }); +}); + +describe('a relaunch in the same pane forgetting the old exit', () => { + /** A mux whose respawn succeeds, so `restartCli()` reaches its success path. */ + const respawningMux = () => { + const cleared: string[] = []; + const mux = { + isAvailable: () => true, + muxSessionExists: () => true, + respawnPane: async () => 4242, + clearPaneExit: (muxName: string) => cleared.push(muxName), + }; + return { mux: mux as unknown as TerminalMultiplexer, cleared }; + }; + + it('clears the exit when restartCli relaunches the CLI', async () => { + // restartCli() is the custom-model endpoint switch. Its caller persists and + // broadcasts straight afterwards, so an exit left in place here is written + // back onto a session that is running again. + const { mux, cleared } = respawningMux(); + const session = new Session({ + workingDir: '/tmp', + mode: 'claude', + useMux: true, + mux, + muxSession: stubMuxSession(), + paneExit: EXIT, + }); + expect(session.paneExit).toEqual(EXIT); + + expect(await session.restartCli()).toBe(true); + + expect(session.paneExit).toBeUndefined(); + expect(session.toState().paneExit).toBeUndefined(); + // Both halves: the record and the mux layer's cache, the latter of which + // also invalidates a pane read already in flight. + expect(cleared).toEqual(['codeman-aaaa']); + }); + + it('leaves the exit alone when the relaunch fails', async () => { + // A failed respawn means the old command is still what the pane last ran. + const { mux, cleared } = respawningMux(); + (mux as unknown as { respawnPane: () => Promise }).respawnPane = async () => null; + const session = new Session({ + workingDir: '/tmp', + mode: 'claude', + useMux: true, + mux, + muxSession: stubMuxSession(), + paneExit: EXIT, + }); + + expect(await session.restartCli()).toBe(false); + + expect(session.paneExit).toEqual(EXIT); + expect(cleared).toEqual([]); + }); +}); + +describe('paneExit round trip through state.json', () => { + let dir: string | null = null; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + it('survives a write and a reload, which is what a reboot restore reads', () => { + dir = mkdtempSync(join(tmpdir(), 'codeman-pane-exit-')); + const file = join(dir, 'state.json'); + const stored: SessionState = { + ...localMuxSession().toState(), + paneExit: { status: 0, signal: undefined, at: 1_700_000_000_000 }, + }; + + const writer = new StateStore(file); + writer.setSession(stored.id, stored); + writer.saveNow(); + + const reader = new StateStore(file); + expect(reader.getSession(stored.id)?.paneExit).toEqual({ at: 1_700_000_000_000, status: 0 }); + }); + + it('reads a record written before the field existed as unknown, needing no migration', () => { + dir = mkdtempSync(join(tmpdir(), 'codeman-pane-exit-')); + const file = join(dir, 'state.json'); + const legacy = localMuxSession().toState(); + delete legacy.paneExit; + + const writer = new StateStore(file); + writer.setSession(legacy.id, legacy); + writer.saveNow(); + + expect(new StateStore(file).getSession(legacy.id)?.paneExit).toBeUndefined(); + }); +}); + +describe('a pane read reaching the session record', () => { + // Drives the real `paneExitsUpdated` wiring rather than asserting on source + // text: a fake reading goes into the mux, the event fires, and the session + // record is checked. This is the path findings about the watcher turn on. + let server: WebServer | null = null; + + afterEach(async () => { + await server?.stop?.(); + server = null; + }); + + const build = () => { + const web = new WebServer(PORT, false, true); + const mux = (web as unknown as { mux: Record }).mux; + const session = localMuxSession(); + (web as unknown as { sessions: Map }).sessions.set(session.id, session); + return { web, mux, session }; + }; + + it('publishes an exit the mux reports for a local pane', () => { + const { web, mux, session } = build(); + server = web; + mux.getPaneExit = () => EXIT; + (mux as unknown as { emit: (e: string) => void }).emit('paneExitsUpdated'); + + expect(session.paneExit).toEqual(EXIT); + expect(session.toState().paneExit).toEqual(EXIT); + }); + + it('leaves status and pid alone while doing it', () => { + const { web, mux, session } = build(); + server = web; + const before = session.toState(); + mux.getPaneExit = () => EXIT; + (mux as unknown as { emit: (e: string) => void }).emit('paneExitsUpdated'); + + const after = session.toState(); + expect(after.status).toBe(before.status); + expect(after.pid).toBe(before.pid); + // `status: 'error'` is the PTY-exit breaker's value; it makes the browser + // offer a restart. A null pid makes it launch a fresh CLI. + expect(after.status).not.toBe('error'); + }); + + it('retracts the exit once the mux reports the pane is back', () => { + const { web, mux, session } = build(); + server = web; + mux.getPaneExit = () => EXIT; + (mux as unknown as { emit: (e: string) => void }).emit('paneExitsUpdated'); + mux.getPaneExit = () => undefined; + (mux as unknown as { emit: (e: string) => void }).emit('paneExitsUpdated'); + + expect(session.paneExit).toBeUndefined(); + }); + + it('publishes nothing for a session the field does not apply to', () => { + const { web, mux } = build(); + server = web; + const remoteSession = localMuxSession({ remote }); + (web as unknown as { sessions: Map }).sessions.set(remoteSession.id, remoteSession); + mux.getPaneExit = () => EXIT; + (mux as unknown as { emit: (e: string) => void }).emit('paneExitsUpdated'); + + expect(remoteSession.paneExit).toBeUndefined(); + }); +}); From c67c130caa6b6962932b0d46704b0778356274d7 Mon Sep 17 00:00:00 2001 From: Michael Grundberg Date: Mon, 21 Sep 2026 10:01:22 +0200 Subject: [PATCH 3/5] feat(web): mark a session tab whose agent has exited MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tab now reads "exited (137)" beside the session name, drawn from the `paneExit` field the server publishes. `applyPaneExitBadge()` owns the DOM work, called from the incremental render path — the only path a live session ever takes, since going from live to exited adds and removes no tab and so never reaches the full rebuild. An unknown answer draws nothing. A death tmux could not explain reads "exited" with no number rather than "exited (0)", so an unexplained death and a clean exit do not look alike. A signal death reads "exited (signal 9)". The badge carries `data-i18n-skip`, like the status pills: it is generated text, `i18n.js` walks inserted content, and a dictionary entry added later would fight the renderer, whose in-place comparison is against English. The tab also carries a `tab-agent-exited` class that mutes the status dot. That dot is drawn from `status`, which stays `idle` or `busy` for an exited pane as the issue requires, so without this a green or pulsing dot sits beside a badge saying the agent is gone — the first thing a tester asked about. `status` itself is untouched, so this is a rendering rule only. The CSS excludes the two alert classes by hand, following the convention the rich-rail dot rules document: a dot turning red or yellow because a session is blocked on a human outranks "the agent exited". The tab keeps its click behavior. X still closes it, and nothing here closes, sweeps or restarts anything. `docs/architecture-invariants.md` gains the mechanism under "Session data and lifecycle", where every comparable one already lives: what the tri-state means, the four shapes it is absent for, why the watcher cannot ride the stats collector, why an absent `#{pane_dead_status}` is not 0, and the three things that must never happen to an exited pane. Refs Ark0N/Codeman#446. Co-Authored-By: Claude Opus 5 (1M context) --- docs/architecture-invariants.md | 4 + src/web/public/app.js | 62 ++++++++++++- src/web/public/styles.css | 40 ++++++++ test/session-pane-exit-ui.test.ts | 147 ++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 test/session-pane-exit-ui.test.ts diff --git a/docs/architecture-invariants.md b/docs/architecture-invariants.md index b024875a..66d19f89 100644 --- a/docs/architecture-invariants.md +++ b/docs/architecture-invariants.md @@ -162,6 +162,10 @@ Tests: `test/docker-hosts.test.ts`, `test/docker-exec-options.test.ts`, `test/do **Circuit breaker**: Prevents respawn thrashing. States: `CLOSED` → `HALF_OPEN` → `OPEN`. Reset: `/api/sessions/:id/ralph-circuit-breaker/reset`. **Distinct: PTY-exit breaker** (COD-115/118/#147, `session-pty-exit-breaker.ts`) trips after repeated rapid PTY exits (crash loops on attach), blocks further auto-restarts, broadcasts SSE `session:respawnBreakerTripped` + push (in `PUSH_EVENT_MAP`). Reset ONLY via an explicit `{clearBreaker:true}` body on `POST /api/sessions/:id/interactive` (sent by the user-facing restart control) — the frontend's auto-reattach in `selectSession()` sends no body and must never clear it. Sessions also scrub inherited `TMUX`/`TMUX_PANE` env so Codeman-in-tmux doesn't nest. Tests: `test/respawn-pty-breaker.test.ts`. +### An exited agent in a live pane (`paneExit`) + +**Codeman creates every pane with `remain-on-exit on`, so a session whose agent exited still looks alive.** `/exit` ends the CLI, tmux keeps the pane and the tmux session, and the `tmux attach-session` process Codeman records as `Session.pid` runs on, so no PTY exit handler fires and the record keeps its pid and `status: 'idle'` (Ark0N/Codeman#446). `SessionState.paneExit` (`{status?, signal?, at}`) is the fact tmux already knows, published through `toState()` so it rides `session:updated` and lands in `state.json` on the same persist — there is no SSE event for it. One batched `tmux list-panes -a` per tick fills it, from `TmuxManager.startPaneExitWatcher()`, which has its OWN always-on interval: the stats collector cannot carry it, because the browser arms and disarms that one with the Monitor panel (`panels-ui.js`) and boot skips it entirely when no session was recovered. ⚠️ **The field is TRI-STATE and its third state is absence**, meaning UNKNOWN, which renders as nothing and must NEVER read as alive; it covers a running pane, a session the read did not list, a failed probe, and every session shape a dead local pane does not describe. `Session.paneExitApplies` is the single place that scoping lives, and it fails closed for four shapes: a direct-PTY session (no pane), a remote SSH session (the local pane is the ssh client, whose death is a transport drop OR an exit — the whole of #355), a docker case (the local pane is a `docker exec` into the container's own tmux), and a session rebuilt from the socket (`MuxSession.discovered`: its synthetic `restored-` id matches no `state.json` entry, so a remote session rediscovered after `mux-sessions.json` was lost would arrive looking local). ⚠️ **Never set `status: 'error'`** for an exited pane — that value is the PTY-exit breaker's and the browser answers it with a "restart it?" confirm — and **never null the `pid`**, which is what makes `selectSession()` re-attach and launch a fresh CLI. Local panes keep `remain-on-exit on`; flipping them to `failed` ends the tmux session, nulls the pid and reintroduces the auto-revive #355 removed. ⚠️ **An absent `#{pane_dead_status}` is not 0**: measured on tmux 3.2a a SIGKILLed pane reports neither a status nor a signal (`#{pane_dead_signal}` did not exist before tmux 3.4), so folding it into 0 would turn an unexplained death into a clean exit. A session answers only when the read listed EXACTLY ONE pane for it, since Codeman never splits a pane and a session the user split by hand has none that speaks for the agent. The three synchronous `isPaneDead()` callers (the `/wait` route, the TUI, the attach path) keep their own probes — this watcher is never fresh enough for them. Tests: `test/session-pane-exit.test.ts`, `test/tmux-manager.test.ts`. + ## Features ### Attachments diff --git a/src/web/public/app.js b/src/web/public/app.js index 4b27df17..4762d978 100644 --- a/src/web/public/app.js +++ b/src/web/public/app.js @@ -325,6 +325,55 @@ function parseSessionPrefix(name) { return null; } +// ═══════════════════════════════════════════════════════════════ +// Exited-agent tab label (Ark0N/Codeman#446) +// ═══════════════════════════════════════════════════════════════ +// The server publishes session.paneExit when the agent inside a local tmux +// pane has exited while remain-on-exit kept the pane. The field is tri-state +// and its third state is absence, which means Codeman does not know — that +// renders as nothing here and must never read as alive. +// +// status and signal are each optional, because tmux can know the pane died +// without reporting how (a SIGKILLed pane on tmux 3.2a reports neither). So an +// absent status shows a bare "exited" rather than "exited (0)": a clean exit +// and an unexplained one must not look the same. +function paneExitLabel(paneExit) { + if (!paneExit || typeof paneExit !== 'object') return ''; + if (typeof paneExit.signal === 'number' && paneExit.signal > 0) return `exited (signal ${paneExit.signal})`; + if (typeof paneExit.status === 'number') return `exited (${paneExit.status})`; + return 'exited'; +} + +// Add, update or remove one tab's exited-agent badge in place. Separate from +// the render loop so it can be exercised directly: this is the only path a +// session going live-to-exited ever takes, since that transition adds and +// removes no tab and so never reaches the full rebuild. +function applyPaneExitBadge(tab, paneExit) { + const label = paneExitLabel(paneExit); + const existing = tab.querySelector('.tab-exited-badge'); + // Quiets the status dot too. That dot reports `status`, which stays `idle` or + // `busy` for an exited pane by design, so without this a green or pulsing dot + // sits next to a badge saying the agent is gone. + tab.classList.toggle('tab-agent-exited', !!label); + if (!label) { + existing?.remove(); + return; + } + if (!existing) { + const badge = document.createElement('span'); + badge.className = 'tab-exited-badge'; + // Generated status text, like the status pills: it carries data-i18n-skip + // rather than a dictionary entry. Without it the translator would rewrite + // the badge and the next render pass would rewrite it back, because the + // comparison below is against the English string. + badge.setAttribute('data-i18n-skip', ''); + badge.textContent = label; + tab.querySelector('.tab-name')?.insertAdjacentElement('afterend', badge); + return; + } + if (existing.textContent !== label) existing.textContent = label; +} + const DEFAULT_SHORTCUTS = [ { id: 'show-shortcuts', @@ -4968,6 +5017,11 @@ class CodemanApp { statusEl.className = `tab-status ${status}`; } + // The exited-agent badge (Ark0N/Codeman#446). A session going from live + // to exited changes no tab count, so the full rebuild below never runs + // for it and this is the only path that ever draws the badge. + applyPaneExitBadge(tab, session.paneExit); + // Rich sidebar meta ("created 3d ago · working 12m" + pill). The stamps // themselves move on _tickSidebarRichTimes(); this is here for the parts // a tick cannot see — the state flipping, and with it the pill, the row @@ -5268,10 +5322,15 @@ class CodemanApp { ? ` data-tab-state="${richRow.state}" data-tab-meta-sig="${richRow.state}:${richRow.since ? richRow.since.at : 0}:${richRow.createdAt}"` : ''; + // '' whenever the server said nothing about this pane's agent, which covers + // a running pane and every session shape the field never applies to + // (direct-PTY, remote SSH, docker). See paneExitLabel(). + const paneExitBadge = paneExitLabel(session.paneExit); + const inlineSessionActions = this.shouldInlineSessionActions(); const tabActionsHtml = `⚙⧉×`; - parts.push(`