diff --git a/src/remote-reconnect.ts b/src/remote-reconnect.ts new file mode 100644 index 00000000..6e5a709a --- /dev/null +++ b/src/remote-reconnect.ts @@ -0,0 +1,184 @@ +/** + * @fileoverview Pure logic for the remote-session auto-reconnect watcher (COD-108). + * + * COD-104 made remote tmux sessions durable + idempotently reattachable, but a + * reconnect only fired at explicit trigger points. COD-108 adds a continuous + * watcher (in `TmuxManager`) that detects a dead remote pane and emits + * `remoteSessionDropped`; `SessionManager`/server then reassembles the respawn + * options and reattaches (re-running the idempotent remote command). + * + * This module holds the SIDE-EFFECT-FREE pieces so they can be unit-tested + * without real tmux: + * - the bounded exponential **backoff schedule** (attempt → delay, capped), + * - the per-session **reconnect state** shape, + * - the **eligibility decision** (`decideReconnect`) given a session + its + * reconnect state + the current time + the guard set. + * + * The watcher in `tmux-manager.ts` owns the live `isPaneDead` probe and the + * timers; everything here is pure and deterministic (time is injected). + * + * @module remote-reconnect + */ + +/** + * Bounded exponential backoff delays (ms) between reconnect attempts. + * Attempt N (1-based) waits `BACKOFF_SCHEDULE_MS[N-1]` from the previous emit + * before the next emit is eligible. After the last entry the session is + * considered `reconnect-exhausted` and the watcher stops emitting for it. + * + * 5s, 15s, 45s, 2m, 5m, 5m → ~6 attempts spanning ~13 minutes. + */ +export const BACKOFF_SCHEDULE_MS: readonly number[] = [5_000, 15_000, 45_000, 120_000, 300_000, 300_000]; + +/** Maximum number of reconnect attempts before exhaustion. */ +export const MAX_RECONNECT_ATTEMPTS = BACKOFF_SCHEDULE_MS.length; + +/** + * Delay (ms) to wait AFTER emitting attempt `attempt` (1-based) before the next + * attempt is eligible. `attempt <= 0` returns the first delay; an attempt at or + * beyond the cap returns the last delay (callers should check exhaustion via + * {@link isExhausted} rather than relying on this for the stop decision). + * + * Pure — no clock, no I/O. + */ +export function reconnectDelayForAttempt(attempt: number): number { + if (!Number.isFinite(attempt) || attempt <= 1) return BACKOFF_SCHEDULE_MS[0]; + const idx = Math.min(Math.floor(attempt) - 1, BACKOFF_SCHEDULE_MS.length - 1); + return BACKOFF_SCHEDULE_MS[idx]; +} + +/** Whether `attempts` reconnect emits have reached/exceeded the cap. Pure. */ +export function isExhausted(attempts: number): boolean { + return attempts >= MAX_RECONNECT_ATTEMPTS; +} + +/** + * Per-session reconnect bookkeeping held by the watcher. All time values are + * epoch ms. `inFlight` guards against stacking respawns when a tick fires while + * a previous reattach is still running. `exhaustedEmitted` ensures the + * `remoteReconnectExhausted` event fires at most once per session. + */ +export interface RemoteReconnectState { + /** Number of `remoteSessionDropped` emits so far (advances per emit). */ + attempts: number; + /** Earliest time (epoch ms) the next emit is eligible. 0 = eligible now. */ + nextEligibleAt: number; + /** A reattach triggered by a prior emit is currently running. */ + inFlight: boolean; + /** Cap reached — stop auto-retrying for this session. */ + exhausted: boolean; + /** The `remoteReconnectExhausted` SSE event has already been emitted. */ + exhaustedEmitted: boolean; +} + +/** A fresh reconnect state (no attempts, immediately eligible). Pure. */ +export function freshReconnectState(): RemoteReconnectState { + return { attempts: 0, nextEligibleAt: 0, inFlight: false, exhausted: false, exhaustedEmitted: false }; +} + +/** + * Advance the backoff after an emit at time `now`. Increments `attempts` and + * schedules `nextEligibleAt = now + delay`. Returns a NEW state object (does + * not mutate the input). Pure. + * + * NOTE: this does NOT set `exhausted`. Exhaustion is a decision the watcher + * makes on the FOLLOWING tick (via {@link decideReconnect} → `exhaust`), so the + * `remoteReconnectExhausted` event fires exactly once after the final attempt's + * backoff window elapses — not pre-emptively on the last emit. + */ +export function advanceBackoff(state: RemoteReconnectState, now: number): RemoteReconnectState { + const attempts = state.attempts + 1; + const delay = reconnectDelayForAttempt(attempts); + return { + ...state, + attempts, + nextEligibleAt: now + delay, + }; +} + +/** Reset after a successful reattach — back to a fresh, eligible state. Pure. */ +export function resetReconnectState(): RemoteReconnectState { + return freshReconnectState(); +} + +/** Minimal session view the decision needs (avoids importing MuxSession here). */ +export interface ReconnectSessionView { + sessionId: string; + /** Truthy when this is a remote (SSH-wrapped) session. */ + isRemote: boolean; + /** Result of `isPaneDead(muxName)` for this session. */ + paneDead: boolean; +} + +/** + * Decision outcomes for a single watcher tick on one session. + * - `emit` → emit `remoteSessionDropped { sessionId, attempt }`, then + * advance backoff (attempt = the returned `attempt`). + * - `exhaust` → cap reached this tick; emit `remoteReconnectExhausted` once. + * - `skip` → do nothing (not remote / pane alive / guarded / in-flight / + * not yet due / already exhausted). + */ +export type ReconnectAction = + | { kind: 'emit'; attempt: number } + | { kind: 'exhaust' } + | { kind: 'skip'; reason: ReconnectSkipReason }; + +export type ReconnectSkipReason = + | 'not-remote' + | 'pane-alive' + | 'guarded' + | 'in-flight' + | 'not-due' + | 'exhausted' + | 'disabled'; + +export interface DecideReconnectInput { + session: ReconnectSessionView; + state: RemoteReconnectState | undefined; + /** Session is in the intentional-teardown guard set (killed/detached/stopping). */ + guarded: boolean; + /** Kill-switch: `remoteAutoReconnect` setting. When false, never reconnect. */ + enabled: boolean; + now: number; +} + +/** + * PURE eligibility decision for one session on one tick. No clock, no I/O — all + * inputs are passed in. The watcher translates the result into emits + state + * transitions. + * + * Order of guards (most-decisive first): + * 1. kill-switch off → skip:disabled + * 2. not a remote session → skip:not-remote + * 3. pane is alive → skip:pane-alive + * 4. intentional teardown guard → skip:guarded (NEVER revive a killed tab) + * 5. a reattach already running → skip:in-flight (no stacked respawns) + * 6. already exhausted → skip:exhausted (one exhaust emit, then quiet) + * 7. cap reached this tick → exhaust + * 8. not yet due (backoff) → skip:not-due + * 9. otherwise → emit (attempt = attempts + 1) + */ +export function decideReconnect(input: DecideReconnectInput): ReconnectAction { + const { session, state, guarded, enabled, now } = input; + + if (!enabled) return { kind: 'skip', reason: 'disabled' }; + if (!session.isRemote) return { kind: 'skip', reason: 'not-remote' }; + if (!session.paneDead) return { kind: 'skip', reason: 'pane-alive' }; + // Intentional kill / detach must NEVER be auto-revived. + if (guarded) return { kind: 'skip', reason: 'guarded' }; + + const s = state ?? freshReconnectState(); + + // Only one reconnect in flight per session — don't stack respawns. + if (s.inFlight) return { kind: 'skip', reason: 'in-flight' }; + + if (s.exhausted) return { kind: 'skip', reason: 'exhausted' }; + + // Cap reached: surface exhaustion once, then go quiet. + if (isExhausted(s.attempts)) return { kind: 'exhaust' }; + + // Backoff gate — only emit when due. + if (now < s.nextEligibleAt) return { kind: 'skip', reason: 'not-due' }; + + return { kind: 'emit', attempt: s.attempts + 1 }; +} diff --git a/src/session.ts b/src/session.ts index a509040a..3f344fc9 100644 --- a/src/session.ts +++ b/src/session.ts @@ -1211,6 +1211,68 @@ export class Session extends EventEmitter { return { isRestored }; } + /** + * COD-108 — re-establish a dropped REMOTE session. Triggered by the + * `TmuxManager` remote-reconnect watcher (via `remoteSessionDropped`): the + * watcher detects a dead remote pane, the session owner reassembles the SAME + * `RespawnPaneOptions` used for Claude-idle respawns and calls + * `respawnPane()` directly. For a remote session that re-runs + * `buildRemoteSessionCommand` (owned → `new-session -A`, non-owned → + * `attach`), which idempotently REATTACHES the still-running durable remote + * tmux session — scrollback + agent intact (proven COD-104/105). + * + * Deliberately does NOT route through the Claude-idle respawn-controller — + * this is a transport re-establish, not a `/clear`/`/compact` cycle. + * + * @returns true if the pane was respawned (reattach issued), false otherwise. + */ + async reattachRemote(): Promise { + if (!this._remote) return false; // not a remote session + if (!this._useMux || !this._mux || !this._muxSession) return false; + const mux = this._mux; + + // If tmux lost the whole session (not just a dead pane), there is nothing to + // respawn into — a genuine death, leave it for normal recovery/reconcile. + if (!mux.muxSessionExists(this._muxSession.muxName)) { + console.log('[Session] reattachRemote: mux session gone, skipping:', this._muxSession.muxName); + return false; + } + + const newPid = await mux.respawnPane(this._buildRespawnPaneOptions()); + if (!newPid) { + console.error('[Session] reattachRemote: respawnPane failed for', this._muxSession.muxName); + return false; + } + console.log('[Session] reattachRemote: reattached remote session', this._muxSession.muxName, 'pid', newPid); + return true; + } + + /** + * Assemble the {@link RespawnPaneOptions} for this session. Single source of + * truth shared by interactive start, shell start (via their inline copies), + * and {@link reattachRemote} so the remote reattach path can never drift from + * the spawn path. + */ + private _buildRespawnPaneOptions(): import('./mux-interface.js').RespawnPaneOptions { + return { + sessionId: this.id, + workingDir: this.workingDir, + mode: this.mode, + niceConfig: this._niceConfig, + model: this._model, + claudeMode: this._claudeMode, + allowedTools: this._allowedTools, + openCodeConfig: this._openCodeConfig, + codexConfig: this._codexConfig, + geminiConfig: this._geminiConfig, + resumeSessionId: this._resumeSessionId, + envOverrides: this._envOverrides, + effort: this._effort, + historyLimit: this._tmuxHistoryLimit, + remote: this._remote, + }; + } + private _handleTerminalOutput(data: string): void { // Codex AND Claude Code emit sequences that wipe xterm.js scrollback, plus // mouse-tracking enables that hijack the scroll wheel so the user can't reach @@ -1334,23 +1396,8 @@ export class Session extends EventEmitter { if (this._useMux && this._mux) { try { const { isRestored } = await this._setupOrAttachMuxSession({ - respawnPaneOptions: { - sessionId: this.id, - workingDir: this.workingDir, - mode: this.mode, - niceConfig: this._niceConfig, - model: this._model, - claudeMode: this._claudeMode, - allowedTools: this._allowedTools, - openCodeConfig: this._openCodeConfig, - codexConfig: this._codexConfig, - geminiConfig: this._geminiConfig, - resumeSessionId: this._resumeSessionId, - envOverrides: this._envOverrides, - effort: this._effort, - historyLimit: this._tmuxHistoryLimit, - remote: this._remote, - }, + // Single source of truth shared with reattachRemote() (COD-108). + respawnPaneOptions: this._buildRespawnPaneOptions(), createSessionOptions: { sessionId: this.id, workingDir: this.workingDir, diff --git a/src/tmux-manager.ts b/src/tmux-manager.ts index 298e212f..d4951474 100644 --- a/src/tmux-manager.ts +++ b/src/tmux-manager.ts @@ -62,6 +62,13 @@ import type { RespawnPaneOptions, PaneCaptureOptions, } from './mux-interface.js'; +import { + decideReconnect, + advanceBackoff, + freshReconnectState, + resetReconnectState, + type RemoteReconnectState, +} from './remote-reconnect.js'; // ============================================================================ // Timing Constants @@ -94,6 +101,9 @@ const GRACEFUL_SHUTDOWN_WAIT_MS = 100; /** Default stats collection interval (2 seconds) */ const DEFAULT_STATS_INTERVAL_MS = 2000; +/** Default remote-reconnect watcher poll interval (5 seconds) — COD-108 */ +const DEFAULT_REMOTE_RECONNECT_INTERVAL_MS = 5000; + /** Stable cwd for tmux server/pane launch; actual session cwd is reached inside the pane. */ const TMUX_LAUNCH_CWD = '/tmp'; @@ -118,6 +128,20 @@ const IS_TEST_MODE = !!process.env.VITEST; /** Path to persisted mux session metadata */ const MUX_SESSIONS_FILE = dataPath('mux-sessions.json'); +/** + * COD-108 kill-switch: `remoteAutoReconnect` app setting (default ON). Read at + * call time (like headroom routing) so a settings change takes effect without a + * restart. Absent/non-boolean ⇒ true (feature on). + */ +function isRemoteAutoReconnectEnabled(): boolean { + try { + const s = JSON.parse(readFileSync(dataPath('settings.json'), 'utf8')) as Record; + return typeof s.remoteAutoReconnect === 'boolean' ? s.remoteAutoReconnect : true; + } catch { + return true; + } +} + /** Regex to validate tmux session names (only allow safe characters) */ const SAFE_MUX_NAME_PATTERN = /^codeman-[a-f0-9-]+$/; @@ -1008,6 +1032,17 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { /** Track last-known pane count per session to avoid unnecessary tmux set-option calls */ private lastPaneCount: Map = new Map(); + // ── COD-108 remote-reconnect watcher state ──────────────────────────────── + /** Periodic watcher that re-establishes dropped remote sessions. */ + private remoteReconnectInterval: NodeJS.Timeout | null = null; + /** Per-session backoff/attempt bookkeeping (sessionId → state). */ + private reconnectState: Map = new Map(); + /** + * Sessions excluded from auto-reconnect because they are being intentionally + * torn down (killed/detached/stopping). A guarded session is NEVER revived. + */ + private reconnectGuard: Set = new Set(); + private trueColorConfigured = false; constructor() { @@ -1679,9 +1714,16 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { return false; } + // COD-108: an intentional kill/detach must NEVER be auto-revived by the + // remote-reconnect watcher. Guard BEFORE any teardown so a tick that fires + // mid-kill (especially the non-owned DETACH early-return below, where the + // dead local pane would otherwise look reconnectable) sees the guard. + this.guardRemoteReconnect(sessionId); + // TEST MODE: Remove from memory only — NEVER touch real tmux sessions if (IS_TEST_MODE) { this.sessions.delete(sessionId); + this.clearRemoteReconnectState(sessionId); this.emit('sessionKilled', { sessionId }); return true; } @@ -1721,6 +1763,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { } this.lastPaneCount.delete(session.muxName); this.sessions.delete(sessionId); + this.clearRemoteReconnectState(sessionId); this.saveSessions(); this.emit('sessionKilled', { sessionId }); return true; @@ -1818,6 +1861,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { this.lastPaneCount.delete(session.muxName); this.sessions.delete(sessionId); + this.clearRemoteReconnectState(sessionId); this.saveSessions(); this.emit('sessionKilled', { sessionId }); @@ -1884,6 +1928,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { } else { dead.push(sessionId); this.sessions.delete(sessionId); + this.clearRemoteReconnectState(sessionId); this.emit('sessionDied', { sessionId }); } } @@ -2155,9 +2200,118 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { this.lastPaneCount.clear(); } + // ── COD-108 remote-session auto-reconnect watcher ───────────────────────── + + /** + * Start the remote-reconnect watcher (COD-108). Each tick, for every tracked + * session with `session.remote` whose local pane is DEAD, not intentionally + * guarded, and within its backoff budget, emit `remoteSessionDropped` so the + * session owner reattaches (re-running the idempotent remote command rejoins + * the durable remote tmux session). After the attempt cap, emit + * `remoteReconnectExhausted` once and go quiet. + * + * No-op tick body under `IS_TEST_MODE` (mirrors `startMouseModeSync`): tests + * drive the logic deterministically via {@link runRemoteReconnectTick}. + */ + startRemoteReconnectWatcher(intervalMs: number = DEFAULT_REMOTE_RECONNECT_INTERVAL_MS): void { + if (this.remoteReconnectInterval) { + clearInterval(this.remoteReconnectInterval); + } + this.remoteReconnectInterval = setInterval(() => { + if (IS_TEST_MODE) return; + try { + this.runRemoteReconnectTick(Date.now(), isRemoteAutoReconnectEnabled()); + } catch (err) { + console.error('[TmuxManager] Remote reconnect watcher error:', err); + } + }, intervalMs); + } + + stopRemoteReconnectWatcher(): void { + if (this.remoteReconnectInterval) { + clearInterval(this.remoteReconnectInterval); + this.remoteReconnectInterval = null; + } + } + + /** + * Run ONE watcher tick. Extracted (and given an injected `now`/`enabled`) so + * the reconnect logic is deterministically testable even though the live + * `setInterval` body no-ops under test mode. For each remote session it + * applies the pure {@link decideReconnect} decision and translates the result + * into events + backoff/state transitions. Public for tests + the watcher. + */ + runRemoteReconnectTick(now: number, enabled: boolean): void { + for (const session of this.sessions.values()) { + if (!session.remote) continue; + const sessionId = session.sessionId; + const state = this.reconnectState.get(sessionId); + const action = decideReconnect({ + session: { + sessionId, + isRemote: true, + paneDead: this.isPaneDead(session.muxName), + }, + state, + guarded: this.reconnectGuard.has(sessionId), + enabled, + now, + }); + + if (action.kind === 'emit') { + const base = state ?? freshReconnectState(); + // Mark in-flight + advance backoff BEFORE emitting so a re-entrant tick + // (or a synchronous listener) can never stack a second reconnect. + this.reconnectState.set(sessionId, { ...advanceBackoff(base, now), inFlight: true }); + this.emit('remoteSessionDropped', { sessionId, attempt: action.attempt }); + } else if (action.kind === 'exhaust') { + const base = state ?? freshReconnectState(); + if (!base.exhaustedEmitted) { + this.reconnectState.set(sessionId, { ...base, exhausted: true, exhaustedEmitted: true }); + this.emit('remoteReconnectExhausted', { sessionId }); + } + } + // 'skip' → nothing to do. + } + } + + /** + * Tell the watcher a reattach attempt for `sessionId` finished. On success, + * reset the backoff so the session is healthy again; on failure, just clear + * the in-flight flag so the next due tick can retry under the existing + * backoff schedule. Called by the session owner after `respawnPane`. + */ + noteRemoteReconnect(sessionId: string, success: boolean): void { + if (success) { + this.reconnectState.set(sessionId, resetReconnectState()); + return; + } + const state = this.reconnectState.get(sessionId); + if (state) this.reconnectState.set(sessionId, { ...state, inFlight: false }); + } + + /** + * Exclude a session from auto-reconnect (intentional teardown). Adds it to the + * guard set and drops any backoff state so a closed/killed tab — especially a + * non-owned remote DETACH — is never auto-revived. Idempotent. + */ + guardRemoteReconnect(sessionId: string): void { + this.reconnectGuard.add(sessionId); + this.reconnectState.delete(sessionId); + } + + /** Clear all per-session reconnect + guard state (e.g. when a session is removed). */ + clearRemoteReconnectState(sessionId: string): void { + this.reconnectState.delete(sessionId); + this.reconnectGuard.delete(sessionId); + } + destroy(): void { this.stopStatsCollection(); this.stopMouseModeSync(); + this.stopRemoteReconnectWatcher(); + this.reconnectState.clear(); + this.reconnectGuard.clear(); } registerSession(session: MuxSession): void { diff --git a/src/web/public/app.js b/src/web/public/app.js index c27c7f48..fb7c292c 100644 --- a/src/web/public/app.js +++ b/src/web/public/app.js @@ -216,6 +216,10 @@ const _SSE_HANDLER_MAP = [ [SSE_EVENTS.MUX_DIED, '_onMuxDied'], [SSE_EVENTS.MUX_STATS_UPDATED, '_onMuxStatsUpdated'], + // Remote auto-reconnect (COD-108) + [SSE_EVENTS.REMOTE_SESSION_RECONNECTED, '_onRemoteSessionReconnected'], + [SSE_EVENTS.REMOTE_RECONNECT_EXHAUSTED, '_onRemoteReconnectExhausted'], + // Ralph [SSE_EVENTS.SESSION_RALPH_LOOP_UPDATE, '_onRalphLoopUpdate'], [SSE_EVENTS.SESSION_RALPH_TODO_UPDATE, '_onRalphTodoUpdate'], diff --git a/src/web/public/constants.js b/src/web/public/constants.js index 1751bfe3..8d57b026 100644 --- a/src/web/public/constants.js +++ b/src/web/public/constants.js @@ -379,6 +379,11 @@ const SSE_EVENTS = { MUX_DIED: 'mux:died', MUX_STATS_UPDATED: 'mux:statsUpdated', + // Remote auto-reconnect (COD-108) + REMOTE_SESSION_DROPPED: 'remote:sessionDropped', + REMOTE_SESSION_RECONNECTED: 'remote:sessionReconnected', + REMOTE_RECONNECT_EXHAUSTED: 'remote:reconnectExhausted', + // Ralph SESSION_RALPH_LOOP_UPDATE: 'session:ralphLoopUpdate', SESSION_RALPH_TODO_UPDATE: 'session:ralphTodoUpdate', diff --git a/src/web/public/index.html b/src/web/public/index.html index 00d5dc1c..62d6ed9c 100644 --- a/src/web/public/index.html +++ b/src/web/public/index.html @@ -1471,6 +1471,14 @@ Use 1M token context window (model: opus[1m]) for all new sessions — ignored when a Claude Model is selected above +
+ + + Automatically re-establish remote (SSH) sessions when the connection drops, reattaching to the durable remote tmux session (on by default; bounded backoff) +