mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-09 08:59:40 +02:00
COD-108 auto-reconnect remote tmux sessions on SSH drop
Continuous remote-only reconnect watcher closing the COD-104 durability arc: when a remote session's local ssh pane dies mid-run, re-establish it automatically instead of leaving a dead pane until the user pokes it. Design decisions (per cod108 design doc): - D1 event->owner: TmuxManager watcher DETECTS a dead remote pane and emits `remoteSessionDropped`; the session owner (server) reassembles the same RespawnPaneOptions and calls Session.reattachRemote() -> respawnPane, which re-runs the idempotent remote command (owned new-session -A / non-owned attach) and REJOINS the still-running durable remote tmux session. The watcher never reassembles options itself, and never routes through the Claude-idle respawn-controller. - D2 bounded backoff: per-session exponential backoff [5s,15s,45s,2m,5m,5m], reset on a successful reattach, `remoteReconnectExhausted` emitted once after the cap. Pure, unit-tested schedule + eligibility decision. - D3 always-on + kill-switch: `remoteAutoReconnect` app setting (default ON), read each tick; when false the watcher does nothing. Guards: killSession() (incl. the non-owned DETACH early-return) and shutdown add the session to an intentional-teardown guard set + clear its backoff BEFORE teardown, so a closed/killed tab is never auto-revived. Exactly one reconnect in flight per session (inFlight guard prevents stacked respawns). Per-session reconnect/guard state cleared on session removal. New: src/remote-reconnect.ts (pure backoff + decideReconnect), TmuxManager startRemoteReconnectWatcher/stop + runRemoteReconnectTick + noteRemoteReconnect + guardRemoteReconnect + clearRemoteReconnectState; Session.reattachRemote() (+ extracted _buildRespawnPaneOptions, shared with interactive start); server wiring + watcher start; 3 SSE events (sse-events.ts + constants.js in sync, broadcast + app.js exhausted "Reconnect" affordance); remoteAutoReconnect schema + settings-ui toggle. Tests: test/remote-auto-reconnect.test.ts (21) - pure schedule, eligibility (guarded never reconnects, non-remote/pane-alive/not-due skip, over-cap exhaust), and manager-level integration (dead remote pane -> dropped -> backoff -> exhausted; guarded emits nothing; reset-on-success; kill-switch off; state-cleared-on-remove). Verified real-remote against aa-desktop: drop local ssh pane -> watcher emitted -> respawnPane reattached the SAME remote session (remote pane_pid unchanged 3939->3939); test session cleaned up, the real host sessions left untouched. Checks: tsc, eslint, check:frontend-syntax, check:public-assets, prettier --check, build all green; tmux-manager/session-routes/session-manager/ sse-registry-parity suites pass. (cherry picked from commit d13d58b1994eb6594fd2eadea208104d36204f9d)
This commit is contained in:
@@ -0,0 +1,184 @@
|
|||||||
|
/**
|
||||||
|
* @fileoverview Pure logic for the remote-session auto-reconnect watcher (COD-108).
|
||||||
|
*
|
||||||
|
* COD-104 made remote tmux sessions durable + idempotently reattachable, but a
|
||||||
|
* reconnect only fired at explicit trigger points. COD-108 adds a continuous
|
||||||
|
* watcher (in `TmuxManager`) that detects a dead remote pane and emits
|
||||||
|
* `remoteSessionDropped`; `SessionManager`/server then reassembles the respawn
|
||||||
|
* options and reattaches (re-running the idempotent remote command).
|
||||||
|
*
|
||||||
|
* This module holds the SIDE-EFFECT-FREE pieces so they can be unit-tested
|
||||||
|
* without real tmux:
|
||||||
|
* - the bounded exponential **backoff schedule** (attempt → delay, capped),
|
||||||
|
* - the per-session **reconnect state** shape,
|
||||||
|
* - the **eligibility decision** (`decideReconnect`) given a session + its
|
||||||
|
* reconnect state + the current time + the guard set.
|
||||||
|
*
|
||||||
|
* The watcher in `tmux-manager.ts` owns the live `isPaneDead` probe and the
|
||||||
|
* timers; everything here is pure and deterministic (time is injected).
|
||||||
|
*
|
||||||
|
* @module remote-reconnect
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bounded exponential backoff delays (ms) between reconnect attempts.
|
||||||
|
* Attempt N (1-based) waits `BACKOFF_SCHEDULE_MS[N-1]` from the previous emit
|
||||||
|
* before the next emit is eligible. After the last entry the session is
|
||||||
|
* considered `reconnect-exhausted` and the watcher stops emitting for it.
|
||||||
|
*
|
||||||
|
* 5s, 15s, 45s, 2m, 5m, 5m → ~6 attempts spanning ~13 minutes.
|
||||||
|
*/
|
||||||
|
export const BACKOFF_SCHEDULE_MS: readonly number[] = [5_000, 15_000, 45_000, 120_000, 300_000, 300_000];
|
||||||
|
|
||||||
|
/** Maximum number of reconnect attempts before exhaustion. */
|
||||||
|
export const MAX_RECONNECT_ATTEMPTS = BACKOFF_SCHEDULE_MS.length;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delay (ms) to wait AFTER emitting attempt `attempt` (1-based) before the next
|
||||||
|
* attempt is eligible. `attempt <= 0` returns the first delay; an attempt at or
|
||||||
|
* beyond the cap returns the last delay (callers should check exhaustion via
|
||||||
|
* {@link isExhausted} rather than relying on this for the stop decision).
|
||||||
|
*
|
||||||
|
* Pure — no clock, no I/O.
|
||||||
|
*/
|
||||||
|
export function reconnectDelayForAttempt(attempt: number): number {
|
||||||
|
if (!Number.isFinite(attempt) || attempt <= 1) return BACKOFF_SCHEDULE_MS[0];
|
||||||
|
const idx = Math.min(Math.floor(attempt) - 1, BACKOFF_SCHEDULE_MS.length - 1);
|
||||||
|
return BACKOFF_SCHEDULE_MS[idx];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether `attempts` reconnect emits have reached/exceeded the cap. Pure. */
|
||||||
|
export function isExhausted(attempts: number): boolean {
|
||||||
|
return attempts >= MAX_RECONNECT_ATTEMPTS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-session reconnect bookkeeping held by the watcher. All time values are
|
||||||
|
* epoch ms. `inFlight` guards against stacking respawns when a tick fires while
|
||||||
|
* a previous reattach is still running. `exhaustedEmitted` ensures the
|
||||||
|
* `remoteReconnectExhausted` event fires at most once per session.
|
||||||
|
*/
|
||||||
|
export interface RemoteReconnectState {
|
||||||
|
/** Number of `remoteSessionDropped` emits so far (advances per emit). */
|
||||||
|
attempts: number;
|
||||||
|
/** Earliest time (epoch ms) the next emit is eligible. 0 = eligible now. */
|
||||||
|
nextEligibleAt: number;
|
||||||
|
/** A reattach triggered by a prior emit is currently running. */
|
||||||
|
inFlight: boolean;
|
||||||
|
/** Cap reached — stop auto-retrying for this session. */
|
||||||
|
exhausted: boolean;
|
||||||
|
/** The `remoteReconnectExhausted` SSE event has already been emitted. */
|
||||||
|
exhaustedEmitted: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A fresh reconnect state (no attempts, immediately eligible). Pure. */
|
||||||
|
export function freshReconnectState(): RemoteReconnectState {
|
||||||
|
return { attempts: 0, nextEligibleAt: 0, inFlight: false, exhausted: false, exhaustedEmitted: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Advance the backoff after an emit at time `now`. Increments `attempts` and
|
||||||
|
* schedules `nextEligibleAt = now + delay`. Returns a NEW state object (does
|
||||||
|
* not mutate the input). Pure.
|
||||||
|
*
|
||||||
|
* NOTE: this does NOT set `exhausted`. Exhaustion is a decision the watcher
|
||||||
|
* makes on the FOLLOWING tick (via {@link decideReconnect} → `exhaust`), so the
|
||||||
|
* `remoteReconnectExhausted` event fires exactly once after the final attempt's
|
||||||
|
* backoff window elapses — not pre-emptively on the last emit.
|
||||||
|
*/
|
||||||
|
export function advanceBackoff(state: RemoteReconnectState, now: number): RemoteReconnectState {
|
||||||
|
const attempts = state.attempts + 1;
|
||||||
|
const delay = reconnectDelayForAttempt(attempts);
|
||||||
|
return {
|
||||||
|
...state,
|
||||||
|
attempts,
|
||||||
|
nextEligibleAt: now + delay,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reset after a successful reattach — back to a fresh, eligible state. Pure. */
|
||||||
|
export function resetReconnectState(): RemoteReconnectState {
|
||||||
|
return freshReconnectState();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Minimal session view the decision needs (avoids importing MuxSession here). */
|
||||||
|
export interface ReconnectSessionView {
|
||||||
|
sessionId: string;
|
||||||
|
/** Truthy when this is a remote (SSH-wrapped) session. */
|
||||||
|
isRemote: boolean;
|
||||||
|
/** Result of `isPaneDead(muxName)` for this session. */
|
||||||
|
paneDead: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decision outcomes for a single watcher tick on one session.
|
||||||
|
* - `emit` → emit `remoteSessionDropped { sessionId, attempt }`, then
|
||||||
|
* advance backoff (attempt = the returned `attempt`).
|
||||||
|
* - `exhaust` → cap reached this tick; emit `remoteReconnectExhausted` once.
|
||||||
|
* - `skip` → do nothing (not remote / pane alive / guarded / in-flight /
|
||||||
|
* not yet due / already exhausted).
|
||||||
|
*/
|
||||||
|
export type ReconnectAction =
|
||||||
|
| { kind: 'emit'; attempt: number }
|
||||||
|
| { kind: 'exhaust' }
|
||||||
|
| { kind: 'skip'; reason: ReconnectSkipReason };
|
||||||
|
|
||||||
|
export type ReconnectSkipReason =
|
||||||
|
| 'not-remote'
|
||||||
|
| 'pane-alive'
|
||||||
|
| 'guarded'
|
||||||
|
| 'in-flight'
|
||||||
|
| 'not-due'
|
||||||
|
| 'exhausted'
|
||||||
|
| 'disabled';
|
||||||
|
|
||||||
|
export interface DecideReconnectInput {
|
||||||
|
session: ReconnectSessionView;
|
||||||
|
state: RemoteReconnectState | undefined;
|
||||||
|
/** Session is in the intentional-teardown guard set (killed/detached/stopping). */
|
||||||
|
guarded: boolean;
|
||||||
|
/** Kill-switch: `remoteAutoReconnect` setting. When false, never reconnect. */
|
||||||
|
enabled: boolean;
|
||||||
|
now: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* PURE eligibility decision for one session on one tick. No clock, no I/O — all
|
||||||
|
* inputs are passed in. The watcher translates the result into emits + state
|
||||||
|
* transitions.
|
||||||
|
*
|
||||||
|
* Order of guards (most-decisive first):
|
||||||
|
* 1. kill-switch off → skip:disabled
|
||||||
|
* 2. not a remote session → skip:not-remote
|
||||||
|
* 3. pane is alive → skip:pane-alive
|
||||||
|
* 4. intentional teardown guard → skip:guarded (NEVER revive a killed tab)
|
||||||
|
* 5. a reattach already running → skip:in-flight (no stacked respawns)
|
||||||
|
* 6. already exhausted → skip:exhausted (one exhaust emit, then quiet)
|
||||||
|
* 7. cap reached this tick → exhaust
|
||||||
|
* 8. not yet due (backoff) → skip:not-due
|
||||||
|
* 9. otherwise → emit (attempt = attempts + 1)
|
||||||
|
*/
|
||||||
|
export function decideReconnect(input: DecideReconnectInput): ReconnectAction {
|
||||||
|
const { session, state, guarded, enabled, now } = input;
|
||||||
|
|
||||||
|
if (!enabled) return { kind: 'skip', reason: 'disabled' };
|
||||||
|
if (!session.isRemote) return { kind: 'skip', reason: 'not-remote' };
|
||||||
|
if (!session.paneDead) return { kind: 'skip', reason: 'pane-alive' };
|
||||||
|
// Intentional kill / detach must NEVER be auto-revived.
|
||||||
|
if (guarded) return { kind: 'skip', reason: 'guarded' };
|
||||||
|
|
||||||
|
const s = state ?? freshReconnectState();
|
||||||
|
|
||||||
|
// Only one reconnect in flight per session — don't stack respawns.
|
||||||
|
if (s.inFlight) return { kind: 'skip', reason: 'in-flight' };
|
||||||
|
|
||||||
|
if (s.exhausted) return { kind: 'skip', reason: 'exhausted' };
|
||||||
|
|
||||||
|
// Cap reached: surface exhaustion once, then go quiet.
|
||||||
|
if (isExhausted(s.attempts)) return { kind: 'exhaust' };
|
||||||
|
|
||||||
|
// Backoff gate — only emit when due.
|
||||||
|
if (now < s.nextEligibleAt) return { kind: 'skip', reason: 'not-due' };
|
||||||
|
|
||||||
|
return { kind: 'emit', attempt: s.attempts + 1 };
|
||||||
|
}
|
||||||
+64
-17
@@ -1211,6 +1211,68 @@ export class Session extends EventEmitter {
|
|||||||
return { isRestored };
|
return { isRestored };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* COD-108 — re-establish a dropped REMOTE session. Triggered by the
|
||||||
|
* `TmuxManager` remote-reconnect watcher (via `remoteSessionDropped`): the
|
||||||
|
* watcher detects a dead remote pane, the session owner reassembles the SAME
|
||||||
|
* `RespawnPaneOptions` used for Claude-idle respawns and calls
|
||||||
|
* `respawnPane()` directly. For a remote session that re-runs
|
||||||
|
* `buildRemoteSessionCommand` (owned → `new-session -A`, non-owned →
|
||||||
|
* `attach`), which idempotently REATTACHES the still-running durable remote
|
||||||
|
* tmux session — scrollback + agent intact (proven COD-104/105).
|
||||||
|
*
|
||||||
|
* Deliberately does NOT route through the Claude-idle respawn-controller —
|
||||||
|
* this is a transport re-establish, not a `/clear`/`/compact` cycle.
|
||||||
|
*
|
||||||
|
* @returns true if the pane was respawned (reattach issued), false otherwise.
|
||||||
|
*/
|
||||||
|
async reattachRemote(): Promise<boolean> {
|
||||||
|
if (!this._remote) return false; // not a remote session
|
||||||
|
if (!this._useMux || !this._mux || !this._muxSession) return false;
|
||||||
|
const mux = this._mux;
|
||||||
|
|
||||||
|
// If tmux lost the whole session (not just a dead pane), there is nothing to
|
||||||
|
// respawn into — a genuine death, leave it for normal recovery/reconcile.
|
||||||
|
if (!mux.muxSessionExists(this._muxSession.muxName)) {
|
||||||
|
console.log('[Session] reattachRemote: mux session gone, skipping:', this._muxSession.muxName);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const newPid = await mux.respawnPane(this._buildRespawnPaneOptions());
|
||||||
|
if (!newPid) {
|
||||||
|
console.error('[Session] reattachRemote: respawnPane failed for', this._muxSession.muxName);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
console.log('[Session] reattachRemote: reattached remote session', this._muxSession.muxName, 'pid', newPid);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assemble the {@link RespawnPaneOptions} for this session. Single source of
|
||||||
|
* truth shared by interactive start, shell start (via their inline copies),
|
||||||
|
* and {@link reattachRemote} so the remote reattach path can never drift from
|
||||||
|
* the spawn path.
|
||||||
|
*/
|
||||||
|
private _buildRespawnPaneOptions(): import('./mux-interface.js').RespawnPaneOptions {
|
||||||
|
return {
|
||||||
|
sessionId: this.id,
|
||||||
|
workingDir: this.workingDir,
|
||||||
|
mode: this.mode,
|
||||||
|
niceConfig: this._niceConfig,
|
||||||
|
model: this._model,
|
||||||
|
claudeMode: this._claudeMode,
|
||||||
|
allowedTools: this._allowedTools,
|
||||||
|
openCodeConfig: this._openCodeConfig,
|
||||||
|
codexConfig: this._codexConfig,
|
||||||
|
geminiConfig: this._geminiConfig,
|
||||||
|
resumeSessionId: this._resumeSessionId,
|
||||||
|
envOverrides: this._envOverrides,
|
||||||
|
effort: this._effort,
|
||||||
|
historyLimit: this._tmuxHistoryLimit,
|
||||||
|
remote: this._remote,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private _handleTerminalOutput(data: string): void {
|
private _handleTerminalOutput(data: string): void {
|
||||||
// Codex AND Claude Code emit sequences that wipe xterm.js scrollback, plus
|
// Codex AND Claude Code emit sequences that wipe xterm.js scrollback, plus
|
||||||
// mouse-tracking enables that hijack the scroll wheel so the user can't reach
|
// mouse-tracking enables that hijack the scroll wheel so the user can't reach
|
||||||
@@ -1334,23 +1396,8 @@ export class Session extends EventEmitter {
|
|||||||
if (this._useMux && this._mux) {
|
if (this._useMux && this._mux) {
|
||||||
try {
|
try {
|
||||||
const { isRestored } = await this._setupOrAttachMuxSession({
|
const { isRestored } = await this._setupOrAttachMuxSession({
|
||||||
respawnPaneOptions: {
|
// Single source of truth shared with reattachRemote() (COD-108).
|
||||||
sessionId: this.id,
|
respawnPaneOptions: this._buildRespawnPaneOptions(),
|
||||||
workingDir: this.workingDir,
|
|
||||||
mode: this.mode,
|
|
||||||
niceConfig: this._niceConfig,
|
|
||||||
model: this._model,
|
|
||||||
claudeMode: this._claudeMode,
|
|
||||||
allowedTools: this._allowedTools,
|
|
||||||
openCodeConfig: this._openCodeConfig,
|
|
||||||
codexConfig: this._codexConfig,
|
|
||||||
geminiConfig: this._geminiConfig,
|
|
||||||
resumeSessionId: this._resumeSessionId,
|
|
||||||
envOverrides: this._envOverrides,
|
|
||||||
effort: this._effort,
|
|
||||||
historyLimit: this._tmuxHistoryLimit,
|
|
||||||
remote: this._remote,
|
|
||||||
},
|
|
||||||
createSessionOptions: {
|
createSessionOptions: {
|
||||||
sessionId: this.id,
|
sessionId: this.id,
|
||||||
workingDir: this.workingDir,
|
workingDir: this.workingDir,
|
||||||
|
|||||||
@@ -62,6 +62,13 @@ import type {
|
|||||||
RespawnPaneOptions,
|
RespawnPaneOptions,
|
||||||
PaneCaptureOptions,
|
PaneCaptureOptions,
|
||||||
} from './mux-interface.js';
|
} from './mux-interface.js';
|
||||||
|
import {
|
||||||
|
decideReconnect,
|
||||||
|
advanceBackoff,
|
||||||
|
freshReconnectState,
|
||||||
|
resetReconnectState,
|
||||||
|
type RemoteReconnectState,
|
||||||
|
} from './remote-reconnect.js';
|
||||||
|
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
// Timing Constants
|
// Timing Constants
|
||||||
@@ -94,6 +101,9 @@ const GRACEFUL_SHUTDOWN_WAIT_MS = 100;
|
|||||||
/** Default stats collection interval (2 seconds) */
|
/** Default stats collection interval (2 seconds) */
|
||||||
const DEFAULT_STATS_INTERVAL_MS = 2000;
|
const DEFAULT_STATS_INTERVAL_MS = 2000;
|
||||||
|
|
||||||
|
/** Default remote-reconnect watcher poll interval (5 seconds) — COD-108 */
|
||||||
|
const DEFAULT_REMOTE_RECONNECT_INTERVAL_MS = 5000;
|
||||||
|
|
||||||
/** Stable cwd for tmux server/pane launch; actual session cwd is reached inside the pane. */
|
/** Stable cwd for tmux server/pane launch; actual session cwd is reached inside the pane. */
|
||||||
const TMUX_LAUNCH_CWD = '/tmp';
|
const TMUX_LAUNCH_CWD = '/tmp';
|
||||||
|
|
||||||
@@ -118,6 +128,20 @@ const IS_TEST_MODE = !!process.env.VITEST;
|
|||||||
/** Path to persisted mux session metadata */
|
/** Path to persisted mux session metadata */
|
||||||
const MUX_SESSIONS_FILE = dataPath('mux-sessions.json');
|
const MUX_SESSIONS_FILE = dataPath('mux-sessions.json');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* COD-108 kill-switch: `remoteAutoReconnect` app setting (default ON). Read at
|
||||||
|
* call time (like headroom routing) so a settings change takes effect without a
|
||||||
|
* restart. Absent/non-boolean ⇒ true (feature on).
|
||||||
|
*/
|
||||||
|
function isRemoteAutoReconnectEnabled(): boolean {
|
||||||
|
try {
|
||||||
|
const s = JSON.parse(readFileSync(dataPath('settings.json'), 'utf8')) as Record<string, unknown>;
|
||||||
|
return typeof s.remoteAutoReconnect === 'boolean' ? s.remoteAutoReconnect : true;
|
||||||
|
} catch {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Regex to validate tmux session names (only allow safe characters) */
|
/** Regex to validate tmux session names (only allow safe characters) */
|
||||||
const SAFE_MUX_NAME_PATTERN = /^codeman-[a-f0-9-]+$/;
|
const SAFE_MUX_NAME_PATTERN = /^codeman-[a-f0-9-]+$/;
|
||||||
|
|
||||||
@@ -1008,6 +1032,17 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
/** Track last-known pane count per session to avoid unnecessary tmux set-option calls */
|
/** Track last-known pane count per session to avoid unnecessary tmux set-option calls */
|
||||||
private lastPaneCount: Map<string, number> = new Map();
|
private lastPaneCount: Map<string, number> = new Map();
|
||||||
|
|
||||||
|
// ── COD-108 remote-reconnect watcher state ────────────────────────────────
|
||||||
|
/** Periodic watcher that re-establishes dropped remote sessions. */
|
||||||
|
private remoteReconnectInterval: NodeJS.Timeout | null = null;
|
||||||
|
/** Per-session backoff/attempt bookkeeping (sessionId → state). */
|
||||||
|
private reconnectState: Map<string, RemoteReconnectState> = new Map();
|
||||||
|
/**
|
||||||
|
* Sessions excluded from auto-reconnect because they are being intentionally
|
||||||
|
* torn down (killed/detached/stopping). A guarded session is NEVER revived.
|
||||||
|
*/
|
||||||
|
private reconnectGuard: Set<string> = new Set();
|
||||||
|
|
||||||
private trueColorConfigured = false;
|
private trueColorConfigured = false;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
@@ -1679,9 +1714,16 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// COD-108: an intentional kill/detach must NEVER be auto-revived by the
|
||||||
|
// remote-reconnect watcher. Guard BEFORE any teardown so a tick that fires
|
||||||
|
// mid-kill (especially the non-owned DETACH early-return below, where the
|
||||||
|
// dead local pane would otherwise look reconnectable) sees the guard.
|
||||||
|
this.guardRemoteReconnect(sessionId);
|
||||||
|
|
||||||
// TEST MODE: Remove from memory only — NEVER touch real tmux sessions
|
// TEST MODE: Remove from memory only — NEVER touch real tmux sessions
|
||||||
if (IS_TEST_MODE) {
|
if (IS_TEST_MODE) {
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
|
this.clearRemoteReconnectState(sessionId);
|
||||||
this.emit('sessionKilled', { sessionId });
|
this.emit('sessionKilled', { sessionId });
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1721,6 +1763,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
}
|
}
|
||||||
this.lastPaneCount.delete(session.muxName);
|
this.lastPaneCount.delete(session.muxName);
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
|
this.clearRemoteReconnectState(sessionId);
|
||||||
this.saveSessions();
|
this.saveSessions();
|
||||||
this.emit('sessionKilled', { sessionId });
|
this.emit('sessionKilled', { sessionId });
|
||||||
return true;
|
return true;
|
||||||
@@ -1818,6 +1861,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
|
|
||||||
this.lastPaneCount.delete(session.muxName);
|
this.lastPaneCount.delete(session.muxName);
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
|
this.clearRemoteReconnectState(sessionId);
|
||||||
this.saveSessions();
|
this.saveSessions();
|
||||||
this.emit('sessionKilled', { sessionId });
|
this.emit('sessionKilled', { sessionId });
|
||||||
|
|
||||||
@@ -1884,6 +1928,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
} else {
|
} else {
|
||||||
dead.push(sessionId);
|
dead.push(sessionId);
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
|
this.clearRemoteReconnectState(sessionId);
|
||||||
this.emit('sessionDied', { sessionId });
|
this.emit('sessionDied', { sessionId });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2155,9 +2200,118 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
|
|||||||
this.lastPaneCount.clear();
|
this.lastPaneCount.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── COD-108 remote-session auto-reconnect watcher ─────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start the remote-reconnect watcher (COD-108). Each tick, for every tracked
|
||||||
|
* session with `session.remote` whose local pane is DEAD, not intentionally
|
||||||
|
* guarded, and within its backoff budget, emit `remoteSessionDropped` so the
|
||||||
|
* session owner reattaches (re-running the idempotent remote command rejoins
|
||||||
|
* the durable remote tmux session). After the attempt cap, emit
|
||||||
|
* `remoteReconnectExhausted` once and go quiet.
|
||||||
|
*
|
||||||
|
* No-op tick body under `IS_TEST_MODE` (mirrors `startMouseModeSync`): tests
|
||||||
|
* drive the logic deterministically via {@link runRemoteReconnectTick}.
|
||||||
|
*/
|
||||||
|
startRemoteReconnectWatcher(intervalMs: number = DEFAULT_REMOTE_RECONNECT_INTERVAL_MS): void {
|
||||||
|
if (this.remoteReconnectInterval) {
|
||||||
|
clearInterval(this.remoteReconnectInterval);
|
||||||
|
}
|
||||||
|
this.remoteReconnectInterval = setInterval(() => {
|
||||||
|
if (IS_TEST_MODE) return;
|
||||||
|
try {
|
||||||
|
this.runRemoteReconnectTick(Date.now(), isRemoteAutoReconnectEnabled());
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[TmuxManager] Remote reconnect watcher error:', err);
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
stopRemoteReconnectWatcher(): void {
|
||||||
|
if (this.remoteReconnectInterval) {
|
||||||
|
clearInterval(this.remoteReconnectInterval);
|
||||||
|
this.remoteReconnectInterval = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run ONE watcher tick. Extracted (and given an injected `now`/`enabled`) so
|
||||||
|
* the reconnect logic is deterministically testable even though the live
|
||||||
|
* `setInterval` body no-ops under test mode. For each remote session it
|
||||||
|
* applies the pure {@link decideReconnect} decision and translates the result
|
||||||
|
* into events + backoff/state transitions. Public for tests + the watcher.
|
||||||
|
*/
|
||||||
|
runRemoteReconnectTick(now: number, enabled: boolean): void {
|
||||||
|
for (const session of this.sessions.values()) {
|
||||||
|
if (!session.remote) continue;
|
||||||
|
const sessionId = session.sessionId;
|
||||||
|
const state = this.reconnectState.get(sessionId);
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: {
|
||||||
|
sessionId,
|
||||||
|
isRemote: true,
|
||||||
|
paneDead: this.isPaneDead(session.muxName),
|
||||||
|
},
|
||||||
|
state,
|
||||||
|
guarded: this.reconnectGuard.has(sessionId),
|
||||||
|
enabled,
|
||||||
|
now,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (action.kind === 'emit') {
|
||||||
|
const base = state ?? freshReconnectState();
|
||||||
|
// Mark in-flight + advance backoff BEFORE emitting so a re-entrant tick
|
||||||
|
// (or a synchronous listener) can never stack a second reconnect.
|
||||||
|
this.reconnectState.set(sessionId, { ...advanceBackoff(base, now), inFlight: true });
|
||||||
|
this.emit('remoteSessionDropped', { sessionId, attempt: action.attempt });
|
||||||
|
} else if (action.kind === 'exhaust') {
|
||||||
|
const base = state ?? freshReconnectState();
|
||||||
|
if (!base.exhaustedEmitted) {
|
||||||
|
this.reconnectState.set(sessionId, { ...base, exhausted: true, exhaustedEmitted: true });
|
||||||
|
this.emit('remoteReconnectExhausted', { sessionId });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 'skip' → nothing to do.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tell the watcher a reattach attempt for `sessionId` finished. On success,
|
||||||
|
* reset the backoff so the session is healthy again; on failure, just clear
|
||||||
|
* the in-flight flag so the next due tick can retry under the existing
|
||||||
|
* backoff schedule. Called by the session owner after `respawnPane`.
|
||||||
|
*/
|
||||||
|
noteRemoteReconnect(sessionId: string, success: boolean): void {
|
||||||
|
if (success) {
|
||||||
|
this.reconnectState.set(sessionId, resetReconnectState());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const state = this.reconnectState.get(sessionId);
|
||||||
|
if (state) this.reconnectState.set(sessionId, { ...state, inFlight: false });
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exclude a session from auto-reconnect (intentional teardown). Adds it to the
|
||||||
|
* guard set and drops any backoff state so a closed/killed tab — especially a
|
||||||
|
* non-owned remote DETACH — is never auto-revived. Idempotent.
|
||||||
|
*/
|
||||||
|
guardRemoteReconnect(sessionId: string): void {
|
||||||
|
this.reconnectGuard.add(sessionId);
|
||||||
|
this.reconnectState.delete(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Clear all per-session reconnect + guard state (e.g. when a session is removed). */
|
||||||
|
clearRemoteReconnectState(sessionId: string): void {
|
||||||
|
this.reconnectState.delete(sessionId);
|
||||||
|
this.reconnectGuard.delete(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
destroy(): void {
|
destroy(): void {
|
||||||
this.stopStatsCollection();
|
this.stopStatsCollection();
|
||||||
this.stopMouseModeSync();
|
this.stopMouseModeSync();
|
||||||
|
this.stopRemoteReconnectWatcher();
|
||||||
|
this.reconnectState.clear();
|
||||||
|
this.reconnectGuard.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
registerSession(session: MuxSession): void {
|
registerSession(session: MuxSession): void {
|
||||||
|
|||||||
@@ -216,6 +216,10 @@ const _SSE_HANDLER_MAP = [
|
|||||||
[SSE_EVENTS.MUX_DIED, '_onMuxDied'],
|
[SSE_EVENTS.MUX_DIED, '_onMuxDied'],
|
||||||
[SSE_EVENTS.MUX_STATS_UPDATED, '_onMuxStatsUpdated'],
|
[SSE_EVENTS.MUX_STATS_UPDATED, '_onMuxStatsUpdated'],
|
||||||
|
|
||||||
|
// Remote auto-reconnect (COD-108)
|
||||||
|
[SSE_EVENTS.REMOTE_SESSION_RECONNECTED, '_onRemoteSessionReconnected'],
|
||||||
|
[SSE_EVENTS.REMOTE_RECONNECT_EXHAUSTED, '_onRemoteReconnectExhausted'],
|
||||||
|
|
||||||
// Ralph
|
// Ralph
|
||||||
[SSE_EVENTS.SESSION_RALPH_LOOP_UPDATE, '_onRalphLoopUpdate'],
|
[SSE_EVENTS.SESSION_RALPH_LOOP_UPDATE, '_onRalphLoopUpdate'],
|
||||||
[SSE_EVENTS.SESSION_RALPH_TODO_UPDATE, '_onRalphTodoUpdate'],
|
[SSE_EVENTS.SESSION_RALPH_TODO_UPDATE, '_onRalphTodoUpdate'],
|
||||||
|
|||||||
@@ -379,6 +379,11 @@ const SSE_EVENTS = {
|
|||||||
MUX_DIED: 'mux:died',
|
MUX_DIED: 'mux:died',
|
||||||
MUX_STATS_UPDATED: 'mux:statsUpdated',
|
MUX_STATS_UPDATED: 'mux:statsUpdated',
|
||||||
|
|
||||||
|
// Remote auto-reconnect (COD-108)
|
||||||
|
REMOTE_SESSION_DROPPED: 'remote:sessionDropped',
|
||||||
|
REMOTE_SESSION_RECONNECTED: 'remote:sessionReconnected',
|
||||||
|
REMOTE_RECONNECT_EXHAUSTED: 'remote:reconnectExhausted',
|
||||||
|
|
||||||
// Ralph
|
// Ralph
|
||||||
SESSION_RALPH_LOOP_UPDATE: 'session:ralphLoopUpdate',
|
SESSION_RALPH_LOOP_UPDATE: 'session:ralphLoopUpdate',
|
||||||
SESSION_RALPH_TODO_UPDATE: 'session:ralphTodoUpdate',
|
SESSION_RALPH_TODO_UPDATE: 'session:ralphTodoUpdate',
|
||||||
|
|||||||
@@ -1471,6 +1471,14 @@
|
|||||||
</label>
|
</label>
|
||||||
<span class="form-hint">Use 1M token context window (model: opus[1m]) for all new sessions — ignored when a Claude Model is selected above</span>
|
<span class="form-hint">Use 1M token context window (model: opus[1m]) for all new sessions — ignored when a Claude Model is selected above</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="form-row form-row-switch">
|
||||||
|
<label>Remote auto-reconnect</label>
|
||||||
|
<label class="switch">
|
||||||
|
<input type="checkbox" id="appSettingsRemoteAutoReconnect">
|
||||||
|
<span class="slider"></span>
|
||||||
|
</label>
|
||||||
|
<span class="form-hint">Automatically re-establish remote (SSH) sessions when the connection drops, reattaching to the durable remote tmux session (on by default; bounded backoff)</span>
|
||||||
|
</div>
|
||||||
<div class="form-row">
|
<div class="form-row">
|
||||||
<label>Thinking Effort</label>
|
<label>Thinking Effort</label>
|
||||||
<select id="appSettingsThinkingEffort" class="form-select">
|
<select id="appSettingsThinkingEffort" class="form-select">
|
||||||
|
|||||||
@@ -82,6 +82,33 @@ Object.assign(CodemanApp.prototype, {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// Remote auto-reconnect (COD-108)
|
||||||
|
_onRemoteSessionReconnected(data) {
|
||||||
|
const id = this.getShortId(data.sessionId);
|
||||||
|
this.showToast(`Remote session ${id} reconnected`, 'success');
|
||||||
|
},
|
||||||
|
|
||||||
|
_onRemoteReconnectExhausted(data) {
|
||||||
|
const sessionId = data.sessionId;
|
||||||
|
const id = this.getShortId(sessionId);
|
||||||
|
// Auto-reconnect gave up after the bounded backoff. Surface a manual
|
||||||
|
// "Reconnect" affordance that re-triggers the attach path (force-reload the
|
||||||
|
// session, which re-runs the create/attach flow against the durable remote).
|
||||||
|
this.showToast(`Remote session ${id} dropped — auto-reconnect gave up`, 'error', {
|
||||||
|
duration: 15000,
|
||||||
|
action: {
|
||||||
|
label: 'Reconnect',
|
||||||
|
onClick: () => {
|
||||||
|
if (this.sessions && this.sessions.has(sessionId)) {
|
||||||
|
this.selectSession(sessionId, { forceReload: true });
|
||||||
|
} else {
|
||||||
|
this.showToast('Session no longer available', 'warning');
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
|
||||||
// Bash tools
|
// Bash tools
|
||||||
_onBashToolStart(data) {
|
_onBashToolStart(data) {
|
||||||
|
|||||||
@@ -359,6 +359,7 @@ Object.assign(CodemanApp.prototype, {
|
|||||||
document.getElementById('appSettingsAgentTeams').checked = settings.agentTeamsEnabled ?? false;
|
document.getElementById('appSettingsAgentTeams').checked = settings.agentTeamsEnabled ?? false;
|
||||||
document.getElementById('appSettingsClaudeModel').value = settings.claudeModel ?? '';
|
document.getElementById('appSettingsClaudeModel').value = settings.claudeModel ?? '';
|
||||||
document.getElementById('appSettingsOpusContext1m').checked = settings.opusContext1mEnabled ?? false;
|
document.getElementById('appSettingsOpusContext1m').checked = settings.opusContext1mEnabled ?? false;
|
||||||
|
document.getElementById('appSettingsRemoteAutoReconnect').checked = settings.remoteAutoReconnect ?? true;
|
||||||
document.getElementById('appSettingsThinkingEffort').value = settings.thinkingEffort ?? '';
|
document.getElementById('appSettingsThinkingEffort').value = settings.thinkingEffort ?? '';
|
||||||
// CPU Priority settings
|
// CPU Priority settings
|
||||||
const niceSettings = settings.nice || {};
|
const niceSettings = settings.nice || {};
|
||||||
@@ -1453,6 +1454,7 @@ Object.assign(CodemanApp.prototype, {
|
|||||||
agentTeamsEnabled: document.getElementById('appSettingsAgentTeams').checked,
|
agentTeamsEnabled: document.getElementById('appSettingsAgentTeams').checked,
|
||||||
claudeModel: document.getElementById('appSettingsClaudeModel').value,
|
claudeModel: document.getElementById('appSettingsClaudeModel').value,
|
||||||
opusContext1mEnabled: document.getElementById('appSettingsOpusContext1m').checked,
|
opusContext1mEnabled: document.getElementById('appSettingsOpusContext1m').checked,
|
||||||
|
remoteAutoReconnect: document.getElementById('appSettingsRemoteAutoReconnect').checked,
|
||||||
thinkingEffort: document.getElementById('appSettingsThinkingEffort').value,
|
thinkingEffort: document.getElementById('appSettingsThinkingEffort').value,
|
||||||
// CPU Priority settings
|
// CPU Priority settings
|
||||||
nice: {
|
nice: {
|
||||||
@@ -1770,6 +1772,8 @@ Object.assign(CodemanApp.prototype, {
|
|||||||
showPlanUsageLimits: false,
|
showPlanUsageLimits: false,
|
||||||
showAttachmentsButton: false,
|
showAttachmentsButton: false,
|
||||||
showRedrawButton: false,
|
showRedrawButton: false,
|
||||||
|
// Remote auto-reconnect (COD-108) — on by default
|
||||||
|
remoteAutoReconnect: true,
|
||||||
// Input
|
// Input
|
||||||
gestureControlEnabled: false,
|
gestureControlEnabled: false,
|
||||||
// Feature toggles - keep tracking on even on mobile
|
// Feature toggles - keep tracking on even on mobile
|
||||||
|
|||||||
@@ -494,6 +494,10 @@ export const SettingsUpdateSchema = z
|
|||||||
/** Model for new Claude sessions (e.g. "claude-fable-5[1m]", "opus[1m]"); takes precedence over opusContext1mEnabled */
|
/** Model for new Claude sessions (e.g. "claude-fable-5[1m]", "opus[1m]"); takes precedence over opusContext1mEnabled */
|
||||||
claudeModel: z.string().max(50).optional(),
|
claudeModel: z.string().max(50).optional(),
|
||||||
opusContext1mEnabled: z.boolean().optional(),
|
opusContext1mEnabled: z.boolean().optional(),
|
||||||
|
// COD-108 remote-session auto-reconnect kill-switch (default ON). When false,
|
||||||
|
// the TmuxManager watcher does nothing — dropped remote sessions are NOT
|
||||||
|
// auto-reattached.
|
||||||
|
remoteAutoReconnect: z.boolean().optional(),
|
||||||
thinkingEffort: z.string().max(20).optional(),
|
thinkingEffort: z.string().max(20).optional(),
|
||||||
// UI visibility
|
// UI visibility
|
||||||
showFontControls: z.boolean().optional(),
|
showFontControls: z.boolean().optional(),
|
||||||
|
|||||||
@@ -351,6 +351,22 @@ export class WebServer extends EventEmitter {
|
|||||||
this.broadcast(SseEvent.MuxStatsUpdated, sessions);
|
this.broadcast(SseEvent.MuxStatsUpdated, sessions);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// COD-108 — remote-session auto-reconnect. The TmuxManager watcher detects a
|
||||||
|
// dead remote pane and emits `remoteSessionDropped`; the session owner (here)
|
||||||
|
// reassembles the respawn options and reattaches via Session.reattachRemote()
|
||||||
|
// (D1: the watcher does NOT reassemble options itself). On success we reset
|
||||||
|
// the watcher's backoff; on failure the backoff schedules the next attempt.
|
||||||
|
this.mux.on('remoteSessionDropped', (data) => {
|
||||||
|
const { sessionId, attempt } = data as { sessionId: string; attempt: number };
|
||||||
|
this.broadcast(SseEvent.RemoteSessionDropped, { sessionId, attempt });
|
||||||
|
void this.handleRemoteSessionDropped(sessionId);
|
||||||
|
});
|
||||||
|
this.mux.on('remoteReconnectExhausted', (data) => {
|
||||||
|
const { sessionId } = data as { sessionId: string };
|
||||||
|
console.warn(`[Server] Remote auto-reconnect exhausted for session ${sessionId}`);
|
||||||
|
this.broadcast(SseEvent.RemoteReconnectExhausted, { sessionId });
|
||||||
|
});
|
||||||
|
|
||||||
// Set up subagent watcher listeners
|
// Set up subagent watcher listeners
|
||||||
this.setupSubagentWatcherListeners();
|
this.setupSubagentWatcherListeners();
|
||||||
this.setupWorkflowRunWatcherListeners();
|
this.setupWorkflowRunWatcherListeners();
|
||||||
@@ -2384,6 +2400,13 @@ export class WebServer extends EventEmitter {
|
|||||||
(this.mux as { startMouseModeSync: (ms?: number) => void }).startMouseModeSync();
|
(this.mux as { startMouseModeSync: (ms?: number) => void }).startMouseModeSync();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// COD-108 — start the remote-session auto-reconnect watcher (tmux only).
|
||||||
|
// Always-on (D3) with a `remoteAutoReconnect` kill-switch the watcher reads
|
||||||
|
// each tick. Start even with no sessions — remote sessions may arrive later.
|
||||||
|
if ('startRemoteReconnectWatcher' in this.mux) {
|
||||||
|
(this.mux as { startRemoteReconnectWatcher: (ms?: number) => void }).startRemoteReconnectWatcher();
|
||||||
|
}
|
||||||
|
|
||||||
if (dead.length > 0) {
|
if (dead.length > 0) {
|
||||||
console.log(`[Server] Cleaned up ${dead.length} dead mux session(s)`);
|
console.log(`[Server] Cleaned up ${dead.length} dead mux session(s)`);
|
||||||
}
|
}
|
||||||
@@ -2392,6 +2415,41 @@ export class WebServer extends EventEmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* COD-108 — handle a `remoteSessionDropped` emit from the watcher: reattach
|
||||||
|
* the dropped remote session and report the outcome back to the watcher so it
|
||||||
|
* can reset/advance its backoff. Re-running the idempotent remote command
|
||||||
|
* REATTACHES the durable remote tmux session (does NOT recreate it).
|
||||||
|
*/
|
||||||
|
private async handleRemoteSessionDropped(sessionId: string): Promise<void> {
|
||||||
|
const session = this.sessions.get(sessionId);
|
||||||
|
// No live Session object (e.g. detached/restored-but-not-attached) — nothing
|
||||||
|
// to drive the reattach; report failure so the watcher backs off and retries.
|
||||||
|
if (!session) {
|
||||||
|
this.noteRemoteReconnect(sessionId, false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let ok = false;
|
||||||
|
try {
|
||||||
|
ok = await session.reattachRemote();
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[Server] Remote reattach failed for ${sessionId}:`, err);
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
this.noteRemoteReconnect(sessionId, ok);
|
||||||
|
if (ok) {
|
||||||
|
this.persistSessionState(session);
|
||||||
|
this.broadcast(SseEvent.RemoteSessionReconnected, { sessionId });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Forward a reattach outcome to the TmuxManager watcher (resets/clears backoff). */
|
||||||
|
private noteRemoteReconnect(sessionId: string, success: boolean): void {
|
||||||
|
if ('noteRemoteReconnect' in this.mux) {
|
||||||
|
(this.mux as { noteRemoteReconnect: (id: string, ok: boolean) => void }).noteRemoteReconnect(sessionId, success);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private initOrchestratorLoop(): import('../orchestrator-loop.js').OrchestratorLoop {
|
private initOrchestratorLoop(): import('../orchestrator-loop.js').OrchestratorLoop {
|
||||||
if (this._orchestratorLoop) return this._orchestratorLoop;
|
if (this._orchestratorLoop) return this._orchestratorLoop;
|
||||||
|
|
||||||
|
|||||||
@@ -150,6 +150,15 @@ export const MuxDied = 'mux:died' as const;
|
|||||||
/** tmux session stats refreshed. */
|
/** tmux session stats refreshed. */
|
||||||
export const MuxStatsUpdated = 'mux:statsUpdated' as const;
|
export const MuxStatsUpdated = 'mux:statsUpdated' as const;
|
||||||
|
|
||||||
|
// ─── Remote auto-reconnect (COD-108) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
/** A remote session's local ssh pane died; an auto-reconnect attempt is starting. */
|
||||||
|
export const RemoteSessionDropped = 'remote:sessionDropped' as const;
|
||||||
|
/** A dropped remote session was successfully re-established (reattached). */
|
||||||
|
export const RemoteSessionReconnected = 'remote:sessionReconnected' as const;
|
||||||
|
/** Auto-reconnect gave up after the bounded backoff cap — manual reconnect needed. */
|
||||||
|
export const RemoteReconnectExhausted = 'remote:reconnectExhausted' as const;
|
||||||
|
|
||||||
// ─── Respawn ─────────────────────────────────────────────────────────────────
|
// ─── Respawn ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Respawn loop started for a session. */
|
/** Respawn loop started for a session. */
|
||||||
@@ -435,6 +444,11 @@ export const SseEvent = {
|
|||||||
MuxDied,
|
MuxDied,
|
||||||
MuxStatsUpdated,
|
MuxStatsUpdated,
|
||||||
|
|
||||||
|
// Remote auto-reconnect (COD-108)
|
||||||
|
RemoteSessionDropped,
|
||||||
|
RemoteSessionReconnected,
|
||||||
|
RemoteReconnectExhausted,
|
||||||
|
|
||||||
// Respawn
|
// Respawn
|
||||||
RespawnStarted,
|
RespawnStarted,
|
||||||
RespawnStopped,
|
RespawnStopped,
|
||||||
|
|||||||
@@ -0,0 +1,298 @@
|
|||||||
|
/**
|
||||||
|
* @fileoverview COD-108 — remote-session auto-reconnect watcher tests.
|
||||||
|
*
|
||||||
|
* Three layers, all tmux-safe (under VITEST TmuxManager no-ops real tmux and
|
||||||
|
* `isPaneDead` returns false, so live behavior is driven via injected stubs):
|
||||||
|
*
|
||||||
|
* (a) PURE backoff schedule — attempt→delay, cap, reset-on-success.
|
||||||
|
* (b) PURE eligibility decision — dead remote pane + due → emit; guarded →
|
||||||
|
* never; non-remote / pane-alive / not-due → skip; over-cap → exhaust.
|
||||||
|
* (c) MANAGER integration — drive ticks with a stubbed pane-death signal +
|
||||||
|
* controllable clock and assert the emit → backoff → exhausted progression,
|
||||||
|
* and that a guarded (intentionally-killed) session emits nothing.
|
||||||
|
*
|
||||||
|
* Port: N/A (no server).
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||||
|
import {
|
||||||
|
BACKOFF_SCHEDULE_MS,
|
||||||
|
MAX_RECONNECT_ATTEMPTS,
|
||||||
|
reconnectDelayForAttempt,
|
||||||
|
isExhausted,
|
||||||
|
freshReconnectState,
|
||||||
|
advanceBackoff,
|
||||||
|
resetReconnectState,
|
||||||
|
decideReconnect,
|
||||||
|
} from '../src/remote-reconnect.js';
|
||||||
|
import type { ReconnectSessionView } from '../src/remote-reconnect.js';
|
||||||
|
import { TmuxManager } from '../src/tmux-manager.js';
|
||||||
|
import type { SessionRemote } from '../src/types.js';
|
||||||
|
|
||||||
|
const REMOTE: SessionRemote = {
|
||||||
|
hostId: 'aa-desktop',
|
||||||
|
label: 'aa-desktop',
|
||||||
|
host: 'aa-desktop',
|
||||||
|
username: 'aakhter',
|
||||||
|
remotePath: '/home/aakhter',
|
||||||
|
owned: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// (a) PURE backoff schedule
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('reconnect backoff schedule (pure)', () => {
|
||||||
|
it('returns the documented bounded exponential delays per attempt', () => {
|
||||||
|
expect(reconnectDelayForAttempt(1)).toBe(5_000);
|
||||||
|
expect(reconnectDelayForAttempt(2)).toBe(15_000);
|
||||||
|
expect(reconnectDelayForAttempt(3)).toBe(45_000);
|
||||||
|
expect(reconnectDelayForAttempt(4)).toBe(120_000);
|
||||||
|
expect(reconnectDelayForAttempt(5)).toBe(300_000);
|
||||||
|
expect(reconnectDelayForAttempt(6)).toBe(300_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clamps below-range and above-range attempts to the schedule bounds', () => {
|
||||||
|
expect(reconnectDelayForAttempt(0)).toBe(BACKOFF_SCHEDULE_MS[0]);
|
||||||
|
expect(reconnectDelayForAttempt(-3)).toBe(BACKOFF_SCHEDULE_MS[0]);
|
||||||
|
expect(reconnectDelayForAttempt(99)).toBe(BACKOFF_SCHEDULE_MS[BACKOFF_SCHEDULE_MS.length - 1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('flags exhaustion only at/after the cap', () => {
|
||||||
|
expect(isExhausted(0)).toBe(false);
|
||||||
|
expect(isExhausted(MAX_RECONNECT_ATTEMPTS - 1)).toBe(false);
|
||||||
|
expect(isExhausted(MAX_RECONNECT_ATTEMPTS)).toBe(true);
|
||||||
|
expect(isExhausted(MAX_RECONNECT_ATTEMPTS + 1)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('advanceBackoff increments attempts and schedules next-eligible from now (immutable)', () => {
|
||||||
|
const s0 = freshReconnectState();
|
||||||
|
const s1 = advanceBackoff(s0, 1_000);
|
||||||
|
expect(s0.attempts).toBe(0); // input untouched
|
||||||
|
expect(s1.attempts).toBe(1);
|
||||||
|
expect(s1.nextEligibleAt).toBe(1_000 + 5_000);
|
||||||
|
expect(s1.exhausted).toBe(false);
|
||||||
|
|
||||||
|
const s2 = advanceBackoff(s1, 10_000);
|
||||||
|
expect(s2.attempts).toBe(2);
|
||||||
|
expect(s2.nextEligibleAt).toBe(10_000 + 15_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reaches the attempt cap after the scheduled number of advances', () => {
|
||||||
|
let s = freshReconnectState();
|
||||||
|
let now = 0;
|
||||||
|
for (let i = 0; i < MAX_RECONNECT_ATTEMPTS; i++) {
|
||||||
|
s = advanceBackoff(s, now);
|
||||||
|
now += reconnectDelayForAttempt(s.attempts);
|
||||||
|
}
|
||||||
|
expect(s.attempts).toBe(MAX_RECONNECT_ATTEMPTS);
|
||||||
|
// advanceBackoff does not itself set `exhausted`; the watcher decides that
|
||||||
|
// on the following tick via isExhausted(attempts).
|
||||||
|
expect(isExhausted(s.attempts)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reset-on-success returns to a fresh, immediately-eligible state', () => {
|
||||||
|
const advanced = advanceBackoff(advanceBackoff(freshReconnectState(), 0), 100);
|
||||||
|
expect(advanced.attempts).toBe(2);
|
||||||
|
const reset = resetReconnectState();
|
||||||
|
expect(reset.attempts).toBe(0);
|
||||||
|
expect(reset.nextEligibleAt).toBe(0);
|
||||||
|
expect(reset.exhausted).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// (b) PURE eligibility decision
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('decideReconnect (pure eligibility)', () => {
|
||||||
|
const deadRemote: ReconnectSessionView = { sessionId: 's1', isRemote: true, paneDead: true };
|
||||||
|
|
||||||
|
it('emits for a dead remote pane that is not guarded and is due', () => {
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: deadRemote,
|
||||||
|
state: freshReconnectState(),
|
||||||
|
guarded: false,
|
||||||
|
enabled: true,
|
||||||
|
now: 0,
|
||||||
|
});
|
||||||
|
expect(action).toEqual({ kind: 'emit', attempt: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('NEVER reconnects a guarded (intentionally killed/detached) session', () => {
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: deadRemote,
|
||||||
|
state: freshReconnectState(),
|
||||||
|
guarded: true,
|
||||||
|
enabled: true,
|
||||||
|
now: 0,
|
||||||
|
});
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'guarded' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips non-remote sessions', () => {
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: { sessionId: 's1', isRemote: false, paneDead: true },
|
||||||
|
state: freshReconnectState(),
|
||||||
|
guarded: false,
|
||||||
|
enabled: true,
|
||||||
|
now: 0,
|
||||||
|
});
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'not-remote' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips when the pane is alive', () => {
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: { sessionId: 's1', isRemote: true, paneDead: false },
|
||||||
|
state: freshReconnectState(),
|
||||||
|
guarded: false,
|
||||||
|
enabled: true,
|
||||||
|
now: 0,
|
||||||
|
});
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'pane-alive' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips when the kill-switch is off', () => {
|
||||||
|
const action = decideReconnect({
|
||||||
|
session: deadRemote,
|
||||||
|
state: freshReconnectState(),
|
||||||
|
guarded: false,
|
||||||
|
enabled: false,
|
||||||
|
now: 0,
|
||||||
|
});
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'disabled' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips when not yet due (within backoff window)', () => {
|
||||||
|
const state = advanceBackoff(freshReconnectState(), 0); // nextEligibleAt = 5000
|
||||||
|
const action = decideReconnect({ session: deadRemote, state, guarded: false, enabled: true, now: 4_999 });
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'not-due' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits again once the backoff window elapses', () => {
|
||||||
|
const state = advanceBackoff(freshReconnectState(), 0); // nextEligibleAt = 5000
|
||||||
|
const action = decideReconnect({ session: deadRemote, state, guarded: false, enabled: true, now: 5_000 });
|
||||||
|
expect(action).toEqual({ kind: 'emit', attempt: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips while a reconnect is already in flight (no stacked respawns)', () => {
|
||||||
|
const state = { ...freshReconnectState(), inFlight: true };
|
||||||
|
const action = decideReconnect({ session: deadRemote, state, guarded: false, enabled: true, now: 10_000 });
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'in-flight' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('exhausts once the attempt cap is reached', () => {
|
||||||
|
const state = { ...freshReconnectState(), attempts: MAX_RECONNECT_ATTEMPTS, nextEligibleAt: 0 };
|
||||||
|
const action = decideReconnect({ session: deadRemote, state, guarded: false, enabled: true, now: 1_000_000 });
|
||||||
|
expect(action).toEqual({ kind: 'exhaust' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stays quiet after exhaustion has been recorded', () => {
|
||||||
|
const state = { ...freshReconnectState(), attempts: MAX_RECONNECT_ATTEMPTS, exhausted: true };
|
||||||
|
const action = decideReconnect({ session: deadRemote, state, guarded: false, enabled: true, now: 1_000_000 });
|
||||||
|
expect(action).toEqual({ kind: 'skip', reason: 'exhausted' });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
// (c) MANAGER integration — drive ticks with a stubbed pane-death + clock
|
||||||
|
// ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('TmuxManager remote reconnect watcher (integration)', () => {
|
||||||
|
let manager: TmuxManager;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
manager = new TmuxManager();
|
||||||
|
});
|
||||||
|
|
||||||
|
function registerRemote(sessionId: string): void {
|
||||||
|
manager.registerSession({
|
||||||
|
sessionId,
|
||||||
|
muxName: `codeman-${sessionId}`,
|
||||||
|
pid: 4242,
|
||||||
|
createdAt: Date.now(),
|
||||||
|
workingDir: '/home/aakhter',
|
||||||
|
mode: 'shell',
|
||||||
|
attached: true,
|
||||||
|
remote: REMOTE,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it('emits remoteSessionDropped when a dead remote pane is observed, then backs off and exhausts', () => {
|
||||||
|
registerRemote('aaaa1111');
|
||||||
|
// Force the watcher to see a dead pane regardless of test-mode isPaneDead.
|
||||||
|
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
|
||||||
|
|
||||||
|
const dropped: Array<{ sessionId: string; attempt: number }> = [];
|
||||||
|
const exhausted: Array<{ sessionId: string }> = [];
|
||||||
|
manager.on('remoteSessionDropped', (d) => dropped.push(d));
|
||||||
|
manager.on('remoteReconnectExhausted', (d) => exhausted.push(d));
|
||||||
|
|
||||||
|
// Drive ticks with a controllable clock. Each emit marks the session
|
||||||
|
// in-flight; a failed reattach (host still down) releases it via
|
||||||
|
// noteRemoteReconnect(false), mirroring the real server loop.
|
||||||
|
let now = 0;
|
||||||
|
for (let i = 0; i < MAX_RECONNECT_ATTEMPTS + 3; i++) {
|
||||||
|
manager.runRemoteReconnectTick(now, /* enabled */ true);
|
||||||
|
manager.noteRemoteReconnect('aaaa1111', false); // reattach failed → clear in-flight
|
||||||
|
// jump the clock past the just-scheduled backoff window
|
||||||
|
now += BACKOFF_SCHEDULE_MS[Math.min(i, BACKOFF_SCHEDULE_MS.length - 1)] + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(dropped.map((d) => d.attempt)).toEqual([1, 2, 3, 4, 5, 6]);
|
||||||
|
expect(dropped.every((d) => d.sessionId === 'aaaa1111')).toBe(true);
|
||||||
|
expect(exhausted).toEqual([{ sessionId: 'aaaa1111' }]); // fired exactly once
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits nothing for a guarded (intentionally killed) session', () => {
|
||||||
|
registerRemote('bbbb2222');
|
||||||
|
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
|
||||||
|
manager.guardRemoteReconnect('bbbb2222'); // simulate killSession/detach guard
|
||||||
|
|
||||||
|
const dropped: unknown[] = [];
|
||||||
|
manager.on('remoteSessionDropped', (d) => dropped.push(d));
|
||||||
|
|
||||||
|
let now = 0;
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
manager.runRemoteReconnectTick(now, true);
|
||||||
|
now += 600_000;
|
||||||
|
}
|
||||||
|
expect(dropped).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resets backoff on a successful reattach (noteRemoteReconnect)', () => {
|
||||||
|
registerRemote('cccc3333');
|
||||||
|
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
|
||||||
|
|
||||||
|
const dropped: Array<{ attempt: number }> = [];
|
||||||
|
manager.on('remoteSessionDropped', (d) => dropped.push(d));
|
||||||
|
|
||||||
|
manager.runRemoteReconnectTick(0, true); // emit attempt 1
|
||||||
|
manager.noteRemoteReconnect('cccc3333', true); // reattach succeeded → reset
|
||||||
|
manager.runRemoteReconnectTick(1, true); // immediately eligible again → attempt 1
|
||||||
|
|
||||||
|
expect(dropped.map((d) => d.attempt)).toEqual([1, 1]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does nothing when the kill-switch (enabled=false) is off', () => {
|
||||||
|
registerRemote('dddd4444');
|
||||||
|
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
|
||||||
|
const dropped: unknown[] = [];
|
||||||
|
manager.on('remoteSessionDropped', (d) => dropped.push(d));
|
||||||
|
|
||||||
|
manager.runRemoteReconnectTick(0, false);
|
||||||
|
expect(dropped).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears per-session reconnect/guard state when the session is removed', () => {
|
||||||
|
registerRemote('eeee5555');
|
||||||
|
manager.guardRemoteReconnect('eeee5555');
|
||||||
|
manager.clearRemoteReconnectState('eeee5555');
|
||||||
|
// After clearing the guard, a fresh dead-pane observation should emit again.
|
||||||
|
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
|
||||||
|
const dropped: unknown[] = [];
|
||||||
|
manager.on('remoteSessionDropped', (d) => dropped.push(d));
|
||||||
|
manager.runRemoteReconnectTick(0, true);
|
||||||
|
expect(dropped).toEqual([{ sessionId: 'eeee5555', attempt: 1 }]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user