COD-108 auto-reconnect remote tmux sessions on SSH drop

Continuous remote-only reconnect watcher closing the COD-104 durability
arc: when a remote session's local ssh pane dies mid-run, re-establish it
automatically instead of leaving a dead pane until the user pokes it.

Design decisions (per cod108 design doc):
- D1 event->owner: TmuxManager watcher DETECTS a dead remote pane and emits
  `remoteSessionDropped`; the session owner (server) reassembles the same
  RespawnPaneOptions and calls Session.reattachRemote() -> respawnPane, which
  re-runs the idempotent remote command (owned new-session -A / non-owned
  attach) and REJOINS the still-running durable remote tmux session. The
  watcher never reassembles options itself, and never routes through the
  Claude-idle respawn-controller.
- D2 bounded backoff: per-session exponential backoff [5s,15s,45s,2m,5m,5m],
  reset on a successful reattach, `remoteReconnectExhausted` emitted once after
  the cap. Pure, unit-tested schedule + eligibility decision.
- D3 always-on + kill-switch: `remoteAutoReconnect` app setting (default ON),
  read each tick; when false the watcher does nothing.

Guards: killSession() (incl. the non-owned DETACH early-return) and shutdown
add the session to an intentional-teardown guard set + clear its backoff
BEFORE teardown, so a closed/killed tab is never auto-revived. Exactly one
reconnect in flight per session (inFlight guard prevents stacked respawns).
Per-session reconnect/guard state cleared on session removal.

New: src/remote-reconnect.ts (pure backoff + decideReconnect), TmuxManager
startRemoteReconnectWatcher/stop + runRemoteReconnectTick + noteRemoteReconnect
+ guardRemoteReconnect + clearRemoteReconnectState; Session.reattachRemote()
(+ extracted _buildRespawnPaneOptions, shared with interactive start); server
wiring + watcher start; 3 SSE events (sse-events.ts + constants.js in sync,
broadcast + app.js exhausted "Reconnect" affordance); remoteAutoReconnect
schema + settings-ui toggle.

Tests: test/remote-auto-reconnect.test.ts (21) - pure schedule, eligibility
(guarded never reconnects, non-remote/pane-alive/not-due skip, over-cap
exhaust), and manager-level integration (dead remote pane -> dropped ->
backoff -> exhausted; guarded emits nothing; reset-on-success; kill-switch
off; state-cleared-on-remove). Verified real-remote against aa-desktop: drop
local ssh pane -> watcher emitted -> respawnPane reattached the SAME remote
session (remote pane_pid unchanged 3939->3939); test session cleaned up, the
real host sessions left untouched.

Checks: tsc, eslint, check:frontend-syntax, check:public-assets, prettier
--check, build all green; tmux-manager/session-routes/session-manager/
sse-registry-parity suites pass.

(cherry picked from commit d13d58b1994eb6594fd2eadea208104d36204f9d)
This commit is contained in:
Aamer Akhter
2026-07-17 15:59:36 -04:00
parent 897bfdff59
commit 6dba8b5227
12 changed files with 824 additions and 17 deletions
+4
View File
@@ -216,6 +216,10 @@ const _SSE_HANDLER_MAP = [
[SSE_EVENTS.MUX_DIED, '_onMuxDied'],
[SSE_EVENTS.MUX_STATS_UPDATED, '_onMuxStatsUpdated'],
// Remote auto-reconnect (COD-108)
[SSE_EVENTS.REMOTE_SESSION_RECONNECTED, '_onRemoteSessionReconnected'],
[SSE_EVENTS.REMOTE_RECONNECT_EXHAUSTED, '_onRemoteReconnectExhausted'],
// Ralph
[SSE_EVENTS.SESSION_RALPH_LOOP_UPDATE, '_onRalphLoopUpdate'],
[SSE_EVENTS.SESSION_RALPH_TODO_UPDATE, '_onRalphTodoUpdate'],
+5
View File
@@ -379,6 +379,11 @@ const SSE_EVENTS = {
MUX_DIED: 'mux:died',
MUX_STATS_UPDATED: 'mux:statsUpdated',
// Remote auto-reconnect (COD-108)
REMOTE_SESSION_DROPPED: 'remote:sessionDropped',
REMOTE_SESSION_RECONNECTED: 'remote:sessionReconnected',
REMOTE_RECONNECT_EXHAUSTED: 'remote:reconnectExhausted',
// Ralph
SESSION_RALPH_LOOP_UPDATE: 'session:ralphLoopUpdate',
SESSION_RALPH_TODO_UPDATE: 'session:ralphTodoUpdate',
+8
View File
@@ -1471,6 +1471,14 @@
</label>
<span class="form-hint">Use 1M token context window (model: opus[1m]) for all new sessions — ignored when a Claude Model is selected above</span>
</div>
<div class="form-row form-row-switch">
<label>Remote auto-reconnect</label>
<label class="switch">
<input type="checkbox" id="appSettingsRemoteAutoReconnect">
<span class="slider"></span>
</label>
<span class="form-hint">Automatically re-establish remote (SSH) sessions when the connection drops, reattaching to the durable remote tmux session (on by default; bounded backoff)</span>
</div>
<div class="form-row">
<label>Thinking Effort</label>
<select id="appSettingsThinkingEffort" class="form-select">
+27
View File
@@ -82,6 +82,33 @@ Object.assign(CodemanApp.prototype, {
}
},
// Remote auto-reconnect (COD-108)
_onRemoteSessionReconnected(data) {
const id = this.getShortId(data.sessionId);
this.showToast(`Remote session ${id} reconnected`, 'success');
},
_onRemoteReconnectExhausted(data) {
const sessionId = data.sessionId;
const id = this.getShortId(sessionId);
// Auto-reconnect gave up after the bounded backoff. Surface a manual
// "Reconnect" affordance that re-triggers the attach path (force-reload the
// session, which re-runs the create/attach flow against the durable remote).
this.showToast(`Remote session ${id} dropped — auto-reconnect gave up`, 'error', {
duration: 15000,
action: {
label: 'Reconnect',
onClick: () => {
if (this.sessions && this.sessions.has(sessionId)) {
this.selectSession(sessionId, { forceReload: true });
} else {
this.showToast('Session no longer available', 'warning');
}
},
},
});
},
// Bash tools
_onBashToolStart(data) {
+4
View File
@@ -359,6 +359,7 @@ Object.assign(CodemanApp.prototype, {
document.getElementById('appSettingsAgentTeams').checked = settings.agentTeamsEnabled ?? false;
document.getElementById('appSettingsClaudeModel').value = settings.claudeModel ?? '';
document.getElementById('appSettingsOpusContext1m').checked = settings.opusContext1mEnabled ?? false;
document.getElementById('appSettingsRemoteAutoReconnect').checked = settings.remoteAutoReconnect ?? true;
document.getElementById('appSettingsThinkingEffort').value = settings.thinkingEffort ?? '';
// CPU Priority settings
const niceSettings = settings.nice || {};
@@ -1453,6 +1454,7 @@ Object.assign(CodemanApp.prototype, {
agentTeamsEnabled: document.getElementById('appSettingsAgentTeams').checked,
claudeModel: document.getElementById('appSettingsClaudeModel').value,
opusContext1mEnabled: document.getElementById('appSettingsOpusContext1m').checked,
remoteAutoReconnect: document.getElementById('appSettingsRemoteAutoReconnect').checked,
thinkingEffort: document.getElementById('appSettingsThinkingEffort').value,
// CPU Priority settings
nice: {
@@ -1770,6 +1772,8 @@ Object.assign(CodemanApp.prototype, {
showPlanUsageLimits: false,
showAttachmentsButton: false,
showRedrawButton: false,
// Remote auto-reconnect (COD-108) — on by default
remoteAutoReconnect: true,
// Input
gestureControlEnabled: false,
// Feature toggles - keep tracking on even on mobile
+4
View File
@@ -494,6 +494,10 @@ export const SettingsUpdateSchema = z
/** Model for new Claude sessions (e.g. "claude-fable-5[1m]", "opus[1m]"); takes precedence over opusContext1mEnabled */
claudeModel: z.string().max(50).optional(),
opusContext1mEnabled: z.boolean().optional(),
// COD-108 remote-session auto-reconnect kill-switch (default ON). When false,
// the TmuxManager watcher does nothing — dropped remote sessions are NOT
// auto-reattached.
remoteAutoReconnect: z.boolean().optional(),
thinkingEffort: z.string().max(20).optional(),
// UI visibility
showFontControls: z.boolean().optional(),
+58
View File
@@ -351,6 +351,22 @@ export class WebServer extends EventEmitter {
this.broadcast(SseEvent.MuxStatsUpdated, sessions);
});
// COD-108 — remote-session auto-reconnect. The TmuxManager watcher detects a
// dead remote pane and emits `remoteSessionDropped`; the session owner (here)
// reassembles the respawn options and reattaches via Session.reattachRemote()
// (D1: the watcher does NOT reassemble options itself). On success we reset
// the watcher's backoff; on failure the backoff schedules the next attempt.
this.mux.on('remoteSessionDropped', (data) => {
const { sessionId, attempt } = data as { sessionId: string; attempt: number };
this.broadcast(SseEvent.RemoteSessionDropped, { sessionId, attempt });
void this.handleRemoteSessionDropped(sessionId);
});
this.mux.on('remoteReconnectExhausted', (data) => {
const { sessionId } = data as { sessionId: string };
console.warn(`[Server] Remote auto-reconnect exhausted for session ${sessionId}`);
this.broadcast(SseEvent.RemoteReconnectExhausted, { sessionId });
});
// Set up subagent watcher listeners
this.setupSubagentWatcherListeners();
this.setupWorkflowRunWatcherListeners();
@@ -2384,6 +2400,13 @@ export class WebServer extends EventEmitter {
(this.mux as { startMouseModeSync: (ms?: number) => void }).startMouseModeSync();
}
// COD-108 — start the remote-session auto-reconnect watcher (tmux only).
// Always-on (D3) with a `remoteAutoReconnect` kill-switch the watcher reads
// each tick. Start even with no sessions — remote sessions may arrive later.
if ('startRemoteReconnectWatcher' in this.mux) {
(this.mux as { startRemoteReconnectWatcher: (ms?: number) => void }).startRemoteReconnectWatcher();
}
if (dead.length > 0) {
console.log(`[Server] Cleaned up ${dead.length} dead mux session(s)`);
}
@@ -2392,6 +2415,41 @@ export class WebServer extends EventEmitter {
}
}
/**
* COD-108 — handle a `remoteSessionDropped` emit from the watcher: reattach
* the dropped remote session and report the outcome back to the watcher so it
* can reset/advance its backoff. Re-running the idempotent remote command
* REATTACHES the durable remote tmux session (does NOT recreate it).
*/
private async handleRemoteSessionDropped(sessionId: string): Promise<void> {
const session = this.sessions.get(sessionId);
// No live Session object (e.g. detached/restored-but-not-attached) — nothing
// to drive the reattach; report failure so the watcher backs off and retries.
if (!session) {
this.noteRemoteReconnect(sessionId, false);
return;
}
let ok = false;
try {
ok = await session.reattachRemote();
} catch (err) {
console.error(`[Server] Remote reattach failed for ${sessionId}:`, err);
ok = false;
}
this.noteRemoteReconnect(sessionId, ok);
if (ok) {
this.persistSessionState(session);
this.broadcast(SseEvent.RemoteSessionReconnected, { sessionId });
}
}
/** Forward a reattach outcome to the TmuxManager watcher (resets/clears backoff). */
private noteRemoteReconnect(sessionId: string, success: boolean): void {
if ('noteRemoteReconnect' in this.mux) {
(this.mux as { noteRemoteReconnect: (id: string, ok: boolean) => void }).noteRemoteReconnect(sessionId, success);
}
}
private initOrchestratorLoop(): import('../orchestrator-loop.js').OrchestratorLoop {
if (this._orchestratorLoop) return this._orchestratorLoop;
+14
View File
@@ -150,6 +150,15 @@ export const MuxDied = 'mux:died' as const;
/** tmux session stats refreshed. */
export const MuxStatsUpdated = 'mux:statsUpdated' as const;
// ─── Remote auto-reconnect (COD-108) ─────────────────────────────────────────
/** A remote session's local ssh pane died; an auto-reconnect attempt is starting. */
export const RemoteSessionDropped = 'remote:sessionDropped' as const;
/** A dropped remote session was successfully re-established (reattached). */
export const RemoteSessionReconnected = 'remote:sessionReconnected' as const;
/** Auto-reconnect gave up after the bounded backoff cap — manual reconnect needed. */
export const RemoteReconnectExhausted = 'remote:reconnectExhausted' as const;
// ─── Respawn ─────────────────────────────────────────────────────────────────
/** Respawn loop started for a session. */
@@ -435,6 +444,11 @@ export const SseEvent = {
MuxDied,
MuxStatsUpdated,
// Remote auto-reconnect (COD-108)
RemoteSessionDropped,
RemoteSessionReconnected,
RemoteReconnectExhausted,
// Respawn
RespawnStarted,
RespawnStopped,