mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-04 22:49:41 +02:00
COD-108 auto-reconnect remote tmux sessions on SSH drop
Continuous remote-only reconnect watcher closing the COD-104 durability arc: when a remote session's local ssh pane dies mid-run, re-establish it automatically instead of leaving a dead pane until the user pokes it. Design decisions (per cod108 design doc): - D1 event->owner: TmuxManager watcher DETECTS a dead remote pane and emits `remoteSessionDropped`; the session owner (server) reassembles the same RespawnPaneOptions and calls Session.reattachRemote() -> respawnPane, which re-runs the idempotent remote command (owned new-session -A / non-owned attach) and REJOINS the still-running durable remote tmux session. The watcher never reassembles options itself, and never routes through the Claude-idle respawn-controller. - D2 bounded backoff: per-session exponential backoff [5s,15s,45s,2m,5m,5m], reset on a successful reattach, `remoteReconnectExhausted` emitted once after the cap. Pure, unit-tested schedule + eligibility decision. - D3 always-on + kill-switch: `remoteAutoReconnect` app setting (default ON), read each tick; when false the watcher does nothing. Guards: killSession() (incl. the non-owned DETACH early-return) and shutdown add the session to an intentional-teardown guard set + clear its backoff BEFORE teardown, so a closed/killed tab is never auto-revived. Exactly one reconnect in flight per session (inFlight guard prevents stacked respawns). Per-session reconnect/guard state cleared on session removal. New: src/remote-reconnect.ts (pure backoff + decideReconnect), TmuxManager startRemoteReconnectWatcher/stop + runRemoteReconnectTick + noteRemoteReconnect + guardRemoteReconnect + clearRemoteReconnectState; Session.reattachRemote() (+ extracted _buildRespawnPaneOptions, shared with interactive start); server wiring + watcher start; 3 SSE events (sse-events.ts + constants.js in sync, broadcast + app.js exhausted "Reconnect" affordance); remoteAutoReconnect schema + settings-ui toggle. Tests: test/remote-auto-reconnect.test.ts (21) - pure schedule, eligibility (guarded never reconnects, non-remote/pane-alive/not-due skip, over-cap exhaust), and manager-level integration (dead remote pane -> dropped -> backoff -> exhausted; guarded emits nothing; reset-on-success; kill-switch off; state-cleared-on-remove). Verified real-remote against aa-desktop: drop local ssh pane -> watcher emitted -> respawnPane reattached the SAME remote session (remote pane_pid unchanged 3939->3939); test session cleaned up, the real host sessions left untouched. Checks: tsc, eslint, check:frontend-syntax, check:public-assets, prettier --check, build all green; tmux-manager/session-routes/session-manager/ sse-registry-parity suites pass. (cherry picked from commit d13d58b1994eb6594fd2eadea208104d36204f9d)
This commit is contained in:
@@ -216,6 +216,10 @@ const _SSE_HANDLER_MAP = [
|
||||
[SSE_EVENTS.MUX_DIED, '_onMuxDied'],
|
||||
[SSE_EVENTS.MUX_STATS_UPDATED, '_onMuxStatsUpdated'],
|
||||
|
||||
// Remote auto-reconnect (COD-108)
|
||||
[SSE_EVENTS.REMOTE_SESSION_RECONNECTED, '_onRemoteSessionReconnected'],
|
||||
[SSE_EVENTS.REMOTE_RECONNECT_EXHAUSTED, '_onRemoteReconnectExhausted'],
|
||||
|
||||
// Ralph
|
||||
[SSE_EVENTS.SESSION_RALPH_LOOP_UPDATE, '_onRalphLoopUpdate'],
|
||||
[SSE_EVENTS.SESSION_RALPH_TODO_UPDATE, '_onRalphTodoUpdate'],
|
||||
|
||||
@@ -379,6 +379,11 @@ const SSE_EVENTS = {
|
||||
MUX_DIED: 'mux:died',
|
||||
MUX_STATS_UPDATED: 'mux:statsUpdated',
|
||||
|
||||
// Remote auto-reconnect (COD-108)
|
||||
REMOTE_SESSION_DROPPED: 'remote:sessionDropped',
|
||||
REMOTE_SESSION_RECONNECTED: 'remote:sessionReconnected',
|
||||
REMOTE_RECONNECT_EXHAUSTED: 'remote:reconnectExhausted',
|
||||
|
||||
// Ralph
|
||||
SESSION_RALPH_LOOP_UPDATE: 'session:ralphLoopUpdate',
|
||||
SESSION_RALPH_TODO_UPDATE: 'session:ralphTodoUpdate',
|
||||
|
||||
@@ -1471,6 +1471,14 @@
|
||||
</label>
|
||||
<span class="form-hint">Use 1M token context window (model: opus[1m]) for all new sessions — ignored when a Claude Model is selected above</span>
|
||||
</div>
|
||||
<div class="form-row form-row-switch">
|
||||
<label>Remote auto-reconnect</label>
|
||||
<label class="switch">
|
||||
<input type="checkbox" id="appSettingsRemoteAutoReconnect">
|
||||
<span class="slider"></span>
|
||||
</label>
|
||||
<span class="form-hint">Automatically re-establish remote (SSH) sessions when the connection drops, reattaching to the durable remote tmux session (on by default; bounded backoff)</span>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label>Thinking Effort</label>
|
||||
<select id="appSettingsThinkingEffort" class="form-select">
|
||||
|
||||
@@ -82,6 +82,33 @@ Object.assign(CodemanApp.prototype, {
|
||||
}
|
||||
},
|
||||
|
||||
// Remote auto-reconnect (COD-108)
|
||||
_onRemoteSessionReconnected(data) {
|
||||
const id = this.getShortId(data.sessionId);
|
||||
this.showToast(`Remote session ${id} reconnected`, 'success');
|
||||
},
|
||||
|
||||
_onRemoteReconnectExhausted(data) {
|
||||
const sessionId = data.sessionId;
|
||||
const id = this.getShortId(sessionId);
|
||||
// Auto-reconnect gave up after the bounded backoff. Surface a manual
|
||||
// "Reconnect" affordance that re-triggers the attach path (force-reload the
|
||||
// session, which re-runs the create/attach flow against the durable remote).
|
||||
this.showToast(`Remote session ${id} dropped — auto-reconnect gave up`, 'error', {
|
||||
duration: 15000,
|
||||
action: {
|
||||
label: 'Reconnect',
|
||||
onClick: () => {
|
||||
if (this.sessions && this.sessions.has(sessionId)) {
|
||||
this.selectSession(sessionId, { forceReload: true });
|
||||
} else {
|
||||
this.showToast('Session no longer available', 'warning');
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
|
||||
// Bash tools
|
||||
_onBashToolStart(data) {
|
||||
|
||||
@@ -359,6 +359,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
document.getElementById('appSettingsAgentTeams').checked = settings.agentTeamsEnabled ?? false;
|
||||
document.getElementById('appSettingsClaudeModel').value = settings.claudeModel ?? '';
|
||||
document.getElementById('appSettingsOpusContext1m').checked = settings.opusContext1mEnabled ?? false;
|
||||
document.getElementById('appSettingsRemoteAutoReconnect').checked = settings.remoteAutoReconnect ?? true;
|
||||
document.getElementById('appSettingsThinkingEffort').value = settings.thinkingEffort ?? '';
|
||||
// CPU Priority settings
|
||||
const niceSettings = settings.nice || {};
|
||||
@@ -1453,6 +1454,7 @@ Object.assign(CodemanApp.prototype, {
|
||||
agentTeamsEnabled: document.getElementById('appSettingsAgentTeams').checked,
|
||||
claudeModel: document.getElementById('appSettingsClaudeModel').value,
|
||||
opusContext1mEnabled: document.getElementById('appSettingsOpusContext1m').checked,
|
||||
remoteAutoReconnect: document.getElementById('appSettingsRemoteAutoReconnect').checked,
|
||||
thinkingEffort: document.getElementById('appSettingsThinkingEffort').value,
|
||||
// CPU Priority settings
|
||||
nice: {
|
||||
@@ -1770,6 +1772,8 @@ Object.assign(CodemanApp.prototype, {
|
||||
showPlanUsageLimits: false,
|
||||
showAttachmentsButton: false,
|
||||
showRedrawButton: false,
|
||||
// Remote auto-reconnect (COD-108) — on by default
|
||||
remoteAutoReconnect: true,
|
||||
// Input
|
||||
gestureControlEnabled: false,
|
||||
// Feature toggles - keep tracking on even on mobile
|
||||
|
||||
@@ -494,6 +494,10 @@ export const SettingsUpdateSchema = z
|
||||
/** Model for new Claude sessions (e.g. "claude-fable-5[1m]", "opus[1m]"); takes precedence over opusContext1mEnabled */
|
||||
claudeModel: z.string().max(50).optional(),
|
||||
opusContext1mEnabled: z.boolean().optional(),
|
||||
// COD-108 remote-session auto-reconnect kill-switch (default ON). When false,
|
||||
// the TmuxManager watcher does nothing — dropped remote sessions are NOT
|
||||
// auto-reattached.
|
||||
remoteAutoReconnect: z.boolean().optional(),
|
||||
thinkingEffort: z.string().max(20).optional(),
|
||||
// UI visibility
|
||||
showFontControls: z.boolean().optional(),
|
||||
|
||||
@@ -351,6 +351,22 @@ export class WebServer extends EventEmitter {
|
||||
this.broadcast(SseEvent.MuxStatsUpdated, sessions);
|
||||
});
|
||||
|
||||
// COD-108 — remote-session auto-reconnect. The TmuxManager watcher detects a
|
||||
// dead remote pane and emits `remoteSessionDropped`; the session owner (here)
|
||||
// reassembles the respawn options and reattaches via Session.reattachRemote()
|
||||
// (D1: the watcher does NOT reassemble options itself). On success we reset
|
||||
// the watcher's backoff; on failure the backoff schedules the next attempt.
|
||||
this.mux.on('remoteSessionDropped', (data) => {
|
||||
const { sessionId, attempt } = data as { sessionId: string; attempt: number };
|
||||
this.broadcast(SseEvent.RemoteSessionDropped, { sessionId, attempt });
|
||||
void this.handleRemoteSessionDropped(sessionId);
|
||||
});
|
||||
this.mux.on('remoteReconnectExhausted', (data) => {
|
||||
const { sessionId } = data as { sessionId: string };
|
||||
console.warn(`[Server] Remote auto-reconnect exhausted for session ${sessionId}`);
|
||||
this.broadcast(SseEvent.RemoteReconnectExhausted, { sessionId });
|
||||
});
|
||||
|
||||
// Set up subagent watcher listeners
|
||||
this.setupSubagentWatcherListeners();
|
||||
this.setupWorkflowRunWatcherListeners();
|
||||
@@ -2384,6 +2400,13 @@ export class WebServer extends EventEmitter {
|
||||
(this.mux as { startMouseModeSync: (ms?: number) => void }).startMouseModeSync();
|
||||
}
|
||||
|
||||
// COD-108 — start the remote-session auto-reconnect watcher (tmux only).
|
||||
// Always-on (D3) with a `remoteAutoReconnect` kill-switch the watcher reads
|
||||
// each tick. Start even with no sessions — remote sessions may arrive later.
|
||||
if ('startRemoteReconnectWatcher' in this.mux) {
|
||||
(this.mux as { startRemoteReconnectWatcher: (ms?: number) => void }).startRemoteReconnectWatcher();
|
||||
}
|
||||
|
||||
if (dead.length > 0) {
|
||||
console.log(`[Server] Cleaned up ${dead.length} dead mux session(s)`);
|
||||
}
|
||||
@@ -2392,6 +2415,41 @@ export class WebServer extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* COD-108 — handle a `remoteSessionDropped` emit from the watcher: reattach
|
||||
* the dropped remote session and report the outcome back to the watcher so it
|
||||
* can reset/advance its backoff. Re-running the idempotent remote command
|
||||
* REATTACHES the durable remote tmux session (does NOT recreate it).
|
||||
*/
|
||||
private async handleRemoteSessionDropped(sessionId: string): Promise<void> {
|
||||
const session = this.sessions.get(sessionId);
|
||||
// No live Session object (e.g. detached/restored-but-not-attached) — nothing
|
||||
// to drive the reattach; report failure so the watcher backs off and retries.
|
||||
if (!session) {
|
||||
this.noteRemoteReconnect(sessionId, false);
|
||||
return;
|
||||
}
|
||||
let ok = false;
|
||||
try {
|
||||
ok = await session.reattachRemote();
|
||||
} catch (err) {
|
||||
console.error(`[Server] Remote reattach failed for ${sessionId}:`, err);
|
||||
ok = false;
|
||||
}
|
||||
this.noteRemoteReconnect(sessionId, ok);
|
||||
if (ok) {
|
||||
this.persistSessionState(session);
|
||||
this.broadcast(SseEvent.RemoteSessionReconnected, { sessionId });
|
||||
}
|
||||
}
|
||||
|
||||
/** Forward a reattach outcome to the TmuxManager watcher (resets/clears backoff). */
|
||||
private noteRemoteReconnect(sessionId: string, success: boolean): void {
|
||||
if ('noteRemoteReconnect' in this.mux) {
|
||||
(this.mux as { noteRemoteReconnect: (id: string, ok: boolean) => void }).noteRemoteReconnect(sessionId, success);
|
||||
}
|
||||
}
|
||||
|
||||
private initOrchestratorLoop(): import('../orchestrator-loop.js').OrchestratorLoop {
|
||||
if (this._orchestratorLoop) return this._orchestratorLoop;
|
||||
|
||||
|
||||
@@ -150,6 +150,15 @@ export const MuxDied = 'mux:died' as const;
|
||||
/** tmux session stats refreshed. */
|
||||
export const MuxStatsUpdated = 'mux:statsUpdated' as const;
|
||||
|
||||
// ─── Remote auto-reconnect (COD-108) ─────────────────────────────────────────
|
||||
|
||||
/** A remote session's local ssh pane died; an auto-reconnect attempt is starting. */
|
||||
export const RemoteSessionDropped = 'remote:sessionDropped' as const;
|
||||
/** A dropped remote session was successfully re-established (reattached). */
|
||||
export const RemoteSessionReconnected = 'remote:sessionReconnected' as const;
|
||||
/** Auto-reconnect gave up after the bounded backoff cap — manual reconnect needed. */
|
||||
export const RemoteReconnectExhausted = 'remote:reconnectExhausted' as const;
|
||||
|
||||
// ─── Respawn ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Respawn loop started for a session. */
|
||||
@@ -435,6 +444,11 @@ export const SseEvent = {
|
||||
MuxDied,
|
||||
MuxStatsUpdated,
|
||||
|
||||
// Remote auto-reconnect (COD-108)
|
||||
RemoteSessionDropped,
|
||||
RemoteSessionReconnected,
|
||||
RemoteReconnectExhausted,
|
||||
|
||||
// Respawn
|
||||
RespawnStarted,
|
||||
RespawnStopped,
|
||||
|
||||
Reference in New Issue
Block a user