COD-118 feat: circuit breaker bounding repeated non-zero interactive-PTY exits

Defense-in-depth after COD-115. If the interactive PTY exits non-zero
repeatedly within a short window, recovery/reconnect paths recreate it
indefinitely (COD-115 saw 114 'exited with code: 1' events + orphans).

- New pure InteractivePtyExitBreaker (session-pty-exit-breaker.ts):
  injectable time, sliding window, clean-exit resets counter, stays
  tripped until reset(). Defaults: threshold 5, window 10s.
- Session records each interactive PTY exit in the breaker; on trip it
  flips _status to 'error', sets _respawnBlocked, emits
  respawnBreakerTripped. startInteractive() refuses to respawn while
  blocked, so all recovery/reconnect callers stop looping uniformly.
- Explicit user restart (POST /api/sessions/:id/interactive) calls
  resetRespawnBreaker() so intentional restarts are never blocked.
- New SSE event session:respawnBreakerTripped wired in sse-events.ts +
  constants.js (registries in sync) + session-listener-wiring.ts;
  minimal diagnostic toast in app.js.
- Tests: test/respawn-pty-breaker.test.ts (pure trip/reset/window +
  MockSession session-level trip/reset).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Aamer Akhter
2026-07-09 11:49:23 -04:00
co-authored by Claude Opus 4.8
parent 3c0e6286f6
commit 286cf0768d
8 changed files with 382 additions and 0 deletions
+49
View File
@@ -82,6 +82,7 @@ import {
import { SessionAutoOps } from './session-auto-ops.js';
import { detectUsageLimitPause } from './usage-limit-patterns.js';
import { SessionTaskCache } from './session-task-cache.js';
import { InteractivePtyExitBreaker } from './session-pty-exit-breaker.js';
import { parseAttachmentMagicLinks } from './attachment-magic.js';
import {
sanitizeAttachmentHistory,
@@ -288,6 +289,13 @@ export class Session extends EventEmitter {
private _pid: number | null = null;
private _status: SessionStatus = 'idle';
private _currentTaskId: string | null = null;
// COD-118: bound repeated non-zero interactive-PTY exits. Recorded in the
// interactive PTY onExit handler; when it trips, the session flips to 'error'
// and startInteractive() refuses to respawn until an explicit user restart
// calls resetRespawnBreaker(). Defense-in-depth over the COD-115 crash-loop.
private readonly _ptyExitBreaker = new InteractivePtyExitBreaker();
private _respawnBlocked = false;
// Use BufferAccumulator for hot-path buffers to reduce GC pressure
private _terminalBuffer = new BufferAccumulator(MAX_TERMINAL_BUFFER_SIZE, TERMINAL_BUFFER_TRIM_SIZE);
private _textOutput = new BufferAccumulator(MAX_TEXT_OUTPUT_SIZE, TEXT_OUTPUT_TRIM_SIZE);
@@ -1256,6 +1264,16 @@ export class Session extends EventEmitter {
throw new Error('Session already has a running process');
}
// COD-118: if the PTY exit breaker has tripped (repeated non-zero exits in a
// short window), refuse to respawn. This is the uniform choke point that stops
// automatic recovery/reconnect callers from re-creating a crash-looping PTY.
// An explicit user restart clears it via resetRespawnBreaker().
if (this._respawnBlocked) {
throw new Error(
'Respawn blocked: interactive PTY exited non-zero too many times in a short window (circuit breaker tripped). Restart the session to clear it.'
);
}
this._resetBuffers();
const modeLabel = getModeLabel(this.mode);
@@ -1496,6 +1514,9 @@ export class Session extends EventEmitter {
this.ptyProcess.onExit(({ exitCode }) => {
console.log('[Session] Interactive PTY exited with code:', exitCode);
// COD-118: record the exit in the circuit breaker BEFORE status bookkeeping.
// A clean (0) exit resets the counter; rapid non-zero repeats trip it.
const breakerResult = this._ptyExitBreaker.recordExit(exitCode, Date.now());
this.ptyProcess = null;
this._pid = null;
this._status = 'idle';
@@ -1523,10 +1544,38 @@ export class Session extends EventEmitter {
if (this._muxSession && this._mux) {
this._mux.setAttached(this.id, false);
}
// COD-118: if the breaker tripped, surface an error state and block the NEXT
// respawn so recovery/reconnect callers stop looping. Still emit 'exit' below
// for normal cleanup. Cleared by an explicit user restart (resetRespawnBreaker()).
if (breakerResult.tripped && !this._respawnBlocked) {
this._respawnBlocked = true;
this._status = 'error';
console.error(
`[Session] PTY exit circuit breaker tripped for ${this.id} (${breakerResult.count} non-zero exits within window); blocking respawn.`
);
this.emit('respawnBreakerTripped', { count: breakerResult.count });
}
this.emit('exit', exitCode);
});
}
/**
* Clear the interactive-PTY exit circuit breaker (COD-118).
*
* Called on an EXPLICIT, user-initiated (re)start so an intentional restart is
* never blocked by a prior crash-loop trip. Automatic recovery/reconnect paths
* must NOT call this — that's the whole point of the breaker.
*/
resetRespawnBreaker(): void {
this._ptyExitBreaker.reset();
this._respawnBlocked = false;
}
/** Whether the interactive-PTY exit circuit breaker is currently tripped (COD-118). */
get respawnBlocked(): boolean {
return this._respawnBlocked;
}
/**
* Process expensive parsers (ANSI strip, Ralph, bash tool, token, CLI info, task descriptions).
* Called on a throttled schedule (every EXPENSIVE_PROCESS_INTERVAL_MS) instead of on every