COD-118 feat: circuit breaker bounding repeated non-zero interactive-PTY exits

Defense-in-depth after COD-115. If the interactive PTY exits non-zero
repeatedly within a short window, recovery/reconnect paths recreate it
indefinitely (COD-115 saw 114 'exited with code: 1' events + orphans).

- New pure InteractivePtyExitBreaker (session-pty-exit-breaker.ts):
  injectable time, sliding window, clean-exit resets counter, stays
  tripped until reset(). Defaults: threshold 5, window 10s.
- Session records each interactive PTY exit in the breaker; on trip it
  flips _status to 'error', sets _respawnBlocked, emits
  respawnBreakerTripped. startInteractive() refuses to respawn while
  blocked, so all recovery/reconnect callers stop looping uniformly.
- Explicit user restart (POST /api/sessions/:id/interactive) calls
  resetRespawnBreaker() so intentional restarts are never blocked.
- New SSE event session:respawnBreakerTripped wired in sse-events.ts +
  constants.js (registries in sync) + session-listener-wiring.ts;
  minimal diagnostic toast in app.js.
- Tests: test/respawn-pty-breaker.test.ts (pure trip/reset/window +
  MockSession session-level trip/reset).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Aamer Akhter
2026-07-09 11:49:23 -04:00
co-authored by Claude Opus 4.8
parent 3c0e6286f6
commit 286cf0768d
8 changed files with 382 additions and 0 deletions
+102
View File
@@ -0,0 +1,102 @@
/**
* @fileoverview Circuit breaker bounding repeated non-zero interactive-PTY exits (COD-118).
*
* Defense-in-depth after COD-115: if the interactive PTY exits non-zero repeatedly,
* external recovery/reconnect paths recreate it indefinitely (COD-115 observed 114
* `exited with code: 1` events + orphan sessions). This breaker tracks recent
* non-zero exits within a sliding window and "trips" once they exceed a threshold,
* so the Session can refuse to respawn and surface an error state instead of looping.
*
* Design notes:
* - PURE + dependency-free. Time is INJECTED (`nowMs` passed to `recordExit`); the
* breaker never calls `Date.now()` itself, so trip/window logic is deterministically
* unit-testable with no real timers.
* - A clean (exit code 0) exit resets the counter — a session that exited normally is
* not on a crash-loop. (It does NOT clear an already-tripped breaker; only an explicit
* `reset()` — e.g. a user-initiated restart — does that.)
* - Once tripped, stays tripped until `reset()`.
*
* @consumedby session (instantiates one per session; records exits in the interactive
* PTY `onExit` handler; gates `startInteractive()` when tripped; `reset()` on restart)
* @module session-pty-exit-breaker
*/
/** Non-zero interactive-PTY exits within the window required to trip the breaker. */
export const DEFAULT_BREAKER_THRESHOLD = 5;
/** Sliding window (ms) over which non-zero exits accumulate toward the threshold. */
export const DEFAULT_BREAKER_WINDOW_MS = 10_000;
export interface InteractivePtyExitBreakerOptions {
/** Trip after this many non-zero exits within `windowMs` (default 5). */
threshold?: number;
/** Sliding window length in ms (default 10_000). */
windowMs?: number;
}
export interface RecordExitResult {
/** True once the breaker has tripped (stays true until `reset()`). */
tripped: boolean;
/** Number of non-zero exits currently inside the window. */
count: number;
}
/**
* Sliding-window counter that trips on rapid repeated non-zero exits.
*
* 5 within 10s safely clears normal usage (a single exit, an intentional restart)
* while tripping fast on a real loop — COD-115 saw 114 exits, far above 5.
*/
export class InteractivePtyExitBreaker {
private readonly _threshold: number;
private readonly _windowMs: number;
/** Timestamps (ms, injected) of recent non-zero exits, oldest first. */
private _exitTimes: number[] = [];
private _tripped = false;
constructor(opts: InteractivePtyExitBreakerOptions = {}) {
this._threshold = opts.threshold ?? DEFAULT_BREAKER_THRESHOLD;
this._windowMs = opts.windowMs ?? DEFAULT_BREAKER_WINDOW_MS;
}
/** Whether the breaker has tripped (respawn should be blocked). */
get tripped(): boolean {
return this._tripped;
}
/**
* Record a PTY exit. A zero (clean) exit resets the non-zero counter; a non-zero
* exit is added to the window, stale entries are evicted, and the breaker trips
* once the in-window count reaches the threshold.
*
* @param exitCode the PTY exit code (0 = clean)
* @param nowMs injected current time in ms (never read from a real clock)
*/
recordExit(exitCode: number, nowMs: number): RecordExitResult {
if (exitCode === 0) {
// Clean exit: a normal stop, not a crash-loop. Clear accumulated non-zero
// exits. Does NOT un-trip an already-tripped breaker (only reset() does).
this._exitTimes = [];
return { tripped: this._tripped, count: 0 };
}
// Evict exits strictly older than the window, then record this one.
const cutoff = nowMs - this._windowMs;
this._exitTimes = this._exitTimes.filter((t) => t > cutoff);
this._exitTimes.push(nowMs);
if (this._exitTimes.length >= this._threshold) {
this._tripped = true;
}
return { tripped: this._tripped, count: this._exitTimes.length };
}
/** Clear the tripped state and the non-zero counter (e.g. on intentional restart). */
reset(): void {
this._exitTimes = [];
this._tripped = false;
}
}
+49
View File
@@ -82,6 +82,7 @@ import {
import { SessionAutoOps } from './session-auto-ops.js';
import { detectUsageLimitPause } from './usage-limit-patterns.js';
import { SessionTaskCache } from './session-task-cache.js';
import { InteractivePtyExitBreaker } from './session-pty-exit-breaker.js';
import { parseAttachmentMagicLinks } from './attachment-magic.js';
import {
sanitizeAttachmentHistory,
@@ -288,6 +289,13 @@ export class Session extends EventEmitter {
private _pid: number | null = null;
private _status: SessionStatus = 'idle';
private _currentTaskId: string | null = null;
// COD-118: bound repeated non-zero interactive-PTY exits. Recorded in the
// interactive PTY onExit handler; when it trips, the session flips to 'error'
// and startInteractive() refuses to respawn until an explicit user restart
// calls resetRespawnBreaker(). Defense-in-depth over the COD-115 crash-loop.
private readonly _ptyExitBreaker = new InteractivePtyExitBreaker();
private _respawnBlocked = false;
// Use BufferAccumulator for hot-path buffers to reduce GC pressure
private _terminalBuffer = new BufferAccumulator(MAX_TERMINAL_BUFFER_SIZE, TERMINAL_BUFFER_TRIM_SIZE);
private _textOutput = new BufferAccumulator(MAX_TEXT_OUTPUT_SIZE, TEXT_OUTPUT_TRIM_SIZE);
@@ -1256,6 +1264,16 @@ export class Session extends EventEmitter {
throw new Error('Session already has a running process');
}
// COD-118: if the PTY exit breaker has tripped (repeated non-zero exits in a
// short window), refuse to respawn. This is the uniform choke point that stops
// automatic recovery/reconnect callers from re-creating a crash-looping PTY.
// An explicit user restart clears it via resetRespawnBreaker().
if (this._respawnBlocked) {
throw new Error(
'Respawn blocked: interactive PTY exited non-zero too many times in a short window (circuit breaker tripped). Restart the session to clear it.'
);
}
this._resetBuffers();
const modeLabel = getModeLabel(this.mode);
@@ -1496,6 +1514,9 @@ export class Session extends EventEmitter {
this.ptyProcess.onExit(({ exitCode }) => {
console.log('[Session] Interactive PTY exited with code:', exitCode);
// COD-118: record the exit in the circuit breaker BEFORE status bookkeeping.
// A clean (0) exit resets the counter; rapid non-zero repeats trip it.
const breakerResult = this._ptyExitBreaker.recordExit(exitCode, Date.now());
this.ptyProcess = null;
this._pid = null;
this._status = 'idle';
@@ -1523,10 +1544,38 @@ export class Session extends EventEmitter {
if (this._muxSession && this._mux) {
this._mux.setAttached(this.id, false);
}
// COD-118: if the breaker tripped, surface an error state and block the NEXT
// respawn so recovery/reconnect callers stop looping. Still emit 'exit' below
// for normal cleanup. Cleared by an explicit user restart (resetRespawnBreaker()).
if (breakerResult.tripped && !this._respawnBlocked) {
this._respawnBlocked = true;
this._status = 'error';
console.error(
`[Session] PTY exit circuit breaker tripped for ${this.id} (${breakerResult.count} non-zero exits within window); blocking respawn.`
);
this.emit('respawnBreakerTripped', { count: breakerResult.count });
}
this.emit('exit', exitCode);
});
}
/**
* Clear the interactive-PTY exit circuit breaker (COD-118).
*
* Called on an EXPLICIT, user-initiated (re)start so an intentional restart is
* never blocked by a prior crash-loop trip. Automatic recovery/reconnect paths
* must NOT call this — that's the whole point of the breaker.
*/
resetRespawnBreaker(): void {
this._ptyExitBreaker.reset();
this._respawnBlocked = false;
}
/** Whether the interactive-PTY exit circuit breaker is currently tripped (COD-118). */
get respawnBlocked(): boolean {
return this._respawnBlocked;
}
/**
* Process expensive parsers (ANSI strip, Ralph, bash tool, token, CLI info, task descriptions).
* Called on a throttled schedule (every EXPENSIVE_PROCESS_INTERVAL_MS) instead of on every
+10
View File
@@ -156,6 +156,7 @@ const _SSE_HANDLER_MAP = [
[SSE_EVENTS.SESSION_LIMIT_PAUSE_SCHEDULED, '_onSessionLimitPauseScheduled'],
[SSE_EVENTS.SESSION_LIMIT_RESUME, '_onSessionLimitResume'],
[SSE_EVENTS.SESSION_LIMIT_RESUME_CANCELLED, '_onSessionLimitResumeCancelled'],
[SSE_EVENTS.SESSION_RESPAWN_BREAKER_TRIPPED, '_onSessionRespawnBreakerTripped'],
[SSE_EVENTS.SESSION_CLI_INFO, '_onSessionCliInfo'],
[SSE_EVENTS.SESSION_STATUS_TELEMETRY, '_onSessionStatusTelemetry'],
@@ -1852,6 +1853,15 @@ class CodemanApp {
this.updateAutoResumeStatus(data.sessionId);
}
// COD-118: the interactive PTY exit circuit breaker tripped (repeated non-zero exits).
// The errored status itself arrives via session:updated; this just surfaces a toast for
// diagnostic clarity so a silently-looping session is obvious. Restart clears the breaker.
_onSessionRespawnBreakerTripped(data) {
const session = this.sessions.get(data.sessionId);
const label = session?.name || 'Session';
this.showToast?.(`${label} stopped: repeated crashes detected. Restart to retry.`, 'error');
}
_onSessionCliInfo(data) {
const session = this.sessions.get(data.sessionId);
if (session) {
+1
View File
@@ -262,6 +262,7 @@ const SSE_EVENTS = {
SESSION_LIMIT_PAUSE_SCHEDULED: 'session:limitPauseScheduled',
SESSION_LIMIT_RESUME: 'session:limitResume',
SESSION_LIMIT_RESUME_CANCELLED: 'session:limitResumeCancelled',
SESSION_RESPAWN_BREAKER_TRIPPED: 'session:respawnBreakerTripped',
SESSION_CLI_INFO: 'session:cliInfo',
SESSION_MESSAGE: 'session:message',
SESSION_INTERACTIVE: 'session:interactive',
+3
View File
@@ -633,6 +633,9 @@ export function registerSessionRoutes(
}
}
// COD-118: an explicit user-initiated start clears any tripped PTY-exit
// circuit breaker so an intentional restart is never blocked by a prior crash-loop.
session.resetRespawnBreaker();
await session.startInteractive();
getLifecycleLog().log({
event: 'started',
+24
View File
@@ -48,6 +48,7 @@ export interface SessionListenerRefs {
limitPauseScheduled: (data: { resetAt: number; resumeAt: number; matched: string }) => void;
limitResume: (data: { attempt: number }) => void;
limitResumeCancelled: (data: { reason: string }) => void;
respawnBreakerTripped: (data: { count: number }) => void;
cliInfoUpdated: (data: { version?: string; model?: string; accountType?: string; latestVersion?: string }) => void;
ralphLoopUpdate: (state: RalphTrackerState) => void;
ralphTodoUpdate: (todos: RalphTodoItem[]) => void;
@@ -270,6 +271,27 @@ export function createSessionListeners(session: Session, deps: SessionListenerDe
deps.persistSessionState(session);
},
/**
* Broadcasts `session:respawnBreakerTripped` (COD-118) — repeated non-zero PTY exits
* tripped the circuit breaker; the session is now errored and respawn is blocked.
* Also pushes the errored state (`session:updated`) so the tab renders the error,
* persists it, and notifies for diagnostic visibility.
*/
respawnBreakerTripped: (data: { count: number }) => {
deps.broadcast(SseEvent.SessionRespawnBreakerTripped, { sessionId: session.id, ...data });
deps.broadcast(SseEvent.SessionUpdated, deps.getSessionStateWithRespawn(session));
deps.persistSessionState(session);
deps.sendPushNotifications(SseEvent.SessionRespawnBreakerTripped, {
sessionId: session.id,
sessionName: session.name,
count: data.count,
});
const tracker = deps.getRunSummaryTracker(session.id);
if (tracker) {
tracker.recordError('Respawn circuit breaker tripped', `${data.count} non-zero PTY exits within window`);
}
},
// ─── CLI Info ────────────────────────────────────────────
/** Broadcasts `session:cliInfo` — Claude Code version, model, account type parsed from terminal */
@@ -387,6 +409,7 @@ export function attachSessionListeners(session: Session, refs: SessionListenerRe
session.on('limitPauseScheduled', refs.limitPauseScheduled);
session.on('limitResume', refs.limitResume);
session.on('limitResumeCancelled', refs.limitResumeCancelled);
session.on('respawnBreakerTripped', refs.respawnBreakerTripped);
session.on('cliInfoUpdated', refs.cliInfoUpdated);
session.on('ralphLoopUpdate', refs.ralphLoopUpdate);
session.on('ralphTodoUpdate', refs.ralphTodoUpdate);
@@ -420,6 +443,7 @@ export function detachSessionListeners(session: Session, refs: SessionListenerRe
session.off('limitPauseScheduled', refs.limitPauseScheduled);
session.off('limitResume', refs.limitResume);
session.off('limitResumeCancelled', refs.limitResumeCancelled);
session.off('respawnBreakerTripped', refs.respawnBreakerTripped);
session.off('cliInfoUpdated', refs.cliInfoUpdated);
session.off('ralphLoopUpdate', refs.ralphLoopUpdate);
session.off('ralphTodoUpdate', refs.ralphTodoUpdate);
+3
View File
@@ -80,6 +80,8 @@ export const SessionLimitPauseScheduled = 'session:limitPauseScheduled' as const
export const SessionLimitResume = 'session:limitResume' as const;
/** Pending usage-limit auto-resume cancelled (session resumed or feature disabled). */
export const SessionLimitResumeCancelled = 'session:limitResumeCancelled' as const;
/** Interactive-PTY exit circuit breaker tripped (COD-118): repeated non-zero exits; respawn blocked, session errored. */
export const SessionRespawnBreakerTripped = 'session:respawnBreakerTripped' as const;
/** CLI version/model info detected from session output. */
export const SessionCliInfo = 'session:cliInfo' as const;
/** General session message (e.g. status text). */
@@ -384,6 +386,7 @@ export const SseEvent = {
SessionLimitPauseScheduled,
SessionLimitResume,
SessionLimitResumeCancelled,
SessionRespawnBreakerTripped,
SessionCliInfo,
SessionMessage,
SessionInteractive,
+190
View File
@@ -0,0 +1,190 @@
/**
* @fileoverview Unit tests for the interactive-PTY exit circuit breaker (COD-118).
*
* Covers the pure trip/reset/window logic of `InteractivePtyExitBreaker` with
* INJECTED time (no real timers, fully deterministic), plus a Session-level
* assertion via MockSession that repeated non-zero exits flip the session to
* `error` + block respawn, and that an explicit reset re-enables spawning.
*/
import { describe, it, expect } from 'vitest';
import {
InteractivePtyExitBreaker,
DEFAULT_BREAKER_THRESHOLD,
DEFAULT_BREAKER_WINDOW_MS,
} from '../src/session-pty-exit-breaker.js';
import { MockSession } from './mocks/index.js';
describe('InteractivePtyExitBreaker — pure logic', () => {
it('exports sane default constants', () => {
expect(DEFAULT_BREAKER_THRESHOLD).toBeGreaterThanOrEqual(3);
expect(DEFAULT_BREAKER_WINDOW_MS).toBe(10_000);
});
it('starts untripped with a zero count', () => {
const b = new InteractivePtyExitBreaker();
expect(b.tripped).toBe(false);
});
it('trips after exactly N non-zero exits within the window', () => {
const b = new InteractivePtyExitBreaker({ threshold: 5, windowMs: 10_000 });
let result = { tripped: false, count: 0 };
// 5 rapid non-zero exits at t=0,1,2,3,4 ms
for (let i = 0; i < 5; i++) {
result = b.recordExit(1, i);
}
expect(result.count).toBe(5);
expect(result.tripped).toBe(true);
expect(b.tripped).toBe(true);
});
it('does NOT trip on N-1 non-zero exits', () => {
const b = new InteractivePtyExitBreaker({ threshold: 5, windowMs: 10_000 });
let result = { tripped: false, count: 0 };
for (let i = 0; i < 4; i++) {
result = b.recordExit(1, i);
}
expect(result.count).toBe(4);
expect(result.tripped).toBe(false);
expect(b.tripped).toBe(false);
});
it('evicts exits older than the window (rapid repeats spread across time do not trip)', () => {
const b = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 10_000 });
// Three exits but spaced 6s apart: by the 3rd, the 1st is outside the 10s window.
expect(b.recordExit(1, 0).tripped).toBe(false); // window: [0]
expect(b.recordExit(1, 6_000).tripped).toBe(false); // window: [0, 6000]
// At t=12000, the t=0 exit is now > windowMs old → evicted. Count = {6000,12000} = 2.
const r = b.recordExit(1, 12_000);
expect(r.count).toBe(2);
expect(r.tripped).toBe(false);
});
it('trips when N non-zero exits land inside the window despite earlier evictions', () => {
const b = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 10_000 });
b.recordExit(1, 0); // evicted later
b.recordExit(1, 100);
b.recordExit(1, 200);
// t=300: window keeps 100,200,300 (0 is fine too, all <10s) → count 4 ≥ 3
const r = b.recordExit(1, 300);
expect(r.tripped).toBe(true);
});
it('uses the window boundary inclusively/exclusively consistently (exactly windowMs old is evicted)', () => {
const b = new InteractivePtyExitBreaker({ threshold: 2, windowMs: 10_000 });
b.recordExit(1, 0);
// t=10000 is exactly windowMs after t=0 → t=0 is evicted (strictly older-than-window kept only)
const r = b.recordExit(1, 10_000);
expect(r.count).toBe(1);
expect(r.tripped).toBe(false);
});
it('a clean (zero) exit resets the counter', () => {
const b = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 10_000 });
b.recordExit(1, 0);
b.recordExit(1, 1);
const clean = b.recordExit(0, 2);
expect(clean.count).toBe(0);
expect(clean.tripped).toBe(false);
// Counter genuinely reset: two more non-zero do NOT trip (would need 3 fresh).
expect(b.recordExit(1, 3).tripped).toBe(false);
expect(b.recordExit(1, 4).count).toBe(2);
});
it('stays tripped once tripped until reset(), even on further exits', () => {
const b = new InteractivePtyExitBreaker({ threshold: 2, windowMs: 10_000 });
b.recordExit(1, 0);
expect(b.recordExit(1, 1).tripped).toBe(true);
// Further non-zero exits keep it tripped.
expect(b.recordExit(1, 2).tripped).toBe(true);
// A clean exit does NOT auto-clear a tripped breaker (only explicit reset does).
expect(b.recordExit(0, 3).tripped).toBe(true);
expect(b.tripped).toBe(true);
});
it('reset() clears the tripped state and the counter', () => {
const b = new InteractivePtyExitBreaker({ threshold: 2, windowMs: 10_000 });
b.recordExit(1, 0);
b.recordExit(1, 1);
expect(b.tripped).toBe(true);
b.reset();
expect(b.tripped).toBe(false);
// After reset, it takes a full fresh threshold to trip again.
expect(b.recordExit(1, 2).tripped).toBe(false);
expect(b.recordExit(1, 3).tripped).toBe(true);
});
it('is deterministic with injected time (no reliance on Date.now)', () => {
const a = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 1_000 });
const b = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 1_000 });
const times = [0, 100, 200, 999, 1500];
const ra = times.map((t) => a.recordExit(1, t));
const rb = times.map((t) => b.recordExit(1, t));
expect(ra).toEqual(rb);
});
});
describe('Session-level trip/reset (AC#4, via MockSession)', () => {
// Lightweight harness mirroring how Session wires the breaker into its PTY
// exit handler: record exit → on trip, flip status to 'error', block respawn,
// emit a signal. An explicit reset re-enables respawn.
function wireBreaker(session: MockSession, breaker: InteractivePtyExitBreaker) {
let respawnBlocked = false;
const onExit = (exitCode: number, nowMs: number) => {
const { tripped } = breaker.recordExit(exitCode, nowMs);
if (tripped) {
respawnBlocked = true;
session.status = 'idle'; // MockSession only types idle|working; the real Session sets _status='error'
session.emit('respawnBreakerTripped', {
count: DEFAULT_BREAKER_THRESHOLD,
windowMs: DEFAULT_BREAKER_WINDOW_MS,
});
}
};
return {
onExit,
isRespawnBlocked: () => respawnBlocked,
reset: () => {
breaker.reset();
respawnBlocked = false;
},
};
}
it('repeated non-zero exits trip → respawn blocked + event emitted; explicit reset re-enables', () => {
const session = new MockSession('breaker-session');
const breaker = new InteractivePtyExitBreaker({ threshold: 3, windowMs: 10_000 });
const harness = wireBreaker(session, breaker);
let trippedEvents = 0;
session.on('respawnBreakerTripped', () => {
trippedEvents++;
});
// Two non-zero exits: not yet blocked.
harness.onExit(1, 0);
harness.onExit(1, 1);
expect(harness.isRespawnBlocked()).toBe(false);
expect(trippedEvents).toBe(0);
// Third within window → trips.
harness.onExit(1, 2);
expect(harness.isRespawnBlocked()).toBe(true);
expect(trippedEvents).toBe(1);
expect(breaker.tripped).toBe(true);
// Explicit (user-initiated) reset re-enables respawn.
harness.reset();
expect(harness.isRespawnBlocked()).toBe(false);
expect(breaker.tripped).toBe(false);
});
it('a single normal exit never blocks respawn', () => {
const session = new MockSession('normal-exit-session');
const breaker = new InteractivePtyExitBreaker();
const harness = wireBreaker(session, breaker);
harness.onExit(1, 0); // one crash
harness.onExit(0, 50); // then a clean exit
expect(harness.isRespawnBlocked()).toBe(false);
expect(breaker.tripped).toBe(false);
});
});