mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-08 00:19:42 +02:00
COD-118 feat: circuit breaker bounding repeated non-zero interactive-PTY exits
Defense-in-depth after COD-115. If the interactive PTY exits non-zero repeatedly within a short window, recovery/reconnect paths recreate it indefinitely (COD-115 saw 114 'exited with code: 1' events + orphans). - New pure InteractivePtyExitBreaker (session-pty-exit-breaker.ts): injectable time, sliding window, clean-exit resets counter, stays tripped until reset(). Defaults: threshold 5, window 10s. - Session records each interactive PTY exit in the breaker; on trip it flips _status to 'error', sets _respawnBlocked, emits respawnBreakerTripped. startInteractive() refuses to respawn while blocked, so all recovery/reconnect callers stop looping uniformly. - Explicit user restart (POST /api/sessions/:id/interactive) calls resetRespawnBreaker() so intentional restarts are never blocked. - New SSE event session:respawnBreakerTripped wired in sse-events.ts + constants.js (registries in sync) + session-listener-wiring.ts; minimal diagnostic toast in app.js. - Tests: test/respawn-pty-breaker.test.ts (pure trip/reset/window + MockSession session-level trip/reset). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
3c0e6286f6
commit
286cf0768d
@@ -82,6 +82,7 @@ import {
|
||||
import { SessionAutoOps } from './session-auto-ops.js';
|
||||
import { detectUsageLimitPause } from './usage-limit-patterns.js';
|
||||
import { SessionTaskCache } from './session-task-cache.js';
|
||||
import { InteractivePtyExitBreaker } from './session-pty-exit-breaker.js';
|
||||
import { parseAttachmentMagicLinks } from './attachment-magic.js';
|
||||
import {
|
||||
sanitizeAttachmentHistory,
|
||||
@@ -288,6 +289,13 @@ export class Session extends EventEmitter {
|
||||
private _pid: number | null = null;
|
||||
private _status: SessionStatus = 'idle';
|
||||
private _currentTaskId: string | null = null;
|
||||
|
||||
// COD-118: bound repeated non-zero interactive-PTY exits. Recorded in the
|
||||
// interactive PTY onExit handler; when it trips, the session flips to 'error'
|
||||
// and startInteractive() refuses to respawn until an explicit user restart
|
||||
// calls resetRespawnBreaker(). Defense-in-depth over the COD-115 crash-loop.
|
||||
private readonly _ptyExitBreaker = new InteractivePtyExitBreaker();
|
||||
private _respawnBlocked = false;
|
||||
// Use BufferAccumulator for hot-path buffers to reduce GC pressure
|
||||
private _terminalBuffer = new BufferAccumulator(MAX_TERMINAL_BUFFER_SIZE, TERMINAL_BUFFER_TRIM_SIZE);
|
||||
private _textOutput = new BufferAccumulator(MAX_TEXT_OUTPUT_SIZE, TEXT_OUTPUT_TRIM_SIZE);
|
||||
@@ -1256,6 +1264,16 @@ export class Session extends EventEmitter {
|
||||
throw new Error('Session already has a running process');
|
||||
}
|
||||
|
||||
// COD-118: if the PTY exit breaker has tripped (repeated non-zero exits in a
|
||||
// short window), refuse to respawn. This is the uniform choke point that stops
|
||||
// automatic recovery/reconnect callers from re-creating a crash-looping PTY.
|
||||
// An explicit user restart clears it via resetRespawnBreaker().
|
||||
if (this._respawnBlocked) {
|
||||
throw new Error(
|
||||
'Respawn blocked: interactive PTY exited non-zero too many times in a short window (circuit breaker tripped). Restart the session to clear it.'
|
||||
);
|
||||
}
|
||||
|
||||
this._resetBuffers();
|
||||
|
||||
const modeLabel = getModeLabel(this.mode);
|
||||
@@ -1496,6 +1514,9 @@ export class Session extends EventEmitter {
|
||||
|
||||
this.ptyProcess.onExit(({ exitCode }) => {
|
||||
console.log('[Session] Interactive PTY exited with code:', exitCode);
|
||||
// COD-118: record the exit in the circuit breaker BEFORE status bookkeeping.
|
||||
// A clean (0) exit resets the counter; rapid non-zero repeats trip it.
|
||||
const breakerResult = this._ptyExitBreaker.recordExit(exitCode, Date.now());
|
||||
this.ptyProcess = null;
|
||||
this._pid = null;
|
||||
this._status = 'idle';
|
||||
@@ -1523,10 +1544,38 @@ export class Session extends EventEmitter {
|
||||
if (this._muxSession && this._mux) {
|
||||
this._mux.setAttached(this.id, false);
|
||||
}
|
||||
// COD-118: if the breaker tripped, surface an error state and block the NEXT
|
||||
// respawn so recovery/reconnect callers stop looping. Still emit 'exit' below
|
||||
// for normal cleanup. Cleared by an explicit user restart (resetRespawnBreaker()).
|
||||
if (breakerResult.tripped && !this._respawnBlocked) {
|
||||
this._respawnBlocked = true;
|
||||
this._status = 'error';
|
||||
console.error(
|
||||
`[Session] PTY exit circuit breaker tripped for ${this.id} (${breakerResult.count} non-zero exits within window); blocking respawn.`
|
||||
);
|
||||
this.emit('respawnBreakerTripped', { count: breakerResult.count });
|
||||
}
|
||||
this.emit('exit', exitCode);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the interactive-PTY exit circuit breaker (COD-118).
|
||||
*
|
||||
* Called on an EXPLICIT, user-initiated (re)start so an intentional restart is
|
||||
* never blocked by a prior crash-loop trip. Automatic recovery/reconnect paths
|
||||
* must NOT call this — that's the whole point of the breaker.
|
||||
*/
|
||||
resetRespawnBreaker(): void {
|
||||
this._ptyExitBreaker.reset();
|
||||
this._respawnBlocked = false;
|
||||
}
|
||||
|
||||
/** Whether the interactive-PTY exit circuit breaker is currently tripped (COD-118). */
|
||||
get respawnBlocked(): boolean {
|
||||
return this._respawnBlocked;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process expensive parsers (ANSI strip, Ralph, bash tool, token, CLI info, task descriptions).
|
||||
* Called on a throttled schedule (every EXPENSIVE_PROCESS_INTERVAL_MS) instead of on every
|
||||
|
||||
Reference in New Issue
Block a user