mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-10 01:09:43 +02:00
fix(tiles): cap each tile's live-output backlog and recover dropped output
The server applies no WebSocket backpressure (16 KB / 8 ms batches, no bufferedAmount check), and a tile wrote every live frame straight into xterm. A flood a tile could not parse as fast (a shell tile running cat on a huge log) piled up in xterm's own write queue without bound, on a main thread up to six tiles share, until xterm's WriteBuffer threw past 50M code units; onmessage's empty catch then dropped every frame silently and nothing recaptured the screen. The primary pane caps its queues and drops then recaptures (_onSessionTerminal, _scheduleDroppedOutputRecovery). Each tile now writes live output through _writeLive: - unparsed code units are counted, each write's callback counting its own back down; frames held behind a replay (_liveQueue) count too; - the budget is TerminalTile.LIVE_BACKLOG_BUDGET, 4 MiB, deliberately not the primary pane's 128 KB: that caps its own rAF-paced queues, while xterm itself paces a tile, and a tight cap would trip on ordinary bursts and blank-and-reload the tile over and over; - past it a frame is dropped, the tile stops writing onto the hole, and one refresh is scheduled, debounced and bounded by the primary pane's own rule (CodemanDroppedOutput: 2 s, DROP_RECOVERY_MAX_ATTEMPTS, never retried after a deadline abort). It is an ordinary refresh, so single-flight, bounded by lines=/tail= and paced by the grid's TileLoadQueue. The flag clears once a capture taken after the last dropped frame has replayed; - a write that throws is the same drop, never a "malformed frame"; - past the bound the flag is released, so a tile is never left frozen; - a reconnect starts the accounting over (an epoch makes callbacks from before it count nothing) and drops a pending recovery, since its own refresh replaces the screen; destroy() cancels it. The live-queue flush after a pull or a refresh goes through the same path, so a throwing write there cannot skip the load's marker and trailing refresh either. Tests (input harness, real constants and fake timers): the default budget lets a 1 MiB unparsed burst through, parsed bytes stop counting, a trip stops writing and ONE debounced refresh recaptures, a write throw takes the same recovery, a hole in the held queue is recovered by another refresh, bounded retries then release, no retry after a deadline, a reconnect resets the count, and destroy cancels. The fake xterm can now hold and release parses and throw on a write. Live writes now carry a callback, so the unit tests match them on the data argument (a `.not.toHaveBeenCalledWith(data)` would otherwise pass for nothing). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -159,7 +159,19 @@
|
|||||||
this._historyPullAt = 0;
|
this._historyPullAt = 0;
|
||||||
this._historyPullUseless = false;
|
this._historyPullUseless = false;
|
||||||
this._liveQueue = null;
|
this._liveQueue = null;
|
||||||
|
this._liveQueueBytes = 0;
|
||||||
this._markerOwed = false;
|
this._markerOwed = false;
|
||||||
|
// Live-output flow control (_writeLive, TerminalTile.LIVE_BACKLOG_BUDGET):
|
||||||
|
// code units written into this xterm and not yet parsed (each write's
|
||||||
|
// callback counts its own back down, unless a reset bumped `_liveEpoch`
|
||||||
|
// since), whether output was dropped and not yet recovered, when the last
|
||||||
|
// frame was dropped, and the debounced, bounded recovery refresh.
|
||||||
|
this._liveInFlight = 0;
|
||||||
|
this._liveEpoch = 0;
|
||||||
|
this._liveDropped = false;
|
||||||
|
this._liveDropAt = 0;
|
||||||
|
this._dropRecoveryTimer = null;
|
||||||
|
this._dropRecoveryAttempt = 0;
|
||||||
this._onWheel = null;
|
this._onWheel = null;
|
||||||
// `{ ws, lastRecvAt }`, registered with the app's input-socket map while
|
// `{ ws, lastRecvAt }`, registered with the app's input-socket map while
|
||||||
// this pane's socket is open, so the exactly-once input queue delivers this
|
// this pane's socket is open, so the exactly-once input queue delivers this
|
||||||
@@ -507,7 +519,12 @@
|
|||||||
this._lastSentDims = null;
|
this._lastSentDims = null;
|
||||||
this._registerInputSocket();
|
this._registerInputSocket();
|
||||||
this._sendResize();
|
this._sendResize();
|
||||||
if (reconnected) this._refreshBuffer();
|
if (reconnected) {
|
||||||
|
// The gap already cost output, and the refresh below replaces the
|
||||||
|
// screen, so live-output accounting starts over with it.
|
||||||
|
this._resetLiveFlow();
|
||||||
|
this._refreshBuffer();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Opens a replacement socket now instead of waiting out the backoff (for an
|
// Opens a replacement socket now instead of waiting out the backoff (for an
|
||||||
@@ -762,7 +779,7 @@
|
|||||||
// TERMINAL_TAIL_SIZE) gets the pull's short budget; an unbounded
|
// TERMINAL_TAIL_SIZE) gets the pull's short budget; an unbounded
|
||||||
// capture (the split's Pane B, up to 32 MB) keeps the request's.
|
// capture (the split's Pane B, up to 32 MB) keeps the request's.
|
||||||
capturedAt = performance.now();
|
capturedAt = performance.now();
|
||||||
this._liveQueue = [];
|
this._openLiveQueue();
|
||||||
if (shell || this.boundedLoad) armDeadline(HISTORY_PULL_TIMEOUT_MS);
|
if (shell || this.boundedLoad) armDeadline(HISTORY_PULL_TIMEOUT_MS);
|
||||||
}
|
}
|
||||||
payload = (await res.json())?.data ?? {};
|
payload = (await res.json())?.data ?? {};
|
||||||
@@ -795,6 +812,9 @@
|
|||||||
clearTimeout(abortTimer);
|
clearTimeout(abortTimer);
|
||||||
this._loadAbort = null;
|
this._loadAbort = null;
|
||||||
this._loadRunning = false;
|
this._loadRunning = false;
|
||||||
|
// Before the flush: a refresh that recovered dropped output lets its
|
||||||
|
// held frames through.
|
||||||
|
if (refresh) this._settleDropRecovery({ replayed, capturedAt, timedOut: !!controller?.signal?.aborted });
|
||||||
// Held frames before the marker, so the marker stays the last thing on
|
// Held frames before the marker, so the marker stays the last thing on
|
||||||
// screen (see _pullHistory()).
|
// screen (see _pullHistory()).
|
||||||
this._flushLiveQueue(replayed ? capturedAt : 0);
|
this._flushLiveQueue(replayed ? capturedAt : 0);
|
||||||
@@ -846,24 +866,149 @@
|
|||||||
// tmux took it, so a frame arriving mid-replay is held with its arrival time
|
// tmux took it, so a frame arriving mid-replay is held with its arrival time
|
||||||
// and written behind the snapshot by that load's _flushLiveQueue() (the
|
// and written behind the snapshot by that load's _flushLiveQueue() (the
|
||||||
// primary pane's _finishBufferLoad `since` rule), never underneath it.
|
// primary pane's _finishBufferLoad `since` rule), never underneath it.
|
||||||
|
// Held frames count against the same budget as unparsed ones: a pull or a
|
||||||
|
// refresh holds output for up to its body budget, and a flood meanwhile
|
||||||
|
// must not grow the queue without bound either.
|
||||||
_onLiveOutput(data) {
|
_onLiveOutput(data) {
|
||||||
if (this._liveQueue) this._liveQueue.push({ at: performance.now(), data });
|
if (!data) return;
|
||||||
else this.terminal?.write(data);
|
if (this._liveQueue) {
|
||||||
|
if (this._liveQueueBytes + data.length > TerminalTile.LIVE_BACKLOG_BUDGET) {
|
||||||
|
this._noteLiveDrop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this._liveQueueBytes += data.length;
|
||||||
|
this._liveQueue.push({ at: performance.now(), data });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this._writeLive(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
_openLiveQueue() {
|
||||||
|
this._liveQueue = [];
|
||||||
|
this._liveQueueBytes = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Releases the frames a load held (_liveQueue) and closes the queue. After a
|
// Releases the frames a load held (_liveQueue) and closes the queue. After a
|
||||||
// replay only those that arrived after the capture are news (`cutoff`, the
|
// replay only those that arrived after the capture are news (`cutoff`, the
|
||||||
// response's arrival; earlier ones are already in it); with no replay
|
// response's arrival; earlier ones are already in it); with no replay
|
||||||
// (`cutoff` 0) every one is.
|
// (`cutoff` 0) every one is. Through _writeLive(), so they are counted (and
|
||||||
|
// a write that throws cannot skip the load's marker and trailing refresh).
|
||||||
_flushLiveQueue(cutoff) {
|
_flushLiveQueue(cutoff) {
|
||||||
const queued = this._liveQueue ?? [];
|
const queued = this._liveQueue ?? [];
|
||||||
this._liveQueue = null;
|
this._liveQueue = null;
|
||||||
|
this._liveQueueBytes = 0;
|
||||||
for (const entry of queued) {
|
for (const entry of queued) {
|
||||||
if (entry.at < cutoff) continue;
|
if (entry.at < cutoff) continue;
|
||||||
this.terminal?.write(entry.data);
|
this._writeLive(entry.data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Writes one live frame into this xterm, under flow control. The server
|
||||||
|
// applies no backpressure (16 KB / 8 ms batches, never a bufferedAmount
|
||||||
|
// check), so a flood a tile cannot parse as fast as it arrives (a shell
|
||||||
|
// tile running `cat` on a huge log, `yes`) used to pile up in xterm's own
|
||||||
|
// write queue without bound, on a main thread six tiles share, until
|
||||||
|
// xterm's WriteBuffer throws past 50M code units and the frames were
|
||||||
|
// silently lost in onmessage's catch. The primary pane caps its queues
|
||||||
|
// and drops then recaptures (_onSessionTerminal, app.js); this is the
|
||||||
|
// tile's equivalent. Past TerminalTile.LIVE_BACKLOG_BUDGET unparsed, a
|
||||||
|
// frame is dropped and the tile stops writing until a refresh recaptures
|
||||||
|
// the screen (_scheduleDropRecovery): every byte after a hole is written
|
||||||
|
// onto a screen out of step with the PTY, which that refresh replaces
|
||||||
|
// anyway. A write that throws is the same drop, never a malformed frame.
|
||||||
|
_writeLive(data) {
|
||||||
|
const terminal = this.terminal;
|
||||||
|
if (!terminal || this._destroyed || !data) return;
|
||||||
|
if (this._liveDropped) {
|
||||||
|
this._noteLiveDrop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const n = data.length;
|
||||||
|
if (this._liveInFlight + n > TerminalTile.LIVE_BACKLOG_BUDGET) {
|
||||||
|
this._noteLiveDrop();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const epoch = this._liveEpoch;
|
||||||
|
this._liveInFlight += n;
|
||||||
|
try {
|
||||||
|
terminal.write(data, () => {
|
||||||
|
if (epoch === this._liveEpoch) this._liveInFlight -= n;
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
if (epoch === this._liveEpoch) this._liveInFlight -= n;
|
||||||
|
this._noteLiveDrop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A live frame was dropped. Marks the tile out of step and arms ONE
|
||||||
|
// recovery; later drops only move the stamp the recovery has to beat.
|
||||||
|
_noteLiveDrop() {
|
||||||
|
this._liveDropAt = performance.now();
|
||||||
|
if (this._liveDropped) return;
|
||||||
|
this._liveDropped = true;
|
||||||
|
this._scheduleDropRecovery();
|
||||||
|
}
|
||||||
|
|
||||||
|
// The primary pane's dropped-output recovery (_scheduleDroppedOutputRecovery,
|
||||||
|
// app.js), aimed at this tile: debounced by DROP_RECOVERY_DELAY_MS so a
|
||||||
|
// sustained flood collapses into one attempt, and run as an ordinary
|
||||||
|
// refresh, which is single-flight, bounded (`lines=`/`tail=`) and waits its
|
||||||
|
// turn in the grid's load queue. _settleDropRecovery() decides what the
|
||||||
|
// refresh it starts achieved.
|
||||||
|
_scheduleDropRecovery() {
|
||||||
|
if (this._dropRecoveryTimer || this._destroyed) return;
|
||||||
|
const delay = global.CodemanDroppedOutput?.DROP_RECOVERY_DELAY_MS ?? 2000;
|
||||||
|
this._dropRecoveryTimer = setTimeout(() => {
|
||||||
|
this._dropRecoveryTimer = null;
|
||||||
|
if (this._destroyed || !this._liveDropped) return;
|
||||||
|
this._dropRecoveryAttempt++;
|
||||||
|
this._refreshBuffer();
|
||||||
|
}, delay);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A refresh finished while output was marked dropped. Recovered when its
|
||||||
|
// replay's capture was taken after the last dropped frame (the response's
|
||||||
|
// arrival, the cutoff every load uses): output flows again. Otherwise one
|
||||||
|
// more attempt, bounded by the primary pane's rule
|
||||||
|
// (shouldRetryDroppedOutputRecovery: DROP_RECOVERY_MAX_ATTEMPTS, and never
|
||||||
|
// after a capture cut off at its deadline, a stalled link); past that the
|
||||||
|
// flag is released so the tile is never left frozen, and it writes on, out
|
||||||
|
// of step, as every tile did before this existed.
|
||||||
|
_settleDropRecovery({ replayed, capturedAt, timedOut }) {
|
||||||
|
if (!this._liveDropped || this._destroyed) return;
|
||||||
|
if (replayed && capturedAt >= this._liveDropAt) {
|
||||||
|
this._liveDropped = false;
|
||||||
|
this._dropRecoveryAttempt = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Another try is already on its way: the debounce, or a trailing refresh.
|
||||||
|
if (this._dropRecoveryTimer || this._bufferRefreshPending) return;
|
||||||
|
const retry =
|
||||||
|
global.CodemanDroppedOutput?.shouldRetryDroppedOutputRecovery?.({
|
||||||
|
repainted: false,
|
||||||
|
timedOut,
|
||||||
|
attempt: Math.max(0, this._dropRecoveryAttempt - 1),
|
||||||
|
stillActive: true,
|
||||||
|
}) === true;
|
||||||
|
if (retry) {
|
||||||
|
this._scheduleDropRecovery();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this._liveDropped = false;
|
||||||
|
this._dropRecoveryAttempt = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Starts live-output accounting over (a reconnect, destroy): write callbacks
|
||||||
|
// still pending from before carry the old epoch and count nothing.
|
||||||
|
_resetLiveFlow() {
|
||||||
|
this._liveEpoch++;
|
||||||
|
this._liveInFlight = 0;
|
||||||
|
this._liveDropped = false;
|
||||||
|
this._dropRecoveryAttempt = 0;
|
||||||
|
clearTimeout(this._dropRecoveryTimer);
|
||||||
|
this._dropRecoveryTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
// The server's `{t:'c'}` frame, which is a refresh, not a wipe. Its one
|
// The server's `{t:'c'}` frame, which is a refresh, not a wipe. Its one
|
||||||
// emitter (Session.startInteractive, session.ts) sends it once a fresh Claude
|
// emitter (Session.startInteractive, session.ts) sends it once a fresh Claude
|
||||||
// pane first shows its prompt: the server has just trimmed its own buffer and
|
// pane first shows its prompt: the server has just trimmed its own buffer and
|
||||||
@@ -1099,7 +1244,7 @@
|
|||||||
// Opened only now: a frame from before the response is either replaced by
|
// Opened only now: a frame from before the response is either replaced by
|
||||||
// the capture or written unchanged, so holding it for the round trip
|
// the capture or written unchanged, so holding it for the round trip
|
||||||
// would buy nothing and freeze the pane for as long as the fetch took.
|
// would buy nothing and freeze the pane for as long as the fetch took.
|
||||||
this._liveQueue = [];
|
this._openLiveQueue();
|
||||||
const payload = (await res.json())?.data;
|
const payload = (await res.json())?.data;
|
||||||
clearTimeout(abortTimer);
|
clearTimeout(abortTimer);
|
||||||
const buffer = payload?.terminalBuffer;
|
const buffer = payload?.terminalBuffer;
|
||||||
@@ -1307,6 +1452,9 @@
|
|||||||
this.mountEl?.removeEventListener('click', this._onClick);
|
this.mountEl?.removeEventListener('click', this._onClick);
|
||||||
this._onClick = null;
|
this._onClick = null;
|
||||||
}
|
}
|
||||||
|
// A disposed xterm never runs its write callbacks, and a pending
|
||||||
|
// recovery would refresh a pane nobody can see.
|
||||||
|
this._resetLiveFlow();
|
||||||
// Page keys still waiting for their flush go nowhere: the pane is gone.
|
// Page keys still waiting for their flush go nowhere: the pane is gone.
|
||||||
clearTimeout(this._scrollFlushTimer);
|
clearTimeout(this._scrollFlushTimer);
|
||||||
this._scrollFlushTimer = null;
|
this._scrollFlushTimer = null;
|
||||||
@@ -1332,6 +1480,15 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Code units of live output a pane lets sit unparsed in its xterm (or held
|
||||||
|
// behind a replay) before it drops a frame and recaptures (_writeLive). Not
|
||||||
|
// the primary pane's 128 KB: that caps its own rAF-paced queues, about two
|
||||||
|
// frames of them, while here xterm itself is the pacer, a burst normally
|
||||||
|
// parses within a frame or two, and a tight cap would trip on ordinary
|
||||||
|
// bursts and blank-and-reload the tile over and over. A few MB keeps a flood
|
||||||
|
// far below xterm's 50M code-unit throw and bounds each tile's memory.
|
||||||
|
TerminalTile.LIVE_BACKLOG_BUDGET = 4 * 1024 * 1024;
|
||||||
|
|
||||||
// The marker a pane writes when its socket drops: a transient drop says it is
|
// The marker a pane writes when its socket drops: a transient drop says it is
|
||||||
// reconnecting; a permanent stop says why, keyed by close code. All start
|
// reconnecting; a permanent stop says why, keyed by close code. All start
|
||||||
// with `[disconnected` so a reader (and a test) can tell any of them apart
|
// with `[disconnected` so a reader (and a test) can tell any of them apart
|
||||||
|
|||||||
@@ -149,9 +149,23 @@ export class FakeTerminal {
|
|||||||
}
|
}
|
||||||
registerLinkProvider() {}
|
registerLinkProvider() {}
|
||||||
writes: string[] = [];
|
writes: string[] = [];
|
||||||
/** Set by a test: write callbacks never run, as on a disposed xterm. */
|
/**
|
||||||
|
* Set by a test: write callbacks do not run, as while xterm is still parsing
|
||||||
|
* (or never, on a disposed xterm). They wait in `heldParses` until `parse()`.
|
||||||
|
*/
|
||||||
holdParse = false;
|
holdParse = false;
|
||||||
|
heldParses: Array<() => void> = [];
|
||||||
|
/** xterm catches up: runs every write callback held so far, in order. */
|
||||||
|
parse() {
|
||||||
|
for (const cb of this.heldParses.splice(0)) cb();
|
||||||
|
}
|
||||||
|
/** Set by a test: the next write of exactly this data throws, as xterm's WriteBuffer does past 50M. */
|
||||||
|
throwOnWrite: string | null = null;
|
||||||
write(data: string, cb?: () => void) {
|
write(data: string, cb?: () => void) {
|
||||||
|
if (this.throwOnWrite !== null && data === this.throwOnWrite) {
|
||||||
|
this.throwOnWrite = null;
|
||||||
|
throw new Error('write data discarded, use flow control to avoid losing data');
|
||||||
|
}
|
||||||
// An empty write puts nothing on screen; the replay queues one only to hear
|
// An empty write puts nothing on screen; the replay queues one only to hear
|
||||||
// (its callback) that everything before it has been parsed.
|
// (its callback) that everything before it has been parsed.
|
||||||
if (data) this.writes.push(data);
|
if (data) this.writes.push(data);
|
||||||
@@ -162,7 +176,9 @@ export class FakeTerminal {
|
|||||||
this.lineCount += data.slice(reset === -1 ? 0 : reset + 2).split('\n').length - 1;
|
this.lineCount += data.slice(reset === -1 ? 0 : reset + 2).split('\n').length - 1;
|
||||||
this.settleRows();
|
this.settleRows();
|
||||||
}
|
}
|
||||||
if (!this.holdParse) cb?.();
|
if (!cb) return;
|
||||||
|
if (this.holdParse) this.heldParses.push(cb);
|
||||||
|
else cb();
|
||||||
}
|
}
|
||||||
clear() {
|
clear() {
|
||||||
this.writes.push('<CLEAR>');
|
this.writes.push('<CLEAR>');
|
||||||
|
|||||||
@@ -802,6 +802,252 @@ describe('TerminalTile claims the keyboard for the app-level shortcuts', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('TerminalTile live-output flow control: a flood cannot pile up in xterm', () => {
|
||||||
|
// The server applies no backpressure, and a tile used to write every live
|
||||||
|
// frame straight into xterm: a flood it could not parse as fast piled up in
|
||||||
|
// xterm's own queue without bound, until xterm's WriteBuffer threw past 50M
|
||||||
|
// code units and onmessage's catch silently lost the frames. Now unparsed
|
||||||
|
// (and held) output is counted per tile; past the budget a frame is dropped,
|
||||||
|
// the tile stops writing onto the hole, and one debounced, bounded refresh
|
||||||
|
// recaptures the screen (the primary pane's _scheduleDroppedOutputRecovery).
|
||||||
|
const TileStatics = TerminalTile as unknown as { LIVE_BACKLOG_BUDGET: number };
|
||||||
|
const defaultBudget = TileStatics.LIVE_BACKLOG_BUDGET;
|
||||||
|
afterEach(() => {
|
||||||
|
TileStatics.LIVE_BACKLOG_BUDGET = defaultBudget;
|
||||||
|
delete windowStub.AbortController;
|
||||||
|
});
|
||||||
|
const out = (ws: FakeSocket, d: string) => ws.receive({ t: 'o', d });
|
||||||
|
const inFlight = (tile: Tile) => (tile as unknown as { _liveInFlight: number })._liveInFlight;
|
||||||
|
function serve(terminalBuffer: string) {
|
||||||
|
fetchMock.mockClear();
|
||||||
|
fetchMock.mockImplementation(async () => ({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: async () => ({ data: { terminalBuffer } }),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
it("the budget is a few MB, not the primary pane's 128 KB: a 1 MiB burst xterm has not parsed yet is still written", async () => {
|
||||||
|
expect(defaultBudget).toBeGreaterThanOrEqual(2 * 1024 * 1024);
|
||||||
|
expect(defaultBudget).toBeLessThanOrEqual(16 * 1024 * 1024);
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
serve('');
|
||||||
|
term.holdParse = true;
|
||||||
|
|
||||||
|
out(ws, 'z'.repeat(1024 * 1024));
|
||||||
|
|
||||||
|
expect(term.writes.at(-1)).toHaveLength(1024 * 1024);
|
||||||
|
await vi.advanceTimersByTimeAsync(10_000);
|
||||||
|
expect(fetchMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('counts output until xterm has parsed it, so parsed bytes no longer hold the budget', async () => {
|
||||||
|
TileStatics.LIVE_BACKLOG_BUDGET = 100;
|
||||||
|
const { tile, ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
term.holdParse = true;
|
||||||
|
|
||||||
|
out(ws, 'a'.repeat(40));
|
||||||
|
out(ws, 'b'.repeat(40));
|
||||||
|
expect(inFlight(tile)).toBe(80);
|
||||||
|
term.parse();
|
||||||
|
expect(inFlight(tile)).toBe(0);
|
||||||
|
out(ws, 'c'.repeat(90));
|
||||||
|
|
||||||
|
expect(term.writes.slice(-3)).toEqual(['a'.repeat(40), 'b'.repeat(40), 'c'.repeat(90)]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('stops writing past the budget, and ONE debounced refresh recaptures the screen', async () => {
|
||||||
|
TileStatics.LIVE_BACKLOG_BUDGET = 100;
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
serve('recovered screen');
|
||||||
|
term.holdParse = true; // xterm falls behind
|
||||||
|
|
||||||
|
out(ws, 'a'.repeat(60));
|
||||||
|
out(ws, 'b'.repeat(60)); // 120 unparsed: past the budget, dropped
|
||||||
|
out(ws, 'c');
|
||||||
|
term.parse(); // xterm catches up, but the stream already has a hole:
|
||||||
|
out(ws, 'd'); // nothing more is written onto it
|
||||||
|
expect(term.writes.filter((w) => /^[abcd]/.test(w))).toEqual(['a'.repeat(60)]);
|
||||||
|
expect(fetchMock).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
term.holdParse = false;
|
||||||
|
await vi.advanceTimersByTimeAsync(1999);
|
||||||
|
expect(fetchMock).not.toHaveBeenCalled(); // debounced, like the primary pane's
|
||||||
|
await vi.advanceTimersByTimeAsync(1);
|
||||||
|
await settle();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(fetchMock.mock.calls[0][0]).toBe('/api/sessions/s-tile/terminal?full=1');
|
||||||
|
expect(term.writes.slice(-2)).toEqual(['\x1bc', 'recovered screen']);
|
||||||
|
|
||||||
|
out(ws, 'live again');
|
||||||
|
expect(term.writes.at(-1)).toBe('live again');
|
||||||
|
await vi.advanceTimersByTimeAsync(10_000);
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1); // one recovery for the whole burst
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a write xterm refuses (its 50M throw) is a drop that schedules the same recovery, never a swallowed 'malformed frame'", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
serve('recovered screen');
|
||||||
|
term.throwOnWrite = 'boom';
|
||||||
|
|
||||||
|
out(ws, 'boom');
|
||||||
|
out(ws, 'after the hole');
|
||||||
|
expect(term.writes).not.toContain('after the hole');
|
||||||
|
|
||||||
|
await vi.advanceTimersByTimeAsync(2000);
|
||||||
|
await settle();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(term.writes.slice(-2)).toEqual(['\x1bc', 'recovered screen']);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('frames held behind a replay count against the budget too, and a hole there is recovered by another refresh', async () => {
|
||||||
|
TileStatics.LIVE_BACKLOG_BUDGET = 100;
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
fetchMock.mockClear();
|
||||||
|
let answer!: (body: string) => void;
|
||||||
|
fetchMock.mockImplementationOnce(async () => ({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: () =>
|
||||||
|
new Promise((resolveBody) => {
|
||||||
|
answer = (terminalBuffer) => resolveBody({ data: { terminalBuffer } });
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
fetchMock.mockImplementation(async () => ({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
json: async () => ({ data: { terminalBuffer: 'second capture' } }),
|
||||||
|
}));
|
||||||
|
|
||||||
|
ws.receive({ t: 'r' });
|
||||||
|
await settle(); // the headers landed: from here frames are held
|
||||||
|
out(ws, 'x'.repeat(60));
|
||||||
|
out(ws, 'y'.repeat(60)); // the held queue would pass the budget: dropped
|
||||||
|
answer('first capture');
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
// The capture predates the hole, so nothing after it is written onto it.
|
||||||
|
expect(term.writes.slice(-2)).toEqual(['\x1bc', 'first capture']);
|
||||||
|
expect(term.writes).not.toContain('y'.repeat(60));
|
||||||
|
await vi.advanceTimersByTimeAsync(2000);
|
||||||
|
await settle();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||||
|
expect(term.writes.slice(-2)).toEqual(['\x1bc', 'second capture']);
|
||||||
|
out(ws, 'live again');
|
||||||
|
expect(term.writes.at(-1)).toBe('live again');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a recovery that keeps failing is retried a bounded number of times, then lets live output through again', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
fetchMock.mockClear();
|
||||||
|
fetchMock.mockImplementation(async () => {
|
||||||
|
throw new Error('offline');
|
||||||
|
});
|
||||||
|
term.throwOnWrite = 'boom';
|
||||||
|
|
||||||
|
out(ws, 'boom');
|
||||||
|
out(ws, 'held back');
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
await vi.advanceTimersByTimeAsync(2000);
|
||||||
|
await settle();
|
||||||
|
}
|
||||||
|
|
||||||
|
const { DROP_RECOVERY_MAX_ATTEMPTS } = windowStub.CodemanDroppedOutput as { DROP_RECOVERY_MAX_ATTEMPTS: number };
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(DROP_RECOVERY_MAX_ATTEMPTS);
|
||||||
|
expect(term.writes).not.toContain('held back');
|
||||||
|
out(ws, 'flowing');
|
||||||
|
expect(term.writes.at(-1)).toBe('flowing'); // never left frozen
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a recovery cut off at its deadline is not retried (a stalled link), and lets live output through again', async () => {
|
||||||
|
windowStub.AbortController = AbortController;
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
fetchMock.mockClear();
|
||||||
|
fetchMock.mockImplementation(
|
||||||
|
(_url: string, init?: { signal?: AbortSignal }) =>
|
||||||
|
new Promise((_resolveFetch, rejectFetch) => {
|
||||||
|
init?.signal?.addEventListener('abort', () => rejectFetch(new Error('aborted')));
|
||||||
|
})
|
||||||
|
);
|
||||||
|
term.throwOnWrite = 'boom';
|
||||||
|
|
||||||
|
out(ws, 'boom');
|
||||||
|
await vi.advanceTimersByTimeAsync(2000);
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
await vi.advanceTimersByTimeAsync(45_000); // the full-capture budget runs out
|
||||||
|
await settle();
|
||||||
|
|
||||||
|
out(ws, 'flowing');
|
||||||
|
expect(term.writes.at(-1)).toBe('flowing');
|
||||||
|
await vi.advanceTimersByTimeAsync(20_000);
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a reconnect starts the count over, and callbacks from before it count nothing', async () => {
|
||||||
|
TileStatics.LIVE_BACKLOG_BUDGET = 100;
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { tile, ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
term.holdParse = true;
|
||||||
|
out(ws, 'a'.repeat(60));
|
||||||
|
expect(inFlight(tile)).toBe(60);
|
||||||
|
|
||||||
|
ws.drop(1006);
|
||||||
|
await vi.advanceTimersByTimeAsync(300);
|
||||||
|
FakeSocket.instances.at(-1)!.open(); // reconnected: its refresh replaces the screen
|
||||||
|
expect(inFlight(tile)).toBe(0);
|
||||||
|
term.parse(); // the old write's callback lands after the reset
|
||||||
|
expect(inFlight(tile)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a reconnect drops a pending recovery: its own refresh replaces the screen', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
serve('');
|
||||||
|
term.throwOnWrite = 'boom';
|
||||||
|
out(ws, 'boom');
|
||||||
|
|
||||||
|
ws.drop(1006);
|
||||||
|
await vi.advanceTimersByTimeAsync(300);
|
||||||
|
const ws2 = FakeSocket.instances.at(-1)!;
|
||||||
|
ws2.open();
|
||||||
|
await settle();
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1); // the reconnect's refresh
|
||||||
|
out(ws2, 'flowing');
|
||||||
|
expect(term.writes.at(-1)).toBe('flowing');
|
||||||
|
await vi.advanceTimersByTimeAsync(5000);
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('destroy() cancels a pending recovery', async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
const { tile, ws, term } = await connectTile(makeApp());
|
||||||
|
ws.open();
|
||||||
|
fetchMock.mockClear();
|
||||||
|
term.throwOnWrite = 'boom';
|
||||||
|
out(ws, 'boom');
|
||||||
|
|
||||||
|
tile.destroy();
|
||||||
|
await vi.advanceTimersByTimeAsync(5000);
|
||||||
|
|
||||||
|
expect(fetchMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe('the server coming back kicks Pane B', () => {
|
describe('the server coming back kicks Pane B', () => {
|
||||||
it("handleInit's reconnect branch asks the split pane's tile to reconnect without waiting out its backoff", () => {
|
it("handleInit's reconnect branch asks the split pane's tile to reconnect without waiting out its backoff", () => {
|
||||||
// handleInit needs a whole app to run, so the wiring is pinned by source;
|
// handleInit needs a whole app to run, so the wiring is pinned by source;
|
||||||
|
|||||||
@@ -185,6 +185,10 @@ const isMarker = (data: unknown) => typeof data === 'string' && data.includes('[
|
|||||||
*/
|
*/
|
||||||
const screenWrites = (pane: { terminal: FakeTerminal }) =>
|
const screenWrites = (pane: { terminal: FakeTerminal }) =>
|
||||||
pane.terminal.write.mock.calls.map((call) => call[0]).filter((data) => data !== '');
|
pane.terminal.write.mock.calls.map((call) => call[0]).filter((data) => data !== '');
|
||||||
|
// Live frames are written with a callback (the tile counts them until xterm
|
||||||
|
// has parsed them, _writeLive), so they are matched on the data argument
|
||||||
|
// alone, never with toHaveBeenCalledWith(data): that would also miss, and a
|
||||||
|
// `.not` on it would then pass for nothing.
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Holds xterm's write callbacks, as a real xterm still parsing a replay does:
|
* Holds xterm's write callbacks, as a real xterm still parsing a replay does:
|
||||||
@@ -738,7 +742,7 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
// keeps painting during the round trip), and the replay then replaces it.
|
// keeps painting during the round trip), and the replay then replaces it.
|
||||||
clock = 1;
|
clock = 1;
|
||||||
pane._onLiveOutput('early');
|
pane._onLiveOutput('early');
|
||||||
expect(term.write).toHaveBeenCalledWith('early');
|
expect(screenWrites(pane)).toContain('early');
|
||||||
await settle();
|
await settle();
|
||||||
|
|
||||||
// 200 rows (more than the pane holds, so it replays) of 400 columns each:
|
// 200 rows (more than the pane holds, so it replays) of 400 columns each:
|
||||||
@@ -748,12 +752,14 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
clock = 2; // the response arrives: this is the cutoff
|
clock = 2; // the response arrives: this is the cutoff
|
||||||
response.resolve(jsonResponse(bigReplay));
|
response.resolve(jsonResponse(bigReplay));
|
||||||
await settle();
|
await settle();
|
||||||
expect(xterm.held).toHaveLength(1);
|
// Two parses pending: 'early' (a live write, counted until xterm parses it)
|
||||||
|
// and the replay's end marker.
|
||||||
|
expect(xterm.held).toHaveLength(2);
|
||||||
|
|
||||||
// Arrives while the snapshot is still being parsed: must not land under it.
|
// Arrives while the snapshot is still being parsed: must not land under it.
|
||||||
clock = 3;
|
clock = 3;
|
||||||
pane._onLiveOutput('late');
|
pane._onLiveOutput('late');
|
||||||
expect(term.write).not.toHaveBeenCalledWith('late');
|
expect(screenWrites(pane)).not.toContain('late');
|
||||||
|
|
||||||
// The replay parsed, then the pull's own settle write before it scrolls.
|
// The replay parsed, then the pull's own settle write before it scrolls.
|
||||||
xterm.parse();
|
xterm.parse();
|
||||||
@@ -778,12 +784,12 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
pane._maybeLoadMoreHistory();
|
pane._maybeLoadMoreHistory();
|
||||||
await settle();
|
await settle();
|
||||||
pane._onLiveOutput('held');
|
pane._onLiveOutput('held');
|
||||||
expect(pane.terminal.write).not.toHaveBeenCalledWith('held');
|
expect(screenWrites(pane)).not.toContain('held');
|
||||||
held.release(rowsOf(30)); // nothing to gain: no replay
|
held.release(rowsOf(30)); // nothing to gain: no replay
|
||||||
await settle();
|
await settle();
|
||||||
|
|
||||||
// Nothing replaced the terminal, so the held frame is news.
|
// Nothing replaced the terminal, so the held frame is news.
|
||||||
expect(pane.terminal.write).toHaveBeenCalledWith('held');
|
expect(screenWrites(pane)).toContain('held');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('a failed fetch releases the flag and the queue, so live output flows again', async () => {
|
it('a failed fetch releases the flag and the queue, so live output flows again', async () => {
|
||||||
@@ -799,9 +805,9 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
|
|
||||||
expect(pane._bufferLoading).toBe(false);
|
expect(pane._bufferLoading).toBe(false);
|
||||||
expect(pane._liveQueue).toBeNull();
|
expect(pane._liveQueue).toBeNull();
|
||||||
expect(pane.terminal.write).toHaveBeenCalledWith('held');
|
expect(screenWrites(pane)).toContain('held');
|
||||||
pane._onLiveOutput('after');
|
pane._onLiveOutput('after');
|
||||||
expect(pane.terminal.write).toHaveBeenLastCalledWith('after');
|
expect(screenWrites(pane).at(-1)).toBe('after');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('a refresh frame during the pull runs once behind it', async () => {
|
it('a refresh frame during the pull runs once behind it', async () => {
|
||||||
@@ -921,7 +927,7 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
expect(pane._liveQueue).toBeNull();
|
expect(pane._liveQueue).toBeNull();
|
||||||
expect(pane.terminal).toBeNull();
|
expect(pane.terminal).toBeNull();
|
||||||
expect(term.write).not.toHaveBeenCalledWith('\x1bc');
|
expect(term.write).not.toHaveBeenCalledWith('\x1bc');
|
||||||
expect(term.write).not.toHaveBeenCalledWith('held');
|
expect(term.write.mock.calls.map((call) => call[0])).not.toContain('held');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('a pull whose request is aborted (the deadline) frees the pane', async () => {
|
it('a pull whose request is aborted (the deadline) frees the pane', async () => {
|
||||||
@@ -937,7 +943,7 @@ describe('TerminalTile scroll-to-top history pull', () => {
|
|||||||
|
|
||||||
expect(pane._bufferLoading).toBe(false);
|
expect(pane._bufferLoading).toBe(false);
|
||||||
expect(pane._liveQueue).toBeNull();
|
expect(pane._liveQueue).toBeNull();
|
||||||
expect(pane.terminal.write).toHaveBeenCalledWith('held');
|
expect(screenWrites(pane)).toContain('held');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('the wheel listener is capture-phase, and only a wheel UP can trigger a pull', async () => {
|
it('the wheel listener is capture-phase, and only a wheel UP can trigger a pull', async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user