fix: avoid event-loop stalls from synchronous tmux/ps calls (#100)

The stats collector (~2s) and mouse-mode sync (5s) ran execSync (pgrep/ps/
list-panes, 5s timeout each) per session on the server's single thread,
blocking the event loop. With several sessions or a momentarily slow tmux this
froze port 3000 for seconds-to-tens-of-seconds while the process stayed alive
and other ports were unaffected — self-healing, so it never restarted and the
60s loopback healthcheck missed it. Convert these hot-path calls to execAsync.

Also add an always-on event-loop lag monitor (utils/event-loop-monitor.ts) that
logs stalls >=1s to the web log, so this otherwise-invisible class of incident
leaves a quantified, timestamped trace.

Co-authored-by: Teigen <teigen@TeigendeMac-mini.local>
This commit is contained in:
Tenggan Zhang
2026-06-01 19:32:24 +02:00
committed by GitHub
co-authored by Teigen
parent 257695ff8e
commit 06f9ff6d9c
4 changed files with 112 additions and 35 deletions
+12 -1
View File
@@ -94,7 +94,8 @@ import {
type ImageDetectedEvent,
DEFAULT_NICE_CONFIG,
} from '../types.js';
import { CleanupManager, KeyedDebouncer, StaleExpirationMap } from '../utils/index.js';
import { CleanupManager, KeyedDebouncer, StaleExpirationMap, startEventLoopMonitor } from '../utils/index.js';
import type { EventLoopMonitorHandle } from '../utils/index.js';
import { MAX_CONCURRENT_SESSIONS, MAX_SSE_CLIENTS } from '../config/map-limits.js';
import { SseEvent } from './sse-events.js';
import type { ScheduledRun } from './ports/index.js';
@@ -231,6 +232,7 @@ export class WebServer extends EventEmitter {
private teamWatcher: TeamWatcher = new TeamWatcher();
private _orchestratorLoop: import('../orchestrator-loop.js').OrchestratorLoop | null = null;
private _pasteImageGcStop: (() => void) | null = null;
private _eventLoopMonitor: EventLoopMonitorHandle | null = null;
private teamWatcherHandlers: {
teamCreated: (config: unknown) => void;
teamUpdated: (config: unknown) => void;
@@ -1551,6 +1553,10 @@ export class WebServer extends EventEmitter {
// older than 7 days from each live session's .claude-images/ hourly.
if (!this.testMode) {
this._pasteImageGcStop = startPasteImageGc({ sessions: this.sessions });
// Surface event-loop stalls (e.g. a slow synchronous tmux/ps call) so the
// intermittent ":3000 briefly unreachable, process never restarts" class of
// incident leaves a quantified log line instead of vanishing silently.
this._eventLoopMonitor = startEventLoopMonitor();
}
await this.app.listen({ port: this.port, host: '0.0.0.0' });
@@ -1911,6 +1917,11 @@ export class WebServer extends EventEmitter {
this._pasteImageGcStop = null;
}
if (this._eventLoopMonitor) {
this._eventLoopMonitor.stop();
this._eventLoopMonitor = null;
}
// Dispose all managed timers (intervals + resettable timeouts)
this.cleanup.dispose();