fix(cases): bound linked-workspace path probes so an unreachable mount cannot freeze the server

A linked case can live on a network mount. When that mount goes away, a
hard mount makes stat() wait indefinitely, and the existsSync() probes in
the case routes and the workspace hook/statusline helpers ran on the event
loop, so a single GET /api/cases (or a session create in that workspace)
froze the whole web server until the mount came back.

Add boundedPathExists() (src/utils/bounded-path-probe.ts): an async stat
that answers "absent" after 1.5 s, shares one in-flight probe per path,
remembers a timed-out path until its stat finally settles, and refuses to
start new probes while two stalled ones still hold libuv threadpool
workers. Route the read-side probes in case-routes.ts and hooks-config.ts
through it. The settings writers in hooks-config.ts use an async lstat
that treats only ENOENT as missing, so an unreachable workspace is never
mistaken for an empty one and has its settings recreated.
This commit is contained in:
Saqeb Akhter
2026-10-04 20:08:05 -04:00
committed by Aamer Akhter
parent ffaa5ee80c
commit 00b935abe6
5 changed files with 274 additions and 23 deletions
+103
View File
@@ -0,0 +1,103 @@
/**
* @fileoverview Tests for boundedPathExists (src/utils/bounded-path-probe.ts):
* a stat() that never settles (an unreachable hard network mount) must not hold
* the caller past the timeout, must not be re-issued while it is still pending,
* and must not let stalled probes pile up in libuv's shared threadpool.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
vi.mock('node:fs/promises', () => ({
default: { stat: vi.fn() },
}));
import fs from 'node:fs/promises';
import { boundedPathExists, PROBE_TIMEOUT_MS } from '../src/utils/bounded-path-probe.js';
const stat = vi.mocked(fs.stat);
/**
* Make the first stat() of each given path hang until released (the mount is
* down); every later stat, and every other path, answers "exists".
*/
function hangOn(paths: string[]): Map<string, () => void> {
const releases = new Map<string, () => void>();
stat.mockImplementation((path) => {
if (!paths.includes(String(path)) || releases.has(String(path))) return Promise.resolve({} as never);
return new Promise((resolve) => {
releases.set(String(path), () => resolve({} as never));
});
});
return releases;
}
afterEach(() => {
vi.useRealTimers();
stat.mockReset();
});
describe('boundedPathExists', () => {
it('reports an existing path as present and a missing one as absent', async () => {
stat.mockImplementation(async (path) => {
if (String(path) === '/present') return {} as never;
throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
});
expect(await boundedPathExists('/present')).toBe(true);
expect(await boundedPathExists('/missing')).toBe(false);
});
it('answers false after the timeout when stat never settles, and does not re-probe until it does', async () => {
vi.useFakeTimers();
const releases = hangOn(['/mnt/stalled/case']);
const result = boundedPathExists('/mnt/stalled/case');
await vi.advanceTimersByTimeAsync(PROBE_TIMEOUT_MS);
expect(await result).toBe(false);
// A second caller gets the cached verdict immediately, without another stat.
expect(await boundedPathExists('/mnt/stalled/case')).toBe(false);
expect(stat).toHaveBeenCalledTimes(1);
// Once the mount answers, the path is probed afresh.
releases.get('/mnt/stalled/case')!();
await vi.advanceTimersByTimeAsync(0);
expect(await boundedPathExists('/mnt/stalled/case')).toBe(true);
expect(stat).toHaveBeenCalledTimes(2);
});
it('shares one in-flight stat between concurrent callers of the same path', async () => {
const releases = hangOn(['/slow']);
const a = boundedPathExists('/slow');
const b = boundedPathExists('/slow');
expect(stat).toHaveBeenCalledTimes(1);
releases.get('/slow')!();
expect(await a).toBe(true);
expect(await b).toBe(true);
});
it('does not give concurrent healthy probes a false negative', async () => {
stat.mockImplementation(async () => ({}) as never);
const results = await Promise.all(['/a', '/b', '/c', '/d', '/e'].map((p) => boundedPathExists(p)));
expect(results).toEqual([true, true, true, true, true]);
});
it('stops issuing new stats once stalled probes would tie up the threadpool', async () => {
vi.useFakeTimers();
const releases = hangOn(['/mnt/stalled/one', '/mnt/stalled/two']);
const first = boundedPathExists('/mnt/stalled/one');
const second = boundedPathExists('/mnt/stalled/two');
await vi.advanceTimersByTimeAsync(PROBE_TIMEOUT_MS);
expect(await first).toBe(false);
expect(await second).toBe(false);
// Both slots are held by stats that never returned: refuse a third.
expect(await boundedPathExists('/healthy/three')).toBe(false);
expect(stat).toHaveBeenCalledTimes(2);
// Once the stalled stats settle, probing resumes normally.
releases.forEach((release) => release());
await vi.advanceTimersByTimeAsync(0);
expect(await boundedPathExists('/healthy/three')).toBe(true);
expect(stat).toHaveBeenCalledTimes(3);
});
});
+44
View File
@@ -35,6 +35,7 @@ vi.mock('node:fs', async (importOriginal) => {
vi.mock('node:fs/promises', () => ({
default: {
stat: vi.fn(),
readdir: vi.fn(async () => []),
readFile: vi.fn(async () => {
const err = new Error('ENOENT') as NodeJS.ErrnoException;
@@ -74,6 +75,7 @@ const mockedReaddirSync = vi.mocked(readdirSync);
const mockedReaddir = vi.mocked(fs.readdir);
const mockedReadFile = vi.mocked(fs.readFile);
const mockedWriteFile = vi.mocked(fs.writeFile);
const mockedStat = vi.mocked(fs.stat);
const mockedCheckRemoteTmux = vi.mocked(checkRemoteTmuxAvailable);
interface CaseRouteHarness {
@@ -127,6 +129,12 @@ describe('case-routes', () => {
// Default: existsSync returns false, readFile throws ENOENT
mockedExistsSync.mockReturnValue(false);
mockedReadFile.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }));
// Async stat (the bounded path probe) follows the mocked existsSync, so a
// test that sets up a path's presence via existsSync drives both the same way.
mockedStat.mockImplementation(async (path) => {
if (mockedExistsSync(path)) return { isDirectory: () => true } as never;
throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
});
});
afterEach(async () => {
@@ -210,6 +218,42 @@ describe('case-routes', () => {
// Should have both regular and linked cases
expect(body.data.length).toBeGreaterThanOrEqual(1);
});
it('still answers promptly when a linked case sits on an unreachable mount', async () => {
// A hard network mount that went away: a synchronous probe blocks the
// thread (simulated by a busy-wait), and an async stat never settles.
const stalledPath = '/mnt/unreachable/linked-nfs';
const BLOCK_MS = 4_000;
mockedReaddir.mockResolvedValue([] as never);
mockedReadFile.mockResolvedValueOnce(JSON.stringify({ 'linked-nfs': stalledPath }) as never);
mockedExistsSync.mockImplementation((p) => {
if (String(p) !== stalledPath) return false;
const until = Date.now() + BLOCK_MS;
while (Date.now() < until) {
// spin: the event loop is frozen for as long as the mount does not answer
}
return true;
});
let release: (() => void) | undefined;
mockedStat.mockImplementation((p) => {
if (String(p) !== stalledPath) {
return Promise.reject(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }));
}
return new Promise((resolve) => {
release = () => resolve({ isDirectory: () => true } as never);
});
});
const started = Date.now();
const res = await harness.app.inject({ method: 'GET', url: '/api/cases' });
const elapsed = Date.now() - started;
release?.();
expect(res.statusCode).toBe(200);
expect(elapsed).toBeLessThan(BLOCK_MS - 1_000);
// The unreachable case is left out rather than holding the list hostage.
expect(JSON.parse(res.body).data).toEqual([]);
});
});
describe('remote host and remote case routes', () => {