fix(cases): keep deleted workspaces deleted, cap pastCap, scope stalls to network mounts

- applyWorkspaceHooks: an "unknown" probe that is not near a stalled path
  (refused by the stall cap, or an unexpected stat error) no longer reads as
  "go ahead". It checks existence with pathExistsForWrite first, so a deleted
  workspace is not recreated by the mkdir -p in ensureCodemanHooks.
- pastCap gets a hard ceiling, PATH_PROBE_STALL_CEILING = UV_THREADPOOL_SIZE
  (default 4) minus one, so explicit requests against several dead paths can
  never take the last libuv worker. The bulk cap now defaults to one below the
  ceiling (2 with the default pool), leaving a slot for an explicit request.
- A stall widens to its mount only for network and FUSE filesystem types read
  from /proc/self/mounts; on a local mount (a path typed under a local /home
  that reaches a NAS through a symlink) it narrows to the stalled path.
- GET /api/cases/:name probes CLAUDE.md with pastCap, like the folder probe.
- Comment in config/path-probe.ts describes the mount-scoped stall.
This commit is contained in:
Aamer Akhter
2026-10-05 09:51:27 -04:00
parent d1bfbb4fcf
commit 9fa44109b8
7 changed files with 171 additions and 38 deletions
+21 -8
View File
@@ -10,8 +10,8 @@
*
* Both are env-overridable, in the same style as the other config modules. A slow
* but healthy mount (an sshfs that needs a couple of seconds on first touch) may want
* a longer timeout; a server started with a larger `UV_THREADPOOL_SIZE` can afford a
* higher stall cap.
* a longer timeout. The stall limits follow `UV_THREADPOOL_SIZE` on their own, so a
* server started with a larger pool gets a higher ceiling without further setup.
*
* @module config/path-probe
*/
@@ -26,10 +26,23 @@ function envInt(name: string, fallback: number, min: number, max: number): numbe
export const PATH_PROBE_TIMEOUT_MS = envInt('CODEMAN_PATH_PROBE_TIMEOUT_MS', 1_500, 100, 60_000);
/**
* Timed-out probes allowed to stay pending before new probes are refused (answered
* "unknown" without a stat). This is a backstop, not the main defence: a stalled
* path already takes its neighbours (same parent directory) out of probing, so the
* cap only engages once three UNRELATED places have stopped answering. The default
* leaves one of libuv's default four workers free for the rest of the process.
* Hard ceiling on timed-out probes left pending, for every caller, `pastCap` ones
* included: the threadpool size minus one, so a dead mount can never take the last
* worker. libuv sizes the pool from `UV_THREADPOOL_SIZE` (4 when unset). A pool of
* one cannot keep a worker free at all, so the ceiling never drops below one.
*/
export const MAX_STALLED_PATH_PROBES = envInt('CODEMAN_PATH_PROBE_MAX_STALLED', 3, 1, 64);
export const PATH_PROBE_STALL_CEILING = Math.max(1, (Number(process.env.UV_THREADPOOL_SIZE) || 4) - 1);
/**
* Timed-out probes allowed to stay pending before new BULK probes are refused
* (answered "unknown" without a stat). This is a backstop, not the main defence: a
* stalled path on a network or FUSE mount already takes the rest of that mount out
* of probing (a stall anywhere else takes out only the stalled path), so the cap
* only engages once that many UNRELATED places have stopped answering. It defaults
* to one below {@link PATH_PROBE_STALL_CEILING} (2 with the default pool), leaving a
* slot a `pastCap` probe may still use, and is never allowed above the ceiling.
*/
export const MAX_STALLED_PATH_PROBES = Math.min(
PATH_PROBE_STALL_CEILING,
envInt('CODEMAN_PATH_PROBE_MAX_STALLED', Math.max(1, PATH_PROBE_STALL_CEILING - 1), 1, 64)
);
+13 -6
View File
@@ -852,13 +852,20 @@ export async function refreshStaleCodemanHooks(casePath: string): Promise<void>
*/
export async function applyWorkspaceHooks(workspace: string, install?: boolean): Promise<void> {
try {
const skip = await absentOrUnreachable(workspace);
if (skip === 'unreachable') {
console.warn(
`[hooks] ${workspace} is not responding (unreachable mount?); Codeman hooks not checked or installed`
);
const state = await probePath(workspace);
if (state === 'absent') return;
if (state === 'unknown') {
if (isNearStalledPath(workspace)) {
console.warn(
`[hooks] ${workspace} is not responding (unreachable mount?); Codeman hooks not checked or installed`
);
return;
}
// Any other "unknown" (the stall cap refused the probe, or the stat failed
// with something other than ENOENT) proves nothing about existence, and the
// install below would mkdir -p a deleted repo back into being: ask directly.
if (!(await pathExistsForWrite(workspace))) return;
}
if (skip) return;
const shouldInstall = install ?? (await readWorkspaceHooksEnabled());
await (shouldInstall ? ensureCodemanHooks(workspace) : refreshStaleCodemanHooks(workspace));
} catch {
+47 -17
View File
@@ -21,11 +21,14 @@
* - a path whose probe timed out is "stalled" until that stat finally settles.
* Paths NEAR a stalled one are answered "unknown" without a new stat, so one
* dead mount costs one worker, not one per case and file on it. "Near" means on
* the same mount: under the deepest mount point holding the stalled path, read
* from `/proc/self/mounts` (procfs, which never waits on the dead filesystem).
* Where that table is unavailable (not Linux), or the deepest mount is `/`, it
* narrows to the stalled path and everything under it. Unrelated paths are
* probed normally;
* the same mount when that mount is a network or FUSE filesystem (NFS, SMB,
* sshfs and the like): under the deepest mount point holding the stalled path,
* with its type, read from `/proc/self/mounts` (procfs, which never waits on the
* dead filesystem). Otherwise it narrows to the stalled path and everything under
* it: when the deepest mount is local (a path typed under a local `/home` can
* reach a NAS through a symlink, and must not take the rest of `/home` with it),
* is `/`, or the table is unavailable (not Linux). Unrelated paths are probed
* normally;
* - once `MAX_STALLED_PATH_PROBES` stalled stats are pending, new probes are
* refused process-wide (answered "unknown"), since each would risk another
* worker. Probes merely in flight do not count, so concurrent healthy probes
@@ -34,7 +37,9 @@
* probe is still bounded and still recorded as stalled if it hangs (so a dead
* path costs at most one worker however often it is retried), but it is not
* refused just because unrelated mounts are dead. Bulk scans (the case list)
* and per-spawn helpers keep the cap.
* and per-spawn helpers keep the cap. `pastCap` still stops at
* `PATH_PROBE_STALL_CEILING` (the threadpool size minus one), so explicit
* requests against several dead paths can never take the last worker.
*
* Both events are logged once (`console.warn`): a path's first stall, and the
* cap engaging, so "my case vanished" and "hooks stopped firing" leave a trace.
@@ -49,7 +54,7 @@
import { readFileSync } from 'node:fs';
import fs from 'node:fs/promises';
import { resolve, sep } from 'node:path';
import { MAX_STALLED_PATH_PROBES, PATH_PROBE_TIMEOUT_MS } from '../config/path-probe.js';
import { MAX_STALLED_PATH_PROBES, PATH_PROBE_STALL_CEILING, PATH_PROBE_TIMEOUT_MS } from '../config/path-probe.js';
/** What a probe could establish about a path. */
export type PathProbeState = 'present' | 'absent' | 'unknown';
@@ -75,29 +80,53 @@ function isWithin(path: string, root: string): boolean {
return path.startsWith(root.endsWith(sep) ? root : root + sep);
}
/** Deepest mount point holding `abs`, from the kernel's mount table; undefined when unreadable. */
function mountPointOf(abs: string): string | undefined {
/** Filesystem types whose stall means the whole mount is gone (network and FUSE). */
const REMOTE_FS_TYPES = new Set([
'nfs',
'nfs4',
'cifs',
'smb3',
'smbfs',
'9p',
'ceph',
'glusterfs',
'afs',
'lustre',
'davfs',
]);
function isRemoteFsType(fsType: string): boolean {
return REMOTE_FS_TYPES.has(fsType) || fsType.startsWith('fuse.');
}
/** Deepest mount holding `abs`, from the kernel's mount table; undefined when unreadable. */
function mountOf(abs: string): { mountPoint: string; fsType: string } | undefined {
let table: string;
try {
table = readFileSync('/proc/self/mounts', 'utf-8');
} catch {
return undefined;
}
let best: string | undefined;
let best: { mountPoint: string; fsType: string } | undefined;
for (const line of table.split('\n')) {
const field = line.split(' ')[1];
if (!field) continue;
const [, field, fsType] = line.split(' ');
if (!field || !fsType) continue;
// The table octal-escapes space, tab, newline and backslash in mount points.
const mountPoint = field.replace(/\\([0-7]{3})/g, (_m, oct: string) => String.fromCharCode(parseInt(oct, 8)));
if (isWithin(abs, mountPoint) && (!best || mountPoint.length > best.length)) best = mountPoint;
if (isWithin(abs, mountPoint) && (!best || mountPoint.length > best.mountPoint.length)) {
best = { mountPoint, fsType };
}
}
return best;
}
/** The subtree a stalled path takes down with it: its mount, else just itself (see the module comment). */
/**
* The subtree a stalled path takes down with it (see the module comment): its
* mount when that is a network or FUSE filesystem, else just the path itself.
*/
function stallScope(abs: string): string {
const mountPoint = mountPointOf(abs);
return mountPoint && mountPoint !== '/' ? mountPoint : abs;
const mount = mountOf(abs);
return mount && mount.mountPoint !== '/' && isRemoteFsType(mount.fsType) ? mount.mountPoint : abs;
}
/**
@@ -130,7 +159,8 @@ export async function probePathKind(path: string, options: PathProbeOptions = {}
let probe = inFlight.get(abs);
if (!probe) {
if (stalled.size >= MAX_STALLED_PATH_PROBES && !options.pastCap) {
// pastCap lifts the bulk cap, never the ceiling that keeps one worker free.
if (stalled.size >= (options.pastCap ? PATH_PROBE_STALL_CEILING : MAX_STALLED_PATH_PROBES)) {
if (!capWarned) {
capWarned = true;
console.warn(
+2 -1
View File
@@ -1665,7 +1665,8 @@ export function registerCaseRoutes(app: FastifyInstance, ctx: EventPort & Config
return {
name,
path: casePath,
hasClaudeMd: await boundedPathExists(join(casePath, 'CLAUDE.md')),
// Probed like the folder above, or a healthy case reads as having no CLAUDE.md under the cap.
hasClaudeMd: (await probePath(join(casePath, 'CLAUDE.md'), { pastCap: true })) === 'present',
...(linked && { linked: true }),
};
});