diff --git a/scripts/self-update.sh b/scripts/self-update.sh index fb3e7874..6ca3dd7b 100755 --- a/scripts/self-update.sh +++ b/scripts/self-update.sh @@ -74,6 +74,15 @@ echo "[self-update] $(date) start tag=$TAG supervisor=$SUPERVISOR repo=$REPO" export PATH="$(dirname "$NODE"):$HOME/.local/bin:$HOME/.npm-global/bin:/usr/local/bin:/opt/homebrew/bin:$PATH" export GIT_TERMINAL_PROMPT=0 +# --node is the server's process.execPath, a VERSIONED path (Homebrew resolves it +# into Cellar/node//). A `brew upgrade node` under a long-running server +# deletes it, and every status write then failed, so the status stayed "queued" +# forever. Fall back to whatever node is on PATH. +if [ ! -x "$NODE" ]; then + echo "[self-update] WARN: $NODE is not executable, falling back to node on PATH" + NODE="$(command -v node || echo node)" +fi + TO_VERSION="${TAG##*@}" # codeman@0.9.4 → 0.9.4 (tag is validated upstream) STASH_REF="" MANUAL_CMD="" @@ -276,7 +285,17 @@ case "$SUPERVISOR" in # domain needs root, but we don't need it — kill the server and launchd # respawns it on the new dist/ within ThrottleInterval seconds. if [[ -n "$SERVER_PID" ]] && kill "$SERVER_PID" 2>/dev/null; then - : # respawn is launchd's job from here + # Respawn is launchd's job, but only once the old process EXITS. A graceful + # shutdown that hangs leaves the port closed and the service down, so + # escalate to SIGKILL (tmux sessions live outside the server and survive). + for _ in $(seq 1 30); do + kill -0 "$SERVER_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$SERVER_PID" 2>/dev/null; then + echo "[self-update] server pid $SERVER_PID still alive 30s after SIGTERM, sending SIGKILL" + kill -9 "$SERVER_PID" 2>/dev/null || true + fi else MANUAL_CMD="sudo launchctl kickstart -k system/com.codeman.web" write_status "completed-needs-manual-restart" "Update staged — restart Codeman to apply v$TO_VERSION." diff --git a/src/cli.ts b/src/cli.ts index 2da7fac7..1b60246c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -129,6 +129,8 @@ program /** Same registry the server resolves case names through (mirrors `case-routes.ts`). */ const LINKED_CASES_FILE = dataPath('linked-cases.json'); +/** Graceful shutdown budget before the process force-exits (see the SIGTERM handler). */ +const SHUTDOWN_FORCE_EXIT_MS = 10_000; /** * Case name to directory, checking `linked-cases.json` FIRST and falling back to the @@ -1002,6 +1004,14 @@ webCmd.action(async (options) => { if (shuttingDown) return; shuttingDown = true; console.log(palette.warn(`\n${signal} received, shutting down gracefully...`)); + // A hung stop() must not keep the process alive: the listener is already + // closed by then, and a KeepAlive LaunchDaemon only respawns the server once + // it EXITS (systemd would SIGKILL after TimeoutStopSec; launchd does not). + // Seen after a self-update on macOS: port closed, process alive, service down. + setTimeout(() => { + console.error(palette.err(`Shutdown did not finish in ${SHUTDOWN_FORCE_EXIT_MS / 1000}s, forcing exit`)); + process.exit(1); + }, SHUTDOWN_FORCE_EXIT_MS).unref(); try { await server.stop(); } catch (err) { diff --git a/src/web/self-update.ts b/src/web/self-update.ts index d25ec186..ac7c97a2 100644 --- a/src/web/self-update.ts +++ b/src/web/self-update.ts @@ -226,6 +226,29 @@ export function reconcileStatusDecision( return null; } +/** + * PURE runtime staleness check, applied whenever the status is READ (not only on + * boot). The live updater heartbeats `updatedAt` every few seconds, so an + * in-flight status whose last write is older than the window has no updater + * behind it. Without this, a status that never advanced (e.g. the updater could + * not run its `--node` binary because Homebrew upgraded node under a long-running + * server, so every status write failed) blocked every later update with "An + * update is already in progress." until the server happened to restart. + * Returns the failed status to persist, or null to leave the status untouched. + */ +export function expireStalledStatus(status: UpdateStatus | null, now: number): UpdateStatus | null { + if (!status || !IN_FLIGHT_PHASES.has(status.phase)) return null; + const lastWrite = status.updatedAt || status.startedAt; + if (now - lastWrite <= RECONCILE_STALE_MS) return null; + return { + ...status, + phase: 'failed', + message: 'Update stopped reporting progress', + error: `no status update for ${Math.round((now - lastWrite) / 60_000)} min during "${status.phase}"`, + updatedAt: now, + }; +} + // ───────────────────────────────────────────────────────────────────────────── // PURE helpers — the container environment gate // ───────────────────────────────────────────────────────────────────────────── @@ -380,6 +403,19 @@ export function writeUpdateStatusAtomic(status: UpdateStatus): void { renameSync(tmp, STATUS_FILE); } +/** Read the status, first failing (and persisting) an in-flight one that stopped heartbeating. */ +function readCurrentUpdateStatus(now = Date.now()): UpdateStatus | null { + const status = readUpdateStatus(); + const expired = expireStalledStatus(status, now); + if (!expired) return status; + try { + writeUpdateStatusAtomic(expired); + } catch { + // Still report the expired view; the next read retries the write. + } + return expired; +} + /** Reconcile the status file on server boot (call once, early in start()). */ export function reconcileUpdateOnBoot(now = Date.now()): void { const status = readUpdateStatus(); @@ -796,7 +832,7 @@ export async function startUpdate(): Promise { : 'This is not a git install. Update with: npm i -g aicodeman@latest', }; } - const existing = readUpdateStatus(); + const existing = readCurrentUpdateStatus(); if (isInFlight(existing)) { return { ok: false, code: 'in-flight', message: 'An update is already in progress.' }; } @@ -885,7 +921,7 @@ export async function startUpdate(): Promise { /** Current status for the polling endpoint; null collapses to an explicit idle. */ export function getUpdateStatusForApi(): UpdateStatus { - const status = readUpdateStatus(); + const status = readCurrentUpdateStatus(); if (status) return status; return { updateId: '', diff --git a/test/self-update.test.ts b/test/self-update.test.ts index 7a928faf..6a41b48c 100644 --- a/test/self-update.test.ts +++ b/test/self-update.test.ts @@ -15,6 +15,7 @@ import { isValidReleaseTag, parseGitHubRepo, reconcileStatusDecision, + expireStalledStatus, } from '../src/web/self-update.js'; import type { UpdateStatus } from '../src/types/update.js'; @@ -167,3 +168,40 @@ describe('reconcileStatusDecision (boot handoff state machine)', () => { expect(reconcileStatusDecision(noTarget, '0.9.4', NOW)).toBeNull(); }); }); + +describe('expireStalledStatus (runtime staleness backstop)', () => { + const NOW = 1_000_000_000_000; + const MIN = 60 * 1000; + const base = (over: Partial): UpdateStatus => ({ + updateId: 'u1', + phase: 'queued', + message: '', + fromVersion: '1.24.7', + toVersion: '1.29.0', + startedAt: NOW - 5_000, + updatedAt: NOW - 5_000, + ...over, + }); + + it('leaves a heartbeating update alone', () => { + expect( + expireStalledStatus(base({ phase: 'installing', startedAt: NOW - 60 * MIN, updatedAt: NOW - 3_000 }), NOW) + ).toBeNull(); + }); + + it('fails a status that stopped heartbeating (queued forever: status writes were failing)', () => { + const out = expireStalledStatus( + base({ startedAt: NOW - 8 * 24 * 60 * MIN, updatedAt: NOW - 8 * 24 * 60 * MIN }), + NOW + ); + expect(out?.phase).toBe('failed'); + expect(out?.error).toContain('queued'); + expect(out?.updatedAt).toBe(NOW); + }); + + it('never touches terminal phases or a missing status', () => { + expect(expireStalledStatus(null, NOW)).toBeNull(); + expect(expireStalledStatus(base({ phase: 'completed', updatedAt: NOW - 60 * MIN }), NOW)).toBeNull(); + expect(expireStalledStatus(base({ phase: 'failed', updatedAt: NOW - 60 * MIN }), NOW)).toBeNull(); + }); +});