diff --git a/scripts/self-update.sh b/scripts/self-update.sh index fb3e7874..e89b7e59 100755 --- a/scripts/self-update.sh +++ b/scripts/self-update.sh @@ -74,6 +74,15 @@ echo "[self-update] $(date) start tag=$TAG supervisor=$SUPERVISOR repo=$REPO" export PATH="$(dirname "$NODE"):$HOME/.local/bin:$HOME/.npm-global/bin:/usr/local/bin:/opt/homebrew/bin:$PATH" export GIT_TERMINAL_PROMPT=0 +# --node is the server's process.execPath, a VERSIONED path (Homebrew resolves it +# into Cellar/node//). A `brew upgrade node` under a long-running server +# deletes it, and every status write then failed, so the status stayed "queued" +# forever. Fall back to whatever node is on PATH. +if [ ! -x "$NODE" ]; then + echo "[self-update] WARN: $NODE is not executable, falling back to node on PATH" + NODE="$(command -v node || echo node)" +fi + TO_VERSION="${TAG##*@}" # codeman@0.9.4 → 0.9.4 (tag is validated upstream) STASH_REF="" MANUAL_CMD="" diff --git a/src/web/self-update.ts b/src/web/self-update.ts index d25ec186..ac7c97a2 100644 --- a/src/web/self-update.ts +++ b/src/web/self-update.ts @@ -226,6 +226,29 @@ export function reconcileStatusDecision( return null; } +/** + * PURE runtime staleness check, applied whenever the status is READ (not only on + * boot). The live updater heartbeats `updatedAt` every few seconds, so an + * in-flight status whose last write is older than the window has no updater + * behind it. Without this, a status that never advanced (e.g. the updater could + * not run its `--node` binary because Homebrew upgraded node under a long-running + * server, so every status write failed) blocked every later update with "An + * update is already in progress." until the server happened to restart. + * Returns the failed status to persist, or null to leave the status untouched. + */ +export function expireStalledStatus(status: UpdateStatus | null, now: number): UpdateStatus | null { + if (!status || !IN_FLIGHT_PHASES.has(status.phase)) return null; + const lastWrite = status.updatedAt || status.startedAt; + if (now - lastWrite <= RECONCILE_STALE_MS) return null; + return { + ...status, + phase: 'failed', + message: 'Update stopped reporting progress', + error: `no status update for ${Math.round((now - lastWrite) / 60_000)} min during "${status.phase}"`, + updatedAt: now, + }; +} + // ───────────────────────────────────────────────────────────────────────────── // PURE helpers — the container environment gate // ───────────────────────────────────────────────────────────────────────────── @@ -380,6 +403,19 @@ export function writeUpdateStatusAtomic(status: UpdateStatus): void { renameSync(tmp, STATUS_FILE); } +/** Read the status, first failing (and persisting) an in-flight one that stopped heartbeating. */ +function readCurrentUpdateStatus(now = Date.now()): UpdateStatus | null { + const status = readUpdateStatus(); + const expired = expireStalledStatus(status, now); + if (!expired) return status; + try { + writeUpdateStatusAtomic(expired); + } catch { + // Still report the expired view; the next read retries the write. + } + return expired; +} + /** Reconcile the status file on server boot (call once, early in start()). */ export function reconcileUpdateOnBoot(now = Date.now()): void { const status = readUpdateStatus(); @@ -796,7 +832,7 @@ export async function startUpdate(): Promise { : 'This is not a git install. Update with: npm i -g aicodeman@latest', }; } - const existing = readUpdateStatus(); + const existing = readCurrentUpdateStatus(); if (isInFlight(existing)) { return { ok: false, code: 'in-flight', message: 'An update is already in progress.' }; } @@ -885,7 +921,7 @@ export async function startUpdate(): Promise { /** Current status for the polling endpoint; null collapses to an explicit idle. */ export function getUpdateStatusForApi(): UpdateStatus { - const status = readUpdateStatus(); + const status = readCurrentUpdateStatus(); if (status) return status; return { updateId: '', diff --git a/test/self-update.test.ts b/test/self-update.test.ts index 7a928faf..6a41b48c 100644 --- a/test/self-update.test.ts +++ b/test/self-update.test.ts @@ -15,6 +15,7 @@ import { isValidReleaseTag, parseGitHubRepo, reconcileStatusDecision, + expireStalledStatus, } from '../src/web/self-update.js'; import type { UpdateStatus } from '../src/types/update.js'; @@ -167,3 +168,40 @@ describe('reconcileStatusDecision (boot handoff state machine)', () => { expect(reconcileStatusDecision(noTarget, '0.9.4', NOW)).toBeNull(); }); }); + +describe('expireStalledStatus (runtime staleness backstop)', () => { + const NOW = 1_000_000_000_000; + const MIN = 60 * 1000; + const base = (over: Partial): UpdateStatus => ({ + updateId: 'u1', + phase: 'queued', + message: '', + fromVersion: '1.24.7', + toVersion: '1.29.0', + startedAt: NOW - 5_000, + updatedAt: NOW - 5_000, + ...over, + }); + + it('leaves a heartbeating update alone', () => { + expect( + expireStalledStatus(base({ phase: 'installing', startedAt: NOW - 60 * MIN, updatedAt: NOW - 3_000 }), NOW) + ).toBeNull(); + }); + + it('fails a status that stopped heartbeating (queued forever: status writes were failing)', () => { + const out = expireStalledStatus( + base({ startedAt: NOW - 8 * 24 * 60 * MIN, updatedAt: NOW - 8 * 24 * 60 * MIN }), + NOW + ); + expect(out?.phase).toBe('failed'); + expect(out?.error).toContain('queued'); + expect(out?.updatedAt).toBe(NOW); + }); + + it('never touches terminal phases or a missing status', () => { + expect(expireStalledStatus(null, NOW)).toBeNull(); + expect(expireStalledStatus(base({ phase: 'completed', updatedAt: NOW - 60 * MIN }), NOW)).toBeNull(); + expect(expireStalledStatus(base({ phase: 'failed', updatedAt: NOW - 60 * MIN }), NOW)).toBeNull(); + }); +});